@datatruck/restic 0.0.2 → 0.0.4

package/lib/actions/backup.js

@@ -3,6 +3,7 @@ import { checkDiskSpace, fetchMultipleDiskStats } from "../utils/fs.js";
 import { MySQLDump } from "../utils/mysql.js";
 import { Action } from "./base.js";
 import { Prune } from "./prune.js";
+import { SnapshotTagEnum } from "@datatruck/cli/repositories/RepositoryAbstract.js";
 import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.js";
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
 import { isLocalDir } from "@datatruck/cli/utils/fs.js";
@@ -23,6 +24,7 @@ export class Backup extends Action {
             },
         });
         let space;
+        let snapshotsAmount;
         let bytes = 0;
         let files = 0;
         return await createRunner(async () => {
@@ -51,6 +53,13 @@ export class Backup extends Action {
                     });
                 },
             });
+            const snapshots = await restic.snapshots({
+                json: true,
+                tags: [
+                    ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkg.name),
+                ],
+            });
+            snapshotsAmount = snapshots.length;
         }).start(async (data) => {
             await this.ntfy.send("Backup", {
                 Repository: repo.name,
@@ -58,6 +67,7 @@ export class Backup extends Action {
                 Size: formatBytes(bytes) +
                     (space !== undefined ? ` (${formatBytes(space.diff, true)})` : ""),
                 Files: files,
+                Snapshots: snapshotsAmount,
                 Duration: data.duration,
                 Error: data.error?.message,
             }, data.error);

package/lib/actions/create.d.ts

@@ -0,0 +1,10 @@
+export type CreateOptions = {
+    cwd: string;
+    config: string;
+    force?: boolean;
+};
+export declare class Create {
+    constructor();
+    protected random(): string;
+    run(options: CreateOptions): Promise<void>;
+}

package/lib/actions/create.js

@@ -0,0 +1,49 @@
+import { formatBytes } from "@datatruck/cli/utils/bytes.js";
+import { fetchDiskStats } from "@datatruck/cli/utils/fs.js";
+import { existsSync } from "fs";
+import { writeFile } from "fs/promises";
+import { hostname } from "os";
+import { join, relative } from "path";
+export class Create {
+    constructor() { }
+    random() {
+        return crypto.randomUUID().replaceAll("-", "");
+    }
+    async run(options) {
+        const stats = await fetchDiskStats(options.cwd);
+        const minFreeSpaceBytes = (stats.total * 25) / 100;
+        const minFreeSpace = formatBytes(minFreeSpaceBytes);
+        const cwd = process.cwd();
+        const config = {
+            $schema: "https://unpkg.com/@datatruck/restic/config.schema.json",
+            hostname: hostname(),
+            minFreeSpace,
+            ntfyToken: this.random(),
+            packages: [
+                {
+                    name: "default",
+                    path: join(cwd, "data").replaceAll("\\", "/"),
+                },
+            ],
+            prunePolicy: {
+                keepDaily: 7,
+                keepMonthly: 12,
+                keepYearly: 5,
+            },
+            repositories: [
+                {
+                    name: "default",
+                    uri: join(cwd, "repo").replaceAll("\\", "/"),
+                    password: this.random(),
+                },
+            ],
+        };
+        const configPath = join(cwd, options.config);
+        if (existsSync(configPath) && !options.force)
+            throw new Error(`Config file already exists at path: ${configPath}`);
+        await writeFile(configPath, JSON.stringify(config, null, 2) + "\n");
+        const relativeConfigPath = relative(cwd, configPath);
+        console.log(`- Created config file at path: ${relativeConfigPath}`);
+        console.info(`- Show log remotly via ntfy: https://ntfy.sh/${config.ntfyToken}`);
+    }
+}

package/lib/actions/prune.d.ts

@@ -5,6 +5,10 @@ export type PruneOptions = {
     repositories?: string[];
 };
 export declare class Prune extends Action {
-    protected runSingle(repoName: string, pkgName: string, policy: PrunePolicy): Promise<number>;
+    protected runSingle(repoName: string, pkgName: string, policy: PrunePolicy): Promise<{
+        diffSize: number;
+        removed: number | undefined;
+    }>;
+    protected fetchPackages(repoName: string): Promise<string[]>;
     run(options: PruneOptions): Promise<void>;
 }

package/lib/actions/prune.js

@@ -6,21 +6,14 @@ import { ResticRepository } from "@datatruck/cli/repositories/ResticRepository.j
 import { formatBytes } from "@datatruck/cli/utils/bytes.js";
 import { isLocalDir } from "@datatruck/cli/utils/fs.js";
 import { progressPercent } from "@datatruck/cli/utils/math.js";
-import { Restic } from "@datatruck/cli/utils/restic.js";
+import { match } from "@datatruck/cli/utils/string.js";
 export class Prune extends Action {
     async runSingle(repoName, pkgName, policy) {
         let stats;
         let space;
+        let removed;
         await createRunner(async () => {
-            const repo = this.cm.findRepository(repoName);
-            const pkg = this.cm.findPackage(pkgName);
-            const restic = new Restic({
-                log: this.verbose,
-                env: {
-                    RESTIC_REPOSITORY: repo.uri,
-                    RESTIC_PASSWORD: repo.password,
-                },
-            });
+            const [restic, repo] = this.cm.createRestic(repoName, this.verbose);
             const targetPath = isLocalDir(repo.uri) ? repo.uri : undefined;
             space = await checkDiskSpace({
                 minFreeSpace: this.config.minFreeSpace,
@@ -32,7 +25,7 @@ export class Prune extends Action {
                         prune: true,
                         args: ["--group-by", ""],
                         tag: [
-                            ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkg.name),
+                            ResticRepository.createSnapshotTag(SnapshotTagEnum.PACKAGE, pkgName),
                         ],
                     });
                     stats = {
@@ -42,11 +35,13 @@ export class Prune extends Action {
                 },
             });
         }).start(async (data) => {
+            if (stats)
+                removed = stats.remove;
             await this.ntfy.send(`Prune`, {
                 Repository: repoName,
                 Package: pkgName,
                 ...(stats && {
-                    Snapshots: `${stats.keep}`,
+                    Keeped: `${stats.keep}`,
                     Removed: `${stats.remove}`,
                 }),
                 ...(space !== undefined && {
@@ -56,37 +51,60 @@ export class Prune extends Action {
                 Error: data.error?.message,
             }, data.error);
         });
-        return space?.diff ?? 0;
+        return {
+            diffSize: space?.diff ?? 0,
+            removed,
+        };
+    }
+    async fetchPackages(repoName) {
+        const [restic] = this.cm.createRestic(repoName, this.verbose);
+        if (!(await restic.checkRepository()))
+            return [];
+        const snapshots = await restic.snapshots({ json: true });
+        const packages = new Set(snapshots.map((s) => {
+            const tags = ResticRepository.parseSnapshotTags(s.tags ?? []);
+            return tags[SnapshotTagEnum.PACKAGE];
+        }));
+        return [...packages].filter((v) => typeof v === "string");
     }
     async run(options) {
         let globalDiffSize;
+        let globalRemoved = 0;
         let localRepositoryPaths = [];
         await createRunner(async () => {
             const repositories = this.cm.filterRepositories(options.repositories);
-            const packages = this.cm.filterPackages(options.packages);
             await this.ntfy.send(`Prune start`, {
                 Repositories: repositories.length,
-                Packages: packages.length,
             });
             localRepositoryPaths = repositories
                 .filter((repo) => isLocalDir(repo.uri))
                 .map((repo) => repo.uri);
+            let some = false;
             for (const repo of repositories) {
-                for (const pkg of packages) {
-                    const policy = pkg.prunePolicy ?? repo.prunePolicy ?? this.config.prunePolicy;
+                const inPackageNames = await this.fetchPackages(repo.name);
+                const packageNames = inPackageNames.filter((pkgName) => options.packages ? match(pkgName, options.packages) : true);
+                for (const pkgName of packageNames) {
+                    const pkg = this.config.packages.find((pkg) => pkg.name === pkgName);
+                    const policy = pkg?.prunePolicy ?? repo.prunePolicy ?? this.config.prunePolicy;
                     if (policy) {
-                        const diffSize = await this.runSingle(repo.name, pkg.name, policy);
+                        const { diffSize, removed } = await this.runSingle(repo.name, pkgName, policy);
+                        some = true;
                         if (diffSize !== undefined)
                             globalDiffSize = (globalDiffSize ?? 0) + diffSize;
+                        if (removed !== undefined)
+                            globalRemoved += removed;
                     }
                 }
             }
+            if (options.packages && !some)
+                throw new Error(`No packages found for filter: ${options.packages.join(", ")}`);
         }).start(async (data) => {
             const diskStats = await safeRun(() => fetchMultipleDiskStats(localRepositoryPaths));
             if (diskStats.error)
                 console.error(diskStats.error);
             await this.ntfy.send(`Prune end`, {
                 Duration: data.duration,
+                Removed: globalRemoved,
                 ...(globalDiffSize !== undefined && {
                     "Diff size": (globalDiffSize > 0 ? "+" : "") + formatBytes(globalDiffSize),
                 }),

package/lib/actions/run.d.ts

@@ -0,0 +1,8 @@
+import { Action } from "./base.js";
+export type RunOptions = {
+    repository: string;
+    args: string[];
+};
+export declare class Run extends Action {
+    run(options: RunOptions): Promise<void>;
+}

package/lib/actions/run.js

@@ -0,0 +1,11 @@
+import { Action } from "./base.js";
+import { spawnSync } from "child_process";
+export class Run extends Action {
+    async run(options) {
+        const [restic] = this.cm.createRestic(options.repository, this.verbose);
+        const p = restic["createProcess"](options.args, { $log: true });
+        const exit = spawnSync(p["command"], p["argv"]?.map((v) => v.toString()), { stdio: "inherit", env: p["options"]?.env });
+        if (exit.status)
+            process.exit(exit.status);
+    }
+}

package/lib/bin.js

@@ -1,72 +1,2 @@
-import { Backup } from "./actions/backup.js";
-import { Copy } from "./actions/copy.js";
-import { Init } from "./actions/init.js";
-import { Prune } from "./actions/prune.js";
-import { parseConfigFile } from "./config.js";
-import { parseStringList } from "@datatruck/cli/utils/string.js";
-import { program } from "commander";
-import { resolve } from "path";
-function getGlobalOptions() {
-    const options = program.opts();
-    return {
-        ...options,
-        config: resolve(options.config),
-        verbose: process.env.DEBUG ? true : options.verbose,
-    };
-}
-async function load() {
-    const globalOptions = getGlobalOptions();
-    const config = await parseConfigFile(globalOptions.config);
-    return { globalOptions, config };
-}
-program
-    .option("-v, --verbose")
-    .option("-c, --config <path>", "Path to config file", "datatruck.restic.json");
-program
-    .command("init")
-    .alias("i")
-    .description("Run init action")
-    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
-    .action(async (options) => {
-    const { config, globalOptions } = await load();
-    const init = new Init(config, globalOptions);
-    await init.run(options);
-});
-program
-    .command("backup")
-    .alias("b")
-    .description("Run backup action")
-    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
-    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
-    .option("--prune", "Prune after backup")
-    .action(async (options) => {
-    const { config, globalOptions } = await load();
-    const backup = new Backup(config, globalOptions);
-    await backup.run(options);
-});
-program
-    .command("copy")
-    .alias("c")
-    .description("Run copy action")
-    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
-    .requiredOption("-s, --source <name>", "Source repository name")
-    .requiredOption("-t, --targets <names>", "Target repository names", (v) => parseStringList(v))
-    .option("--prune", "Prune after copy")
-    .action(async (options) => {
-    const { config, globalOptions } = await load();
-    const copy = new Copy(config, globalOptions);
-    await copy.run(options);
-});
-program
-    .command("prune")
-    .alias("p")
-    .description("Run prune action")
-    .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
-    .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
-    .option("--prune", "Prune after copy")
-    .action(async (options) => {
-    const { config, globalOptions } = await load();
-    const prune = new Prune(config, globalOptions);
-    await prune.run(options);
-});
-program.parse();
+import { createBin } from "./create-bin.js";
+createBin().parse();

package/lib/config.d.ts

@@ -1,4 +1,5 @@
 import { MySQLDumpOptions } from "./utils/mysql.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
 export type GlobalConfig = {
     config?: string;
     verbose?: boolean;
@@ -78,4 +79,10 @@ export declare class ConfigManager {
         exclude?: string[];
         prunePolicy?: PrunePolicy;
     };
+    createRestic(repoName: string, verbose: boolean | undefined): readonly [Restic, {
+        name: string;
+        password: string;
+        uri: string;
+        prunePolicy?: PrunePolicy;
+    }];
 }

package/lib/config.js

@@ -1,4 +1,5 @@
 import { parseJSONFile } from "./utils/fs.js";
+import { Restic } from "@datatruck/cli/utils/restic.js";
 import { match } from "@datatruck/cli/utils/string.js";
 import { Ajv } from "ajv";
 export function defineConfig(config) {
@@ -29,13 +30,17 @@ export class ConfigManager {
     filterRepositories(filter) {
         const repositories = this.config.repositories.filter((v) => filter ? match(v.name, filter) : true);
         if (!repositories.length)
-            throw new Error(`No repositories found for filter: ${filter?.join(", ")}`);
+            throw new Error(filter
+                ? `No repositories found for filter: ${filter.join(", ")}`
+                : `No repositories found`);
         return repositories;
     }
     filterPackages(filter) {
         const packages = this.config.packages.filter((v) => filter ? match(v.name, filter) : true);
         if (!packages.length)
-            throw new Error(`No packages found for filter: ${filter?.join(", ")}`);
+            throw new Error(filter
+                ? `No packages found for filter: ${filter.join(", ")}`
+                : `No packages found`);
         return packages;
     }
     findRepository(name) {
@@ -50,4 +55,15 @@ export class ConfigManager {
             throw new Error(`Package '${name}' not found`);
         return pkg;
     }
+    createRestic(repoName, verbose) {
+        const repo = this.findRepository(repoName);
+        const restic = new Restic({
+            log: verbose,
+            env: {
+                RESTIC_REPOSITORY: repo.uri,
+                RESTIC_PASSWORD: repo.password,
+            },
+        });
+        return [restic, repo];
+    }
 }

package/lib/create-bin.d.ts

@@ -0,0 +1,3 @@
+import { Config } from "./config.js";
+import { Command } from "commander";
+export declare function createBin(inConfig?: Config): Command;

package/lib/create-bin.js

@@ -0,0 +1,112 @@
+import { Backup } from "./actions/backup.js";
+import { Copy } from "./actions/copy.js";
+import { Create } from "./actions/create.js";
+import { Init } from "./actions/init.js";
+import { Prune } from "./actions/prune.js";
+import { Run } from "./actions/run.js";
+import { parseConfigFile } from "./config.js";
+import { parseStringList } from "@datatruck/cli/utils/string.js";
+import { Command } from "commander";
+import { resolve } from "path";
+export function createBin(inConfig) {
+    async function load() {
+        const globalOptions = { ...program.opts() };
+        if (globalOptions.config)
+            globalOptions.config = resolve(globalOptions.config);
+        const config = inConfig ?? (await parseConfigFile(globalOptions.config));
+        globalOptions.verbose = process.env.DEBUG
+            ? true
+            : globalOptions.verbose || config.verbose;
+        return { config, globalOptions };
+    }
+    const program = new Command();
+    program.option("-v, --verbose");
+    if (!inConfig)
+        program.option("-c, --config <path>", "Path to config file", "datatruck.restic.json");
+    program
+        .command("create")
+        .description("Create config file")
+        .option("--cwd <path>", "Current working directory to create config file in", ".")
+        .option("-c,--config <path>", "Output path for config file", "datatruck.restic.json")
+        .option("-f,--force", "Force overwrite if config file already exists")
+        .action(async (options) => {
+        const create = new Create();
+        await create.run(options);
+    });
+    program
+        .command("run", {})
+        .description("Run arbitrary restic command")
+        .argument("<repository>", "Repository name")
+        .argument("[args...]", "Restic arguments")
+        .allowUnknownOption()
+        .allowExcessArguments()
+        .action(async (repository, args) => {
+        const { config, globalOptions } = await load();
+        const run = new Run(config, globalOptions);
+        await run.run({ repository, args });
+    });
+    program
+        .command("init")
+        .alias("i")
+        .description("Run init action")
+        .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
+        .action(async (options) => {
+        const { config, globalOptions } = await load();
+        const init = new Init(config, globalOptions);
+        await init.run(options);
+    });
+    program
+        .command("backup")
+        .alias("b")
+        .description("Run backup action")
+        .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
+        .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
+        .option("--prune", "Prune after backup")
+        .action(async (options) => {
+        const { config, globalOptions } = await load();
+        const backup = new Backup(config, globalOptions);
+        await backup.run(options);
+    });
+    program
+        .command("copy")
+        .alias("c")
+        .description("Run copy action")
+        .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
+        .requiredOption("-s, --source <name>", "Source repository name")
+        .requiredOption("-t, --targets <names>", "Target repository names", (v) => parseStringList(v))
+        .option("--prune", "Prune after copy")
+        .action(async (options) => {
+        const { config, globalOptions } = await load();
+        const copy = new Copy(config, globalOptions);
+        await copy.run(options);
+    });
+    program
+        .command("prune")
+        .alias("p")
+        .description("Run prune action")
+        .option("-r, --repositories <names>", "Repository names", (v) => parseStringList(v))
+        .option("-p, --packages <packages>", "Package names", (v) => parseStringList(v))
+        .option("--prune", "Prune after copy")
+        .action(async (options) => {
+        const { config, globalOptions } = await load();
+        const prune = new Prune(config, globalOptions);
+        await prune.run(options);
+    });
+    const parse = program.parse.bind(program);
+    program.parse = function (args, options) {
+        if (!args)
+            args = process.argv;
+        const [node, script, ...rest] = args;
+        const commandIndex = rest.findIndex((v) => !v.startsWith("-"));
+        const command = rest[commandIndex];
+        if (command === "run") {
+            args = [
+                node,
+                script,
+                ...rest.flatMap((value, index) => commandIndex === index ? [value, "--"] : value),
+            ];
+        }
+        return parse(args, options);
+    };
+    return program;
+}

package/lib/index.d.ts

@@ -2,4 +2,5 @@ export { Backup, type BackupOptions } from "./actions/backup.js";
 export { Copy, type CopyOptions } from "./actions/copy.js";
 export { Init, type InitOptions } from "./actions/init.js";
 export { Prune, type PruneOptions } from "./actions/prune.js";
-export { type Config, type GlobalConfig, parseConfigFile, defineConfig, validateConfig, } from "./config.js";
+export { type Config, type GlobalConfig, type PrunePolicy, parseConfigFile, defineConfig, validateConfig, } from "./config.js";
+export { createBin } from "./create-bin.js";

package/lib/index.js

@@ -3,3 +3,4 @@ export { Copy } from "./actions/copy.js";
 export { Init } from "./actions/init.js";
 export { Prune } from "./actions/prune.js";
 export { parseConfigFile, defineConfig, validateConfig, } from "./config.js";
+export { createBin } from "./create-bin.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/restic",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "description": "Tool for creating and managing backups",
   "homepage": "https://github.com/swordev/datatruck#readme",
   "bugs": {