@datatruck/cli 0.1.0 → 0.2.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @datatruck/cli
 
+## 0.2.0
+
+### Minor Changes
+
+- [`120460c`](https://github.com/swordev/datatruck/commit/120460c8824cef4184e43f571a4cc0798b899b66) Thanks [@juanrgm](https://github.com/juanrgm)! - Enable `include` option in restic repository
+
+### Patch Changes
+
+- [`e30ede3`](https://github.com/swordev/datatruck/commit/e30ede371bc7ab3fc1cd47758fdac7a28e8e2705) Thanks [@juanrgm](https://github.com/juanrgm)! - Resolve `RESTIC_PASSWORD_FILE` path
+
+* [`8539d28`](https://github.com/swordev/datatruck/commit/8539d285b2c51d700aa811cd772d573fa0d613eb) Thanks [@juanrgm](https://github.com/juanrgm)! - Allow empty backup in restic repository
+
 ## 0.1.0
 
 ### Minor Changes

package/Repository/ResticRepository.js

@@ -1,13 +1,18 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResticRepository = exports.resticPackageRepositoryDefinition = exports.resticRepositoryDefinition = exports.resticRepositoryName = void 0;
 const AppError_1 = require("../Error/AppError");
 const ResticUtil_1 = require("../util/ResticUtil");
+const cli_util_1 = require("../util/cli-util");
 const paths_util_1 = require("../util/datatruck/paths-util");
 const fs_util_1 = require("../util/fs-util");
 const string_util_1 = require("../util/string-util");
 const RepositoryAbstract_1 = require("./RepositoryAbstract");
 const assert_1 = require("assert");
+const fast_glob_1 = __importDefault(require("fast-glob"));
 const micromatch_1 = require("micromatch");
 const path_1 = require("path");
 exports.resticRepositoryName = "restic";
@@ -54,7 +59,7 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         if (this.env)
             return this.env;
         return (this.env = {
-            RESTIC_PASSWORD_FILE: this.config.passwordFile,
+            RESTIC_PASSWORD_FILE: (0, path_1.resolve)(this.config.passwordFile),
             RESTIC_REPOSITORY: await ResticUtil_1.ResticUtil.formatRepository(this.config.repository),
         });
     }
@@ -150,14 +155,28 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         const pkg = data.package;
         const sourcePath = data.targetPath ?? data.package.path;
         (0, assert_1.ok)(sourcePath);
-        const include = (await (0, paths_util_1.parsePaths)(pkg.include ?? [], {
+        const include = await (0, paths_util_1.parsePaths)(pkg.include ?? ["**"], {
             cwd: sourcePath,
             verbose: data.options.verbose,
-        })).map(path_1.normalize);
-        // https://github.com/restic/restic/issues/233
-        // https://github.com/restic/restic/pull/2311
-        if (include.length)
-            throw new AppError_1.AppError(`Include is not supported`);
+        });
+        const exclude = pkg.exclude
+            ? await (0, paths_util_1.parsePaths)(pkg.exclude, {
+                cwd: sourcePath,
+                verbose: data.options.verbose,
+            })
+            : undefined;
+        const stream = fast_glob_1.default.stream(include, {
+            cwd: sourcePath,
+            ignore: exclude,
+            dot: true,
+            onlyFiles: true,
+            markDirectories: true,
+        });
+        if (data.options.verbose)
+            (0, cli_util_1.logExec)(`Writing paths lists`);
+        const gitignorePath = await (0, fs_util_1.writeGitIgnoreList)({
+            paths: stream,
+        });
         if (data.options.tags?.some((tag) => tag.startsWith(ResticRepository.refPrefix)))
             throw new AppError_1.AppError(`Tag prefix is not allowed`);
         const packageTag = ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.PACKAGE, data.package.name);
@@ -170,15 +189,13 @@ class ResticRepository extends RepositoryAbstract_1.RepositoryAbstract {
         await restic.backup({
             cwd: sourcePath,
             paths: ["."],
+            allowEmptySnapshot: true,
+            excludeFile: [gitignorePath],
             parent: lastSnapshot?.id,
             // https://github.com/restic/restic/pull/3200
             ...((await restic.checkBackupSetPathSupport()) && {
                 setPaths: [`/datatruck/${data.package.name}`],
             }),
-            exclude: (await (0, paths_util_1.parsePaths)(pkg.exclude ?? [], {
-                cwd: sourcePath,
-                verbose: data.options.verbose,
-            })).map(path_1.normalize),
             tags: [
                 ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.ID, data.snapshot.id),
                 ResticRepository.buildSnapshotTag(RepositoryAbstract_1.SnapshotTagEnum.SHORT_ID, data.snapshot.id.slice(0, 8)),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "dependencies": {
     "ajv": "^8.11.0",
     "chalk": "^4.1.2",

package/util/ResticUtil.d.ts

@@ -1,4 +1,4 @@
-import { ExecSettingsInterface } from "./process-util";
+import { ExecResultType, ExecSettingsInterface } from "./process-util";
 import { UriType } from "./string-util";
 export declare type RepositoryType = {
     name?: string;
@@ -43,7 +43,7 @@ export declare class ResticUtil {
     static formatRepository(input: RepositoryType, hidePassword?: boolean): Promise<string>;
     exec(args: string[], settings?: ExecSettingsInterface, options?: {
         cwd?: string;
-    }): Promise<import("./process-util").ExecResultType>;
+    }): Promise<ExecResultType>;
     checkRepository(): Promise<boolean>;
     forget(options: {
         snapshotId?: string;
@@ -81,12 +81,14 @@ export declare class ResticUtil {
         paths: string[];
         setPaths?: string[];
         exclude?: string[];
+        excludeFile?: string[];
         parent?: string;
+        allowEmptySnapshot?: boolean;
         onStream?: (data: BackupStreamType) => Promise<void>;
-    }): Promise<import("./process-util").ExecResultType>;
+    }): Promise<ExecResultType>;
     restore(options: {
         id: string;
         target: string;
         onStream?: (data: BackupStreamType) => Promise<void>;
-    }): Promise<import("./process-util").ExecResultType>;
+    }): Promise<ExecResultType>;
 }

package/util/ResticUtil.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ResticUtil = void 0;
+const fs_util_1 = require("./fs-util");
 const process_util_1 = require("./process-util");
 const string_util_1 = require("./string-util");
 const fs_extra_1 = require("fs-extra");
+const promises_1 = require("fs/promises");
 const path_1 = require("path");
 class ResticUtil {
     constructor(options) {
@@ -13,7 +15,7 @@ class ResticUtil {
         if (input.backend === "local") {
             if (typeof input.path !== "string")
                 throw new Error(`Invalid path at "${input.name}" repository: ${input.path}`);
-            return (0, path_1.normalize)(input.path);
+            return (0, path_1.resolve)(input.path);
         }
         if (input.passwordFile)
             input = {
@@ -36,7 +38,7 @@ class ResticUtil {
                     stderr: true,
                     colorize: true,
                     allToStderr: true,
-                    envNames: ["RESTIC_REPOSITORY"],
+                    envNames: ["RESTIC_REPOSITORY", "RESTIC_PASSWORD_FILE"],
                 }
                 : {},
             ...(settings ?? {}),
@@ -99,10 +101,11 @@ class ResticUtil {
         return result.stderr.includes("flag needs an argument");
     }
     async backup(options) {
-        return await this.exec([
+        const exec = async () => await this.exec([
             "backup",
             "--json",
             ...(options.exclude?.flatMap((v) => ["-e", v]) ?? []),
+            ...(options.excludeFile?.flatMap((v) => ["--exclude-file", v]) ?? []),
             ...(options.tags?.flatMap((v) => ["--tag", v]) ?? []),
             ...(options.setPaths?.flatMap((v) => ["--set-path", v]) ?? []),
             ...(options.parent ? ["--parent", options.parent] : []),
@@ -123,6 +126,25 @@ class ResticUtil {
         }, {
             cwd: options.cwd,
         });
+        try {
+            return await exec();
+        }
+        catch (error) {
+            if (options.allowEmptySnapshot &&
+                error.message.includes("unable to save snapshot: snapshot is empty")) {
+                const emptyPath = await (0, fs_util_1.mkTmpDir)("empty");
+                await (0, promises_1.writeFile)(`${emptyPath}/.empty`, "");
+                return await this.backup({
+                    ...options,
+                    cwd: emptyPath,
+                    allowEmptySnapshot: false,
+                    paths: ["."],
+                    exclude: [],
+                    excludeFile: [],
+                });
+            }
+            throw error;
+        }
     }
     async restore(options) {
         return await this.exec(["restore", "--json", options.id, "--target", options.target], {

package/util/fs-util.d.ts

@@ -23,6 +23,9 @@ export declare function readPartialFile(path: string, positions: [number, number
 export declare function checkFile(path: string): Promise<boolean>;
 export declare function checkDir(path: string): Promise<boolean>;
 export declare function forEachFile(dirPath: string, cb: (path: string, dir: boolean) => void, includeDir?: boolean): Promise<void>;
+export declare function writeGitIgnoreList(options: {
+    paths: NodeJS.ReadableStream | string[];
+}): Promise<string>;
 export declare function writePathLists(options: {
     paths: NodeJS.ReadableStream | string[];
     packs?: {

package/util/fs-util.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writePathLists = exports.forEachFile = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = void 0;
+exports.writePathLists = exports.writeGitIgnoreList = exports.forEachFile = exports.checkDir = exports.checkFile = exports.readPartialFile = exports.mkTmpDir = exports.tmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.existsFile = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.readdirIfExists = exports.writeJSONFile = exports.existsDir = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isDirEmpty = exports.isLocalDir = void 0;
 const path_util_1 = require("./path-util");
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
@@ -190,6 +190,40 @@ async function forEachFile(dirPath, cb, includeDir) {
     }
 }
 exports.forEachFile = forEachFile;
+async function writeGitIgnoreList(options) {
+    const tempDir = await mkTmpDir("gitignore-list");
+    const path = (0, path_1.join)(tempDir, `.gitignore`);
+    const stream = (0, fs_2.createWriteStream)(path);
+    const dirs = new Set();
+    stream.write("*\n");
+    for await (const value of options.paths) {
+        const dir = (0, path_1.dirname)(value.toString());
+        if (dir !== ".") {
+            let parentPath;
+            for (const value of dir.split("/")) {
+                if (!parentPath) {
+                    parentPath = `!${value}`;
+                }
+                else {
+                    parentPath += `/${value}`;
+                }
+                if (!dirs.has(parentPath)) {
+                    stream.write(`${parentPath}\n`);
+                    stream.write(`${parentPath.slice(1)}/*\n`);
+                    dirs.add(parentPath);
+                }
+            }
+        }
+        stream.write(`!${value}\n`);
+    }
+    await new Promise((resolve, reject) => {
+        stream.close();
+        stream.on("close", resolve);
+        stream.on("error", reject);
+    });
+    return path;
+}
+exports.writeGitIgnoreList = writeGitIgnoreList;
 async function writePathLists(options) {
     const tempDir = await mkTmpDir("path-lists");
     const includedPaths = [];