@datatruck/cli 0.26.2 → 0.28.0

package/utils/tar.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractTar = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
+exports.extractTar = exports.normalizeTarPath = exports.createTar = exports.checkPigzLib = exports.listTar = exports.getTarVendor = exports.compressDefinition = void 0;
 const cli_1 = require("./cli");
 const fs_1 = require("./fs");
 const math_1 = require("./math");
@@ -52,13 +52,18 @@ async function getTarVendor(cache = true, log = false) {
 }
 exports.getTarVendor = getTarVendor;
 async function listTar(options) {
+    const vendor = await getTarVendor(true, options.verbose);
     let total = 0;
-    await (0, process_1.exec)("tar", ["-tf", toLocalPath(options.input), "--force-local"], {}, {
+    await (0, process_1.exec)("tar", [
+        "-tf",
+        toLocalPath(options.input),
+        ...(vendor === "bsdtar" ? [] : ["--force-local"]),
+    ], {}, {
         log: options.verbose,
         stdout: {
             parseLines: "skip-empty",
             onData: (path) => {
-                options.onEntry?.({ path });
+                options.onEntry?.({ path: normalizeTarPath(path) });
                 total++;
             },
         },
@@ -91,7 +96,9 @@ async function ifX(input, cb) {
 }
 async function createTar(options) {
     const vendor = await getTarVendor(true, options.verbose);
-    const total = await (0, fs_1.countFileLines)(options.includeList);
+    const total = "include" in options
+        ? options.include.length
+        : await (0, fs_1.countFileLines)(options.includeList);
     const compress = await ifX(options.compress, async (compress) => ({
         ...compress,
         cores: await resolveCores(compress.cores),
@@ -103,16 +110,27 @@ async function createTar(options) {
             GZIP_OPT: compress.level.toString(),
         }),
     };
+    const onData = (line) => {
+        current++;
+        let path = vendor === "bsdtar" ? line.slice(2) : line;
+        options.onEntry?.({
+            path: normalizeTarPath(path),
+            progress: {
+                total,
+                current,
+                percent: (0, math_1.progressPercent)(total, current),
+            },
+        });
+    };
     await (0, process_1.exec)("tar", [
         "--no-recursion",
         "-C",
         toLocalPath(options.path),
         compress?.cores === 1 ? "-czvf" : "-cvf",
         toLocalPath(options.output),
-        "-T",
-        toLocalPath(options.includeList),
-        "--ignore-failed-read",
-        "--force-local",
+        // https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=172293
+        ...(vendor === "bsdtar" ? [] : ["--ignore-failed-read"]),
+        ...(vendor === "bsdtar" ? [] : ["--force-local"]),
         ...(compress && compress.cores > 1
             ? [
                 "-I",
@@ -126,6 +144,9 @@ async function createTar(options) {
                     .join(" ")}"`,
             ]
             : []),
+        ...("includeList" in options
+            ? ["-T", toLocalPath(options.includeList)]
+            : ["--", ...options.include]),
     ], {
         ...(compress &&
             compress.cores > 1 && {
@@ -137,24 +158,16 @@ async function createTar(options) {
         },
     }, {
         log: options.verbose ? { envNames: Object.keys(env) } : false,
-        stderr: { toExitCode: true },
-        stdout: options.onEntry
+        ...(vendor === "bsdtar"
             ? {
-                parseLines: "skip-empty",
-                onData: (line) => {
-                    current++;
-                    const path = vendor === "bsdtar" ? line.slice(2) : line;
-                    options.onEntry?.({
-                        path,
-                        progress: {
-                            total,
-                            current,
-                            percent: (0, math_1.progressPercent)(total, current),
-                        },
-                    });
-                },
+                stderr: options.onEntry
+                    ? { toExitCode: true, parseLines: "skip-empty", onData }
+                    : { toExitCode: true },
             }
-            : undefined,
+            : {
+                stderr: { toExitCode: true },
+                stdout: { parseLines: "skip-empty", onData },
+            }),
     });
 }
 exports.createTar = createTar;
@@ -164,9 +177,18 @@ exports.createTar = createTar;
 function toLocalPath(path) {
     return (0, os_1.platform)() === "win32" ? path.replace(/\\/g, "/") : path;
 }
+/**
+ * bsdtar (only windows?) fails if path ends with slash
+ */
+function normalizeTarPath(path) {
+    return path.endsWith("/") ? path.slice(0, -1) : path;
+}
+exports.normalizeTarPath = normalizeTarPath;
 async function extractTar(options) {
-    let total = options.total ??
-        (await listTar({ input: options.input, verbose: options.verbose }));
+    let total = options.onEntry
+        ? options.total ??
+            (await listTar({ input: options.input, verbose: options.verbose }))
+        : undefined;
     if (!(await (0, fs_1.existsDir)(options.output))) {
         if (options.verbose)
             (0, cli_1.logExec)("mkdir", ["-p", options.output]);
@@ -177,12 +199,25 @@ async function extractTar(options) {
         cores: await resolveCores(decompress.cores),
     }));
     let current = 0;
+    const vendor = await getTarVendor(true, options.verbose);
+    const onData = (line) => {
+        const path = vendor === "bsdtar" ? line.slice(2) : line;
+        current++;
+        options.onEntry?.({
+            path: normalizeTarPath(path),
+            progress: {
+                total: total,
+                current,
+                percent: (0, math_1.progressPercent)(total, current),
+            },
+        });
+    };
     await (0, process_1.exec)("tar", [
         decompress?.cores === 1 ? "-xzvpf" : "-xvpf",
         toLocalPath(options.input),
         "-C",
         toLocalPath(options.output),
-        "--force-local",
+        ...(vendor === "bsdtar" ? [] : ["--force-local"]),
         ...(decompress && decompress.cores > 1
             ? ["-I", `"pigz -p ${decompress.cores}"`]
             : []),
@@ -193,25 +228,16 @@ async function extractTar(options) {
         }),
     }, {
         log: options.verbose,
-        stderr: {
-            toExitCode: true,
-        },
-        stdout: options.onEntry
+        ...(vendor === "bsdtar"
             ? {
-                parseLines: "skip-empty",
-                onData: (path) => {
-                    current++;
-                    options.onEntry?.({
-                        path,
-                        progress: {
-                            total,
-                            current,
-                            percent: (0, math_1.progressPercent)(total, current),
-                        },
-                    });
-                },
+                stderr: options.onEntry
+                    ? { toExitCode: true, parseLines: "skip-empty", onData }
+                    : { toExitCode: true },
             }
-            : undefined,
+            : {
+                stderr: { toExitCode: true },
+                stdout: { parseLines: "skip-empty", onData },
+            }),
     });
 }
 exports.extractTar = extractTar;

package/utils/temp.d.ts

@@ -0,0 +1,26 @@
+/// <reference types="node" />
+export declare function parentTmpDir(): string;
+export declare function sessionTmpDir(): string;
+export declare function ensureFreeDiskTempSpace(size: number | string): Promise<void>;
+export declare function isTmpDir(path: string): boolean;
+export declare function rmTmpDir(input: string | string[]): Promise<void>;
+export declare function tmpDir(...keys: [string, ...string[]]): string;
+export declare function mkTmpDir(...keys: [string, ...string[]]): Promise<string>;
+export declare function useTempDir(...keys: [string, ...string[]]): Promise<AsyncDisposable & {
+    path: string;
+}>;
+export declare function useTempFile(path: string): AsyncDisposable & {
+    path: string;
+};
+export declare class CleanupListener {
+    readonly paths: string[];
+    stop(): void;
+    dispose(): Promise<void>;
+}
+export declare class GargabeCollector {
+    protected listeners: Set<CleanupListener>;
+    get pending(): boolean;
+    cleanup(cb?: () => any): Promise<void>;
+    cleanupOnFinish(cb: () => any): Promise<void>;
+    cleanupIfFail(cb: () => any): Promise<CleanupListener>;
+}

package/utils/temp.js

@@ -0,0 +1,133 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GargabeCollector = exports.CleanupListener = exports.useTempFile = exports.useTempDir = exports.mkTmpDir = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.ensureFreeDiskTempSpace = exports.sessionTmpDir = exports.parentTmpDir = void 0;
+const globalData_1 = __importDefault(require("../globalData"));
+const fs_1 = require("./fs");
+const crypto_1 = require("crypto");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+Symbol.dispose ??= Symbol("Symbol.dispose");
+Symbol.asyncDispose ??= Symbol("Symbol.asyncDispose");
+function parentTmpDir() {
+    return (0, path_1.join)(globalData_1.default.tempDir, "datatruck-temp");
+}
+exports.parentTmpDir = parentTmpDir;
+function sessionTmpDir() {
+    return (0, path_1.join)(parentTmpDir(), process.pid.toString());
+}
+exports.sessionTmpDir = sessionTmpDir;
+async function ensureFreeDiskTempSpace(size) {
+    const path = sessionTmpDir();
+    await (0, fs_1.mkdirIfNotExists)(sessionTmpDir());
+    await (0, fs_1.ensureFreeDiskSpace)([path], size);
+}
+exports.ensureFreeDiskTempSpace = ensureFreeDiskTempSpace;
+function isTmpDir(path) {
+    return path.startsWith(sessionTmpDir()) && path.includes("datatruck-temp");
+}
+exports.isTmpDir = isTmpDir;
+async function rmTmpDir(input) {
+    if (typeof input === "string") {
+        if (!isTmpDir(input))
+            throw new Error(`Path is not a temp dir: ${input}`);
+        await (0, promises_1.rm)(input, { recursive: true });
+    }
+    else {
+        for (const path of input)
+            await rmTmpDir(path);
+    }
+}
+exports.rmTmpDir = rmTmpDir;
+function tmpDir(...keys) {
+    const id = (0, crypto_1.randomUUID)().slice(0, 8);
+    const path = (0, path_1.join)(sessionTmpDir(), [...keys, id].map(encodeURIComponent).join("-"));
+    for (const listener of listeners)
+        listener.paths.push(path);
+    return path;
+}
+exports.tmpDir = tmpDir;
+const listeners = new Set();
+async function mkTmpDir(...keys) {
+    const path = tmpDir(...keys);
+    await (0, promises_1.mkdir)(path, { recursive: true });
+    return path;
+}
+exports.mkTmpDir = mkTmpDir;
+async function useTempDir(...keys) {
+    const path = await mkTmpDir(...keys);
+    return {
+        path,
+        async [Symbol.asyncDispose]() {
+            try {
+                await rmTmpDir(path);
+            }
+            catch (_) { }
+        },
+    };
+}
+exports.useTempDir = useTempDir;
+function useTempFile(path) {
+    return {
+        path,
+        async [Symbol.asyncDispose]() {
+            try {
+                await (0, promises_1.rm)(path, { recursive: true });
+            }
+            catch (_) { }
+        },
+    };
+}
+exports.useTempFile = useTempFile;
+class CleanupListener {
+    paths = [];
+    stop() {
+        listeners.delete(this);
+    }
+    async dispose() {
+        this.stop();
+        await rmTmpDir(this.paths);
+    }
+}
+exports.CleanupListener = CleanupListener;
+class GargabeCollector {
+    listeners = new Set();
+    get pending() {
+        return this.listeners.size > 0;
+    }
+    async cleanup(cb) {
+        try {
+            await cb?.();
+        }
+        finally {
+            for (const listener of this.listeners) {
+                this.listeners.delete(listener);
+                await listener.dispose();
+            }
+        }
+    }
+    async cleanupOnFinish(cb) {
+        const cleanup = new CleanupListener();
+        try {
+            await cb();
+        }
+        finally {
+            cleanup.dispose();
+        }
+    }
+    async cleanupIfFail(cb) {
+        const cleanup = new CleanupListener();
+        try {
+            await cb();
+        }
+        catch (error) {
+            await cleanup.dispose();
+            throw error;
+        }
+        this.listeners.add(cleanup);
+        return cleanup;
+    }
+}
+exports.GargabeCollector = GargabeCollector;

package/utils/virtual-fs.d.ts

@@ -0,0 +1,37 @@
+import { DiskStats } from "./fs";
+export declare function resolvePath(path: string): string;
+export type FsOptions = {
+    backend: string;
+};
+export declare abstract class AbstractFs {
+    readonly options: FsOptions;
+    constructor(options: FsOptions);
+    resolvePath(path: string): string;
+    abstract isLocal(): boolean;
+    isRemote(): boolean;
+    abstract existsDir(path: string): Promise<boolean>;
+    abstract mkdir(path: string): Promise<void>;
+    abstract readFile(path: string): Promise<string>;
+    abstract rmAll(path: string): Promise<void>;
+    abstract readFileIfExists(path: string): Promise<string | undefined>;
+    abstract readdir(path: string): Promise<string[]>;
+    abstract ensureEmptyDir(path: string): Promise<void>;
+    abstract writeFile(path: string, contents: string): Promise<void>;
+    abstract upload(source: string, target: string): Promise<void>;
+    abstract download(source: string, target: string): Promise<void>;
+    abstract fetchDiskStats(source: string): Promise<DiskStats>;
+}
+export declare class LocalFs extends AbstractFs {
+    isLocal(): boolean;
+    existsDir(path: string): Promise<boolean>;
+    mkdir(path: string): Promise<void>;
+    ensureEmptyDir(path: string): Promise<void>;
+    readFile(path: string): Promise<string>;
+    readFileIfExists(inPath: string): Promise<string | undefined>;
+    readdir(path: string): Promise<string[]>;
+    writeFile(path: string, contents: string): Promise<void>;
+    rmAll(path: string): Promise<void>;
+    fetchDiskStats(source: string): Promise<DiskStats>;
+    upload(source: string, target: string): Promise<void>;
+    download(source: string, target: string): Promise<void>;
+}

package/utils/virtual-fs.js

@@ -0,0 +1,65 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LocalFs = exports.AbstractFs = exports.resolvePath = void 0;
+const fs_1 = require("./fs");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+function resolvePath(path) {
+    const { pathname } = new URL(`file:///${path}`);
+    return pathname;
+}
+exports.resolvePath = resolvePath;
+class AbstractFs {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    resolvePath(path) {
+        return (0, path_1.resolve)((0, path_1.join)(this.options.backend ?? ".", resolvePath(path)));
+    }
+    isRemote() {
+        return !this.isLocal();
+    }
+}
+exports.AbstractFs = AbstractFs;
+class LocalFs extends AbstractFs {
+    isLocal() {
+        return true;
+    }
+    async existsDir(path) {
+        return (0, fs_1.existsDir)(this.resolvePath(path));
+    }
+    async mkdir(path) {
+        await (0, fs_1.mkdirIfNotExists)(this.resolvePath(path));
+    }
+    async ensureEmptyDir(path) {
+        await (0, fs_1.ensureEmptyDir)(this.resolvePath(path));
+    }
+    async readFile(path) {
+        return (await (0, promises_1.readFile)(this.resolvePath(path))).toString();
+    }
+    async readFileIfExists(inPath) {
+        return (await (0, fs_1.existsFile)(this.resolvePath(inPath)))
+            ? await this.readFile(inPath)
+            : undefined;
+    }
+    async readdir(path) {
+        return await (0, promises_1.readdir)(this.resolvePath(path));
+    }
+    async writeFile(path, contents) {
+        await (0, promises_1.writeFile)(this.resolvePath(path), contents);
+    }
+    async rmAll(path) {
+        await (0, promises_1.rm)(this.resolvePath(path), { recursive: true });
+    }
+    async fetchDiskStats(source) {
+        return await (0, fs_1.fetchDiskStats)(this.resolvePath(source));
+    }
+    async upload(source, target) {
+        await (0, promises_1.cp)(source, this.resolvePath(target));
+    }
+    async download(source, target) {
+        await (0, promises_1.cp)(this.resolvePath(source), target);
+    }
+}
+exports.LocalFs = LocalFs;

package/Action/BackupSessionsAction.d.ts

@@ -1,13 +0,0 @@
-import type { ConfigType } from "../Config/Config";
-import { ReadDataType } from "../SessionDriver/SessionDriverAbstract";
-import { BackupSessionManager } from "../SessionManager/BackupSessionManager";
-import { IfRequireKeys } from "../utils/ts";
-export type BackupSessionsActionOptionsType = ReadDataType & {
-    verbose?: boolean;
-};
-export declare class BackupSessionsAction<TRequired extends boolean = true> {
-    readonly config: ConfigType;
-    readonly options: IfRequireKeys<TRequired, BackupSessionsActionOptionsType>;
-    constructor(config: ConfigType, options: IfRequireKeys<TRequired, BackupSessionsActionOptionsType>);
-    exec(session: BackupSessionManager): Promise<import("../SessionDriver/SessionDriverAbstract").ReadResultType[]>;
-}

package/Action/BackupSessionsAction.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BackupSessionsAction = void 0;
-class BackupSessionsAction {
-    config;
-    options;
-    constructor(config, options) {
-        this.config = config;
-        this.options = options;
-    }
-    async exec(session) {
-        await session.initDrivers();
-        const result = await session.readAll(this.options);
-        await session.endDrivers();
-        return result;
-    }
-}
-exports.BackupSessionsAction = BackupSessionsAction;

package/Action/RestoreSessionsAction.d.ts

@@ -1,13 +0,0 @@
-import type { ConfigType } from "../Config/Config";
-import { ReadDataType } from "../SessionDriver/SessionDriverAbstract";
-import { RestoreSessionManager } from "../SessionManager/RestoreSessionManager";
-import { IfRequireKeys } from "../utils/ts";
-export type RestoreSessionsActionOptionsType = ReadDataType & {
-    verbose?: boolean;
-};
-export declare class RestoreSessionsAction<TRequired extends boolean = true> {
-    readonly config: ConfigType;
-    readonly options: IfRequireKeys<TRequired, RestoreSessionsActionOptionsType>;
-    constructor(config: ConfigType, options: IfRequireKeys<TRequired, RestoreSessionsActionOptionsType>);
-    exec(manager: RestoreSessionManager): Promise<import("../SessionDriver/SessionDriverAbstract").ReadResultType[]>;
-}

package/Action/RestoreSessionsAction.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RestoreSessionsAction = void 0;
-class RestoreSessionsAction {
-    config;
-    options;
-    constructor(config, options) {
-        this.config = config;
-        this.options = options;
-    }
-    async exec(manager) {
-        await manager.initDrivers();
-        const result = await manager.readAll(this.options);
-        await manager.endDrivers();
-        return result;
-    }
-}
-exports.RestoreSessionsAction = RestoreSessionsAction;

package/Command/BackupSessionsCommand.d.ts

@@ -1,12 +0,0 @@
-import { If } from "../utils/ts";
-import { CommandAbstract } from "./CommandAbstract";
-export type BackupSessionsCommandOptionsType<TResolved = false> = {
-    package?: If<TResolved, string[]>;
-    repository?: If<TResolved, string[]>;
-    tag?: If<TResolved, string[]>;
-    limit?: number;
-};
-export declare class BackupSessionsCommand extends CommandAbstract<BackupSessionsCommandOptionsType<false>, BackupSessionsCommandOptionsType<true>> {
-    onOptions(): import("../utils/cli").OptionsType<BackupSessionsCommandOptionsType<false>, BackupSessionsCommandOptionsType<true>>;
-    onExec(): Promise<number>;
-}

package/Command/BackupSessionsCommand.js

@@ -1,92 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BackupSessionsCommand = void 0;
-const BackupSessionsAction_1 = require("../Action/BackupSessionsAction");
-const ConfigAction_1 = require("../Action/ConfigAction");
-const SqliteSessionDriver_1 = require("../SessionDriver/SqliteSessionDriver");
-const BackupSessionManager_1 = require("../SessionManager/BackupSessionManager");
-const DataFormat_1 = require("../utils/DataFormat");
-const cli_1 = require("../utils/cli");
-const string_1 = require("../utils/string");
-const CommandAbstract_1 = require("./CommandAbstract");
-class BackupSessionsCommand extends CommandAbstract_1.CommandAbstract {
-    onOptions() {
-        return this.returnsOptions({
-            package: {
-                description: "Filter by package names",
-                option: "-p,--package <values>",
-                parser: string_1.parseStringList,
-            },
-            repository: {
-                description: "Filter by repository names",
-                option: "-r,--repository <values>",
-                parser: string_1.parseStringList,
-            },
-            tag: {
-                description: "Filter by tags",
-                option: "-t,--tag <values>",
-                parser: string_1.parseStringList,
-            },
-            limit: {
-                description: "Limit",
-                option: "-l,--limit <value>",
-                defaults: 10,
-                parser: Number,
-            },
-        });
-    }
-    async onExec() {
-        const verbose = this.globalOptions.verbose ?? 0;
-        const config = await ConfigAction_1.ConfigAction.fromGlobalOptions(this.globalOptions);
-        const action = new BackupSessionsAction_1.BackupSessionsAction(config, {
-            packageNames: this.options.package,
-            repositoryNames: this.options.repository,
-            tags: this.options.tag,
-            limit: this.options.limit,
-            verbose: verbose > 0,
-        });
-        const manager = new BackupSessionManager_1.BackupSessionManager({
-            driver: new SqliteSessionDriver_1.SqliteSessionDriver({
-                verbose: verbose > 1,
-            }),
-            verbose: verbose > 1,
-            progressInterval: this.globalOptions.progressInterval,
-        });
-        const items = await action.exec(manager);
-        const dataFormat = new DataFormat_1.DataFormat({
-            items,
-            table: {
-                labels: [
-                    "   ",
-                    "Id.",
-                    "Snapshot id.",
-                    "Date",
-                    "Repository",
-                    "Repository type",
-                    "Package",
-                    "",
-                ],
-                handler: (item) => [
-                    (0, cli_1.resultColumn)(item.error, item.state),
-                    item.id,
-                    item.snapshotId.slice(0, 8),
-                    (0, string_1.formatDateTime)(item.creationDate),
-                    item.repositoryName,
-                    item.repositoryType,
-                    item.packageName,
-                    (0, cli_1.errorColumn)(item.error, verbose),
-                ],
-            },
-        });
-        if (this.globalOptions.outputFormat)
-            console.info(dataFormat.format(this.globalOptions.outputFormat, {
-                tpl: {
-                    sids: () => items.map((i) => i.snapshotId).join(),
-                    ssids: () => items.map((i) => i.snapshotId.slice(0, 8)).join(),
-                    pkgNames: () => items.map((i) => i.packageName).join(),
-                },
-            }));
-        return 0;
-    }
-}
-exports.BackupSessionsCommand = BackupSessionsCommand;

package/Command/RestoreSessionsCommand.d.ts

@@ -1,12 +0,0 @@
-import { If } from "../utils/ts";
-import { CommandAbstract } from "./CommandAbstract";
-export type RestoreSessionsCommandOptionsType<TResolved = false> = {
-    package?: If<TResolved, string[]>;
-    repository?: If<TResolved, string[]>;
-    tag?: If<TResolved, string[]>;
-    limit?: number | null;
-};
-export declare class RestoreSessionsCommand extends CommandAbstract<RestoreSessionsCommandOptionsType<false>, RestoreSessionsCommandOptionsType<true>> {
-    onOptions(): import("../utils/cli").OptionsType<RestoreSessionsCommandOptionsType<false>, RestoreSessionsCommandOptionsType<true>>;
-    onExec(): Promise<number>;
-}