@datatruck/cli 0.26.1 → 0.27.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.26.1",
+  "version": "0.27.0",
   "dependencies": {
     "@supercharge/promise-pool": "^3.1.0",
     "ajv": "^8.12.0",

package/utils/cli.js

@@ -96,7 +96,9 @@ exports.errorColumn = errorColumn;
 function parseOptions(object, options) {
     const result = {};
     for (const key in options) {
-        const value = object?.[key] ?? options[key].defaults;
+        const isNegative = options[key].option.startsWith("--no");
+        const defaultsValue = isNegative ? true : options[key].defaults;
+        const value = object?.[key] ?? defaultsValue;
         const parser = options[key].parser;
         if (typeof value !== "undefined") {
             result[key] = parser ? parser(value) : value;

package/utils/datatruck/client.d.ts

@@ -0,0 +1,22 @@
+import { AbstractFs, FsOptions } from "../virtual-fs";
+export declare class RemoteFs extends AbstractFs {
+    readonly options: FsOptions;
+    protected url: string;
+    protected headers: Record<string, string>;
+    constructor(options: FsOptions);
+    isLocal(): boolean;
+    protected fetchJson(name: string, params: any[]): Promise<any>;
+    protected post(name: string, params: any[], data: string): Promise<void>;
+    existsDir(path: string): Promise<any>;
+    mkdir(path: string): Promise<any>;
+    readFile(path: string): Promise<any>;
+    readdir(path: string): Promise<any>;
+    readFileIfExists(path: string): Promise<string | undefined>;
+    ensureEmptyDir(path: string): Promise<void>;
+    writeFile(path: string, contents: string): Promise<void>;
+    rmAll(path: string): Promise<void>;
+    upload(source: string, target: string): Promise<void>;
+    download(source: string, target: string, timeout?: number): Promise<void>;
+}
+export declare function isRemoteBackend(backend: string): boolean;
+export declare function createFs(backend: string): AbstractFs;

package/utils/datatruck/client.js

@@ -0,0 +1,96 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createFs = exports.isRemoteBackend = exports.RemoteFs = void 0;
+const http_1 = require("../http");
+const virtual_fs_1 = require("../virtual-fs");
+const server_1 = require("./server");
+class RemoteFs extends virtual_fs_1.AbstractFs {
+    options;
+    url;
+    headers;
+    constructor(options) {
+        super(options);
+        this.options = options;
+        const url = new URL(options.backend);
+        this.headers = {
+            [server_1.headerKey.user]: url.username,
+            [server_1.headerKey.password]: url.password,
+        };
+        url.username = "";
+        url.password = "";
+        this.url = url.href;
+        if (this.url.endsWith("/"))
+            this.url = this.url.slice(0, -1);
+    }
+    isLocal() {
+        return false;
+    }
+    async fetchJson(name, params) {
+        return await (0, http_1.fetchJson)(`${this.url}/${name}`, {
+            headers: this.headers,
+            query: {
+                params: JSON.stringify(params),
+            },
+        });
+    }
+    async post(name, params, data) {
+        return await (0, http_1.post)(`${this.url}/${name}`, data, {
+            headers: this.headers,
+            query: {
+                params: JSON.stringify(params),
+            },
+        });
+    }
+    async existsDir(path) {
+        return await this.fetchJson("existsDir", [path]);
+    }
+    async mkdir(path) {
+        return await this.fetchJson("mkdir", [path]);
+    }
+    async readFile(path) {
+        return await this.fetchJson("readFile", [path]);
+    }
+    async readdir(path) {
+        return await this.fetchJson("readdir", [path]);
+    }
+    async readFileIfExists(path) {
+        return await this.fetchJson("readFileIfExists", [path]);
+    }
+    async ensureEmptyDir(path) {
+        return await this.fetchJson("readdir", [path]);
+    }
+    async writeFile(path, contents) {
+        await this.post("writeFile", [path], contents);
+    }
+    async rmAll(path) {
+        await this.fetchJson("rmAll", [path]);
+    }
+    async upload(source, target) {
+        await (0, http_1.uploadFile)(`${this.url}/upload`, source, {
+            headers: this.headers,
+            query: {
+                params: JSON.stringify([target]),
+            },
+        });
+    }
+    async download(source, target, timeout = 100000) {
+        await (0, http_1.downloadFile)(`${this.url}/download`, target, {
+            timeout,
+            headers: this.headers,
+            query: {
+                params: JSON.stringify([source]),
+            },
+        });
+    }
+}
+exports.RemoteFs = RemoteFs;
+function isRemoteBackend(backend) {
+    return backend.startsWith("http:") || backend.startsWith("https:");
+}
+exports.isRemoteBackend = isRemoteBackend;
+function createFs(backend) {
+    return isRemoteBackend(backend)
+        ? new RemoteFs({ backend })
+        : new virtual_fs_1.LocalFs({ backend });
+}
+exports.createFs = createFs;

package/utils/datatruck/config.d.ts

@@ -31,18 +31,18 @@ export declare const pkgPathParams: {
     [name in "temp" | Exclude<keyof ResolvePackagePathParamsType, "path">]: string;
 };
 export declare const pkgIncludeParams: {
+    action: string;
     packageName: string;
     temp: string;
     snapshotId: string;
     snapshotDate: string;
-    action: string;
 };
 export declare const pkgExcludeParams: {
+    action: string;
     packageName: string;
     temp: string;
     snapshotId: string;
     snapshotDate: string;
-    action: string;
 };
 export declare const pkgRestorePathParams: {
     [name in "temp" | keyof ResolvePackagePathParamsType]: string;
@@ -52,33 +52,33 @@ export declare const dbNameParams: {
 };
 export declare const params: {
     pkgPath: {
+        action: string;
         packageName: string;
         temp: string;
         snapshotId: string;
         snapshotDate: string;
-        action: string;
     };
     pkgRestorePath: {
         path: string;
+        action: string;
         packageName: string;
         temp: string;
         snapshotId: string;
         snapshotDate: string;
-        action: string;
     };
     pkgInclude: {
+        action: string;
         packageName: string;
         temp: string;
         snapshotId: string;
         snapshotDate: string;
-        action: string;
     };
     pkgExclude: {
+        action: string;
         packageName: string;
         temp: string;
         snapshotId: string;
         snapshotDate: string;
-        action: string;
     };
     dbName: {
         snapshotId: string;

package/utils/datatruck/server.d.ts

@@ -0,0 +1,21 @@
+/// <reference types="node" />
+import { IncomingMessage } from "http";
+type User = {
+    name: string;
+    password: string;
+};
+export type DatatruckServerOptions = {
+    path?: string;
+    log?: boolean;
+    listen?: {
+        port?: number;
+        address?: string;
+    };
+    users?: User[];
+};
+export declare const headerKey: {
+    user: string;
+    password: string;
+};
+export declare function createDatatruckServer(options: DatatruckServerOptions): import("node:http").Server<typeof IncomingMessage, typeof import("node:http").ServerResponse>;
+export {};

package/utils/datatruck/server.js

@@ -0,0 +1,96 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDatatruckServer = exports.headerKey = void 0;
+const http_1 = require("../http");
+const virtual_fs_1 = require("../virtual-fs");
+const fs_1 = require("fs");
+const http_2 = require("http");
+function parseUrl(inUrl) {
+    const url = new URL(`http://127.0.0.1${inUrl}`);
+    const inParams = url.searchParams.get("params");
+    const action = url.pathname.slice(1);
+    if (typeof inParams === "string") {
+        const params = JSON.parse(inParams);
+        if (!Array.isArray(params))
+            throw new Error(`Invalid params`);
+        return { action, params };
+    }
+    else {
+        return { action, params: [] };
+    }
+}
+exports.headerKey = {
+    user: "x-dtt-user",
+    password: "x-dtt-password",
+};
+function validateRequest(req, users) {
+    const name = req.headers[exports.headerKey.user];
+    const password = req.headers[exports.headerKey.password];
+    return users.some((user) => user.name.length &&
+        user.password.length &&
+        user.name === name &&
+        user.password === password);
+}
+function createDatatruckServer(options) {
+    const log = options.log ?? true;
+    return (0, http_2.createServer)(async (req, res) => {
+        try {
+            if (req.url === "/" || req.url === "/favicon.ico") {
+                return res.end();
+            }
+            else if (!validateRequest(req, options.users || [])) {
+                res.statusCode = 401;
+                return res.end();
+            }
+            if (log)
+                console.info(`> ${req.url}`);
+            const fs = new virtual_fs_1.LocalFs({
+                backend: options.path ?? ".",
+            });
+            const { action, params } = parseUrl(req.url);
+            if (action === "upload") {
+                const [target] = params;
+                const file = (0, fs_1.createWriteStream)(fs.resolvePath(target));
+                req.pipe(file);
+                await new Promise((resolve, reject) => {
+                    req.on("error", reject);
+                    file.on("error", reject);
+                    file.on("close", resolve);
+                });
+            }
+            else if (action === "download") {
+                const [target] = params;
+                const file = (0, fs_1.createReadStream)(fs.resolvePath(target));
+                file.pipe(res);
+                await new Promise((resolve, reject) => {
+                    req.on("error", reject);
+                    file.on("error", reject);
+                    res.on("error", reject);
+                    res.on("close", resolve);
+                });
+            }
+            else if (action === "writeFile") {
+                const data = await (0, http_1.readRequestData)(req);
+                const [target] = params;
+                await fs.writeFile(target, data);
+            }
+            else {
+                const object = fs[action]?.bind(fs);
+                if (!object)
+                    throw new Error(`Invalid action: ${action}`);
+                const json = await object(...params);
+                if (json !== undefined)
+                    res.write(JSON.stringify(json));
+            }
+            res.end();
+            if (log)
+                console.info(`<${action}`);
+        }
+        catch (error) {
+            if (log)
+                console.error(`<${req.url}`, error);
+            res.statusCode = 500;
+        }
+    });
+}
+exports.createDatatruckServer = createDatatruckServer;

package/utils/fs.d.ts

@@ -87,18 +87,24 @@ export declare function cpy(options: {
     files: number;
     dirs: number;
 }>;
+type ProgressObject = {
+    disposed: boolean;
+    total: number;
+    current: number;
+    update: (description: string, path?: string, increment?: boolean) => Promise<void>;
+};
+export declare function createProgress(options: {
+    onProgress: (data: Progress) => Promise<void>;
+}): ProgressObject;
 export declare function createFileScanner(options: {
     glob: Options & {
         include: string[];
     };
     onProgress: (data: Progress) => Promise<void>;
-}): Promise<{
-    disposed: boolean;
-    total: number;
-    current: number;
-    progress: (description: string, path?: string, increment?: boolean) => Promise<void>;
+}): Promise<ProgressObject & {
+    progress: ProgressObject["update"];
+    start: (cb?: (entry: Required<Entry>) => any) => Promise<void>;
     end: () => Promise<void>;
-    start: (cb?: ((entry: Required<Entry>) => any) | undefined) => Promise<void>;
 }>;
 type StreamItem = {
     key: string;
@@ -121,4 +127,5 @@ export declare function createWriteStreamPool(options: {
 export declare function countFileLines(path: string): Promise<number>;
 export declare function fetchData<T>(input: T, onPath?: (input: Exclude<T, string>) => string | undefined): Promise<string | null>;
 export declare function safeRename(oldPath: string, newPath: string): Promise<void>;
+export declare function tryRm(path: string): Promise<void>;
 export {};

package/utils/fs.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
+exports.tryRm = exports.safeRename = exports.fetchData = exports.countFileLines = exports.createWriteStreamPool = exports.createFileScanner = exports.createProgress = exports.cpy = exports.isNotFoundError = exports.updateFileStats = exports.copyFileWithStreams = exports.waitForClose = exports.writeGitIgnoreList = exports.fastglobToGitIgnore = exports.forEachFile = exports.readDir = exports.readPartialFile = exports.mkTmpDir = exports.fastFolderSizeAsync = exports.tmpDir = exports.rmTmpDir = exports.isTmpDir = exports.sessionTmpDir = exports.parentTmpDir = exports.findFile = exports.parsePackageFile = exports.parseFile = exports.parseFileExtensions = exports.writeJSONFile = exports.existsFile = exports.existsDir = exports.safeStat = exports.ensureEmptyDir = exports.mkdirIfNotExists = exports.isLocalDir = exports.isEmptyDir = exports.isWSLSystem = void 0;
 const globalData_1 = __importDefault(require("../globalData"));
 const math_1 = require("./math");
 const path_1 = require("./path");
@@ -404,57 +404,65 @@ async function cpy(options) {
     return stats;
 }
 exports.cpy = cpy;
-async function createFileScanner(options) {
-    const object = {
+function createProgress(options) {
+    const progress = {
         disposed: false,
         total: 0,
         current: 0,
-        progress: async (description, path, increment = true) => {
-            if (object.disposed)
+        update: async (description, path, increment = true) => {
+            if (progress.disposed)
                 return;
             if (path && increment)
-                object.current++;
+                progress.current++;
             await options.onProgress({
                 relative: {
                     description,
                     payload: path,
                 },
                 absolute: {
-                    total: object.total,
-                    current: object.current,
-                    percent: (0, math_1.progressPercent)(object.total, object.current),
+                    total: progress.total,
+                    current: progress.current,
+                    percent: (0, math_1.progressPercent)(progress.total, progress.current),
                 },
             });
         },
+    };
+    return progress;
+}
+exports.createProgress = createProgress;
+async function createFileScanner(options) {
+    const progress = createProgress(options);
+    Object.assign(progress, {
+        progress: progress.update,
         end: async () => {
-            if (!object.disposed)
-                await object.progress("Finished");
-            object.disposed = true;
+            if (!progress.disposed)
+                await progress.update("Finished");
+            progress.disposed = true;
         },
         start: async (cb) => {
             let lastTime = performance.now();
-            await object.progress("Scanning files");
+            await progress.update("Scanning files");
             for await (const entry of pathIterator(stream)) {
                 if (cb) {
                     if (await cb(entry))
-                        object.total++;
+                        progress.total++;
                 }
                 else {
-                    object.total++;
+                    progress.total++;
                 }
                 if (lastTime - performance.now() > 500)
-                    await object.progress("Scanning files");
+                    await progress.update("Scanning files");
             }
-            await object.progress("Scanned files");
+            await progress.update("Scanned files");
         },
-    };
+    });
     const stream = fast_glob_1.default.stream(options.glob.include, {
         dot: true,
         markDirectories: true,
         stats: true,
         ...options.glob,
     });
-    return object;
+    return progress;
 }
 exports.createFileScanner = createFileScanner;
 function createWriteStreamPool(options) {
@@ -560,3 +568,10 @@ async function safeRename(oldPath, newPath) {
     }
 }
 exports.safeRename = safeRename;
+async function tryRm(path) {
+    try {
+        await (0, promises_1.rm)(path);
+    }
+    catch (_) { }
+}
+exports.tryRm = tryRm;

package/utils/http.d.ts

@@ -0,0 +1,21 @@
+/// <reference types="node" />
+import { IncomingMessage, Server } from "http";
+export declare function closeServer(server: Server): Promise<void>;
+export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
+export declare function fetchJson<T = any>(url: string, options?: {
+    headers?: Record<string, string>;
+    query?: Record<string, string>;
+}): Promise<T>;
+export declare function post(url: string, data: string, options?: {
+    headers?: Record<string, string>;
+    query?: Record<string, string>;
+}): Promise<void>;
+export declare function downloadFile(url: string, output: string, options?: {
+    headers?: Record<string, string>;
+    query?: Record<string, string>;
+    timeout?: number;
+}): Promise<void>;
+export declare function uploadFile(url: string, path: string, options?: {
+    headers?: Record<string, string>;
+    query?: Record<string, string>;
+}): Promise<void>;

package/utils/http.js

@@ -0,0 +1,154 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.readRequestData = exports.closeServer = void 0;
+const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
+const http_1 = require("http");
+const https_1 = require("https");
+const request = (url, options, callback) => {
+    return url.startsWith("https://")
+        ? (0, https_1.request)(url, options, callback)
+        : (0, http_1.request)(url, options, callback);
+};
+function href(inUrl, query) {
+    const url = new URL(inUrl);
+    for (const key in query || {})
+        url.searchParams.set(key, query[key]);
+    return url.href;
+}
+async function closeServer(server) {
+    await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
+}
+exports.closeServer = closeServer;
+function readRequestData(req) {
+    let data;
+    return new Promise((resolve, reject) => {
+        req
+            .on("error", reject)
+            .on("data", (chunk) => {
+            if (data === undefined)
+                data = "";
+            data += chunk;
+        })
+            .on("close", async () => {
+            resolve(data);
+        });
+    });
+}
+exports.readRequestData = readRequestData;
+async function fetchJson(url, options = {}) {
+    return new Promise((resolve, reject) => {
+        let data;
+        request(href(url, options.query), {
+            method: "GET",
+            headers: options.headers,
+        }, (res) => {
+            if (res.statusCode !== 200)
+                return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
+            res
+                .on("data", (chunk) => {
+                if (data === undefined)
+                    data = "";
+                data += chunk;
+            })
+                .on("error", reject)
+                .on("close", () => {
+                if (data === undefined) {
+                    resolve(undefined);
+                }
+                else {
+                    try {
+                        resolve(JSON.parse(data));
+                    }
+                    catch (error) {
+                        reject(error);
+                    }
+                }
+            });
+        })
+            .on("error", reject)
+            .end();
+    });
+}
+exports.fetchJson = fetchJson;
+async function post(url, data, options = {}) {
+    await new Promise((resolve, reject) => {
+        const req = request(href(url, options.query), { method: "POST", headers: options.headers }, (res) => {
+            res.on("error", reject);
+            if (res.statusCode !== 200) {
+                reject(new Error(`Post failed: ${res.statusCode} ${res.statusMessage}`));
+            }
+            else {
+                resolve();
+            }
+        });
+        req.on("error", reject);
+        req.write(data);
+        req.end();
+    });
+}
+exports.post = post;
+async function downloadFile(url, output, options = {}) {
+    const timeout = options.timeout ?? 3600 * 100;
+    const file = (0, fs_1.createWriteStream)(output);
+    await new Promise((resolve, reject) => {
+        const req = request(href(url, options.query), {
+            headers: options.headers,
+        }, (res) => {
+            if (res.statusCode === 200) {
+                res
+                    .on("error", async (error) => {
+                    file.destroy();
+                    try {
+                        await (0, promises_1.unlink)(output);
+                    }
+                    catch (_) { }
+                    reject(error);
+                })
+                    .pipe(file);
+                file.on("finish", () => {
+                    file.close((error) => {
+                        error ? reject(error) : resolve();
+                    });
+                });
+            }
+            else {
+                reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
+            }
+        }).on("error", async (error) => {
+            try {
+                await (0, promises_1.unlink)(output);
+            }
+            catch (_) { }
+            reject(error);
+        });
+        req.setTimeout(timeout, () => {
+            req.destroy();
+            reject(new Error(`Request timeout after ${timeout / 1000}s`));
+        });
+        req.end();
+    });
+}
+exports.downloadFile = downloadFile;
+async function uploadFile(url, path, options = {}) {
+    const { size } = await (0, promises_1.stat)(path);
+    const readStream = (0, fs_1.createReadStream)(path);
+    await new Promise((resolve, reject) => {
+        const req = request(href(url, options.query), {
+            method: "POST",
+            headers: {
+                ...options.headers,
+                "Content-length": size,
+            },
+        }, (res) => {
+            if (res.statusCode !== 200) {
+                reject(new Error(`Upload failed: ${res.statusCode} ${res.statusMessage}`));
+            }
+            else {
+                resolve();
+            }
+        }).on("error", reject);
+        readStream.on("error", reject).pipe(req);
+    });
+}
+exports.uploadFile = uploadFile;

package/utils/virtual-fs.d.ts

@@ -0,0 +1,33 @@
+export declare function resolvePath(path: string): string;
+export type FsOptions = {
+    backend: string;
+};
+export declare abstract class AbstractFs {
+    readonly options: FsOptions;
+    constructor(options: FsOptions);
+    resolvePath(path: string): string;
+    abstract isLocal(): boolean;
+    abstract existsDir(path: string): Promise<boolean>;
+    abstract mkdir(path: string): Promise<void>;
+    abstract readFile(path: string): Promise<string>;
+    abstract rmAll(path: string): Promise<void>;
+    abstract readFileIfExists(path: string): Promise<string | undefined>;
+    abstract readdir(path: string): Promise<string[]>;
+    abstract ensureEmptyDir(path: string): Promise<void>;
+    abstract writeFile(path: string, contents: string): Promise<void>;
+    abstract upload(url: string, path: string): Promise<void>;
+    abstract download(url: string, path: string): Promise<void>;
+}
+export declare class LocalFs extends AbstractFs {
+    isLocal(): boolean;
+    existsDir(path: string): Promise<boolean>;
+    mkdir(path: string): Promise<void>;
+    ensureEmptyDir(path: string): Promise<void>;
+    readFile(path: string): Promise<string>;
+    readFileIfExists(inPath: string): Promise<string | undefined>;
+    readdir(path: string): Promise<string[]>;
+    writeFile(path: string, contents: string): Promise<void>;
+    rmAll(path: string): Promise<void>;
+    upload(source: string, target: string): Promise<void>;
+    download(source: string, target: string): Promise<void>;
+}