@datatruck/cli 0.36.4 → 0.36.6

package/lib/utils/crypto.d.ts

@@ -1 +1,2 @@
 export declare function calcFileHash(path: string, algorithm: string): Promise<string>;
+export declare function assertFileChecksum(path: string, checksum: string, algorithm: string): Promise<void>;

package/lib/utils/crypto.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.calcFileHash = void 0;
+exports.assertFileChecksum = exports.calcFileHash = void 0;
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
 function calcFileHash(path, algorithm) {
@@ -13,3 +13,9 @@ function calcFileHash(path, algorithm) {
     });
 }
 exports.calcFileHash = calcFileHash;
+async function assertFileChecksum(path, checksum, algorithm) {
+    const fileChecksum = await calcFileHash(path, algorithm);
+    if (fileChecksum !== checksum)
+        throw new Error(`Invalid checksum file: ${checksum} != ${fileChecksum}`);
+}
+exports.assertFileChecksum = assertFileChecksum;

package/lib/utils/datatruck/client.d.ts

@@ -1,3 +1,4 @@
+/// <reference types="node" />
 import { DiskStats } from "../fs";
 import { BasicProgress } from "../progress";
 import { AbstractFs, FsOptions } from "../virtual-fs";
@@ -12,12 +13,12 @@ export declare class RemoteFs extends AbstractFs {
     });
     isLocal(): boolean;
     protected fetchJson(name: string, params: any[]): Promise<any>;
-    protected post(name: string, params: any[], data: string): Promise<void>;
-    existsDir(path: string): Promise<any>;
-    rename(source: string, target: string): Promise<any>;
-    mkdir(path: string): Promise<any>;
-    readFile(path: string): Promise<any>;
-    readdir(path: string): Promise<any>;
+    protected post(name: string, params: any[], data: string): Promise<Response>;
+    existsDir(path: string): Promise<boolean>;
+    rename(source: string, target: string): Promise<void>;
+    mkdir(path: string): Promise<void>;
+    readFile(path: string): Promise<string>;
+    readdir(path: string): Promise<string[]>;
     readFileIfExists(path: string): Promise<string | undefined>;
     ensureEmptyDir(path: string): Promise<void>;
     writeFile(path: string, contents: string): Promise<void>;

package/lib/utils/datatruck/client.js

@@ -27,19 +27,19 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
         return false;
     }
     async fetchJson(name, params) {
-        return await (0, http_1.fetchJson)(`${this.url}/${name}`, {
+        const url = (0, http_1.createHref)(`${this.url}/${name}`, {
+            params: JSON.stringify(params),
+        });
+        return await (0, http_1.fetchJson)(url, {
             headers: this.headers,
-            query: {
-                params: JSON.stringify(params),
-            },
         });
     }
     async post(name, params, data) {
-        return await (0, http_1.post)(`${this.url}/${name}`, data, {
+        const url = (0, http_1.createHref)(`${this.url}/${name}`, {
+            params: JSON.stringify(params),
+        });
+        return await (0, http_1.post)(url, data, {
             headers: this.headers,
-            query: {
-                params: JSON.stringify(params),
-            },
         });
     }
     async existsDir(path) {
@@ -67,7 +67,7 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
         await this.post("writeFile", [path], contents);
     }
     async rmAll(path) {
-        await this.fetchJson("rmAll", [path]);
+        return await this.fetchJson("rmAll", [path]);
     }
     async fetchDiskStats(path) {
         if (this.options.verbose)
@@ -77,20 +77,23 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
     async upload(source, target) {
         if (this.options.verbose)
             (0, cli_1.logExec)("fs.upload", [source, target]);
-        await (0, http_1.uploadFile)(`${this.url}/upload`, source, {
+        const url = (0, http_1.createHref)(`${this.url}/upload`, {
+            params: JSON.stringify([target]),
+        });
+        return await (0, http_1.uploadFile)(url, source, {
             headers: this.headers,
-            query: {
-                params: JSON.stringify([target]),
-            },
+            checksum: true,
         });
     }
     async download(source, target, options = {}) {
         if (this.options.verbose)
             (0, cli_1.logExec)("fs.download", [source, target]);
-        return await (0, http_1.downloadFile)(`${this.url}/download`, target, {
+        const url = (0, http_1.createHref)(`${this.url}/download`, {
+            params: JSON.stringify([source]),
+        });
+        return await (0, http_1.downloadFile)(url, target, {
             ...options,
             headers: this.headers,
-            query: { params: JSON.stringify([source]) },
         });
     }
 }

package/lib/utils/datatruck/repository-server.js

@@ -4,9 +4,8 @@ exports.createDatatruckRepositoryServer = exports.headerKey = void 0;
 const ConfigAction_1 = require("../../actions/ConfigAction");
 const cli_1 = require("../cli");
 const http_1 = require("../http");
+const math_1 = require("../math");
 const virtual_fs_1 = require("../virtual-fs");
-const fs_1 = require("fs");
-const promises_1 = require("fs/promises");
 const http_2 = require("http");
 exports.headerKey = {
     user: "x-dtt-user",
@@ -58,60 +57,43 @@ const getRemoteAddress = (req, options) => {
         : undefined) ?? req.socket.remoteAddress);
 };
 function createDatatruckRepositoryServer(inOptions, config = {}) {
+    const counter = new math_1.Counter();
     return (0, http_2.createServer)(async (req, res) => {
+        const url = req.url || "";
+        if (url === "/" || url === "/favicon.ico")
+            return res.end();
+        const id = counter.next();
+        let requestError;
+        let responseError;
+        req.on("error", (error) => (requestError = error));
+        res.on("error", (error) => (responseError = error));
         try {
-            if (req.url === "/" || req.url === "/favicon.ico")
-                return res.end();
-            const { repository, action, params } = parseUrl(req.url);
-            if (!repository || !action) {
-                res.statusCode = 404;
-                return res.end();
-            }
+            const { repository, action, params } = parseUrl(url);
+            if (!repository || !action)
+                return res.writeHead(404);
             const fileOptions = config.configPath
                 ? (await ConfigAction_1.ConfigAction.findAndParseFile(config.configPath)).server
                     ?.repository
                 : undefined;
             const options = fileOptions ?? inOptions;
             const backend = findRepositoryBackend(req, repository, options);
-            if (!backend) {
-                res.statusCode = 401;
-                return res.end();
-            }
+            if (!backend)
+                return res.writeHead(401);
             if (config.log)
-                (0, cli_1.logJson)("repository-server", "request", {
-                    repository,
-                    url: req.url,
-                });
-            const fs = new virtual_fs_1.LocalFs({
-                backend: backend.path,
-            });
+                (0, cli_1.logJson)("repository-server", "request", { id, repository, url });
+            const fs = new virtual_fs_1.LocalFs({ backend: backend.path });
             if (action === "comcheck") {
                 res.write(JSON.stringify({ success: true }));
             }
             else if (action === "upload") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
-                const file = (0, fs_1.createWriteStream)(path);
-                req.pipe(file);
-                await new Promise((resolve, reject) => {
-                    req.on("error", reject);
-                    file.on("error", reject);
-                    file.on("close", resolve);
-                });
+                await (0, http_1.recvFile)(req, res, path);
             }
             else if (action === "download") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
-                const file = (0, fs_1.createReadStream)(path);
-                const fileStat = await (0, promises_1.stat)(path);
-                res.setHeader("Content-Length", fileStat.size);
-                file.pipe(res);
-                await new Promise((resolve, reject) => {
-                    req.on("error", reject);
-                    file.on("error", reject);
-                    res.on("error", reject);
-                    res.on("close", resolve);
-                });
+                await (0, http_1.sendFile)(req, res, path);
             }
             else if (action === "writeFile") {
                 const data = await (0, http_1.readRequestData)(req);
@@ -127,21 +109,27 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
                     res.write(JSON.stringify(json));
             }
             if (config.log)
-                (0, cli_1.logJson)("repository-server", "request finished", {
-                    url: req.url,
-                });
-            res.end();
+                (0, cli_1.logJson)("repository-server", "request finished", { id });
         }
         catch (error) {
             if (config.log) {
-                (0, cli_1.logJson)("repository-server", "request failed", {
-                    url: req.url,
-                });
+                (0, cli_1.logJson)("repository-server", "request failed", { id });
                 console.error(error);
             }
-            res.statusCode = 500;
-            res.statusMessage = error.message;
-            res.end();
+            if (!res.writableEnded && !res.headersSent)
+                res.writeHead(500, error.message);
+        }
+        finally {
+            if (requestError) {
+                (0, cli_1.logJson)("repository-server", "request error", { id });
+                console.error(requestError);
+            }
+            if (responseError) {
+                (0, cli_1.logJson)("repository-server", "response error", { id });
+                console.error(responseError);
+            }
+            if (!res.writableEnded)
+                res.end();
         }
     });
 }

package/lib/utils/http.d.ts

@@ -1,31 +1,27 @@
 /// <reference types="node" />
+/// <reference types="node" />
 import { BasicProgress } from "./progress";
-import { IncomingMessage, Server } from "http";
+import { IncomingMessage, Server, ServerResponse } from "http";
+export declare function createHref(inUrl: string, query?: Record<string, string>): string;
 export declare function closeServer(server: Server): Promise<void>;
 export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
-export type FetchOptions = {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
-    statusError?: boolean;
-};
-export declare function fetch(url: string, options?: FetchOptions): Promise<{
-    data: string | undefined;
-    status: number | undefined;
-}>;
-export declare function fetchJson<T = any>(url: string, options?: FetchOptions): Promise<T | undefined>;
-export declare function post(url: string, data: string, options?: {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
+export declare const safeFetch: typeof fetch;
+export declare function fetchJson<T = any>(url: string, options?: RequestInit): Promise<T | undefined>;
+export declare function post(url: string, data: string, options?: Omit<RequestInit, "method" | "body">): Promise<Response>;
+export declare function parseContentLength(value: string | undefined): number;
+export declare function sendFile(req: IncomingMessage, res: ServerResponse, path: string, options?: {
+    end?: boolean;
+    checksum?: boolean;
+}): Promise<void>;
+export declare function recvFile(req: IncomingMessage, res: ServerResponse, path: string, options?: {
+    end?: boolean;
 }): Promise<void>;
-export declare function downloadFile(url: string, output: string, options?: {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
+export declare function downloadFile(url: string, output: string, options?: Omit<RequestInit, "signal"> & {
     timeout?: number;
     onProgress?: (progress: BasicProgress) => void;
 }): Promise<{
     bytes: number;
 }>;
-export declare function uploadFile(url: string, path: string, options?: {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
+export declare function uploadFile(url: string, path: string, options?: Omit<RequestInit, "method" | "body"> & {
+    checksum?: boolean;
 }): Promise<void>;

package/lib/utils/http.js

@@ -1,22 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.fetch = exports.readRequestData = exports.closeServer = void 0;
+exports.uploadFile = exports.downloadFile = exports.recvFile = exports.sendFile = exports.parseContentLength = exports.post = exports.fetchJson = exports.safeFetch = exports.readRequestData = exports.closeServer = exports.createHref = void 0;
+const crypto_1 = require("./crypto");
 const math_1 = require("./math");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
-const http_1 = require("http");
-const https_1 = require("https");
-const request = (url, options, callback) => {
-    return url.startsWith("https://")
-        ? (0, https_1.request)(url, options, callback)
-        : (0, http_1.request)(url, options, callback);
-};
-function href(inUrl, query) {
+const stream_1 = require("stream");
+const promises_2 = require("stream/promises");
+function createHref(inUrl, query) {
     const url = new URL(inUrl);
     for (const key in query || {})
         url.searchParams.set(key, query[key]);
     return url.href;
 }
+exports.createHref = createHref;
 async function closeServer(server) {
     await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
 }
@@ -37,140 +34,150 @@ function readRequestData(req) {
     });
 }
 exports.readRequestData = readRequestData;
-async function fetch(url, options = {}) {
-    const throwStatusCodeError = options.statusError ?? true;
-    return new Promise((resolve, reject) => {
-        let data;
-        request(href(url, options.query), {
-            method: "GET",
-            headers: options.headers,
-        }, (res) => {
-            if (throwStatusCodeError && res.statusCode !== 200)
-                return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
-            res
-                .on("data", (chunk) => {
-                if (data === undefined)
-                    data = "";
-                data += chunk;
-            })
-                .on("error", reject)
-                .on("close", () => {
-                resolve({ data, status: res.statusCode });
-            });
-        })
-            .on("error", reject)
-            .end();
-    });
-}
-exports.fetch = fetch;
+const safeFetch = async (...args) => {
+    const res = await fetch(...args);
+    if (res.status !== 200)
+        throw new Error(`Fetch request failed: ${res.status} ${res.statusText}`);
+    return res;
+};
+exports.safeFetch = safeFetch;
 async function fetchJson(url, options = {}) {
-    const res = await fetch(url, options);
-    if (res.data === undefined)
-        return;
-    return JSON.parse(res.data);
+    const res = await (0, exports.safeFetch)(url, options);
+    const data = await res.text();
+    return data.length ? JSON.parse(data) : undefined;
 }
 exports.fetchJson = fetchJson;
 async function post(url, data, options = {}) {
-    await new Promise((resolve, reject) => {
-        const req = request(href(url, options.query), { method: "POST", headers: options.headers }, (res) => {
+    return await (0, exports.safeFetch)(url, { ...options, method: "POST", body: data });
+}
+exports.post = post;
+function parseContentLength(value) {
+    if (!value || !/^\d+$/.test(value))
+        throw new Error(`Invalid 'content-length': ${value}`);
+    return Number(value);
+}
+exports.parseContentLength = parseContentLength;
+async function sendFile(req, res, path, options = {}) {
+    let file;
+    try {
+        file = (0, fs_1.createReadStream)(path);
+        const fileStat = await (0, promises_1.stat)(path);
+        res.setHeader("Content-Length", fileStat.size);
+        if (options.checksum)
+            res.setHeader("x-checksum", await (0, crypto_1.calcFileHash)(path, "sha1"));
+        file.pipe(res);
+        await new Promise((resolve, reject) => {
+            file.on("error", reject);
+            req.on("error", reject);
+            res.on("error", reject).on("close", resolve);
+        });
+    }
+    finally {
+        file?.close();
+        if (options.end)
+            res.end();
+    }
+}
+exports.sendFile = sendFile;
+async function recvFile(req, res, path, options = {}) {
+    let file;
+    try {
+        file = (0, fs_1.createWriteStream)(path);
+        req.pipe(file);
+        await new Promise((resolve, reject) => {
+            file.on("error", reject).on("close", resolve);
+            req.on("error", reject);
             res.on("error", reject);
-            if (res.statusCode !== 200) {
-                reject(new Error(`Post failed: ${res.statusCode} ${res.statusMessage}`));
-            }
-            else {
-                resolve();
-            }
         });
-        req.on("error", reject);
-        req.write(data);
-        req.end();
-    });
+        const checksum = res.getHeader("x-checksum");
+        if (typeof checksum === "string")
+            await (0, crypto_1.assertFileChecksum)(path, checksum, "sha1");
+    }
+    finally {
+        file?.close();
+        if (options.end)
+            res.end();
+    }
 }
-exports.post = post;
+exports.recvFile = recvFile;
 async function downloadFile(url, output, options = {}) {
-    const timeout = options.timeout ?? 3600 * 1000; // 60m
+    const { timeout, onProgress, ...fetchOptions } = options;
     const file = (0, fs_1.createWriteStream)(output);
-    let total = 0;
-    await new Promise((resolve, reject) => {
-        const req = request(href(url, options.query), {
-            headers: options.headers,
-        }, (res) => {
-            const contentLength = res.headers["content-length"] ?? "";
-            if (!/^\d+$/.test(contentLength))
-                return reject(new Error(`Invalid 'content-length': ${contentLength}`));
-            total = Number(contentLength);
-            let current = 0;
-            if (res.statusCode === 200) {
-                if (options.onProgress) {
-                    res.on("data", (chunk) => {
-                        current += chunk.byteLength;
-                        options.onProgress({
-                            percent: (0, math_1.progressPercent)(total, current),
-                            current,
-                            total,
-                        });
-                    });
-                }
-                res
-                    .on("error", async (error) => {
+    let checksum;
+    const length = { total: 0, current: 0 };
+    let requestError;
+    try {
+        const res = await (0, exports.safeFetch)(url, {
+            ...fetchOptions,
+            signal: AbortSignal.timeout(timeout ?? 3600 * 1000), // 60m
+        });
+        length.total = parseContentLength(res.headers.get("content-length") ?? undefined);
+        checksum = res.headers.get("x-checksum") ?? undefined;
+        const body = stream_1.Readable.fromWeb(res.body);
+        const progress = onProgress &&
+            new stream_1.Transform({
+                transform(chunk, encoding, callback) {
+                    let error;
                     try {
-                        file.destroy();
+                        length.current += chunk.byteLength;
+                        onProgress({
+                            percent: (0, math_1.progressPercent)(length.total, length.current),
+                            current: length.current,
+                            total: length.total,
+                        });
                     }
-                    catch (_) { }
-                    try {
-                        await (0, promises_1.unlink)(output);
+                    catch (progressError) {
+                        error = progressError;
                     }
-                    catch (_) { }
-                    reject(error);
-                })
-                    .pipe(file);
-                file.on("finish", () => {
-                    file.close((error) => {
-                        error ? reject(error) : resolve();
-                    });
-                });
-            }
-            else {
-                reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
-            }
-        }).on("error", async (error) => {
-            try {
-                await (0, promises_1.unlink)(output);
-            }
-            catch (_) { }
-            reject(error);
-        });
-        req.setTimeout(timeout, () => {
-            req.destroy();
-            reject(new Error(`Request timeout after ${timeout / 1000}s`));
-        });
-        req.end();
-    });
-    const { size: bytes } = await (0, promises_1.stat)(output);
-    if (total !== bytes)
-        throw new Error(`Invalid download size: ${total} != ${bytes}`);
-    return { bytes };
+                    callback(error, chunk);
+                },
+            });
+        if (progress) {
+            await (0, promises_2.pipeline)(body, progress, file);
+        }
+        else {
+            await (0, promises_2.pipeline)(body, file);
+        }
+        const { size: fileLength } = await (0, promises_1.stat)(output);
+        if (length.total !== fileLength)
+            throw new Error(`Invalid download size: ${length.total} != ${fileLength}`);
+    }
+    catch (error) {
+        try {
+            await (0, promises_1.unlink)(output);
+        }
+        catch (_) { }
+        throw error;
+    }
+    if (checksum)
+        await (0, crypto_1.assertFileChecksum)(output, checksum, "sha1");
+    if (requestError)
+        throw requestError;
+    return { bytes: length.total };
 }
 exports.downloadFile = downloadFile;
 async function uploadFile(url, path, options = {}) {
     const { size } = await (0, promises_1.stat)(path);
-    const readStream = (0, fs_1.createReadStream)(path);
-    await new Promise((resolve, reject) => {
-        const req = request(href(url, options.query), {
+    const file = (0, fs_1.createReadStream)(path);
+    try {
+        const res = await fetch(url, {
+            ...options,
             method: "POST",
+            duplex: "half",
             headers: {
                 ...options.headers,
-                "Content-length": size,
+                "Content-Length": size.toString(),
+                ...(options.checksum && {
+                    "x-checksum": await (0, crypto_1.calcFileHash)(path, "sha1"),
+                }),
             },
-        }, (res) => {
-            if (res.statusCode !== 200) {
-                reject(new Error(`Upload failed: ${res.statusCode} ${res.statusMessage}`));
-            }
-            else {
-                resolve();
-            }
-        }).on("error", reject);
-        readStream.on("error", reject).pipe(req);
-    });
+            body: file,
+        });
+        if (res.status !== 200)
+            new Error(`Upload failed: ${res.status} ${res.statusText}`);
+    }
+    finally {
+        file.close();
+    }
 }
 exports.uploadFile = uploadFile;

package/lib/utils/math.d.ts

@@ -1 +1,7 @@
 export declare function progressPercent(total: number, current: number): number;
+export declare class Counter {
+    protected maxValue: number;
+    protected value: number;
+    constructor(maxValue?: number);
+    next(): number;
+}

package/lib/utils/math.js

@@ -1,9 +1,22 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.progressPercent = void 0;
+exports.Counter = exports.progressPercent = void 0;
 function progressPercent(total, current) {
     if (total === 0 && current === 0)
         return 0;
     return Number(((current / total) * 100).toFixed(2));
 }
 exports.progressPercent = progressPercent;
+class Counter {
+    maxValue;
+    value = 0;
+    constructor(maxValue = 4294967295) {
+        this.maxValue = maxValue;
+    }
+    next() {
+        if (this.maxValue && this.value >= this.maxValue)
+            this.value = 0;
+        return ++this.value;
+    }
+}
+exports.Counter = Counter;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.36.4",
+  "version": "0.36.6",
   "description": "Tool for creating and managing backups",
   "homepage": "https://github.com/swordev/datatruck#readme",
   "bugs": {