@datatruck/cli 0.36.5 → 0.36.6

package/lib/utils/crypto.d.ts

@@ -1 +1,2 @@
 export declare function calcFileHash(path: string, algorithm: string): Promise<string>;
+export declare function assertFileChecksum(path: string, checksum: string, algorithm: string): Promise<void>;

package/lib/utils/crypto.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.calcFileHash = void 0;
+exports.assertFileChecksum = exports.calcFileHash = void 0;
 const crypto_1 = require("crypto");
 const fs_1 = require("fs");
 function calcFileHash(path, algorithm) {
@@ -13,3 +13,9 @@ function calcFileHash(path, algorithm) {
     });
 }
 exports.calcFileHash = calcFileHash;
+async function assertFileChecksum(path, checksum, algorithm) {
+    const fileChecksum = await calcFileHash(path, algorithm);
+    if (fileChecksum !== checksum)
+        throw new Error(`Invalid checksum file: ${checksum} != ${fileChecksum}`);
+}
+exports.assertFileChecksum = assertFileChecksum;

package/lib/utils/datatruck/client.d.ts

@@ -1,3 +1,4 @@
+/// <reference types="node" />
 import { DiskStats } from "../fs";
 import { BasicProgress } from "../progress";
 import { AbstractFs, FsOptions } from "../virtual-fs";
@@ -12,12 +13,12 @@ export declare class RemoteFs extends AbstractFs {
     });
     isLocal(): boolean;
     protected fetchJson(name: string, params: any[]): Promise<any>;
-    protected post(name: string, params: any[], data: string): Promise<void>;
-    existsDir(path: string): Promise<any>;
-    rename(source: string, target: string): Promise<any>;
-    mkdir(path: string): Promise<any>;
-    readFile(path: string): Promise<any>;
-    readdir(path: string): Promise<any>;
+    protected post(name: string, params: any[], data: string): Promise<Response>;
+    existsDir(path: string): Promise<boolean>;
+    rename(source: string, target: string): Promise<void>;
+    mkdir(path: string): Promise<void>;
+    readFile(path: string): Promise<string>;
+    readdir(path: string): Promise<string[]>;
     readFileIfExists(path: string): Promise<string | undefined>;
     ensureEmptyDir(path: string): Promise<void>;
     writeFile(path: string, contents: string): Promise<void>;

package/lib/utils/datatruck/client.js

@@ -27,19 +27,19 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
         return false;
     }
     async fetchJson(name, params) {
-        return await (0, http_1.fetchJson)(`${this.url}/${name}`, {
+        const url = (0, http_1.createHref)(`${this.url}/${name}`, {
+            params: JSON.stringify(params),
+        });
+        return await (0, http_1.fetchJson)(url, {
             headers: this.headers,
-            query: {
-                params: JSON.stringify(params),
-            },
         });
     }
     async post(name, params, data) {
-        return await (0, http_1.post)(`${this.url}/${name}`, data, {
+        const url = (0, http_1.createHref)(`${this.url}/${name}`, {
+            params: JSON.stringify(params),
+        });
+        return await (0, http_1.post)(url, data, {
             headers: this.headers,
-            query: {
-                params: JSON.stringify(params),
-            },
         });
     }
     async existsDir(path) {
@@ -67,7 +67,7 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
         await this.post("writeFile", [path], contents);
     }
     async rmAll(path) {
-        await this.fetchJson("rmAll", [path]);
+        return await this.fetchJson("rmAll", [path]);
     }
     async fetchDiskStats(path) {
         if (this.options.verbose)
@@ -77,20 +77,23 @@ class RemoteFs extends virtual_fs_1.AbstractFs {
     async upload(source, target) {
         if (this.options.verbose)
             (0, cli_1.logExec)("fs.upload", [source, target]);
-        await (0, http_1.uploadFile)(`${this.url}/upload`, source, {
+        const url = (0, http_1.createHref)(`${this.url}/upload`, {
+            params: JSON.stringify([target]),
+        });
+        return await (0, http_1.uploadFile)(url, source, {
             headers: this.headers,
-            query: {
-                params: JSON.stringify([target]),
-            },
+            checksum: true,
         });
     }
     async download(source, target, options = {}) {
         if (this.options.verbose)
             (0, cli_1.logExec)("fs.download", [source, target]);
-        return await (0, http_1.downloadFile)(`${this.url}/download`, target, {
+        const url = (0, http_1.createHref)(`${this.url}/download`, {
+            params: JSON.stringify([source]),
+        });
+        return await (0, http_1.downloadFile)(url, target, {
             ...options,
             headers: this.headers,
-            query: { params: JSON.stringify([source]) },
         });
     }
 }

package/lib/utils/datatruck/repository-server.js

@@ -6,10 +6,7 @@ const cli_1 = require("../cli");
 const http_1 = require("../http");
 const math_1 = require("../math");
 const virtual_fs_1 = require("../virtual-fs");
-const fs_1 = require("fs");
-const promises_1 = require("fs/promises");
 const http_2 = require("http");
-const promises_2 = require("stream/promises");
 exports.headerKey = {
     user: "x-dtt-user",
     password: "x-dtt-password",
@@ -91,16 +88,12 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
             else if (action === "upload") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
-                const file = (0, fs_1.createWriteStream)(path);
-                await (0, promises_2.pipeline)(req, file);
+                await (0, http_1.recvFile)(req, res, path);
             }
             else if (action === "download") {
                 const [target] = params;
                 const path = fs.resolvePath(target);
-                const file = (0, fs_1.createReadStream)(path);
-                const fileStat = await (0, promises_1.stat)(path);
-                res.setHeader("Content-Length", fileStat.size);
-                await (0, promises_2.pipeline)(file, res, { end: false });
+                await (0, http_1.sendFile)(req, res, path);
             }
             else if (action === "writeFile") {
                 const data = await (0, http_1.readRequestData)(req);
@@ -123,7 +116,7 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
                 (0, cli_1.logJson)("repository-server", "request failed", { id });
                 console.error(error);
             }
-            if (!res.headersSent)
+            if (!res.writableEnded && !res.headersSent)
                 res.writeHead(500, error.message);
         }
         finally {
@@ -135,12 +128,8 @@ function createDatatruckRepositoryServer(inOptions, config = {}) {
                 (0, cli_1.logJson)("repository-server", "response error", { id });
                 console.error(responseError);
             }
-            if (requestError || responseError) {
-                res.destroy();
-            }
-            else {
+            if (!res.writableEnded)
                 res.end();
-            }
         }
     });
 }

package/lib/utils/http.d.ts

@@ -1,31 +1,27 @@
 /// <reference types="node" />
+/// <reference types="node" />
 import { BasicProgress } from "./progress";
-import { IncomingMessage, Server } from "http";
+import { IncomingMessage, Server, ServerResponse } from "http";
+export declare function createHref(inUrl: string, query?: Record<string, string>): string;
 export declare function closeServer(server: Server): Promise<void>;
 export declare function readRequestData(req: IncomingMessage): Promise<string | undefined>;
-export type FetchOptions = {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
-    statusError?: boolean;
-};
-export declare function fetch(url: string, options?: FetchOptions): Promise<{
-    data: string | undefined;
-    status: number | undefined;
-}>;
-export declare function fetchJson<T = any>(url: string, options?: FetchOptions): Promise<T | undefined>;
-export declare function post(url: string, data: string, options?: {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
+export declare const safeFetch: typeof fetch;
+export declare function fetchJson<T = any>(url: string, options?: RequestInit): Promise<T | undefined>;
+export declare function post(url: string, data: string, options?: Omit<RequestInit, "method" | "body">): Promise<Response>;
+export declare function parseContentLength(value: string | undefined): number;
+export declare function sendFile(req: IncomingMessage, res: ServerResponse, path: string, options?: {
+    end?: boolean;
+    checksum?: boolean;
+}): Promise<void>;
+export declare function recvFile(req: IncomingMessage, res: ServerResponse, path: string, options?: {
+    end?: boolean;
 }): Promise<void>;
-export declare function downloadFile(url: string, output: string, options?: {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
+export declare function downloadFile(url: string, output: string, options?: Omit<RequestInit, "signal"> & {
     timeout?: number;
     onProgress?: (progress: BasicProgress) => void;
 }): Promise<{
     bytes: number;
 }>;
-export declare function uploadFile(url: string, path: string, options?: {
-    headers?: Record<string, string>;
-    query?: Record<string, string>;
+export declare function uploadFile(url: string, path: string, options?: Omit<RequestInit, "method" | "body"> & {
+    checksum?: boolean;
 }): Promise<void>;

package/lib/utils/http.js

@@ -1,22 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadFile = exports.downloadFile = exports.post = exports.fetchJson = exports.fetch = exports.readRequestData = exports.closeServer = void 0;
+exports.uploadFile = exports.downloadFile = exports.recvFile = exports.sendFile = exports.parseContentLength = exports.post = exports.fetchJson = exports.safeFetch = exports.readRequestData = exports.closeServer = exports.createHref = void 0;
+const crypto_1 = require("./crypto");
 const math_1 = require("./math");
 const fs_1 = require("fs");
 const promises_1 = require("fs/promises");
-const http_1 = require("http");
-const https_1 = require("https");
-const request = (url, options, callback) => {
-    return url.startsWith("https://")
-        ? (0, https_1.request)(url, options, callback)
-        : (0, http_1.request)(url, options, callback);
-};
-function href(inUrl, query) {
+const stream_1 = require("stream");
+const promises_2 = require("stream/promises");
+function createHref(inUrl, query) {
     const url = new URL(inUrl);
     for (const key in query || {})
         url.searchParams.set(key, query[key]);
     return url.href;
 }
+exports.createHref = createHref;
 async function closeServer(server) {
     await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
 }
@@ -37,140 +34,150 @@ function readRequestData(req) {
     });
 }
 exports.readRequestData = readRequestData;
-async function fetch(url, options = {}) {
-    const throwStatusCodeError = options.statusError ?? true;
-    return new Promise((resolve, reject) => {
-        let data;
-        request(href(url, options.query), {
-            method: "GET",
-            headers: options.headers,
-        }, (res) => {
-            if (throwStatusCodeError && res.statusCode !== 200)
-                return reject(new Error(`GET failed: ${res.statusCode} ${res.statusMessage}`));
-            res
-                .on("data", (chunk) => {
-                if (data === undefined)
-                    data = "";
-                data += chunk;
-            })
-                .on("error", reject)
-                .on("close", () => {
-                resolve({ data, status: res.statusCode });
-            });
-        })
-            .on("error", reject)
-            .end();
-    });
-}
-exports.fetch = fetch;
+const safeFetch = async (...args) => {
+    const res = await fetch(...args);
+    if (res.status !== 200)
+        throw new Error(`Fetch request failed: ${res.status} ${res.statusText}`);
+    return res;
+};
+exports.safeFetch = safeFetch;
 async function fetchJson(url, options = {}) {
-    const res = await fetch(url, options);
-    if (res.data === undefined)
-        return;
-    return JSON.parse(res.data);
+    const res = await (0, exports.safeFetch)(url, options);
+    const data = await res.text();
+    return data.length ? JSON.parse(data) : undefined;
 }
 exports.fetchJson = fetchJson;
 async function post(url, data, options = {}) {
-    await new Promise((resolve, reject) => {
-        const req = request(href(url, options.query), { method: "POST", headers: options.headers }, (res) => {
+    return await (0, exports.safeFetch)(url, { ...options, method: "POST", body: data });
+}
+exports.post = post;
+function parseContentLength(value) {
+    if (!value || !/^\d+$/.test(value))
+        throw new Error(`Invalid 'content-length': ${value}`);
+    return Number(value);
+}
+exports.parseContentLength = parseContentLength;
+async function sendFile(req, res, path, options = {}) {
+    let file;
+    try {
+        file = (0, fs_1.createReadStream)(path);
+        const fileStat = await (0, promises_1.stat)(path);
+        res.setHeader("Content-Length", fileStat.size);
+        if (options.checksum)
+            res.setHeader("x-checksum", await (0, crypto_1.calcFileHash)(path, "sha1"));
+        file.pipe(res);
+        await new Promise((resolve, reject) => {
+            file.on("error", reject);
+            req.on("error", reject);
+            res.on("error", reject).on("close", resolve);
+        });
+    }
+    finally {
+        file?.close();
+        if (options.end)
+            res.end();
+    }
+}
+exports.sendFile = sendFile;
+async function recvFile(req, res, path, options = {}) {
+    let file;
+    try {
+        file = (0, fs_1.createWriteStream)(path);
+        req.pipe(file);
+        await new Promise((resolve, reject) => {
+            file.on("error", reject).on("close", resolve);
+            req.on("error", reject);
             res.on("error", reject);
-            if (res.statusCode !== 200) {
-                reject(new Error(`Post failed: ${res.statusCode} ${res.statusMessage}`));
-            }
-            else {
-                resolve();
-            }
         });
-        req.on("error", reject);
-        req.write(data);
-        req.end();
-    });
+        const checksum = res.getHeader("x-checksum");
+        if (typeof checksum === "string")
+            await (0, crypto_1.assertFileChecksum)(path, checksum, "sha1");
+    }
+    finally {
+        file?.close();
+        if (options.end)
+            res.end();
+    }
 }
-exports.post = post;
+exports.recvFile = recvFile;
 async function downloadFile(url, output, options = {}) {
-    const timeout = options.timeout ?? 3600 * 1000; // 60m
+    const { timeout, onProgress, ...fetchOptions } = options;
     const file = (0, fs_1.createWriteStream)(output);
-    let total = 0;
-    await new Promise((resolve, reject) => {
-        const req = request(href(url, options.query), {
-            headers: options.headers,
-        }, (res) => {
-            const contentLength = res.headers["content-length"] ?? "";
-            if (!/^\d+$/.test(contentLength))
-                return reject(new Error(`Invalid 'content-length': ${contentLength}`));
-            total = Number(contentLength);
-            let current = 0;
-            if (res.statusCode === 200) {
-                if (options.onProgress) {
-                    res.on("data", (chunk) => {
-                        current += chunk.byteLength;
-                        options.onProgress({
-                            percent: (0, math_1.progressPercent)(total, current),
-                            current,
-                            total,
-                        });
-                    });
-                }
-                res
-                    .on("error", async (error) => {
+    let checksum;
+    const length = { total: 0, current: 0 };
+    let requestError;
+    try {
+        const res = await (0, exports.safeFetch)(url, {
+            ...fetchOptions,
+            signal: AbortSignal.timeout(timeout ?? 3600 * 1000), // 60m
+        });
+        length.total = parseContentLength(res.headers.get("content-length") ?? undefined);
+        checksum = res.headers.get("x-checksum") ?? undefined;
+        const body = stream_1.Readable.fromWeb(res.body);
+        const progress = onProgress &&
+            new stream_1.Transform({
+                transform(chunk, encoding, callback) {
+                    let error;
                     try {
-                        file.destroy();
+                        length.current += chunk.byteLength;
+                        onProgress({
+                            percent: (0, math_1.progressPercent)(length.total, length.current),
+                            current: length.current,
+                            total: length.total,
+                        });
                     }
-                    catch (_) { }
-                    try {
-                        await (0, promises_1.unlink)(output);
+                    catch (progressError) {
+                        error = progressError;
                     }
-                    catch (_) { }
-                    reject(error);
-                })
-                    .pipe(file);
-                file.on("finish", () => {
-                    file.close((error) => {
-                        error ? reject(error) : resolve();
-                    });
-                });
-            }
-            else {
-                reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
-            }
-        }).on("error", async (error) => {
-            try {
-                await (0, promises_1.unlink)(output);
-            }
-            catch (_) { }
-            reject(error);
-        });
-        req.setTimeout(timeout, () => {
-            req.destroy();
-            reject(new Error(`Request timeout after ${timeout / 1000}s`));
-        });
-        req.end();
-    });
-    const { size: bytes } = await (0, promises_1.stat)(output);
-    if (total !== bytes)
-        throw new Error(`Invalid download size: ${total} != ${bytes}`);
-    return { bytes };
+                    callback(error, chunk);
+                },
+            });
+        if (progress) {
+            await (0, promises_2.pipeline)(body, progress, file);
+        }
+        else {
+            await (0, promises_2.pipeline)(body, file);
+        }
+        const { size: fileLength } = await (0, promises_1.stat)(output);
+        if (length.total !== fileLength)
+            throw new Error(`Invalid download size: ${length.total} != ${fileLength}`);
+    }
+    catch (error) {
+        try {
+            await (0, promises_1.unlink)(output);
+        }
+        catch (_) { }
+        throw error;
+    }
+    if (checksum)
+        await (0, crypto_1.assertFileChecksum)(output, checksum, "sha1");
+    if (requestError)
+        throw requestError;
+    return { bytes: length.total };
 }
 exports.downloadFile = downloadFile;
 async function uploadFile(url, path, options = {}) {
     const { size } = await (0, promises_1.stat)(path);
-    const readStream = (0, fs_1.createReadStream)(path);
-    await new Promise((resolve, reject) => {
-        const req = request(href(url, options.query), {
+    const file = (0, fs_1.createReadStream)(path);
+    try {
+        const res = await fetch(url, {
+            ...options,
             method: "POST",
+            duplex: "half",
             headers: {
                 ...options.headers,
-                "Content-length": size,
+                "Content-Length": size.toString(),
+                ...(options.checksum && {
+                    "x-checksum": await (0, crypto_1.calcFileHash)(path, "sha1"),
+                }),
             },
-        }, (res) => {
-            if (res.statusCode !== 200) {
-                reject(new Error(`Upload failed: ${res.statusCode} ${res.statusMessage}`));
-            }
-            else {
-                resolve();
-            }
-        }).on("error", reject);
-        readStream.on("error", reject).pipe(req);
-    });
+            body: file,
+        });
+        if (res.status !== 200)
+            new Error(`Upload failed: ${res.status} ${res.statusText}`);
+    }
+    finally {
+        file.close();
+    }
 }
 exports.uploadFile = uploadFile;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@datatruck/cli",
-  "version": "0.36.5",
+  "version": "0.36.6",
   "description": "Tool for creating and managing backups",
   "homepage": "https://github.com/swordev/datatruck#readme",
   "bugs": {