@dynamicweb/cli 1.0.13 → 1.0.15

package/bin/commands/files.js

@@ -1,11 +1,12 @@
 import fetch from 'node-fetch';
 import path from 'path';
 import fs from 'fs';
-import extract from 'extract-zip';
 import FormData from 'form-data';
 import { setupEnv, getAgent } from './env.js';
 import { setupUser } from './login.js';
-import { interactiveConfirm } from '../utils.js';
+import { interactiveConfirm, formatBytes, createThrottledStatusUpdater } from '../utils.js';
+import { downloadWithProgress, tryGetFileNameFromResponse } from '../downloader.js';
+import { extractWithProgress } from '../extractor.js';
 
 export function filesCommand() {
     return {
@@ -38,7 +39,7 @@ export function filesCommand() {
             .option('overwrite', {
                 alias: 'o',
                 type: 'boolean',
-                describe: 'Used with import, will overwrite existing files at destrination if set to true'
+                describe: 'Used with import, will overwrite existing files at destination if set to true'
             })
             .option('createEmpty', {
                 type: 'boolean',
@@ -187,8 +188,7 @@ async function download(env, user, dirPath, outPath, recursive, outname, raw, ia
 
     console.log('Downloading', dirPath === '/.' ? 'Base' : dirPath, 'Recursive=' + recursive);
 
-    let filename;
-    fetch(`${env.protocol}://${env.host}/Admin/Api/${endpoint}`, {
+    const res = await fetch(`${env.protocol}://${env.host}/Admin/Api/${endpoint}`, {
         method: 'POST',
         body: JSON.stringify(data),
         headers: {
@@ -196,33 +196,40 @@ async function download(env, user, dirPath, outPath, recursive, outname, raw, ia
             'Content-Type': 'application/json'
         },
         agent: getAgent(env.protocol)
-    }).then((res) => {
-        const header = res.headers.get('content-disposition');
-        const parts = header?.split(';');
-        if (!parts) {
-            console.log(`No files found in directory '${dirPath}', if you want to download all folders recursively include the -r flag`);
-            return;
-        }
-        filename = parts[1].split('=')[1].replace('+', ' ');
-        if (outname) filename = outname;
-        return res;
-    }).then(async (res) => {
-        if (!filename) return;
-        let filePath = path.resolve(`${path.resolve(outPath)}/${filename}`)
-        const fileStream = fs.createWriteStream(filePath);
-        await new Promise((resolve, reject) => {
-            res.body.pipe(fileStream);
-            res.body.on("error", reject);
-            fileStream.on("finish", resolve);
-        });
-        console.log(`Finished downloading`, dirPath === '/.' ? '.' : dirPath, 'Recursive=' + recursive);
-        if (!raw) {
-            let filenameWithoutExtension = filename.replace('.zip', '')
-            await extract(filePath, { dir: `${path.resolve(outPath)}/${filenameWithoutExtension === 'Base' ? '' : filenameWithoutExtension}` }, function (err) {})
-            fs.unlink(filePath, function(err) {})
+    });
+
+    const filename = outname || tryGetFileNameFromResponse(res, dirPath);
+    if (!filename) return;
+
+    let filePath = path.resolve(`${path.resolve(outPath)}/${filename}`)
+    let updater = createThrottledStatusUpdater();
+    await downloadWithProgress(res, filePath, {
+        onData: (received) => {
+            updater.update(`Received:\t${formatBytes(received)}`);
         }
-        return res;
     });
+    updater.stop();
+
+    console.log(`Finished downloading`, dirPath === '/.' ? '.' : dirPath, 'Recursive=' + recursive);
+
+    if (!raw) {
+        console.log(`\nExtracting ${filename} to ${outPath}`);
+
+        const filenameWithoutExtension = filename.replace('.zip', '');
+        const destinationPath = `${path.resolve(outPath)}/${filenameWithoutExtension === 'Base' ? '' : filenameWithoutExtension}`;
+
+        updater = createThrottledStatusUpdater();
+        await extractWithProgress(filePath, destinationPath, {
+            onEntry: (processedEntries, totalEntries, percent) => {
+                updater.update(`Extracted:\t${processedEntries} of ${totalEntries} files (${percent}%)`);
+            }
+        });
+        updater.stop();
+
+        console.log(`Finished extracting ${filename} to ${outPath}\n`);
+
+        fs.unlink(filePath, function(err) {});
+    }
 }
 
 async function getFilesStructure(env, user, dirPath, recursive, includeFiles) {

package/bin/downloader.js

@@ -0,0 +1,67 @@
+import { Response } from 'node-fetch';
+import fs from 'fs';
+
+/**
+ * Extracts the file name from the HTTP response.
+ * @param {Response} res - The HTTP response object.
+ * @returns {string} The extracted file name.
+ */
+export function getFileNameFromResponse(res, dirPath) {
+    const header = res.headers.get('content-disposition');
+    const parts = header?.split(';');
+    
+    if (!parts) {
+        const msg = `No files found in directory '${dirPath}', if you want to download all folders recursively include the -r flag`;
+        throw new Error(msg);
+    }
+
+    return parts[1].split('=')[1].replace('+', ' ');
+}
+
+/**
+ * Attempts to extract the file name from an HTTP response.
+ * If extraction fails, logs the error message to the console.
+ *
+ * @param {Object} res - The HTTP response object to extract the file name from.
+ * @param {string} dirPath - The directory path to use for file name resolution.
+ * @returns {string|null} The extracted file name, or null if extraction fails.
+ */
+export function tryGetFileNameFromResponse(res, dirPath) {
+    try {
+        return getFileNameFromResponse(res, dirPath);
+    } catch (err) {
+        console.error(err.message);
+        return null;
+    }
+}
+
+/**
+ * Downloads a file with progress reporting.
+ * @param {Response} res - The response from which to read the stream data.
+ * @param {string} filePath - The path to write the file to.
+ * @param {Object} options - Options for the download.
+ * @param {(receivedBytes: number, elapsedMillis: number, isFirstChunk: boolean) => void} options.onData - Callback invoked with each data chunk.
+ * @returns {Promise<void>} Resolves when the download is complete.
+ */
+export function downloadWithProgress(res, filePath, options) {
+    const fileStream = fs.createWriteStream(filePath);
+    return new Promise((resolve, reject) => {
+        let receivedBytes = 0;
+        let startTime = Date.now();
+
+        res.body.pipe(fileStream);
+        res.body.on("error", reject);
+        fileStream.on("finish", resolve);
+
+        res.body.on("data", chunk => {
+            const isFirstChunk = receivedBytes === 0;
+            const elapsed = Date.now() - startTime;
+            
+            receivedBytes += chunk.length;
+
+            if (options?.onData) {
+                options.onData(receivedBytes, elapsed, isFirstChunk);
+            }
+        });
+    });
+}

package/bin/extractor.js

@@ -0,0 +1,28 @@
+import extract from 'extract-zip';
+
+/**
+ * Extracts files from a zip archive with progress reporting.
+ *
+ * @param {string} filePath - The path to the zip file to extract.
+ * @param {string} destinationPath - The directory where files will be extracted.
+ * @param {Object} [options] - Optional settings.
+ * @param {(processedEntries: number, totalEntries: number, percent: string) => void} [options.onEntry] - Callback invoked on each entry extracted.
+ *   Receives the number of processed files, total entry count, and percent complete as arguments.
+ * @returns {Promise<void>} A promise that resolves when extraction is complete.
+ */
+export function extractWithProgress(filePath, destinationPath, options) {
+    let processedEntries = 0;
+
+    return extract(filePath, {
+        dir: destinationPath,
+        onEntry: (_, zipFile) => {
+            processedEntries++;
+
+            const percent = Math.floor((processedEntries / zipFile.entryCount) * 100).toFixed(0);
+
+            if (options?.onEntry) {
+                options.onEntry(processedEntries, zipFile.entryCount, percent);
+            }
+        }
+    }, function (err) {});
+}

package/bin/utils.js

@@ -1,4 +1,7 @@
 import yargsInteractive from 'yargs-interactive';
+import logUpdate from 'log-update';
+
+const WRITE_THROTTLE_MS = 500;
 
 export async function interactiveConfirm(question, func) {
     await yargsInteractive()
@@ -17,4 +20,106 @@ export async function interactiveConfirm(question, func) {
             if (!result.confirm) return;
             func()
         });
-}
+}
+
+/**
+ * Creates a throttled status updater that periodically logs status messages and elapsed time.
+ *
+ * The updater allows you to update the displayed status message at any time, but throttles the actual log updates
+ * to the specified interval. When called with no message, the updater stops and finalizes the log output.
+ *
+ * @param {number} [intervalMs=WRITE_THROTTLE_MS] - The interval in milliseconds at which to update the log output.
+ * @returns {{
+ *   update: (message?: string) => void,
+ *   stop: () => void
+ * }} An object with `update` and `stop` methods:
+ *   - `update(message)`: Updates the status message and starts the updater if not already running. If called with no message, stops the updater.
+ *   - `stop()`: Stops the updater and finalizes the log output.
+ *
+ * @example
+ * const updater = createThrottledStatusUpdater(500);
+ * updater.update('Processing...');
+ * // ... later
+ * updater.update('Still working...');
+ * // ... when done
+ * updater.update(); // or updater.stop();
+ */
+export function createThrottledStatusUpdater(intervalMs = WRITE_THROTTLE_MS) {
+    let latestMessage;
+    let timer = null;
+    let stopped = false;
+    let startTime;
+
+    function write() {
+        const elapsed = `Elapsed:\t${formatElapsed(Date.now() - startTime)}`;
+        const lines = latestMessage ? [latestMessage, elapsed] : [elapsed];
+        logUpdate(lines.join('\n'));
+    }
+
+    function start() {
+        if (timer) return;
+
+        startTime = Date.now();
+
+        timer = setInterval(() => {
+            if (stopped) return;
+            write();
+        }, intervalMs);
+    }
+
+    function stop() {
+        stopped = true;
+        if (timer) clearInterval(timer);
+        write();
+        logUpdate.done();
+    }
+
+    // The updater function you call from anywhere
+    function update(message) {
+        if (!message) {
+            stop();
+            return;
+        }
+        latestMessage = message;
+        start();
+    }
+
+    return {
+        update,
+        stop
+    };
+}
+
+/**
+ * Converts a number of bytes into a human-readable string with appropriate units.
+ *
+ * @param {number} bytes - The number of bytes to format.
+ * @returns {string} The formatted string representing the size in appropriate units (Bytes, KB, MB, GB, TB, PB).
+ */
+export function formatBytes(bytes) {
+    if (bytes === 0) return '0 Bytes';
+    const k = 1024;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return (bytes / Math.pow(k, i)).toFixed(2) + ' ' + sizes[i];
+}
+
+/**
+ * Formats a duration given in milliseconds into a human-readable string.
+ *
+ * The output is in the format:
+ * - "Xh Ym Zs" if the duration is at least 1 hour,
+ * - "Ym Zs" if the duration is at least 1 minute but less than 1 hour,
+ * - "Zs" if the duration is less than 1 minute.
+ *
+ * @param {number} ms - The duration in milliseconds.
+ * @returns {string} The formatted elapsed time string.
+ */
+export function formatElapsed(ms) {
+    const sec = Math.floor(ms / 1000);
+    const min = Math.floor(sec / 60);
+    const hr = Math.floor(min / 60);
+    if (hr > 0) return `${hr}h ${min % 60}m ${sec % 60}s`;
+    if (min > 0) return `${min}m ${sec % 60}s`;
+    return `${sec}s`;
+}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@dynamicweb/cli",
   "type": "module",
   "description": "Dynamicweb CLI is a commandline tool for interacting with Dynamicweb 10 solutions.",
-  "version": "1.0.13",
+  "version": "1.0.15",
   "main": "bin/index.js",
   "files": [
     "bin/*"
@@ -33,6 +33,7 @@
     "fetch": "^1.1.0",
     "form-data": "^4.0.0",
     "https": "^1.0.0",
+    "log-update": "^6.1.0",
     "node-fetch": "^3.2.10",
     "path": "^0.12.7",
     "yargs": "^17.5.1",