@vbarbarosh/node-helpers 3.6.0 → 3.8.0

package/package.json

@@ -2,7 +2,7 @@
     "author": "Vladimir Barbarosh <vladimir.barbarosh@gmail.com>",
     "license": "MIT",
     "name": "@vbarbarosh/node-helpers",
-    "version": "3.6.0",
+    "version": "3.8.0",
     "description": "A set of helpers for JavaScript/Node.js",
     "files": [
         "src"
@@ -28,7 +28,8 @@
         "form-data": "^4.0.0",
         "json-stable-stringify": "^1.0.2",
         "sanitize-filename": "^1.6.3",
-        "split": "^1.0.1"
+        "split": "^1.0.1",
+        "ssh2": "^1.15.0"
     },
     "devDependencies": {
         "mocha": "^10.2.0",

package/src/fastdl.js

@@ -2,6 +2,7 @@ const format_bytes = require('./format_bytes');
 const format_seconds = require('./format_seconds');
 const format_thousands = require('./format_thousands');
 const fs = require('fs');
+const fs_path_basename = require('./fs_path_basename');
 const make_progress = require('./progress');
 const parallel = require('./parallel');
 const stream = require('stream');
@@ -14,16 +15,17 @@ const stream = require('stream');
  *     read_stream_with_range: (first, last) => http_get_stream_range(url, first, last),
  * });
  */
-async function fastdl({file, read_stream_with_range, concurrency = 60, log = v => console.log(v)})
+async function fastdl({file, read_stream_with_range, concurrency = 60, user_friendly_status = v => console.log(v)})
 {
+    const _ = user_friendly_status;
     const M = 1024*1024;
     const chunk_min_bytes = M;
     const chunk_max_bytes = 50*M;
 
-    log(`Truncating destination file [${file}]...`);
+    _(`Truncating destination file [${fs_path_basename(file)}]...`);
     await fs.promises.writeFile(file, '');
 
-    log('Requesting first chunk to determine total size...');
+    _('Requesting first chunk to determine total size...');
     const rs0 = await read_stream_with_range(0, chunk_min_bytes);
 
     const total = rs0.content_range.total;
@@ -39,10 +41,15 @@ async function fastdl({file, read_stream_with_range, concurrency = 60, log = v =
         const p = progress;
         const bb = format_bytes;
         const ss = format_seconds;
-        log(`${bb(p.done)} of ${bb(p.total)} ${(p.percentage*100).toFixed(2)}% at ${bb(p.rate)}/s ETA ${ss(p.eta)} duration=${ss(p.duration)} connections=${connections}`);
+        if (p.rate) {
+            _(`${bb(p.done)} of ${bb(p.total)} [${(p.percentage*100).toFixed(2)}%] at ${bb(p.rate)}/s ETA ${ss(p.eta)} duration=${ss(p.duration)} connections=${connections}`);
+        }
+        else {
+            _(`${bb(p.done)} of ${bb(p.total)} [${(p.percentage*100).toFixed(2)}%] at ~ ETA ${ss(p.eta)} duration=${ss(p.duration)} connections=${connections}`);
+        }
     }
 
-    log(`${format_bytes(total)} [${format_thousands(total)} bytes] to download`);
+    _(`${format_bytes(total)} [${format_thousands(total)} bytes] to download`);
 
     try {
         await parallel({concurrency, spawn});

package/src/progress.js

@@ -8,27 +8,41 @@
 // https://bramcohen.livejournal.com/24122.html
 function progress(total)
 {
-    let done = 0;
+    const history = [];
     const time0 = Date.now();
     const out = {
         done: 0,
         total,
         eta: null,
+        // `rate` instead of `bps` (bytes per speed) because this function could count any values (e.g.
+        // copied bytes over time, or handled jobs over time, or created object over time, etc.)
         rate: null,
         duration: 0,
         progress: null,
         percentage: null,
         add: function (delta = 0) {
+            const now_minus_2sec = Date.now() - 2000;
+            history.push({time: Date.now(), delta});
+            // Keep last 2 seconds
+            history.splice(0, history.findLastIndex(v => v.time < now_minus_2sec));
             out.done += delta;
             out.refresh();
         },
         update: function (done) {
-            out.done = done;
-            out.refresh();
+            out.add(done - out.done);
         },
         refresh: function () {
+            const now_minus_2sec = Date.now() - 2000;
+            const tmp = history.filter(v => v.time >= now_minus_2sec);
+            if (tmp.length > 1) {
+                const items = tmp.reduce((a,v) => a + v.delta, 0);
+                const time_sec = (tmp.pop().time - tmp[0].time)/1000;
+                out.rate = items/time_sec;
+            }
+            else {
+                out.rate = out.done/out.duration;
+            }
             out.duration = (Date.now() - time0)/1000;
-            out.rate = out.done/out.duration;
             out.percentage = !out.total ? null : out.done/out.total;
             out.eta = !out.total ? null : (out.total - out.done)/out.rate;
         },

package/src/sftp_get_file_info.js

@@ -0,0 +1,44 @@
+const Promise = require('bluebird');
+const ignore = require('./ignore');
+const ssh2 = require('ssh2');
+const waitcb = require('./waitcb');
+
+async function sftp_get_file_info(url, {log = ignore})
+{
+    const u = new URL(url);
+    const host = u.host;
+    const port = +u.port || 22;
+    const username = decodeURIComponent(u.username);
+    const password = decodeURIComponent(u.password);
+    const pathname = u.pathname;
+
+    log('Establishing connection...');
+    const conn = new ssh2.Client();
+    try {
+        await new Promise(function (resolve, reject) {
+            conn.once('ready', resolve);
+            conn.once('error', reject);
+            conn.connect({host, port, username, password});
+        });
+
+        log('Asking for an sftp service...');
+        const sftp = await waitcb(cb => conn.sftp(cb));
+
+        log('Requesting file info...');
+        const [stat, lstat, readdir] = await Promise.all([
+            waitcb(cb => sftp.stat(pathname, cb)),
+            waitcb(cb => sftp.lstat(pathname, cb)),
+            waitcb(cb => sftp.readdir(pathname, cb)),
+        ]);
+        return {
+            stat,
+            lstat,
+            readdir,
+        };
+    }
+    finally {
+        conn.end();
+    }
+}
+
+module.exports = sftp_get_file_info;

package/src/sftp_get_stream_range.js

@@ -0,0 +1,47 @@
+const ignore = require('./ignore');
+const ssh2 = require('ssh2');
+const waitcb = require('./waitcb');
+
+const ___stat = Symbol('sftp_get_stream_range.stat');
+
+async function sftp_get_stream_range(url, first, last, {log = ignore} = {})
+{
+    const u = new URL(url);
+    const host = u.host;
+    const port = +u.port || 22;
+    const username = decodeURIComponent(u.username);
+    const password = decodeURIComponent(u.password);
+    const pathname = u.pathname;
+
+    log('Establishing connection...');
+    const conn = new ssh2.Client();
+    await new Promise(function (resolve, reject) {
+        conn.on('ready', resolve);
+        conn.on('error', reject);
+        conn.connect({host, port, username, password});
+    });
+
+    log('Asking for an sftp service...');
+    const sftp = await waitcb(cb => conn.sftp(cb));
+
+    log('Requesting file info...');
+    const stat = url[___stat] || (url[___stat] = await waitcb(cb => sftp.stat(pathname, cb)));
+
+    if (first < 0 || last >= stat.size) {
+        throw new Error(`Invalid range: [first=${first}][last=${last}]`);
+    }
+
+    const out = sftp.createReadStream(pathname, {start: first, end: last})
+    out.content_range = {
+        type: 'bytes',
+        first,
+        last,
+        total: stat.size,
+    };
+    out.once('end', function () {
+        conn.end();
+    });
+    return out;
+}
+
+module.exports = sftp_get_stream_range;

package/src/waitcb.js

@@ -0,0 +1,18 @@
+const Promise = require('bluebird');
+
+/**
+ * Wait for a Node-like function to finish (which will call `callback`
+ * with 2 arguments: `error` and `value`).
+ *
+ * await waitcb(cb => fs.promises.writeFile('a', 'hello\n', cb));
+ */
+function waitcb(fn)
+{
+    return new Promise(function (resolve, reject) {
+        fn(function (error, out) {
+            error ? reject(error) : resolve(out);
+        });
+    });
+}
+
+module.exports = waitcb;