@vbarbarosh/node-helpers 3.5.3 → 3.7.0

package/package.json

@@ -2,7 +2,7 @@
     "author": "Vladimir Barbarosh <vladimir.barbarosh@gmail.com>",
     "license": "MIT",
     "name": "@vbarbarosh/node-helpers",
-    "version": "3.5.3",
+    "version": "3.7.0",
     "description": "A set of helpers for JavaScript/Node.js",
     "files": [
         "src"
@@ -28,7 +28,8 @@
         "form-data": "^4.0.0",
         "json-stable-stringify": "^1.0.2",
         "sanitize-filename": "^1.6.3",
-        "split": "^1.0.1"
+        "split": "^1.0.1",
+        "ssh2": "^1.15.0"
     },
     "devDependencies": {
         "mocha": "^10.2.0",

package/src/errors/NotImplemented.js

@@ -0,0 +1,8 @@
+class NotImplemented extends Error
+{
+    constructor(message = 'Not Implemented') {
+        super(message);
+    }
+}
+
+module.exports = NotImplemented;

package/src/fastdl.js

@@ -0,0 +1,86 @@
+const format_bytes = require('./format_bytes');
+const format_seconds = require('./format_seconds');
+const format_thousands = require('./format_thousands');
+const fs = require('fs');
+const make_progress = require('./progress');
+const parallel = require('./parallel');
+const stream = require('stream');
+
+/**
+ * Download a file in several connections in parallel.
+ *
+ * await fastdl({
+ *     file: fs_path_basename(new URL(url).pathname),
+ *     read_stream_with_range: (first, last) => http_get_stream_range(url, first, last),
+ * });
+ */
+async function fastdl({file, read_stream_with_range, concurrency = 60, log = v => console.log(v)})
+{
+    const M = 1024*1024;
+    const chunk_min_bytes = M;
+    const chunk_max_bytes = 50*M;
+
+    log(`Truncating destination file [${file}]...`);
+    await fs.promises.writeFile(file, '');
+
+    log('Requesting first chunk to determine total size...');
+    const rs0 = await read_stream_with_range(0, chunk_min_bytes);
+
+    const total = rs0.content_range.total;
+    const chunk_size = Math.max(chunk_min_bytes, Math.min(chunk_max_bytes, Math.trunc(total/concurrency)));
+    const progress = make_progress(total);
+
+    let connections = 0;
+    let next_first = 0;
+    let total_written = 0;
+
+    const timer = setInterval(tick, 1000);
+    function tick() {
+        const p = progress;
+        const bb = format_bytes;
+        const ss = format_seconds;
+        log(`${bb(p.done)} of ${bb(p.total)} ${(p.percentage*100).toFixed(2)}% at ${bb(p.rate)}/s ETA ${ss(p.eta)} duration=${ss(p.duration)} connections=${connections}`);
+    }
+
+    log(`${format_bytes(total)} [${format_thousands(total)} bytes] to download`);
+
+    try {
+        await parallel({concurrency, spawn});
+    }
+    finally {
+        tick();
+        clearInterval(timer);
+    }
+
+    if (total !== total_written) {
+        throw new Error(`Total bytes written differs from size of a file: total[${total}] - total_written[${total_written}] = ${total - total_written}`);
+    }
+
+    function spawn() {
+        if (next_first >= total) {
+            return;
+        }
+        const first = next_first;
+        const last = Math.min(total - 1, first + (first === 0 ? rs0.content_range.last : chunk_size));
+        next_first = last + 1;
+        connections++;
+        return Promise.resolve(run()).finally(() => connections--);
+        async function run() {
+            const rs = (first === 0) ? rs0 : await read_stream_with_range(first, last);
+            const acc = new stream.PassThrough({
+                transform(buf, enc, cb) {
+                    total_written += buf.length;
+                    progress.add(buf.length);
+                    cb(null, buf);
+                }
+            });
+            const ws = fs.createWriteStream(file, {
+                flags: fs.constants.O_WRONLY, // |fs.constants.O_CREAT,
+                start: first,
+            });
+            await stream.promises.pipeline(rs, acc, ws);
+        }
+    }
+}
+
+module.exports = fastdl;

package/src/logger.js

@@ -0,0 +1,25 @@
+const NotImplemented = require('./errors/NotImplemented');
+
+// grep_cle1grnu10000tu72diepbfet
+const re = /^((?:\[\s*[^\]]*\])*)\s*(.*)$/m;
+
+/**
+ * log = logger();
+ * log.add('hey')
+ * log.add('[tag] heyhey')
+ */
+function logger(options = {})
+{
+    function out(message) {
+        const v = typeof message === 'string' ? JSON.stringify(message.trim()).slice(1, -1) : JSON.stringify(message);
+        const [, a, m] = v.match(re);
+        const time = new Date().toJSON();
+        console.log(`${a} ${m}`.trim());
+    };
+    out.spawn = function () {
+        throw new NotImplemented();
+    };
+    return out;
+}
+
+module.exports = logger;

package/src/parallel.js

@@ -0,0 +1,51 @@
+const NotImplemented = require('./errors/NotImplemented');
+
+/**
+ * - Run until `spawn` return `null`.
+ * - Keep no more than `concurrency` number of workers at a time.
+ */
+async function parallel({concurrency, spawn, progress})
+{
+    const running = [];
+    return new Promise(function (resolve, reject) {
+        const timer = setInterval(tick, 1000);
+        schedule();
+        if (running.length) {
+            tick();
+        }
+        function schedule() {
+            while (running.length < concurrency) {
+                const w = spawn();
+                if (!w) {
+                    break;
+                }
+                const item = Promise.resolve(w).then(resolved, rejected);
+                running.push(item);
+                function resolved() {
+                    const i = running.indexOf(item);
+                    if (i === -1) {
+                        throw new NotImplemented();
+                    }
+                    running.splice(i, 1);
+                    schedule();
+                }
+                function rejected(error) {
+                    console.log(error);
+                    throw new NotImplemented();
+                }
+            }
+            if (running.length === 0) {
+                tick();
+                clearInterval(timer);
+                resolve();
+            }
+        }
+        function tick() {
+            if (progress) {
+                progress(running);
+            }
+        }
+    });
+}
+
+module.exports = parallel;

package/src/progress.js

@@ -8,27 +8,41 @@
 // https://bramcohen.livejournal.com/24122.html
 function progress(total)
 {
-    let done = 0;
+    const history = [];
     const time0 = Date.now();
     const out = {
         done: 0,
         total,
         eta: null,
+        // `rate` instead of `bps` (bytes per speed) because this function could count any values (e.g.
+        // copied bytes over time, or handled jobs over time, or created object over time, etc.)
         rate: null,
         duration: 0,
         progress: null,
         percentage: null,
         add: function (delta = 0) {
+            const now_minus_5_seconds = Date.now() - 5000;
+            history.push({time: Date.now(), delta});
+            // Keep last 5 seconds
+            history.splice(0, history.findLastIndex(v => v.time < now_minus_5_seconds));
             out.done += delta;
             out.refresh();
         },
         update: function (done) {
-            out.done = done;
-            out.refresh();
+            out.add(done - out.done);
         },
         refresh: function () {
+            const now_minus_5_seconds = Date.now() - 5000;
+            const tmp = history.filter(v => v.time >= now_minus_5_seconds);
+            if (tmp.length > 1) {
+                const items = tmp.reduce((a,v) => a + v.delta, 0);
+                const time_sec = (tmp.pop().time - tmp[0].time)/1000;
+                out.rate = items/time_sec;
+            }
+            else {
+                out.rate = out.done/out.duration;
+            }
             out.duration = (Date.now() - time0)/1000;
-            out.rate = out.done/out.duration;
             out.percentage = !out.total ? null : out.done/out.total;
             out.eta = !out.total ? null : (out.total - out.done)/out.rate;
         },

package/src/sftp_get_file_info.js

@@ -0,0 +1,44 @@
+const Promise = require('bluebird');
+const ignore = require('./ignore');
+const ssh2 = require('ssh2');
+const waitcb = require('./waitcb');
+
+async function sftp_get_file_info(url, {log = ignore})
+{
+    const u = new URL(url);
+    const host = u.host;
+    const port = +u.port || 22;
+    const username = decodeURIComponent(u.username);
+    const password = decodeURIComponent(u.password);
+    const pathname = u.pathname;
+
+    log('Establishing connection...');
+    const conn = new ssh2.Client();
+    try {
+        await new Promise(function (resolve, reject) {
+            conn.once('ready', resolve);
+            conn.once('error', reject);
+            conn.connect({host, port, username, password});
+        });
+
+        log('Asking for an sftp service...');
+        const sftp = await waitcb(cb => conn.sftp(cb));
+
+        log('Requesting file info...');
+        const [stat, lstat, readdir] = await Promise.all([
+            waitcb(cb => sftp.stat(pathname, cb)),
+            waitcb(cb => sftp.lstat(pathname, cb)),
+            waitcb(cb => sftp.readdir(pathname, cb)),
+        ]);
+        return {
+            stat,
+            lstat,
+            readdir,
+        };
+    }
+    finally {
+        conn.end();
+    }
+}
+
+module.exports = sftp_get_file_info;

package/src/sftp_get_stream_range.js

@@ -0,0 +1,47 @@
+const ignore = require('./ignore');
+const ssh2 = require('ssh2');
+const waitcb = require('./waitcb');
+
+const ___stat = Symbol('sftp_get_stream_range.stat');
+
+async function sftp_get_stream_range(url, first, last, {log = ignore} = {})
+{
+    const u = new URL(url);
+    const host = u.host;
+    const port = +u.port || 22;
+    const username = decodeURIComponent(u.username);
+    const password = decodeURIComponent(u.password);
+    const pathname = u.pathname;
+
+    log('Establishing connection...');
+    const conn = new ssh2.Client();
+    await new Promise(function (resolve, reject) {
+        conn.on('ready', resolve);
+        conn.on('error', reject);
+        conn.connect({host, port, username, password});
+    });
+
+    log('Asking for an sftp service...');
+    const sftp = await waitcb(cb => conn.sftp(cb));
+
+    log('Requesting file info...');
+    const stat = url[___stat] || (url[___stat] = await waitcb(cb => sftp.stat(pathname, cb)));
+
+    if (first < 0 || last >= stat.size) {
+        throw new Error(`Invalid range: [first=${first}][last=${last}]`);
+    }
+
+    const out = sftp.createReadStream(pathname, {start: first, end: last})
+    out.content_range = {
+        type: 'bytes',
+        first,
+        last,
+        total: stat.size,
+    };
+    out.once('end', function () {
+        conn.end();
+    });
+    return out;
+}
+
+module.exports = sftp_get_stream_range;

package/src/waitcb.js

@@ -0,0 +1,18 @@
+const Promise = require('bluebird');
+
+/**
+ * Wait for a Node-like function to finish (which will call `callback`
+ * with 2 arguments: `error` and `value`).
+ *
+ * await waitcb(cb => fs.promises.writeFile('a', 'hello\n', cb));
+ */
+function waitcb(fn)
+{
+    return new Promise(function (resolve, reject) {
+        fn(function (error, out) {
+            error ? reject(error) : resolve(out);
+        });
+    });
+}
+
+module.exports = waitcb;