@vbarbarosh/node-helpers 3.5.2 → 3.6.0

package/package.json

@@ -2,7 +2,7 @@
     "author": "Vladimir Barbarosh <vladimir.barbarosh@gmail.com>",
     "license": "MIT",
     "name": "@vbarbarosh/node-helpers",
-    "version": "3.5.2",
+    "version": "3.6.0",
     "description": "A set of helpers for JavaScript/Node.js",
     "files": [
         "src"
@@ -27,14 +27,15 @@
         "bluebird": "^3.7.2",
         "form-data": "^4.0.0",
         "json-stable-stringify": "^1.0.2",
-        "sanitize-filename": "^1.6.3"
+        "sanitize-filename": "^1.6.3",
+        "split": "^1.0.1"
     },
     "devDependencies": {
         "mocha": "^10.2.0",
-        "throttle": "^1.0.3",
         "redis-v1": "npm:redis@1",
         "redis-v2": "npm:redis@2",
         "redis-v3": "npm:redis@3",
-        "redis-v4": "npm:redis@4"
+        "redis-v4": "npm:redis@4",
+        "throttle": "^1.0.3"
     }
 }

package/src/errors/NotImplemented.js

@@ -0,0 +1,8 @@
+class NotImplemented extends Error
+{
+    constructor(message = 'Not Implemented') {
+        super(message);
+    }
+}
+
+module.exports = NotImplemented;

package/src/fastdl.js

@@ -0,0 +1,86 @@
+const format_bytes = require('./format_bytes');
+const format_seconds = require('./format_seconds');
+const format_thousands = require('./format_thousands');
+const fs = require('fs');
+const make_progress = require('./progress');
+const parallel = require('./parallel');
+const stream = require('stream');
+
+/**
+ * Download a file in several connections in parallel.
+ *
+ * await fastdl({
+ *     file: fs_path_basename(new URL(url).pathname),
+ *     read_stream_with_range: (first, last) => http_get_stream_range(url, first, last),
+ * });
+ */
+async function fastdl({file, read_stream_with_range, concurrency = 60, log = v => console.log(v)})
+{
+    const M = 1024*1024;
+    const chunk_min_bytes = M;
+    const chunk_max_bytes = 50*M;
+
+    log(`Truncating destination file [${file}]...`);
+    await fs.promises.writeFile(file, '');
+
+    log('Requesting first chunk to determine total size...');
+    const rs0 = await read_stream_with_range(0, chunk_min_bytes);
+
+    const total = rs0.content_range.total;
+    const chunk_size = Math.max(chunk_min_bytes, Math.min(chunk_max_bytes, Math.trunc(total/concurrency)));
+    const progress = make_progress(total);
+
+    let connections = 0;
+    let next_first = 0;
+    let total_written = 0;
+
+    const timer = setInterval(tick, 1000);
+    function tick() {
+        const p = progress;
+        const bb = format_bytes;
+        const ss = format_seconds;
+        log(`${bb(p.done)} of ${bb(p.total)} ${(p.percentage*100).toFixed(2)}% at ${bb(p.rate)}/s ETA ${ss(p.eta)} duration=${ss(p.duration)} connections=${connections}`);
+    }
+
+    log(`${format_bytes(total)} [${format_thousands(total)} bytes] to download`);
+
+    try {
+        await parallel({concurrency, spawn});
+    }
+    finally {
+        tick();
+        clearInterval(timer);
+    }
+
+    if (total !== total_written) {
+        throw new Error(`Total bytes written differs from size of a file: total[${total}] - total_written[${total_written}] = ${total - total_written}`);
+    }
+
+    function spawn() {
+        if (next_first >= total) {
+            return;
+        }
+        const first = next_first;
+        const last = Math.min(total - 1, first + (first === 0 ? rs0.content_range.last : chunk_size));
+        next_first = last + 1;
+        connections++;
+        return Promise.resolve(run()).finally(() => connections--);
+        async function run() {
+            const rs = (first === 0) ? rs0 : await read_stream_with_range(first, last);
+            const acc = new stream.PassThrough({
+                transform(buf, enc, cb) {
+                    total_written += buf.length;
+                    progress.add(buf.length);
+                    cb(null, buf);
+                }
+            });
+            const ws = fs.createWriteStream(file, {
+                flags: fs.constants.O_WRONLY, // |fs.constants.O_CREAT,
+                start: first,
+            });
+            await stream.promises.pipeline(rs, acc, ws);
+        }
+    }
+}
+
+module.exports = fastdl;

package/src/http_get_stream_range.js

@@ -2,10 +2,10 @@ const axios = require('axios');
 const format_thousands = require('./format_thousands');
 const parse_http_content_range = require('./parse_http_content_range');
 
-async function http_get_stream_range(url, begin, end)
+async function http_get_stream_range(url, first, last)
 {
-    const a = Number.isInteger(begin) ? begin : '';
-    const b = Number.isInteger(end) ? (end - 1) : '';
+    const a = Number.isInteger(first) ? first : '';
+    const b = Number.isInteger(last) ? last : '';
     const headers = {Range: `bytes=${a}-${b}`};
     const res = await axios.get(url, {responseType: 'stream', headers});
     const out = res.data;

package/src/logger.js

@@ -0,0 +1,25 @@
+const NotImplemented = require('./errors/NotImplemented');
+
+// grep_cle1grnu10000tu72diepbfet
+const re = /^((?:\[\s*[^\]]*\])*)\s*(.*)$/m;
+
+/**
+ * log = logger();
+ * log.add('hey')
+ * log.add('[tag] heyhey')
+ */
+function logger(options = {})
+{
+    function out(message) {
+        const v = typeof message === 'string' ? JSON.stringify(message.trim()).slice(1, -1) : JSON.stringify(message);
+        const [, a, m] = v.match(re);
+        const time = new Date().toJSON();
+        console.log(`${a} ${m}`.trim());
+    };
+    out.spawn = function () {
+        throw new NotImplemented();
+    };
+    return out;
+}
+
+module.exports = logger;

package/src/parallel.js

@@ -0,0 +1,51 @@
+const NotImplemented = require('./errors/NotImplemented');
+
+/**
+ * - Run until `spawn` return `null`.
+ * - Keep no more than `concurrency` number of workers at a time.
+ */
+async function parallel({concurrency, spawn, progress})
+{
+    const running = [];
+    return new Promise(function (resolve, reject) {
+        const timer = setInterval(tick, 1000);
+        schedule();
+        if (running.length) {
+            tick();
+        }
+        function schedule() {
+            while (running.length < concurrency) {
+                const w = spawn();
+                if (!w) {
+                    break;
+                }
+                const item = Promise.resolve(w).then(resolved, rejected);
+                running.push(item);
+                function resolved() {
+                    const i = running.indexOf(item);
+                    if (i === -1) {
+                        throw new NotImplemented();
+                    }
+                    running.splice(i, 1);
+                    schedule();
+                }
+                function rejected(error) {
+                    console.log(error);
+                    throw new NotImplemented();
+                }
+            }
+            if (running.length === 0) {
+                tick();
+                clearInterval(timer);
+                resolve();
+            }
+        }
+        function tick() {
+            if (progress) {
+                progress(running);
+            }
+        }
+    });
+}
+
+module.exports = parallel;

package/src/stream_data_ln.test.js

@@ -1,15 +1,25 @@
 const assert = require('assert');
 const fs = require('fs');
+const split = require('split');
+const stream = require('stream');
 const stream_data_ln = require('./stream_data_ln');
 const stream_promise = require('./stream_promise');
 
 describe('stream_data_ln', function () {
     it('should handle basic input', async function () {
-        let tmp = '';
+        let actual = '';
         const s = fs.createReadStream(__filename);
-        const off = stream_data_ln(s, v => tmp += v + '\n');
+        const off = stream_data_ln(s, v => actual += v + '\n');
         await stream_promise(s);
         off();
-        assert.deepStrictEqual(tmp, fs.readFileSync(__filename, {encoding: 'utf8'}));
+        assert.deepStrictEqual(actual, fs.readFileSync(__filename, {encoding: 'utf8'}));
+    });
+    it('split', async function () {
+        let actual = '';
+        const s = fs.createReadStream(__filename);
+        const ss = s.pipe(split('\n', null, {trailing: false}));
+        ss.on('data', v => actual += v + '\n');
+        await stream.promises.finished(ss);
+        assert.deepStrictEqual(actual, fs.readFileSync(__filename, {encoding: 'utf8'}));
     });
 });