@vbarbarosh/node-helpers 3.35.0 → 3.37.0

package/package.json

@@ -2,7 +2,7 @@
     "author": "Vladimir Barbarosh <vladimir.barbarosh@gmail.com>",
     "license": "MIT",
     "name": "@vbarbarosh/node-helpers",
-    "version": "3.35.0",
+    "version": "3.37.0",
     "description": "A set of helpers for JavaScript/Node.js",
     "files": [
         "src"

package/src/format_bytes.js

@@ -1,17 +1,17 @@
 function format_bytes(bytes)
 {
     const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
-    if (Number.isNaN(bytes*1)) {
+    if (typeof bytes !== 'number' || Number.isNaN(bytes)) {
         return 'n/a';
     }
-    if (bytes == 0) {
+    if (!bytes) {
         return '0KB';
     }
     if (bytes < 1024) {
         return '1KB';
     }
     const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)), 10);
-    return `${(bytes / (1024 ** i)).toFixed(bytes > 1024*1024 ? 1 : 0)}${sizes[i]}`;
+    return `${(bytes / (1024 ** i)).toFixed(bytes > 1024*1024 ? 2 : 0)}${sizes[i]}`.replace(/\.00/, '.0');
 }
 
 module.exports = format_bytes;

package/src/format_bytes.test.js

@@ -0,0 +1,22 @@
+const assert = require('assert');
+const format_bytes = require('./format_bytes');
+
+const items = [
+    ['', 'n/a'],
+    [null, 'n/a'],
+    [undefined, 'n/a'],
+    [0, '0KB'],
+    [1, '1KB'],
+    [1023, '1KB'],
+    [2048, '2KB'],
+    [35.45*1024*1024, '35.45MB'],
+    [3.92*1024*1024*1024, '3.92GB'],
+];
+
+describe('format_bytes', function () {
+    items.forEach(function ([input, expected]) {
+        it(`${input} → ${expected}`, function () {
+            assert.strictEqual(expected, format_bytes(input));
+        });
+    });
+});

package/src/parse_bytes.js

@@ -0,0 +1,14 @@
+function parse_bytes(bytes)
+{
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
+    if (typeof bytes !== 'string') {
+        return Number.NaN;
+    }
+    const m = bytes.toUpperCase().match(/(\d+([.]?\d+)?)(([KMGTP]I?)?B)/);
+    if (!m) {
+        return Number.NaN;
+    }
+    return Math.floor(m[1]*Math.pow(1024, sizes.indexOf(m[3].replace('I', ''))));
+}
+
+module.exports = parse_bytes;

package/src/parse_bytes.test.js

@@ -0,0 +1,21 @@
+const assert = require('assert');
+const parse_bytes = require('./parse_bytes');
+
+const items = [
+    ['', Number.NaN],
+    [null, Number.NaN],
+    [undefined, Number.NaN],
+    ['0KB', 0],
+    ['1KB', 1024],
+    ['2KB', 2048],
+    ['35.45MB', Math.floor(35.45*1024*1024)],
+    ['3.92GB', Math.floor(3.92*1024*1024*1024)],
+];
+
+describe('parse_bytes', function () {
+    items.forEach(function ([input, expected]) {
+        it(`${input} → ${expected}`, function () {
+            assert.strictEqual(expected, parse_bytes(input));
+        });
+    });
+});

package/src/shell_curl_progress.js

@@ -0,0 +1,35 @@
+const Promise = require('bluebird');
+const child_process = require('child_process');
+const stream = require('stream');
+const stream_curl_progress = require('./stream_curl_progress');
+const stream_each = require('./stream_each');
+
+async function shell_curl_progress(args, {options, user_friendly_status})
+{
+    // $ man curl
+    // > If you want a progress meter for HTTP POST or PUT requests,
+    // > you need to redirect the response output to a file, using
+    // > shell redirect (>), -o, --output or similar.
+
+    const tmp = ['--progress-meter'];
+    if (!args.includes('-o')) {
+        tmp.push('-o', '/dev/null');
+    }
+    const proc = child_process.spawn(args[0], args.slice(1).concat(tmp), {...options, stdio: ['inherit', 'inherit', 'pipe']});
+
+    const promises = [];
+    promises.push(new Promise(function (resolve, reject) {
+        proc.once('error', reject);
+        proc.once('exit', code => code ? reject(new Error(`Process terminated with code ${code}`)) : resolve());
+    }));
+    promises.push(stream.promises.pipeline(proc.stderr, stream_curl_progress(), stream_each(progress_fn)));
+
+    await Promise.all(promises);
+
+    function progress_fn(v) {
+        const speed = v.speed === '0' ? '~' : `${v.speed}/s`;
+        user_friendly_status(`${v.perc}% | ${v.done} of ${v.total} at ${speed} ETA ${v.eta} duration=${v.duration}`);
+    }
+}
+
+module.exports = shell_curl_progress;

package/src/shell_ytdlp_progress.js

@@ -1,11 +1,13 @@
 const child_process = require('child_process');
+const format_seconds = require('./format_seconds');
 const stream = require('stream');
 const stream_each = require('./stream_each');
 const stream_ytdlp_progress = require('./stream_ytdlp_progress');
 
-async function shell_ytdlp_progress(args, {progress_fn, ...options})
+async function shell_ytdlp_progress(args, {user_friendly_status, ...options})
 {
     const proc = child_process.spawn('yt-dlp', ['--progress', ...args], {...options, stdio: 'pipe'});
+    const time0 = Date.now();
 
     const promises = [];
     promises.push(new Promise(function (resolve, reject) {
@@ -16,6 +18,11 @@ async function shell_ytdlp_progress(args, {progress_fn, ...options})
     promises.push(stream.promises.pipeline(proc.stderr, stream_each(v => console.log(`[stderr] ${v}`))));
 
     await Promise.all(promises);
+
+    function progress_fn(v) {
+        const duration = format_seconds((Date.now() - time0)/1000);
+        user_friendly_status(`${v.perc} | [${v.current_part}/${v.total_parts}] ${v.done} of ${v.total} at ${v.speed} ETA ${v.eta} duration=${duration}`);
+    }
 }
 
 module.exports = shell_ytdlp_progress;

package/src/stream_curl_progress.js

@@ -0,0 +1,46 @@
+const stream = require('stream');
+const stream_filter = require('./stream_filter');
+const stream_lines = require('./stream_lines');
+const stream_map = require('./stream_map');
+const stream_skip = require('./stream_skip');
+
+function stream_curl_progress()
+{
+    return stream.compose(
+        stream_lines(),
+        stream_skip(2),
+        stream_map(function (v) {
+            // curl: (22) The requested URL returned error: 500
+            if (v.startsWith('curl:')) {
+                process.stderr.write(`${v}\n`);
+                return null;
+            }
+
+            // % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+            //                                Dload  Upload   Total   Spent    Left  Speed
+            // 0 3217M    0     0    0  128k      0   205k  4:27:16 --:--:--  4:27:16  205k
+            // 0 3217M    0     0    0 6016k      0  3751k  0:14:38  0:00:01  0:14:37 3750k
+            // ...
+            // 18 3217M    0     0   18  587M      0  18.5M  0:02:53  0:00:31  0:02:22 19.0M
+            const [perc, total, perc2, recv, perc3, transferred, speed_down, speed_up, time_total, duration, eta, speed] = v.trim().split(/\s+/);
+            return {
+                perc,
+                done: recv === '0' ? transferred : recv,
+                total,
+                perc2,
+                recv,
+                perc3,
+                transferred,
+                speed_down,
+                speed_up,
+                time_total: time_total === '--:--:--' ? '~' : time_total,
+                duration: duration === '--:--:--' ? '~' : duration,
+                eta: eta === '--:--:--' ? '~' : eta,
+                speed
+            };
+        }),
+        stream_filter(v => v),
+    );
+}
+
+module.exports = stream_curl_progress;

package/src/stream_skip.js

@@ -0,0 +1,25 @@
+const stream = require('stream');
+
+/**
+ * Skip first `n` records.
+ */
+function stream_skip(n)
+{
+    let current = 0;
+    return new stream.Transform({
+        objectMode: true,
+        transform: function (item, encoding, callback) {
+            try {
+                if (current++ >= n) {
+                    this.push(item);
+                }
+                callback();
+            }
+            catch (error) {
+                callback(error);
+            }
+        },
+    });
+}
+
+module.exports = stream_skip;

package/src/stream_ytdlp_progress.js

@@ -1,3 +1,6 @@
+const format_bytes = require('./format_bytes');
+const format_percents = require('./format_percents');
+const parse_bytes = require('./parse_bytes');
 const stream = require('stream');
 const stream_filter = require('./stream_filter');
 const stream_lines = require('./stream_lines');
@@ -5,12 +8,30 @@ const stream_map = require('./stream_map');
 
 function stream_ytdlp_progress()
 {
-    const re = /^\[download] (.*) of (.*) at (.*) ETA (.*)$/;
+    let current_part = 1;
+    const downloading_formats = [];
     return stream.compose(
         stream_lines(),
-        stream_map(function (v) {
-            const m = re.exec(v);
-            return !m ? false : {done: m[1].trim(), of: m[2].trim(), speed: m[3].trim(), eta: m[4].trim()};
+        stream_map(function (line) {
+            let m;
+            if (m = line.match(/^\[info] \S+: Downloading \d+ format\(s\): (.+)$/)) {
+                downloading_formats.push(...m[1].split('+'));
+                return false;
+            }
+            if (m = line.match(/^\[download] Destination: .*f(\d+)\.[^.]+/)) {
+                current_part = downloading_formats.indexOf(m[1]) + 1;
+                return false;
+            }
+            if (m = line.match(/^\[download] (.*) of (.*) at (.*) ETA (.*)$/)) {
+                const percentage = parseFloat(m[1].trim())/100;
+                const total = parse_bytes(m[2].trim());
+                const speed = m[3].trim().replace('i', '');
+                const eta = m[4].trim();
+                return {current_part, total_parts: downloading_formats.length,
+                    perc: format_percents(percentage), done: format_bytes(percentage*total),
+                    total: format_bytes(total), speed, eta};
+            }
+            return false;
         }),
         stream_filter(v => v),
     );