@vbarbarosh/node-helpers 3.11.0 → 3.13.0

package/package.json

@@ -2,7 +2,7 @@
     "author": "Vladimir Barbarosh <vladimir.barbarosh@gmail.com>",
     "license": "MIT",
     "name": "@vbarbarosh/node-helpers",
-    "version": "3.11.0",
+    "version": "3.13.0",
     "description": "A set of helpers for JavaScript/Node.js",
     "files": [
         "src"

package/src/mongo_stream_upsert.js

@@ -0,0 +1,41 @@
+const Promise = require('bluebird');
+const stream = require('stream');
+const wait_while = require('./wait_while');
+
+/**
+ * Returns a writable stream for upsert mongo collection.
+ */
+function mongo_stream_upsert({collection, concurrency = 1})
+{
+    return new stream.Writable({
+        objectMode: true,
+        construct(next) {
+            this.running = 0;
+            next();
+        },
+        write: async function (items, enc, next) {
+            try {
+                if (!Array.isArray(items)) {
+                    next(new Error('An array of objects is expected.'));
+                    return;
+                }
+                await wait_while(() => this.running >= concurrency);
+                const operations = items.map(function (item) {
+                    return {replaceOne: {filter: {_id: item._id}, replacement: item, upsert: true}};
+                });
+                this.running++;
+                Promise.resolve(collection.bulkWrite(operations, {multi: true})).finally(() => this.running--);
+                next();
+            }
+            catch (error) {
+                next(error);
+            }
+        },
+        final: async function (next) {
+            await wait_while(() => this.running > 0);
+            next();
+        },
+    });
+}
+
+module.exports = mongo_stream_upsert;

package/src/stream_group.js

@@ -0,0 +1,27 @@
+const stream = require('stream');
+
+/**
+ * Returns a transform stream used to group object into chunks of objects.
+ */
+function stream_group(chunk_size)
+{
+    const chunk = [];
+    return new stream.Transform({
+        objectMode: true,
+        transform: function (item, enc, next) {
+            chunk.push(item);
+            if (chunk.length >= chunk_size) {
+                this.push(chunk.splice(0));
+            }
+            next();
+        },
+        flush: function (next) {
+            if (chunk.length) {
+                this.push(chunk.splice(0));
+                next();
+            }
+        },
+    });
+}
+
+module.exports = stream_group;

package/src/stream_multiplex.js

@@ -0,0 +1,51 @@
+const ignore = require('./ignore');
+const stream = require('stream');
+const wait_while = require('./wait_while');
+
+function stream_multiplex(...streams)
+{
+    // `new stream.Writable()` will ignore second, third, etc. calls to `destroy`
+    return stream.Writable({
+        objectMode: true,
+        construct: function (next) {
+            streams.forEach(s => s.once('error', e => this.destroy(e)));
+            next();
+        },
+        destroy: async function (error, next) {
+            let running = streams.length;
+            streams.forEach(s => s.destroy(error, () => running--));
+            await wait_while(() => running > 0);
+            next();
+        },
+        write: async function (chunk, enc, next) {
+            let running = streams.length;
+            streams.forEach(function (stream) {
+                stream.write(chunk, function (error) {
+                    running--;
+                    if (error) {
+                        next(error);
+                        next = ignore;
+                    }
+                });
+            });
+            await wait_while(() => running > 0);
+            next();
+        },
+        final: async function (next) {
+            let running = streams.length;
+            streams.forEach(function (stream) {
+                stream.end(function (error) {
+                    running--;
+                    if (error) {
+                        next(error);
+                        next = ignore;
+                    }
+                });
+            });
+            await wait_while(() => running > 0);
+            next();
+        },
+    });
+}
+
+module.exports = stream_multiplex;

package/src/stream_multiplex.test.js

@@ -0,0 +1,67 @@
+const assert = require('assert');
+const ignore = require('./ignore');
+const stream = require('stream');
+const stream_multiplex = require('./stream_multiplex');
+
+describe('stream_multiplex', function () {
+    it('basic', async function () {
+        const input = [1, 2, 3, 'str1'];
+        const rs = stream.Readable.from(input);
+        const ws1 = stream_debug();
+        const ws2 = stream_debug();
+        await stream.promises.pipeline(rs, stream_multiplex(ws1, ws2));
+        assert.deepStrictEqual(ws1.items, input);
+        assert.deepStrictEqual(ws2.items, input);
+        assert.deepStrictEqual(ws1.events.map(v => v.name), ['finish', 'close']);
+        assert.deepStrictEqual(ws2.events.map(v => v.name), ['finish', 'close']);
+    });
+    it('should throw after first error', async function () {
+        const input = [1, 2, 3, 'str1'];
+        const fail_at = 2;
+        const rs = stream.Readable.from(input);
+        const ws1 = stream_debug();
+        const ws2 = stream_debug();
+        const p = stream.promises.pipeline(rs, stream_multiplex(ws1, ws2, stream_fail_at(fail_at)));
+        await assert.rejects(p, new Error(`Failed at ${fail_at}`));
+        assert.deepStrictEqual(ws1.items, input.slice(0, fail_at));
+        assert.deepStrictEqual(ws2.items, input.slice(0, fail_at));
+        assert.deepStrictEqual(ws1.events.map(v => v.name), ['error', 'close']);
+        assert.deepStrictEqual(ws2.events.map(v => v.name), ['error', 'close']);
+    });
+});
+
+function stream_debug()
+{
+    const out = new stream.Writable({
+        objectMode: true,
+        write: function (item, enc, next) {
+            out.items.push(item);
+            next();
+        },
+    });
+    out.items = [];
+    out.events = [];
+    out.once('close', (...args) => out.events.push({name: 'close', args})); // result of `destroy`
+    out.once('drain', (...args) => out.events.push({name: 'drain', args}));
+    out.once('error', (...args) => out.events.push({name: 'error', args}));
+    out.once('finish', (...args) => out.events.push({name: 'finish', args})); // result of `end`
+    out.once('pipe', (...args) => out.events.push({name: 'pipe', args}));
+    out.once('unpipe', (...args) => out.events.push({name: 'unpipe', args}));
+    return out;
+}
+
+function stream_fail_at(at)
+{
+    let counter = 0;
+    return new stream.Writable({
+        objectMode: true,
+        write: function (item, enc, next) {
+            if (++counter >= at) {
+                next(new Error(`Failed at ${at}`));
+            }
+            else {
+                next();
+            }
+        },
+    });
+}

package/src/wait_while.js

@@ -0,0 +1,26 @@
+const Promise = require('bluebird');
+
+/**
+ * Wait until `fn` returns `true`.
+ */
+function wait_while(fn)
+{
+    return new Promise(function (resolve, reject) {
+        setTimeout(tick, 1);
+        async function tick() {
+            try {
+                if (await fn()) {
+                    setTimeout(tick, 1);
+                }
+                else {
+                    resolve();
+                }
+            }
+            catch (error) {
+                reject(error);
+            }
+        }
+    });
+}
+
+module.exports = wait_while;