@vbarbarosh/node-helpers 3.10.1 → 3.12.0

package/package.json

@@ -2,7 +2,7 @@
     "author": "Vladimir Barbarosh <vladimir.barbarosh@gmail.com>",
     "license": "MIT",
     "name": "@vbarbarosh/node-helpers",
-    "version": "3.10.1",
+    "version": "3.12.0",
     "description": "A set of helpers for JavaScript/Node.js",
     "files": [
         "src"

package/src/mongo_stream_upsert.js

@@ -0,0 +1,41 @@
+const Promise = require('bluebird');
+const stream = require('stream');
+const wait_while = require('./wait_while');
+
+/**
+ * Returns a writable stream for upsert mongo collection.
+ */
+function mongo_stream_upsert({collection, concurrency = 1})
+{
+    return new stream.Writable({
+        objectMode: true,
+        construct(next) {
+            this.running = 0;
+            next();
+        },
+        write: async function (items, enc, next) {
+            try {
+                if (!Array.isArray(items)) {
+                    next(new Error('An array of objects is expected.'));
+                    return;
+                }
+                await wait_while(() => this.running >= concurrency);
+                const operations = items.map(function (item) {
+                    return {replaceOne: {filter: {_id: item._id}, replacement: item, upsert: true}};
+                });
+                this.running++;
+                Promise.resolve(collection.bulkWrite(operations, {multi: true})).finally(() => this.running--);
+                next();
+            }
+            catch (error) {
+                next(error);
+            }
+        },
+        final: async function (next) {
+            await wait_while(() => this.running > 0);
+            next();
+        },
+    });
+}
+
+module.exports = mongo_stream_upsert;

package/src/stream_group.js

@@ -0,0 +1,27 @@
+const stream = require('stream');
+
+/**
+ * Returns a transform stream used to group object into chunks of objects.
+ */
+function stream_group(chunk_size)
+{
+    const chunk = [];
+    return new stream.Transform({
+        objectMode: true,
+        transform: function (item, enc, next) {
+            chunk.push(item);
+            if (chunk.length >= chunk_size) {
+                this.push(chunk.splice(0));
+            }
+            next();
+        },
+        flush: function (next) {
+            if (chunk.length) {
+                this.push(chunk.splice(0));
+                next();
+            }
+        },
+    });
+}
+
+module.exports = stream_group;

package/src/urlclean.js

@@ -0,0 +1,21 @@
+const querystring = require('querystring');
+
+/**
+ * Replace unnecessary encodes symbols with chars. The returned url will be
+ * more human frieldy, at the same time it still be valid url.
+ */
+function urlclean(url)
+{
+    const i = url.indexOf('?') + 1;
+    if (i) {
+        const s = querystring.stringify(querystring.parse(url.slice(i)), '&', '=', {
+            encodeURIComponent: function (v) {
+                return encodeURIComponent(v).replaceAll('%3A', ':').replaceAll('%2F', '/');
+            },
+        });
+        return url.slice(0, i) + s;
+    }
+    return url;
+}
+
+module.exports = urlclean;

package/src/urlclean.test.js

@@ -0,0 +1,18 @@
+const assert = require('assert');
+const urlclean = require('./urlclean');
+
+const items = [
+    ['', ''],
+    ['http://127.0.0.1:3000/proxy', 'http://127.0.0.1:3000/proxy'],
+    ['http://127.0.0.1:3000/proxy?url=http://127.0.0.1:3000/static/&redirects=2', 'http://127.0.0.1:3000/proxy?url=http://127.0.0.1:3000/static/&redirects=2'],
+    ['http://127.0.0.1:3000/proxy?url=http%3A%2F%2F127.0.0.1%3A3000%2Fstatic%2F&redirects=1', 'http://127.0.0.1:3000/proxy?url=http://127.0.0.1:3000/static/&redirects=1'],
+];
+
+describe('urlclean', function () {
+    items.forEach(function ([input, expected]) {
+        it(input, function () {
+            const actual = urlclean(input);
+            assert.strictEqual(actual, expected);
+        });
+    });
+});

package/src/wait_while.js

@@ -0,0 +1,26 @@
+const Promise = require('bluebird');
+
+/**
+ * Wait until `fn` returns `true`.
+ */
+function wait_while(fn)
+{
+    return new Promise(function (resolve, reject) {
+        setTimeout(tick, 1);
+        async function tick() {
+            try {
+                if (await fn()) {
+                    setTimeout(tick, 1);
+                }
+                else {
+                    resolve();
+                }
+            }
+            catch (error) {
+                reject(error);
+            }
+        }
+    });
+}
+
+module.exports = wait_while;