@vbarbarosh/node-helpers 1.1.4 → 2.0.1

package/src/fs_mkdirp.js

@@ -1,26 +1,32 @@
-import fs_lstat from './fs_lstat';
-import fs_mkdir from './fs_mkdir';
-import fs_path_dirname from './fs_path_dirname';
+const fs_exists = require('./fs_exists');
+const fs_mkdir = require('./fs_mkdir');
+const fs_path_dirname = require('./fs_path_dirname');
 
 async function fs_mkdirp(pathname)
 {
     const parents = [];
 
     for (let p = pathname; p && p != '/'; p = fs_path_dirname(p)) {
-        try {
-            await fs_lstat(p);
+        if (await fs_exists(p)) {
             break;
         }
-        catch (error) {
-            parents.unshift(p);
-        }
+        parents.push(p);
     }
 
-    for (let i = 0, end = parents.length; i < end; ++i) {
-        await fs_mkdir(parents[i]);
+    while (parents.length) {
+        try {
+            await fs_mkdir(parents.pop());
+        }
+        catch (error) {
+            // Error: EEXIST: file already exists, mkdir [...]
+            if (error.code == 'EEXIST') {
+                break;
+            }
+            throw error;
+        }
     }
 
     return pathname;
 }
 
-export default fs_mkdirp;
+module.exports = fs_mkdirp;

package/src/fs_mkdirp.test.js

@@ -0,0 +1,15 @@
+const assert = require('assert');
+const fs_fi = require('./fs_fi');
+const fs_mkdirp = require('./fs_mkdirp');
+const fs_path_resolve = require('./fs_path_resolve');
+const fs_tempdir = require('./fs_tempdir');
+
+describe('fs_mkdirp', function () {
+    it('should accept empty array', async function () {
+        await fs_tempdir(async function (d) {
+            const tmp = fs_path_resolve(d, 'a/b/c/d/e');
+            await fs_mkdirp(tmp);
+            assert(await fs_fi(tmp).then(v => v.isDirectory()));
+        });
+    });
+});

package/src/fs_path_basename.js

@@ -1,8 +1,8 @@
-import path from 'path';
+const path = require('path');
 
 function fs_path_basename(p)
 {
     return path.basename(p);
 }
 
-export default fs_path_basename;
+module.exports = fs_path_basename;

package/src/fs_path_dirname.js

@@ -1,8 +1,8 @@
-import path from 'path';
+const path = require('path');
 
 function fs_path_dirname(p)
 {
     return path.dirname(p);
 }
 
-export default fs_path_dirname;
+module.exports = fs_path_dirname;

package/src/fs_path_join.js

@@ -1,8 +1,8 @@
-import path from 'path';
+const path = require('path');
 
 function fs_path_join(...parts)
 {
     return path.join(...parts);
 }
 
-export default fs_path_join;
+module.exports = fs_path_join;

package/src/fs_path_resolve.js

@@ -1,8 +1,8 @@
-import path from 'path';
+const path = require('path');
 
 function fs_path_resolve(...parts)
 {
     return path.resolve(...parts);
 }
 
-export default fs_path_resolve;
+module.exports = fs_path_resolve;

package/src/fs_path_sanitize.js

@@ -1,8 +1,8 @@
-import sanitize_filename from 'sanitize-filename';
+const sanitize_filename = require('sanitize-filename');
 
 function fs_path_sanitize(filename)
 {
     return sanitize_filename(filename);
 }
 
-export default fs_path_sanitize;
+module.exports = fs_path_sanitize;

package/src/fs_path_tempdir.js

@@ -1,8 +1,8 @@
-import os from 'os';
+const os = require('os');
 
 function fs_path_tempdir()
 {
     return os.tmpdir();
 }
 
-export default fs_path_tempdir;
+module.exports = fs_path_tempdir;

package/src/fs_read.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import util from 'util';
+const fs = require('fs');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
-export default util.promisify(fs.readFile);
+module.exports = util.promisify(fs.readFile);

package/src/fs_read_json.js

@@ -1,4 +1,4 @@
-import fs_read from './fs_read';
+const fs_read = require('./fs_read');
 
 // XXX Should support catchReturn, e.g.
 //     fs_read_json(file).catchReturn(default_value)
@@ -7,4 +7,4 @@ function fs_read_json(file)
     return fs_read(file, {encoding: 'utf8'}).then(JSON.parse);
 }
 
-export default fs_read_json;
+module.exports = fs_read_json;

package/src/fs_read_stream.js

@@ -1 +1,3 @@
-export {createReadStream as default} from 'fs';
+const fs = require('fs');
+
+module.exports = fs.createReadStream;

package/src/fs_read_utf8.js

@@ -1,8 +1,8 @@
-import fs_read from './fs_read';
+const fs_read = require('./fs_read');
 
 async function fs_read_utf8(file)
 {
     return await fs_read(file, {encoding: 'utf8'});
 }
 
-export default fs_read_utf8;
+module.exports = fs_read_utf8;

package/src/fs_readdir.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import util from 'util';
+const fs = require('fs');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
-export default util.promisify(fs.readdir);
+module.exports = util.promisify(fs.readdir);

package/src/fs_readlink.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import util from 'util';
+const fs = require('fs');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
-export default util.promisify(fs.readlink);
+module.exports = util.promisify(fs.readlink);

package/src/fs_rename.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import util from 'util';
+const fs = require('fs');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
-export default util.promisify(fs.rename);
+module.exports = util.promisify(fs.rename);

package/src/fs_rm.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import util from 'util';
+const fs = require('fs');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
-export default util.promisify(fs.unlink);
+module.exports = util.promisify(fs.unlink);

package/src/fs_rmdir.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import util from 'util';
+const fs = require('fs');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
-export default util.promisify(fs.rmdir);
+module.exports = util.promisify(fs.rmdir);

package/src/fs_rmrf.js

@@ -1,9 +1,9 @@
-import Promise from 'bluebird';
-import fs_fi from './fs_fi';
-import fs_rm from './fs_rm';
-import fs_rmdir from './fs_rmdir';
-import fs_path_resolve from './fs_path_resolve';
-import fs_readdir from './fs_readdir';
+const Promise = require('bluebird');
+const fs_fi = require('./fs_fi');
+const fs_path_resolve = require('./fs_path_resolve');
+const fs_readdir = require('./fs_readdir');
+const fs_rm = require('./fs_rm');
+const fs_rmdir = require('./fs_rmdir');
 
 async function fs_rmrf(pathname)
 {
@@ -22,4 +22,4 @@ async function fs_rmrf(pathname)
     }
 }
 
-export default fs_rmrf;
+module.exports = fs_rmrf;

package/src/fs_size.js

@@ -1,8 +1,8 @@
-import fs_fi from './fs_fi';
+const fs_fi = require('./fs_fi');
 
 async function fs_size(file)
 {
     return fs_fi(file).then(v => v.size);
 }
 
-export default fs_size;
+module.exports = fs_size;

package/src/fs_tempdir.js

@@ -1,8 +1,8 @@
-import fs from 'fs';
-import fs_path_resolve from './fs_path_resolve';
-import fs_path_tempdir from './fs_path_tempdir';
-import fs_rmrf from './fs_rmrf';
-import util from 'util';
+const fs = require('fs');
+const fs_path_resolve = require('./fs_path_resolve');
+const fs_path_tempdir = require('./fs_path_tempdir');
+const fs_rmrf = require('./fs_rmrf');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
 const mkdtemp = util.promisify(fs.mkdtemp);
@@ -39,4 +39,4 @@ async function fs_tempdir(fn)
     }
 }
 
-export default fs_tempdir;
+module.exports = fs_tempdir;

package/src/fs_write.js

@@ -1,5 +1,5 @@
-import fs from 'fs';
-import util from 'util';
+const fs = require('fs');
+const util = require('util');
 
 // https://nodejs.org/api/util.html#util_util_promisify_original
-export default util.promisify(fs.writeFile);
+module.exports = util.promisify(fs.writeFile);

package/src/fs_write_stream.js

@@ -1 +1,3 @@
-export {createWriteStream as default} from 'fs';
+const fs = require('fs');
+
+module.exports = fs.createWriteStream;

package/src/http_delete.js

@@ -1,8 +1,8 @@
-import axios from 'axios';
+const axios = require('axios');
 
 function http_delete(url, options)
 {
     return axios.delete(url, {responseType: 'json', ...options}).then(v => v.data);
 }
 
-export default http_delete;
+module.exports = http_delete;

package/src/http_get_blob.js

@@ -1,8 +1,8 @@
-import axios from 'axios';
+const axios = require('axios');
 
 function http_get_blob(url, options)
 {
     return axios.get(url, {responseType: 'blob', ...options}).then(v => v.data);
 }
 
-export default http_get_blob;
+module.exports = http_get_blob;

package/src/http_get_buffer.js

@@ -1,8 +1,8 @@
-import axios from 'axios';
+const axios = require('axios');
 
 function http_get_buffer(url, options)
 {
     return axios.get(url, {responseType: 'arraybuffer', ...options}).then(v => v.data);
 }
 
-export default http_get_buffer;
+module.exports = http_get_buffer;

package/src/http_get_file.js

@@ -1,9 +1,9 @@
-import * as fs from 'fs';
-import * as stream from 'stream';
-import axios from 'axios';
-import {promisify} from 'util';
+const axios = require('axios');
+const fs = require('fs');
+const stream = require('stream');
+const util = require('util');
 
-const stream_finished = promisify(stream.finished);
+const stream_finished = util.promisify(stream.finished);
 
 // https://stackoverflow.com/a/61269447
 function http_get_file(url, out_file, options)
@@ -14,4 +14,4 @@ function http_get_file(url, out_file, options)
     });
 }
 
-export default http_get_file;
+module.exports = http_get_file;

package/src/http_get_json.js

@@ -1,8 +1,8 @@
-import axios from 'axios';
+const axios = require('axios');
 
 function http_get_json(url, options)
 {
     return axios.get(url, {responseType: 'json', ...options}).then(v => v.data);
 }
 
-export default http_get_json;
+module.exports = http_get_json;

package/src/http_patch_json.js

@@ -1,8 +1,8 @@
-import axios from 'axios';
+const axios = require('axios');
 
 function http_patch_json(url, body, options)
 {
     return axios.patch(url, body, options).then(v => v.data);
 }
 
-export default http_patch_json;
+module.exports = http_patch_json;

package/src/http_post_json.js

@@ -1,8 +1,8 @@
-import axios from 'axios';
+const axios = require('axios');
 
 function http_post_json(url, body, options)
 {
     return axios.post(url, body, options).then(v => v.data);
 }
 
-export default http_post_json;
+module.exports = http_post_json;

package/src/http_post_multipart.js

@@ -1,5 +1,5 @@
-import FormData from 'form-data';
-import axios from 'axios';
+const FormData = require('form-data');
+const axios = require('axios');
 
 // Provide examples for the following common tasks:
 //
@@ -60,4 +60,4 @@ function http_post_multipart(url, items, options)
     return Promise.resolve(axios.post(url, form, options)).then(v => v.data);
 }
 
-export default http_post_multipart;
+module.exports = http_post_multipart;

package/src/http_post_urlencoded.js

@@ -1,9 +1,9 @@
-import axios from 'axios';
-import qs from 'querystring';
+const axios = require('axios');
+const qs = require('querystring');
 
 function http_post_urlencoded(url, body, options)
 {
     return axios.post(url, qs.stringify(body), options).then(v => v.data);
 }
 
-export default http_post_urlencoded;
+module.exports = http_post_urlencoded;

package/src/http_put_file.js

@@ -1,7 +1,7 @@
-import fs from 'fs';
-import fs_size from './fs_size';
-import http from 'http';
-import https from 'https';
+const fs = require('fs');
+const fs_size = require('./fs_size');
+const http = require('http');
+const https = require('https');
 
 async function http_put_file(url, file, options = {})
 {
@@ -17,7 +17,7 @@ async function http_put_file(url, file, options = {})
     const total = await fs_size(file);
     return new Promise(function (resolve, reject) {
         const chunks = [];
-        const request = (url.startsWith('http:') ? http : https).request(url, {method: 'PUT', headers: {'Content-Length': total}}, function (response) {
+        const request = (url.startsWith('http:') ? http : https).request(url, {method: 'PUT', headers: {...options.headers, 'Content-Length': total}}, function (response) {
             response.on('error', reject);
             if (options.progress_download) {
                 const total_download = response.headers['content-length'] ? parseInt(response.headers['content-length']) : null;
@@ -57,4 +57,4 @@ async function http_put_file(url, file, options = {})
     });
 }
 
-export default http_put_file;
+module.exports = http_put_file;

package/src/http_put_json.js

@@ -1,8 +1,8 @@
-import axios from 'axios';
+const axios = require('axios');
 
 function http_put_json(url, body, options)
 {
     return axios.put(url, body, options).then(v => v.data);
 }
 
-export default http_put_json;
+module.exports = http_put_json;

package/src/identity.js

@@ -3,4 +3,4 @@ function identity(value)
     return value;
 }
 
-export default identity;
+module.exports = identity;

package/src/ignore.js

@@ -2,4 +2,4 @@ function ignore()
 {
 }
 
-export default ignore;
+module.exports = ignore;

package/src/json_stringify_stable.js

@@ -1,3 +1,3 @@
-import json_stringify_stable from 'json-stable-stringify';
+const json_stringify_stable = require('json-stable-stringify');
 
-export default json_stringify_stable;
+module.exports = json_stringify_stable;

package/src/now_atom.js

@@ -9,4 +9,4 @@ function now_atom()
     return (new Date()).toJSON();
 }
 
-export default now_atom;
+module.exports = now_atom;

package/src/random_int.js

@@ -0,0 +1,6 @@
+function random_int(min, max)
+{
+    return Math.floor(Math.random()*(max - min + 1) + min);
+}
+
+module.exports = random_int;

package/src/redis_connect.js

@@ -0,0 +1,21 @@
+const Redis = require('redis');
+
+/**
+ * tls://11.11.11.11:6378?ip-google=11.11.11.11&ip-external=22.22.22.22&password=xx-xx-xx&ssl[verify_peer]=0&ssl[verify_peer_name]=0
+ *
+ * @param url
+ * @returns {*}
+ */
+function redis_connect(url)
+{
+    if (url.match(/^tls:/)) {
+        return Redis.createClient(url.replace(/^tls:/, 'rediss:'), {
+            tls: {
+                rejectUnauthorized: false,
+            },
+        });
+    }
+    return Redis.createClient(url.replace(/^tcp:/, 'redis:'));
+}
+
+module.exports = redis_connect;

package/src/redis_lshift.js

@@ -0,0 +1,8 @@
+async function redis_lshift(redis, queue, limit = 1)
+{
+    const last = limit - 1;
+    const [items] = await redis.multi().lrange(queue, 0, last).ltrim(queue, limit, -1).exec_p();
+    return items;
+}
+
+module.exports = redis_lshift;

package/src/redis_poll.js

@@ -0,0 +1,106 @@
+const Promise = require('bluebird');
+const random_int = require('./random_int');
+const redis_zshift = require('./redis_zshift');
+
+async function redis_poll(options)
+{
+    const {log, version} = options;
+    const {
+        poll_message,
+        push_response,
+        process_message,
+        log_waiter_begin,
+        log_waiter_end,
+        log_waiter_sleep,
+        log_waiter_rpush,
+        log_waiter_error,
+        log_waiter_error_parse,
+        log_waiter_error_no_uid,
+    } = options;
+
+    log(`[${log_waiter_begin}] v${version}`);
+    try {
+        for (let iter = 1; iter <= 24*60*60; ++iter) {
+
+            // const message_string = await redis.lpop_p(redis_input_queue);
+            const message = await poll_message();
+            if (!message) {
+                const ms = random_int(500, 1500);
+                log(`[${log_waiter_sleep}] ${ms}ms`);
+                await Promise.delay(ms);
+                continue;
+            }
+
+            try {
+                const uid = message.uid;
+
+                if (!uid) {
+                    log(`[${log_waiter_error_no_uid}] uid is missed; skipped`);
+                    continue;
+                }
+                if (!message.expires_at) {
+                    throw new Error('expires_at is missed');
+                }
+
+                // This is necessary because we use signed urls which has limited lifetime.
+                // Basically, `expires_at` tell when signed url will be expired.
+                if (new Date(message.expires_at).getTime() <= new Date().getTime()) {
+                    throw new Error('Expired');
+                }
+
+                const value = await worker(log.spawn(), message, options);
+                await push_response({uid: message.uid, version, type: 'resolve', value});
+
+                log(`[${log_waiter_rpush}]`);
+            }
+            catch (error) {
+                const response = {uid: message.uid, version, type: 'reject', value: `${(error && (error.stack || error.message)) || 'Error N/A'}`};
+                log(`[${log_waiter_error}] ${JSON.stringify({response})}`);
+                await push_response(response);
+            }
+        }
+    }
+    finally {
+        log(`[${log_waiter_end}]`);
+    }
+}
+
+async function worker(log, message, options)
+{
+    const {redis, redis_output_queue, spawn_command, version} = options;
+    const {
+        callback,
+        log_worker_begin,
+        log_worker_end_ok,
+        log_worker_end_error,
+        log_worker_stdout,
+        log_worker_stderr,
+        log_worker_user_friendly_status,
+    } = options;
+
+    log(`[${log_worker_begin}] ${message.uid}`);
+
+    const uid = message.uid;
+
+    try {
+        // GOTCHA
+        // Promise won't update its status until at least one message would
+        // be taken from output queue. Pushing generic "Started...." message
+        // will tell the promise to refresh its status.
+        await user_friendly_status('Started...');
+        const out = await callback({message, log: log.spawn(), user_friendly_status});
+        log(`[${log_worker_end_ok}]`);
+        return out;
+    }
+    catch (error) {
+        log(`[${log_worker_end_error}] ${error.message}`);
+        throw error;
+    }
+
+    async function user_friendly_status(value) {
+        log(`[${log_worker_user_friendly_status}] ${value}`);
+        await redis.rpush_p(redis_output_queue, JSON.stringify({uid, version, type: 'user_friendly_status', value}));
+    }
+}
+
+module.exports = redis_poll;