@ezs/basics 2.6.0 → 2.7.0

package/README.md

@@ -20,6 +20,7 @@ npm install @ezs/basics
 -   [CSVParse](#csvparse)
 -   [CSVString](#csvstring)
 -   [FILELoad](#fileload)
+-   [FILEMerge](#filemerge)
 -   [FILESave](#filesave)
 -   [INIString](#inistring)
 -   [JSONParse](#jsonparse)
@@ -251,6 +252,37 @@ Output:
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
+### FILEMerge
+
+Take `Object` or `Buffer` and throw only one document
+
+```json
+[ fi1e1.csv, file2.csv ]
+```
+
+Script:
+
+```ini
+[use]
+plugin = basics
+
+[FILELoad]
+[FILEMerge]
+[replace]
+path = contentOfFile1AndFile2
+value = self()
+```
+
+Output:
+
+```json
+[
+(...)
+]
+```
+
+Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
+
 ### FILESave
 
 Take data, convert it to buffer and append it to file
@@ -591,6 +623,7 @@ It returns to the output stream
 
 -   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Regex to select the files to extract (optional, default `"**\/*.json"`)
 -   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of each file (optional, default `true`)
+-   `text` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** The content of each file is converted to a string (otherwise it remains a buffer) (optional, default `true`)
 -   `compress` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Enable gzip compression (optional, default `false`)
 
 Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<{id: [String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), value: [String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)}>** 

package/lib/file-merge.js

@@ -0,0 +1,57 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = FILEMerge;
+/**
+ * Take `Object` or `Buffer` and throw only one document
+ *
+ * ```json
+ * [ fi1e1.csv, file2.csv ]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [use]
+ * plugin = basics
+ *
+ * [FILELoad]
+ * [FILEMerge]
+ * [replace]
+ * path = contentOfFile1AndFile2
+ * value = self()
+ *
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [
+ * (...)
+ * ]
+ * ```
+ *
+ * @name FILEMerge
+ * @returns {Object}
+ */
+function FILEMerge(data, feed) {
+  if (this.isFirst()) {
+    this.chunks = [];
+    this.length = 0;
+    this.isBuffer = Buffer.isBuffer(data);
+  }
+  if (this.isLast()) {
+    feed.write(this.isBuffer ? Buffer.concat(this.chunks, this.length) : this.chunks.join(''));
+    feed.close();
+    return;
+  }
+  if (this.isBuffer) {
+    this.length += data.length;
+  } else {
+    this.length = this.chunks.length;
+  }
+  this.chunks.push(data);
+  feed.end();
+}

package/lib/index.js

@@ -37,6 +37,7 @@ var _tarDump = _interopRequireDefault(require("./tar-dump"));
 var _iniString = _interopRequireDefault(require("./ini-string"));
 var _fileSave = _interopRequireDefault(require("./file-save"));
 var _fileLoad = _interopRequireDefault(require("./file-load"));
+var _fileMerge = _interopRequireDefault(require("./file-merge"));
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 const funcs = {
   BUFObject: _bufObject.default,
@@ -72,6 +73,7 @@ const funcs = {
   INIString: _iniString.default,
   FILESave: _fileSave.default,
   FILELoad: _fileLoad.default,
+  FILEMerge: _fileMerge.default,
   // aliases
   bufferify: _bufObject.default.BUFObject,
   concat: _txtConcat.default.TXTConcat,

package/lib/obj-namespaces.js

@@ -4,11 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = OBJNamespaces;
-var _lodash = _interopRequireDefault(require("lodash.zipobject"));
-var _lodash2 = _interopRequireDefault(require("lodash.escaperegexp"));
-var _lodash3 = _interopRequireDefault(require("lodash.mapkeys"));
-var _lodash4 = _interopRequireDefault(require("lodash.mapvalues"));
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+var _lodash = require("lodash");
 /**
  * Take `Object` and throw the same object, all keys parsed to replace namespaces with their prefixes
  *
@@ -78,13 +74,13 @@ function OBJNamespaces(data, feed) {
   if (this.isFirst()) {
     const prefixes = [].concat(this.getParam('prefix')).map(x => String(x).trim()).filter(Boolean);
     const namespaces = [].concat(this.getParam('namespace')).filter(Boolean).slice(0, prefixes.length).map(x => String(x).trim());
-    this.mapping = (0, _lodash.default)(namespaces, prefixes);
-    this.expression = RegExp(Object.keys(this.mapping).map(_lodash2.default).join('|'), 'g');
+    this.mapping = (0, _lodash.zipObject)(namespaces, prefixes);
+    this.expression = RegExp(Object.keys(this.mapping).map(_lodash.escapeRegExp).join('|'), 'g');
     this.references = [].concat(this.getParam('reference')).filter(Boolean).map(x => RegExp(String(x).trim(), 'g'));
   }
-  const result = (0, _lodash3.default)(data, (val, key) => String(key).replace(this.expression, matched => this.mapping[matched]));
+  const result = (0, _lodash.mapKeys)(data, (val, key) => String(key).replace(this.expression, matched => this.mapping[matched]));
   if (this.references.length > 0) {
-    const result1 = (0, _lodash4.default)(result, (value, key) => {
+    const result1 = (0, _lodash.mapValues)(result, (value, key) => {
       if (this.references.some(x => key.search(x) !== -1)) {
         return String(value).replace(this.expression, matched => this.mapping[matched]);
       }

package/lib/tar-dump.js

@@ -9,7 +9,7 @@ var _util = require("util");
 var _fs = require("fs");
 var _tarStream = _interopRequireDefault(require("tar-stream"));
 var _zlib = require("zlib");
-var _lodash = _interopRequireDefault(require("lodash.merge"));
+var _lodash = require("lodash");
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 // Avoid importing from fs/promise to be compatible with node 12
 const readFilePromise = (0, _util.promisify)(_fs.readFile);
@@ -48,7 +48,7 @@ function TARDump(data, feed) {
       'processingMSTime': this.getCumulativeTimeMS()
     };
     const manifestArray = [metadata].concat(this.getParam('manifest', [])).filter(Boolean);
-    const manifest = (0, _lodash.default)(...manifestArray);
+    const manifest = (0, _lodash.merge)(...manifestArray);
     this.pack.entry({
       name: 'manifest.json'
     }, JSON.stringify(manifest, null, '  '));

package/lib/tar-extract.js

@@ -25,6 +25,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * @name TARExtract
  * @param {String} [path="**\/*.json"] Regex to select the files to extract
  * @param {String} [json=true] Parse as JSON the content of each file
+ * @param {Boolean} [text=true] The content of each file is converted to a string (otherwise it remains a buffer)
  * @param {Boolean} [compress=false] Enable gzip compression
  * @returns {{id: String, value: String}[]}
  */
@@ -34,7 +35,8 @@ function TARExtract(data, feed) {
     const {
       ezs
     } = this;
-    const json = this.getParam('json', true);
+    const text = this.getParam('text', true);
+    const json = text ? this.getParam('json', true) : false;
     const compress = this.getParam('compress', false);
     this.input = ezs.createStream(ezs.objectMode());
     this.output = ezs.createStream(ezs.objectMode());
@@ -43,11 +45,21 @@ function TARExtract(data, feed) {
       extract.on('entry', async (header, stream, next) => {
         if (_micromatch.default.isMatch(header.name, filesPatern)) {
           try {
-            const contentRaw = await (0, _getStream.default)(stream);
             if (json) {
-              const contentJson = JSON.parse(contentRaw);
+              const contentText = await (0, _getStream.default)(stream);
+              const contentJson = JSON.parse(contentText);
               return (0, _streamWrite.default)(this.output, contentJson, () => next());
             }
+            if (text) {
+              const contentText = await (0, _getStream.default)(stream);
+              return (0, _streamWrite.default)(this.output, {
+                id: header.name,
+                value: contentText
+              }, () => next());
+            }
+            const contentRaw = await (0, _getStream.default)(stream, {
+              encoding: 'buffer'
+            });
             return (0, _streamWrite.default)(this.output, {
               id: header.name,
               value: contentRaw

package/lib/txt-inflection.js

@@ -4,9 +4,8 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = void 0;
-var _lodash = _interopRequireDefault(require("lodash.get"));
+var _lodash = require("lodash");
 var _inflection = require("inflection");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 const transformer = transformations => str => str && typeof str === 'string' ? (0, _inflection.transform)(str, transformations) : str;
 const TXTInflection = (data, feed, ctx) => {
   if (ctx.isLast()) {
@@ -14,7 +13,7 @@ const TXTInflection = (data, feed, ctx) => {
   }
   const transformations = [].concat(ctx.getParam('transform', [])).filter(Boolean);
   const path = ctx.getParam('path', 'value');
-  const value = (0, _lodash.default)(data, path, '');
+  const value = (0, _lodash.get)(data, path, '');
   const process = transformer(transformations);
   const result = Array.isArray(value) ? value.map(item => process(item)) : process(value);
   feed.write({

package/lib/txt-sentences.js

@@ -4,8 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = void 0;
-var _lodash = _interopRequireDefault(require("lodash.get"));
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+var _lodash = require("lodash");
 const UPPER_LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
 const SENTENCE_INIT = '  ';
 const SENTENCE_ENDING = '.?!';
@@ -46,7 +45,7 @@ const TXTSentences = (data, feed, ctx) => {
     return feed.close();
   }
   const path = ctx.getParam('path', 'value');
-  const value = (0, _lodash.default)(data, path);
+  const value = (0, _lodash.get)(data, path);
   const str = Array.isArray(value) ? value.map(item => typeof item === 'string' ? item : '').join(' ') : value;
   const sentences = str ? segmentSentences(str) : [];
   feed.write({

package/lib/url-fetch.js

@@ -5,8 +5,7 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = URLFetch;
 var _debug = _interopRequireDefault(require("debug"));
-var _lodash = _interopRequireDefault(require("lodash.get"));
-var _lodash2 = _interopRequireDefault(require("lodash.set"));
+var _lodash = require("lodash");
 var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 var _asyncRetry = _interopRequireDefault(require("async-retry"));
@@ -43,7 +42,7 @@ async function URLFetch(data, feed) {
   const mimetype = String(this.getParam('mimetype', 'application/json'));
   const controller = new _nodeAbortController.default();
   const key = Array.isArray(path) ? path.shift() : path;
-  const body = (0, _lodash.default)(data, key);
+  const body = (0, _lodash.get)(data, key);
   const parameters = {
     timeout,
     headers,
@@ -53,9 +52,9 @@ async function URLFetch(data, feed) {
     retries
   };
   if (body) {
-    (0, _lodash2.default)(parameters, 'method', 'POST');
-    (0, _lodash2.default)(parameters, 'body', Buffer.isBuffer(body) ? body : JSON.stringify(body));
-    (0, _lodash2.default)(parameters, 'headers.content-type', mimetype);
+    (0, _lodash.set)(parameters, 'method', 'POST');
+    (0, _lodash.set)(parameters, 'body', Buffer.isBuffer(body) ? body : JSON.stringify(body));
+    (0, _lodash.set)(parameters, 'headers.content-type', mimetype);
   }
   try {
     const response = await (0, _asyncRetry.default)((0, _request.default)(url, parameters), options);
@@ -67,7 +66,7 @@ async function URLFetch(data, feed) {
       } : {
         input: data
       };
-      (0, _lodash2.default)(result, target, value);
+      (0, _lodash.set)(result, target, value);
       return feed.send(result);
     }
     return feed.send(value);

package/lib/url-request.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = URLRequest;
-var _lodash = _interopRequireDefault(require("lodash.set"));
+var _lodash = require("lodash");
 var _debug = _interopRequireDefault(require("debug"));
 var _url = require("url");
 var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
@@ -93,11 +93,11 @@ async function URLRequest(data, feed) {
       } : {
         url: data
       };
-      (0, _lodash.default)(result, target, value);
-      inserts.forEach(i => (0, _lodash.default)(result, i, response.headers.get(i)));
+      (0, _lodash.set)(result, target, value);
+      inserts.forEach(i => (0, _lodash.set)(result, i, response.headers.get(i)));
       return feed.send(result);
     }
-    inserts.forEach(i => (0, _lodash.default)(value, i, response.headers.get(i)));
+    inserts.forEach(i => (0, _lodash.set)(value, i, response.headers.get(i)));
     return feed.send(value);
   } catch (e) {
     onError(e);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "2.6.0",
+  "version": "2.7.0",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
@@ -17,13 +17,7 @@
     "get-stream": "6.0.1",
     "higher-path": "1.0.0",
     "inflection": "2.0.1",
-    "lodash.escaperegexp": "4.1.2",
-    "lodash.get": "4.4.2",
-    "lodash.mapkeys": "4.6.0",
-    "lodash.mapvalues": "4.6.0",
-    "lodash.merge": "4.6.2",
-    "lodash.set": "4.3.2",
-    "lodash.zipobject": "4.1.3",
+    "lodash": "4.17.21",
     "make-dir": "4.0.0",
     "micromatch": "4.0.4",
     "node-abort-controller": "1.1.0",
@@ -42,7 +36,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "3c080564bbee1b2e2f74fa453fb6781ad5aeb145",
+  "gitHead": "0cbe93a845e6d4fe89501d3d7b4d2bd0f77f0297",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"