@ezs/basics 1.20.0 → 1.22.1

package/CHANGELOG.md

@@ -3,6 +3,44 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.22.1](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.22.0...@ezs/basics@1.22.1) (2022-09-07)
+
+
+### Bug Fixes
+
+* 🐛 stop loop (and refactoring) ([2c162aa](https://github.com/Inist-CNRS/ezs/commit/2c162aa64a4603f79c5bdddbca8b58272278f745))
+
+
+
+
+
+# [1.22.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.21.0...@ezs/basics@1.22.0) (2022-09-05)
+
+
+### Bug Fixes
+
+* use location to check filename ([cb9c6e2](https://github.com/Inist-CNRS/ezs/commit/cb9c6e2f42d12c287d806b4ccebf8a4f8607e755))
+
+
+### Features
+
+* 🎸 add file load ([a06c221](https://github.com/Inist-CNRS/ezs/commit/a06c221c3463b4299383db5c613f5cabc687961c))
+
+
+
+
+
+# [1.21.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.20.0...@ezs/basics@1.21.0) (2022-07-11)
+
+
+### Features
+
+* 🎸 add target option to URLRequest ([3861c5e](https://github.com/Inist-CNRS/ezs/commit/3861c5e10b4dcfd060b49fa9b23f2bcf98d8c942))
+
+
+
+
+
 # [1.20.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.19.0...@ezs/basics@1.20.0) (2022-06-24)
 
 

package/README.md

@@ -18,6 +18,7 @@ npm install @ezs/basics
 -   [CSVObject](#csvobject)
 -   [CSVParse](#csvparse)
 -   [CSVString](#csvstring)
+-   [FILELoad](#fileload)
 -   [FILESave](#filesave)
 -   [INIString](#inistring)
 -   [JSONParse](#jsonparse)
@@ -192,6 +193,41 @@ a;b;c
 
 Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
 
+### FILELoad
+
+Take `Object` containing filename et throw content by chunk
+
+```json
+[ fi1e1.csv, file2.csv ]
+```
+
+Script:
+
+```ini
+[use]
+plugin = analytics
+plugin = basics
+
+[FILELoad]
+location = /tmp
+[CSVParse]
+```
+
+Output:
+
+```json
+[
+(...)
+]
+```
+
+#### Parameters
+
+-   `location` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Directory location (optional, default `TMPDIR`)
+-   `compress` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Enable gzip compression (optional, default `false`)
+
+Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
+
 ### FILESave
 
 Take data, convert it to buffer and append it to file
@@ -723,7 +759,7 @@ Output:
 ```json
 [
      {
-         "result": "a" 
+         "result": "a"
      }
  ]
 ```
@@ -732,9 +768,11 @@ Output:
 
 -   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch
 -   `json` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** parse result as json (optional, default `true`)
+-   `target` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** choose the key to set
 -   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
 -   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
+-   `insert` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** a header response value in the result
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 

package/lib/file-load.js

@@ -0,0 +1,73 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = FILELoad;
+
+var _zlib = require("zlib");
+
+var _fs = require("fs");
+
+var _path = require("path");
+
+var _os = require("os");
+
+/**
+ * Take `Object` containing filename et throw content by chunk
+ *
+ * ```json
+ * [ fi1e1.csv, file2.csv ]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [use]
+ * plugin = analytics
+ * plugin = basics
+ *
+ * [FILELoad]
+ * location = /tmp
+ * [CSVParse]
+ *
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [
+ * (...)
+ * ]
+ * ```
+ *
+ * @name FILELoad
+ * @param {String} [location=TMPDIR] Directory location
+ * @param {Boolean} [compress=false] Enable gzip compression
+ * @returns {Object}
+ */
+function FILELoad(data, feed) {
+  if (this.isLast()) {
+    feed.close();
+    return;
+  }
+
+  const cwd = process.cwd();
+  const tpd = (0, _os.tmpdir)();
+  const location = this.getParam('location', tpd);
+  const compress = this.getParam('compress', false);
+  const locations = [cwd, tpd, location];
+  const file = locations.filter(Boolean).map(dir => (0, _path.resolve)(dir, String(data).trim())).filter(fil => (0, _fs.existsSync)(fil)).shift();
+
+  if (!file) {
+    feed.end();
+    return;
+  }
+
+  if (compress) {
+    feed.flow((0, _fs.createReadStream)(file).pipe((0, _zlib.createGunzip)()));
+    return;
+  }
+
+  feed.flow((0, _fs.createReadStream)(file));
+}

package/lib/file-save.js

@@ -19,6 +19,8 @@ var _makeDir = _interopRequireDefault(require("make-dir"));
 
 var _streamWrite = _interopRequireDefault(require("stream-write"));
 
+var _debug = _interopRequireDefault(require("debug"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 /**
@@ -56,8 +58,10 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * @param {Boolean} [compress=false] Enable gzip compression
  * @returns {Object}
  */
-async function FILESave(data, feed) {
-  if (!this.handle) {
+function FILESave(data, feed) {
+  const buf = Buffer.from(String(data));
+
+  if (this.isFirst()) {
     const identifier = String(this.getParam('identifier'));
     const location = this.getParam('location', (0, _os.tmpdir)());
     const compress = this.getParam('compress', false);
@@ -66,29 +70,37 @@ async function FILESave(data, feed) {
       _makeDir.default.sync(location);
     }
 
-    if (compress) {
-      this.filename = _path.default.resolve(location, `${identifier}.gz`);
-      this.handle = (0, _zlib.createGzip)();
-      this.handleEnd = this.handle.pipe((0, _fs.createWriteStream)(this.filename));
-    } else {
-      this.filename = _path.default.resolve(location, identifier);
-      this.handle = (0, _fs.createWriteStream)(this.filename);
-      this.handleEnd = this.handle;
-    }
-  }
+    const name = compress ? `${identifier}.gz` : identifier;
 
-  if (this.isLast()) {
-    this.handleEnd.on('close', () => {
-      (0, _fs.lstat)(this.filename, (err, stat) => {
-        feed.write({
-          filename: this.filename,
-          ...stat
+    const filename = _path.default.resolve(location, name);
+
+    this.input = compress ? (0, _zlib.createGzip)() : (0, _fs.createWriteStream)(filename);
+    this.whenFinish = new Promise((resolve, reject) => {
+      const output = compress ? this.input.pipe((0, _fs.createWriteStream)(filename)) : this.input;
+      output.once('error', err => {
+        (0, _debug.default)('ezs')(`WARNING: ${filename} not saved. ${err}`);
+        reject(err);
+      });
+      output.once('close', () => {
+        (0, _debug.default)('ezs')(`${filename} saved.`);
+        (0, _fs.lstat)(filename, (err, stat) => {
+          if (err) {
+            return reject(err);
+          }
+
+          return resolve({
+            filename,
+            ...stat
+          });
         });
-        return feed.close();
       });
     });
-    return this.handle.end();
   }
 
-  (0, _streamWrite.default)(this.handle, Buffer.from(String(data)), () => feed.end());
+  if (this.isLast()) {
+    this.input.end();
+    return this.whenFinish.then(stats => feed.write(stats)).catch(err => feed.stop(err)).finally(() => feed.close());
+  }
+
+  (0, _streamWrite.default)(this.input, buf, () => feed.end());
 }

package/lib/index.js

@@ -59,6 +59,8 @@ var _iniString = _interopRequireDefault(require("./ini-string"));
 
 var _fileSave = _interopRequireDefault(require("./file-save"));
 
+var _fileLoad = _interopRequireDefault(require("./file-load"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 const funcs = {
@@ -89,6 +91,7 @@ const funcs = {
   ZIPExtract: _zipExtract.default,
   INIString: _iniString.default,
   FILESave: _fileSave.default,
+  FILELoad: _fileLoad.default,
   // aliases
   bufferify: _bufObject.default.BUFObject,
   concat: _txtConcat.default.TXTConcat,

package/lib/request.js

@@ -13,7 +13,10 @@ const request = (url, parameters) => async () => {
   const response = await (0, _fetchWithProxy.default)(url, parameters);
 
   if (!response.ok) {
-    throw new Error(response.statusText);
+    const err = new Error(response.statusText);
+    const text = await response.text();
+    err.responseText = text;
+    throw err;
   }
 
   return response;

package/lib/url-fetch.js

@@ -44,7 +44,7 @@ async function URLFetch(data, feed) {
 
   const url = this.getParam('url');
   const path = this.getParam('path');
-  const target = this.getParam('target');
+  const target = [].concat(this.getParam('target')).filter(Boolean).shift();
   const json = Boolean(this.getParam('json', false));
   const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
@@ -74,8 +74,10 @@ async function URLFetch(data, feed) {
     const func = json ? 'json' : 'text';
     const value = await response[func]();
 
-    if (target && typeof target === 'string' && typeof data === 'object') {
-      const result = { ...data
+    if (target) {
+      const result = typeof data === 'object' ? { ...data
+      } : {
+        input: data
       };
       (0, _lodash2.default)(result, target, value);
       return feed.send(result);

package/lib/url-request.js

@@ -5,6 +5,8 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = URLRequest;
 
+var _lodash = _interopRequireDefault(require("lodash.set"));
+
 var _debug = _interopRequireDefault(require("debug"));
 
 var _url = require("url");
@@ -41,7 +43,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * ```json
  * [
  *      {
- *          "result": "a" 
+ *          "result": "a"
  *      }
  *  ]
  * ```
@@ -49,9 +51,11 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * @name URLRequest
  * @param {String} [url] URL to fetch
  * @param {Boolean} [json=true] parse result as json
+ * @param {String} [target] choose the key to set
  * @param {Number} [timeout=1000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
  * @param {Number} [retries=5] The maximum amount of times to retry the connection
+ * @param {String} [insert] a header response value in the result
  * @returns {Object}
  */
 async function URLRequest(data, feed) {
@@ -61,10 +65,12 @@ async function URLRequest(data, feed) {
 
   const url = this.getParam('url');
   const json = Boolean(this.getParam('json', true));
+  const target = [].concat(this.getParam('target')).filter(Boolean).shift();
   const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
   const timeout = Number(this.getParam('timeout')) || 1000;
   const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
+  const inserts = [].concat(this.getParam('insert')).filter(Boolean);
   const cURL = new _url.URL(url || data);
   const controller = new _nodeAbortController.default();
   const parameters = {
@@ -75,7 +81,10 @@ async function URLRequest(data, feed) {
   const options = {
     retries
   };
-  cURL.search = new _url.URLSearchParams(data);
+
+  if (url) {
+    cURL.search = new _url.URLSearchParams(data);
+  }
 
   const onError = e => {
     controller.abort();
@@ -93,7 +102,19 @@ async function URLRequest(data, feed) {
     const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
     const func = json ? 'json' : 'text';
     const value = await response[func]();
-    feed.send(value);
+
+    if (target) {
+      const result = typeof data === 'object' ? { ...data
+      } : {
+        url: data
+      };
+      (0, _lodash.default)(result, target, value);
+      inserts.forEach(i => (0, _lodash.default)(result, i, response.headers.get(i)));
+      return feed.send(result);
+    }
+
+    inserts.forEach(i => (0, _lodash.default)(value, i, response.headers.get(i)));
+    return feed.send(value);
   } catch (e) {
     onError(e);
   }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "1.20.0",
+  "version": "1.22.1",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
@@ -36,7 +36,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "4a9a6877830fb4a769c1aae19d798b7985783760",
+  "gitHead": "ec76c69ceefbb96d50a208cba0a1b3cc9d40c3d1",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"