@ezs/basics 1.16.0 → 1.18.0

package/CHANGELOG.md

@@ -3,6 +3,39 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [1.18.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.17.1...@ezs/basics@1.18.0) (2022-06-21)
+
+
+### Features
+
+* 🎸 add [FILESave] ([2e7dfd7](https://github.com/Inist-CNRS/ezs/commit/2e7dfd76f769418a56f7b9b6cb23d156f37b6789))
+
+
+
+
+
+## [1.17.1](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.17.0...@ezs/basics@1.17.1) (2022-04-01)
+
+
+### Bug Fixes
+
+* erratic error with store ([a26febc](https://github.com/Inist-CNRS/ezs/commit/a26febc4fe7bc0a66a7d32781dc6ef175f707f0a))
+
+
+
+
+
+# [1.17.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.16.0...@ezs/basics@1.17.0) (2022-03-25)
+
+
+### Features
+
+* 🎸 add retry feature ([166c913](https://github.com/Inist-CNRS/ezs/commit/166c913302b4d305ea34d3b84f9b8be6b5188cbb))
+
+
+
+
+
 # [1.16.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.15.5...@ezs/basics@1.16.0) (2022-02-07)
 
 

package/README.md

@@ -18,6 +18,7 @@ npm install @ezs/basics
 -   [CSVObject](#csvobject)
 -   [CSVParse](#csvparse)
 -   [CSVString](#csvstring)
+-   [FILESave](#filesave)
 -   [INIString](#inistring)
 -   [JSONParse](#jsonparse)
 -   [JSONString](#jsonstring)
@@ -189,6 +190,43 @@ a;b;c
 
 Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
 
+### FILESave
+
+Take data, convert it to buffer and append it to file
+
+##### Example
+
+Input:
+
+```json
+[
+  {"a": "a"},
+  {"a": "b"},
+  {"a": "c" }
+]
+```
+
+Script:
+
+```ini
+[FILESave]
+location = /tmp
+identifier = toto
+```
+
+Output:
+
+```json
+["/tmp/truc.json"]
+```
+
+#### Parameters
+
+-   `location` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Directory location (optional, default `TMPDIR`)
+-   `identifier` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)?** File name
+
+Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
+
 ### INIString
 
 Take `Object` and generate INI
@@ -548,6 +586,7 @@ Useful to send JSON data to an API and get results.
 -   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of URL (optional, default `false`)
 -   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors (optional, default `false`)
+-   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
@@ -566,6 +605,7 @@ Or if no target is specified, the output will be the returned content of URL.
 -   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** timeout in milliseconds (optional, default `1000`)
 -   `mimetype` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** mimetype for value of path  (if presents) (optional, default `"application/json"`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** ignore all errors, the target field will remain undefined (optional, default `false`)
+-   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
@@ -660,6 +700,7 @@ Output:
 -   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose the path to split JSON result (optional, default `"*"`)
 -   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
+-   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 

package/lib/file-save.js

@@ -0,0 +1,81 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = FILESave;
+
+var _fs = require("fs");
+
+var _path = _interopRequireDefault(require("path"));
+
+var _os = require("os");
+
+var _pathExists = _interopRequireDefault(require("path-exists"));
+
+var _makeDir = _interopRequireDefault(require("make-dir"));
+
+var _streamWrite = _interopRequireDefault(require("stream-write"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Take data, convert it to buffer and append it to file
+ *
+ * #### Example
+ *
+ * Input:
+ *
+ * ```json
+ * [
+ *   {"a": "a"},
+ *   {"a": "b"},
+ *   {"a": "c" }
+ * ]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [FILESave]
+ * location = /tmp
+ * identifier = toto
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * ["/tmp/truc.json"]
+ * ```
+ *
+ * @name FILESave
+ * @param {String} [location=TMPDIR] Directory location
+ * @param {Number} [identifier] File name
+ * @returns {Object}
+ */
+async function FILESave(data, feed) {
+  if (!this.handle) {
+    const identifier = String(this.getParam('identifier'));
+    const location = this.getParam('location', (0, _os.tmpdir)());
+
+    if (!_pathExists.default.sync(location)) {
+      _makeDir.default.sync(location);
+    }
+
+    this.filename = _path.default.resolve(location, identifier);
+    this.handle = (0, _fs.createWriteStream)(this.filename);
+  }
+
+  if (this.isLast()) {
+    this.handle.close();
+    return (0, _fs.lstat)(this.filename, (err, stat) => {
+      feed.write({
+        filename: this.filename,
+        ...stat
+      });
+      return feed.close();
+    });
+  }
+
+  (0, _streamWrite.default)(this.handle, Buffer.from(String(data)), () => feed.end());
+}

package/lib/index.js

@@ -53,6 +53,8 @@ var _zipExtract = _interopRequireDefault(require("./zip-extract"));
 
 var _iniString = _interopRequireDefault(require("./ini-string"));
 
+var _fileSave = _interopRequireDefault(require("./file-save"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 const funcs = {
@@ -80,6 +82,7 @@ const funcs = {
   TXTZip: _txtZip.default,
   ZIPExtract: _zipExtract.default,
   INIString: _iniString.default,
+  FILESave: _fileSave.default,
   // aliases
   bufferify: _bufObject.default.BUFObject,
   concat: _txtConcat.default.TXTConcat,

package/lib/request.js

@@ -0,0 +1,23 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const request = (url, parameters) => async () => {
+  const response = await (0, _fetchWithProxy.default)(url, parameters);
+
+  if (!response.ok) {
+    throw new Error(response.statusText);
+  }
+
+  return response;
+};
+
+var _default = request;
+exports.default = _default;

package/lib/url-connect.js

@@ -13,10 +13,12 @@ var _streamWrite = _interopRequireDefault(require("stream-write"));
 
 var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
 
-var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
-
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 
+var _asyncRetry = _interopRequireDefault(require("async-retry"));
+
+var _request = _interopRequireDefault(require("./request"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 /**
@@ -31,10 +33,12 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * @param {String} [json=false] Parse as JSON the content of URL
  * @param {Number} [timeout=1000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors
+ * @param {Number} [retries=5] The maximum amount of times to retry the connection
  * @returns {Object}
  */
-function URLConnect(data, feed) {
+async function URLConnect(data, feed) {
   const url = this.getParam('url');
+  const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
   const json = this.getParam('json', true);
   const {
@@ -46,29 +50,23 @@ function URLConnect(data, feed) {
     const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
     const controller = new _nodeAbortController.default();
     this.input = ezs.createStream(ezs.objectMode());
-    this.whenReady = (0, _fetchWithProxy.default)(url, {
+    const output = ezs.createStream(ezs.objectMode());
+    this.whenFinish = feed.flow(output);
+    (0, _streamWrite.default)(this.input, data, () => feed.end());
+    const bodyIn = this.input.pipe(ezs('dump')).pipe(ezs.toBuffer());
+    const parameters = {
       method: 'POST',
-      body: this.input.pipe(ezs('dump')).pipe(ezs.toBuffer()),
+      body: bodyIn,
       timeout,
       headers,
       signal: controller.signal
-    }).then(({
-      body,
-      status,
-      statusText
-    }) => {
-      if (status !== 200) {
-        const msg = `Received status code ${status} (${statusText})`;
-        throw new Error(msg);
-      }
+    };
+    const options = {
+      retries
+    };
 
-      const output = json ? body.pipe(_JSONStream.default.parse('*')) : body;
-      output.once('error', () => controller.abort());
-      this.whenFinish = feed.flow(output);
-      return Promise.resolve(true);
-    }).catch(e => {
+    const onError = e => {
       controller.abort();
-      this.whenFinish = Promise.resolve(true);
 
       if (!noerror) {
         (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLConnect] <${e}>`);
@@ -77,13 +75,26 @@ function URLConnect(data, feed) {
         (0, _debug.default)('ezs')(`Ignore item #${this.getIndex()} [URLConnect] <${e}>`);
       }
 
-      return Promise.resolve(true);
-    });
+      output.end();
+    };
+
+    try {
+      const response = await (0, _asyncRetry.default)((0, _request.default)(url, parameters), options);
+      const bodyOut = json ? response.body.pipe(_JSONStream.default.parse('*')) : response.body;
+      bodyOut.once('error', onError);
+      bodyOut.pipe(output);
+    } catch (e) {
+      onError(e);
+    }
+
+    ;
+    return;
   }
 
   if (this.isLast()) {
-    this.whenReady.finally(() => this.whenFinish.finally(() => feed.close()));
-    return this.input.end();
+    this.input.end();
+    this.whenFinish.finally(() => feed.close());
+    return;
   }
 
   (0, _streamWrite.default)(this.input, data, () => feed.end());

package/lib/url-fetch.js

@@ -13,10 +13,12 @@ var _lodash2 = _interopRequireDefault(require("lodash.set"));
 
 var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
 
-var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
-
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 
+var _asyncRetry = _interopRequireDefault(require("async-retry"));
+
+var _request = _interopRequireDefault(require("./request"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 /**
@@ -32,6 +34,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * @param {Number} [timeout=1000] timeout in milliseconds
  * @param {String} [mimetype="application/json"] mimetype for value of path  (if presents)
  * @param {Boolean} [noerror=false] ignore all errors, the target field will remain undefined
+ * @param {Number} [retries=5] The maximum amount of times to retry the connection
  * @returns {Object}
  */
 async function URLFetch(data, feed) {
@@ -43,6 +46,7 @@ async function URLFetch(data, feed) {
   const path = this.getParam('path');
   const target = this.getParam('target');
   const json = Boolean(this.getParam('json', false));
+  const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
   const timeout = Number(this.getParam('timeout')) || 1000;
   const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
@@ -55,6 +59,9 @@ async function URLFetch(data, feed) {
     headers,
     signal: controller.signal
   };
+  const options = {
+    retries
+  };
 
   if (body) {
     (0, _lodash2.default)(parameters, 'method', 'POST');
@@ -63,13 +70,7 @@ async function URLFetch(data, feed) {
   }
 
   try {
-    const response = await (0, _fetchWithProxy.default)(url, parameters);
-
-    if (!response.ok) {
-      const msg = `Received status code ${response.status} (${response.statusText})`;
-      throw new Error(msg);
-    }
-
+    const response = await (0, _asyncRetry.default)((0, _request.default)(url, parameters), options);
     const func = json ? 'json' : 'text';
     const value = await response[func]();
 

package/lib/url-stream.js

@@ -13,10 +13,12 @@ var _nodeAbortController = _interopRequireDefault(require("node-abort-controller
 
 var _JSONStream = _interopRequireDefault(require("JSONStream"));
 
-var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
-
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 
+var _asyncRetry = _interopRequireDefault(require("async-retry"));
+
+var _request = _interopRequireDefault(require("./request"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 /**
@@ -82,6 +84,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * @param {String} [path="*"] choose the path to split JSON result
  * @param {Number} [timeout=1000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
+ * @param {Number} [retries=5] The maximum amount of times to retry the connection
  * @returns {Object}
  */
 async function URLStream(data, feed) {
@@ -91,32 +94,26 @@ async function URLStream(data, feed) {
 
   const url = this.getParam('url');
   const path = this.getParam('path', '*');
+  const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
   const timeout = Number(this.getParam('timeout')) || 1000;
   const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
   const cURL = new _url.URL(url || data);
   const controller = new _nodeAbortController.default();
+  const parameters = {
+    timeout,
+    headers,
+    signal: controller.signal
+  };
+  const options = {
+    retries
+  };
 
   if (url) {
     cURL.search = new _url.URLSearchParams(data);
   }
 
-  try {
-    const response = await (0, _fetchWithProxy.default)(cURL.href, {
-      timeout,
-      headers,
-      signal: controller.signal
-    });
-
-    if (!response.ok) {
-      const msg = `Received status code ${response.status} (${response.statusText})`;
-      throw new Error(msg);
-    }
-
-    const output = path ? response.body.pipe(_JSONStream.default.parse(path)) : response.body;
-    output.once('error', () => controller.abort());
-    await feed.flow(output);
-  } catch (e) {
+  const onError = e => {
     controller.abort();
 
     if (noerror) {
@@ -126,5 +123,14 @@ async function URLStream(data, feed) {
 
     (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLStream] <${e}>`);
     return feed.send(e);
+  };
+
+  try {
+    const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
+    const output = path ? response.body.pipe(_JSONStream.default.parse(path)) : response.body;
+    output.once('error', onError);
+    await feed.flow(output);
+  } catch (e) {
+    onError(e);
   }
 }

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "1.16.0",
+  "version": "1.18.0",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
     "JSONStream": "1.3.5",
+    "async-retry": "1.3.3",
     "better-https-proxy-agent": "1.0.9",
     "csv-string": "3.2.0",
     "debug": "4.3.3",
@@ -17,9 +18,11 @@
     "lodash.mapvalues": "4.6.0",
     "lodash.set": "4.3.2",
     "lodash.zipobject": "4.1.3",
+    "make-dir": "3.1.0",
     "micromatch": "4.0.4",
     "node-abort-controller": "1.1.0",
     "parse-headers": "2.0.4",
+    "path-exists": "4.0.0",
     "stream-write": "1.0.1",
     "tmp-filepath": "2.0.0",
     "unzipper": "0.10.11",
@@ -33,7 +36,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "b0ced7e413e748195400608f9d3dc33d4487f112",
+  "gitHead": "346fbfb891990dda7dd2d445e1f32ab2276c7020",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"