@ezs/basics 1.17.1 → 1.20.0

package/CHANGELOG.md

@@ -3,6 +3,39 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [1.20.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.19.0...@ezs/basics@1.20.0) (2022-06-24)
+
+
+### Features
+
+* 🎸 add [URLPager] ([59dcbdc](https://github.com/Inist-CNRS/ezs/commit/59dcbdca0c33eaf2ebf5a195de153cec802d1bc1))
+
+
+
+
+
+# [1.19.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.18.0...@ezs/basics@1.19.0) (2022-06-22)
+
+
+### Features
+
+* 🎸 file save gz ([33df6cd](https://github.com/Inist-CNRS/ezs/commit/33df6cdc40af50c10511cca1449c5edb3a6878f2))
+
+
+
+
+
+# [1.18.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.17.1...@ezs/basics@1.18.0) (2022-06-21)
+
+
+### Features
+
+* 🎸 add [FILESave] ([2e7dfd7](https://github.com/Inist-CNRS/ezs/commit/2e7dfd76f769418a56f7b9b6cb23d156f37b6789))
+
+
+
+
+
 ## [1.17.1](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.17.0...@ezs/basics@1.17.1) (2022-04-01)
 
 

package/README.md

@@ -18,6 +18,7 @@ npm install @ezs/basics
 -   [CSVObject](#csvobject)
 -   [CSVParse](#csvparse)
 -   [CSVString](#csvstring)
+-   [FILESave](#filesave)
 -   [INIString](#inistring)
 -   [JSONParse](#jsonparse)
 -   [JSONString](#jsonstring)
@@ -31,7 +32,9 @@ npm install @ezs/basics
 -   [TXTZip](#txtzip)
 -   [URLConnect](#urlconnect)
 -   [URLFetch](#urlfetch)
+-   [URLPagination](#urlpagination)
 -   [URLParse](#urlparse)
+-   [URLRequest](#urlrequest)
 -   [URLStream](#urlstream)
 -   [URLString](#urlstring)
 -   [XMLConvert](#xmlconvert)
@@ -189,6 +192,44 @@ a;b;c
 
 Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
 
+### FILESave
+
+Take data, convert it to buffer and append it to file
+
+##### Example
+
+Input:
+
+```json
+[
+  {"a": "a"},
+  {"a": "b"},
+  {"a": "c" }
+]
+```
+
+Script:
+
+```ini
+[FILESave]
+location = /tmp
+identifier = toto
+```
+
+Output:
+
+```json
+[{ filename: "/tmp/toto", size: XXX, ... }]
+```
+
+#### Parameters
+
+-   `location` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Directory location (optional, default `TMPDIR`)
+-   `identifier` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** File name
+-   `compress` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Enable gzip compression (optional, default `false`)
+
+Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
+
 ### INIString
 
 Take `Object` and generate INI
@@ -571,6 +612,68 @@ Or if no target is specified, the output will be the returned content of URL.
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
+### URLPagination
+
+Take `Object` and multiple it to make it one object per page
+
+Input:
+
+```json
+[{"q": "a"}]
+```
+
+Script:
+
+```ini
+[URLRequest]
+url = https://api.search.net
+
+[URLPagination]
+total = get('total')
+```
+
+Output:
+
+```json
+[
+     {
+         "q": "a",
+         "total": 22
+         "offset": 0,
+         "pageNumber": 1,
+         "totalPages", 3,
+         "maxPages": 1000,
+         "limit": 10
+     },
+     {
+         "q": "a",
+         "total": 22
+         "offset": 10,
+         "pageNumber": 2,
+         "totalPages", 3,
+         "maxPages": 1000,
+         "limit": 10
+     },
+     {
+         "q": "a",
+         "total": 22
+         "offset": 20,
+         "pageNumber": 3,
+         "totalPages", 3,
+         "maxPages": 1000,
+         "limit": 10
+     }
+ ]
+```
+
+#### Parameters
+
+-   `total` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** total to use for the pagination (optional, default `0`)
+-   `limit` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** limit to use to pagination (optional, default `10`)
+-   `maxPages` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** maxPages to use to pagination (optional, default `1000`)
+
+Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
+
 ### URLParse
 
 Take an URL `String`, parse it and return `Object`.
@@ -598,6 +701,43 @@ See:
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
+### URLRequest
+
+Take `Object` as parameters of URL, throw each chunk from the result
+
+Input:
+
+```json
+[{"q": "a"}]
+```
+
+Script:
+
+```ini
+[URLRequest]
+url = https://api.search.net
+```
+
+Output:
+
+```json
+[
+     {
+         "result": "a" 
+     }
+ ]
+```
+
+#### Parameters
+
+-   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch
+-   `json` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** parse result as json (optional, default `true`)
+-   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
+-   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
+-   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
+
+Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
+
 ### URLStream
 
 Take `String` as URL, throw each chunk from the result or

package/lib/file-save.js

@@ -0,0 +1,94 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = FILESave;
+
+var _fs = require("fs");
+
+var _zlib = require("zlib");
+
+var _path = _interopRequireDefault(require("path"));
+
+var _os = require("os");
+
+var _pathExists = _interopRequireDefault(require("path-exists"));
+
+var _makeDir = _interopRequireDefault(require("make-dir"));
+
+var _streamWrite = _interopRequireDefault(require("stream-write"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Take data, convert it to buffer and append it to file
+ *
+ * #### Example
+ *
+ * Input:
+ *
+ * ```json
+ * [
+ *   {"a": "a"},
+ *   {"a": "b"},
+ *   {"a": "c" }
+ * ]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [FILESave]
+ * location = /tmp
+ * identifier = toto
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [{ filename: "/tmp/toto", size: XXX, ... }]
+ * ```
+ *
+ * @name FILESave
+ * @param {String} [location=TMPDIR] Directory location
+ * @param {String} [identifier] File name
+ * @param {Boolean} [compress=false] Enable gzip compression
+ * @returns {Object}
+ */
+async function FILESave(data, feed) {
+  if (!this.handle) {
+    const identifier = String(this.getParam('identifier'));
+    const location = this.getParam('location', (0, _os.tmpdir)());
+    const compress = this.getParam('compress', false);
+
+    if (!_pathExists.default.sync(location)) {
+      _makeDir.default.sync(location);
+    }
+
+    if (compress) {
+      this.filename = _path.default.resolve(location, `${identifier}.gz`);
+      this.handle = (0, _zlib.createGzip)();
+      this.handleEnd = this.handle.pipe((0, _fs.createWriteStream)(this.filename));
+    } else {
+      this.filename = _path.default.resolve(location, identifier);
+      this.handle = (0, _fs.createWriteStream)(this.filename);
+      this.handleEnd = this.handle;
+    }
+  }
+
+  if (this.isLast()) {
+    this.handleEnd.on('close', () => {
+      (0, _fs.lstat)(this.filename, (err, stat) => {
+        feed.write({
+          filename: this.filename,
+          ...stat
+        });
+        return feed.close();
+      });
+    });
+    return this.handle.end();
+  }
+
+  (0, _streamWrite.default)(this.handle, Buffer.from(String(data)), () => feed.end());
+}

package/lib/index.js

@@ -41,6 +41,10 @@ var _urlFetch = _interopRequireDefault(require("./url-fetch"));
 
 var _urlParse = _interopRequireDefault(require("./url-parse"));
 
+var _urlRequest = _interopRequireDefault(require("./url-request"));
+
+var _urlPagination = _interopRequireDefault(require("./url-pagination"));
+
 var _urlString = _interopRequireDefault(require("./url-string"));
 
 var _urlStream = _interopRequireDefault(require("./url-stream"));
@@ -53,6 +57,8 @@ var _zipExtract = _interopRequireDefault(require("./zip-extract"));
 
 var _iniString = _interopRequireDefault(require("./ini-string"));
 
+var _fileSave = _interopRequireDefault(require("./file-save"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 const funcs = {
@@ -73,6 +79,8 @@ const funcs = {
   JSONParse: _jsonParse.default,
   JSONString: _jsonString.default,
   URLFetch: _urlFetch.default,
+  URLPagination: _urlPagination.default,
+  URLRequest: _urlRequest.default,
   URLParse: _urlParse.default,
   URLString: _urlString.default,
   URLStream: _urlStream.default,
@@ -80,6 +88,7 @@ const funcs = {
   TXTZip: _txtZip.default,
   ZIPExtract: _zipExtract.default,
   INIString: _iniString.default,
+  FILESave: _fileSave.default,
   // aliases
   bufferify: _bufObject.default.BUFObject,
   concat: _txtConcat.default.TXTConcat,

package/lib/url-pager.js

@@ -0,0 +1,179 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = URLPager;
+
+var _debug = _interopRequireDefault(require("debug"));
+
+var _url = require("url");
+
+var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
+
+var _lodash = _interopRequireDefault(require("lodash.get"));
+
+var _parseHeaders = _interopRequireDefault(require("parse-headers"));
+
+var _asyncRetry = _interopRequireDefault(require("async-retry"));
+
+var _request = _interopRequireDefault(require("./request"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Take `Object` as parameters of URL, throw each chunk from the result
+ *
+ *
+ * Input:
+ *
+ * ```json
+ * [{"q": "a"}]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [URLPager]
+ * url = https://api.search.net
+ * path = total
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 0,
+ *          "pageNumber": 1,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      },
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 10,
+ *          "pageNumber": 2,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      },
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 20,
+ *          "pageNumber": 3,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      }
+ *  ]
+ * ```
+ *
+ * #### Example with URLs
+ *
+ * Input:
+ *
+ * ```json
+ * [
+ *   "https://httpbin.org/get?a=a",
+ *   "https://httpbin.org/get?a=b",
+ *   "https://httpbin.org/get?a=c"
+ * ]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [URLPager]
+ * path = .args
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [{"a": "a"}, {"a": "b"}, {"a": "c" }]
+ * ```
+ *
+ * @name URLPager
+ * @param {String} [url] URL to fetch (by default input string is taken)
+ * @param {String} [path=total] choose the path to find the number of result
+ * @param {Number} [timeout=1000] Timeout in milliseconds
+ * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
+ * @param {Number} [retries=5] The maximum amount of times to retry the connection
+ * @returns {Object}
+ */
+async function URLPager(data, feed) {
+  if (this.isLast()) {
+    return feed.close();
+  }
+
+  const url = this.getParam('url');
+  const path = this.getParam('path', 'total');
+  const limit = Number(this.getParam('limit', 10));
+  const maxPages = Number(this.getParam('maxPages', 1000));
+  const retries = Number(this.getParam('retries', 5));
+  const noerror = Boolean(this.getParam('noerror', false));
+  const timeout = Number(this.getParam('timeout')) || 1000;
+  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
+  const cURL = new _url.URL(url || data);
+  const controller = new _nodeAbortController.default();
+  const parameters = {
+    timeout,
+    headers,
+    signal: controller.signal
+  };
+  const options = {
+    retries
+  };
+  cURL.search = new _url.URLSearchParams(data);
+
+  const onError = e => {
+    controller.abort();
+
+    if (noerror) {
+      (0, _debug.default)('ezs')(`Ignore item #${this.getIndex()} [URLPager] <${e}>`);
+      return feed.send(data);
+    }
+
+    (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLPager] <${e}>`);
+    return feed.send(e);
+  };
+
+  try {
+    const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
+    const json = await response.json();
+    const total = (0, _lodash.default)(json, path);
+
+    if (total === 0) {
+      return onError(new Error('No result.'));
+    }
+
+    if (total === undefined) {
+      return onError(new Error('Unexpected response.'));
+    }
+
+    let totalPages = Math.ceil(json.total / limit);
+
+    if (totalPages > maxPages) {
+      totalPages = maxPages;
+    }
+
+    for (let pageNumber = 1; pageNumber <= totalPages; pageNumber += 1) {
+      feed.write({ ...data,
+        offset: (pageNumber - 1) * limit,
+        pageNumber,
+        totalPages,
+        maxPages,
+        limit
+      });
+    }
+
+    feed.end();
+  } catch (e) {
+    onError(e);
+  }
+}

package/lib/url-pagination.js

@@ -0,0 +1,103 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = URLPagination;
+
+/**
+ * Take `Object` and multiple it to make it one object per page
+ *
+ *
+ * Input:
+ *
+ * ```json
+ * [{"q": "a"}]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [URLRequest]
+ * url = https://api.search.net
+ *
+ * [URLPagination]
+ * total = get('total')
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 0,
+ *          "pageNumber": 1,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      },
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 10,
+ *          "pageNumber": 2,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      },
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 20,
+ *          "pageNumber": 3,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      }
+ *  ]
+ * ```
+ *
+ *
+ * @name URLPagination
+ * @param {Number} [total=0] total to use for the pagination
+ * @param {Number} [limit=10] limit to use to pagination
+ * @param {Number} [maxPages=1000] maxPages to use to pagination
+ * @returns {Object}
+ */
+function URLPagination(data, feed) {
+  if (this.isLast()) {
+    return feed.close();
+  }
+
+  const total = Number(this.getParam('total', 0));
+  const limit = Number(this.getParam('limit', 10));
+  const maxPages = Number(this.getParam('maxPages', 1000));
+
+  if (total === 0) {
+    return feed.send(new Error('No result.'));
+  }
+
+  if (Number.isNaN(total)) {
+    return feed.send(new Error('Unexpected response.'));
+  }
+
+  let totalPages = Math.ceil(total / limit);
+
+  if (totalPages > maxPages) {
+    totalPages = maxPages;
+  }
+
+  for (let pageNumber = 1; pageNumber <= totalPages; pageNumber += 1) {
+    feed.write({ ...data,
+      offset: (pageNumber - 1) * limit,
+      pageNumber,
+      totalPages,
+      maxPages,
+      limit
+    });
+  }
+
+  feed.end();
+}

package/lib/url-request.js

@@ -0,0 +1,100 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = URLRequest;
+
+var _debug = _interopRequireDefault(require("debug"));
+
+var _url = require("url");
+
+var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
+
+var _parseHeaders = _interopRequireDefault(require("parse-headers"));
+
+var _asyncRetry = _interopRequireDefault(require("async-retry"));
+
+var _request = _interopRequireDefault(require("./request"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Take `Object` as parameters of URL, throw each chunk from the result
+ *
+ *
+ * Input:
+ *
+ * ```json
+ * [{"q": "a"}]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [URLRequest]
+ * url = https://api.search.net
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [
+ *      {
+ *          "result": "a" 
+ *      }
+ *  ]
+ * ```
+ *
+ * @name URLRequest
+ * @param {String} [url] URL to fetch
+ * @param {Boolean} [json=true] parse result as json
+ * @param {Number} [timeout=1000] Timeout in milliseconds
+ * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
+ * @param {Number} [retries=5] The maximum amount of times to retry the connection
+ * @returns {Object}
+ */
+async function URLRequest(data, feed) {
+  if (this.isLast()) {
+    return feed.close();
+  }
+
+  const url = this.getParam('url');
+  const json = Boolean(this.getParam('json', true));
+  const retries = Number(this.getParam('retries', 5));
+  const noerror = Boolean(this.getParam('noerror', false));
+  const timeout = Number(this.getParam('timeout')) || 1000;
+  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
+  const cURL = new _url.URL(url || data);
+  const controller = new _nodeAbortController.default();
+  const parameters = {
+    timeout,
+    headers,
+    signal: controller.signal
+  };
+  const options = {
+    retries
+  };
+  cURL.search = new _url.URLSearchParams(data);
+
+  const onError = e => {
+    controller.abort();
+
+    if (noerror) {
+      (0, _debug.default)('ezs')(`Ignore item #${this.getIndex()} [URLRequest] <${e}>`);
+      return feed.send(data);
+    }
+
+    (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLRequest] <${e}>`);
+    return feed.send(e);
+  };
+
+  try {
+    const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
+    const func = json ? 'json' : 'text';
+    const value = await response[func]();
+    feed.send(value);
+  } catch (e) {
+    onError(e);
+  }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "1.17.1",
+  "version": "1.20.0",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
@@ -18,9 +18,11 @@
     "lodash.mapvalues": "4.6.0",
     "lodash.set": "4.3.2",
     "lodash.zipobject": "4.1.3",
+    "make-dir": "3.1.0",
     "micromatch": "4.0.4",
     "node-abort-controller": "1.1.0",
     "parse-headers": "2.0.4",
+    "path-exists": "4.0.0",
     "stream-write": "1.0.1",
     "tmp-filepath": "2.0.0",
     "unzipper": "0.10.11",
@@ -34,7 +36,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "0ccde3c9e03a451c95a20372ff41437c9ec214e7",
+  "gitHead": "4a9a6877830fb4a769c1aae19d798b7985783760",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"