@ezs/basics 2.10.2 → 2.12.0

package/README.md

@@ -734,10 +734,11 @@ Useful to send JSON data to an API and get results.
 *   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch
 *   `streaming` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Direct connection to the Object Stream server (disables the retries setting) (optional, default `false`)
 *   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of URL (optional, default `false`)
-*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
+*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `5000`)
 *   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors (optional, default `false`)
 *   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
 *   `encoder` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The statement to encode each chunk to a string (optional, default `dump`)
+*   `method` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The method to use for the HTTP request (optional, default `POST`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
@@ -754,7 +755,7 @@ Or if no target is specified, the output will be the returned content of URL.
 *   `target` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** choose the key to set
 *   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** parse as JSON the content of URL (optional, default `false`)
 *   `dataurl` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** encode content into DATA Url (optional, default `false`)
-*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** timeout in milliseconds (optional, default `1000`)
+*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** timeout in milliseconds (optional, default `5000`)
 *   `mimetype` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** mimetype for value of path  (if presents) (optional, default `"application/json"`)
 *   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** ignore all errors, the target field will remain undefined (optional, default `false`)
 *   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
@@ -882,7 +883,7 @@ Output:
 *   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch
 *   `json` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** parse result as json (optional, default `true`)
 *   `target` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** choose the key to set
-*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
+*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `5000`)
 *   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
 *   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
 *   `insert` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** a header response value in the result
@@ -951,7 +952,7 @@ Output:
 
 *   `url` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** URL to fetch (by default input string is taken)
 *   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** choose the path to split JSON result (optional, default `"*"`)
-*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
+*   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `5000`)
 *   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors, the target field will remain undefined (optional, default `false`)
 *   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
 

package/lib/index.js

@@ -71,13 +71,6 @@ const funcs = {
   INIString: _iniString.default,
   FILESave: _fileSave.default,
   FILELoad: _fileLoad.default,
-  FILEMerge: _fileMerge.default,
-  // aliases
-  bufferify: _bufObject.default.BUFObject,
-  concat: _txtConcat.default.TXTConcat,
-  standardize: _objStandardize.default.OBJStandardize,
-  split: _txtParse.default.TXTParse,
-  segmenter: _txtParse.default.TXTParse
+  FILEMerge: _fileMerge.default
 };
-var _default = exports.default = funcs;
-module.exports = funcs;
+var _default = exports.default = funcs;

package/lib/request.js

@@ -4,10 +4,27 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = void 0;
-var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 const request = (url, parameters) => async bail => {
-  const response = await (0, _fetchWithProxy.default)(url, parameters);
+  const hasBody = parameters.body !== undefined;
+  let response;
+  try {
+    response = await fetch(url, {
+      ...parameters,
+      ...(hasBody && {
+        duplex: 'half'
+      })
+    });
+  } catch (raw) {
+    // Normalise l'erreur quelle que soit la source (undici, DOMException, etc.)
+    const err = raw instanceof Error ? raw : Object.assign(new Error(raw?.message ?? String(raw)), {
+      name: raw?.name ?? 'FetchError',
+      cause: raw
+    });
+    if (err.name === 'AbortError' || err.name === 'TimeoutError') {
+      return bail(err);
+    }
+    throw err;
+  }
   if (!response.ok) {
     // response.status >= 200 && response.status < 300
     const err = new Error(response.statusText);

package/lib/url-connect.js

@@ -5,15 +5,15 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = URLConnect;
 var _JSONStream = _interopRequireDefault(require("JSONStream"));
+var _stream = require("stream");
 var _from = _interopRequireDefault(require("from"));
 var _debug = _interopRequireDefault(require("debug"));
 var _streamWrite = _interopRequireDefault(require("stream-write"));
-var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 var _asyncRetry = _interopRequireDefault(require("async-retry"));
 var _getStream = _interopRequireDefault(require("get-stream"));
-var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const restMethods = ['POST', 'GET', 'DELETE', 'PUT', 'PATCH', 'HEAD', 'OPTIONS', 'TRACE'];
 /**
  * Take an `Object` and send it to an URL.
  *
@@ -25,10 +25,11 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
  * @param {String} [url] URL to fetch
  * @param {String} [streaming=false] Direct connection to the Object Stream server (disables the retries setting)
  * @param {String} [json=false] Parse as JSON the content of URL
- * @param {Number} [timeout=1000] Timeout in milliseconds
+ * @param {Number} [timeout=5000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors
  * @param {Number} [retries=5] The maximum amount of times to retry the connection
  * @param {String} [encoder=dump] The statement to encode each chunk to a string
+ * @param {String} [method=POST] The method to use for the HTTP request
  * @returns {Object}
  */
 async function URLConnect(data, feed) {
@@ -36,13 +37,14 @@ async function URLConnect(data, feed) {
   const streaming = Boolean(this.getParam('streaming', false));
   const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
+  const method = ['POST'].concat(this.getParam('method')).filter(item => restMethods.includes(item)).pop();
   const json = this.getParam('json', true);
   const encoder = this.getParam('encoder', 'dump');
   const {
     ezs
   } = this;
   if (this.isFirst()) {
-    const timeout = Number(this.getParam('timeout')) || 1000;
+    const timeout = Number(this.getParam('timeout', 5000));
     const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
     this.input = ezs.createStream(ezs.objectMode());
     const output = ezs.createStream(ezs.objectMode());
@@ -58,19 +60,23 @@ async function URLConnect(data, feed) {
     }
     const parameters = {
       timeout,
-      method: 'POST',
+      method,
       body: bodyIn,
       headers
     };
+    const controller = new AbortController();
     try {
       await (0, _asyncRetry.default)(async (bail, numberOfTimes) => {
         if (numberOfTimes > 1) {
           (0, _debug.default)('ezs:debug')(`Attempts to reconnect (${numberOfTimes})`);
         }
-        const controller = new _nodeAbortController.default();
-        const response = await (0, _fetchWithProxy.default)(url, {
+        const hasBody = parameters.body !== undefined;
+        const response = await fetch(url, {
           ...parameters,
-          signal: controller.signal
+          ...(hasBody && {
+            duplex: 'half'
+          }),
+          signal: AbortSignal.any([controller.signal, AbortSignal.timeout(timeout)])
         });
         if (!response.ok) {
           const err = new Error(response.statusText);
@@ -78,16 +84,17 @@ async function URLConnect(data, feed) {
           err.responseText = text;
           throw err;
         }
+        const bodyStream = _stream.Readable.fromWeb(response.body);
         if (streaming) {
-          const bodyOut = json ? response.body.pipe(_JSONStream.default.parse('*')) : response.body;
+          const bodyOut = json ? bodyStream.pipe(_JSONStream.default.parse('*')) : bodyStream;
           bodyOut.once('error', e => {
-            controller.abort();
             output.emit('error', e);
+            //controller.abort();
           });
           return bodyOut.pipe(output);
         }
         if (json) {
-          const bodyOutRaw = await (0, _getStream.default)(response.body);
+          const bodyOutRaw = await (0, _getStream.default)(bodyStream);
           if (bodyOutRaw === '') {
             throw new Error('URL returned an empty response');
           }
@@ -99,11 +106,12 @@ async function URLConnect(data, feed) {
           }
           return (0, _from.default)(bodyOutArray).pipe(output);
         }
-        return response.body.pipe(output);
+        return bodyStream.pipe(output);
       }, {
         retries: streaming ? 0 : retries
       });
     } catch (e) {
+      controller.abort();
       if (!noerror) {
         (0, _debug.default)('ezs:warn')(`Break item #${this.getIndex()} [URLConnect]`, ezs.serializeError(e));
         feed.stop(e);

package/lib/url-fetch.js

@@ -6,7 +6,6 @@ Object.defineProperty(exports, "__esModule", {
 exports.default = URLFetch;
 var _debug = _interopRequireDefault(require("debug"));
 var _lodash = require("lodash");
-var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 var _asyncRetry = _interopRequireDefault(require("async-retry"));
 var _request = _interopRequireDefault(require("./request"));
@@ -24,7 +23,7 @@ const createObjectURL = (arrayBuffer, mimeType = 'application/octet-stream') =>
  * @param {String} [target] choose the key to set
  * @param {String} [json=false] parse as JSON the content of URL
  * @param {String} [dataurl=false] encode content into DATA Url
- * @param {Number} [timeout=1000] timeout in milliseconds
+ * @param {Number} [timeout=5000] timeout in milliseconds
  * @param {String} [mimetype="application/json"] mimetype for value of path  (if presents)
  * @param {Boolean} [noerror=false] ignore all errors, the target field will remain undefined
  * @param {Number} [retries=5] The maximum amount of times to retry the connection
@@ -41,16 +40,15 @@ async function URLFetch(data, feed) {
   const dataurl = Boolean(this.getParam('dataurl', false));
   const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
-  const timeout = Number(this.getParam('timeout')) || 1000;
+  const timeout = Number(this.getParam('timeout', 5000));
   const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
   const mimetype = String(this.getParam('mimetype', 'application/json'));
-  const controller = new _nodeAbortController.default();
+  const controller = new AbortController();
   const key = Array.isArray(path) ? path.shift() : path;
   const body = (0, _lodash.get)(data, key);
   const parameters = {
-    timeout,
     headers,
-    signal: controller.signal
+    signal: AbortSignal.any([controller.signal, AbortSignal.timeout(timeout)])
   };
   const options = {
     retries

package/lib/url-request.js

@@ -7,7 +7,6 @@ exports.default = URLRequest;
 var _lodash = require("lodash");
 var _debug = _interopRequireDefault(require("debug"));
 var _url = require("url");
-var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 var _asyncRetry = _interopRequireDefault(require("async-retry"));
 var _request = _interopRequireDefault(require("./request"));
@@ -43,7 +42,7 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
  * @param {String} [url] URL to fetch
  * @param {Boolean} [json=true] parse result as json
  * @param {String} [target] choose the key to set
- * @param {Number} [timeout=1000] Timeout in milliseconds
+ * @param {Number} [timeout=5000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
  * @param {Number} [retries=5] The maximum amount of times to retry the connection
  * @param {String} [insert] a header response value in the result
@@ -58,15 +57,14 @@ async function URLRequest(data, feed) {
   const target = [].concat(this.getParam('target')).filter(Boolean).shift();
   const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
-  const timeout = Number(this.getParam('timeout')) || 1000;
+  const timeout = Number(this.getParam('timeout', 5000));
   const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
   const inserts = [].concat(this.getParam('insert')).filter(Boolean);
   const cURL = new _url.URL(url || data);
-  const controller = new _nodeAbortController.default();
+  const controller = new AbortController();
   const parameters = {
-    timeout,
     headers,
-    signal: controller.signal
+    signal: AbortSignal.any([controller.signal, AbortSignal.timeout(timeout)])
   };
   const options = {
     retries

package/lib/url-stream.js

@@ -5,8 +5,8 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = URLStream;
 var _debug = _interopRequireDefault(require("debug"));
+var _stream = require("stream");
 var _url = require("url");
-var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
 var _JSONStream = _interopRequireDefault(require("JSONStream"));
 var _parseHeaders = _interopRequireDefault(require("parse-headers"));
 var _asyncRetry = _interopRequireDefault(require("async-retry"));
@@ -73,7 +73,7 @@ function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e
  * @name URLStream
  * @param {String} [url] URL to fetch (by default input string is taken)
  * @param {String} [path="*"] choose the path to split JSON result
- * @param {Number} [timeout=1000] Timeout in milliseconds
+ * @param {Number} [timeout=5000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
  * @param {Number} [retries=5] The maximum amount of times to retry the connection
  * @returns {Object}
@@ -89,14 +89,13 @@ async function URLStream(data, feed) {
   const path = this.getParam('path', '*');
   const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
-  const timeout = Number(this.getParam('timeout')) || 1000;
+  const timeout = Number(this.getParam('timeout', 5000));
   const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
   const cURL = new _url.URL(url || data);
-  const controller = new _nodeAbortController.default();
+  const controller = new AbortController();
   const parameters = {
-    timeout,
     headers,
-    signal: controller.signal
+    signal: AbortSignal.any([controller.signal, AbortSignal.timeout(timeout)])
   };
   const options = {
     retries
@@ -105,7 +104,6 @@ async function URLStream(data, feed) {
     cURL.search = new _url.URLSearchParams(data);
   }
   const onError = e => {
-    controller.abort();
     if (noerror) {
       (0, _debug.default)('ezs:info')(`Ignore item #${this.getIndex()} [URLStream]`, ezs.serializeError(e));
       return feed.send(data);
@@ -115,10 +113,12 @@ async function URLStream(data, feed) {
   };
   try {
     const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
-    const output = path ? response.body.pipe(_JSONStream.default.parse(path)) : response.body;
+    const bodyStream = _stream.Readable.fromWeb(response.body);
+    const output = path ? bodyStream.pipe(_JSONStream.default.parse(path)) : bodyStream;
     output.once('error', onError);
     await feed.flow(output);
   } catch (e) {
     onError(e);
+    controller.abort();
   }
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "2.10.2",
+  "version": "2.12.0",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
@@ -16,7 +16,7 @@
     "from": "0.1.7",
     "get-stream": "6.0.1",
     "higher-path": "1.0.0",
-    "lodash": "4.17.21",
+    "lodash": "4.18.1",
     "make-dir": "4.0.0",
     "micromatch": "4.0.8",
     "node-abort-controller": "1.1.0",
@@ -36,7 +36,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "0a1bbe0441bf56a543a006a6d521210b50ee4835",
+  "gitHead": "d495361ad3a11b258c4370e97bfa6899f5c9dfbf",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"

package/lib/fetch.js

@@ -1,74 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = fetch;
-
-var _crossFetch = _interopRequireDefault(require("cross-fetch"));
-
-var _proxyFromEnv = require("proxy-from-env");
-
-var _http = _interopRequireDefault(require("http"));
-
-var _https = _interopRequireDefault(require("https"));
-
-var _betterHttpsProxyAgent = require("better-https-proxy-agent");
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-const DefaultOptions = {
-  keepAlive: true,
-  timeout: 1000,
-  keepAliveMsecs: 500,
-  maxSockets: 200,
-  maxFreeSockets: 5,
-  maxCachedSessions: 500
-};
-
-const chooseAgent = (parsedURL, options) => {
-  const proxyurl = (0, _proxyFromEnv.getProxyForUrl)(parsedURL.href);
-
-  if (proxyurl) {
-    const proxyRequestOptions = new URL(proxyurl);
-    return new _betterHttpsProxyAgent.Agent(options, proxyRequestOptions);
-  }
-
-  if (parsedURL.protocol === 'https:') {
-    return new _https.default.Agent(options);
-  }
-
-  return new _http.default.Agent(options);
-};
-
-function fetch(url, options) {
-  const opts = options || {};
-  const {
-    keepAlive,
-    timeout,
-    keepAliveMsecs,
-    maxSockets,
-    maxFreeSockets,
-    maxCachedSessions
-  } = { ...options,
-    ...DefaultOptions
-  };
-  let agent = chooseAgent(new URL(url), {
-    keepAlive,
-    timeout,
-    keepAliveMsecs,
-    maxSockets,
-    maxFreeSockets,
-    maxCachedSessions
-  });
-  opts.agent = agent;
-
-  if (opts.signal) {
-    opts.signal.addEventListener('abort', () => {
-      agent.destroy();
-      agent = null;
-    });
-  }
-
-  return (0, _crossFetch.default)(url, options);
-}

package/lib/file-load-parquet.js

@@ -1,37 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = FILELoadParquet;
-var _fs = require("fs");
-var _path = require("path");
-var _os = require("os");
-var _higherPath = _interopRequireDefault(require("higher-path"));
-var _hyparquet = require("hyparquet");
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
-/**
- * @name FileLoadParquet
- * @param {String} [location=TMPDIR] Directory location
- * @returns {Object}
- */
-async function FILELoadParquet(data, feed) {
-  if (this.isLast()) {
-    return feed.close();
-  }
-  const location = (0, _path.normalize)(this.getParam('location', '/'));
-  const locations = [(0, _higherPath.default)((0, _os.tmpdir)(), location), (0, _higherPath.default)(process.cwd(), location)];
-  const filename = locations.filter(Boolean).map(dir => (0, _path.resolve)(dir, String(data).trim())).filter(fil => (0, _fs.existsSync)(fil)).shift();
-  if (!filename) {
-    return feed.stop(new Error('File location check failed.'));
-  }
-  (0, _fs.accessSync)((0, _path.dirname)(filename), _fs.constants.R_OK | _fs.constants.W_OK);
-  (0, _fs.accessSync)(filename, _fs.constants.R_OK | _fs.constants.W_OK);
-  await (0, _hyparquet.parquetRead)({
-    file: await (0, _hyparquet.asyncBufferFromFile)(filename),
-    rowFormat: 'object',
-    onComplete: rows => feed.write(rows)
-  });
-  return feed.end();
-}
-;

package/lib/obj-columns.js

@@ -1,53 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = OBJColumns;
-
-/**
- * Take an `Object` and flatten it to get only one level of keys.
- *
- * <caption>Input:</caption>
- *
- * ```json
- * [{
- *   "foo": {
- *     "hello": "world"
- *   },
- *   "bar": "anything else",
- *   "baz": 1
- * }]
- * ```
- *
- * <caption>Output:</caption>
- *
- * ```json
- * [{
- *   "foo": "{\"hello\":\"world\"}",
- *   "bar": "anything else",
- *   "baz": 1
- * }]
- * ```
- *
- * @name OBJColumns
- * @alias flatten
- * @param {undefined} none
- * @returns {Object}
- */
-function OBJColumns(data, feed) {
-  if (this.isLast()) {
-    feed.close();
-    return;
-  }
-
-  const obj = {};
-  Object.keys(data).sort((x, y) => x.localeCompare(y)).forEach(key => {
-    if (typeof data[key] === 'object') {
-      obj[key] = JSON.stringify(data[key]);
-    } else {
-      obj[key] = data[key];
-    }
-  });
-  feed.send(obj);
-}

package/lib/skos-object.js

@@ -1,73 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-
-function Concept(data, feed) {
-  const obj = {};
-  Object.keys(data).forEach(key => {
-    const newkey = key.replace('skos$', '');
-
-    if (Array.isArray(data[key])) {
-      data[key].filter(x => x.xml$lang).forEach(item => {
-        const localkey = newkey.concat('@').concat(item.xml$lang);
-        obj[localkey] = item.$t;
-      });
-    } else if (data[key].rdf$resource && !obj[newkey]) {
-      obj[newkey] = data[key].rdf$resource;
-    } else if (data[key].rdf$resource && obj[newkey]) {
-      obj[newkey] = [obj[newkey], data[key].rdf$resource];
-    } else if (data[key].$t && data[key].xml$lang) {
-      const localkey = newkey.concat('@').concat(data[key].xml$lang);
-      obj[localkey] = data[key].$t;
-    } else if (data[key].$t && Array.isArray(obj[newkey])) {
-      obj[newkey].push(data[key].$t);
-    } else if (data[key].$t && obj[newkey]) {
-      obj[newkey] = [obj[newkey], data[key].$t];
-    } else if (data[key].$t && !obj[newkey]) {
-      obj[newkey] = data[key].$t;
-    } else if (typeof data[key] === 'object') {
-      obj[newkey] = (this.getIndex().toString(36) + Math.random().toString(36).substr(2, 5)).toUpperCase();
-      let counter = 0;
-      Object.keys(data[key]).forEach(key2 => {
-        if (typeof data[key][key2] === 'object') {
-          data[key][key2].rdf$about = obj[newkey];
-          Concept.call(this, data[key][key2], feed);
-          counter += 1;
-        }
-      });
-
-      if (counter === 0) {
-        delete obj[newkey];
-      }
-    } else {
-      obj[newkey] = data[key];
-    }
-  });
-  feed.write(obj);
-}
-
-function SKOSObject(data, feed) {
-  if (this.isLast()) {
-    feed.close();
-  } else {
-    Concept.call(this, data, feed);
-    feed.end();
-  }
-}
-/**
- * Take `Object` generated by XMLMapping & SKOS data and
- * create a new basic object with only keys & values
- *
- * @name SKOSObject
- * @param {undefined} none
- * @returns {Object}
- */
-
-
-var _default = {
-  SKOSObject
-};
-exports.default = _default;

package/lib/txt-inflection.js

@@ -1,61 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-var _lodash = require("lodash");
-var _inflection = require("inflection");
-const transformer = transformations => str => str && typeof str === 'string' ? (0, _inflection.transform)(str, transformations) : str;
-const TXTInflection = (data, feed, ctx) => {
-  if (ctx.isLast()) {
-    return feed.close();
-  }
-  const transformations = [].concat(ctx.getParam('transform', [])).filter(Boolean);
-  const path = ctx.getParam('path', 'value');
-  const value = (0, _lodash.get)(data, path, '');
-  const process = transformer(transformations);
-  const result = Array.isArray(value) ? value.map(item => process(item)) : process(value);
-  feed.write({
-    ...data,
-    [path]: result
-  });
-  return feed.end();
-};
-
-/**
- * Take a `String` and inflect it with or more transformers from this list
- *  pluralize, singularize, camelize, underscore, humanize, capitalize,
- *  dasherize, titleize, demodulize, tableize, classify, foreign_key, ordinalize
- *
- * Input:
- *
- * ```json
- * { "id": 1, "value": "all job" }
- * ```
- * Script:
- * ```ini
- * [TXTInflection]
- * transform = pluralize
- * transform = capitalize
- * transform = dasherize
- * ```
- *
- * Output:
- *
- * ```json
- * { "id": 1, "value": "All-jobs" }
- * ```
- *
- * @name TXTInflection
- * @param {String} [path="value"] path of the field to segment
- * @param {String} [transform] name of a transformer
- * @returns {String[]}
- * @deprecated
- * see https://inist-cnrs.github.io/ezs/#/plugin-strings?id=inflection
- * see https://www.npmjs.com/package/inflection
- */
-var _default = {
-  TXTInflection
-};
-exports.default = _default;

package/lib/txt-sentences.js

@@ -1,82 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-var _lodash = require("lodash");
-const UPPER_LETTERS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ';
-const SENTENCE_INIT = '  ';
-const SENTENCE_ENDING = '.?!';
-
-/*
- * Segment sentences from `str` into an array
- * @param {string} str
- * @returns {string[]}
- */
-const segmentSentences = str => {
-  const characters = Array.from(str);
-  const sentences = characters.reduce(
-  /*
-   * @param {string[]} prevSentences
-   * @param {string} character
-   * @return {string[]}
-   */
-  (prevSentences, character) => {
-    const currentSentence = prevSentences.slice(-1)[0];
-    const [char1, char2] = currentSentence.slice(-2);
-    if (SENTENCE_ENDING.includes(character)) {
-      if (character !== '.') {
-        return [...prevSentences.slice(0, -1), currentSentence + character, SENTENCE_INIT];
-      }
-      if (char1 !== ' ') {
-        return [...prevSentences.slice(0, -1), currentSentence + character, SENTENCE_INIT];
-      }
-      if (!UPPER_LETTERS.includes(char2)) {
-        return [...prevSentences.slice(0, -1), currentSentence + character, SENTENCE_INIT];
-      }
-    }
-    return [...prevSentences.slice(0, -1), currentSentence + character];
-  }, [SENTENCE_INIT]).filter(sentence => sentence !== SENTENCE_INIT).map(sentence => sentence.trimStart());
-  return sentences;
-};
-const TXTSentences = (data, feed, ctx) => {
-  if (ctx.isLast()) {
-    return feed.close();
-  }
-  const path = ctx.getParam('path', 'value');
-  const value = (0, _lodash.get)(data, path);
-  const str = Array.isArray(value) ? value.map(item => typeof item === 'string' ? item : '').join(' ') : value;
-  const sentences = str ? segmentSentences(str) : [];
-  feed.write({
-    ...data,
-    [path]: sentences
-  });
-  return feed.end();
-};
-
-/**
- * Take a `String` and split it into an array of sentences.
- *
- * Input:
- *
- * ```json
- * { "id": 1, "value": "First sentence? Second sentence. My name is Bond, J. Bond." }
- * ```
- *
- * Output:
- *
- * ```json
- * { "id": 1, "value": ["First sentence?", "Second sentence.", "My name is Bond, J. Bond."] }
- * ```
- *
- * @name TXTSentences
- * @param {String} [path="value"] path of the field to segment
- * @returns {String[]}
- * @deprecated
- * see https://inist-cnrs.github.io/ezs/#/plugin-strings?id=sentences
- */
-var _default = {
-  TXTSentences
-};
-exports.default = _default;

package/lib/url-pager.js

@@ -1,179 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = URLPager;
-
-var _debug = _interopRequireDefault(require("debug"));
-
-var _url = require("url");
-
-var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
-
-var _lodash = _interopRequireDefault(require("lodash.get"));
-
-var _parseHeaders = _interopRequireDefault(require("parse-headers"));
-
-var _asyncRetry = _interopRequireDefault(require("async-retry"));
-
-var _request = _interopRequireDefault(require("./request"));
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-/**
- * Take `Object` as parameters of URL, throw each chunk from the result
- *
- *
- * Input:
- *
- * ```json
- * [{"q": "a"}]
- * ```
- *
- * Script:
- *
- * ```ini
- * [URLPager]
- * url = https://api.search.net
- * path = total
- * ```
- *
- * Output:
- *
- * ```json
- * [
- *      {
- *          "q": "a",
- *          "total": 22
- *          "offset": 0,
- *          "pageNumber": 1,
- *          "totalPages", 3,
- *          "maxPages": 1000,
- *          "limit": 10
- *      },
- *      {
- *          "q": "a",
- *          "total": 22
- *          "offset": 10,
- *          "pageNumber": 2,
- *          "totalPages", 3,
- *          "maxPages": 1000,
- *          "limit": 10
- *      },
- *      {
- *          "q": "a",
- *          "total": 22
- *          "offset": 20,
- *          "pageNumber": 3,
- *          "totalPages", 3,
- *          "maxPages": 1000,
- *          "limit": 10
- *      }
- *  ]
- * ```
- *
- * #### Example with URLs
- *
- * Input:
- *
- * ```json
- * [
- *   "https://httpbin.org/get?a=a",
- *   "https://httpbin.org/get?a=b",
- *   "https://httpbin.org/get?a=c"
- * ]
- * ```
- *
- * Script:
- *
- * ```ini
- * [URLPager]
- * path = .args
- * ```
- *
- * Output:
- *
- * ```json
- * [{"a": "a"}, {"a": "b"}, {"a": "c" }]
- * ```
- *
- * @name URLPager
- * @param {String} [url] URL to fetch (by default input string is taken)
- * @param {String} [path=total] choose the path to find the number of result
- * @param {Number} [timeout=1000] Timeout in milliseconds
- * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
- * @param {Number} [retries=5] The maximum amount of times to retry the connection
- * @returns {Object}
- */
-async function URLPager(data, feed) {
-  if (this.isLast()) {
-    return feed.close();
-  }
-
-  const url = this.getParam('url');
-  const path = this.getParam('path', 'total');
-  const limit = Number(this.getParam('limit', 10));
-  const maxPages = Number(this.getParam('maxPages', 1000));
-  const retries = Number(this.getParam('retries', 5));
-  const noerror = Boolean(this.getParam('noerror', false));
-  const timeout = Number(this.getParam('timeout')) || 1000;
-  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
-  const cURL = new _url.URL(url || data);
-  const controller = new _nodeAbortController.default();
-  const parameters = {
-    timeout,
-    headers,
-    signal: controller.signal
-  };
-  const options = {
-    retries
-  };
-  cURL.search = new _url.URLSearchParams(data);
-
-  const onError = e => {
-    controller.abort();
-
-    if (noerror) {
-      (0, _debug.default)('ezs')(`Ignore item #${this.getIndex()} [URLPager] <${e}>`);
-      return feed.send(data);
-    }
-
-    (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLPager] <${e}>`);
-    return feed.send(e);
-  };
-
-  try {
-    const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
-    const json = await response.json();
-    const total = (0, _lodash.default)(json, path);
-
-    if (total === 0) {
-      return onError(new Error('No result.'));
-    }
-
-    if (total === undefined) {
-      return onError(new Error('Unexpected response.'));
-    }
-
-    let totalPages = Math.ceil(json.total / limit);
-
-    if (totalPages > maxPages) {
-      totalPages = maxPages;
-    }
-
-    for (let pageNumber = 1; pageNumber <= totalPages; pageNumber += 1) {
-      feed.write({ ...data,
-        offset: (pageNumber - 1) * limit,
-        pageNumber,
-        totalPages,
-        maxPages,
-        limit
-      });
-    }
-
-    feed.end();
-  } catch (e) {
-    onError(e);
-  }
-}

package/lib/utils.js

@@ -1,18 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = exports.writeTo = void 0;
-
-const writeTo = function writeTo(stream, data, cb) {
-  if (!stream.write(data)) {
-    stream.once('drain', cb);
-  } else {
-    process.nextTick(cb);
-  }
-};
-
-exports.writeTo = writeTo;
-var _default = writeTo;
-exports.default = _default;