@ezs/basics 2.0.0 → 2.1.1

package/CHANGELOG.md

@@ -3,6 +3,31 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [2.1.1](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.1.0...@ezs/basics@2.1.1) (2023-05-12)
+
+**Note:** Version bump only for package @ezs/basics
+
+
+
+
+
+# [2.1.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.0.0...@ezs/basics@2.1.0) (2023-04-12)
+
+
+### Bug Fixes
+
+* 🐛 improve coverage ([1173137](https://github.com/Inist-CNRS/ezs/commit/1173137e5a2cd6f1f6bfda8ad2a89720ae991e91))
+* 🐛 missing files ([cd091ac](https://github.com/Inist-CNRS/ezs/commit/cd091acd91a6b0042e6c0e1b9cfef87e64126007))
+
+
+### Features
+
+* 🎸 add [TXTInflection] ([880b2bc](https://github.com/Inist-CNRS/ezs/commit/880b2bc4795f8bfb51fe11803cebfe38d2097487))
+
+
+
+
+
 # [2.0.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.24.0...@ezs/basics@2.0.0) (2023-03-29)
 
 

package/README.md

@@ -23,11 +23,13 @@ npm install @ezs/basics
 -   [INIString](#inistring)
 -   [JSONParse](#jsonparse)
 -   [JSONString](#jsonstring)
+-   [OBJColumns](#objcolumns)
 -   [OBJCount](#objcount)
 -   [OBJFlatten](#objflatten)
 -   [OBJNamespaces](#objnamespaces)
 -   [OBJStandardize](#objstandardize)
 -   [TXTConcat](#txtconcat)
+-   [TXTInflection](#txtinflection)
 -   [TXTObject](#txtobject)
 -   [TXTParse](#txtparse)
 -   [TXTSentences](#txtsentences)
@@ -391,6 +393,38 @@ Output:
 
 Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
 
+### OBJColumns
+
+Take an `Object` and flatten it to get only one level of keys.
+
+<caption>Input:</caption>
+
+```json
+[{
+  "foo": {
+    "hello": "world"
+  },
+  "bar": "anything else",
+  "baz": 1
+}]
+```
+
+<caption>Output:</caption>
+
+```json
+[{
+  "foo": "{\"hello\":\"world\"}",
+  "bar": "anything else",
+  "baz": 1
+}]
+```
+
+#### Parameters
+
+-   `none` **[undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined)** 
+
+Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
+
 ### OBJCount
 
 Count how many objects are received, and yield the total.
@@ -557,6 +591,43 @@ Output:
 
 Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
 
+### TXTInflection
+
+-   **See: <https://www.npmjs.com/package/inflection>
+    **
+
+Take a `String` and inflect it with or more transformers from this list
+ pluralize, singularize, camelize, underscore, humanize, capitalize,
+ dasherize, titleize, demodulize, tableize, classify, foreign_key, ordinalize
+
+Input:
+
+```json
+{ "id": 1, "value": "all job" }
+```
+
+Script:
+
+```ini
+[TXTInflection]
+transform = pluralize
+transform = capitalize
+transform = dasherize
+```
+
+Output:
+
+```json
+{ "id": 1, "value": "All-jobs" }
+```
+
+#### Parameters
+
+-   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** path of the field to segment (optional, default `"value"`)
+-   `transform` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)?** name of a transformer
+
+Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)&lt;[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** 
+
 ### TXTObject
 
 Take an array of values and generate an array containing objects with the
@@ -657,6 +728,7 @@ read to be buffered and sent to the server (n times)
 -   `timeout` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** Timeout in milliseconds (optional, default `1000`)
 -   `noerror` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Ignore all errors (optional, default `false`)
 -   `retries` **[Number](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Number)** The maximum amount of times to retry the connection (optional, default `5`)
+-   `encoder` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** The statement to encode each chunk to a string (optional, default `dump`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 

package/lib/fetch.js

@@ -0,0 +1,74 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = fetch;
+
+var _crossFetch = _interopRequireDefault(require("cross-fetch"));
+
+var _proxyFromEnv = require("proxy-from-env");
+
+var _http = _interopRequireDefault(require("http"));
+
+var _https = _interopRequireDefault(require("https"));
+
+var _betterHttpsProxyAgent = require("better-https-proxy-agent");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const DefaultOptions = {
+  keepAlive: true,
+  timeout: 1000,
+  keepAliveMsecs: 500,
+  maxSockets: 200,
+  maxFreeSockets: 5,
+  maxCachedSessions: 500
+};
+
+const chooseAgent = (parsedURL, options) => {
+  const proxyurl = (0, _proxyFromEnv.getProxyForUrl)(parsedURL.href);
+
+  if (proxyurl) {
+    const proxyRequestOptions = new URL(proxyurl);
+    return new _betterHttpsProxyAgent.Agent(options, proxyRequestOptions);
+  }
+
+  if (parsedURL.protocol === 'https:') {
+    return new _https.default.Agent(options);
+  }
+
+  return new _http.default.Agent(options);
+};
+
+function fetch(url, options) {
+  const opts = options || {};
+  const {
+    keepAlive,
+    timeout,
+    keepAliveMsecs,
+    maxSockets,
+    maxFreeSockets,
+    maxCachedSessions
+  } = { ...options,
+    ...DefaultOptions
+  };
+  let agent = chooseAgent(new URL(url), {
+    keepAlive,
+    timeout,
+    keepAliveMsecs,
+    maxSockets,
+    maxFreeSockets,
+    maxCachedSessions
+  });
+  opts.agent = agent;
+
+  if (opts.signal) {
+    opts.signal.addEventListener('abort', () => {
+      agent.destroy();
+      agent = null;
+    });
+  }
+
+  return (0, _crossFetch.default)(url, options);
+}

package/lib/index.js

@@ -23,6 +23,8 @@ var _txtParse = _interopRequireDefault(require("./txt-parse"));
 
 var _txtSentences = _interopRequireDefault(require("./txt-sentences"));
 
+var _txtInflection = _interopRequireDefault(require("./txt-inflection"));
+
 var _xmlParse = _interopRequireDefault(require("./xml-parse"));
 
 var _xmlString = _interopRequireDefault(require("./xml-string"));
@@ -75,6 +77,7 @@ const funcs = {
   TXTObject: _txtObject.default,
   TXTConcat: _txtConcat.default,
   TXTSentences: _txtSentences.default,
+  TXTInflection: _txtInflection.default,
   XMLParse: _xmlParse.default,
   XMLString: _xmlString.default,
   XMLConvert: _xmlConvert.default,

package/lib/obj-columns.js

@@ -0,0 +1,53 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = OBJColumns;
+
+/**
+ * Take an `Object` and flatten it to get only one level of keys.
+ *
+ * <caption>Input:</caption>
+ *
+ * ```json
+ * [{
+ *   "foo": {
+ *     "hello": "world"
+ *   },
+ *   "bar": "anything else",
+ *   "baz": 1
+ * }]
+ * ```
+ *
+ * <caption>Output:</caption>
+ *
+ * ```json
+ * [{
+ *   "foo": "{\"hello\":\"world\"}",
+ *   "bar": "anything else",
+ *   "baz": 1
+ * }]
+ * ```
+ *
+ * @name OBJColumns
+ * @alias flatten
+ * @param {undefined} none
+ * @returns {Object}
+ */
+function OBJColumns(data, feed) {
+  if (this.isLast()) {
+    feed.close();
+    return;
+  }
+
+  const obj = {};
+  Object.keys(data).sort((x, y) => x.localeCompare(y)).forEach(key => {
+    if (typeof data[key] === 'object') {
+      obj[key] = JSON.stringify(data[key]);
+    } else {
+      obj[key] = data[key];
+    }
+  });
+  feed.send(obj);
+}

package/lib/skos-object.js

@@ -0,0 +1,73 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+function Concept(data, feed) {
+  const obj = {};
+  Object.keys(data).forEach(key => {
+    const newkey = key.replace('skos$', '');
+
+    if (Array.isArray(data[key])) {
+      data[key].filter(x => x.xml$lang).forEach(item => {
+        const localkey = newkey.concat('@').concat(item.xml$lang);
+        obj[localkey] = item.$t;
+      });
+    } else if (data[key].rdf$resource && !obj[newkey]) {
+      obj[newkey] = data[key].rdf$resource;
+    } else if (data[key].rdf$resource && obj[newkey]) {
+      obj[newkey] = [obj[newkey], data[key].rdf$resource];
+    } else if (data[key].$t && data[key].xml$lang) {
+      const localkey = newkey.concat('@').concat(data[key].xml$lang);
+      obj[localkey] = data[key].$t;
+    } else if (data[key].$t && Array.isArray(obj[newkey])) {
+      obj[newkey].push(data[key].$t);
+    } else if (data[key].$t && obj[newkey]) {
+      obj[newkey] = [obj[newkey], data[key].$t];
+    } else if (data[key].$t && !obj[newkey]) {
+      obj[newkey] = data[key].$t;
+    } else if (typeof data[key] === 'object') {
+      obj[newkey] = (this.getIndex().toString(36) + Math.random().toString(36).substr(2, 5)).toUpperCase();
+      let counter = 0;
+      Object.keys(data[key]).forEach(key2 => {
+        if (typeof data[key][key2] === 'object') {
+          data[key][key2].rdf$about = obj[newkey];
+          Concept.call(this, data[key][key2], feed);
+          counter += 1;
+        }
+      });
+
+      if (counter === 0) {
+        delete obj[newkey];
+      }
+    } else {
+      obj[newkey] = data[key];
+    }
+  });
+  feed.write(obj);
+}
+
+function SKOSObject(data, feed) {
+  if (this.isLast()) {
+    feed.close();
+  } else {
+    Concept.call(this, data, feed);
+    feed.end();
+  }
+}
+/**
+ * Take `Object` generated by XMLMapping & SKOS data and
+ * create a new basic object with only keys & values
+ *
+ * @name SKOSObject
+ * @param {undefined} none
+ * @returns {Object}
+ */
+
+
+var _default = {
+  SKOSObject
+};
+exports.default = _default;

package/lib/txt-inflection.js

@@ -0,0 +1,66 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+var _lodash = _interopRequireDefault(require("lodash.get"));
+
+var _inflection = require("inflection");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const transformer = transformations => str => str && typeof str === 'string' ? (0, _inflection.transform)(str, transformations) : str;
+
+const TXTInflection = (data, feed, ctx) => {
+  if (ctx.isLast()) {
+    return feed.close();
+  }
+
+  const transformations = [].concat(ctx.getParam('transform', [])).filter(Boolean);
+  const path = ctx.getParam('path', 'value');
+  const value = (0, _lodash.default)(data, path, '');
+  const process = transformer(transformations);
+  const result = Array.isArray(value) ? value.map(item => process(item)) : process(value);
+  feed.write({ ...data,
+    [path]: result
+  });
+  return feed.end();
+};
+/**
+ * Take a `String` and inflect it with or more transformers from this list
+ *  pluralize, singularize, camelize, underscore, humanize, capitalize,
+ *  dasherize, titleize, demodulize, tableize, classify, foreign_key, ordinalize
+ *
+ * Input:
+ *
+ * ```json
+ * { "id": 1, "value": "all job" }
+ * ```
+ * Script:
+ * ```ini
+ * [TXTInflection]
+ * transform = pluralize
+ * transform = capitalize
+ * transform = dasherize
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * { "id": 1, "value": "All-jobs" }
+ * ```
+ *
+ * @name TXTInflection
+ * @param {String} [path="value"] path of the field to segment
+ * @param {String} [transform] name of a transformer
+ * @returns {String[]}
+ * @see https://www.npmjs.com/package/inflection
+ */
+
+
+var _default = {
+  TXTInflection
+};
+exports.default = _default;

package/lib/url-connect.js

@@ -43,6 +43,7 @@ function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { de
  * @param {Number} [timeout=1000] Timeout in milliseconds
  * @param {Boolean} [noerror=false] Ignore all errors
  * @param {Number} [retries=5] The maximum amount of times to retry the connection
+ * @param {String} [encoder=dump] The statement to encode each chunk to a string
  * @returns {Object}
  */
 async function URLConnect(data, feed) {
@@ -50,6 +51,7 @@ async function URLConnect(data, feed) {
   const retries = Number(this.getParam('retries', 5));
   const noerror = Boolean(this.getParam('noerror', false));
   const json = this.getParam('json', true);
+  const encoder = this.getParam('encoder', 'dump');
   const {
     ezs
   } = this;
@@ -61,7 +63,7 @@ async function URLConnect(data, feed) {
     const output = ezs.createStream(ezs.objectMode());
     this.whenFinish = feed.flow(output);
     (0, _streamWrite.default)(this.input, data, () => feed.end());
-    const streamIn = this.input.pipe(ezs('dump'));
+    const streamIn = this.input.pipe(ezs(encoder));
     let bodyIn;
 
     if (retries === 1) {

package/lib/url-pager.js

@@ -0,0 +1,179 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = URLPager;
+
+var _debug = _interopRequireDefault(require("debug"));
+
+var _url = require("url");
+
+var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
+
+var _lodash = _interopRequireDefault(require("lodash.get"));
+
+var _parseHeaders = _interopRequireDefault(require("parse-headers"));
+
+var _asyncRetry = _interopRequireDefault(require("async-retry"));
+
+var _request = _interopRequireDefault(require("./request"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Take `Object` as parameters of URL, throw each chunk from the result
+ *
+ *
+ * Input:
+ *
+ * ```json
+ * [{"q": "a"}]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [URLPager]
+ * url = https://api.search.net
+ * path = total
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 0,
+ *          "pageNumber": 1,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      },
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 10,
+ *          "pageNumber": 2,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      },
+ *      {
+ *          "q": "a",
+ *          "total": 22
+ *          "offset": 20,
+ *          "pageNumber": 3,
+ *          "totalPages", 3,
+ *          "maxPages": 1000,
+ *          "limit": 10
+ *      }
+ *  ]
+ * ```
+ *
+ * #### Example with URLs
+ *
+ * Input:
+ *
+ * ```json
+ * [
+ *   "https://httpbin.org/get?a=a",
+ *   "https://httpbin.org/get?a=b",
+ *   "https://httpbin.org/get?a=c"
+ * ]
+ * ```
+ *
+ * Script:
+ *
+ * ```ini
+ * [URLPager]
+ * path = .args
+ * ```
+ *
+ * Output:
+ *
+ * ```json
+ * [{"a": "a"}, {"a": "b"}, {"a": "c" }]
+ * ```
+ *
+ * @name URLPager
+ * @param {String} [url] URL to fetch (by default input string is taken)
+ * @param {String} [path=total] choose the path to find the number of result
+ * @param {Number} [timeout=1000] Timeout in milliseconds
+ * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
+ * @param {Number} [retries=5] The maximum amount of times to retry the connection
+ * @returns {Object}
+ */
+async function URLPager(data, feed) {
+  if (this.isLast()) {
+    return feed.close();
+  }
+
+  const url = this.getParam('url');
+  const path = this.getParam('path', 'total');
+  const limit = Number(this.getParam('limit', 10));
+  const maxPages = Number(this.getParam('maxPages', 1000));
+  const retries = Number(this.getParam('retries', 5));
+  const noerror = Boolean(this.getParam('noerror', false));
+  const timeout = Number(this.getParam('timeout')) || 1000;
+  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
+  const cURL = new _url.URL(url || data);
+  const controller = new _nodeAbortController.default();
+  const parameters = {
+    timeout,
+    headers,
+    signal: controller.signal
+  };
+  const options = {
+    retries
+  };
+  cURL.search = new _url.URLSearchParams(data);
+
+  const onError = e => {
+    controller.abort();
+
+    if (noerror) {
+      (0, _debug.default)('ezs')(`Ignore item #${this.getIndex()} [URLPager] <${e}>`);
+      return feed.send(data);
+    }
+
+    (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLPager] <${e}>`);
+    return feed.send(e);
+  };
+
+  try {
+    const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
+    const json = await response.json();
+    const total = (0, _lodash.default)(json, path);
+
+    if (total === 0) {
+      return onError(new Error('No result.'));
+    }
+
+    if (total === undefined) {
+      return onError(new Error('Unexpected response.'));
+    }
+
+    let totalPages = Math.ceil(json.total / limit);
+
+    if (totalPages > maxPages) {
+      totalPages = maxPages;
+    }
+
+    for (let pageNumber = 1; pageNumber <= totalPages; pageNumber += 1) {
+      feed.write({ ...data,
+        offset: (pageNumber - 1) * limit,
+        pageNumber,
+        totalPages,
+        maxPages,
+        limit
+      });
+    }
+
+    feed.end();
+  } catch (e) {
+    onError(e);
+  }
+}

package/lib/utils.js

@@ -0,0 +1,18 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = exports.writeTo = void 0;
+
+const writeTo = function writeTo(stream, data, cb) {
+  if (!stream.write(data)) {
+    stream.once('drain', cb);
+  } else {
+    process.nextTick(cb);
+  }
+};
+
+exports.writeTo = writeTo;
+var _default = writeTo;
+exports.default = _default;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "2.0.0",
+  "version": "2.1.1",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
@@ -15,6 +15,7 @@
     "from": "0.1.7",
     "get-stream": "6.0.1",
     "higher-path": "1.0.0",
+    "inflection": "2.0.1",
     "lodash.escaperegexp": "4.1.2",
     "lodash.get": "4.4.2",
     "lodash.mapkeys": "4.6.0",
@@ -39,7 +40,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "314a26ebcbf109ffdb6936ee4b8564e19dd17cbc",
+  "gitHead": "54e12b40d2cd6ccb6754f06ab5cae566c60bb3a9",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"