@ezs/basics 2.1.3 → 2.3.0

package/CHANGELOG.md

@@ -3,6 +3,28 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [2.3.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.2.0...@ezs/basics@2.3.0) (2023-06-23)
+
+
+### Features
+
+* **strings:** Add inflection and sentences statements ([67ba36c](https://github.com/Inist-CNRS/ezs/commit/67ba36c8ba68c1798f5430a087172fed45d2b3c0))
+
+
+
+
+
+# [2.2.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.1.3...@ezs/basics@2.2.0) (2023-06-22)
+
+
+### Features
+
+* 🎸 add tar-extract ([a471aca](https://github.com/Inist-CNRS/ezs/commit/a471aca64b7ab08d92237917264a23080a2c3072))
+
+
+
+
+
 ## [2.1.3](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.1.2...@ezs/basics@2.1.3) (2023-05-30)
 
 **Note:** Version bump only for package @ezs/basics

package/README.md

@@ -27,6 +27,7 @@ npm install @ezs/basics
 -   [OBJFlatten](#objflatten)
 -   [OBJNamespaces](#objnamespaces)
 -   [OBJStandardize](#objstandardize)
+-   [TARExtract](#tarextract)
 -   [TXTConcat](#txtconcat)
 -   [TXTInflection](#txtinflection)
 -   [TXTObject](#txtobject)
@@ -536,6 +537,27 @@ Output:
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
+### TARExtract
+
+Take the content of a tar file, extract some files.
+The JSON object is sent to the output stream for each file.
+It returns to the output stream
+
+```json
+{
+   "id": "file name",
+   "value": "file contents"
+}
+```
+
+#### Parameters
+
+-   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Regex to select the files to extract (optional, default `"**\/*.json"`)
+-   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of each file (optional, default `true`)
+-   `compress` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Enable gzip compression (optional, default `false`)
+
+Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<{id: [String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), value: [String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)}>** 
+
 ### TXTConcat
 
 Concatenate all `String` items into one string
@@ -560,9 +582,6 @@ Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/G
 
 ### TXTInflection
 
--   **See: <https://www.npmjs.com/package/inflection>
-    **
-
 Take a `String` and inflect it with or more transformers from this list
  pluralize, singularize, camelize, underscore, humanize, capitalize,
  dasherize, titleize, demodulize, tableize, classify, foreign_key, ordinalize
@@ -595,6 +614,12 @@ Output:
 
 Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)&lt;[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** 
 
+**Meta**
+
+-   **deprecated**: see <https://inist-cnrs.github.io/ezs/#/plugin-strings?id=inflection>
+    see <https://www.npmjs.com/package/inflection>
+
+
 ### TXTObject
 
 Take an array of values and generate an array containing objects with the
@@ -662,6 +687,11 @@ Output:
 
 Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)&lt;[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)>** 
 
+**Meta**
+
+-   **deprecated**: see <https://inist-cnrs.github.io/ezs/#/plugin-strings?id=sentences>
+
+
 ### TXTZip
 
 Take a `String` and zip it.

package/lib/index.js

@@ -59,6 +59,8 @@ var _txtZip = _interopRequireDefault(require("./txt-zip"));
 
 var _zipExtract = _interopRequireDefault(require("./zip-extract"));
 
+var _tarExtract = _interopRequireDefault(require("./tar-extract"));
+
 var _iniString = _interopRequireDefault(require("./ini-string"));
 
 var _fileSave = _interopRequireDefault(require("./file-save"));
@@ -95,6 +97,7 @@ const funcs = {
   URLConnect: _urlConnect.default,
   TXTZip: _txtZip.default,
   ZIPExtract: _zipExtract.default,
+  TARExtract: _tarExtract.default,
   INIString: _iniString.default,
   FILESave: _fileSave.default,
   FILELoad: _fileLoad.default,

package/lib/tar-extract.js

@@ -0,0 +1,90 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = TARExtract;
+
+var _tarStream = _interopRequireDefault(require("tar-stream"));
+
+var _micromatch = _interopRequireDefault(require("micromatch"));
+
+var _zlib = require("zlib");
+
+var _getStream = _interopRequireDefault(require("get-stream"));
+
+var _streamWrite = _interopRequireDefault(require("stream-write"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Take the content of a tar file, extract some files.
+ * The JSON object is sent to the output stream for each file.
+ * It returns to the output stream
+ *
+ * ```json
+ * {
+ *    "id": "file name",
+ *    "value": "file contents"
+ * }
+ * ```
+ *
+ * @name TARExtract
+ * @param {String} [path="**\/*.json"] Regex to select the files to extract
+ * @param {String} [json=true] Parse as JSON the content of each file
+ * @param {Boolean} [compress=false] Enable gzip compression
+ * @returns {{id: String, value: String}[]}
+ */
+function TARExtract(data, feed) {
+  const filesPatern = this.getParam('path', '**/*.json');
+
+  if (this.isFirst()) {
+    const {
+      ezs
+    } = this;
+    const json = this.getParam('json', true);
+    const compress = this.getParam('compress', false);
+    this.input = ezs.createStream(ezs.objectMode());
+    this.output = ezs.createStream(ezs.objectMode());
+
+    const extract = _tarStream.default.extract();
+
+    this.whenEnd = new Promise((resolve, reject) => {
+      extract.on('entry', async (header, stream, next) => {
+        if (_micromatch.default.isMatch(header.name, filesPatern)) {
+          const contentRaw = await (0, _getStream.default)(stream);
+
+          if (json) {
+            const contentJson = JSON.parse(contentRaw);
+            return (0, _streamWrite.default)(this.output, contentJson, () => next());
+          }
+
+          return (0, _streamWrite.default)(this.output, {
+            id: header.name,
+            value: contentRaw
+          }, () => next());
+        }
+
+        return next();
+      });
+      extract.on('error', reject);
+      extract.on('finish', resolve);
+    });
+
+    if (compress) {
+      this.input.pipe((0, _zlib.createGunzip)()).pipe(extract);
+    } else {
+      this.input.pipe(extract);
+    }
+
+    this.whenFinish = feed.flow(this.output);
+  }
+
+  if (this.isLast()) {
+    this.whenEnd.finally(() => this.output.end());
+    this.whenFinish.finally(() => feed.close());
+    this.input.end();
+  } else {
+    (0, _streamWrite.default)(this.input, data, () => feed.end());
+  }
+}

package/lib/txt-inflection.js

@@ -56,7 +56,9 @@ const TXTInflection = (data, feed, ctx) => {
  * @param {String} [path="value"] path of the field to segment
  * @param {String} [transform] name of a transformer
  * @returns {String[]}
- * @see https://www.npmjs.com/package/inflection
+ * @deprecated
+ * see https://inist-cnrs.github.io/ezs/#/plugin-strings?id=inflection
+ * see https://www.npmjs.com/package/inflection
  */
 
 

package/lib/txt-sentences.js

@@ -81,6 +81,8 @@ const TXTSentences = (data, feed, ctx) => {
  * @name TXTSentences
  * @param {String} [path="value"] path of the field to segment
  * @returns {String[]}
+ * @deprecated
+ * see https://inist-cnrs.github.io/ezs/#/plugin-strings?id=sentences
  */
 
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "2.1.3",
+  "version": "2.3.0",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
@@ -28,6 +28,7 @@
     "parse-headers": "2.0.4",
     "path-exists": "4.0.0",
     "stream-write": "1.0.1",
+    "tar-stream": "3.1.4",
     "tmp-filepath": "2.0.0",
     "unzipper": "0.10.11",
     "xml-mapping": "1.7.2",
@@ -40,7 +41,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "2296d653ec1f8ce2ee9999ed77e6c803faa74de9",
+  "gitHead": "d363643a24747f3fd2230deb8d3eb68aa313369d",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"

package/lib/fetch.js

@@ -1,74 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = fetch;
-
-var _crossFetch = _interopRequireDefault(require("cross-fetch"));
-
-var _proxyFromEnv = require("proxy-from-env");
-
-var _http = _interopRequireDefault(require("http"));
-
-var _https = _interopRequireDefault(require("https"));
-
-var _betterHttpsProxyAgent = require("better-https-proxy-agent");
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-const DefaultOptions = {
-  keepAlive: true,
-  timeout: 1000,
-  keepAliveMsecs: 500,
-  maxSockets: 200,
-  maxFreeSockets: 5,
-  maxCachedSessions: 500
-};
-
-const chooseAgent = (parsedURL, options) => {
-  const proxyurl = (0, _proxyFromEnv.getProxyForUrl)(parsedURL.href);
-
-  if (proxyurl) {
-    const proxyRequestOptions = new URL(proxyurl);
-    return new _betterHttpsProxyAgent.Agent(options, proxyRequestOptions);
-  }
-
-  if (parsedURL.protocol === 'https:') {
-    return new _https.default.Agent(options);
-  }
-
-  return new _http.default.Agent(options);
-};
-
-function fetch(url, options) {
-  const opts = options || {};
-  const {
-    keepAlive,
-    timeout,
-    keepAliveMsecs,
-    maxSockets,
-    maxFreeSockets,
-    maxCachedSessions
-  } = { ...options,
-    ...DefaultOptions
-  };
-  let agent = chooseAgent(new URL(url), {
-    keepAlive,
-    timeout,
-    keepAliveMsecs,
-    maxSockets,
-    maxFreeSockets,
-    maxCachedSessions
-  });
-  opts.agent = agent;
-
-  if (opts.signal) {
-    opts.signal.addEventListener('abort', () => {
-      agent.destroy();
-      agent = null;
-    });
-  }
-
-  return (0, _crossFetch.default)(url, options);
-}

package/lib/obj-columns.js

@@ -1,53 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = OBJColumns;
-
-/**
- * Take an `Object` and flatten it to get only one level of keys.
- *
- * <caption>Input:</caption>
- *
- * ```json
- * [{
- *   "foo": {
- *     "hello": "world"
- *   },
- *   "bar": "anything else",
- *   "baz": 1
- * }]
- * ```
- *
- * <caption>Output:</caption>
- *
- * ```json
- * [{
- *   "foo": "{\"hello\":\"world\"}",
- *   "bar": "anything else",
- *   "baz": 1
- * }]
- * ```
- *
- * @name OBJColumns
- * @alias flatten
- * @param {undefined} none
- * @returns {Object}
- */
-function OBJColumns(data, feed) {
-  if (this.isLast()) {
-    feed.close();
-    return;
-  }
-
-  const obj = {};
-  Object.keys(data).sort((x, y) => x.localeCompare(y)).forEach(key => {
-    if (typeof data[key] === 'object') {
-      obj[key] = JSON.stringify(data[key]);
-    } else {
-      obj[key] = data[key];
-    }
-  });
-  feed.send(obj);
-}

package/lib/skos-object.js

@@ -1,73 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = void 0;
-
-function Concept(data, feed) {
-  const obj = {};
-  Object.keys(data).forEach(key => {
-    const newkey = key.replace('skos$', '');
-
-    if (Array.isArray(data[key])) {
-      data[key].filter(x => x.xml$lang).forEach(item => {
-        const localkey = newkey.concat('@').concat(item.xml$lang);
-        obj[localkey] = item.$t;
-      });
-    } else if (data[key].rdf$resource && !obj[newkey]) {
-      obj[newkey] = data[key].rdf$resource;
-    } else if (data[key].rdf$resource && obj[newkey]) {
-      obj[newkey] = [obj[newkey], data[key].rdf$resource];
-    } else if (data[key].$t && data[key].xml$lang) {
-      const localkey = newkey.concat('@').concat(data[key].xml$lang);
-      obj[localkey] = data[key].$t;
-    } else if (data[key].$t && Array.isArray(obj[newkey])) {
-      obj[newkey].push(data[key].$t);
-    } else if (data[key].$t && obj[newkey]) {
-      obj[newkey] = [obj[newkey], data[key].$t];
-    } else if (data[key].$t && !obj[newkey]) {
-      obj[newkey] = data[key].$t;
-    } else if (typeof data[key] === 'object') {
-      obj[newkey] = (this.getIndex().toString(36) + Math.random().toString(36).substr(2, 5)).toUpperCase();
-      let counter = 0;
-      Object.keys(data[key]).forEach(key2 => {
-        if (typeof data[key][key2] === 'object') {
-          data[key][key2].rdf$about = obj[newkey];
-          Concept.call(this, data[key][key2], feed);
-          counter += 1;
-        }
-      });
-
-      if (counter === 0) {
-        delete obj[newkey];
-      }
-    } else {
-      obj[newkey] = data[key];
-    }
-  });
-  feed.write(obj);
-}
-
-function SKOSObject(data, feed) {
-  if (this.isLast()) {
-    feed.close();
-  } else {
-    Concept.call(this, data, feed);
-    feed.end();
-  }
-}
-/**
- * Take `Object` generated by XMLMapping & SKOS data and
- * create a new basic object with only keys & values
- *
- * @name SKOSObject
- * @param {undefined} none
- * @returns {Object}
- */
-
-
-var _default = {
-  SKOSObject
-};
-exports.default = _default;

package/lib/url-pager.js

@@ -1,179 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = URLPager;
-
-var _debug = _interopRequireDefault(require("debug"));
-
-var _url = require("url");
-
-var _nodeAbortController = _interopRequireDefault(require("node-abort-controller"));
-
-var _lodash = _interopRequireDefault(require("lodash.get"));
-
-var _parseHeaders = _interopRequireDefault(require("parse-headers"));
-
-var _asyncRetry = _interopRequireDefault(require("async-retry"));
-
-var _request = _interopRequireDefault(require("./request"));
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-/**
- * Take `Object` as parameters of URL, throw each chunk from the result
- *
- *
- * Input:
- *
- * ```json
- * [{"q": "a"}]
- * ```
- *
- * Script:
- *
- * ```ini
- * [URLPager]
- * url = https://api.search.net
- * path = total
- * ```
- *
- * Output:
- *
- * ```json
- * [
- *      {
- *          "q": "a",
- *          "total": 22
- *          "offset": 0,
- *          "pageNumber": 1,
- *          "totalPages", 3,
- *          "maxPages": 1000,
- *          "limit": 10
- *      },
- *      {
- *          "q": "a",
- *          "total": 22
- *          "offset": 10,
- *          "pageNumber": 2,
- *          "totalPages", 3,
- *          "maxPages": 1000,
- *          "limit": 10
- *      },
- *      {
- *          "q": "a",
- *          "total": 22
- *          "offset": 20,
- *          "pageNumber": 3,
- *          "totalPages", 3,
- *          "maxPages": 1000,
- *          "limit": 10
- *      }
- *  ]
- * ```
- *
- * #### Example with URLs
- *
- * Input:
- *
- * ```json
- * [
- *   "https://httpbin.org/get?a=a",
- *   "https://httpbin.org/get?a=b",
- *   "https://httpbin.org/get?a=c"
- * ]
- * ```
- *
- * Script:
- *
- * ```ini
- * [URLPager]
- * path = .args
- * ```
- *
- * Output:
- *
- * ```json
- * [{"a": "a"}, {"a": "b"}, {"a": "c" }]
- * ```
- *
- * @name URLPager
- * @param {String} [url] URL to fetch (by default input string is taken)
- * @param {String} [path=total] choose the path to find the number of result
- * @param {Number} [timeout=1000] Timeout in milliseconds
- * @param {Boolean} [noerror=false] Ignore all errors, the target field will remain undefined
- * @param {Number} [retries=5] The maximum amount of times to retry the connection
- * @returns {Object}
- */
-async function URLPager(data, feed) {
-  if (this.isLast()) {
-    return feed.close();
-  }
-
-  const url = this.getParam('url');
-  const path = this.getParam('path', 'total');
-  const limit = Number(this.getParam('limit', 10));
-  const maxPages = Number(this.getParam('maxPages', 1000));
-  const retries = Number(this.getParam('retries', 5));
-  const noerror = Boolean(this.getParam('noerror', false));
-  const timeout = Number(this.getParam('timeout')) || 1000;
-  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
-  const cURL = new _url.URL(url || data);
-  const controller = new _nodeAbortController.default();
-  const parameters = {
-    timeout,
-    headers,
-    signal: controller.signal
-  };
-  const options = {
-    retries
-  };
-  cURL.search = new _url.URLSearchParams(data);
-
-  const onError = e => {
-    controller.abort();
-
-    if (noerror) {
-      (0, _debug.default)('ezs')(`Ignore item #${this.getIndex()} [URLPager] <${e}>`);
-      return feed.send(data);
-    }
-
-    (0, _debug.default)('ezs')(`Break item #${this.getIndex()} [URLPager] <${e}>`);
-    return feed.send(e);
-  };
-
-  try {
-    const response = await (0, _asyncRetry.default)((0, _request.default)(cURL.href, parameters), options);
-    const json = await response.json();
-    const total = (0, _lodash.default)(json, path);
-
-    if (total === 0) {
-      return onError(new Error('No result.'));
-    }
-
-    if (total === undefined) {
-      return onError(new Error('Unexpected response.'));
-    }
-
-    let totalPages = Math.ceil(json.total / limit);
-
-    if (totalPages > maxPages) {
-      totalPages = maxPages;
-    }
-
-    for (let pageNumber = 1; pageNumber <= totalPages; pageNumber += 1) {
-      feed.write({ ...data,
-        offset: (pageNumber - 1) * limit,
-        pageNumber,
-        totalPages,
-        maxPages,
-        limit
-      });
-    }
-
-    feed.end();
-  } catch (e) {
-    onError(e);
-  }
-}

package/lib/utils.js

@@ -1,18 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.default = exports.writeTo = void 0;
-
-const writeTo = function writeTo(stream, data, cb) {
-  if (!stream.write(data)) {
-    stream.once('drain', cb);
-  } else {
-    process.nextTick(cb);
-  }
-};
-
-exports.writeTo = writeTo;
-var _default = writeTo;
-exports.default = _default;