@ezs/basics 2.1.1 → 2.2.0

package/CHANGELOG.md

@@ -3,6 +3,36 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [2.2.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.1.3...@ezs/basics@2.2.0) (2023-06-22)
+
+
+### Features
+
+* 🎸 add tar-extract ([a471aca](https://github.com/Inist-CNRS/ezs/commit/a471aca64b7ab08d92237917264a23080a2c3072))
+
+
+
+
+
+## [2.1.3](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.1.2...@ezs/basics@2.1.3) (2023-05-30)
+
+**Note:** Version bump only for package @ezs/basics
+
+
+
+
+
+## [2.1.2](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.1.1...@ezs/basics@2.1.2) (2023-05-30)
+
+
+### Bug Fixes
+
+* 🐛 retry without json parse ([90e17b5](https://github.com/Inist-CNRS/ezs/commit/90e17b5ea2cbf60f18a9ecbf29b338b1395e318e))
+
+
+
+
+
 ## [2.1.1](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@2.1.0...@ezs/basics@2.1.1) (2023-05-12)
 
 **Note:** Version bump only for package @ezs/basics

package/README.md

@@ -23,11 +23,11 @@ npm install @ezs/basics
 -   [INIString](#inistring)
 -   [JSONParse](#jsonparse)
 -   [JSONString](#jsonstring)
--   [OBJColumns](#objcolumns)
 -   [OBJCount](#objcount)
 -   [OBJFlatten](#objflatten)
 -   [OBJNamespaces](#objnamespaces)
 -   [OBJStandardize](#objstandardize)
+-   [TARExtract](#tarextract)
 -   [TXTConcat](#txtconcat)
 -   [TXTInflection](#txtinflection)
 -   [TXTObject](#txtobject)
@@ -393,38 +393,6 @@ Output:
 
 Returns **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** 
 
-### OBJColumns
-
-Take an `Object` and flatten it to get only one level of keys.
-
-<caption>Input:</caption>
-
-```json
-[{
-  "foo": {
-    "hello": "world"
-  },
-  "bar": "anything else",
-  "baz": 1
-}]
-```
-
-<caption>Output:</caption>
-
-```json
-[{
-  "foo": "{\"hello\":\"world\"}",
-  "bar": "anything else",
-  "baz": 1
-}]
-```
-
-#### Parameters
-
--   `none` **[undefined](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/undefined)** 
-
-Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
-
 ### OBJCount
 
 Count how many objects are received, and yield the total.
@@ -569,6 +537,27 @@ Output:
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 
+### TARExtract
+
+Take the content of a tar file, extract some files.
+The JSON object is sent to the output stream for each file.
+It returns to the output stream
+
+```json
+{
+   "id": "file name",
+   "value": "file contents"
+}
+```
+
+#### Parameters
+
+-   `path` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Regex to select the files to extract (optional, default `"**\/*.json"`)
+-   `json` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** Parse as JSON the content of each file (optional, default `true`)
+-   `compress` **[Boolean](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Boolean)** Enable gzip compression (optional, default `false`)
+
+Returns **[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)&lt;{id: [String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String), value: [String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)}>** 
+
 ### TXTConcat
 
 Concatenate all `String` items into one string

package/lib/index.js

@@ -59,6 +59,8 @@ var _txtZip = _interopRequireDefault(require("./txt-zip"));
 
 var _zipExtract = _interopRequireDefault(require("./zip-extract"));
 
+var _tarExtract = _interopRequireDefault(require("./tar-extract"));
+
 var _iniString = _interopRequireDefault(require("./ini-string"));
 
 var _fileSave = _interopRequireDefault(require("./file-save"));
@@ -95,6 +97,7 @@ const funcs = {
   URLConnect: _urlConnect.default,
   TXTZip: _txtZip.default,
   ZIPExtract: _zipExtract.default,
+  TARExtract: _tarExtract.default,
   INIString: _iniString.default,
   FILESave: _fileSave.default,
   FILELoad: _fileLoad.default,

package/lib/tar-extract.js

@@ -0,0 +1,90 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = TARExtract;
+
+var _tarStream = _interopRequireDefault(require("tar-stream"));
+
+var _micromatch = _interopRequireDefault(require("micromatch"));
+
+var _zlib = require("zlib");
+
+var _getStream = _interopRequireDefault(require("get-stream"));
+
+var _streamWrite = _interopRequireDefault(require("stream-write"));
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+/**
+ * Take the content of a tar file, extract some files.
+ * The JSON object is sent to the output stream for each file.
+ * It returns to the output stream
+ *
+ * ```json
+ * {
+ *    "id": "file name",
+ *    "value": "file contents"
+ * }
+ * ```
+ *
+ * @name TARExtract
+ * @param {String} [path="**\/*.json"] Regex to select the files to extract
+ * @param {String} [json=true] Parse as JSON the content of each file
+ * @param {Boolean} [compress=false] Enable gzip compression
+ * @returns {{id: String, value: String}[]}
+ */
+function TARExtract(data, feed) {
+  const filesPatern = this.getParam('path', '**/*.json');
+
+  if (this.isFirst()) {
+    const {
+      ezs
+    } = this;
+    const json = this.getParam('json', true);
+    const compress = this.getParam('compress', false);
+    this.input = ezs.createStream(ezs.objectMode());
+    this.output = ezs.createStream(ezs.objectMode());
+
+    const extract = _tarStream.default.extract();
+
+    this.whenEnd = new Promise((resolve, reject) => {
+      extract.on('entry', async (header, stream, next) => {
+        if (_micromatch.default.isMatch(header.name, filesPatern)) {
+          const contentRaw = await (0, _getStream.default)(stream);
+
+          if (json) {
+            const contentJson = JSON.parse(contentRaw);
+            return (0, _streamWrite.default)(this.output, contentJson, () => next());
+          }
+
+          return (0, _streamWrite.default)(this.output, {
+            id: header.name,
+            value: contentRaw
+          }, () => next());
+        }
+
+        return next();
+      });
+      extract.on('error', reject);
+      extract.on('finish', resolve);
+    });
+
+    if (compress) {
+      this.input.pipe((0, _zlib.createGunzip)()).pipe(extract);
+    } else {
+      this.input.pipe(extract);
+    }
+
+    this.whenFinish = feed.flow(this.output);
+  }
+
+  if (this.isLast()) {
+    this.whenEnd.finally(() => this.output.end());
+    this.whenFinish.finally(() => feed.close());
+    this.input.end();
+  } else {
+    (0, _streamWrite.default)(this.input, data, () => feed.end());
+  }
+}

package/lib/url-connect.js

@@ -106,9 +106,13 @@ async function URLConnect(data, feed) {
           });
           bodyOut.pipe(output);
         } else {
-          const bodyOutRaw = await (0, _getStream.default)(response.body);
-          const bodyOutArray = JSON.parse(bodyOutRaw);
-          (0, _from.default)(bodyOutArray).pipe(output);
+          if (json) {
+            const bodyOutRaw = await (0, _getStream.default)(response.body);
+            const bodyOutArray = JSON.parse(bodyOutRaw);
+            (0, _from.default)(bodyOutArray).pipe(output);
+          } else {
+            response.body.pipe(output);
+          }
         }
       }, {
         retries

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ezs/basics",
   "description": "Basics statements for EZS",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "author": "Nicolas Thouvenin <nthouvenin@gmail.com>",
   "bugs": "https://github.com/Inist-CNRS/ezs/issues",
   "dependencies": {
@@ -28,6 +28,7 @@
     "parse-headers": "2.0.4",
     "path-exists": "4.0.0",
     "stream-write": "1.0.1",
+    "tar-stream": "3.1.4",
     "tmp-filepath": "2.0.0",
     "unzipper": "0.10.11",
     "xml-mapping": "1.7.2",
@@ -40,7 +41,7 @@
   "directories": {
     "test": "test"
   },
-  "gitHead": "54e12b40d2cd6ccb6754f06ab5cae566c60bb3a9",
+  "gitHead": "6868634896b6171224fe54c59360a3668d71bd5f",
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "keywords": [
     "ezs"