@ezs/basics 1.13.6 → 1.15.3

package/CHANGELOG.md

@@ -3,6 +3,55 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [1.15.3](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.15.2...@ezs/basics@1.15.3) (2022-01-31)
+
+**Note:** Version bump only for package @ezs/basics
+
+
+
+
+
+## [1.15.2](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.15.1...@ezs/basics@1.15.2) (2022-01-27)
+
+**Note:** Version bump only for package @ezs/basics
+
+
+
+
+
+## [1.15.1](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.15.0...@ezs/basics@1.15.1) (2021-10-05)
+
+
+### Bug Fixes
+
+* 🐛 fix wrong options + miss dep.x ([5b38d05](https://github.com/Inist-CNRS/ezs/commit/5b38d05199a9a49c73d264f4ddb9a45dd0e64c7e))
+
+
+
+
+
+# [1.15.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.14.0...@ezs/basics@1.15.0) (2021-10-05)
+
+
+### Features
+
+* 🎸 introduce new parser ([66a408b](https://github.com/Inist-CNRS/ezs/commit/66a408b0be2f6f2374d7193df2576e2fa383d369))
+
+
+
+
+
+# [1.14.0](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.13.6...@ezs/basics@1.14.0) (2021-08-27)
+
+
+### Features
+
+* 🎸 support http headers ([0d11b7c](https://github.com/Inist-CNRS/ezs/commit/0d11b7c46a44040c48aa0d0fcb7954611dd36658))
+
+
+
+
+
 ## [1.13.6](https://github.com/Inist-CNRS/ezs/compare/@ezs/basics@1.13.5...@ezs/basics@1.13.6) (2021-07-30)
 
 **Note:** Version bump only for package @ezs/basics

package/README.md

@@ -284,6 +284,7 @@ Output:
 #### Parameters
 
 -   `separator` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** to split at every JSONPath found (optional, default `"*"`)
+-   `legacy` **[String](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** use legacy or newer parser (separator should be different) (optional, default `true`)
 
 Returns **[Object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** 
 

package/lib/fetch.js

@@ -0,0 +1,74 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = fetch;
+
+var _crossFetch = _interopRequireDefault(require("cross-fetch"));
+
+var _proxyFromEnv = require("proxy-from-env");
+
+var _http = _interopRequireDefault(require("http"));
+
+var _https = _interopRequireDefault(require("https"));
+
+var _betterHttpsProxyAgent = require("better-https-proxy-agent");
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+const DefaultOptions = {
+  keepAlive: true,
+  timeout: 1000,
+  keepAliveMsecs: 500,
+  maxSockets: 200,
+  maxFreeSockets: 5,
+  maxCachedSessions: 500
+};
+
+const chooseAgent = (parsedURL, options) => {
+  const proxyurl = (0, _proxyFromEnv.getProxyForUrl)(parsedURL.href);
+
+  if (proxyurl) {
+    const proxyRequestOptions = new URL(proxyurl);
+    return new _betterHttpsProxyAgent.Agent(options, proxyRequestOptions);
+  }
+
+  if (parsedURL.protocol === 'https:') {
+    return new _https.default.Agent(options);
+  }
+
+  return new _http.default.Agent(options);
+};
+
+function fetch(url, options) {
+  const opts = options || {};
+  const {
+    keepAlive,
+    timeout,
+    keepAliveMsecs,
+    maxSockets,
+    maxFreeSockets,
+    maxCachedSessions
+  } = { ...options,
+    ...DefaultOptions
+  };
+  let agent = chooseAgent(new URL(url), {
+    keepAlive,
+    timeout,
+    keepAliveMsecs,
+    maxSockets,
+    maxFreeSockets,
+    maxCachedSessions
+  });
+  opts.agent = agent;
+
+  if (opts.signal) {
+    opts.signal.addEventListener('abort', () => {
+      agent.destroy();
+      agent = null;
+    });
+  }
+
+  return (0, _crossFetch.default)(url, options);
+}

package/lib/json-parse.js

@@ -7,25 +7,35 @@ exports.default = void 0;
 
 var _JSONStream = _interopRequireDefault(require("JSONStream"));
 
+var _stream = require("stream");
+
+var _yajsonStream = _interopRequireDefault(require("yajson-stream"));
+
 var _streamWrite = _interopRequireDefault(require("stream-write"));
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 function JSONParse(data, feed) {
-  if (!this.handle) {
+  if (this.isFirst()) {
+    const legacy = this.getParam('legacy', true);
     const separator = this.getParam('separator', '*');
-    this.handle = _JSONStream.default.parse(separator);
-    this.handle.on('data', obj => feed.write(obj));
+
+    if (legacy) {
+      this.input = _JSONStream.default.parse(separator);
+      this.whenFinish = feed.flow(this.input);
+    } else {
+      this.input = new _stream.PassThrough();
+      const stream = this.input.pipe(this.ezs.toBuffer()).pipe((0, _yajsonStream.default)(separator)).pipe(this.ezs((d, f) => f.send(d ? d.value : d)));
+      this.whenFinish = feed.flow(stream);
+    }
   }
 
-  if (!this.isLast()) {
-    (0, _streamWrite.default)(this.handle, data, () => feed.end());
-  } else {
-    this.handle.end();
-    process.nextTick(() => {
-      feed.close();
-    });
+  if (this.isLast()) {
+    this.whenFinish.finally(() => feed.close());
+    return this.input.end();
   }
+
+  (0, _streamWrite.default)(this.input, data, () => feed.end());
 }
 /**
  * Parse a `String` to JSON and generate objects.
@@ -69,6 +79,7 @@ function JSONParse(data, feed) {
  *
  * @name JSONParse
  * @param {String} [separator="*"] to split at every JSONPath found
+ * @param {String} [legacy=true] use legacy or newer parser (separator should be different)
  * @returns {Object}
  */
 

package/lib/skos-object.js

@@ -0,0 +1,73 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = void 0;
+
+function Concept(data, feed) {
+  const obj = {};
+  Object.keys(data).forEach(key => {
+    const newkey = key.replace('skos$', '');
+
+    if (Array.isArray(data[key])) {
+      data[key].filter(x => x.xml$lang).forEach(item => {
+        const localkey = newkey.concat('@').concat(item.xml$lang);
+        obj[localkey] = item.$t;
+      });
+    } else if (data[key].rdf$resource && !obj[newkey]) {
+      obj[newkey] = data[key].rdf$resource;
+    } else if (data[key].rdf$resource && obj[newkey]) {
+      obj[newkey] = [obj[newkey], data[key].rdf$resource];
+    } else if (data[key].$t && data[key].xml$lang) {
+      const localkey = newkey.concat('@').concat(data[key].xml$lang);
+      obj[localkey] = data[key].$t;
+    } else if (data[key].$t && Array.isArray(obj[newkey])) {
+      obj[newkey].push(data[key].$t);
+    } else if (data[key].$t && obj[newkey]) {
+      obj[newkey] = [obj[newkey], data[key].$t];
+    } else if (data[key].$t && !obj[newkey]) {
+      obj[newkey] = data[key].$t;
+    } else if (typeof data[key] === 'object') {
+      obj[newkey] = (this.getIndex().toString(36) + Math.random().toString(36).substr(2, 5)).toUpperCase();
+      let counter = 0;
+      Object.keys(data[key]).forEach(key2 => {
+        if (typeof data[key][key2] === 'object') {
+          data[key][key2].rdf$about = obj[newkey];
+          Concept.call(this, data[key][key2], feed);
+          counter += 1;
+        }
+      });
+
+      if (counter === 0) {
+        delete obj[newkey];
+      }
+    } else {
+      obj[newkey] = data[key];
+    }
+  });
+  feed.write(obj);
+}
+
+function SKOSObject(data, feed) {
+  if (this.isLast()) {
+    feed.close();
+  } else {
+    Concept.call(this, data, feed);
+    feed.end();
+  }
+}
+/**
+ * Take `Object` generated by XMLMapping & SKOS data and
+ * create a new basic object with only keys & values
+ *
+ * @name SKOSObject
+ * @param {undefined} none
+ * @returns {Object}
+ */
+
+
+var _default = {
+  SKOSObject
+};
+exports.default = _default;

package/lib/url-connect.js

@@ -15,6 +15,8 @@ var _nodeAbortController = _interopRequireDefault(require("node-abort-controller
 
 var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
 
+var _parseHeaders = _interopRequireDefault(require("parse-headers"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 /**
@@ -41,6 +43,7 @@ function URLConnect(data, feed) {
 
   if (this.isFirst()) {
     const timeout = Number(this.getParam('timeout')) || 1000;
+    const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
     const controller = new _nodeAbortController.default();
     this.input = ezs.createStream(ezs.objectMode());
     this.whenReady = (0, _fetchWithProxy.default)(url, {

package/lib/url-fetch.js

@@ -15,6 +15,8 @@ var _nodeAbortController = _interopRequireDefault(require("node-abort-controller
 
 var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
 
+var _parseHeaders = _interopRequireDefault(require("parse-headers"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 /**
@@ -43,12 +45,14 @@ async function URLFetch(data, feed) {
   const json = Boolean(this.getParam('json', false));
   const noerror = Boolean(this.getParam('noerror', false));
   const timeout = Number(this.getParam('timeout')) || 1000;
+  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
   const mimetype = String(this.getParam('mimetype', 'application/json'));
   const controller = new _nodeAbortController.default();
   const key = Array.isArray(path) ? path.shift() : path;
   const body = (0, _lodash.default)(data, key);
   const parameters = {
     timeout,
+    headers,
     signal: controller.signal
   };
 

package/lib/url-stream.js

@@ -15,6 +15,8 @@ var _JSONStream = _interopRequireDefault(require("JSONStream"));
 
 var _fetchWithProxy = _interopRequireDefault(require("fetch-with-proxy"));
 
+var _parseHeaders = _interopRequireDefault(require("parse-headers"));
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 /**
@@ -91,6 +93,7 @@ async function URLStream(data, feed) {
   const path = this.getParam('path', '*');
   const noerror = Boolean(this.getParam('noerror', false));
   const timeout = Number(this.getParam('timeout')) || 1000;
+  const headers = (0, _parseHeaders.default)([].concat(this.getParam('header')).filter(Boolean).join('\n'));
   const cURL = new _url.URL(url || data);
   const controller = new _nodeAbortController.default();
 
@@ -100,6 +103,7 @@ async function URLStream(data, feed) {
 
   try {
     const response = await (0, _fetchWithProxy.default)(cURL.href, {
+      headers,
       timeout,
       signal: controller.signal
     });

package/lib/utils.js

@@ -0,0 +1,18 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.default = exports.writeTo = void 0;
+
+const writeTo = function writeTo(stream, data, cb) {
+  if (!stream.write(data)) {
+    stream.once('drain', cb);
+  } else {
+    process.nextTick(cb);
+  }
+};
+
+exports.writeTo = writeTo;
+var _default = writeTo;
+exports.default = _default;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ezs/basics",
-  "version": "1.13.6",
+  "version": "1.15.3",
   "description": "Basics statements for EZS",
   "directories": {
     "test": "test"
@@ -19,12 +19,12 @@
   },
   "homepage": "https://github.com/Inist-CNRS/ezs/tree/master/packages/basics#readme",
   "dependencies": {
-    "JSONStream": "~1.3.5",
+    "JSONStream": "1.3.5",
     "better-https-proxy-agent": "1.0.8",
-    "csv-string": "~3.2.0",
-    "debug": "4.1.1",
+    "csv-string": "3.2.0",
+    "debug": "4.3.2",
     "fetch-with-proxy": "3.0.1",
-    "flat": "~5.0.2",
+    "flat": "5.0.2",
     "lodash.escaperegexp": "4.1.2",
     "lodash.get": "4.4.2",
     "lodash.mapkeys": "4.6.0",
@@ -33,11 +33,13 @@
     "lodash.zipobject": "4.1.3",
     "micromatch": "4.0.2",
     "node-abort-controller": "1.1.0",
+    "parse-headers": "2.0.4",
     "stream-write": "1.0.1",
     "tmp-filepath": "2.0.0",
     "unzipper": "0.10.11",
-    "xml-mapping": "~1.7.1",
-    "xml-splitter": "~1.2.1"
+    "xml-mapping": "1.7.1",
+    "xml-splitter": "1.2.1",
+    "yajson-stream": "1.3.3"
   },
   "main": "./lib/index.js",
   "scripts": {
@@ -52,11 +54,10 @@
     "access": "public"
   },
   "devDependencies": {
-    "@ezs/core": "1.3.1",
-    "pako": "~1.0.11"
+    "pako": "1.0.11"
   },
   "peerDependencies": {
-    "@ezs/core": "*"
+    "@ezs/core": "1.27.0"
   },
-  "gitHead": "cf443b00f83d11656a21f89e8471e5247c1c029f"
+  "gitHead": "2df397fbd92f852e7d7a1d434809d1b0367238c5"
 }