@hestia-earth/utils 0.10.28 → 0.10.29

package/dist/validate-terms.js

@@ -51,6 +51,11 @@ var __read = (this && this.__read) || function (o, n) {
     }
     return ar;
 };
+var __spreadArray = (this && this.__spreadArray) || function (to, from) {
+    for (var i = 0, il = from.length, j = to.length; i < il; i++, j++)
+        to[j] = from[i];
+    return to;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.run = exports.validateTerms = void 0;
 var fs_1 = require("fs");
@@ -119,30 +124,59 @@ var getAllTerms = function (data) {
     })
         .filter(function (term) { return (!!term) && (term['@type'] === 'Term') && (!!term['@id'] || !!term.name); });
 };
-var validateTerms = function (terms) { return __awaiter(void 0, void 0, void 0, function () {
-    var query, results;
-    return __generator(this, function (_a) {
-        switch (_a.label) {
+var searchLimit = 500;
+var searchTerms = function (terms) { return __awaiter(void 0, void 0, void 0, function () {
+    var searchedTerms, results, _a, _b, _c;
+    return __generator(this, function (_d) {
+        switch (_d.label) {
             case 0:
-                query = {
-                    bool: {
-                        must: [{
-                                match: { '@type': 'Term' }
-                            }],
-                        should: terms.map(function (_a) {
-                            var id = _a["@id"], name = _a.name;
-                            return ({ match: id ? { '@id.keyword': id } : { 'name.keyword': name } });
-                        }),
-                        minimum_should_match: 1
-                    }
-                };
+                searchedTerms = terms.slice(0, searchLimit);
+                if (!searchedTerms.length) return [3, 2];
                 return [4, search({
-                        limit: 1000,
+                        limit: searchLimit,
                         fields: ['@id', 'name'],
-                        query: query
+                        query: {
+                            bool: {
+                                must: [{
+                                        match: { '@type': 'Term' }
+                                    }],
+                                should: searchedTerms
+                                    .map(function (_a) {
+                                    var id = _a["@id"], name = _a.name;
+                                    return ({ match: id ? { '@id.keyword': id } : { 'name.keyword': name } });
+                                }),
+                                minimum_should_match: 1
+                            }
+                        }
                     })];
             case 1:
-                results = (_a.sent()).results;
+                _a = _d.sent();
+                return [3, 3];
+            case 2:
+                _a = { results: [] };
+                _d.label = 3;
+            case 3:
+                results = (_a).results;
+                _b = [__spreadArray([], __read(results))];
+                if (!(terms.length > searchLimit)) return [3, 5];
+                return [4, searchTerms(terms.slice(searchLimit))];
+            case 4:
+                _c = _d.sent();
+                return [3, 6];
+            case 5:
+                _c = [];
+                _d.label = 6;
+            case 6: return [2, __spreadArray.apply(void 0, _b.concat([__read.apply(void 0, [(_c)])]))];
+        }
+    });
+}); };
+var validateTerms = function (terms) { return __awaiter(void 0, void 0, void 0, function () {
+    var results;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0: return [4, searchTerms(terms)];
+            case 1:
+                results = _a.sent();
                 return [2, terms.map(function (_a) {
                         var id = _a["@id"], name = _a.name;
                         var exists = results.find(function (res) { return res['@id'] === id || res.name === name; });
@@ -152,43 +186,37 @@ var validateTerms = function (terms) { return __awaiter(void 0, void 0, void 0,
     });
 }); };
 exports.validateTerms = validateTerms;
-var validateFile = function (_a) {
-    var filepath = _a.filepath, data = _a.data;
-    return __awaiter(void 0, void 0, void 0, function () {
-        var terms, missing;
-        return __generator(this, function (_b) {
-            switch (_b.label) {
-                case 0:
-                    terms = array_1.unique(getAllTerms(data));
-                    return [4, exports.validateTerms(terms)];
-                case 1:
-                    missing = _b.sent();
-                    return [2, {
-                            filepath: filepath,
-                            missing: missing
-                        }];
-            }
-        });
+var validateFile = function (existingTerms) { return function (_a) {
+    var filepath = _a.filepath, terms = _a.terms;
+    return ({
+        filepath: filepath,
+        missing: terms.map(function (_a) {
+            var id = _a["@id"], name = _a.name;
+            return existingTerms.find(function (res) { return res['@id'] === id || res.name === name; })
+                ? true
+                : id || name;
+        }).filter(function (val) { return val !== true; })
     });
-};
+}; };
 var loadFile = function (filepath) { return __awaiter(void 0, void 0, void 0, function () {
-    var _a, _b;
-    var _c;
-    return __generator(this, function (_d) {
-        switch (_d.label) {
+    var _a, _b, _c;
+    var _d;
+    return __generator(this, function (_e) {
+        switch (_e.label) {
             case 0:
-                _c = {
+                _d = {
                     filepath: filepath
                 };
-                _b = (_a = JSON).parse;
+                _a = getAllTerms;
+                _c = (_b = JSON).parse;
                 return [4, readFile(filepath, encoding)];
-            case 1: return [2, (_c.data = _b.apply(_a, [_d.sent()]),
-                    _c)];
+            case 1: return [2, (_d.terms = _a.apply(void 0, [_c.apply(_b, [_e.sent()])]),
+                    _d)];
         }
     });
 }); };
 var run = function () { return __awaiter(void 0, void 0, void 0, function () {
-    var filepaths, files, results, missingFiles;
+    var filepaths, fileData, allTerms, existingTerms, results, missingFiles;
     return __generator(this, function (_a) {
         switch (_a.label) {
             case 0: return [4, recursiveFiles(folder)];
@@ -196,10 +224,15 @@ var run = function () { return __awaiter(void 0, void 0, void 0, function () {
                 filepaths = _a.sent();
                 return [4, Promise.all(filepaths.map(loadFile))];
             case 2:
-                files = _a.sent();
-                return [4, Promise.all(files.map(validateFile))];
+                fileData = _a.sent();
+                allTerms = array_1.unique(fileData.flatMap(function (_a) {
+                    var terms = _a.terms;
+                    return terms;
+                }));
+                return [4, searchTerms(allTerms)];
             case 3:
-                results = (_a.sent()).filter(Boolean);
+                existingTerms = _a.sent();
+                results = fileData.map(validateFile(existingTerms)).filter(Boolean);
                 missingFiles = results.filter(function (_a) {
                     var missing = _a.missing;
                     return missing.length;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hestia-earth/utils",
-  "version": "0.10.28",
+  "version": "0.10.29",
   "description": "Hestia Utils library",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -53,4 +53,4 @@
   "peerDependencies": {
     "@hestia-earth/glossary": ">=0.8.0"
   }
-}
+}