@hestia-earth/utils 0.17.15 → 0.17.16

package/dist/esm/validate-terms.js

@@ -0,0 +1,267 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+import { promises } from 'fs';
+import { join } from 'path';
+import { request } from 'https';
+import { unique } from './array';
+var API_URL = process.env.API_URL || 'https://api.hestia.earth';
+var _a = __read(process.argv.slice(2), 2), folder = _a[0], extension = _a[1];
+var readdir = promises.readdir, lstat = promises.lstat, readFile = promises.readFile;
+var encoding = 'utf8';
+var Term = /** @class */ (function () {
+    function Term() {
+    }
+    return Term;
+}());
+var search = function (body) {
+    return new Promise(function (resolve, reject) {
+        var req = request("".concat(API_URL, "/terms/search"), {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' }
+        }, function (res) {
+            res.setEncoding(encoding);
+            var data = [];
+            res.on('data', function (chunk) { return data.push(chunk); });
+            res.on('end', function () { return resolve(JSON.parse(data.join(''))); });
+            res.on('error', reject);
+        });
+        req.write(JSON.stringify(body));
+        req.end();
+    });
+};
+var recursiveFiles = function (directory) { return __awaiter(void 0, void 0, void 0, function () {
+    var _a, _b;
+    return __generator(this, function (_c) {
+        switch (_c.label) {
+            case 0:
+                _b = (_a = Promise).all;
+                return [4 /*yield*/, readdir(directory)];
+            case 1: return [4 /*yield*/, _b.apply(_a, [(_c.sent()).map(function (entry) { return __awaiter(void 0, void 0, void 0, function () {
+                        var _a;
+                        return __generator(this, function (_b) {
+                            switch (_b.label) {
+                                case 0: return [4 /*yield*/, lstat(join(directory, entry))];
+                                case 1:
+                                    if (!(_b.sent()).isDirectory()) return [3 /*break*/, 3];
+                                    return [4 /*yield*/, recursiveFiles(join(directory, entry))];
+                                case 2:
+                                    _a = _b.sent();
+                                    return [3 /*break*/, 4];
+                                case 3:
+                                    _a = join(directory, entry).endsWith(".".concat(extension || 'jsonld'))
+                                        ? [join(directory, entry)]
+                                        : [];
+                                    _b.label = 4;
+                                case 4: return [2 /*return*/, _a];
+                            }
+                        });
+                    }); })])];
+            case 2: return [2 /*return*/, (_c.sent()).flat()];
+        }
+    });
+}); };
+var getAllTerms = function (data) {
+    return Object.keys(data)
+        .filter(function (key) { return Array.isArray(data[key]) || (typeof data[key] === 'object' && '@type' in data[key]); })
+        .flatMap(function (key) {
+        var value = data[key];
+        var isArray = Array.isArray(value);
+        return isArray ? value.flatMap(getAllTerms) : value['@type'] === 'Term' ? value : getAllTerms(value);
+    })
+        .filter(function (term) { return !!term && term['@type'] === 'Term' && (!!term['@id'] || !!term.name); });
+};
+// search is limited to 1000 parameters
+var searchLimit = 500;
+var searchTerms = function (terms) { return __awaiter(void 0, void 0, void 0, function () {
+    var searchedTerms, results, _a, _b, _c;
+    return __generator(this, function (_d) {
+        switch (_d.label) {
+            case 0:
+                searchedTerms = terms.slice(0, searchLimit);
+                if (!searchedTerms.length) return [3 /*break*/, 2];
+                return [4 /*yield*/, search({
+                        limit: searchLimit,
+                        fields: ['@id', 'name'],
+                        query: {
+                            bool: {
+                                must: [
+                                    {
+                                        match: { '@type': 'Term' }
+                                    }
+                                ],
+                                should: searchedTerms.map(function (_a) {
+                                    var id = _a["@id"], name = _a.name;
+                                    return ({
+                                        match: id ? { '@id.keyword': id } : { 'name.keyword': name }
+                                    });
+                                }),
+                                minimum_should_match: 1
+                            }
+                        }
+                    })];
+            case 1:
+                _a = _d.sent();
+                return [3 /*break*/, 3];
+            case 2:
+                _a = { results: [] };
+                _d.label = 3;
+            case 3:
+                results = (_a).results;
+                _b = [__spreadArray([], __read(results), false)];
+                if (!(terms.length > searchLimit)) return [3 /*break*/, 5];
+                return [4 /*yield*/, searchTerms(terms.slice(searchLimit))];
+            case 4:
+                _c = _d.sent();
+                return [3 /*break*/, 6];
+            case 5:
+                _c = [];
+                _d.label = 6;
+            case 6: return [2 /*return*/, __spreadArray.apply(void 0, _b.concat([__read.apply(void 0, [(_c)]), false]))];
+        }
+    });
+}); };
+/**
+ * Validate a list of Terms.
+ *
+ * @param terms The list of Terms to validate.
+ * @returns The list of ids/names that could not be found (invalid).
+ */
+export var validateTerms = function (terms) { return __awaiter(void 0, void 0, void 0, function () {
+    var results;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0: return [4 /*yield*/, searchTerms(terms)];
+            case 1:
+                results = _a.sent();
+                return [2 /*return*/, terms
+                        .map(function (_a) {
+                        var id = _a["@id"], name = _a.name;
+                        var exists = results.find(function (res) { return res['@id'] === id || res.name === name; });
+                        return exists ? true : id || name;
+                    })
+                        .filter(function (val) { return val !== true; })];
+        }
+    });
+}); };
+var validateFile = function (existingTerms) {
+    return function (_a) {
+        var filepath = _a.filepath, terms = _a.terms;
+        return ({
+            filepath: filepath,
+            missing: terms
+                .map(function (_a) {
+                var id = _a["@id"], name = _a.name;
+                return existingTerms.find(function (res) { return res['@id'] === id || res.name === name; }) ? true : id || name;
+            })
+                .filter(function (val) { return val !== true; })
+        });
+    };
+};
+var loadFile = function (filepath) { return __awaiter(void 0, void 0, void 0, function () {
+    var _a, _b, _c;
+    var _d;
+    return __generator(this, function (_e) {
+        switch (_e.label) {
+            case 0:
+                _d = {
+                    filepath: filepath
+                };
+                _a = getAllTerms;
+                _c = (_b = JSON).parse;
+                return [4 /*yield*/, readFile(filepath, encoding)];
+            case 1: return [2 /*return*/, (_d.terms = _a.apply(void 0, [_c.apply(_b, [_e.sent()])]),
+                    _d)];
+        }
+    });
+}); };
+export var run = function () { return __awaiter(void 0, void 0, void 0, function () {
+    var filepaths, fileData, allTerms, existingTerms, results, missingFiles;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0: return [4 /*yield*/, recursiveFiles(folder)];
+            case 1:
+                filepaths = _a.sent();
+                return [4 /*yield*/, Promise.all(filepaths.map(loadFile))];
+            case 2:
+                fileData = _a.sent();
+                allTerms = unique(fileData.flatMap(function (_a) {
+                    var terms = _a.terms;
+                    return terms;
+                }));
+                return [4 /*yield*/, searchTerms(allTerms)];
+            case 3:
+                existingTerms = _a.sent();
+                results = fileData.map(validateFile(existingTerms)).filter(Boolean);
+                missingFiles = results.filter(function (_a) {
+                    var missing = _a.missing;
+                    return missing.length;
+                });
+                if (missingFiles.length) {
+                    throw new Error("\n  Terms not found:\n  \t".concat(missingFiles.map(function (_a) {
+                        var filepath = _a.filepath, missing = _a.missing;
+                        return "- ".concat(filepath, ": ").concat(missing.join(', '));
+                    }).join('\n\t'), "\n    ").trim());
+                }
+                return [2 /*return*/, filepaths];
+        }
+    });
+}); };

package/dist/validate-terms.js

@@ -77,7 +77,7 @@ var Term = /** @class */ (function () {
 }());
 var search = function (body) {
     return new Promise(function (resolve, reject) {
-        var req = (0, https_1.request)("".concat(API_URL, "/search"), {
+        var req = (0, https_1.request)("".concat(API_URL, "/terms/search"), {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' }
         }, function (res) {

package/package.json

@@ -1,14 +1,23 @@
 {
   "name": "@hestia-earth/utils",
-  "version": "0.17.15",
+  "version": "0.17.16",
   "description": "HESTIA Utils library",
   "main": "dist/index.js",
+  "module": "dist/esm/index.js",
   "typings": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/esm/index.js",
+      "require": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "sideEffects": false,
   "bin": {
     "hestia-validate-terms": "dist/bin/bin-validate-terms.js"
   },
   "scripts": {
-    "build": "rm -rf dist && tsc -p tsconfig.dist.json",
+    "build": "rm -rf dist && tsc -p tsconfig.dist.json && tsc -p tsconfig.dist.esm.json",
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "npm run lint -- --fix",
     "test": "jest",