@hestia-earth/schema-convert 30.0.1 → 30.2.0

package/bin/bin-convert-to-csv-pivot.js

@@ -2,9 +2,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 var convert_csv_1 = require("../convert-csv");
+var now = new Date().getTime();
 (0, convert_csv_1.run)(true)
     .then(function (paths) {
-    console.log('Done converting', paths.length, 'files.');
+    console.log('Done converting', paths.length, 'files in', (new Date().getTime() - now) / 1000, 's');
     console.log(paths.join('\n'));
     process.exit(0);
 })

package/bin/bin-convert-to-csv.js

@@ -2,9 +2,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 var convert_csv_1 = require("../convert-csv");
+var now = new Date().getTime();
 (0, convert_csv_1.run)()
     .then(function (paths) {
-    console.log('Done converting', paths.length, 'files.');
+    console.log('Done converting', paths.length, 'files in', (new Date().getTime() - now) / 1000, 's');
     console.log(paths.join('\n'));
     process.exit(0);
 })

package/bin/bin-convert-to-json.js

@@ -2,9 +2,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 var convert_json_1 = require("../convert-json");
+var now = new Date().getTime();
 (0, convert_json_1.run)()
     .then(function (paths) {
-    console.log('Done converting', paths.length, 'files.');
+    console.log('Done converting', paths.length, 'files in', (new Date().getTime() - now) / 1000, 's');
     console.log(paths.join('\n'));
     process.exit(0);
 })

package/bin/bin-pivot-file.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/bin/bin-pivot-file.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var pivot_file_1 = require("../pivot-file");
+var now = new Date().getTime();
+(0, pivot_file_1.run)()
+    .then(function (paths) {
+    console.log('Done pivoting', paths.length, 'files in', (new Date().getTime() - now) / 1000, 's');
+    console.log(paths.join('\n'));
+    process.exit(0);
+})
+    .catch(function (err) {
+    console.log('Error pivoting files.');
+    console.error(err);
+    process.exit(1);
+});

package/convert-csv.js

@@ -54,24 +54,14 @@ var __read = (this && this.__read) || function (o, n) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.run = void 0;
 var fs_1 = require("fs");
-var path_1 = require("path");
+var json_schema_1 = require("@hestia-earth/json-schema");
+var file_utils_1 = require("./file-utils");
 var csv_pivot_1 = require("./csv-pivot");
 var csv_1 = require("./csv");
 var _a = __read(process.argv.slice(2), 1), path = _a[0];
-var readdir = fs_1.promises.readdir, readFile = fs_1.promises.readFile, writeFile = fs_1.promises.writeFile, lstat = fs_1.promises.lstat;
+var readFile = fs_1.promises.readFile, writeFile = fs_1.promises.writeFile;
 var encoding = 'utf8';
-var listFiles = function (folder) { return __awaiter(void 0, void 0, void 0, function () {
-    var files;
-    return __generator(this, function (_a) {
-        switch (_a.label) {
-            case 0: return [4 /*yield*/, readdir(folder)];
-            case 1:
-                files = _a.sent();
-                return [2 /*return*/, files.filter(function (file) { return file.endsWith('.jsonld') || file.endsWith('.json'); })];
-        }
-    });
-}); };
-var processFile = function (filepath, pivot) { return __awaiter(void 0, void 0, void 0, function () {
+var processFile = function (schemas, filepath, pivot) { return __awaiter(void 0, void 0, void 0, function () {
     var data, content, nodes, res, destPath;
     return __generator(this, function (_a) {
         switch (_a.label) {
@@ -80,7 +70,7 @@ var processFile = function (filepath, pivot) { return __awaiter(void 0, void 0,
                 data = _a.sent();
                 content = JSON.parse(data.trim());
                 nodes = Array.isArray(content) ? content : 'nodes' in content ? content.nodes : [content];
-                res = pivot ? (0, csv_pivot_1.toCsvPivot)(nodes) : (0, csv_1.toCsv)(nodes);
+                res = pivot ? (0, csv_pivot_1.toCsvPivot)(schemas, nodes) : (0, csv_1.toCsv)(nodes);
                 destPath = filepath.replace('.jsonld', '.csv').replace('.json', '.csv');
                 return [4 /*yield*/, writeFile(destPath, res, encoding)];
             case 2:
@@ -92,22 +82,15 @@ var processFile = function (filepath, pivot) { return __awaiter(void 0, void 0,
 var run = function (pivot) {
     if (pivot === void 0) { pivot = false; }
     return __awaiter(void 0, void 0, void 0, function () {
-        var stat, _a, _b, _c;
-        return __generator(this, function (_d) {
-            switch (_d.label) {
-                case 0: return [4 /*yield*/, lstat(path)];
+        var schemas, files;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    schemas = (0, json_schema_1.loadSchemas)();
+                    return [4 /*yield*/, (0, file_utils_1.listFiles)(path, '.jsonld', '.json')];
                 case 1:
-                    stat = _d.sent();
-                    _b = (_a = Promise).all;
-                    if (!stat.isDirectory()) return [3 /*break*/, 3];
-                    return [4 /*yield*/, listFiles(path)];
-                case 2:
-                    _c = (_d.sent()).map(function (file) { return processFile((0, path_1.join)(path, file), pivot); });
-                    return [3 /*break*/, 4];
-                case 3:
-                    _c = [processFile(path, pivot)];
-                    _d.label = 4;
-                case 4: return [2 /*return*/, _b.apply(_a, [_c])];
+                    files = _a.sent();
+                    return [2 /*return*/, Promise.all(files.map(function (file) { return processFile(schemas, file, pivot); }))];
             }
         });
     });

package/convert-json.js

@@ -54,23 +54,13 @@ var __read = (this && this.__read) || function (o, n) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.run = void 0;
 var fs_1 = require("fs");
-var path_1 = require("path");
 var json_schema_1 = require("@hestia-earth/json-schema");
+var file_utils_1 = require("./file-utils");
 var json_1 = require("./json");
 var _a = __read(process.argv.slice(2), 1), path = _a[0];
-var readdir = fs_1.promises.readdir, readFile = fs_1.promises.readFile, writeFile = fs_1.promises.writeFile, lstat = fs_1.promises.lstat;
+var readFile = fs_1.promises.readFile, writeFile = fs_1.promises.writeFile;
 var encoding = 'utf8';
-var listFiles = function (folder) { return __awaiter(void 0, void 0, void 0, function () {
-    var files;
-    return __generator(this, function (_a) {
-        switch (_a.label) {
-            case 0: return [4 /*yield*/, readdir(folder)];
-            case 1:
-                files = _a.sent();
-                return [2 /*return*/, files.filter(function (file) { return file.endsWith('.csv'); })];
-        }
-    });
-}); };
+var ext = '.csv';
 var processFile = function (schemas, filepath) { return __awaiter(void 0, void 0, void 0, function () {
     var content, nodes, destPath;
     return __generator(this, function (_a) {
@@ -81,7 +71,7 @@ var processFile = function (schemas, filepath) { return __awaiter(void 0, void 0
                 return [4 /*yield*/, (0, json_1.toJson)(schemas, content)];
             case 2:
                 nodes = _a.sent();
-                destPath = filepath.replace('.csv', '.json');
+                destPath = filepath.replace(ext, '.json');
                 return [4 /*yield*/, writeFile(destPath, JSON.stringify({ nodes: nodes }, null, 2), encoding)];
             case 3:
                 _a.sent();
@@ -90,23 +80,15 @@ var processFile = function (schemas, filepath) { return __awaiter(void 0, void 0
     });
 }); };
 var run = function () { return __awaiter(void 0, void 0, void 0, function () {
-    var stat, schemas, _a, _b, _c;
-    return __generator(this, function (_d) {
-        switch (_d.label) {
-            case 0: return [4 /*yield*/, lstat(path)];
-            case 1:
-                stat = _d.sent();
+    var schemas, files;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0:
                 schemas = (0, json_schema_1.loadSchemas)();
-                _b = (_a = Promise).all;
-                if (!stat.isDirectory()) return [3 /*break*/, 3];
-                return [4 /*yield*/, listFiles(path)];
-            case 2:
-                _c = (_d.sent()).map(function (file) { return processFile(schemas, (0, path_1.join)(path, file)); });
-                return [3 /*break*/, 4];
-            case 3:
-                _c = [processFile(schemas, path)];
-                _d.label = 4;
-            case 4: return [2 /*return*/, _b.apply(_a, [_c])];
+                return [4 /*yield*/, (0, file_utils_1.listFiles)(path, ext)];
+            case 1:
+                files = _a.sent();
+                return [2 /*return*/, Promise.all(files.map(function (file) { return processFile(schemas, file); }))];
         }
     });
 }); };

package/csv-pivot.d.ts

@@ -1,9 +1,9 @@
+import { definitions } from '@hestia-earth/json-schema/types';
 export declare const toCsv: (nodes: any[], keys?: string[], excludeKeys?: string[]) => string;
-export declare const pivotNode: (node: any) => Partial<any>;
 /**
  * CSV format for data processing.
  *
  * @param nodes List of nodes to convert.
  * @returns CSV content formatted with pivoted blank nodes.
  */
-export declare const toCsvPivot: (nodes: any[]) => string;
+export declare const toCsvPivot: (schemas: definitions, nodes: any[]) => string;

package/csv-pivot.js

@@ -1,63 +1,22 @@
 "use strict";
-var __assign = (this && this.__assign) || function () {
-    __assign = Object.assign || function(t) {
-        for (var s, i = 1, n = arguments.length; i < n; i++) {
-            s = arguments[i];
-            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
-                t[p] = s[p];
-        }
-        return t;
-    };
-    return __assign.apply(this, arguments);
-};
-var __read = (this && this.__read) || function (o, n) {
-    var m = typeof Symbol === "function" && o[Symbol.iterator];
-    if (!m) return o;
-    var i = m.call(o), r, ar = [], e;
-    try {
-        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
-    }
-    catch (error) { e = { error: error }; }
-    finally {
-        try {
-            if (r && !r.done && (m = i["return"])) m.call(i);
-        }
-        finally { if (e) throw e.error; }
-    }
-    return ar;
-};
-var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
-    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
-        if (ar || !(i in from)) {
-            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
-            ar[i] = from[i];
-        }
-    }
-    return to.concat(ar || Array.prototype.slice.call(from));
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toCsvPivot = exports.pivotNode = exports.toCsv = void 0;
+exports.toCsvPivot = exports.toCsv = void 0;
 var json_2_csv_1 = require("json-2-csv");
 var utils_1 = require("./utils");
-var omitKeys = __spreadArray([
-    '@context',
-    'schemaVersion',
-    'dataPrivate'
-], __read(['bibliography', 'site', 'cycle', 'impactAssessment', 'defaultSource', 'source'].flatMap(function (k) { return [
-    "".concat(k, ".id"),
-    "".concat(k, ".type"),
-    "".concat(k, ".@type")
-]; })), false);
-var convertValue = function (value) {
-    return Array.isArray(value) ? value.map(String).join(';') : typeof value === 'object' ? JSON.stringify(value) : value;
-};
+var json_pivot_1 = require("./json-pivot");
 var parseValue = function (value) {
-    var result = convertValue(value);
+    var result = (0, utils_1.convertValue)(value);
     return typeof result === undefined || result === null ? '' : result.toString();
 };
+var withType = function (nodes) {
+    return nodes.map(function (node) {
+        var _a;
+        return (_a = {}, _a["".concat(node['@type'][0].toLowerCase()).concat(node['@type'].substring(1))] = node, _a);
+    });
+};
 var toCsv = function (nodes, keys, excludeKeys) {
-    if (excludeKeys === void 0) { excludeKeys = omitKeys; }
-    return (0, json_2_csv_1.json2csv)(nodes, {
+    if (excludeKeys === void 0) { excludeKeys = utils_1.omitKeys; }
+    return (0, json_2_csv_1.json2csv)(withType(nodes), {
         keys: keys,
         excludeKeys: excludeKeys,
         emptyFieldValue: '-',
@@ -68,39 +27,11 @@ var toCsv = function (nodes, keys, excludeKeys) {
     });
 };
 exports.toCsv = toCsv;
-var arrayKeys = function (node) {
-    return Object.keys(node).filter(function (key) {
-        return Array.isArray(node[key]) &&
-            node[key].length &&
-            typeof node[key][0] === 'object' &&
-            ('type' in node[key][0] || '@type' in node[key][0]);
-    });
-};
-var termId = function (term) { return (term ? term['@id'] || term.id : undefined); };
-var convertTerm = function (term) { return (term ? { methodModel: termId(term) } : {}); };
-var pivotList = function (values) {
-    return values.reduce(function (prev, _a) {
-        var _b;
-        var _t = _a["@type"], term = _a.term, methodModel = _a.methodModel, value = _a.value;
-        var key = termId(term);
-        var data = (0, utils_1.reduceUndefinedValues)(__assign(__assign({}, convertTerm(methodModel)), { value: convertValue(value) }));
-        return __assign(__assign({}, prev), (key && Object.keys(data).length ? (_b = {}, _b[key] = (prev[key] || []).concat([data]), _b) : {}));
-    }, {});
-};
-var pivotNode = function (node) {
-    var keys = arrayKeys(node);
-    return (0, utils_1.reduceUndefinedValues)(__assign(__assign({}, (0, utils_1.omit)(node, __spreadArray(__spreadArray([], __read(omitKeys), false), __read(keys), false))), keys.reduce(function (prev, key) {
-        var _a;
-        var value = pivotList(node[key]);
-        return __assign(__assign({}, prev), (Object.keys(value).length ? (_a = {}, _a[key] = value, _a) : {}));
-    }, {})));
-};
-exports.pivotNode = pivotNode;
 /**
  * CSV format for data processing.
  *
  * @param nodes List of nodes to convert.
  * @returns CSV content formatted with pivoted blank nodes.
  */
-var toCsvPivot = function (nodes) { return (0, exports.toCsv)(nodes.map(exports.pivotNode)); };
+var toCsvPivot = function (schemas, nodes) { return (0, exports.toCsv)(nodes.map(function (node) { return (0, json_pivot_1.pivotNode)(schemas, node); })); };
 exports.toCsvPivot = toCsvPivot;

package/file-utils.d.ts

@@ -0,0 +1 @@
+export declare const listFiles: (path: string, ...extensions: string[]) => Promise<string[]>;

package/file-utils.js

@@ -0,0 +1,97 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.listFiles = void 0;
+var fs_1 = require("fs");
+var path_1 = require("path");
+var lstat = fs_1.promises.lstat, readdir = fs_1.promises.readdir;
+var listFiles = function (path) {
+    var extensions = [];
+    for (var _i = 1; _i < arguments.length; _i++) {
+        extensions[_i - 1] = arguments[_i];
+    }
+    return __awaiter(void 0, void 0, void 0, function () {
+        var stat, files, _a, _b, _c;
+        return __generator(this, function (_d) {
+            switch (_d.label) {
+                case 0: return [4 /*yield*/, lstat(path)];
+                case 1:
+                    stat = _d.sent();
+                    _b = (_a = Promise).all;
+                    if (!stat.isDirectory()) return [3 /*break*/, 3];
+                    return [4 /*yield*/, readdir(path)];
+                case 2:
+                    _c = (_d.sent()).map(function (file) { return exports.listFiles.apply(void 0, __spreadArray([(0, path_1.join)(path, file)], __read(extensions), false)); });
+                    return [3 /*break*/, 4];
+                case 3:
+                    _c = [path];
+                    _d.label = 4;
+                case 4: return [4 /*yield*/, _b.apply(_a, [_c])];
+                case 5:
+                    files = (_d.sent()).flat();
+                    return [2 /*return*/, files.filter(function (file) { return extensions.includes((0, path_1.extname)(file)); })];
+            }
+        });
+    });
+};
+exports.listFiles = listFiles;

package/index.d.ts

@@ -1,3 +1,4 @@
 export { toCsvPivot } from './csv-pivot';
+export { pivotNode, pivotNodes } from './json-pivot';
 export * from './csv';
 export * from './json';

package/index.js

@@ -14,8 +14,11 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toCsvPivot = void 0;
+exports.pivotNodes = exports.pivotNode = exports.toCsvPivot = void 0;
 var csv_pivot_1 = require("./csv-pivot");
 Object.defineProperty(exports, "toCsvPivot", { enumerable: true, get: function () { return csv_pivot_1.toCsvPivot; } });
+var json_pivot_1 = require("./json-pivot");
+Object.defineProperty(exports, "pivotNode", { enumerable: true, get: function () { return json_pivot_1.pivotNode; } });
+Object.defineProperty(exports, "pivotNodes", { enumerable: true, get: function () { return json_pivot_1.pivotNodes; } });
 __exportStar(require("./csv"), exports);
 __exportStar(require("./json"), exports);

package/json-pivot.d.ts

@@ -0,0 +1,17 @@
+import { SchemaType, NodeType } from '@hestia-earth/schema';
+import { definitions } from '@hestia-earth/json-schema/types';
+export interface INode {
+    type?: SchemaType;
+    '@type'?: NodeType;
+    [key: string]: any;
+}
+/**
+ * Pivot node and return in compacted format.
+ *
+ * @param schemas The definitions of the HESTIA Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
+ * @param node The node to pivot.
+ * @param keepType To keep the `@type`/`type` key in the pivoted format.
+ * @returns
+ */
+export declare const pivotNode: (schemas: definitions, { "@type": _type, type, ...node }: INode, keepType?: boolean) => INode;
+export declare const pivotNodes: (schemas: definitions, nodes: any[]) => INode[];

package/json-pivot.js

@@ -0,0 +1,222 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pivotNodes = exports.pivotNode = void 0;
+var schema_1 = require("@hestia-earth/schema");
+var schema_utils_1 = require("@hestia-earth/json-schema/schema-utils");
+var get = require('lodash.get');
+var utils_1 = require("./utils");
+var ignoreKeys = __spreadArray([
+    '@context',
+    '@type',
+    'type',
+    'added',
+    'addedVersion',
+    'updated',
+    'updatedVersion',
+    '_cache',
+    // ignore all term keys as only the `@id` is included in the key
+    'term'
+], __read(schema_1.termFields.map(function (v) { return "term.".concat(v); })), false);
+var headerJoin = '+';
+var headerParentKey = function (key) { return key.split(headerJoin)[0]; };
+var uniqueData = function (value, key) {
+    // in some cases, we are getting a list using "inputs.@id", but `get` will not be able to access this
+    var parentData = key.includes('.') ? get(value, (0, utils_1.parentKey)(key), undefined) : undefined;
+    return Array.isArray(parentData) ? parentData.map(function (d) { return uniqueData(d, (0, utils_1.childkey)(key)); }) : get(value, key, undefined);
+};
+var valueAsString = function (value) {
+    return (Array.isArray(value) ? value : [value]).map(function (v) { return "".concat(v); }).join(';');
+};
+var valueAsKey = function (value) { return ([value.match(/[\s\"\'\.;]/g)].some(Boolean) ? "[".concat(value, "]") : value); };
+var pivotValue = function (schemas, nodeType, key, value) {
+    var _a;
+    var _b, _c;
+    // nodeType = Cycle
+    // key = emissions
+    // value = {'term':{'@id':'gwp100}, 'dates':['2021','2022'], 'value':[10,12]}
+    var uniqueFields = (_c = (_b = schema_1.uniquenessFields[nodeType]) === null || _b === void 0 ? void 0 : _b[key]) !== null && _c !== void 0 ? _c : [];
+    // uniqueFields = ['term.@id', 'dates']
+    var pivotedKey = uniqueFields.length
+        ? uniqueFields
+            .map(function (field) {
+            var data = uniqueData(value, field);
+            return typeof data === 'undefined'
+                ? undefined
+                : field === 'term.@id' || uniqueFields.length === 1
+                    ? valueAsKey(valueAsString(data))
+                    : "".concat((0, utils_1.parentKey)(field), "[").concat(valueAsString(data), "]");
+        })
+            .filter(Boolean)
+            .join(headerJoin)
+        : '';
+    // pivotedKey = 'gwp100+dates[2021;2022]'
+    var data = (0, utils_1.omit)((0, exports.pivotNode)(schemas, value, false), __spreadArray(__spreadArray([], __read(uniqueFields), false), __read(ignoreKeys), false));
+    return _a = {}, _a[pivotedKey] = data, _a;
+};
+var simplifiedKeys = function (keys) { return (0, utils_1.findNonDuplicates)(keys.map(headerParentKey)); };
+var mergeUnpivotedBlankNode = function (values, existingValue) {
+    return Object.assign.apply(Object, __spreadArray(__spreadArray([{}], __read(values.map(function (v) {
+        var _a;
+        return (_a = {}, _a[v] = {}, _a);
+    })), false), [existingValue || {}], false));
+};
+var mergeUnpivotedNodes = function (values, uniqueArrayKey) { return values.map(function (v) {
+    var _a;
+    return (_a = {}, _a[uniqueArrayKey] = v, _a);
+}); };
+var mergeUnpivotedByType = {
+    number: function (values) { return Number(values[0]); },
+    string: function (values) { return values[0]; },
+    boolean: function (values) { return Boolean(values[0]); },
+    null: function () { return null; },
+    array: function (values, definition, uniqueArrayKey, existingValue) {
+        var _a;
+        return ((_a = definition === null || definition === void 0 ? void 0 : definition.items) === null || _a === void 0 ? void 0 : _a.$ref)
+            ? mergeUnpivotedByType.$ref(values, definition.items, uniqueArrayKey, existingValue)
+            : __spreadArray(__spreadArray([], __read((existingValue || [])), false), __read(values), false).map(function (v) { return mergeUnpivotedByType[definition.items.type || 'string']([v]); })
+                .join(';');
+    },
+    $ref: function (values, _a, uniqueArrayKey, existingValue) {
+        var $ref = _a.$ref;
+        return (0, schema_1.isBlankNode)({ type: (0, schema_1.refToSchemaType)($ref) })
+            ? mergeUnpivotedBlankNode(values, existingValue)
+            : mergeUnpivotedNodes(values, uniqueArrayKey);
+    }
+};
+var mergeUnpivotedValue = function (definition, uniqueArrayItem, data, nodeValue, pivotedKey) {
+    var _a;
+    var _b = __read([pivotedKey.split('[')[0], pivotedKey.split('[')[1].replace(']', '').split(';')], 2), key = _b[0], newValues = _b[1];
+    var newValueDefinition = (_a = definition === null || definition === void 0 ? void 0 : definition.properties) === null || _a === void 0 ? void 0 : _a[key];
+    var uniqueArrayKey = (0, utils_1.childkey)((uniqueArrayItem !== null && uniqueArrayItem !== void 0 ? uniqueArrayItem : []).find(function (v) { return v.startsWith(key); }));
+    var valueType = newValueDefinition.type;
+    var functionValueType = '$ref' in newValueDefinition ? '$ref' : Array.isArray(valueType) ? valueType[0] : valueType;
+    data[key] = mergeUnpivotedByType[functionValueType](newValues, newValueDefinition, uniqueArrayKey, nodeValue[key]);
+    return data;
+};
+var unpivotBlankNode = function (definition, uniqueArrayItem, key, value) { return (__assign(__assign({}, value), key
+    .split(headerJoin)
+    .slice(1)
+    .reduce(function (prev, curr) { return mergeUnpivotedValue(definition, uniqueArrayItem, prev, value, curr); }, {}))); };
+/**
+ * Pivot list of Blank Node and simplify when only one instance is found
+ *
+ * @param nodeType
+ * @param key
+ * @param value
+ */
+var pivotBlankNodes = function (schemas, nodeType, key, value) {
+    var _a, _b;
+    var uniqueFields = (_b = (_a = schema_1.uniquenessFields[nodeType]) === null || _a === void 0 ? void 0 : _a[key]) !== null && _b !== void 0 ? _b : [];
+    var data = Object.assign.apply(Object, __spreadArray([{}], __read(value.map(function (val) { return pivotValue(schemas, nodeType, key, val); })), false));
+    var dataKeys = Object.keys(data);
+    var uniqueKeys = simplifiedKeys(dataKeys);
+    return (uniqueFields === null || uniqueFields === void 0 ? void 0 : uniqueFields[0]) === 'term.@id'
+        ? Object.fromEntries(Object.entries(data).map(function (_a) {
+            var _b, _c, _d, _e;
+            var _f = __read(_a, 2), entryKey = _f[0], value = _f[1];
+            var isUnique = entryKey !== headerParentKey(entryKey) && uniqueKeys.includes(headerParentKey(entryKey));
+            var blankNodeType = (0, schema_utils_1.keyType)(schemas[nodeType], key);
+            return [
+                isUnique ? headerParentKey(entryKey) : entryKey,
+                isUnique
+                    ? unpivotBlankNode(schemas[blankNodeType], (_e = (_d = (_c = (_b = schemas[nodeType]) === null || _b === void 0 ? void 0 : _b.properties) === null || _c === void 0 ? void 0 : _c[key]) === null || _d === void 0 ? void 0 : _d.uniqueArrayItem) !== null && _e !== void 0 ? _e : [], entryKey, value)
+                    : value
+            ];
+        }))
+        : data;
+};
+/**
+ * Pivot node and return in compacted format.
+ *
+ * @param schemas The definitions of the HESTIA Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
+ * @param node The node to pivot.
+ * @param keepType To keep the `@type`/`type` key in the pivoted format.
+ * @returns
+ */
+var pivotNode = function (schemas, _a, keepType) {
+    var _type = _a["@type"], type = _a.type, node = __rest(_a, ['@type', "type"]);
+    if (keepType === void 0) { keepType = true; }
+    return Object.fromEntries(__spreadArray([
+        keepType && (_type || type) ? [_type ? '@type' : 'type', _type || type] : null
+    ], __read(Object.entries(node)
+        .filter(function (_a) {
+        var _b = __read(_a, 1), key = _b[0];
+        return !ignoreKeys.includes(key) && ((_type || type) !== schema_1.NodeType.Term || !schema_1.termFields.includes(key));
+    })
+        .map(function (_a) {
+        var _b = __read(_a, 2), key = _b[0], value = _b[1];
+        return [
+            key,
+            value === null
+                ? null
+                : typeof value === 'object'
+                    ? Array.isArray(value)
+                        ? value.every(schema_1.isNode)
+                            ? value.map(function (v) { return (0, exports.pivotNode)(schemas, v, false); })
+                            : value.every(schema_1.isBlankNode)
+                                ? pivotBlankNodes(schemas, _type || type, key, value)
+                                : valueAsString(value)
+                        : (0, exports.pivotNode)(schemas, value, false) // deep pivoting
+                    : valueAsString(value)
+        ];
+    })), false).filter(Boolean));
+};
+exports.pivotNode = pivotNode;
+var pivotNodes = function (schemas, nodes) {
+    return nodes.map(function (node) {
+        try {
+            return (0, exports.pivotNode)(schemas, node);
+        }
+        catch (err) {
+            throw new Error("Error pivoting ".concat(node['@type'] || node.type, " with id \"").concat(node['@id'] || node.id, "\":\n").concat(err.toString()));
+        }
+    });
+};
+exports.pivotNodes = pivotNodes;

package/json.d.ts

@@ -34,16 +34,33 @@ export declare enum ErrorKeys {
     DuplicatedIdFields = "duplicated-id-fields",
     ObjectArrayInvalid = "object-array-invalid"
 }
+interface IConversionParams {
+    /**
+     * Ignore all internal fields (see `internal: true` in the schema). Defaults to `true`.
+     */
+    ignoreInternal?: boolean;
+    /**
+     * Add default values (see `default: true` in the schema). Defaults to `true`.
+     */
+    addDefaults?: boolean;
+}
+interface IExtendedConversionParams extends IConversionParams {
+    /**
+     * If it's an existing Node on HESTIA, i.e. the `@id` is used instead of `id`.
+     */
+    existingNode?: boolean;
+}
 export declare const throwCSVErrors: <T extends ICSVError>(errors: T[]) => never;
 export declare const cleanStringValue: (value: string) => string;
-export declare const formatNode: (schemas: definitions, type: SchemaType, data: ICSVContent, ignoreInternal: boolean, top?: boolean) => any;
+export declare const formatNode: (schemas: definitions, type: SchemaType, data: ICSVContent, params?: IExtendedConversionParams) => any;
 export declare const filterEmptyNode: (schemas: definitions, node: any) => boolean;
 /**
  * Convert CSV to HESTIA JSON-LD format.
  *
  * @param schemas The definitions of the HESTIA Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
  * @param content The content of the CSV as a string
+ * @param params Conversion parameters.
  * @returns A list of JSON-LD content.
  */
-export declare const toJson: (schemas: definitions, content: string) => Promise<any[]>;
+export declare const toJson: (schemas: definitions, content: string, params?: IConversionParams) => Promise<any[]>;
 export {};

package/json.js

@@ -128,7 +128,8 @@ var internalProperties = function (_a) {
 var isEmptyCell = function (value) { return value === IGNORE_FIELD_KEY || value === ''; };
 var nonEmptyCell = function (value) { return !isEmptyCell(value); };
 var isEmptyNode = function (node, schemas) {
-    var schema = node.type ? schemas[node.type] : null;
+    var type = node.type || node['@type'];
+    var schema = type ? schemas[type] : null;
     var properties = Object.keys(node).filter(function (key) { return !types_1.excludedDefaultProperties.includes(key); });
     var defaultProperties = schema ? getDefaultProperties(schema).map(function (_a) {
         var property = _a.property;
@@ -359,8 +360,8 @@ exports.cleanStringValue = cleanStringValue;
 var propertyTypeToValue = {
     null: function (value) {
         var args = [];
-        for (var _a = 1; _a < arguments.length; _a++) {
-            args[_a - 1] = arguments[_a];
+        for (var _i = 1; _i < arguments.length; _i++) {
+            args[_i - 1] = arguments[_i];
         }
         return (isEmptyCell(value) ? null : propertyTypeToValue.auto.apply(propertyTypeToValue, __spreadArray([value], __read(args), false)));
     },
@@ -386,35 +387,35 @@ var propertyTypeToValue = {
                         throw new Error('failed to parse boolean, expected true or false');
                     })();
     },
-    object: function (value, schemas, _a, _i) {
+    object: function (value, schemas, _a, params) {
         var $ref = _a.$ref, required = _a.required, geojson = _a.geojson;
         return geojson
             ? propertyTypeToValue.geojson(value)
             : $ref
-                ? propertyTypeToValue.$ref(value, schemas, { $ref: $ref }, _i)
-                : propertyTypeToValue.required(value, schemas, { required: required }, _i);
+                ? propertyTypeToValue.$ref(value, schemas, { $ref: $ref }, params)
+                : propertyTypeToValue.required(value, schemas, { required: required }, params);
     },
-    array: function (value, schemas, _a, _i) {
+    array: function (value, schemas, _a, params) {
         var items = _a.items;
         return (Array.isArray(value) ? value : value.split(arrayDelimiter()))
             .map(handleArrayError(function (val) {
             return items
                 ? '$ref' in items
-                    ? propertyTypeToValue.object(val, schemas, items, _i)
+                    ? propertyTypeToValue.object(val, schemas, items, params)
                     : Array.isArray(items.type)
                         ? val === IGNORE_FIELD_KEY
                             ? null
-                            : propertyTypeToValue.auto(val, schemas, items, _i)
-                        : propertyTypeToValue[items.type](val, schemas, items, _i)
+                            : propertyTypeToValue.auto(val, schemas, items, params)
+                        : propertyTypeToValue[items.type](val, schemas, items, params)
                 : val;
         }))
             .filter(function (val) { return !isEmptyValue(val, schemas); });
     },
     // try to determine the type automatically
-    auto: function (value, schemas, _d, _i) {
+    auto: function (value, schemas, _d, params) {
         // iris are mapped as {@id: val}
         return (0, utils_1.isIri)(value)
-            ? propertyTypeToValue.required(value, schemas, { required: ['@id'] }, _i)
+            ? propertyTypeToValue.required(value, schemas, { required: ['@id'] }, params)
             : (0, utils_1.isBoolean)(value)
                 ? propertyTypeToValue.boolean(value)
                 : (0, utils_1.isNumber)(value)
@@ -428,17 +429,21 @@ var propertyTypeToValue = {
         var data = propertyRequiredValue(value, required);
         return (0, utils_1.isEmpty)(data) ? {} : data;
     },
-    $ref: function (value, schemas, _a, _i) {
+    $ref: function (value, schemas, _a, params) {
         var $ref = _a.$ref;
         var schemaType = (0, schema_1.refToSchemaType)($ref);
         var schema = schemaType ? schemas[schemaType] : undefined;
         var data = schema
-            ? mapContent(schemas, schema, _i)(schemaRefValue(value, schemaType))
+            ? mapContent(schemas, schema, params)(schemaRefValue(value, schemaType))
             : $ref in propertyTypeToValue
                 ? propertyTypeToValue[$ref](value)
                 : safeParseJSON(value);
-        var includeDefaults = (0, schema_1.isBlankNode)(data) || schemaType === schema_1.SchemaType.Actor; // only blank nodes or actors allowed
-        return (0, utils_1.isEmpty)(data) ? {} : !!schema ? extendDataFromSchema(data, schema, schemaType, _i, includeDefaults) : data;
+        var addDefaults = (0, schema_1.isBlankNode)(data) || schemaType === schema_1.SchemaType.Actor; // only blank nodes or actors allowed
+        return (0, utils_1.isEmpty)(data)
+            ? {}
+            : !!schema
+                ? extendDataFromSchema(data, schema, schemaType, __assign(__assign({}, params), { addDefaults: addDefaults }))
+                : data;
     },
     geojson: function (value) { return (isEmptyCell(value) ? undefined : parseGeoJSONValue(value)); }
 };
@@ -488,19 +493,18 @@ var filterProperties = function (data, excludedProps) {
 };
 var nodeType = function (id, type, existingNode) {
     if (existingNode === void 0) { existingNode = false; }
-    return type ? ((0, schema_1.isTypeNode)(type) ? (existingNode ? { '@type': type } : { id: id, type: type }) : { type: type }) : {};
+    return type ? (existingNode ? { '@type': type } : { id: id, type: type }) : {};
 };
-var extendDataFromSchema = function (_a, schema, type, ignoreInternal, addDefaults) {
+var extendDataFromSchema = function (_a, schema, type, _b) {
     var id = _a.id, _t = _a.type, data = __rest(_a, ["id", "type"]);
-    if (ignoreInternal === void 0) { ignoreInternal = false; }
-    if (addDefaults === void 0) { addDefaults = false; }
+    var _c = _b === void 0 ? {} : _b, ignoreInternal = _c.ignoreInternal, addDefaults = _c.addDefaults, existingNode = _c.existingNode;
     return (0, utils_1.reduceUndefinedValues)(__assign(__assign(__assign({}, filterProperties(data, ignoreInternal ? [] : internalProperties(schema))), (addDefaults && !('@id' in data) && schema
         ? getDefaultProperties(schema, data, ignoreInternal).reduce(function (prev, _a) {
             var property = _a.property, defaultValue = _a.defaultValue;
             prev[property] = defaultValue;
             return prev;
         }, {})
-        : {})), nodeType(id, type, '@id' in data)), true);
+        : {})), nodeType(id, type, '@id' in data || existingNode)), true);
 };
 var validateArrayType = function (schema, def, key, value) {
     return typeof value !== 'object' ||
@@ -516,7 +520,7 @@ var validateArrayType = function (schema, def, key, value) {
             }
         ]);
 };
-var mapContent = function (schemas, schema, ignoreInternal) { return function (json) {
+var mapContent = function (schemas, schema, params) { return function (json) {
     var values = Object.keys(json)
         .filter(nonEmptyCell)
         .map(function (key) {
@@ -531,11 +535,12 @@ var mapContent = function (schemas, schema, ignoreInternal) { return function (j
             var newValue = key === VALUE_TYPE_KEY
                 ? ''
                 : propertyInvalidFormat(schema, key, value_1, function () {
-                    return Array.isArray(type_1)
-                        ? value_1
-                        : propertyTypeToValue[type_1](value_1, schemas, propertyDefinition_1, ignoreInternal);
+                    return Array.isArray(type_1) ? value_1 : propertyTypeToValue[type_1](value_1, schemas, propertyDefinition_1, params);
                 });
-            return { errors: [], value: type_1 !== 'null' && isEmptyValue(newValue, schemas) ? undefined : [key, newValue] };
+            return {
+                errors: [],
+                value: type_1 !== 'null' && isEmptyValue(newValue, schemas) ? undefined : [key, newValue]
+            };
         }
         catch (err) {
             var errors_1 = JSON.parse(err.message).errors;
@@ -556,23 +561,26 @@ var mapContent = function (schemas, schema, ignoreInternal) { return function (j
             return value;
         })), false).filter(Boolean));
 }; };
-var formatNode = function (schemas, type, data, ignoreInternal, top) {
-    if (top === void 0) { top = false; }
+var formatNode = function (schemas, type, data, params) {
+    if (params === void 0) { params = {}; }
     var schema = type in schemas && exports.acceptedNodeTypes.includes(type) ? schemas[type] : schemaNotFoundError(type);
-    var content = mapContent(schemas, schema, ignoreInternal)(data);
-    return (0, utils_1.reduceUndefinedValues)(__assign({ id: isEmptyCell(data.id) ? undefined : data.id }, extendDataFromSchema(content, schema, type, ignoreInternal, top)), true);
+    var content = mapContent(schemas, schema, params)(data);
+    return (0, utils_1.reduceUndefinedValues)(__assign({ id: isEmptyCell(data.id) ? undefined : data.id }, extendDataFromSchema(content, schema, type, params)), true);
 };
 exports.formatNode = formatNode;
 var filterEmptyNode = function (schemas, node) {
-    var schema = node.type ? schemas[node.type] : null;
+    var type = node.type || node['@type'];
+    var schema = type ? schemas[type] : null;
     // make sure defaultProperties are not counted
     return !!schema && !isEmptyNode(node, schemas);
 };
 exports.filterEmptyNode = filterEmptyNode;
-var convetToNode = function (schemas) { return function (data) {
+var convetToNode = function (schemas, params) { return function (data) {
     return Object.keys(data)
         .filter(utils_1.nonEmptyValue)
-        .map(function (type) { return (0, exports.formatNode)(schemas, (0, schema_1.typeToSchemaType)(type), data[type], true, true); })
+        .map(function (type) {
+        return (0, exports.formatNode)(schemas, (0, schema_1.typeToSchemaType)(type), data[type], __assign(__assign({}, params), { existingNode: '@id' in data[type] }));
+    })
         .filter(function (node) { return (0, exports.filterEmptyNode)(schemas, node); });
 }; };
 /**
@@ -580,20 +588,25 @@ var convetToNode = function (schemas) { return function (data) {
  *
  * @param schemas The definitions of the HESTIA Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
  * @param content The content of the CSV as a string
+ * @param params Conversion parameters.
  * @returns A list of JSON-LD content.
  */
-var toJson = function (schemas, content) { return __awaiter(void 0, void 0, void 0, function () {
-    var nodes;
-    return __generator(this, function (_a) {
-        switch (_a.label) {
-            case 0: return [4 /*yield*/, csvtojson({
-                    delimiter: 'auto',
-                    ignoreColumns: /^$/
-                }).fromString(content)];
-            case 1:
-                nodes = (_a.sent()).flatMap(convetToNode(schemas));
-                return [2 /*return*/, (0, default_values_1.setDefaultValues)(nodes)];
-        }
+var toJson = function (schemas, content, params) {
+    if (params === void 0) { params = { ignoreInternal: true, addDefaults: true }; }
+    return __awaiter(void 0, void 0, void 0, function () {
+        var nodes, converted;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0: return [4 /*yield*/, csvtojson({
+                        delimiter: 'auto',
+                        ignoreColumns: /^$/
+                    }).fromString(content)];
+                case 1:
+                    nodes = _a.sent();
+                    converted = nodes.flatMap(convetToNode(schemas, params));
+                    return [2 /*return*/, (0, default_values_1.setDefaultValues)(converted)];
+            }
+        });
     });
-}); };
+};
 exports.toJson = toJson;

package/package.json

@@ -1,13 +1,14 @@
 {
   "name": "@hestia-earth/schema-convert",
-  "version": "30.0.1",
+  "version": "30.2.0",
   "description": "HESTIA Schema Converters",
   "main": "index.js",
   "typings": "index.d.ts",
   "bin": {
     "hestia-convert-to-csv-pivot": "bin/bin-convert-to-csv-pivot.js",
     "hestia-convert-to-csv": "bin/bin-convert-to-csv.js",
-    "hestia-convert-to-json": "bin/bin-convert-to-json.js"
+    "hestia-convert-to-json": "bin/bin-convert-to-json.js",
+    "hestia-pivot-file": "bin/bin-pivot-file.js"
   },
   "repository": {
     "type": "git",

package/pivot-file.d.ts

@@ -0,0 +1 @@
+export declare const run: () => Promise<any[]>;

package/pivot-file.js

@@ -0,0 +1,134 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.run = void 0;
+var fs_1 = require("fs");
+var path_1 = require("path");
+var json_schema_1 = require("@hestia-earth/json-schema");
+var file_utils_1 = require("./file-utils");
+var json_pivot_1 = require("./json-pivot");
+var csv_pivot_1 = require("./csv-pivot");
+var _a = __read(process.argv.slice(2), 2), path = _a[0], format = _a[1];
+var readFile = fs_1.promises.readFile, writeFile = fs_1.promises.writeFile;
+var encoding = 'utf8';
+var Format;
+(function (Format) {
+    Format["csv"] = "csv";
+    Format["json"] = "json";
+})(Format || (Format = {}));
+var formats = Object.values(Format);
+if (format && !formats.includes(format)) {
+    throw new Error("Invalid format. Please use either ".concat(formats.map(function (v) { return "\"".concat(v, "\""); }).join(' or ')));
+}
+var saveToFileExt = {
+    json: function (filepath, nodes) {
+        return writeFile(filepath.replace((0, path_1.extname)(filepath), '.json'), JSON.stringify(nodes, null, 2), encoding);
+    },
+    csv: function (filepath, nodes) { return writeFile(filepath.replace((0, path_1.extname)(filepath), '.csv'), (0, csv_pivot_1.toCsv)(nodes), encoding); }
+};
+var processJsonFile = function (schemas, filepath) { return __awaiter(void 0, void 0, void 0, function () {
+    var data, _a, _b, pivoted;
+    return __generator(this, function (_c) {
+        switch (_c.label) {
+            case 0:
+                _b = (_a = JSON).parse;
+                return [4 /*yield*/, readFile(filepath, encoding)];
+            case 1:
+                data = _b.apply(_a, [_c.sent()]);
+                pivoted = Array.isArray(data)
+                    ? (0, json_pivot_1.pivotNodes)(schemas, data)
+                    : 'nodes' in data
+                        ? { nodes: (0, json_pivot_1.pivotNodes)(schemas, data.nodes) }
+                        : (0, json_pivot_1.pivotNode)(schemas, data);
+                return [2 /*return*/, saveToFileExt[format](filepath, pivoted)];
+        }
+    });
+}); };
+var processCsvFile = function (schemas, filepath) { return __awaiter(void 0, void 0, void 0, function () {
+    var toJson, content, nodes, pivoted;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0: return [4 /*yield*/, Promise.resolve().then(function () { return require('./json'); })];
+            case 1:
+                toJson = (_a.sent()).toJson;
+                return [4 /*yield*/, readFile(filepath, encoding)];
+            case 2:
+                content = _a.sent();
+                return [4 /*yield*/, toJson(schemas, content, { ignoreInternal: false, addDefaults: false })];
+            case 3:
+                nodes = _a.sent();
+                pivoted = (0, json_pivot_1.pivotNodes)(schemas, nodes);
+                return [2 /*return*/, saveToFileExt[format](filepath, pivoted)];
+        }
+    });
+}); };
+var processFileExt = {
+    '.json': processJsonFile,
+    '.jsonld': processJsonFile,
+    '.csv': processCsvFile
+};
+var run = function () { return __awaiter(void 0, void 0, void 0, function () {
+    var schemas, files;
+    return __generator(this, function (_a) {
+        switch (_a.label) {
+            case 0:
+                schemas = (0, json_schema_1.loadSchemas)();
+                return [4 /*yield*/, (0, file_utils_1.listFiles)(path, '.csv', '.jsonld', '.json')];
+            case 1:
+                files = _a.sent();
+                return [2 /*return*/, Promise.all(files.map(function (file) { return processFileExt[(0, path_1.extname)(file)](schemas, file); }))];
+        }
+    });
+}); };
+exports.run = run;

package/utils.d.ts

@@ -1,3 +1,6 @@
+export declare const omitKeys: string[];
+export declare const parentKey: (key: string) => string;
+export declare const childkey: (key: string) => string;
 export declare const omit: (data: any, keys: string[]) => any;
 export declare const isExpandable: (val: any) => any;
 export declare const isIri: (value?: string) => boolean;
@@ -12,3 +15,6 @@ export declare const keyToLabel: (key: string) => string;
 export declare const diffInDays: (date1: Date | string, date2: Date | string) => number;
 export declare const daysBefore: (date?: Date, days?: number) => Date;
 export declare const daysInYear: (year: number) => 366 | 365;
+export declare const convertValue: (value: any) => any;
+export declare const findDuplicates: (values: string[]) => string[];
+export declare const findNonDuplicates: (values: string[]) => string[];

package/utils.js

@@ -10,12 +10,69 @@ var __assign = (this && this.__assign) || function () {
     };
     return __assign.apply(this, arguments);
 };
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.daysInYear = exports.daysBefore = exports.diffInDays = exports.keyToLabel = exports.ellipsis = exports.reduceUndefinedValues = exports.nonEmptyNode = exports.nonEmptyValue = exports.isEmpty = exports.isNumber = exports.isBoolean = exports.isIri = exports.isExpandable = exports.omit = void 0;
+exports.findNonDuplicates = exports.findDuplicates = exports.convertValue = exports.daysInYear = exports.daysBefore = exports.diffInDays = exports.keyToLabel = exports.ellipsis = exports.reduceUndefinedValues = exports.nonEmptyNode = exports.nonEmptyValue = exports.isEmpty = exports.isNumber = exports.isBoolean = exports.isIri = exports.isExpandable = exports.omit = exports.childkey = exports.parentKey = exports.omitKeys = void 0;
+var get = require('lodash.get');
 var unset = require('lodash.unset');
+exports.omitKeys = __spreadArray([
+    '@context',
+    'schemaVersion',
+    'dataPrivate'
+], __read(['bibliography', 'site', 'cycle', 'impactAssessment', 'defaultSource', 'source'].flatMap(function (k) { return [
+    "".concat(k, ".id"),
+    "".concat(k, ".type"),
+    "".concat(k, ".@type")
+]; })), false);
+var parentKey = function (key) { return key.split('.')[0]; };
+exports.parentKey = parentKey;
+var childkey = function (key) { return key.split('.').slice(1).join('.'); };
+exports.childkey = childkey;
 var omit = function (data, keys) {
     var obj = __assign({}, data);
-    keys.forEach(function (key) { return unset(obj, key); });
+    keys.forEach(function (key) {
+        unset(obj, key);
+        // handle arrays and notation without array
+        var _parentKey = (0, exports.parentKey)(key);
+        var _childkey = (0, exports.childkey)(key);
+        var parentData = key.includes('.') ? get(obj, _parentKey, undefined) : undefined;
+        if (typeof parentData === 'object') {
+            if (Array.isArray(parentData)) {
+                parentData.forEach(function (_v, index) {
+                    unset(obj, "".concat(_parentKey, "[").concat(index, "].").concat(_childkey));
+                });
+                // only keep values that are not empty
+                obj[_parentKey] = obj[_parentKey].filter(function (v) { return !(0, exports.isEmpty)(v); });
+            }
+            if ((0, exports.isEmpty)(obj[_parentKey])) {
+                unset(obj, _parentKey);
+            }
+        }
+    });
     return obj;
 };
 exports.omit = omit;
@@ -90,3 +147,17 @@ var daysBefore = function (date, days) {
 exports.daysBefore = daysBefore;
 var daysInYear = function (year) { return ((year % 4 === 0 && year % 100 > 0) || year % 400 === 0 ? 366 : 365); };
 exports.daysInYear = daysInYear;
+var convertValue = function (value) {
+    return Array.isArray(value) ? value.map(String).join(';') : typeof value === 'object' ? JSON.stringify(value) : value;
+};
+exports.convertValue = convertValue;
+var count = function (values) { return values.reduce(function (a, b) {
+    var _a;
+    return (__assign(__assign({}, a), (_a = {}, _a[b] = (a[b] || 0) + 1, _a)));
+}, {}); };
+var duplicates = function (dict) { return Object.keys(dict).filter(function (a) { return dict[a] > 1; }); };
+var nonDuplicates = function (dict) { return Object.keys(dict).filter(function (a) { return dict[a] === 1; }); };
+var findDuplicates = function (values) { return duplicates(count(values)); };
+exports.findDuplicates = findDuplicates;
+var findNonDuplicates = function (values) { return nonDuplicates(count(values)); };
+exports.findNonDuplicates = findNonDuplicates;