@hestia-earth/schema-convert 7.2.0 → 7.4.0

package/json.d.ts

@@ -1,2 +1,37 @@
-import { definitions } from '@hestia-earth/json-schema';
+import { SchemaType } from '@hestia-earth/schema';
+import { definitions } from '@hestia-earth/json-schema/types';
+export interface INode {
+    type: SchemaType;
+    id?: string;
+    name?: string;
+}
+export interface ICSVContent {
+    [key: string]: string | ICSVContent;
+}
+interface ICSVHeader {
+    name: string;
+    index: number;
+}
+export interface ICSVError {
+    message: string;
+    schema?: SchemaType;
+    schemaKey?: string;
+    key?: string;
+    property?: string;
+    value?: any;
+    error?: string;
+    headers?: ICSVHeader[];
+}
+export declare const throwCSVError: <T extends ICSVError>(error: T) => never;
+export declare const cleanStringValue: (value: string) => string;
+export declare const formatNode: (schemas: definitions, type: SchemaType, data: ICSVContent, ignoreInternal: boolean, top?: boolean) => any;
+export declare const filterEmptyNode: (schemas: definitions, { schemaVersion, ...node }: any) => boolean;
+/**
+ * Convert CSV to Hestia JSON-LD format.
+ *
+ * @param schemas The definitions of the Hestia Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
+ * @param content The content of the CSV as a string
+ * @returns A list of JSON-LD content.
+ */
 export declare const toJson: (schemas: definitions, content: string) => Promise<any[]>;
+export {};

package/json.js

@@ -46,51 +46,251 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
         if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
     }
 };
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toJson = void 0;
+exports.toJson = exports.filterEmptyNode = exports.formatNode = exports.cleanStringValue = exports.throwCSVError = void 0;
 var csvtojson = require("csvtojson");
 var schema_1 = require("@hestia-earth/schema");
+var types_1 = require("@hestia-earth/json-schema/types");
 var utils_1 = require("./utils");
-var propertyRef = function (schema, key) {
-    var property = schema && schema.properties ? schema.properties[key] : {};
-    return property ? ('items' in property ? property.items['$ref'] : property['$ref']) : null;
-};
-var keyType = function (schema, key) {
-    var ref = propertyRef(schema, key);
-    return ref ? schema_1.refToSchemaType(ref) : null;
-};
-var formatNode = function (schemas, schema, node) {
-    return typeof node === 'object' ?
-        Object.keys(node)
-            .map(function (key) {
-            var isArray = Array.isArray(node[key]);
-            var type = keyType(schema, key);
-            var nodeSchema = type ? schemas[type] : schema;
-            var value = (isArray ? node[key] : [node[key]])
-                .map(function (v) { return formatNode(schemas, nodeSchema, v); })
-                .filter(utils_1.nonEmptyNode)
-                .map(function (v) { return typeof v === 'object' ? (__assign(__assign({}, v), (type ? { type: type } : {}))) : v; });
-            return {
-                key: key,
-                isArray: isArray,
-                value: isArray ? value : value[0]
-            };
-        })
-            .filter(function (_a) {
-            var key = _a.key, value = _a.value;
-            return utils_1.nonEmptyValue(key) && utils_1.nonEmptyNode(value);
-        })
-            .reduce(function (prev, _a) {
+var IGNORE_FIELD_KEY = '-';
+var VALUE_TYPE_KEY = 'valueType';
+var DEFAULT_ARRAY_DELIMITER = ';';
+var arrayDelimiter = function () {
+    try {
+        return process.env.CSV_ARRAY_DELIMITER || DEFAULT_ARRAY_DELIMITER;
+    }
+    catch (err) {
+        return DEFAULT_ARRAY_DELIMITER;
+    }
+};
+var safeParseJSON = function (obj) {
+    // throw already handled error or throw a generic invalid-format error
+    var data;
+    try {
+        data = JSON.parse(obj);
+    }
+    catch (err) { }
+    return data;
+};
+var internalProperties = function (_a) {
+    var properties = _a.properties;
+    return Object.keys(properties).filter(function (property) { return properties[property].internal; });
+};
+var nonEmptyCell = function (value) { return value !== IGNORE_FIELD_KEY && value !== ''; };
+var compileFullKey = function (key1, key2) {
+    // handle arrays
+    var joinKey = (key2 || '').startsWith('[') ? '' : '.';
+    return [key1, key2].filter(Boolean).join(joinKey);
+};
+var propertyRequiredValue = function (value, required) {
+    var _a;
+    var val = typeof value === 'object' && required[0] in value ? value[required[0]] : value;
+    return nonEmptyCell(val) ? (_a = {}, _a[required[0]] = val, _a) : {};
+};
+var parseError = function (err) { return safeParseJSON(err.message); };
+exports.throwCSVError = function (error) {
+    throw new Error(JSON.stringify(error));
+};
+var schemaNotFoundError = function () { return exports.throwCSVError({
+    message: 'schema-not-found'
+}); };
+var propertyNotFoundError = function (schema, key, value) { return exports.throwCSVError({
+    schema: schema.title,
+    message: 'property-not-found',
+    schemaKey: key,
+    key: key,
+    value: value
+}); };
+var propertyInvalidFormat = function (schema, key, value, func) {
+    try {
+        return func();
+    }
+    catch (err) {
+        var data = parseError(err);
+        // throw already handled error or throw a generic invalid-format error
+        return exports.throwCSVError(data ? __assign(__assign({}, data), { key: compileFullKey(key, data.key) }) : {
+            schema: schema.title,
+            message: 'property-invalid-format',
+            schemaKey: key,
+            key: key,
+            value: value,
+            error: err === null || err === void 0 ? void 0 : err.message
+        });
+    }
+};
+var handleArrayError = function (func) { return function (value, index) {
+    try {
+        return func(value, index);
+    }
+    catch (err) {
+        var data = parseError(err);
+        // throw already handled error or throw a generic invalid-format error
+        return data ?
+            exports.throwCSVError(__assign(__assign({}, data), { key: compileFullKey("" + index, data.key) })) :
+            (function () {
+                throw err;
+            })();
+    }
+}; };
+/**
+ * If the user provided a non-object where an object was expected, assume it was meant to be the `name` property.
+ *
+ * @param value The value mapped as a Ref
+ */
+var schemaRefValue = function (value) { return typeof value === 'object' ? value : ({ name: value }); };
+exports.cleanStringValue = function (value) { return (value.match(/^[\d]+\.[\d]+\.[\d]+$/) === null ? value.replace(/\.0$/, '') : value).trim(); };
+var propertyTypeToValue = {
+    string: function (value) { return exports.cleanStringValue(value || ''); },
+    number: function (value) {
+        return isNaN(+value) ? (function () {
+            throw new Error('failed to parse number');
+        })() : +value;
+    },
+    integer: function (value) { return +value; },
+    boolean: function (value) { return value.toLowerCase() === 'true'; },
+    object: function (value, schemas, _a, _i) {
+        var $ref = _a.$ref, required = _a.required;
+        return $ref ?
+            propertyTypeToValue.$ref(value, schemas, { $ref: $ref }, _i) :
+            propertyTypeToValue.required(value, schemas, { required: required }, _i);
+    },
+    array: function (value, schemas, _a, _i) {
+        var items = _a.items;
+        return (Array.isArray(value) ? value : value.split(arrayDelimiter()))
+            .map(handleArrayError(function (val) { return items ? ('$ref' in items ?
+            propertyTypeToValue.object(val, schemas, items, _i) :
+            Array.isArray(items.type) ?
+                propertyTypeToValue.auto(val, schemas, items, _i) :
+                propertyTypeToValue[items.type](val, schemas, items, _i)) : val; }))
+            .filter(function (val) { return !utils_1.isEmpty(val); });
+    },
+    // try to determine the type automatically
+    auto: function (value, schemas, _d, _i) {
+        // iris are mapped as {@id: val}
+        return utils_1.isIri(value) ? propertyTypeToValue.required(value, schemas, { required: ['@id'] }, _i) : (utils_1.isBoolean(value) ? propertyTypeToValue.boolean(value) : (utils_1.isNumber(value) ? propertyTypeToValue.number(value) :
+            propertyTypeToValue.string(value)));
+    },
+    required: function (value, _schemas, _a) {
+        var required = _a.required;
+        var data = propertyRequiredValue(value, required);
+        return utils_1.isEmpty(data) ? {} : data;
+    },
+    $ref: function (value, schemas, _a, _i) {
+        var $ref = _a.$ref;
+        var schemaType = schema_1.refToSchemaType($ref);
+        var schema = schemaType ? schemas[schemaType] : undefined;
+        var data = schema ? mapContent(schemas, schema, _i)(schemaRefValue(value)) : safeParseJSON(value);
+        return utils_1.isEmpty(data) ? {} : (schema ? extendDataFromSchema(data, schema, schemaType, _i) : data);
+    }
+};
+var getPropertyDefinition = function (schema, key, ignoreErrors, value) {
+    if (ignoreErrors === void 0) { ignoreErrors = false; }
+    return key in schema.properties ? schema.properties[key] : !ignoreErrors && propertyNotFoundError(schema, key, value);
+};
+var getValueType = function (schema, json) {
+    return VALUE_TYPE_KEY in schema.properties ? (
+    // valueType can be present but skipped
+    (json[VALUE_TYPE_KEY] !== IGNORE_FIELD_KEY ? json[VALUE_TYPE_KEY] : null) ||
+        schema.properties[VALUE_TYPE_KEY].default) : null;
+};
+var getPropertyType = function (def, key, valueType) {
+    // default type is object (like $ref)
+    return (key === 'value' && valueType ? valueType : (Array.isArray(def.type) ? 'auto' : def.type)) || 'object';
+};
+var setDefaultProperty = function (prop, def, node, ignoreInternal) {
+    if (ignoreInternal === void 0) { ignoreInternal = false; }
+    return prop !== VALUE_TYPE_KEY && (
+    // only get default if internal or not present
+    (!ignoreInternal && def.internal) || !node || !(prop in node));
+};
+var getDefaultProperties = function (schema, node, ignoreInternal) {
+    if (ignoreInternal === void 0) { ignoreInternal = false; }
+    return types_1.defaultProperties(schema)
+        .map(function (property) {
+        var def = schema.properties[property];
+        return setDefaultProperty(property, def, node, ignoreInternal) ? { property: property, defaultValue: def.default } : null;
+    }, {})
+        .filter(Boolean);
+};
+var filterProperties = function (data, excludedProps) {
+    return Object.keys(data)
+        .filter(function (key) { return !excludedProps.includes(key); })
+        .reduce(function (prev, curr) {
+        var _a;
+        return (__assign(__assign({}, prev), (_a = {}, _a[curr] = data[curr], _a)));
+    }, {});
+};
+var nodeType = function (id, type, existingNode) {
+    if (existingNode === void 0) { existingNode = false; }
+    return type ? (schema_1.isTypeNode(type) ? (existingNode ? { '@type': type } : { id: id, type: type }) : { type: type }) : {};
+};
+var extendDataFromSchema = function (_a, schema, type, ignoreInternal, top) {
+    if (ignoreInternal === void 0) { ignoreInternal = false; }
+    if (top === void 0) { top = false; }
+    var id = _a.id, _t = _a.type, data = __rest(_a, ["id", "type"]);
+    return utils_1.reduceUndefinedValues(__assign(__assign(__assign({}, filterProperties(data, ignoreInternal ? [] : internalProperties(schema))), (top && schema ?
+        getDefaultProperties(schema, data, ignoreInternal).reduce(function (prev, _a) {
             var _b;
-            var key = _a.key, value = _a.value;
-            return (__assign(__assign({}, prev), (_b = {}, _b[key] = value, _b)));
+            var property = _a.property, defaultValue = _a.defaultValue;
+            return (__assign(__assign({}, prev), (_b = {}, _b[property] = defaultValue, _b)));
         }, {}) :
-        node;
+        {})), nodeType(id, type, '@id' in data)));
+};
+var isEmptyValue = function (value) {
+    return (value['@type'] || value.type) === schema_1.SchemaType.Term ? !(value['@id'] || value.id || value.name) : utils_1.isEmpty(value);
+};
+var mapContent = function (schemas, schema, ignoreInternal) { return function (json) {
+    return Object.keys(json)
+        .filter(function (key) { return nonEmptyCell(key) && nonEmptyCell(json[key]); })
+        .reduce(function (prev, key) {
+        var _a;
+        var value = json[key];
+        var propertyDefinition = getPropertyDefinition(schema, key, false, json);
+        var valueType = getValueType(schema, json);
+        var type = getPropertyType(propertyDefinition, key, valueType);
+        var newValue = key === VALUE_TYPE_KEY ? '' : propertyInvalidFormat(schema, key, value, function () {
+            return Array.isArray(type) ? value : propertyTypeToValue[type](value, schemas, propertyDefinition, ignoreInternal);
+        });
+        return __assign(__assign(__assign({}, prev), (schema.$id ? { type: schema.title } : {})), (isEmptyValue(newValue) ? {} : (_a = {}, _a[key] = newValue, _a)));
+    }, {});
+}; };
+exports.formatNode = function (schemas, type, data, ignoreInternal, top) {
+    if (top === void 0) { top = false; }
+    var schema = type in schemas ? schemas[type] : schemaNotFoundError();
+    var content = mapContent(schemas, schema, ignoreInternal)(data);
+    return utils_1.reduceUndefinedValues(__assign({ id: !nonEmptyCell(data.id) ? undefined : data.id }, extendDataFromSchema(content, schema, type, ignoreInternal, top)));
+};
+exports.filterEmptyNode = function (schemas, _a) {
+    var schemaVersion = _a.schemaVersion, node = __rest(_a, ["schemaVersion"]);
+    // make sure defaultProperties are not counted
+    var schema = node.type ? schemas[node.type] : null;
+    var minKeysLength = schema ? getDefaultProperties(schema).length + 1 : 1;
+    return !utils_1.isEmpty(node, minKeysLength) && !!schema;
 };
-var convetToNode = function (schemas) { return function (data) { return Object.keys(data)
-    .filter(utils_1.nonEmptyValue)
-    .map(function (type) { return (__assign({ type: schema_1.typeToSchemaType(type) }, formatNode(schemas, schemas[schema_1.typeToSchemaType(type)], data[type]))); })
-    .filter(utils_1.nonEmptyNode); }; };
+var convetToNode = function (schemas) { return function (data) {
+    return Object.keys(data)
+        .filter(utils_1.nonEmptyValue)
+        .map(function (type) { return exports.formatNode(schemas, schema_1.typeToSchemaType(type), data[type], true, true); })
+        .filter(function (node) { return exports.filterEmptyNode(schemas, node); });
+}; };
+/**
+ * Convert CSV to Hestia JSON-LD format.
+ *
+ * @param schemas The definitions of the Hestia Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
+ * @param content The content of the CSV as a string
+ * @returns A list of JSON-LD content.
+ */
 exports.toJson = function (schemas, content) { return __awaiter(void 0, void 0, void 0, function () { return __generator(this, function (_a) {
     switch (_a.label) {
         case 0: return [4 /*yield*/, csvtojson({ delimiter: 'auto' }).fromString(content)];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hestia-earth/schema-convert",
-  "version": "7.2.0",
+  "version": "7.4.0",
   "description": "Hestia Schema Converters",
   "main": "index.js",
   "typings": "index.d.ts",

package/utils.d.ts

@@ -1,5 +1,9 @@
 export declare const omit: (data: any, keys: string[]) => any;
 export declare const isExpandable: (val: any) => number | boolean;
+export declare const isIri: (value?: string) => boolean;
+export declare const isBoolean: (value: string) => boolean;
+export declare const isNumber: (n: string | number) => boolean;
+export declare const isEmpty: (value: any, minKeys?: number) => boolean;
 export declare const nonEmptyValue: (value: any) => boolean;
 export declare const nonEmptyNode: (node: any) => boolean;
 export declare const reduceUndefinedValues: <T>(obj: T) => Partial<T>;

package/utils.js

@@ -11,7 +11,7 @@ var __assign = (this && this.__assign) || function () {
     return __assign.apply(this, arguments);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.reduceUndefinedValues = exports.nonEmptyNode = exports.nonEmptyValue = exports.isExpandable = exports.omit = void 0;
+exports.reduceUndefinedValues = exports.nonEmptyNode = exports.nonEmptyValue = exports.isEmpty = exports.isNumber = exports.isBoolean = exports.isIri = exports.isExpandable = exports.omit = void 0;
 var unset = require('lodash.unset');
 exports.omit = function (data, keys) {
     var obj = __assign({}, data);
@@ -19,21 +19,25 @@ exports.omit = function (data, keys) {
     return obj;
 };
 exports.isExpandable = function (val) { return !!val && typeof val === 'object' && (Array.isArray(val) ? val.every(exports.isExpandable) : Object.keys(val).length); };
+exports.isIri = function (value) { return (value || '').startsWith('http'); };
+exports.isBoolean = function (value) { return value.toLowerCase() === 'true' || value.toLowerCase() === 'false'; };
+exports.isNumber = function (n) { return !isNaN(parseFloat("" + n)) && isFinite(parseFloat("" + n)); };
 var ignoreKey = function (key) { return !['@type', 'type'].includes(key); };
-var isEmpty = function (value) {
+exports.isEmpty = function (value, minKeys) {
+    if (minKeys === void 0) { minKeys = 1; }
     return value === null ||
         typeof value === 'undefined' ||
         (typeof value === 'object' ?
             (Array.isArray(value) ?
                 !value.length :
-                Object.keys(value).filter(ignoreKey).length < 1) :
+                Object.keys(value).filter(function (key) { return key !== 'type'; }).length < minKeys) :
             value === '');
 };
-exports.nonEmptyValue = function (value) { return !isEmpty(value) && value !== '-'; };
+exports.nonEmptyValue = function (value) { return !exports.isEmpty(value) && value !== '-'; };
 exports.nonEmptyNode = function (node) {
     return typeof node === 'object' ?
         Object.keys(node)
-            .filter(function (key) { return ignoreKey(key) && !isEmpty(node[key]); })
+            .filter(function (key) { return ignoreKey(key) && !exports.isEmpty(node[key]); })
             .length > 0 :
         exports.nonEmptyValue(node);
 };