@hestia-earth/schema-convert 37.1.0 → 37.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/csv-pivot.js +21 -0
- package/esm/csv.js +110 -0
- package/esm/default-values.js +66 -0
- package/esm/index.js +6 -0
- package/esm/json-pivot.js +163 -0
- package/esm/json.js +451 -0
- package/esm/utils.js +92 -0
- package/index.d.ts +1 -0
- package/index.js +9 -1
- package/package.json +49 -1
package/esm/csv-pivot.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { convertValue, omitKeys, jsonToCsv, uncapitalize } from './utils';
|
|
2
|
+
import { pivotNode } from './json-pivot';
|
|
3
|
+
const parseValue = (value) => {
|
|
4
|
+
const result = convertValue(value);
|
|
5
|
+
return typeof result === undefined || result === null ? '' : result.toString();
|
|
6
|
+
};
|
|
7
|
+
const withType = (nodes) => nodes.map(node => ({ [uncapitalize(node['@type'] || node.type)]: node }));
|
|
8
|
+
export const toCsv = (nodes, keys, excludeKeys = omitKeys) => jsonToCsv(withType(nodes), {
|
|
9
|
+
keys,
|
|
10
|
+
excludeKeys,
|
|
11
|
+
emptyFieldValue: '-',
|
|
12
|
+
parseValue
|
|
13
|
+
});
|
|
14
|
+
/**
|
|
15
|
+
* CSV format for data processing.
|
|
16
|
+
*
|
|
17
|
+
* @pram schemas Schema definitions to use for pivoting.
|
|
18
|
+
* @param nodes List of nodes to convert.
|
|
19
|
+
* @returns CSV content formatted with pivoted blank nodes.
|
|
20
|
+
*/
|
|
21
|
+
export const toCsvPivot = (schemas, nodes) => toCsv(nodes.map(node => pivotNode(schemas, node)));
|
package/esm/csv.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
2
|
+
var t = {};
|
|
3
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
4
|
+
t[p] = s[p];
|
|
5
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
6
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
7
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
8
|
+
t[p[i]] = s[p[i]];
|
|
9
|
+
}
|
|
10
|
+
return t;
|
|
11
|
+
};
|
|
12
|
+
import { NodeType } from '@hestia-earth/schema';
|
|
13
|
+
import { loadSchemas } from '@hestia-earth/json-schema';
|
|
14
|
+
import { isCSVIncluded, schemaFromKey } from '@hestia-earth/json-schema/schema-utils';
|
|
15
|
+
const get = require('lodash.get');
|
|
16
|
+
import { isExpandable } from './utils';
|
|
17
|
+
const noValue = '-';
|
|
18
|
+
export const csvColSep = ',';
|
|
19
|
+
const ignoreValues = {
|
|
20
|
+
Date: val => val instanceof Date,
|
|
21
|
+
GeoJSON: val => typeof val === 'object' && ('features' in val || 'feature' in val || 'geometry' in val)
|
|
22
|
+
};
|
|
23
|
+
const isIgnored = (val) => Object.values(ignoreValues).some(func => func(val));
|
|
24
|
+
/**
|
|
25
|
+
* Get the headers in a CSV file.
|
|
26
|
+
*
|
|
27
|
+
* @param csv The CSV content as string.
|
|
28
|
+
* @returns headers as a list.
|
|
29
|
+
*/
|
|
30
|
+
export const headersFromCsv = (csv) => csv.split('\n')[0].split(csvColSep);
|
|
31
|
+
const convertValue = {
|
|
32
|
+
default: (value) => `${value}`,
|
|
33
|
+
object: (value) => (value === null ? noValue : value instanceof Date ? value.toJSON() : JSON.stringify(value))
|
|
34
|
+
};
|
|
35
|
+
const escapeField = (value) => {
|
|
36
|
+
const convertType = typeof value in convertValue ? typeof value : 'default';
|
|
37
|
+
const val = `${convertValue[convertType](value)}`.replace(/"/g, '""').replace(/\r/g, '').replace('\n', '');
|
|
38
|
+
return val.includes(csvColSep) ? `"${val}"` : val;
|
|
39
|
+
};
|
|
40
|
+
const skipColumns = ['type', '@type', '@context'];
|
|
41
|
+
const typeToColumnName = (type = '') => `${type.substring(0, 1).toLowerCase()}${type.substring(1)}`;
|
|
42
|
+
const getChildrenHeaders = (key, value, useBrackets) => {
|
|
43
|
+
const headers = getHeaders(value, useBrackets);
|
|
44
|
+
return Array.isArray(value)
|
|
45
|
+
? headers.map(header => {
|
|
46
|
+
const [index, ...rest] = header.split('.');
|
|
47
|
+
return `${key}${useBrackets ? `[${index}]` : `.${index}`}.${rest.join('.')}`;
|
|
48
|
+
})
|
|
49
|
+
: headers.map(header => `${key}.${header}`);
|
|
50
|
+
};
|
|
51
|
+
const getHeaders = (node, useBrackets) => Object.keys(node)
|
|
52
|
+
.flatMap(key => {
|
|
53
|
+
const value = node[key];
|
|
54
|
+
return isExpandable(value) && !isIgnored(value) ? getChildrenHeaders(key, value, useBrackets) : key;
|
|
55
|
+
})
|
|
56
|
+
.filter(key => !skipColumns.includes(key));
|
|
57
|
+
const isIndexedNode = (headers) => {
|
|
58
|
+
const topLevel = headers.filter(header => !header.includes('.'));
|
|
59
|
+
return topLevel.some(header => header === '@id');
|
|
60
|
+
};
|
|
61
|
+
const defaultOptions = {
|
|
62
|
+
includeExising: true,
|
|
63
|
+
useBrackets: false,
|
|
64
|
+
includeAllHeaders: false,
|
|
65
|
+
selectedHeaders: []
|
|
66
|
+
};
|
|
67
|
+
const filterTermHeaders = (schemas, header) => {
|
|
68
|
+
const { title } = schemaFromKey(schemas, header);
|
|
69
|
+
return title !== NodeType.Term || header.endsWith('id') || header.endsWith('name');
|
|
70
|
+
};
|
|
71
|
+
const filterHeaders = (headers) => {
|
|
72
|
+
const schemas = loadSchemas();
|
|
73
|
+
const filterIncluded = isCSVIncluded(schemas);
|
|
74
|
+
return headers.filter(header => filterIncluded(header) && filterTermHeaders(schemas, header));
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* Convert a list of Nodes to CSV format.
|
|
78
|
+
*
|
|
79
|
+
* @param nodes List of nodes to convert.
|
|
80
|
+
* @param options Conversion options.
|
|
81
|
+
* @returns CSV content formatted for upload on Hestia.
|
|
82
|
+
*/
|
|
83
|
+
export const toCsv = (nodes, options) => {
|
|
84
|
+
const { includeExising, useBrackets, includeAllHeaders, selectedHeaders } = Object.assign(Object.assign({}, defaultOptions), options);
|
|
85
|
+
const headersByType = nodes.reduce((prev, _a) => {
|
|
86
|
+
var { type, '@type': _t } = _a, node = __rest(_a, ["type", '@type']);
|
|
87
|
+
type = type || _t;
|
|
88
|
+
prev[type] = type in prev ? prev[type] : [];
|
|
89
|
+
const headers = getHeaders(node, useBrackets);
|
|
90
|
+
prev[type] = Array.from(new Set([...prev[type], ...(!includeExising && isIndexedNode(headers) ? [] : headers)]));
|
|
91
|
+
return prev;
|
|
92
|
+
}, {});
|
|
93
|
+
const headers = Object.keys(headersByType).flatMap(type => headersByType[type].map(key => `${typeToColumnName(type)}.${key}`));
|
|
94
|
+
const allHeaders = selectedHeaders.length ? selectedHeaders : includeAllHeaders ? headers : filterHeaders(headers);
|
|
95
|
+
return [
|
|
96
|
+
allHeaders.join(csvColSep),
|
|
97
|
+
nodes
|
|
98
|
+
// remove all top-level nodes that have an @id - already uploaded
|
|
99
|
+
.filter(node => includeExising || !node['@id'])
|
|
100
|
+
.map(node => allHeaders
|
|
101
|
+
.map(header => {
|
|
102
|
+
const [type, ...rest] = header.split('.');
|
|
103
|
+
const ntype = node.type || node['@type'];
|
|
104
|
+
const value = typeToColumnName(ntype) === type ? get(node, rest.join('.'), noValue) : noValue;
|
|
105
|
+
return Array.isArray(value) ? escapeField(value.join(';')) : escapeField(value);
|
|
106
|
+
})
|
|
107
|
+
.join(csvColSep))
|
|
108
|
+
.join('\n')
|
|
109
|
+
].join('\n');
|
|
110
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { SchemaType, CycleFunctionalUnit } from '@hestia-earth/schema';
|
|
2
|
+
import { ellipsis, keyToLabel, reduceUndefinedValues } from './utils';
|
|
3
|
+
const findLinkedNode = (nodes, type, node) => node
|
|
4
|
+
? nodes.find(v => (v.type === type && v.id === node.id) || (v['@type'] === type && v['@id'] === node['@id'])) || node
|
|
5
|
+
: undefined;
|
|
6
|
+
export const siteLocationName = (region, country) => [
|
|
7
|
+
region === null || region === void 0 ? void 0 : region.name,
|
|
8
|
+
// make sure country doesnt appear in region already, if so don't add it
|
|
9
|
+
(country === null || country === void 0 ? void 0 : country.name) ? ((region === null || region === void 0 ? void 0 : region.name) && region.name.includes(country === null || country === void 0 ? void 0 : country.name) ? undefined : country === null || country === void 0 ? void 0 : country.name) : undefined
|
|
10
|
+
]
|
|
11
|
+
.filter(Boolean)
|
|
12
|
+
.join(', ');
|
|
13
|
+
export const primaryProduct = (products, defaultValue = {}) => (products && products.length
|
|
14
|
+
? products.find(product => product.primary || product.economicValueShare > 50) || products[0]
|
|
15
|
+
: null) || defaultValue;
|
|
16
|
+
const defaultName = (product, country, region, endDate, treatment, description) => [
|
|
17
|
+
(product === null || product === void 0 ? void 0 : product.name) || 'No Product',
|
|
18
|
+
siteLocationName(region, country),
|
|
19
|
+
endDate,
|
|
20
|
+
ellipsis(treatment, 20),
|
|
21
|
+
ellipsis(description, 30)
|
|
22
|
+
]
|
|
23
|
+
.filter(Boolean)
|
|
24
|
+
.join(' - ');
|
|
25
|
+
export const cycleDefaultName = ({ endDate, treatment, description, products }, site) => { var _a; return defaultName((_a = primaryProduct(products)) === null || _a === void 0 ? void 0 : _a.term, site === null || site === void 0 ? void 0 : site.country, site === null || site === void 0 ? void 0 : site.region, endDate, treatment, description); };
|
|
26
|
+
export const defaultSiteArea = (cycle) => reduceUndefinedValues({
|
|
27
|
+
siteArea: cycle.functionalUnit === CycleFunctionalUnit['1 ha'] ? cycle.siteArea || 1 : undefined
|
|
28
|
+
});
|
|
29
|
+
export const extendCycle = (nodes, cycle) => (Object.assign(Object.assign(Object.assign({}, cycle), { name: cycleDefaultName(cycle, findLinkedNode(nodes, SchemaType.Site, cycle.site)) }), defaultSiteArea(cycle)));
|
|
30
|
+
export const impactAssessmentDefaultName = ({ product, country, region, endDate }) => defaultName(product === null || product === void 0 ? void 0 : product.term, country, region, endDate);
|
|
31
|
+
export const extendImpactAssessment = (nodes, impactAssessment) => {
|
|
32
|
+
var _a;
|
|
33
|
+
const cycle = findLinkedNode(nodes, SchemaType.Cycle, impactAssessment.cycle);
|
|
34
|
+
// replace the product if there is a single match in the Cycle products
|
|
35
|
+
const products = (_a = cycle === null || cycle === void 0 ? void 0 : cycle.products) === null || _a === void 0 ? void 0 : _a.filter(v => { var _a, _b, _c; return ((_a = v.term) === null || _a === void 0 ? void 0 : _a['@id']) === ((_c = (_b = impactAssessment === null || impactAssessment === void 0 ? void 0 : impactAssessment.product) === null || _b === void 0 ? void 0 : _b.term) === null || _c === void 0 ? void 0 : _c['@id']); });
|
|
36
|
+
const product = (products === null || products === void 0 ? void 0 : products.length) === 1 ? products[0] : impactAssessment.product;
|
|
37
|
+
return Object.assign(Object.assign(Object.assign({}, impactAssessment), { name: impactAssessment.name || impactAssessmentDefaultName(impactAssessment), source: impactAssessment.source || (cycle === null || cycle === void 0 ? void 0 : cycle.defaultSource), product }), (cycle
|
|
38
|
+
? {
|
|
39
|
+
cycle: reduceUndefinedValues({
|
|
40
|
+
id: cycle.id,
|
|
41
|
+
type: SchemaType.Cycle,
|
|
42
|
+
name: cycle.name || cycleDefaultName(cycle, findLinkedNode(nodes, SchemaType.Site, cycle.site)),
|
|
43
|
+
description: cycle.description
|
|
44
|
+
})
|
|
45
|
+
}
|
|
46
|
+
: {}));
|
|
47
|
+
};
|
|
48
|
+
export const siteDefaultName = ({ siteType, region, country, description }, organisation) => [
|
|
49
|
+
siteType ? keyToLabel(siteType) : null,
|
|
50
|
+
organisation === null || organisation === void 0 ? void 0 : organisation.name,
|
|
51
|
+
siteLocationName(region, country),
|
|
52
|
+
ellipsis(description, 30)
|
|
53
|
+
]
|
|
54
|
+
.filter(Boolean)
|
|
55
|
+
.join(' - ');
|
|
56
|
+
export const extendSite = (nodes, site) => {
|
|
57
|
+
const org = findLinkedNode(nodes, SchemaType.Organisation, site.organisation);
|
|
58
|
+
return Object.assign(Object.assign({}, site), { name: siteDefaultName(site, org) });
|
|
59
|
+
};
|
|
60
|
+
const extendNodeType = {
|
|
61
|
+
[SchemaType.Cycle]: extendCycle,
|
|
62
|
+
[SchemaType.ImpactAssessment]: extendImpactAssessment,
|
|
63
|
+
[SchemaType.Site]: extendSite
|
|
64
|
+
};
|
|
65
|
+
const extendNode = (nodes) => (node) => node.type in extendNodeType ? reduceUndefinedValues(extendNodeType[node.type](nodes, node), true) : node;
|
|
66
|
+
export const setDefaultValues = (nodes) => nodes.map(extendNode(nodes));
|
package/esm/index.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { jsonToCsv } from './utils';
|
|
2
|
+
export { toCsvPivot } from './csv-pivot';
|
|
3
|
+
export { pivotNode, pivotNodes } from './json-pivot';
|
|
4
|
+
export * from './csv';
|
|
5
|
+
export * from './json';
|
|
6
|
+
export { cycleDefaultName, extendCycle, impactAssessmentDefaultName, extendImpactAssessment, siteLocationName, defaultSiteArea, extendSite } from './default-values';
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
2
|
+
var t = {};
|
|
3
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
4
|
+
t[p] = s[p];
|
|
5
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
6
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
7
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
8
|
+
t[p[i]] = s[p[i]];
|
|
9
|
+
}
|
|
10
|
+
return t;
|
|
11
|
+
};
|
|
12
|
+
import { NodeType, uniquenessFields, isBlankNode, isNode, termFields, refToSchemaType } from '@hestia-earth/schema';
|
|
13
|
+
import { keyType } from '@hestia-earth/json-schema/schema-utils';
|
|
14
|
+
const get = require('lodash.get');
|
|
15
|
+
import { childkey, findNonDuplicates, omit, parentKey } from './utils';
|
|
16
|
+
const ignoreKeys = [
|
|
17
|
+
'@context',
|
|
18
|
+
'@type',
|
|
19
|
+
'type',
|
|
20
|
+
'added',
|
|
21
|
+
'addedVersion',
|
|
22
|
+
'updated',
|
|
23
|
+
'updatedVersion',
|
|
24
|
+
'_cache',
|
|
25
|
+
// ignore all term keys as only the `@id` is included in the key
|
|
26
|
+
'term',
|
|
27
|
+
...termFields.map(v => `term.${v}`)
|
|
28
|
+
];
|
|
29
|
+
const headerJoin = '+';
|
|
30
|
+
const headerParentKey = (key) => key.split(headerJoin)[0];
|
|
31
|
+
const uniqueData = (value, key) => {
|
|
32
|
+
// in some cases, we are getting a list using "inputs.@id", but `get` will not be able to access this
|
|
33
|
+
const parentData = key.includes('.') ? get(value, parentKey(key), undefined) : undefined;
|
|
34
|
+
return Array.isArray(parentData) ? parentData.map(d => uniqueData(d, childkey(key))) : get(value, key, undefined);
|
|
35
|
+
};
|
|
36
|
+
const valueAsString = (value) => (Array.isArray(value) ? value : [value]).map(v => `${v}`).join(';');
|
|
37
|
+
const valueAsKey = (value) => ([value.match(/[\s\"\'\.;]/g)].some(Boolean) ? `[${value}]` : value);
|
|
38
|
+
const pivotValue = (schemas, nodeType, key, value) => {
|
|
39
|
+
var _a, _b;
|
|
40
|
+
// nodeType = Cycle
|
|
41
|
+
// key = emissions
|
|
42
|
+
// value = {'term':{'@id':'gwp100}, 'dates':['2021','2022'], 'value':[10,12]}
|
|
43
|
+
const uniqueFields = (_b = (_a = uniquenessFields[nodeType]) === null || _a === void 0 ? void 0 : _a[key]) !== null && _b !== void 0 ? _b : [];
|
|
44
|
+
// uniqueFields = ['term.@id', 'dates']
|
|
45
|
+
const pivotedKey = uniqueFields.length
|
|
46
|
+
? uniqueFields
|
|
47
|
+
.map(field => {
|
|
48
|
+
const data = uniqueData(value, field);
|
|
49
|
+
return typeof data === 'undefined'
|
|
50
|
+
? undefined
|
|
51
|
+
: field === 'term.@id' || uniqueFields.length === 1
|
|
52
|
+
? valueAsKey(valueAsString(data))
|
|
53
|
+
: `${parentKey(field)}[${valueAsString(data)}]`;
|
|
54
|
+
})
|
|
55
|
+
.filter(Boolean)
|
|
56
|
+
.join(headerJoin)
|
|
57
|
+
: '';
|
|
58
|
+
// pivotedKey = 'gwp100+dates[2021;2022]'
|
|
59
|
+
const data = omit(pivotNode(schemas, value, false), [...uniqueFields, ...ignoreKeys]);
|
|
60
|
+
return { [pivotedKey]: data };
|
|
61
|
+
};
|
|
62
|
+
const simplifiedKeys = (keys) => findNonDuplicates(keys.map(headerParentKey));
|
|
63
|
+
const mergeUnpivotedBlankNode = (values, existingValue) => Object.assign({}, ...values.map(v => ({ [v]: {} })), existingValue || {});
|
|
64
|
+
const mergeUnpivotedNodes = (values, uniqueArrayKey) => values.map(v => ({ [uniqueArrayKey]: v }));
|
|
65
|
+
const mergeUnpivotedByType = {
|
|
66
|
+
number: values => Number(values[0]),
|
|
67
|
+
string: values => values[0],
|
|
68
|
+
boolean: values => Boolean(values[0]),
|
|
69
|
+
null: () => null,
|
|
70
|
+
array: (values, definition, uniqueArrayKey, existingValue) => {
|
|
71
|
+
var _a;
|
|
72
|
+
return ((_a = definition === null || definition === void 0 ? void 0 : definition.items) === null || _a === void 0 ? void 0 : _a.$ref)
|
|
73
|
+
? mergeUnpivotedByType.$ref(values, definition.items, uniqueArrayKey, existingValue)
|
|
74
|
+
: [...(existingValue || []), ...values]
|
|
75
|
+
.map(v => mergeUnpivotedByType[definition.items.type || 'string']([v]))
|
|
76
|
+
.join(';');
|
|
77
|
+
},
|
|
78
|
+
$ref: (values, { $ref }, uniqueArrayKey, existingValue) => isBlankNode({ type: refToSchemaType($ref) })
|
|
79
|
+
? mergeUnpivotedBlankNode(values, existingValue)
|
|
80
|
+
: mergeUnpivotedNodes(values, uniqueArrayKey)
|
|
81
|
+
};
|
|
82
|
+
const mergeUnpivotedValue = (definition, uniqueArrayItem, data, nodeValue, pivotedKey) => {
|
|
83
|
+
var _a;
|
|
84
|
+
const [key, newValues] = [pivotedKey.split('[')[0], pivotedKey.split('[')[1].replace(']', '').split(';')];
|
|
85
|
+
const newValueDefinition = (_a = definition === null || definition === void 0 ? void 0 : definition.properties) === null || _a === void 0 ? void 0 : _a[key];
|
|
86
|
+
const uniqueArrayKey = childkey((uniqueArrayItem !== null && uniqueArrayItem !== void 0 ? uniqueArrayItem : []).find(v => v.startsWith(key)));
|
|
87
|
+
const valueType = newValueDefinition.type;
|
|
88
|
+
const functionValueType = '$ref' in newValueDefinition ? '$ref' : Array.isArray(valueType) ? valueType[0] : valueType;
|
|
89
|
+
data[key] = mergeUnpivotedByType[functionValueType](newValues, newValueDefinition, uniqueArrayKey, nodeValue[key]);
|
|
90
|
+
return data;
|
|
91
|
+
};
|
|
92
|
+
const unpivotBlankNode = (definition, uniqueArrayItem, key, value) => (Object.assign(Object.assign({}, value), key
|
|
93
|
+
.split(headerJoin)
|
|
94
|
+
.slice(1)
|
|
95
|
+
.reduce((prev, curr) => mergeUnpivotedValue(definition, uniqueArrayItem, prev, value, curr), {})));
|
|
96
|
+
/**
|
|
97
|
+
* Pivot list of Blank Node and simplify when only one instance is found
|
|
98
|
+
*
|
|
99
|
+
* @param nodeType
|
|
100
|
+
* @param key
|
|
101
|
+
* @param value
|
|
102
|
+
*/
|
|
103
|
+
const pivotBlankNodes = (schemas, nodeType, key, value) => {
|
|
104
|
+
var _a, _b;
|
|
105
|
+
const uniqueFields = (_b = (_a = uniquenessFields[nodeType]) === null || _a === void 0 ? void 0 : _a[key]) !== null && _b !== void 0 ? _b : [];
|
|
106
|
+
const data = Object.assign({}, ...value.map(val => pivotValue(schemas, nodeType, key, val)));
|
|
107
|
+
const dataKeys = Object.keys(data);
|
|
108
|
+
const uniqueKeys = simplifiedKeys(dataKeys);
|
|
109
|
+
return (uniqueFields === null || uniqueFields === void 0 ? void 0 : uniqueFields[0]) === 'term.@id'
|
|
110
|
+
? Object.fromEntries(Object.entries(data).map(([entryKey, value]) => {
|
|
111
|
+
var _a, _b, _c, _d;
|
|
112
|
+
const isUnique = entryKey !== headerParentKey(entryKey) && uniqueKeys.includes(headerParentKey(entryKey));
|
|
113
|
+
const blankNodeType = keyType(schemas[nodeType], key);
|
|
114
|
+
return [
|
|
115
|
+
isUnique ? headerParentKey(entryKey) : entryKey,
|
|
116
|
+
isUnique
|
|
117
|
+
? unpivotBlankNode(schemas[blankNodeType], (_d = (_c = (_b = (_a = schemas[nodeType]) === null || _a === void 0 ? void 0 : _a.properties) === null || _b === void 0 ? void 0 : _b[key]) === null || _c === void 0 ? void 0 : _c.uniqueArrayItem) !== null && _d !== void 0 ? _d : [], entryKey, value)
|
|
118
|
+
: value
|
|
119
|
+
];
|
|
120
|
+
}))
|
|
121
|
+
: data;
|
|
122
|
+
};
|
|
123
|
+
/**
|
|
124
|
+
* Pivot node and return in compacted format.
|
|
125
|
+
*
|
|
126
|
+
* @param schemas The definitions of the HESTIA Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
|
|
127
|
+
* @param node The node to pivot.
|
|
128
|
+
* @param keepType To keep the `@type`/`type` key in the pivoted format.
|
|
129
|
+
* @returns
|
|
130
|
+
*/
|
|
131
|
+
export const pivotNode = (schemas, _a, keepType) => {
|
|
132
|
+
var { '@type': _type, type } = _a, node = __rest(_a, ['@type', "type"]);
|
|
133
|
+
if (keepType === void 0) { keepType = true; }
|
|
134
|
+
return Object.fromEntries([
|
|
135
|
+
keepType && (_type || type) ? [_type ? '@type' : 'type', _type || type] : null,
|
|
136
|
+
...Object.entries(node)
|
|
137
|
+
.filter(([key]) => !ignoreKeys.includes(key) && ((_type || type) !== NodeType.Term || !termFields.includes(key)))
|
|
138
|
+
.map(([key, value]) => [
|
|
139
|
+
key,
|
|
140
|
+
value === null
|
|
141
|
+
? null
|
|
142
|
+
: typeof value === 'object'
|
|
143
|
+
? Array.isArray(value)
|
|
144
|
+
? value.every(isNode)
|
|
145
|
+
? value.map(v => pivotNode(schemas, v, false))
|
|
146
|
+
: value.every(isBlankNode)
|
|
147
|
+
? pivotBlankNodes(schemas, _type || type, key, value)
|
|
148
|
+
: valueAsString(value)
|
|
149
|
+
: pivotNode(schemas, value, false) // deep pivoting
|
|
150
|
+
: valueAsString(value)
|
|
151
|
+
])
|
|
152
|
+
].filter(Boolean));
|
|
153
|
+
};
|
|
154
|
+
export const pivotNodes = (schemas, nodes) => {
|
|
155
|
+
return nodes.map(node => {
|
|
156
|
+
try {
|
|
157
|
+
return pivotNode(schemas, node);
|
|
158
|
+
}
|
|
159
|
+
catch (err) {
|
|
160
|
+
throw new Error(`Error pivoting ${node['@type'] || node.type} with id "${node['@id'] || node.id}":\n${err.toString()}`);
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
};
|
package/esm/json.js
ADDED
|
@@ -0,0 +1,451 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
11
|
+
var t = {};
|
|
12
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
13
|
+
t[p] = s[p];
|
|
14
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
15
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
16
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
17
|
+
t[p[i]] = s[p[i]];
|
|
18
|
+
}
|
|
19
|
+
return t;
|
|
20
|
+
};
|
|
21
|
+
import * as csvtojson from 'csvtojson';
|
|
22
|
+
import * as levenshtein from 'fast-levenshtein';
|
|
23
|
+
import { wktToGeoJSON } from '@terraformer/wkt';
|
|
24
|
+
import { NodeType, typeToSchemaType, refToSchemaType, SchemaType, isBlankNode, isTypeNode } from '@hestia-earth/schema';
|
|
25
|
+
import { defaultProperties, excludedDefaultProperties } from '@hestia-earth/json-schema/types';
|
|
26
|
+
import { nonEmptyValue, reduceUndefinedValues, isEmpty, isIri, isBoolean, isNumber } from './utils';
|
|
27
|
+
import { setDefaultValues } from './default-values';
|
|
28
|
+
export const acceptedNodeTypes = Object.values(NodeType);
|
|
29
|
+
export var ErrorKeys;
|
|
30
|
+
(function (ErrorKeys) {
|
|
31
|
+
ErrorKeys["SchemaNotFound"] = "schema-not-found";
|
|
32
|
+
ErrorKeys["PropertyNotFound"] = "property-not-found";
|
|
33
|
+
ErrorKeys["PropertyInvalidFormat"] = "property-invalid-format";
|
|
34
|
+
ErrorKeys["DuplicatedIdFields"] = "duplicated-id-fields";
|
|
35
|
+
ErrorKeys["ObjectArrayInvalid"] = "object-array-invalid";
|
|
36
|
+
})(ErrorKeys || (ErrorKeys = {}));
|
|
37
|
+
const IGNORE_FIELD_KEY = '-';
|
|
38
|
+
const VALUE_TYPE_KEY = 'valueType';
|
|
39
|
+
const DEFAULT_ARRAY_DELIMITER = ';';
|
|
40
|
+
const arrayDelimiter = () => {
|
|
41
|
+
try {
|
|
42
|
+
return process.env.CSV_ARRAY_DELIMITER || DEFAULT_ARRAY_DELIMITER;
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
return DEFAULT_ARRAY_DELIMITER;
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
const safeParseJSON = (obj) => {
|
|
49
|
+
// throw already handled error or throw a generic invalid-format error
|
|
50
|
+
let data;
|
|
51
|
+
try {
|
|
52
|
+
data = JSON.parse(obj);
|
|
53
|
+
}
|
|
54
|
+
catch (err) { }
|
|
55
|
+
return data;
|
|
56
|
+
};
|
|
57
|
+
const internalProperties = ({ properties }) => Object.keys(properties).filter(property => properties[property].internal);
|
|
58
|
+
const isEmptyCell = (value) => value === IGNORE_FIELD_KEY || value === '';
|
|
59
|
+
const nonEmptyCell = (value) => !isEmptyCell(value);
|
|
60
|
+
const isEmptyNode = (node, schemas) => {
|
|
61
|
+
const type = node.type || node['@type'];
|
|
62
|
+
const schema = type ? schemas[type] : null;
|
|
63
|
+
const properties = Object.keys(node).filter(key => !excludedDefaultProperties.includes(key));
|
|
64
|
+
const defaultProperties = schema ? getDefaultProperties(schema).map(({ property }) => property) : [];
|
|
65
|
+
// get the list of properties that do not have a default value
|
|
66
|
+
const nonDefaultProperties = properties.filter(p => !defaultProperties.includes(p));
|
|
67
|
+
return nonDefaultProperties.length === 0;
|
|
68
|
+
};
|
|
69
|
+
const isEmptyValueType = {
|
|
70
|
+
undefined: () => true,
|
|
71
|
+
number: (value) => isNaN(value),
|
|
72
|
+
[SchemaType.Term]: (value) => !(value['@id'] || value.id || value.name),
|
|
73
|
+
blankNode: isEmptyNode,
|
|
74
|
+
array: (value) => value.filter(v => !isEmpty(v)).length === 0,
|
|
75
|
+
object: (value, schemas) => value === null
|
|
76
|
+
? false
|
|
77
|
+
: Array.isArray(value)
|
|
78
|
+
? isEmptyValueType.array(value)
|
|
79
|
+
: (value['@type'] || value.type) === SchemaType.Term
|
|
80
|
+
? isEmptyValueType[SchemaType.Term](value)
|
|
81
|
+
: isBlankNode(value)
|
|
82
|
+
? isEmptyValueType.blankNode(value, schemas)
|
|
83
|
+
: Object.keys(value).length === 0
|
|
84
|
+
};
|
|
85
|
+
const isEmptyValue = (value, schemas) => typeof value in isEmptyValueType ? isEmptyValueType[typeof value](value, schemas) : isEmptyCell(value);
|
|
86
|
+
const compileFullKey = (key1, key2) => {
|
|
87
|
+
// handle arrays
|
|
88
|
+
const joinKey = (key2 || '').startsWith('[') ? '' : '.';
|
|
89
|
+
return [key1, key2].filter(Boolean).join(joinKey);
|
|
90
|
+
};
|
|
91
|
+
const propertyRequiredValue = (value, required) => {
|
|
92
|
+
const val = typeof value === 'object' && required[0] in value ? value[required[0]] : value;
|
|
93
|
+
return nonEmptyCell(val) ? { [required[0]]: val } : {};
|
|
94
|
+
};
|
|
95
|
+
const parseErrors = (err) => { var _a; return ((_a = safeParseJSON(err.message)) === null || _a === void 0 ? void 0 : _a.errors) || []; };
|
|
96
|
+
const throwError = (error) => {
|
|
97
|
+
throw new Error(error);
|
|
98
|
+
};
|
|
99
|
+
export const throwCSVErrors = (errors) => throwError(JSON.stringify({ errors }));
|
|
100
|
+
const schemaNotFoundError = (schema) => throwCSVErrors([
|
|
101
|
+
{
|
|
102
|
+
message: ErrorKeys.SchemaNotFound,
|
|
103
|
+
schema
|
|
104
|
+
}
|
|
105
|
+
]);
|
|
106
|
+
const computeSuggestions = ({ title, properties }, key) => {
|
|
107
|
+
const internal = internalProperties({ properties });
|
|
108
|
+
const allKeys = Object.keys(properties).filter(k => ![...excludedDefaultProperties, ...internal].includes(k));
|
|
109
|
+
return [title === SchemaType.Term].every(Boolean)
|
|
110
|
+
? ['@id', 'name']
|
|
111
|
+
: allKeys.filter(v => levenshtein.get(v, key) <= 3);
|
|
112
|
+
};
|
|
113
|
+
const propertyNotFoundError = (schema, key, value) => throwCSVErrors([
|
|
114
|
+
{
|
|
115
|
+
schema: schema.title,
|
|
116
|
+
message: ErrorKeys.PropertyNotFound,
|
|
117
|
+
schemaKey: key,
|
|
118
|
+
key,
|
|
119
|
+
value,
|
|
120
|
+
suggestions: computeSuggestions(schema, key)
|
|
121
|
+
}
|
|
122
|
+
]);
|
|
123
|
+
const propertyInvalidFormat = (schema, key, value, func) => {
|
|
124
|
+
try {
|
|
125
|
+
return func();
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
const errors = parseErrors(err);
|
|
129
|
+
// throw already handled error or throw a generic invalid-format error
|
|
130
|
+
return throwCSVErrors(errors.length
|
|
131
|
+
? errors.map(data => (Object.assign(Object.assign({}, data), { key: compileFullKey(key, data.key) })))
|
|
132
|
+
: [
|
|
133
|
+
{
|
|
134
|
+
schema: schema.title,
|
|
135
|
+
message: ErrorKeys.PropertyInvalidFormat,
|
|
136
|
+
schemaKey: key,
|
|
137
|
+
key,
|
|
138
|
+
value,
|
|
139
|
+
error: err === null || err === void 0 ? void 0 : err.message
|
|
140
|
+
}
|
|
141
|
+
]);
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
const validateDuplicatedIdFields = (schema, key, value) => typeof value === 'object' && [value.id, value['@id']].every(v => !!v && nonEmptyCell(v))
|
|
145
|
+
? throwCSVErrors([
|
|
146
|
+
{
|
|
147
|
+
schema: schema.title,
|
|
148
|
+
message: ErrorKeys.DuplicatedIdFields,
|
|
149
|
+
schemaKey: key,
|
|
150
|
+
key,
|
|
151
|
+
value
|
|
152
|
+
}
|
|
153
|
+
])
|
|
154
|
+
: null;
|
|
155
|
+
const handleArrayError = func => (value, index) => {
|
|
156
|
+
try {
|
|
157
|
+
return func(value, index);
|
|
158
|
+
}
|
|
159
|
+
catch (err) {
|
|
160
|
+
const errors = parseErrors(err);
|
|
161
|
+
// throw already handled error or throw a generic invalid-format error
|
|
162
|
+
return errors.length
|
|
163
|
+
? throwCSVErrors(errors.map(data => (Object.assign(Object.assign({}, data), { key: compileFullKey(`${index}`, data.key) }))))
|
|
164
|
+
: (() => {
|
|
165
|
+
throw err;
|
|
166
|
+
})();
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
const allowedGeoJSONTypes = ['FeatureCollection', 'Feature', 'GeometryCollection'];
|
|
170
|
+
const geoJSONGeomtryTypeValidation = {
|
|
171
|
+
Point: () => throwError('use "latitude" and "longitude" instead of "Point"'),
|
|
172
|
+
MultiPoint: () => throwError('use a "Polygon" instead of "MultiPoint"'),
|
|
173
|
+
LineString: () => throwError('use a "Polygon" instead of "LineString"'),
|
|
174
|
+
MultiLineString: () => throwError('use a "MultiPolygon" instead of "MultiLineString"')
|
|
175
|
+
};
|
|
176
|
+
const validateGeoJSONGeometryType = ({ type }) => type in geoJSONGeomtryTypeValidation ? geoJSONGeomtryTypeValidation[type]() : true;
|
|
177
|
+
/**
|
|
178
|
+
* Validate the GeoJSON format provided.
|
|
179
|
+
* GeoJSON is only relevant if it includes a `Polygon` or `MultiPolygon`, which would represent an area.
|
|
180
|
+
* If a single `Point` is provided, `latitude` and `longitude` should be used instead.
|
|
181
|
+
*
|
|
182
|
+
* @param value The GeoJSON as object.
|
|
183
|
+
*/
|
|
184
|
+
const validateGeoJSONFormat = (value) => (!allowedGeoJSONTypes.includes(value.type)
|
|
185
|
+
? throwError(`Invalid GeoJSON "type". Allowed values are: ${allowedGeoJSONTypes.join(', ')}`)
|
|
186
|
+
: 'geometries' in value
|
|
187
|
+
? !value.geometries.length
|
|
188
|
+
? throwError('This GeoJSON appears to be empty. Please either add coordinates or remove the field.')
|
|
189
|
+
: (value.geometries || []).every(validateGeoJSONGeometryType)
|
|
190
|
+
: 'geometry' in value
|
|
191
|
+
? validateGeoJSONGeometryType(value.geometry)
|
|
192
|
+
: 'features' in value
|
|
193
|
+
? !value.features.length
|
|
194
|
+
? throwError('This GeoJSON appears to be empty. Please either add coordinates or remove the field.')
|
|
195
|
+
: (value.features || []).every(validateGeoJSONFormat)
|
|
196
|
+
: true)
|
|
197
|
+
? value
|
|
198
|
+
: null;
|
|
199
|
+
const parseWtkValue = (value) => {
|
|
200
|
+
try {
|
|
201
|
+
return {
|
|
202
|
+
type: 'FeatureCollection',
|
|
203
|
+
features: [
|
|
204
|
+
{
|
|
205
|
+
type: 'Feature',
|
|
206
|
+
properties: {},
|
|
207
|
+
geometry: wktToGeoJSON(value)
|
|
208
|
+
}
|
|
209
|
+
]
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
catch (error) {
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
|
+
};
|
|
216
|
+
const parseJsonValue = (value) => {
|
|
217
|
+
try {
|
|
218
|
+
const data = JSON.parse(value);
|
|
219
|
+
return ['Polygon', 'MultiPolygon'].includes(data.type)
|
|
220
|
+
? {
|
|
221
|
+
type: 'FeatureCollection',
|
|
222
|
+
features: [
|
|
223
|
+
{
|
|
224
|
+
type: 'Feature',
|
|
225
|
+
properties: {},
|
|
226
|
+
geometry: data
|
|
227
|
+
}
|
|
228
|
+
]
|
|
229
|
+
}
|
|
230
|
+
: ['Feature'].includes(data.type)
|
|
231
|
+
? {
|
|
232
|
+
type: 'FeatureCollection',
|
|
233
|
+
features: [
|
|
234
|
+
Object.assign(Object.assign({}, data), { properties: {} // make sure they are set or it is invalid
|
|
235
|
+
})
|
|
236
|
+
]
|
|
237
|
+
}
|
|
238
|
+
: data;
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
return throwError('Unable to parse GeoJSON value. Please make sure the formatting is correct.');
|
|
242
|
+
}
|
|
243
|
+
};
|
|
244
|
+
const parseGeoJSONValue = (value) => {
|
|
245
|
+
const result = parseWtkValue(value) || parseJsonValue(value);
|
|
246
|
+
return result ? validateGeoJSONFormat(result) : value;
|
|
247
|
+
};
|
|
248
|
+
/**
|
|
249
|
+
* If the user provided a non-object where an object was expected, assume it was meant to be the `name` property.
|
|
250
|
+
* For Blank Nodes, we assume it is the `term.name`.
|
|
251
|
+
* For non-Term Nodes, we assume it is the `id`.
|
|
252
|
+
*
|
|
253
|
+
* @param value The value mapped as a Ref
|
|
254
|
+
*/
|
|
255
|
+
const schemaRefValue = (value, type) => typeof value === 'object'
|
|
256
|
+
? value
|
|
257
|
+
: type === SchemaType.Term
|
|
258
|
+
? { name: value }
|
|
259
|
+
: isBlankNode({ type })
|
|
260
|
+
? { term: { name: value } }
|
|
261
|
+
: { id: value };
|
|
262
|
+
const isSemver = (value) => value.match(/^[\d]+\.[\d]+\.[\d]+$/) !== null;
|
|
263
|
+
export const cleanStringValue = (value) => (isNumber(value) && !isSemver(value) ? value.replace(/\.0$/, '') : value).trim();
|
|
264
|
+
const propertyTypeToValue = {
|
|
265
|
+
null: (value, ...args) => (isEmptyCell(value) ? null : propertyTypeToValue.auto(value, ...args)),
|
|
266
|
+
string: (value) => cleanStringValue(value || ''),
|
|
267
|
+
number: (value) => isEmptyCell(value)
|
|
268
|
+
? undefined
|
|
269
|
+
: isNaN(+value)
|
|
270
|
+
? (() => {
|
|
271
|
+
throw new Error('failed to parse number');
|
|
272
|
+
})()
|
|
273
|
+
: +value,
|
|
274
|
+
integer: (value) => (isEmptyCell(value) ? undefined : +value),
|
|
275
|
+
boolean: (value) => isEmptyCell(value)
|
|
276
|
+
? undefined
|
|
277
|
+
: isBoolean(value)
|
|
278
|
+
? value.toLowerCase() === 'true'
|
|
279
|
+
: isNumber(value)
|
|
280
|
+
? +value === 1 // handle setting 1 or 1.0 as true
|
|
281
|
+
: (() => {
|
|
282
|
+
throw new Error('failed to parse boolean, expected true or false');
|
|
283
|
+
})(),
|
|
284
|
+
object: (value, schemas, { $ref, required, geojson }, params) => geojson
|
|
285
|
+
? propertyTypeToValue.geojson(value)
|
|
286
|
+
: $ref
|
|
287
|
+
? propertyTypeToValue.$ref(value, schemas, { $ref }, params)
|
|
288
|
+
: propertyTypeToValue.required(value, schemas, { required }, params),
|
|
289
|
+
array: (value, schemas, { items }, params) => (Array.isArray(value) ? value : value.split(arrayDelimiter()))
|
|
290
|
+
.map(handleArrayError(val => items
|
|
291
|
+
? '$ref' in items
|
|
292
|
+
? propertyTypeToValue.object(val, schemas, items, params)
|
|
293
|
+
: Array.isArray(items.type) || items.anyOf
|
|
294
|
+
? val === IGNORE_FIELD_KEY
|
|
295
|
+
? null
|
|
296
|
+
: propertyTypeToValue.auto(val, schemas, items, params)
|
|
297
|
+
: propertyTypeToValue[items.type](val, schemas, items, params)
|
|
298
|
+
: val))
|
|
299
|
+
.filter(val => !isEmptyValue(val, schemas)),
|
|
300
|
+
// try to determine the type automatically
|
|
301
|
+
auto: (value, schemas, _d, params) =>
|
|
302
|
+
// iris are mapped as {@id: val}
|
|
303
|
+
isIri(value)
|
|
304
|
+
? propertyTypeToValue.required(value, schemas, { required: ['@id'] }, params)
|
|
305
|
+
: isBoolean(value)
|
|
306
|
+
? propertyTypeToValue.boolean(value)
|
|
307
|
+
: isNumber(value)
|
|
308
|
+
? propertyTypeToValue.number(value)
|
|
309
|
+
: value === 'null'
|
|
310
|
+
? null
|
|
311
|
+
: propertyTypeToValue.string(value),
|
|
312
|
+
required: (value, _schemas, { required }) => {
|
|
313
|
+
const data = propertyRequiredValue(value, required);
|
|
314
|
+
return isEmpty(data) ? {} : data;
|
|
315
|
+
},
|
|
316
|
+
$ref: (value, schemas, { $ref }, params) => {
|
|
317
|
+
const schemaType = refToSchemaType($ref);
|
|
318
|
+
const schema = schemaType ? schemas[schemaType] : undefined;
|
|
319
|
+
const data = schema
|
|
320
|
+
? mapContent(schemas, schema, params)(schemaRefValue(value, schemaType))
|
|
321
|
+
: $ref in propertyTypeToValue
|
|
322
|
+
? propertyTypeToValue[$ref](value)
|
|
323
|
+
: safeParseJSON(value);
|
|
324
|
+
const addDefaults = isBlankNode(data) || schemaType === SchemaType.Actor; // only blank nodes or actors allowed
|
|
325
|
+
return isEmpty(data)
|
|
326
|
+
? {}
|
|
327
|
+
: !!schema
|
|
328
|
+
? extendDataFromSchema(data, schema, schemaType, Object.assign(Object.assign({}, params), { addDefaults }))
|
|
329
|
+
: data;
|
|
330
|
+
},
|
|
331
|
+
geojson: value => (isEmptyCell(value) ? undefined : parseGeoJSONValue(value))
|
|
332
|
+
};
|
|
333
|
+
const getPropertyDefinition = (schema, key, ignoreErrors = false, value) => key in schema.properties ? schema.properties[key] : !ignoreErrors && propertyNotFoundError(schema, key, value);
|
|
334
|
+
const getValueType = (schema, json) => VALUE_TYPE_KEY in schema.properties
|
|
335
|
+
? // valueType can be present but skipped
|
|
336
|
+
(json[VALUE_TYPE_KEY] !== IGNORE_FIELD_KEY ? json[VALUE_TYPE_KEY] : null) ||
|
|
337
|
+
schema.properties[VALUE_TYPE_KEY].default
|
|
338
|
+
: null;
|
|
339
|
+
const getPropertyType = (def, key, valueType) =>
|
|
340
|
+
// default type is object (like $ref)
|
|
341
|
+
(key === 'value' && valueType
|
|
342
|
+
? valueType
|
|
343
|
+
: Array.isArray(def.type)
|
|
344
|
+
? def.type.includes('null')
|
|
345
|
+
? 'null'
|
|
346
|
+
: 'auto'
|
|
347
|
+
: def.type) || 'object';
|
|
348
|
+
const setDefaultProperty = (prop, def, node, ignoreInternal = false) => prop !== VALUE_TYPE_KEY &&
|
|
349
|
+
// only get default if internal or not present
|
|
350
|
+
((!ignoreInternal && def.internal) || !node || !(prop in node));
|
|
351
|
+
const getDefaultProperties = (schema, node, ignoreInternal = false) => defaultProperties(schema)
|
|
352
|
+
.map((property) => {
|
|
353
|
+
const def = schema.properties[property];
|
|
354
|
+
return setDefaultProperty(property, def, node, ignoreInternal) ? { property, defaultValue: def.default } : null;
|
|
355
|
+
}, {})
|
|
356
|
+
.filter(Boolean);
|
|
357
|
+
const filterProperties = (data, excludedProps) => Object.keys(data)
|
|
358
|
+
.filter(key => !excludedProps.includes(key))
|
|
359
|
+
.reduce((prev, curr) => (Object.assign(Object.assign({}, prev), { [curr]: data[curr] })), {});
|
|
360
|
+
const nodeType = (id, type, existingNode = false) => type
|
|
361
|
+
? existingNode
|
|
362
|
+
? isTypeNode(type)
|
|
363
|
+
? { '@type': type, '@id': id }
|
|
364
|
+
: { '@type': type }
|
|
365
|
+
: isTypeNode(type)
|
|
366
|
+
? { type, id }
|
|
367
|
+
: { type }
|
|
368
|
+
: {};
|
|
369
|
+
const extendDataFromSchema = (_a, schema, type, _b) => {
|
|
370
|
+
var { id, '@id': _id, type: _t } = _a, data = __rest(_a, ["id", '@id', "type"]);
|
|
371
|
+
var _c = _b === void 0 ? {} : _b, ignoreInternal = _c.ignoreInternal, addDefaults = _c.addDefaults, convertToExistingNodes = _c.convertToExistingNodes;
|
|
372
|
+
return reduceUndefinedValues(Object.assign(Object.assign(Object.assign({}, filterProperties(data, ignoreInternal ? [] : internalProperties(schema))), (addDefaults && !_id && schema
|
|
373
|
+
? getDefaultProperties(schema, data, ignoreInternal).reduce((prev, { property, defaultValue }) => {
|
|
374
|
+
prev[property] = defaultValue;
|
|
375
|
+
return prev;
|
|
376
|
+
}, {})
|
|
377
|
+
: {})), nodeType(id || _id, type, !!_id || convertToExistingNodes)), true);
|
|
378
|
+
};
|
|
379
|
+
const validateArrayType = (schema, def, key, value) => typeof value !== 'object' ||
|
|
380
|
+
def.type !== 'array' ||
|
|
381
|
+
Array.isArray(value) ||
|
|
382
|
+
throwCSVErrors([
|
|
383
|
+
{
|
|
384
|
+
schema: schema.title,
|
|
385
|
+
message: ErrorKeys.ObjectArrayInvalid,
|
|
386
|
+
schemaKey: key,
|
|
387
|
+
key,
|
|
388
|
+
value
|
|
389
|
+
}
|
|
390
|
+
]);
|
|
391
|
+
const mapContent = (schemas, schema, params) => (json) => {
|
|
392
|
+
const values = Object.keys(json)
|
|
393
|
+
.filter(nonEmptyCell)
|
|
394
|
+
.map(key => {
|
|
395
|
+
try {
|
|
396
|
+
const value = json[key];
|
|
397
|
+
// make sure we are not using both `id` and `@id` fields
|
|
398
|
+
validateDuplicatedIdFields(schema, key, value);
|
|
399
|
+
const propertyDefinition = getPropertyDefinition(schema, key, false, json);
|
|
400
|
+
const valueType = getValueType(schema, json);
|
|
401
|
+
const type = getPropertyType(propertyDefinition, key, valueType);
|
|
402
|
+
validateArrayType(schema, propertyDefinition, key, value);
|
|
403
|
+
const newValue = key === VALUE_TYPE_KEY
|
|
404
|
+
? ''
|
|
405
|
+
: propertyInvalidFormat(schema, key, value, () => Array.isArray(type) ? value : propertyTypeToValue[type](value, schemas, propertyDefinition, params));
|
|
406
|
+
return {
|
|
407
|
+
errors: [],
|
|
408
|
+
value: type !== 'null' && isEmptyValue(newValue, schemas) ? undefined : [key, newValue]
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
catch (err) {
|
|
412
|
+
const { errors } = JSON.parse(err.message);
|
|
413
|
+
return { errors };
|
|
414
|
+
}
|
|
415
|
+
});
|
|
416
|
+
const errors = values.filter(({ errors }) => (errors === null || errors === void 0 ? void 0 : errors.length) > 0).flatMap(({ errors }) => errors);
|
|
417
|
+
return (errors === null || errors === void 0 ? void 0 : errors.length)
|
|
418
|
+
? throwError(JSON.stringify({ errors }))
|
|
419
|
+
: Object.fromEntries([schema.$id ? ['type', schema.title] : null, ...values.map(({ value }) => value)].filter(Boolean));
|
|
420
|
+
};
|
|
421
|
+
export const formatNode = (schemas, type, data, params = {}) => {
|
|
422
|
+
const schema = type in schemas && acceptedNodeTypes.includes(type) ? schemas[type] : schemaNotFoundError(type);
|
|
423
|
+
const content = mapContent(schemas, schema, params)(data);
|
|
424
|
+
return reduceUndefinedValues(extendDataFromSchema(content, schema, type, params), true);
|
|
425
|
+
};
|
|
426
|
+
export const filterEmptyNode = (schemas, node) => {
|
|
427
|
+
const type = node.type || node['@type'];
|
|
428
|
+
const schema = type ? schemas[type] : null;
|
|
429
|
+
// make sure defaultProperties are not counted
|
|
430
|
+
return !!schema && !isEmptyNode(node, schemas);
|
|
431
|
+
};
|
|
432
|
+
const convetToNode = (schemas, params) => (data) => Object.keys(data)
|
|
433
|
+
.filter(nonEmptyValue)
|
|
434
|
+
.map(type => formatNode(schemas, typeToSchemaType(type) || type, data[type], Object.assign(Object.assign({}, params), { convertToExistingNodes: params.convertToExistingNodes || (typeof data[type] === 'object' ? '@id' in data[type] : false) })))
|
|
435
|
+
.filter(node => filterEmptyNode(schemas, node));
|
|
436
|
+
/**
|
|
437
|
+
* Convert CSV to HESTIA JSON-LD format.
|
|
438
|
+
*
|
|
439
|
+
* @param schemas The definitions of the HESTIA Schema (`import { loadSchemas } from "@hestia-earth/json-schema"`)
|
|
440
|
+
* @param content The content of the CSV as a string
|
|
441
|
+
* @param params Conversion parameters.
|
|
442
|
+
* @returns A list of JSON-LD content.
|
|
443
|
+
*/
|
|
444
|
+
export const toJson = (schemas, content, params = { ignoreInternal: true, addDefaults: true }) => __awaiter(void 0, void 0, void 0, function* () {
|
|
445
|
+
const nodes = yield csvtojson({
|
|
446
|
+
delimiter: 'auto',
|
|
447
|
+
ignoreColumns: /^$/
|
|
448
|
+
}).fromString(content);
|
|
449
|
+
const converted = nodes.flatMap(convetToNode(schemas, params));
|
|
450
|
+
return setDefaultValues(converted);
|
|
451
|
+
});
|
package/esm/utils.js
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { json2csv } from 'json-2-csv';
|
|
2
|
+
const get = require('lodash.get');
|
|
3
|
+
const unset = require('lodash.unset');
|
|
4
|
+
export const omitKeys = [
|
|
5
|
+
'@context',
|
|
6
|
+
'schemaVersion',
|
|
7
|
+
'dataPrivate',
|
|
8
|
+
...['bibliography', 'site', 'cycle', 'impactAssessment', 'defaultSource', 'source'].flatMap(k => [
|
|
9
|
+
`${k}.type`,
|
|
10
|
+
`${k}.@type`
|
|
11
|
+
])
|
|
12
|
+
];
|
|
13
|
+
export const parentKey = (key) => key.split('.')[0];
|
|
14
|
+
export const childkey = (key) => key.split('.').slice(1).join('.');
|
|
15
|
+
export const omit = (data, keys) => {
|
|
16
|
+
const obj = Object.assign({}, data);
|
|
17
|
+
keys.forEach(key => {
|
|
18
|
+
unset(obj, key);
|
|
19
|
+
// handle arrays and notation without array
|
|
20
|
+
const _parentKey = parentKey(key);
|
|
21
|
+
const _childkey = childkey(key);
|
|
22
|
+
const parentData = key.includes('.') ? get(obj, _parentKey, undefined) : undefined;
|
|
23
|
+
if (typeof parentData === 'object') {
|
|
24
|
+
if (Array.isArray(parentData)) {
|
|
25
|
+
parentData.forEach((_v, index) => {
|
|
26
|
+
unset(obj, `${_parentKey}[${index}].${_childkey}`);
|
|
27
|
+
});
|
|
28
|
+
// only keep values that are not empty
|
|
29
|
+
obj[_parentKey] = obj[_parentKey].filter(v => !isEmpty(v));
|
|
30
|
+
}
|
|
31
|
+
if (isEmpty(obj[_parentKey])) {
|
|
32
|
+
unset(obj, _parentKey);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
return obj;
|
|
37
|
+
};
|
|
38
|
+
export const isExpandable = (val) => !!val && typeof val === 'object' && (Array.isArray(val) ? val.every(isExpandable) : Object.keys(val).length);
|
|
39
|
+
export const isIri = (value) => (value || '').startsWith('http');
|
|
40
|
+
export const isBoolean = (value) => value.toLowerCase() === 'true' || value.toLowerCase() === 'false';
|
|
41
|
+
export const isNumber = (n) => !isNaN(parseFloat(`${n}`)) && isFinite(parseFloat(`${n}`));
|
|
42
|
+
const ignoreKey = (key) => !['@type', 'type'].includes(key);
|
|
43
|
+
export const isEmpty = (value, minKeys = 1) => value === null ||
|
|
44
|
+
typeof value === 'undefined' ||
|
|
45
|
+
(typeof value === 'object'
|
|
46
|
+
? Array.isArray(value)
|
|
47
|
+
? !value.length
|
|
48
|
+
: Object.keys(value).filter(key => key !== 'type').length < minKeys
|
|
49
|
+
: value === '');
|
|
50
|
+
export const nonEmptyValue = (value) => !isEmpty(value) && value !== '-';
|
|
51
|
+
export const nonEmptyNode = (node) => typeof node === 'object'
|
|
52
|
+
? Object.keys(node).filter(key => ignoreKey(key) && !isEmpty(node[key])).length > 0
|
|
53
|
+
: nonEmptyValue(node);
|
|
54
|
+
const isUndefined = (value, allowNull) => (value === null && !allowNull) ||
|
|
55
|
+
typeof value === 'undefined' ||
|
|
56
|
+
(typeof value === 'object' && value !== null && !(value instanceof Date) && !Object.keys(value).length);
|
|
57
|
+
export const reduceUndefinedValues = (obj, allowNull = false) => Object.keys(obj).reduce((prev, key) => {
|
|
58
|
+
const value = obj[key];
|
|
59
|
+
return Object.assign(Object.assign({}, prev), (isUndefined(value, allowNull) ? {} : { [key]: value }));
|
|
60
|
+
}, {});
|
|
61
|
+
export const ellipsis = (text = '', maxlength = 20) => text.length > maxlength ? `${text.substring(0, maxlength)}...` : text;
|
|
62
|
+
export const keyToLabel = (key) => `${key[0].toUpperCase()}${key
|
|
63
|
+
.replace(/([a-z])([A-Z])/g, '$1 $2')
|
|
64
|
+
.replace(/([_])([a-zA-Z])/g, g => ` ${g[1].toUpperCase()}`)
|
|
65
|
+
.substring(1)}`;
|
|
66
|
+
export const diffInDays = (date1, date2) => Math.abs(Math.round((new Date(date2).getTime() - new Date(date1).getTime()) / (24 * 60 * 60 * 1000)));
|
|
67
|
+
export const daysBefore = (date = new Date(), days = 1) => {
|
|
68
|
+
const newDate = new Date(date);
|
|
69
|
+
newDate.setDate(newDate.getDate() - days);
|
|
70
|
+
return newDate;
|
|
71
|
+
};
|
|
72
|
+
export const daysInYear = (year) => ((year % 4 === 0 && year % 100 > 0) || year % 400 === 0 ? 366 : 365);
|
|
73
|
+
export const convertValue = (value) => Array.isArray(value) ? value.map(String).join(';') : typeof value === 'object' ? JSON.stringify(value) : value;
|
|
74
|
+
const count = (values) => values.reduce((a, b) => (Object.assign(Object.assign({}, a), { [b]: (a[b] || 0) + 1 })), {});
|
|
75
|
+
const duplicates = (dict) => Object.keys(dict).filter(a => dict[a] > 1);
|
|
76
|
+
const nonDuplicates = (dict) => Object.keys(dict).filter(a => dict[a] === 1);
|
|
77
|
+
export const findDuplicates = (values) => duplicates(count(values));
|
|
78
|
+
export const findNonDuplicates = (values) => nonDuplicates(count(values));
|
|
79
|
+
export const uncapitalize = (text) => `${text[0].toLowerCase()}${text.substring(1)}`;
|
|
80
|
+
/**
|
|
81
|
+
* Move every item on the same row.
|
|
82
|
+
*
|
|
83
|
+
* @param data
|
|
84
|
+
* @returns List of original records with less gaps in data.
|
|
85
|
+
*/
|
|
86
|
+
const compactDataForCsv = (data) => data.reduce((prev, curr) => {
|
|
87
|
+
const [[key, value]] = Object.entries(curr);
|
|
88
|
+
const prevNoKeyIndex = prev.findIndex(p => !Object.keys(p).includes(key));
|
|
89
|
+
prevNoKeyIndex >= 0 ? (prev[prevNoKeyIndex][key] = value) : prev.push(curr);
|
|
90
|
+
return prev;
|
|
91
|
+
}, []);
|
|
92
|
+
export const jsonToCsv = (values, options = {}) => json2csv(compactDataForCsv(values), Object.assign({ emptyFieldValue: '', expandNestedObjects: true, expandArrayObjects: true, arrayIndexesAsKeys: true, sortHeader: true }, options));
|
package/index.d.ts
CHANGED
|
@@ -3,3 +3,4 @@ export { toCsvPivot } from './csv-pivot';
|
|
|
3
3
|
export { pivotNode, pivotNodes } from './json-pivot';
|
|
4
4
|
export * from './csv';
|
|
5
5
|
export * from './json';
|
|
6
|
+
export { cycleDefaultName, extendCycle, impactAssessmentDefaultName, extendImpactAssessment, siteLocationName, defaultSiteArea, extendSite } from './default-values';
|
package/index.js
CHANGED
|
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.pivotNodes = exports.pivotNode = exports.toCsvPivot = exports.jsonToCsv = void 0;
|
|
17
|
+
exports.extendSite = exports.defaultSiteArea = exports.siteLocationName = exports.extendImpactAssessment = exports.impactAssessmentDefaultName = exports.extendCycle = exports.cycleDefaultName = exports.pivotNodes = exports.pivotNode = exports.toCsvPivot = exports.jsonToCsv = void 0;
|
|
18
18
|
var utils_1 = require("./utils");
|
|
19
19
|
Object.defineProperty(exports, "jsonToCsv", { enumerable: true, get: function () { return utils_1.jsonToCsv; } });
|
|
20
20
|
var csv_pivot_1 = require("./csv-pivot");
|
|
@@ -24,3 +24,11 @@ Object.defineProperty(exports, "pivotNode", { enumerable: true, get: function ()
|
|
|
24
24
|
Object.defineProperty(exports, "pivotNodes", { enumerable: true, get: function () { return json_pivot_1.pivotNodes; } });
|
|
25
25
|
__exportStar(require("./csv"), exports);
|
|
26
26
|
__exportStar(require("./json"), exports);
|
|
27
|
+
var default_values_1 = require("./default-values");
|
|
28
|
+
Object.defineProperty(exports, "cycleDefaultName", { enumerable: true, get: function () { return default_values_1.cycleDefaultName; } });
|
|
29
|
+
Object.defineProperty(exports, "extendCycle", { enumerable: true, get: function () { return default_values_1.extendCycle; } });
|
|
30
|
+
Object.defineProperty(exports, "impactAssessmentDefaultName", { enumerable: true, get: function () { return default_values_1.impactAssessmentDefaultName; } });
|
|
31
|
+
Object.defineProperty(exports, "extendImpactAssessment", { enumerable: true, get: function () { return default_values_1.extendImpactAssessment; } });
|
|
32
|
+
Object.defineProperty(exports, "siteLocationName", { enumerable: true, get: function () { return default_values_1.siteLocationName; } });
|
|
33
|
+
Object.defineProperty(exports, "defaultSiteArea", { enumerable: true, get: function () { return default_values_1.defaultSiteArea; } });
|
|
34
|
+
Object.defineProperty(exports, "extendSite", { enumerable: true, get: function () { return default_values_1.extendSite; } });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hestia-earth/schema-convert",
|
|
3
|
-
"version": "37.
|
|
3
|
+
"version": "37.2.0",
|
|
4
4
|
"description": "HESTIA Schema Converters",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"typings": "index.d.ts",
|
|
@@ -35,5 +35,53 @@
|
|
|
35
35
|
"json-2-csv": "^5.4.0",
|
|
36
36
|
"lodash.get": "^4.0.0",
|
|
37
37
|
"lodash.unset": "^4.0.0"
|
|
38
|
+
},
|
|
39
|
+
"sideEffects": false,
|
|
40
|
+
"module": "esm/index.js",
|
|
41
|
+
"exports": {
|
|
42
|
+
".": {
|
|
43
|
+
"import": "./esm/index.js",
|
|
44
|
+
"require": "./index.js"
|
|
45
|
+
},
|
|
46
|
+
"./convert-csv": {
|
|
47
|
+
"import": "./esm/convert-csv.js",
|
|
48
|
+
"require": "./convert-csv.js"
|
|
49
|
+
},
|
|
50
|
+
"./convert-json": {
|
|
51
|
+
"import": "./esm/convert-json.js",
|
|
52
|
+
"require": "./convert-json.js"
|
|
53
|
+
},
|
|
54
|
+
"./csv-pivot": {
|
|
55
|
+
"import": "./esm/csv-pivot.js",
|
|
56
|
+
"require": "./csv-pivot.js"
|
|
57
|
+
},
|
|
58
|
+
"./csv": {
|
|
59
|
+
"import": "./esm/csv.js",
|
|
60
|
+
"require": "./csv.js"
|
|
61
|
+
},
|
|
62
|
+
"./default-values": {
|
|
63
|
+
"import": "./esm/default-values.js",
|
|
64
|
+
"require": "./default-values.js"
|
|
65
|
+
},
|
|
66
|
+
"./file-utils": {
|
|
67
|
+
"import": "./esm/file-utils.js",
|
|
68
|
+
"require": "./file-utils.js"
|
|
69
|
+
},
|
|
70
|
+
"./json-pivot": {
|
|
71
|
+
"import": "./esm/json-pivot.js",
|
|
72
|
+
"require": "./json-pivot.js"
|
|
73
|
+
},
|
|
74
|
+
"./json": {
|
|
75
|
+
"import": "./esm/json.js",
|
|
76
|
+
"require": "./json.js"
|
|
77
|
+
},
|
|
78
|
+
"./pivot-file": {
|
|
79
|
+
"import": "./esm/pivot-file.js",
|
|
80
|
+
"require": "./pivot-file.js"
|
|
81
|
+
},
|
|
82
|
+
"./utils": {
|
|
83
|
+
"import": "./esm/utils.js",
|
|
84
|
+
"require": "./utils.js"
|
|
85
|
+
}
|
|
38
86
|
}
|
|
39
87
|
}
|