@firecms/data_import_export 3.0.0-canary.8 → 3.0.0-canary.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +113 -21
- package/README.md +1 -1
- package/dist/components/DataNewPropertiesMapping.d.ts +3 -5
- package/dist/components/ImportFileUpload.d.ts +1 -1
- package/dist/export_import/BasicExportAction.d.ts +7 -0
- package/dist/export_import/ExportCollectionAction.d.ts +1 -1
- package/dist/export_import/export.d.ts +15 -4
- package/dist/export_import/index.d.ts +4 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.es.js +940 -570
- package/dist/index.es.js.map +1 -1
- package/dist/index.umd.js +2 -2
- package/dist/index.umd.js.map +1 -1
- package/dist/types/column_mapping.d.ts +5 -7
- package/dist/useImportExportPlugin.d.ts +1 -1
- package/dist/utils/data.d.ts +3 -10
- package/dist/utils/file_headers.d.ts +1 -0
- package/dist/utils/file_to_json.d.ts +6 -1
- package/dist/utils/get_properties_mapping.d.ts +0 -3
- package/dist/utils/index.d.ts +0 -1
- package/package.json +20 -34
- package/src/components/DataNewPropertiesMapping.tsx +153 -40
- package/src/components/ImportFileUpload.tsx +12 -4
- package/src/components/ImportNewPropertyFieldPreview.tsx +7 -2
- package/src/export_import/BasicExportAction.tsx +147 -0
- package/src/export_import/ExportCollectionAction.tsx +45 -9
- package/src/export_import/ImportCollectionAction.tsx +22 -22
- package/src/export_import/export.ts +63 -29
- package/src/export_import/index.ts +4 -0
- package/src/hooks/useImportConfig.tsx +6 -0
- package/src/index.ts +1 -0
- package/src/types/column_mapping.ts +6 -6
- package/src/useImportExportPlugin.tsx +2 -2
- package/src/utils/data.ts +36 -126
- package/src/utils/file_headers.ts +90 -0
- package/src/utils/file_to_json.ts +33 -15
- package/src/utils/get_properties_mapping.ts +63 -59
- package/src/utils/index.ts +0 -1
package/src/utils/data.ts
CHANGED
@@ -1,41 +1,44 @@
|
|
1
|
-
import {
|
1
|
+
import {
|
2
|
+
Entity,
|
3
|
+
EntityReference,
|
4
|
+
getPropertyInPath,
|
5
|
+
isPropertyBuilder,
|
6
|
+
mergeDeep,
|
7
|
+
Properties,
|
8
|
+
Property,
|
9
|
+
PropertyOrBuilder,
|
10
|
+
ResolvedProperty,
|
11
|
+
resolveProperty
|
12
|
+
} from "@firecms/core";
|
2
13
|
import { unflattenObject } from "./file_to_json";
|
3
|
-
|
4
|
-
|
5
|
-
from: DataType;
|
6
|
-
fromSubtype?: DataType;
|
7
|
-
to: DataType;
|
8
|
-
toSubtype?: DataType;
|
9
|
-
}
|
14
|
+
import { getIn } from "@firecms/formex";
|
15
|
+
import { inferTypeFromValue } from "@firecms/schema_inference";
|
10
16
|
|
11
17
|
export function convertDataToEntity(data: Record<any, any>,
|
12
18
|
idColumn: string | undefined,
|
13
19
|
headersMapping: Record<string, string | null>,
|
14
20
|
properties: Properties,
|
15
|
-
|
16
|
-
|
21
|
+
path: string,
|
22
|
+
defaultValues: Record<string, any>): Entity<any> {
|
17
23
|
const flatObject = flattenEntry(data);
|
18
24
|
if (idColumn)
|
19
25
|
delete flatObject[idColumn];
|
20
26
|
const mappedKeysObject = Object.entries(flatObject)
|
21
27
|
.map(([key, value]) => {
|
22
|
-
const mappedKey = headersMapping
|
28
|
+
const mappedKey = getIn(headersMapping, key) ?? key;
|
23
29
|
|
24
30
|
const mappedProperty = getPropertyInPath(properties, mappedKey);
|
25
|
-
if (!mappedProperty)
|
31
|
+
if (!mappedProperty) {
|
26
32
|
return {};
|
27
|
-
|
28
|
-
const propertyMapping = propertiesMapping[mappedKey];
|
29
|
-
let valueResult = value;
|
30
|
-
if (propertyMapping) {
|
31
|
-
valueResult = processValueMapping(value, propertyMapping);
|
32
33
|
}
|
34
|
+
const processedValue = processValueMapping(value, mappedProperty);
|
33
35
|
return ({
|
34
|
-
[mappedKey]:
|
36
|
+
[mappedKey]: processedValue
|
35
37
|
});
|
36
38
|
})
|
37
39
|
.reduce((acc, curr) => ({ ...acc, ...curr }), {});
|
38
|
-
|
40
|
+
|
41
|
+
const values = mergeDeep(defaultValues ?? {}, unflattenObject(mappedKeysObject));
|
39
42
|
let id = idColumn ? data[idColumn] : undefined;
|
40
43
|
if (typeof id === "string") {
|
41
44
|
id = id.trim();
|
@@ -60,7 +63,7 @@ export function flattenEntry(obj: any, parent = ""): any {
|
|
60
63
|
return Object.keys(obj).reduce((acc, key) => {
|
61
64
|
const prefixedKey = parent ? `${parent}.${key}` : key;
|
62
65
|
|
63
|
-
if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) {
|
66
|
+
if (typeof obj[key] === "object" && !(obj[key] instanceof Date) && obj[key] !== null && !Array.isArray(obj[key])) {
|
64
67
|
Object.assign(acc, flattenEntry(obj[key], prefixedKey));
|
65
68
|
} else {
|
66
69
|
// @ts-ignore
|
@@ -71,116 +74,23 @@ export function flattenEntry(obj: any, parent = ""): any {
|
|
71
74
|
}, {});
|
72
75
|
}
|
73
76
|
|
74
|
-
|
75
|
-
|
76
|
-
// fullKey,
|
77
|
-
// value,
|
78
|
-
// idColumn,
|
79
|
-
// headersMapping,
|
80
|
-
// properties,
|
81
|
-
// propertiesMapping
|
82
|
-
// }: {
|
83
|
-
// key?: string,
|
84
|
-
// fullKey: string,
|
85
|
-
// value: any,
|
86
|
-
// idColumn?: string,
|
87
|
-
// headersMapping: Record<string, string | null>,
|
88
|
-
// properties: Properties,
|
89
|
-
// propertiesMapping: Record<string, DataTypeMapping>
|
90
|
-
// }): any {
|
91
|
-
//
|
92
|
-
// if (value === undefined) return value;
|
93
|
-
// if (value === null) return value;
|
94
|
-
//
|
95
|
-
// if (Array.isArray(value)) {
|
96
|
-
// const mappedKey = headersMapping[fullKey] || fullKey;
|
97
|
-
// const valueMapping = propertiesMapping[mappedKey];
|
98
|
-
// return processValueMapping(value, valueMapping);
|
99
|
-
// // return value.map(v => convertDataEntryValue({ value: v, fullKey, idColumn, headersMapping, propertiesMapping }));
|
100
|
-
// } else if (typeof value === "object") {
|
101
|
-
// return convertDataObjectValue({
|
102
|
-
// key,
|
103
|
-
// fullKey,
|
104
|
-
// value,
|
105
|
-
// idColumn,
|
106
|
-
// headersMapping,
|
107
|
-
// properties,
|
108
|
-
// propertiesMapping
|
109
|
-
// });
|
110
|
-
// } else {
|
111
|
-
// const mappedKey = headersMapping[fullKey] || fullKey;
|
112
|
-
// const valueMapping = propertiesMapping[mappedKey];
|
113
|
-
// return processValueMapping(value, valueMapping);
|
114
|
-
// }
|
115
|
-
// }
|
77
|
+
export function processValueMapping(value: any, property?: PropertyOrBuilder): any {
|
78
|
+
if (value === null) return null;
|
116
79
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
// propertiesMapping
|
125
|
-
// }: {
|
126
|
-
// key?: string,
|
127
|
-
// fullKey?: string,
|
128
|
-
// value: object,
|
129
|
-
// idColumn?: string,
|
130
|
-
// headersMapping: Record<string, string | null>,
|
131
|
-
// properties: Properties,
|
132
|
-
// propertiesMapping: Record<string, DataTypeMapping>
|
133
|
-
// }): object {
|
134
|
-
//
|
135
|
-
// if (value instanceof Date) {
|
136
|
-
// const mappedKey = fullKey ? headersMapping[fullKey] || fullKey : undefined;
|
137
|
-
// const valueMapping = mappedKey ? propertiesMapping[mappedKey] : undefined;
|
138
|
-
// return processValueMapping(value, valueMapping);
|
139
|
-
// }
|
140
|
-
//
|
141
|
-
// return Object.entries(value)
|
142
|
-
// .map(([childKey, childValue]) => {
|
143
|
-
// const childFullKey = fullKey ? `${fullKey}.${childKey}` : childKey;
|
144
|
-
// const mappedKey = headersMapping[childFullKey] ?? childFullKey;
|
145
|
-
// const mappedKeyLastPart = mappedKey.split(".").slice(-1)[0];
|
146
|
-
// const property = getPropertyInPath(properties, mappedKey);
|
147
|
-
// // if (!property) {
|
148
|
-
// // console.log("ppp", { properties, mappedKey, })
|
149
|
-
// // return {};
|
150
|
-
// // }
|
151
|
-
// return ({
|
152
|
-
// [mappedKeyLastPart]: convertDataEntryValue({
|
153
|
-
// key: childKey,
|
154
|
-
// fullKey: childFullKey,
|
155
|
-
// value: childValue,
|
156
|
-
// idColumn,
|
157
|
-
// headersMapping,
|
158
|
-
// properties,
|
159
|
-
// propertiesMapping
|
160
|
-
// })
|
161
|
-
// });
|
162
|
-
// })
|
163
|
-
// .reduce((acc, curr) => ({ ...acc, ...curr }), {});
|
164
|
-
// }
|
80
|
+
if (property === undefined) return value;
|
81
|
+
const usedProperty: ResolvedProperty | null = resolveProperty({
|
82
|
+
propertyOrBuilder: property,
|
83
|
+
})
|
84
|
+
if (usedProperty === null) return value;
|
85
|
+
const from = inferTypeFromValue(value);
|
86
|
+
const to = usedProperty.dataType;
|
165
87
|
|
166
|
-
|
167
|
-
|
168
|
-
const {
|
169
|
-
from,
|
170
|
-
to
|
171
|
-
} = valueMapping;
|
172
|
-
if (from === "array" && to === "array" && valueMapping.fromSubtype && valueMapping.toSubtype && Array.isArray(value)) {
|
173
|
-
return value.map(v => processValueMapping(v, {
|
174
|
-
from: valueMapping.fromSubtype!,
|
175
|
-
to: valueMapping.toSubtype!
|
176
|
-
}));
|
88
|
+
if (from === "array" && to === "array" && Array.isArray(value) && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
|
89
|
+
return value.map(v => processValueMapping(v, usedProperty.of as Property));
|
177
90
|
} else if (from === "string" && to === "number" && typeof value === "string") {
|
178
91
|
return Number(value);
|
179
|
-
} else if (from === "string" && to === "array" &&
|
180
|
-
return value.split(",").map((v: string) => processValueMapping(v,
|
181
|
-
from: "string",
|
182
|
-
to: valueMapping.toSubtype!
|
183
|
-
}));
|
92
|
+
} else if (from === "string" && to === "array" && typeof value === "string" && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
|
93
|
+
return value.split(",").map((v: string) => processValueMapping(v, usedProperty.of));
|
184
94
|
} else if (from === "string" && to === "boolean") {
|
185
95
|
return value === "true";
|
186
96
|
} else if (from === "number" && to === "boolean") {
|
@@ -0,0 +1,90 @@
|
|
1
|
+
import * as XLSX from "xlsx";
|
2
|
+
export function getXLSXHeaders(sheet: any) {
|
3
|
+
let header = 0; let offset = 1;
|
4
|
+
const hdr = [];
|
5
|
+
const o:any = {};
|
6
|
+
if (sheet == null || sheet["!ref"] == null) return [];
|
7
|
+
const range = o.range !== undefined ? o.range : sheet["!ref"];
|
8
|
+
let r;
|
9
|
+
if (o.header === 1) header = 1;
|
10
|
+
else if (o.header === "A") header = 2;
|
11
|
+
else if (Array.isArray(o.header)) header = 3;
|
12
|
+
switch (typeof range) {
|
13
|
+
case "string":
|
14
|
+
r = safeDecodeRange(range);
|
15
|
+
break;
|
16
|
+
case "number":
|
17
|
+
r = safeDecodeRange(sheet["!ref"]);
|
18
|
+
r.s.r = range;
|
19
|
+
break;
|
20
|
+
default:
|
21
|
+
r = range;
|
22
|
+
}
|
23
|
+
if (header > 0) offset = 0;
|
24
|
+
const rr = XLSX.utils.encode_row(r.s.r);
|
25
|
+
const cols = new Array(r.e.c - r.s.c + 1);
|
26
|
+
for (let C = r.s.c; C <= r.e.c; ++C) {
|
27
|
+
cols[C] = XLSX.utils.encode_col(C);
|
28
|
+
const val = sheet[cols[C] + rr];
|
29
|
+
switch (header) {
|
30
|
+
case 1:
|
31
|
+
hdr.push(C);
|
32
|
+
break;
|
33
|
+
case 2:
|
34
|
+
hdr.push(cols[C]);
|
35
|
+
break;
|
36
|
+
case 3:
|
37
|
+
hdr.push(o.header[C - r.s.c]);
|
38
|
+
break;
|
39
|
+
default:
|
40
|
+
if (val === undefined) continue;
|
41
|
+
hdr.push(XLSX.utils.format_cell(val));
|
42
|
+
}
|
43
|
+
}
|
44
|
+
return hdr;
|
45
|
+
}
|
46
|
+
|
47
|
+
function safeDecodeRange(range:any) {
|
48
|
+
const o = {
|
49
|
+
s: {
|
50
|
+
c: 0,
|
51
|
+
r: 0
|
52
|
+
},
|
53
|
+
e: {
|
54
|
+
c: 0,
|
55
|
+
r: 0
|
56
|
+
}
|
57
|
+
};
|
58
|
+
let idx = 0; let i = 0; let cc = 0;
|
59
|
+
const len = range.length;
|
60
|
+
for (idx = 0; i < len; ++i) {
|
61
|
+
if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
|
62
|
+
idx = 26 * idx + cc;
|
63
|
+
}
|
64
|
+
o.s.c = --idx;
|
65
|
+
|
66
|
+
for (idx = 0; i < len; ++i) {
|
67
|
+
if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
|
68
|
+
idx = 10 * idx + cc;
|
69
|
+
}
|
70
|
+
o.s.r = --idx;
|
71
|
+
|
72
|
+
if (i === len || range.charCodeAt(++i) === 58) {
|
73
|
+
o.e.c = o.s.c;
|
74
|
+
o.e.r = o.s.r;
|
75
|
+
return o;
|
76
|
+
}
|
77
|
+
|
78
|
+
for (idx = 0; i !== len; ++i) {
|
79
|
+
if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
|
80
|
+
idx = 26 * idx + cc;
|
81
|
+
}
|
82
|
+
o.e.c = --idx;
|
83
|
+
|
84
|
+
for (idx = 0; i !== len; ++i) {
|
85
|
+
if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
|
86
|
+
idx = 10 * idx + cc;
|
87
|
+
}
|
88
|
+
o.e.r = --idx;
|
89
|
+
return o;
|
90
|
+
}
|
@@ -1,38 +1,56 @@
|
|
1
1
|
import * as XLSX from "xlsx";
|
2
|
+
import { getXLSXHeaders } from "./file_headers";
|
2
3
|
|
3
|
-
|
4
|
+
type ConversionResult = {
|
5
|
+
data: object[];
|
6
|
+
propertiesOrder: string[]
|
7
|
+
}
|
8
|
+
|
9
|
+
export function convertFileToJson(file: File): Promise<ConversionResult> {
|
4
10
|
return new Promise((resolve, reject) => {
|
5
|
-
// check if file is a JSON file
|
6
11
|
if (file.type === "application/json") {
|
7
12
|
console.debug("Converting JSON file to JSON", file.name);
|
8
13
|
const reader = new FileReader();
|
9
14
|
reader.onload = function (e) {
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
15
|
+
try {
|
16
|
+
const data = e.target?.result as string;
|
17
|
+
const jsonData = JSON.parse(data);
|
18
|
+
if (!Array.isArray(jsonData)) {
|
19
|
+
reject(new Error("JSON file should contain an array of objects"));
|
20
|
+
} else {
|
21
|
+
// Assuming all objects in the array have the same structure/order
|
22
|
+
const propertiesOrder = jsonData.length > 0 ? Object.keys(jsonData[0]) : [];
|
23
|
+
resolve({
|
24
|
+
data: jsonData,
|
25
|
+
propertiesOrder
|
26
|
+
});
|
27
|
+
}
|
28
|
+
} catch (e) {
|
29
|
+
console.error("Error parsing JSON file", e);
|
30
|
+
reject(e);
|
14
31
|
}
|
15
|
-
resolve(jsonData);
|
16
32
|
};
|
17
33
|
reader.readAsText(file);
|
18
34
|
} else {
|
19
35
|
console.debug("Converting Excel file to JSON", file.name);
|
20
36
|
const reader = new FileReader();
|
21
37
|
reader.onload = function (e) {
|
22
|
-
|
23
38
|
const data = new Uint8Array(e.target?.result as ArrayBuffer);
|
24
|
-
const workbook = XLSX.read(data,
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
});
|
39
|
+
const workbook = XLSX.read(data, {
|
40
|
+
type: "array",
|
41
|
+
codepage: 65001,
|
42
|
+
cellDates: true,
|
43
|
+
});
|
30
44
|
const worksheetName = workbook.SheetNames[0];
|
31
45
|
const worksheet = workbook.Sheets[worksheetName];
|
32
46
|
const parsedData: Array<any> = XLSX.utils.sheet_to_json(worksheet);
|
47
|
+
const headers = getXLSXHeaders(worksheet);
|
33
48
|
const cleanedData = parsedData.map(mapJsonParse);
|
34
49
|
const jsonData = cleanedData.map(unflattenObject);
|
35
|
-
resolve(
|
50
|
+
resolve({
|
51
|
+
data: jsonData,
|
52
|
+
propertiesOrder: headers
|
53
|
+
});
|
36
54
|
};
|
37
55
|
reader.readAsArrayBuffer(file);
|
38
56
|
}
|
@@ -1,59 +1,63 @@
|
|
1
|
-
import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
|
2
|
-
import { DataTypeMapping } from "../types";
|
3
|
-
|
4
|
-
export function getPropertiesMapping(originProperties: Properties,
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
1
|
+
// import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
|
2
|
+
// import { DataTypeMapping } from "../types";
|
3
|
+
//
|
4
|
+
// export function getPropertiesMapping(originProperties: Properties,
|
5
|
+
// newProperties: Properties,
|
6
|
+
// headersMapping: Record<string, string | null>): Record<string, DataTypeMapping> {
|
7
|
+
//
|
8
|
+
// function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
|
9
|
+
//
|
10
|
+
// const dataMapping: Record<string, DataTypeMapping> = {};
|
11
|
+
//
|
12
|
+
// Object.keys(properties).forEach((key) => {
|
13
|
+
//
|
14
|
+
// const currentKey = namespace ? `${namespace}.${key}` : key;
|
15
|
+
//
|
16
|
+
// const property = getPropertyInPath(properties, key) as Property;
|
17
|
+
// // reverse lookup
|
18
|
+
// const mappedKey = Object.entries(headersMapping).find(([_, value]) => value === currentKey)?.[0];
|
19
|
+
// const inferredProperty = mappedKey ? getPropertyInPath(originProperties, mappedKey) as Property : null;
|
20
|
+
//
|
21
|
+
// if (property) {
|
22
|
+
// if (property.dataType === "map" && property.properties) {
|
23
|
+
// const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
|
24
|
+
// Object.keys(nestedMapping).forEach((nestedKey) => {
|
25
|
+
// dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
|
26
|
+
// });
|
27
|
+
// return;
|
28
|
+
// }
|
29
|
+
//
|
30
|
+
// if (inferredProperty) {
|
31
|
+
//
|
32
|
+
// const from = inferredProperty.dataType;
|
33
|
+
// const to = property.dataType;
|
34
|
+
// let fromSubtype: DataType | undefined;
|
35
|
+
// let toSubtype: DataType | undefined;
|
36
|
+
//
|
37
|
+
// if (property.dataType === "array" && property.of) {
|
38
|
+
// toSubtype = (property.of as Property).dataType;
|
39
|
+
// }
|
40
|
+
//
|
41
|
+
// if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
|
42
|
+
// fromSubtype = (inferredProperty.of as Property).dataType;
|
43
|
+
// }
|
44
|
+
//
|
45
|
+
// if (from !== to || fromSubtype !== toSubtype) {
|
46
|
+
// dataMapping[currentKey] = {
|
47
|
+
// from,
|
48
|
+
// to,
|
49
|
+
// fromSubtype,
|
50
|
+
// toSubtype
|
51
|
+
// };
|
52
|
+
// }
|
53
|
+
// }
|
54
|
+
//
|
55
|
+
// }
|
56
|
+
//
|
57
|
+
// });
|
58
|
+
//
|
59
|
+
// return dataMapping;
|
60
|
+
// }
|
61
|
+
//
|
62
|
+
// return updateMapping(newProperties);
|
63
|
+
// }
|
package/src/utils/index.ts
CHANGED