@firecms/data_import_export 3.0.0-canary.5 → 3.0.0-canary.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/LICENSE +113 -21
  2. package/README.md +1 -1
  3. package/dist/components/DataNewPropertiesMapping.d.ts +3 -5
  4. package/dist/components/ImportFileUpload.d.ts +1 -1
  5. package/dist/export_import/BasicExportAction.d.ts +7 -0
  6. package/dist/export_import/ExportCollectionAction.d.ts +2 -1
  7. package/dist/export_import/ImportCollectionAction.d.ts +3 -1
  8. package/dist/export_import/export.d.ts +4 -4
  9. package/dist/export_import/index.d.ts +4 -0
  10. package/dist/index.d.ts +1 -0
  11. package/dist/index.es.js +901 -546
  12. package/dist/index.es.js.map +1 -1
  13. package/dist/index.umd.js +2 -2
  14. package/dist/index.umd.js.map +1 -1
  15. package/dist/types/column_mapping.d.ts +5 -7
  16. package/dist/useImportExportPlugin.d.ts +2 -1
  17. package/dist/utils/data.d.ts +3 -10
  18. package/dist/utils/file_headers.d.ts +1 -0
  19. package/dist/utils/file_to_json.d.ts +6 -1
  20. package/dist/utils/get_properties_mapping.d.ts +0 -3
  21. package/dist/utils/index.d.ts +0 -1
  22. package/package.json +20 -18
  23. package/src/components/DataNewPropertiesMapping.tsx +155 -41
  24. package/src/components/ImportFileUpload.tsx +12 -4
  25. package/src/components/ImportNewPropertyFieldPreview.tsx +7 -2
  26. package/src/components/ImportSaveInProgress.tsx +24 -2
  27. package/src/export_import/BasicExportAction.tsx +137 -0
  28. package/src/export_import/ExportCollectionAction.tsx +15 -7
  29. package/src/export_import/ImportCollectionAction.tsx +52 -31
  30. package/src/export_import/export.ts +50 -30
  31. package/src/export_import/index.ts +4 -0
  32. package/src/hooks/useImportConfig.tsx +6 -0
  33. package/src/index.ts +1 -0
  34. package/src/types/column_mapping.ts +6 -6
  35. package/src/useImportExportPlugin.tsx +4 -3
  36. package/src/utils/data.ts +50 -127
  37. package/src/utils/file_headers.ts +90 -0
  38. package/src/utils/file_to_json.ts +33 -15
  39. package/src/utils/get_properties_mapping.ts +63 -59
  40. package/src/utils/index.ts +0 -1
package/src/utils/data.ts CHANGED
@@ -1,43 +1,59 @@
1
- import { DataType, Entity, EntityReference, getPropertyInPath, Properties } from "@firecms/core";
1
+ import {
2
+ Entity,
3
+ EntityReference,
4
+ getPropertyInPath,
5
+ isPropertyBuilder,
6
+ mergeDeep,
7
+ Properties,
8
+ Property,
9
+ PropertyOrBuilder,
10
+ ResolvedProperty,
11
+ resolveProperty
12
+ } from "@firecms/core";
2
13
  import { unflattenObject } from "./file_to_json";
3
-
4
- type DataTypeMapping = {
5
- from: DataType;
6
- fromSubtype?: DataType;
7
- to: DataType;
8
- toSubtype?: DataType;
9
- }
14
+ import { getIn } from "@firecms/formex";
15
+ import { inferTypeFromValue } from "@firecms/schema_inference";
10
16
 
11
17
  export function convertDataToEntity(data: Record<any, any>,
12
18
  idColumn: string | undefined,
13
19
  headersMapping: Record<string, string | null>,
14
20
  properties: Properties,
15
- propertiesMapping: Record<string, DataTypeMapping>,
16
- path: string): Entity<any> {
21
+ path: string,
22
+ defaultValues: Record<string, any>): Entity<any> {
17
23
  const flatObject = flattenEntry(data);
18
24
  if (idColumn)
19
25
  delete flatObject[idColumn];
20
26
  const mappedKeysObject = Object.entries(flatObject)
21
27
  .map(([key, value]) => {
22
- const mappedKey = headersMapping[key] ?? key;
28
+ const mappedKey = getIn(headersMapping, key) ?? key;
23
29
 
24
30
  const mappedProperty = getPropertyInPath(properties, mappedKey);
25
- if (!mappedProperty)
31
+ if (!mappedProperty) {
26
32
  return {};
27
-
28
- const propertyMapping = propertiesMapping[mappedKey];
29
- let valueResult = value;
30
- if (propertyMapping) {
31
- valueResult = processValueMapping(value, propertyMapping);
32
33
  }
34
+ const processedValue = processValueMapping(value, mappedProperty);
33
35
  return ({
34
- [mappedKey]: valueResult
36
+ [mappedKey]: processedValue
35
37
  });
36
38
  })
37
39
  .reduce((acc, curr) => ({ ...acc, ...curr }), {});
38
- const values = unflattenObject(mappedKeysObject);
40
+
41
+ const values = mergeDeep(defaultValues ?? {}, unflattenObject(mappedKeysObject));
42
+ let id = idColumn ? data[idColumn] : undefined;
43
+ if (typeof id === "string") {
44
+ id = id.trim();
45
+ } else if (typeof id === "number") {
46
+ id = id.toString();
47
+ } else if (typeof id === "boolean") {
48
+ id = id.toString();
49
+ } else if (id instanceof Date) {
50
+ id = id.toISOString();
51
+ } else if (id && "toString" in id) {
52
+ id = id.toString();
53
+ }
54
+
39
55
  return {
40
- id: idColumn ? data[idColumn] : undefined,
56
+ id,
41
57
  values,
42
58
  path
43
59
  };
@@ -47,7 +63,7 @@ export function flattenEntry(obj: any, parent = ""): any {
47
63
  return Object.keys(obj).reduce((acc, key) => {
48
64
  const prefixedKey = parent ? `${parent}.${key}` : key;
49
65
 
50
- if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) {
66
+ if (typeof obj[key] === "object" && !(obj[key] instanceof Date) && obj[key] !== null && !Array.isArray(obj[key])) {
51
67
  Object.assign(acc, flattenEntry(obj[key], prefixedKey));
52
68
  } else {
53
69
  // @ts-ignore
@@ -58,116 +74,23 @@ export function flattenEntry(obj: any, parent = ""): any {
58
74
  }, {});
59
75
  }
60
76
 
61
- // export function convertDataEntryValue({
62
- // key,
63
- // fullKey,
64
- // value,
65
- // idColumn,
66
- // headersMapping,
67
- // properties,
68
- // propertiesMapping
69
- // }: {
70
- // key?: string,
71
- // fullKey: string,
72
- // value: any,
73
- // idColumn?: string,
74
- // headersMapping: Record<string, string | null>,
75
- // properties: Properties,
76
- // propertiesMapping: Record<string, DataTypeMapping>
77
- // }): any {
78
- //
79
- // if (value === undefined) return value;
80
- // if (value === null) return value;
81
- //
82
- // if (Array.isArray(value)) {
83
- // const mappedKey = headersMapping[fullKey] || fullKey;
84
- // const valueMapping = propertiesMapping[mappedKey];
85
- // return processValueMapping(value, valueMapping);
86
- // // return value.map(v => convertDataEntryValue({ value: v, fullKey, idColumn, headersMapping, propertiesMapping }));
87
- // } else if (typeof value === "object") {
88
- // return convertDataObjectValue({
89
- // key,
90
- // fullKey,
91
- // value,
92
- // idColumn,
93
- // headersMapping,
94
- // properties,
95
- // propertiesMapping
96
- // });
97
- // } else {
98
- // const mappedKey = headersMapping[fullKey] || fullKey;
99
- // const valueMapping = propertiesMapping[mappedKey];
100
- // return processValueMapping(value, valueMapping);
101
- // }
102
- // }
77
+ export function processValueMapping(value: any, property?: PropertyOrBuilder): any {
78
+ if (value === null) return null;
103
79
 
104
- // export function convertDataObjectValue({
105
- // key,
106
- // fullKey,
107
- // value,
108
- // idColumn,
109
- // headersMapping,
110
- // properties,
111
- // propertiesMapping
112
- // }: {
113
- // key?: string,
114
- // fullKey?: string,
115
- // value: object,
116
- // idColumn?: string,
117
- // headersMapping: Record<string, string | null>,
118
- // properties: Properties,
119
- // propertiesMapping: Record<string, DataTypeMapping>
120
- // }): object {
121
- //
122
- // if (value instanceof Date) {
123
- // const mappedKey = fullKey ? headersMapping[fullKey] || fullKey : undefined;
124
- // const valueMapping = mappedKey ? propertiesMapping[mappedKey] : undefined;
125
- // return processValueMapping(value, valueMapping);
126
- // }
127
- //
128
- // return Object.entries(value)
129
- // .map(([childKey, childValue]) => {
130
- // const childFullKey = fullKey ? `${fullKey}.${childKey}` : childKey;
131
- // const mappedKey = headersMapping[childFullKey] ?? childFullKey;
132
- // const mappedKeyLastPart = mappedKey.split(".").slice(-1)[0];
133
- // const property = getPropertyInPath(properties, mappedKey);
134
- // // if (!property) {
135
- // // console.log("ppp", { properties, mappedKey, })
136
- // // return {};
137
- // // }
138
- // return ({
139
- // [mappedKeyLastPart]: convertDataEntryValue({
140
- // key: childKey,
141
- // fullKey: childFullKey,
142
- // value: childValue,
143
- // idColumn,
144
- // headersMapping,
145
- // properties,
146
- // propertiesMapping
147
- // })
148
- // });
149
- // })
150
- // .reduce((acc, curr) => ({ ...acc, ...curr }), {});
151
- // }
80
+ if (property === undefined) return value;
81
+ const usedProperty: ResolvedProperty | null = resolveProperty({
82
+ propertyOrBuilder: property,
83
+ })
84
+ if (usedProperty === null) return value;
85
+ const from = inferTypeFromValue(value);
86
+ const to = usedProperty.dataType;
152
87
 
153
- export function processValueMapping(value: any, valueMapping?: DataTypeMapping): any {
154
- if (valueMapping === undefined) return value;
155
- const {
156
- from,
157
- to
158
- } = valueMapping;
159
- if (from === "array" && to === "array" && valueMapping.fromSubtype && valueMapping.toSubtype && Array.isArray(value)) {
160
- return value.map(v => processValueMapping(v, {
161
- from: valueMapping.fromSubtype!,
162
- to: valueMapping.toSubtype!
163
- }));
88
+ if (from === "array" && to === "array" && Array.isArray(value) && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
89
+ return value.map(v => processValueMapping(v, usedProperty.of as Property));
164
90
  } else if (from === "string" && to === "number" && typeof value === "string") {
165
91
  return Number(value);
166
- } else if (from === "string" && to === "array" && valueMapping.toSubtype && typeof value === "string") {
167
- return value.split(",").map((v: string) => processValueMapping(v, {
168
- from: "string",
169
- to: valueMapping.toSubtype!
170
- }));
92
+ } else if (from === "string" && to === "array" && typeof value === "string" && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
93
+ return value.split(",").map((v: string) => processValueMapping(v, usedProperty.of));
171
94
  } else if (from === "string" && to === "boolean") {
172
95
  return value === "true";
173
96
  } else if (from === "number" && to === "boolean") {
@@ -0,0 +1,90 @@
1
+ import * as XLSX from "xlsx";
2
+ export function getXLSXHeaders(sheet: any) {
3
+ let header = 0; let offset = 1;
4
+ const hdr = [];
5
+ const o:any = {};
6
+ if (sheet == null || sheet["!ref"] == null) return [];
7
+ const range = o.range !== undefined ? o.range : sheet["!ref"];
8
+ let r;
9
+ if (o.header === 1) header = 1;
10
+ else if (o.header === "A") header = 2;
11
+ else if (Array.isArray(o.header)) header = 3;
12
+ switch (typeof range) {
13
+ case "string":
14
+ r = safeDecodeRange(range);
15
+ break;
16
+ case "number":
17
+ r = safeDecodeRange(sheet["!ref"]);
18
+ r.s.r = range;
19
+ break;
20
+ default:
21
+ r = range;
22
+ }
23
+ if (header > 0) offset = 0;
24
+ const rr = XLSX.utils.encode_row(r.s.r);
25
+ const cols = new Array(r.e.c - r.s.c + 1);
26
+ for (let C = r.s.c; C <= r.e.c; ++C) {
27
+ cols[C] = XLSX.utils.encode_col(C);
28
+ const val = sheet[cols[C] + rr];
29
+ switch (header) {
30
+ case 1:
31
+ hdr.push(C);
32
+ break;
33
+ case 2:
34
+ hdr.push(cols[C]);
35
+ break;
36
+ case 3:
37
+ hdr.push(o.header[C - r.s.c]);
38
+ break;
39
+ default:
40
+ if (val === undefined) continue;
41
+ hdr.push(XLSX.utils.format_cell(val));
42
+ }
43
+ }
44
+ return hdr;
45
+ }
46
+
47
+ function safeDecodeRange(range:any) {
48
+ const o = {
49
+ s: {
50
+ c: 0,
51
+ r: 0
52
+ },
53
+ e: {
54
+ c: 0,
55
+ r: 0
56
+ }
57
+ };
58
+ let idx = 0; let i = 0; let cc = 0;
59
+ const len = range.length;
60
+ for (idx = 0; i < len; ++i) {
61
+ if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
62
+ idx = 26 * idx + cc;
63
+ }
64
+ o.s.c = --idx;
65
+
66
+ for (idx = 0; i < len; ++i) {
67
+ if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
68
+ idx = 10 * idx + cc;
69
+ }
70
+ o.s.r = --idx;
71
+
72
+ if (i === len || range.charCodeAt(++i) === 58) {
73
+ o.e.c = o.s.c;
74
+ o.e.r = o.s.r;
75
+ return o;
76
+ }
77
+
78
+ for (idx = 0; i !== len; ++i) {
79
+ if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
80
+ idx = 26 * idx + cc;
81
+ }
82
+ o.e.c = --idx;
83
+
84
+ for (idx = 0; i !== len; ++i) {
85
+ if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
86
+ idx = 10 * idx + cc;
87
+ }
88
+ o.e.r = --idx;
89
+ return o;
90
+ }
@@ -1,38 +1,56 @@
1
1
  import * as XLSX from "xlsx";
2
+ import { getXLSXHeaders } from "./file_headers";
2
3
 
3
- export function convertFileToJson(file: File): Promise<object[]> {
4
+ type ConversionResult = {
5
+ data: object[];
6
+ propertiesOrder: string[]
7
+ }
8
+
9
+ export function convertFileToJson(file: File): Promise<ConversionResult> {
4
10
  return new Promise((resolve, reject) => {
5
- // check if file is a JSON file
6
11
  if (file.type === "application/json") {
7
12
  console.debug("Converting JSON file to JSON", file.name);
8
13
  const reader = new FileReader();
9
14
  reader.onload = function (e) {
10
- const data = e.target?.result as string;
11
- const jsonData = JSON.parse(data);
12
- if (!Array.isArray(jsonData)) {
13
- reject(new Error("JSON file should contain an array of objects"));
15
+ try {
16
+ const data = e.target?.result as string;
17
+ const jsonData = JSON.parse(data);
18
+ if (!Array.isArray(jsonData)) {
19
+ reject(new Error("JSON file should contain an array of objects"));
20
+ } else {
21
+ // Assuming all objects in the array have the same structure/order
22
+ const propertiesOrder = jsonData.length > 0 ? Object.keys(jsonData[0]) : [];
23
+ resolve({
24
+ data: jsonData,
25
+ propertiesOrder
26
+ });
27
+ }
28
+ } catch (e) {
29
+ console.error("Error parsing JSON file", e);
30
+ reject(e);
14
31
  }
15
- resolve(jsonData);
16
32
  };
17
33
  reader.readAsText(file);
18
34
  } else {
19
35
  console.debug("Converting Excel file to JSON", file.name);
20
36
  const reader = new FileReader();
21
37
  reader.onload = function (e) {
22
-
23
38
  const data = new Uint8Array(e.target?.result as ArrayBuffer);
24
- const workbook = XLSX.read(data,
25
- {
26
- type: "array",
27
- codepage: 65001,
28
- cellDates: true,
29
- });
39
+ const workbook = XLSX.read(data, {
40
+ type: "array",
41
+ codepage: 65001,
42
+ cellDates: true,
43
+ });
30
44
  const worksheetName = workbook.SheetNames[0];
31
45
  const worksheet = workbook.Sheets[worksheetName];
32
46
  const parsedData: Array<any> = XLSX.utils.sheet_to_json(worksheet);
47
+ const headers = getXLSXHeaders(worksheet);
33
48
  const cleanedData = parsedData.map(mapJsonParse);
34
49
  const jsonData = cleanedData.map(unflattenObject);
35
- resolve(jsonData);
50
+ resolve({
51
+ data: jsonData,
52
+ propertiesOrder: headers
53
+ });
36
54
  };
37
55
  reader.readAsArrayBuffer(file);
38
56
  }
@@ -1,59 +1,63 @@
1
- import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
2
- import { DataTypeMapping } from "../types";
3
-
4
- export function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping> {
5
-
6
- function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
7
-
8
- const dataMapping: Record<string, DataTypeMapping> = {};
9
-
10
- Object.keys(properties).forEach((key) => {
11
-
12
- const currentKey = namespace ? `${namespace}.${key}` : key;
13
-
14
- const property = getPropertyInPath(properties, key) as Property;
15
- const inferredProperty = getPropertyInPath(originProperties, currentKey) as Property;
16
-
17
- if (property) {
18
- if (property.dataType === "map" && property.properties) {
19
- const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
20
- Object.keys(nestedMapping).forEach((nestedKey) => {
21
- dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
22
- });
23
- return;
24
- }
25
-
26
- if (inferredProperty) {
27
-
28
- const from = inferredProperty.dataType;
29
- const to = property.dataType;
30
- let fromSubtype: DataType | undefined;
31
- let toSubtype: DataType | undefined;
32
-
33
- if (property.dataType === "array" && property.of) {
34
- toSubtype = (property.of as Property).dataType;
35
- }
36
-
37
- if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
38
- fromSubtype = (inferredProperty.of as Property).dataType;
39
- }
40
-
41
- if (from !== to || fromSubtype !== toSubtype) {
42
- dataMapping[key] = {
43
- from,
44
- to,
45
- fromSubtype,
46
- toSubtype
47
- };
48
- }
49
- }
50
-
51
- }
52
-
53
- });
54
-
55
- return dataMapping;
56
- }
57
-
58
- return updateMapping(newProperties);
59
- }
1
+ // import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
2
+ // import { DataTypeMapping } from "../types";
3
+ //
4
+ // export function getPropertiesMapping(originProperties: Properties,
5
+ // newProperties: Properties,
6
+ // headersMapping: Record<string, string | null>): Record<string, DataTypeMapping> {
7
+ //
8
+ // function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
9
+ //
10
+ // const dataMapping: Record<string, DataTypeMapping> = {};
11
+ //
12
+ // Object.keys(properties).forEach((key) => {
13
+ //
14
+ // const currentKey = namespace ? `${namespace}.${key}` : key;
15
+ //
16
+ // const property = getPropertyInPath(properties, key) as Property;
17
+ // // reverse lookup
18
+ // const mappedKey = Object.entries(headersMapping).find(([_, value]) => value === currentKey)?.[0];
19
+ // const inferredProperty = mappedKey ? getPropertyInPath(originProperties, mappedKey) as Property : null;
20
+ //
21
+ // if (property) {
22
+ // if (property.dataType === "map" && property.properties) {
23
+ // const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
24
+ // Object.keys(nestedMapping).forEach((nestedKey) => {
25
+ // dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
26
+ // });
27
+ // return;
28
+ // }
29
+ //
30
+ // if (inferredProperty) {
31
+ //
32
+ // const from = inferredProperty.dataType;
33
+ // const to = property.dataType;
34
+ // let fromSubtype: DataType | undefined;
35
+ // let toSubtype: DataType | undefined;
36
+ //
37
+ // if (property.dataType === "array" && property.of) {
38
+ // toSubtype = (property.of as Property).dataType;
39
+ // }
40
+ //
41
+ // if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
42
+ // fromSubtype = (inferredProperty.of as Property).dataType;
43
+ // }
44
+ //
45
+ // if (from !== to || fromSubtype !== toSubtype) {
46
+ // dataMapping[currentKey] = {
47
+ // from,
48
+ // to,
49
+ // fromSubtype,
50
+ // toSubtype
51
+ // };
52
+ // }
53
+ // }
54
+ //
55
+ // }
56
+ //
57
+ // });
58
+ //
59
+ // return dataMapping;
60
+ // }
61
+ //
62
+ // return updateMapping(newProperties);
63
+ // }
@@ -1,4 +1,3 @@
1
1
  export * from "./file_to_json";
2
2
  export * from "./data";
3
3
  export * from "./get_import_inference_type";
4
- export * from "./get_properties_mapping";