@firecms/data_import_export 3.0.0-canary.8 → 3.0.0-canary.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/LICENSE +113 -21
  2. package/README.md +1 -1
  3. package/dist/components/DataNewPropertiesMapping.d.ts +3 -5
  4. package/dist/components/ImportFileUpload.d.ts +1 -1
  5. package/dist/export_import/BasicExportAction.d.ts +7 -0
  6. package/dist/export_import/ExportCollectionAction.d.ts +1 -1
  7. package/dist/export_import/export.d.ts +15 -4
  8. package/dist/export_import/index.d.ts +4 -0
  9. package/dist/index.d.ts +1 -0
  10. package/dist/index.es.js +940 -570
  11. package/dist/index.es.js.map +1 -1
  12. package/dist/index.umd.js +2 -2
  13. package/dist/index.umd.js.map +1 -1
  14. package/dist/types/column_mapping.d.ts +5 -7
  15. package/dist/useImportExportPlugin.d.ts +1 -1
  16. package/dist/utils/data.d.ts +3 -10
  17. package/dist/utils/file_headers.d.ts +1 -0
  18. package/dist/utils/file_to_json.d.ts +6 -1
  19. package/dist/utils/get_properties_mapping.d.ts +0 -3
  20. package/dist/utils/index.d.ts +0 -1
  21. package/package.json +20 -34
  22. package/src/components/DataNewPropertiesMapping.tsx +153 -40
  23. package/src/components/ImportFileUpload.tsx +12 -4
  24. package/src/components/ImportNewPropertyFieldPreview.tsx +7 -2
  25. package/src/export_import/BasicExportAction.tsx +147 -0
  26. package/src/export_import/ExportCollectionAction.tsx +45 -9
  27. package/src/export_import/ImportCollectionAction.tsx +22 -22
  28. package/src/export_import/export.ts +63 -29
  29. package/src/export_import/index.ts +4 -0
  30. package/src/hooks/useImportConfig.tsx +6 -0
  31. package/src/index.ts +1 -0
  32. package/src/types/column_mapping.ts +6 -6
  33. package/src/useImportExportPlugin.tsx +2 -2
  34. package/src/utils/data.ts +36 -126
  35. package/src/utils/file_headers.ts +90 -0
  36. package/src/utils/file_to_json.ts +33 -15
  37. package/src/utils/get_properties_mapping.ts +63 -59
  38. package/src/utils/index.ts +0 -1
package/src/utils/data.ts CHANGED
@@ -1,41 +1,44 @@
1
- import { DataType, Entity, EntityReference, getPropertyInPath, Properties } from "@firecms/core";
1
+ import {
2
+ Entity,
3
+ EntityReference,
4
+ getPropertyInPath,
5
+ isPropertyBuilder,
6
+ mergeDeep,
7
+ Properties,
8
+ Property,
9
+ PropertyOrBuilder,
10
+ ResolvedProperty,
11
+ resolveProperty
12
+ } from "@firecms/core";
2
13
  import { unflattenObject } from "./file_to_json";
3
-
4
- type DataTypeMapping = {
5
- from: DataType;
6
- fromSubtype?: DataType;
7
- to: DataType;
8
- toSubtype?: DataType;
9
- }
14
+ import { getIn } from "@firecms/formex";
15
+ import { inferTypeFromValue } from "@firecms/schema_inference";
10
16
 
11
17
  export function convertDataToEntity(data: Record<any, any>,
12
18
  idColumn: string | undefined,
13
19
  headersMapping: Record<string, string | null>,
14
20
  properties: Properties,
15
- propertiesMapping: Record<string, DataTypeMapping>,
16
- path: string): Entity<any> {
21
+ path: string,
22
+ defaultValues: Record<string, any>): Entity<any> {
17
23
  const flatObject = flattenEntry(data);
18
24
  if (idColumn)
19
25
  delete flatObject[idColumn];
20
26
  const mappedKeysObject = Object.entries(flatObject)
21
27
  .map(([key, value]) => {
22
- const mappedKey = headersMapping[key] ?? key;
28
+ const mappedKey = getIn(headersMapping, key) ?? key;
23
29
 
24
30
  const mappedProperty = getPropertyInPath(properties, mappedKey);
25
- if (!mappedProperty)
31
+ if (!mappedProperty) {
26
32
  return {};
27
-
28
- const propertyMapping = propertiesMapping[mappedKey];
29
- let valueResult = value;
30
- if (propertyMapping) {
31
- valueResult = processValueMapping(value, propertyMapping);
32
33
  }
34
+ const processedValue = processValueMapping(value, mappedProperty);
33
35
  return ({
34
- [mappedKey]: valueResult
36
+ [mappedKey]: processedValue
35
37
  });
36
38
  })
37
39
  .reduce((acc, curr) => ({ ...acc, ...curr }), {});
38
- const values = unflattenObject(mappedKeysObject);
40
+
41
+ const values = mergeDeep(defaultValues ?? {}, unflattenObject(mappedKeysObject));
39
42
  let id = idColumn ? data[idColumn] : undefined;
40
43
  if (typeof id === "string") {
41
44
  id = id.trim();
@@ -60,7 +63,7 @@ export function flattenEntry(obj: any, parent = ""): any {
60
63
  return Object.keys(obj).reduce((acc, key) => {
61
64
  const prefixedKey = parent ? `${parent}.${key}` : key;
62
65
 
63
- if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) {
66
+ if (typeof obj[key] === "object" && !(obj[key] instanceof Date) && obj[key] !== null && !Array.isArray(obj[key])) {
64
67
  Object.assign(acc, flattenEntry(obj[key], prefixedKey));
65
68
  } else {
66
69
  // @ts-ignore
@@ -71,116 +74,23 @@ export function flattenEntry(obj: any, parent = ""): any {
71
74
  }, {});
72
75
  }
73
76
 
74
- // export function convertDataEntryValue({
75
- // key,
76
- // fullKey,
77
- // value,
78
- // idColumn,
79
- // headersMapping,
80
- // properties,
81
- // propertiesMapping
82
- // }: {
83
- // key?: string,
84
- // fullKey: string,
85
- // value: any,
86
- // idColumn?: string,
87
- // headersMapping: Record<string, string | null>,
88
- // properties: Properties,
89
- // propertiesMapping: Record<string, DataTypeMapping>
90
- // }): any {
91
- //
92
- // if (value === undefined) return value;
93
- // if (value === null) return value;
94
- //
95
- // if (Array.isArray(value)) {
96
- // const mappedKey = headersMapping[fullKey] || fullKey;
97
- // const valueMapping = propertiesMapping[mappedKey];
98
- // return processValueMapping(value, valueMapping);
99
- // // return value.map(v => convertDataEntryValue({ value: v, fullKey, idColumn, headersMapping, propertiesMapping }));
100
- // } else if (typeof value === "object") {
101
- // return convertDataObjectValue({
102
- // key,
103
- // fullKey,
104
- // value,
105
- // idColumn,
106
- // headersMapping,
107
- // properties,
108
- // propertiesMapping
109
- // });
110
- // } else {
111
- // const mappedKey = headersMapping[fullKey] || fullKey;
112
- // const valueMapping = propertiesMapping[mappedKey];
113
- // return processValueMapping(value, valueMapping);
114
- // }
115
- // }
77
+ export function processValueMapping(value: any, property?: PropertyOrBuilder): any {
78
+ if (value === null) return null;
116
79
 
117
- // export function convertDataObjectValue({
118
- // key,
119
- // fullKey,
120
- // value,
121
- // idColumn,
122
- // headersMapping,
123
- // properties,
124
- // propertiesMapping
125
- // }: {
126
- // key?: string,
127
- // fullKey?: string,
128
- // value: object,
129
- // idColumn?: string,
130
- // headersMapping: Record<string, string | null>,
131
- // properties: Properties,
132
- // propertiesMapping: Record<string, DataTypeMapping>
133
- // }): object {
134
- //
135
- // if (value instanceof Date) {
136
- // const mappedKey = fullKey ? headersMapping[fullKey] || fullKey : undefined;
137
- // const valueMapping = mappedKey ? propertiesMapping[mappedKey] : undefined;
138
- // return processValueMapping(value, valueMapping);
139
- // }
140
- //
141
- // return Object.entries(value)
142
- // .map(([childKey, childValue]) => {
143
- // const childFullKey = fullKey ? `${fullKey}.${childKey}` : childKey;
144
- // const mappedKey = headersMapping[childFullKey] ?? childFullKey;
145
- // const mappedKeyLastPart = mappedKey.split(".").slice(-1)[0];
146
- // const property = getPropertyInPath(properties, mappedKey);
147
- // // if (!property) {
148
- // // console.log("ppp", { properties, mappedKey, })
149
- // // return {};
150
- // // }
151
- // return ({
152
- // [mappedKeyLastPart]: convertDataEntryValue({
153
- // key: childKey,
154
- // fullKey: childFullKey,
155
- // value: childValue,
156
- // idColumn,
157
- // headersMapping,
158
- // properties,
159
- // propertiesMapping
160
- // })
161
- // });
162
- // })
163
- // .reduce((acc, curr) => ({ ...acc, ...curr }), {});
164
- // }
80
+ if (property === undefined) return value;
81
+ const usedProperty: ResolvedProperty | null = resolveProperty({
82
+ propertyOrBuilder: property,
83
+ })
84
+ if (usedProperty === null) return value;
85
+ const from = inferTypeFromValue(value);
86
+ const to = usedProperty.dataType;
165
87
 
166
- export function processValueMapping(value: any, valueMapping?: DataTypeMapping): any {
167
- if (valueMapping === undefined) return value;
168
- const {
169
- from,
170
- to
171
- } = valueMapping;
172
- if (from === "array" && to === "array" && valueMapping.fromSubtype && valueMapping.toSubtype && Array.isArray(value)) {
173
- return value.map(v => processValueMapping(v, {
174
- from: valueMapping.fromSubtype!,
175
- to: valueMapping.toSubtype!
176
- }));
88
+ if (from === "array" && to === "array" && Array.isArray(value) && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
89
+ return value.map(v => processValueMapping(v, usedProperty.of as Property));
177
90
  } else if (from === "string" && to === "number" && typeof value === "string") {
178
91
  return Number(value);
179
- } else if (from === "string" && to === "array" && valueMapping.toSubtype && typeof value === "string") {
180
- return value.split(",").map((v: string) => processValueMapping(v, {
181
- from: "string",
182
- to: valueMapping.toSubtype!
183
- }));
92
+ } else if (from === "string" && to === "array" && typeof value === "string" && usedProperty.of && !isPropertyBuilder(usedProperty.of as PropertyOrBuilder)) {
93
+ return value.split(",").map((v: string) => processValueMapping(v, usedProperty.of));
184
94
  } else if (from === "string" && to === "boolean") {
185
95
  return value === "true";
186
96
  } else if (from === "number" && to === "boolean") {
@@ -0,0 +1,90 @@
1
+ import * as XLSX from "xlsx";
2
+ export function getXLSXHeaders(sheet: any) {
3
+ let header = 0; let offset = 1;
4
+ const hdr = [];
5
+ const o:any = {};
6
+ if (sheet == null || sheet["!ref"] == null) return [];
7
+ const range = o.range !== undefined ? o.range : sheet["!ref"];
8
+ let r;
9
+ if (o.header === 1) header = 1;
10
+ else if (o.header === "A") header = 2;
11
+ else if (Array.isArray(o.header)) header = 3;
12
+ switch (typeof range) {
13
+ case "string":
14
+ r = safeDecodeRange(range);
15
+ break;
16
+ case "number":
17
+ r = safeDecodeRange(sheet["!ref"]);
18
+ r.s.r = range;
19
+ break;
20
+ default:
21
+ r = range;
22
+ }
23
+ if (header > 0) offset = 0;
24
+ const rr = XLSX.utils.encode_row(r.s.r);
25
+ const cols = new Array(r.e.c - r.s.c + 1);
26
+ for (let C = r.s.c; C <= r.e.c; ++C) {
27
+ cols[C] = XLSX.utils.encode_col(C);
28
+ const val = sheet[cols[C] + rr];
29
+ switch (header) {
30
+ case 1:
31
+ hdr.push(C);
32
+ break;
33
+ case 2:
34
+ hdr.push(cols[C]);
35
+ break;
36
+ case 3:
37
+ hdr.push(o.header[C - r.s.c]);
38
+ break;
39
+ default:
40
+ if (val === undefined) continue;
41
+ hdr.push(XLSX.utils.format_cell(val));
42
+ }
43
+ }
44
+ return hdr;
45
+ }
46
+
47
+ function safeDecodeRange(range:any) {
48
+ const o = {
49
+ s: {
50
+ c: 0,
51
+ r: 0
52
+ },
53
+ e: {
54
+ c: 0,
55
+ r: 0
56
+ }
57
+ };
58
+ let idx = 0; let i = 0; let cc = 0;
59
+ const len = range.length;
60
+ for (idx = 0; i < len; ++i) {
61
+ if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
62
+ idx = 26 * idx + cc;
63
+ }
64
+ o.s.c = --idx;
65
+
66
+ for (idx = 0; i < len; ++i) {
67
+ if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
68
+ idx = 10 * idx + cc;
69
+ }
70
+ o.s.r = --idx;
71
+
72
+ if (i === len || range.charCodeAt(++i) === 58) {
73
+ o.e.c = o.s.c;
74
+ o.e.r = o.s.r;
75
+ return o;
76
+ }
77
+
78
+ for (idx = 0; i !== len; ++i) {
79
+ if ((cc = range.charCodeAt(i) - 64) < 1 || cc > 26) break;
80
+ idx = 26 * idx + cc;
81
+ }
82
+ o.e.c = --idx;
83
+
84
+ for (idx = 0; i !== len; ++i) {
85
+ if ((cc = range.charCodeAt(i) - 48) < 0 || cc > 9) break;
86
+ idx = 10 * idx + cc;
87
+ }
88
+ o.e.r = --idx;
89
+ return o;
90
+ }
@@ -1,38 +1,56 @@
1
1
  import * as XLSX from "xlsx";
2
+ import { getXLSXHeaders } from "./file_headers";
2
3
 
3
- export function convertFileToJson(file: File): Promise<object[]> {
4
+ type ConversionResult = {
5
+ data: object[];
6
+ propertiesOrder: string[]
7
+ }
8
+
9
+ export function convertFileToJson(file: File): Promise<ConversionResult> {
4
10
  return new Promise((resolve, reject) => {
5
- // check if file is a JSON file
6
11
  if (file.type === "application/json") {
7
12
  console.debug("Converting JSON file to JSON", file.name);
8
13
  const reader = new FileReader();
9
14
  reader.onload = function (e) {
10
- const data = e.target?.result as string;
11
- const jsonData = JSON.parse(data);
12
- if (!Array.isArray(jsonData)) {
13
- reject(new Error("JSON file should contain an array of objects"));
15
+ try {
16
+ const data = e.target?.result as string;
17
+ const jsonData = JSON.parse(data);
18
+ if (!Array.isArray(jsonData)) {
19
+ reject(new Error("JSON file should contain an array of objects"));
20
+ } else {
21
+ // Assuming all objects in the array have the same structure/order
22
+ const propertiesOrder = jsonData.length > 0 ? Object.keys(jsonData[0]) : [];
23
+ resolve({
24
+ data: jsonData,
25
+ propertiesOrder
26
+ });
27
+ }
28
+ } catch (e) {
29
+ console.error("Error parsing JSON file", e);
30
+ reject(e);
14
31
  }
15
- resolve(jsonData);
16
32
  };
17
33
  reader.readAsText(file);
18
34
  } else {
19
35
  console.debug("Converting Excel file to JSON", file.name);
20
36
  const reader = new FileReader();
21
37
  reader.onload = function (e) {
22
-
23
38
  const data = new Uint8Array(e.target?.result as ArrayBuffer);
24
- const workbook = XLSX.read(data,
25
- {
26
- type: "array",
27
- codepage: 65001,
28
- cellDates: true,
29
- });
39
+ const workbook = XLSX.read(data, {
40
+ type: "array",
41
+ codepage: 65001,
42
+ cellDates: true,
43
+ });
30
44
  const worksheetName = workbook.SheetNames[0];
31
45
  const worksheet = workbook.Sheets[worksheetName];
32
46
  const parsedData: Array<any> = XLSX.utils.sheet_to_json(worksheet);
47
+ const headers = getXLSXHeaders(worksheet);
33
48
  const cleanedData = parsedData.map(mapJsonParse);
34
49
  const jsonData = cleanedData.map(unflattenObject);
35
- resolve(jsonData);
50
+ resolve({
51
+ data: jsonData,
52
+ propertiesOrder: headers
53
+ });
36
54
  };
37
55
  reader.readAsArrayBuffer(file);
38
56
  }
@@ -1,59 +1,63 @@
1
- import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
2
- import { DataTypeMapping } from "../types";
3
-
4
- export function getPropertiesMapping(originProperties: Properties, newProperties: Properties): Record<string, DataTypeMapping> {
5
-
6
- function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
7
-
8
- const dataMapping: Record<string, DataTypeMapping> = {};
9
-
10
- Object.keys(properties).forEach((key) => {
11
-
12
- const currentKey = namespace ? `${namespace}.${key}` : key;
13
-
14
- const property = getPropertyInPath(properties, key) as Property;
15
- const inferredProperty = getPropertyInPath(originProperties, currentKey) as Property;
16
-
17
- if (property) {
18
- if (property.dataType === "map" && property.properties) {
19
- const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
20
- Object.keys(nestedMapping).forEach((nestedKey) => {
21
- dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
22
- });
23
- return;
24
- }
25
-
26
- if (inferredProperty) {
27
-
28
- const from = inferredProperty.dataType;
29
- const to = property.dataType;
30
- let fromSubtype: DataType | undefined;
31
- let toSubtype: DataType | undefined;
32
-
33
- if (property.dataType === "array" && property.of) {
34
- toSubtype = (property.of as Property).dataType;
35
- }
36
-
37
- if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
38
- fromSubtype = (inferredProperty.of as Property).dataType;
39
- }
40
-
41
- if (from !== to || fromSubtype !== toSubtype) {
42
- dataMapping[key] = {
43
- from,
44
- to,
45
- fromSubtype,
46
- toSubtype
47
- };
48
- }
49
- }
50
-
51
- }
52
-
53
- });
54
-
55
- return dataMapping;
56
- }
57
-
58
- return updateMapping(newProperties);
59
- }
1
+ // import { DataType, getPropertyInPath, Properties, Property } from "@firecms/core";
2
+ // import { DataTypeMapping } from "../types";
3
+ //
4
+ // export function getPropertiesMapping(originProperties: Properties,
5
+ // newProperties: Properties,
6
+ // headersMapping: Record<string, string | null>): Record<string, DataTypeMapping> {
7
+ //
8
+ // function updateMapping(properties: Record<string, Property>, namespace?: string): Record<string, DataTypeMapping> {
9
+ //
10
+ // const dataMapping: Record<string, DataTypeMapping> = {};
11
+ //
12
+ // Object.keys(properties).forEach((key) => {
13
+ //
14
+ // const currentKey = namespace ? `${namespace}.${key}` : key;
15
+ //
16
+ // const property = getPropertyInPath(properties, key) as Property;
17
+ // // reverse lookup
18
+ // const mappedKey = Object.entries(headersMapping).find(([_, value]) => value === currentKey)?.[0];
19
+ // const inferredProperty = mappedKey ? getPropertyInPath(originProperties, mappedKey) as Property : null;
20
+ //
21
+ // if (property) {
22
+ // if (property.dataType === "map" && property.properties) {
23
+ // const nestedMapping = updateMapping(property.properties as Record<string, Property>, currentKey);
24
+ // Object.keys(nestedMapping).forEach((nestedKey) => {
25
+ // dataMapping[`${currentKey}.${nestedKey}`] = nestedMapping[nestedKey];
26
+ // });
27
+ // return;
28
+ // }
29
+ //
30
+ // if (inferredProperty) {
31
+ //
32
+ // const from = inferredProperty.dataType;
33
+ // const to = property.dataType;
34
+ // let fromSubtype: DataType | undefined;
35
+ // let toSubtype: DataType | undefined;
36
+ //
37
+ // if (property.dataType === "array" && property.of) {
38
+ // toSubtype = (property.of as Property).dataType;
39
+ // }
40
+ //
41
+ // if (inferredProperty?.dataType === "array" && inferredProperty?.of) {
42
+ // fromSubtype = (inferredProperty.of as Property).dataType;
43
+ // }
44
+ //
45
+ // if (from !== to || fromSubtype !== toSubtype) {
46
+ // dataMapping[currentKey] = {
47
+ // from,
48
+ // to,
49
+ // fromSubtype,
50
+ // toSubtype
51
+ // };
52
+ // }
53
+ // }
54
+ //
55
+ // }
56
+ //
57
+ // });
58
+ //
59
+ // return dataMapping;
60
+ // }
61
+ //
62
+ // return updateMapping(newProperties);
63
+ // }
@@ -1,4 +1,3 @@
1
1
  export * from "./file_to_json";
2
2
  export * from "./data";
3
3
  export * from "./get_import_inference_type";
4
- export * from "./get_properties_mapping";