@rebasepro/admin 0.13.1-canary.gf57a27e → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{CollectionEditorDialog-BnfOcqM_.js → CollectionEditorDialog-CSheud_E.js} +19 -9
- package/dist/CollectionEditorDialog-CSheud_E.js.map +1 -0
- package/dist/{PropertyEditView-vLXglzIy.js → PropertyEditView-CWZlC1gq.js} +5 -5
- package/dist/PropertyEditView-CWZlC1gq.js.map +1 -0
- package/dist/{RouterCollectionsStudioView-DF3aiOuv.js → RouterCollectionsStudioView-CwTpdzWF.js} +4 -4
- package/dist/{RouterCollectionsStudioView-DF3aiOuv.js.map → RouterCollectionsStudioView-CwTpdzWF.js.map} +1 -1
- package/dist/collection_editor_ui.js +4 -4
- package/dist/components/ClearFilterSortButton.d.ts +7 -1
- package/dist/components/CollectionViewBinding/SortButton.d.ts +7 -1
- package/dist/components/RelationSelector.d.ts +13 -0
- package/dist/data_export/export/export.d.ts +7 -0
- package/dist/data_export/export/fetch_export_data.d.ts +48 -0
- package/dist/data_export/export/index.d.ts +1 -0
- package/dist/data_import/utils/csv.d.ts +30 -0
- package/dist/data_import/utils/index.d.ts +2 -0
- package/dist/data_import/utils/save_entities.d.ts +38 -0
- package/dist/data_import/utils/transforms.d.ts +9 -0
- package/dist/editor.js +3 -0
- package/dist/editor.js.map +1 -1
- package/dist/{export-C26xQN-g.js → export-L53WektO.js} +175 -39
- package/dist/export-L53WektO.js.map +1 -0
- package/dist/{history-DSS-Ebwh.js → history-D5SKygpJ.js} +2 -2
- package/dist/{history-DSS-Ebwh.js.map → history-D5SKygpJ.js.map} +1 -1
- package/dist/{import-Dhx96B6x.js → import-C_4edkoD.js} +5 -4
- package/dist/import-C_4edkoD.js.map +1 -0
- package/dist/index.js +93 -74
- package/dist/index.js.map +1 -1
- package/dist/preview/components/date_fns_locales.d.ts +9 -0
- package/dist/{util-Bt6VadeZ.js → util-C-D5yD2n.js} +567 -113
- package/dist/util-C-D5yD2n.js.map +1 -0
- package/package.json +10 -9
- package/src/collection_editor/ui/collection_editor/AICollectionGeneratorPopover.tsx +8 -3
- package/src/collection_editor/ui/collection_editor/CollectionDetailsForm.tsx +5 -2
- package/src/collection_editor/ui/collection_editor/CollectionJsonImportDialog.tsx +1 -0
- package/src/collection_editor/ui/collection_editor/CollectionRLSTab.tsx +9 -8
- package/src/collection_editor/ui/collection_editor/GeneralSettingsForm.tsx +5 -2
- package/src/collection_editor/ui/collection_editor/properties/MapPropertyField.tsx +1 -1
- package/src/collection_editor/ui/collection_editor/properties/advanced/AdvancedPropertyValidation.tsx +5 -2
- package/src/collection_editor/ui/collection_editor/properties/conditions/ConditionsEditor.tsx +1 -0
- package/src/collection_editor/useJsonCollectionsConfigController.tsx +92 -84
- package/src/collection_editor/useLocalCollectionsConfigController.tsx +10 -3
- package/src/components/ClearFilterSortButton.tsx +16 -7
- package/src/components/CollectionTableBinding/table_bindings.tsx +7 -1
- package/src/components/CollectionViewBinding/CollectionBoardViewBinding.tsx +13 -7
- package/src/components/CollectionViewBinding/CollectionViewBinding.tsx +1 -0
- package/src/components/CollectionViewBinding/CollectionViewStartActions.tsx +9 -0
- package/src/components/CollectionViewBinding/SortButton.tsx +16 -4
- package/src/components/CollectionViewBinding/ViewModeToggle.tsx +10 -5
- package/src/components/CollectionViewBinding/useBoardDataController.tsx +11 -3
- package/src/components/DefaultDrawer.tsx +7 -0
- package/src/components/DrawerNavigationItem.tsx +12 -0
- package/src/components/RelationSelector.tsx +52 -5
- package/src/components/SearchIconsView.tsx +14 -33
- package/src/components/SelectableTable/filters/StringNumberFilterField.tsx +1 -0
- package/src/components/UserSelector.tsx +47 -5
- package/src/data_export/export/ExportCollectionAction.tsx +101 -48
- package/src/data_export/export/export.ts +63 -16
- package/src/data_export/export/fetch_export_data.ts +90 -0
- package/src/data_export/export/index.ts +1 -0
- package/src/data_import/components/DataNewPropertiesMapping.tsx +20 -3
- package/src/data_import/components/ImportNewPropertyFieldPreview.tsx +1 -0
- package/src/data_import/components/ImportSaveInProgress.tsx +35 -36
- package/src/data_import/import/ImportCollectionAction.tsx +13 -2
- package/src/data_import/utils/csv.ts +153 -0
- package/src/data_import/utils/data.ts +7 -1
- package/src/data_import/utils/file_to_json.ts +77 -6
- package/src/data_import/utils/index.ts +2 -0
- package/src/data_import/utils/save_entities.ts +138 -0
- package/src/data_import/utils/transforms.ts +23 -0
- package/src/editor/components/image-bubble.tsx +2 -0
- package/src/editor/selectors/link-selector.tsx +1 -0
- package/src/form/field_bindings/BinaryFieldBinding.tsx +1 -0
- package/src/form/field_bindings/DateTimeFieldBinding.tsx +4 -0
- package/src/form/field_bindings/KeyValueFieldBinding.tsx +3 -0
- package/src/form/field_bindings/TextFieldBinding.tsx +18 -0
- package/src/form/field_bindings/VectorFieldBinding.tsx +1 -0
- package/src/preview/components/DatePreview.tsx +15 -2
- package/src/preview/components/ReferencePreview.tsx +5 -8
- package/src/preview/components/RelationPreview.tsx +8 -7
- package/src/preview/components/date_fns_locales.ts +125 -0
- package/dist/CollectionEditorDialog-BnfOcqM_.js.map +0 -1
- package/dist/PropertyEditView-vLXglzIy.js.map +0 -1
- package/dist/export-C26xQN-g.js.map +0 -1
- package/dist/import-Dhx96B6x.js.map +0 -1
- package/dist/util-Bt6VadeZ.js.map +0 -1
|
@@ -35,7 +35,7 @@ import { DataNewPropertiesMapping } from "../components/DataNewPropertiesMapping
|
|
|
35
35
|
import { ImportFileUpload } from "../components/ImportFileUpload";
|
|
36
36
|
import { ImportSaveInProgress } from "../components/ImportSaveInProgress";
|
|
37
37
|
import { ImportConfig } from "../types";
|
|
38
|
-
import { slugify } from "@rebasepro/utils";
|
|
38
|
+
import { isPrototypePollutingKey, slugify } from "@rebasepro/utils";
|
|
39
39
|
|
|
40
40
|
type ImportState = "initial" | "mapping" | "preview" | "import_data_saving";
|
|
41
41
|
|
|
@@ -400,7 +400,15 @@ export function ImportDataPreview<M extends Record<string, unknown>>({
|
|
|
400
400
|
return <CollectionTableBinding
|
|
401
401
|
title={<div>
|
|
402
402
|
<Typography variant={"subtitle2"}>Imported data preview</Typography>
|
|
403
|
-
|
|
403
|
+
{/* Conditional because it is only true when a column was chosen as
|
|
404
|
+
the id: without one, every row is created and nothing can be
|
|
405
|
+
overwritten. With one, the import asks for an upsert — which it
|
|
406
|
+
did not, back when this sentence was unconditional and false. */}
|
|
407
|
+
<Typography variant={"caption"}>
|
|
408
|
+
{importConfig.idColumn
|
|
409
|
+
? "Entities with the same id will be overwritten"
|
|
410
|
+
: "All rows will be imported as new entities"}
|
|
411
|
+
</Typography>
|
|
404
412
|
</div>}
|
|
405
413
|
tableController={{
|
|
406
414
|
data: importConfig.entities,
|
|
@@ -421,6 +429,9 @@ function buildHeadersMappingFromData(objArr: object[], properties?: Properties)
|
|
|
421
429
|
const headersMapping: Record<string, string> = {};
|
|
422
430
|
objArr.filter(Boolean).forEach((obj) => {
|
|
423
431
|
Object.keys(obj).forEach((key) => {
|
|
432
|
+
// The keys are the uploaded file's header row; `headersMapping[key] = …`
|
|
433
|
+
// is the same setter the import pipeline refuses elsewhere.
|
|
434
|
+
if (isPrototypePollutingKey(key)) return;
|
|
424
435
|
const child = (obj as Record<string, unknown>)[key];
|
|
425
436
|
if (child != null && typeof child === "object" && !Array.isArray(child)) {
|
|
426
437
|
const childProperty = properties?.[key];
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { isPrototypePollutingKey } from "@rebasepro/utils";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Delimiters sniffed from the header row, most common first.
|
|
5
|
+
*
|
|
6
|
+
* A spreadsheet saved as "CSV" in a locale that uses the comma as a decimal
|
|
7
|
+
* separator is semicolon-delimited, and a tab-separated file saved as `.csv`
|
|
8
|
+
* happens often enough to be worth reading.
|
|
9
|
+
*/
|
|
10
|
+
const CANDIDATE_DELIMITERS = [",", ";", "\t"];
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Split CSV text into rows of raw cells, RFC 4180 style.
|
|
14
|
+
*
|
|
15
|
+
* Quoted fields may contain the delimiter, newlines and doubled quotes; this is
|
|
16
|
+
* the exact inverse of what the export's `entryToCSVRow` writes, so a file this
|
|
17
|
+
* admin produced reads back into the cells it was built from.
|
|
18
|
+
*/
|
|
19
|
+
export function parseCsvRows(text: string, delimiter = ","): string[][] {
|
|
20
|
+
const rows: string[][] = [];
|
|
21
|
+
let row: string[] = [];
|
|
22
|
+
let field = "";
|
|
23
|
+
let quoted = false;
|
|
24
|
+
let inQuotes = false;
|
|
25
|
+
let i = 0;
|
|
26
|
+
|
|
27
|
+
const endField = () => {
|
|
28
|
+
row.push(field);
|
|
29
|
+
field = "";
|
|
30
|
+
quoted = false;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
while (i < text.length) {
|
|
34
|
+
const char = text[i];
|
|
35
|
+
|
|
36
|
+
if (inQuotes) {
|
|
37
|
+
if (char === "\"") {
|
|
38
|
+
// A doubled quote is a literal one; a single quote closes the field.
|
|
39
|
+
if (text[i + 1] === "\"") {
|
|
40
|
+
field += "\"";
|
|
41
|
+
i += 2;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
inQuotes = false;
|
|
45
|
+
i++;
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
field += char;
|
|
49
|
+
i++;
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (char === "\"" && field === "") {
|
|
54
|
+
inQuotes = true;
|
|
55
|
+
quoted = true;
|
|
56
|
+
i++;
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
if (char === delimiter) {
|
|
60
|
+
endField();
|
|
61
|
+
i++;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
if (char === "\r") {
|
|
65
|
+
// CRLF: the line ends on the \n. A lone CR outside quotes is dropped.
|
|
66
|
+
i++;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (char === "\n") {
|
|
70
|
+
endField();
|
|
71
|
+
rows.push(row);
|
|
72
|
+
row = [];
|
|
73
|
+
i++;
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
field += char;
|
|
77
|
+
i++;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// A file that does not end in a newline still has a last row.
|
|
81
|
+
if (field !== "" || quoted || row.length > 0) {
|
|
82
|
+
endField();
|
|
83
|
+
rows.push(row);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return rows;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Pick the delimiter that splits the header row into the most cells. A
|
|
91
|
+
* single-column file matches nothing and stays on the comma.
|
|
92
|
+
*/
|
|
93
|
+
export function detectCsvDelimiter(text: string): string {
|
|
94
|
+
const sample = text.slice(0, 64 * 1024);
|
|
95
|
+
let best = CANDIDATE_DELIMITERS[0];
|
|
96
|
+
let bestCount = 1;
|
|
97
|
+
for (const candidate of CANDIDATE_DELIMITERS) {
|
|
98
|
+
const count = parseCsvRows(sample, candidate)[0]?.length ?? 0;
|
|
99
|
+
if (count > bestCount) {
|
|
100
|
+
best = candidate;
|
|
101
|
+
bestCount = count;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return best;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export interface ParsedCsv {
|
|
108
|
+
headers: string[];
|
|
109
|
+
data: Record<string, string>[];
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Parse CSV text into one object per row, keyed by the header row.
|
|
114
|
+
*
|
|
115
|
+
* The picker accepts `.csv` and the dialog promises "Upload a CSV, Excel or JSON
|
|
116
|
+
* file", but every non-JSON file used to be handed to ExcelJS'
|
|
117
|
+
* `workbook.xlsx.load`, which is a zip reader — so every CSV failed with
|
|
118
|
+
* "Can't find end of central directory : is this a zip file ?". The export
|
|
119
|
+
* writes CSV by default, so the round trip could not close.
|
|
120
|
+
*
|
|
121
|
+
* A blank header keeps its column rather than collapsing it, so the cells to its
|
|
122
|
+
* right stay under their own names.
|
|
123
|
+
*/
|
|
124
|
+
export function parseCsvToObjects(text: string): ParsedCsv {
|
|
125
|
+
// A BOM is one character of the first header, and Excel writes one.
|
|
126
|
+
const cleaned = text.charCodeAt(0) === 0xFEFF ? text.slice(1) : text;
|
|
127
|
+
const delimiter = detectCsvDelimiter(cleaned);
|
|
128
|
+
const rows = parseCsvRows(cleaned, delimiter)
|
|
129
|
+
.filter(row => row.some(cell => cell.trim() !== ""));
|
|
130
|
+
|
|
131
|
+
if (rows.length === 0) {
|
|
132
|
+
return { headers: [],
|
|
133
|
+
data: [] };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const headers = rows[0].map((header, index) => header.trim() || `Column${index + 1}`);
|
|
137
|
+
|
|
138
|
+
const data = rows.slice(1).map((cells) => {
|
|
139
|
+
const obj: Record<string, string> = {};
|
|
140
|
+
headers.forEach((header, index) => {
|
|
141
|
+
// The header row is uploaded data: `obj["__proto__"] = …` is the
|
|
142
|
+
// prototype setter, not a column. Refused, as elsewhere in import.
|
|
143
|
+
if (isPrototypePollutingKey(header)) return;
|
|
144
|
+
const cell = cells[index];
|
|
145
|
+
if (cell === undefined) return;
|
|
146
|
+
obj[header] = cell;
|
|
147
|
+
});
|
|
148
|
+
return obj;
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
return { headers,
|
|
152
|
+
data };
|
|
153
|
+
}
|
|
@@ -5,7 +5,7 @@ import { isPropertyBuilder } from "@rebasepro/common";
|
|
|
5
5
|
import { unflattenObject } from "./file_to_json";
|
|
6
6
|
import { getIn } from "@rebasepro/forms";
|
|
7
7
|
import { inferTypeFromValue } from "@rebasepro/inference";
|
|
8
|
-
import { mergeDeep } from "@rebasepro/utils";
|
|
8
|
+
import { isPrototypePollutingKey, mergeDeep } from "@rebasepro/utils";
|
|
9
9
|
|
|
10
10
|
export function convertDataToEntity(authController: AuthController,
|
|
11
11
|
navigation: CollectionRegistryController,
|
|
@@ -57,6 +57,12 @@ export function convertDataToEntity(authController: AuthController,
|
|
|
57
57
|
|
|
58
58
|
export function flattenEntry(obj: Record<string, unknown>, parent = ""): Record<string, unknown> {
|
|
59
59
|
return Object.keys(obj).reduce<Record<string, unknown>>((acc, key) => {
|
|
60
|
+
// Keys come from the uploaded file, so `acc[key] = …` is the class-22
|
|
61
|
+
// primitive when the key is `__proto__`; refuse the column instead.
|
|
62
|
+
if (isPrototypePollutingKey(key)) {
|
|
63
|
+
console.warn(`Skipping column "${key}": a header may not reach the prototype chain`);
|
|
64
|
+
return acc;
|
|
65
|
+
}
|
|
60
66
|
const prefixedKey = parent ? `${parent}.${key}` : key;
|
|
61
67
|
|
|
62
68
|
if (typeof obj[key] === "object" && !(obj[key] instanceof Date) && obj[key] !== null && !Array.isArray(obj[key])) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import ExcelJS from "exceljs";
|
|
2
1
|
import { getWorksheetHeaders } from "./file_headers";
|
|
3
2
|
import { mapJsonParse, unflattenObject } from "./transforms";
|
|
3
|
+
import { parseCsvToObjects } from "./csv";
|
|
4
|
+
import { isPrototypePollutingKey } from "@rebasepro/utils";
|
|
4
5
|
|
|
5
6
|
// Re-export for backwards compat (tests import from this file)
|
|
6
7
|
export { unflattenObject } from "./transforms";
|
|
@@ -10,9 +11,78 @@ type ConversionResult = {
|
|
|
10
11
|
propertiesOrder: string[]
|
|
11
12
|
}
|
|
12
13
|
|
|
14
|
+
/**
|
|
15
|
+
* ExcelJS, fetched the first time somebody actually opens a workbook.
|
|
16
|
+
*
|
|
17
|
+
* It was a top-level `import ExcelJS from "exceljs"`, and that one line put
|
|
18
|
+
* 940 kB of spreadsheet reader into the admin's entry chunk — preloaded on the
|
|
19
|
+
* login screen, before anyone has authenticated, let alone clicked Import.
|
|
20
|
+
* `CollectionViewActions` lazy-loads the import and export actions and says so
|
|
21
|
+
* in a comment, but the package's own barrel (`src/index.ts` re-exports
|
|
22
|
+
* `./data_import`) puts this module back in the entry's static graph, so the
|
|
23
|
+
* `lazy()` bought nothing. A static import inside a module the barrel reaches
|
|
24
|
+
* is eager no matter what the component above it does; the only thing that
|
|
25
|
+
* makes a dependency lazy is importing it lazily, here.
|
|
26
|
+
*/
|
|
27
|
+
type ExcelJsModule = typeof import("exceljs");
|
|
28
|
+
|
|
29
|
+
let excelJsModule: Promise<ExcelJsModule> | undefined;
|
|
30
|
+
|
|
31
|
+
function loadExcelJs(): Promise<ExcelJsModule> {
|
|
32
|
+
// exceljs is CommonJS, so the namespace a bundler hands back wraps the real
|
|
33
|
+
// module under `default`. Native ESM (and the type declarations) put the
|
|
34
|
+
// members at the top level. Accept either.
|
|
35
|
+
excelJsModule ??= import("exceljs").then(mod => (mod as { default?: ExcelJsModule }).default ?? mod);
|
|
36
|
+
return excelJsModule;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Whether this file is delimited text rather than a workbook.
|
|
41
|
+
*
|
|
42
|
+
* Browsers report `text/csv`, `application/csv` or nothing at all for the same
|
|
43
|
+
* `.csv`, so the extension is the reliable half of the test.
|
|
44
|
+
*/
|
|
45
|
+
function isCsvFile(file: File): boolean {
|
|
46
|
+
const name = (file.name ?? "").toLowerCase();
|
|
47
|
+
if (name.endsWith(".csv") || name.endsWith(".tsv")) return true;
|
|
48
|
+
return file.type === "text/csv" || file.type === "application/csv";
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Turn the parsed cells of one sheet/file into the entity-shaped rows the import maps. */
|
|
52
|
+
function toImportRows(rows: Array<Record<string, unknown>>): object[] {
|
|
53
|
+
return rows.map(mapJsonParse).map(unflattenObject);
|
|
54
|
+
}
|
|
55
|
+
|
|
13
56
|
export function convertFileToJson(file: File): Promise<ConversionResult> {
|
|
14
57
|
return new Promise((resolve, reject) => {
|
|
15
|
-
if (file
|
|
58
|
+
if (isCsvFile(file)) {
|
|
59
|
+
// Not the ExcelJS branch: `workbook.xlsx.load` is a zip reader, and
|
|
60
|
+
// every CSV — the format this admin exports by default — died on it
|
|
61
|
+
// with "Can't find end of central directory : is this a zip file ?".
|
|
62
|
+
console.debug("Converting CSV file to JSON", file.name);
|
|
63
|
+
const reader = new FileReader();
|
|
64
|
+
reader.onload = function (e) {
|
|
65
|
+
try {
|
|
66
|
+
const { headers, data } = parseCsvToObjects(e.target?.result as string);
|
|
67
|
+
if (headers.length === 0) {
|
|
68
|
+
reject(new Error("The CSV file is empty"));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
resolve({
|
|
72
|
+
data: toImportRows(data),
|
|
73
|
+
propertiesOrder: headers
|
|
74
|
+
});
|
|
75
|
+
} catch (err) {
|
|
76
|
+
console.error("Error parsing CSV file", err);
|
|
77
|
+
reject(err);
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
reader.onerror = () => reject(reader.error ?? new Error("Could not read the file"));
|
|
81
|
+
// Explicit UTF-8: the pre-ExcelJS implementation passed
|
|
82
|
+
// `codepage: 65001` for the same reason, and the browser's default
|
|
83
|
+
// guess mangles accented and CJK text.
|
|
84
|
+
reader.readAsText(file, "utf-8");
|
|
85
|
+
} else if (file.type === "application/json") {
|
|
16
86
|
console.debug("Converting JSON file to JSON", file.name);
|
|
17
87
|
const reader = new FileReader();
|
|
18
88
|
reader.onload = function (e) {
|
|
@@ -40,6 +110,7 @@ export function convertFileToJson(file: File): Promise<ConversionResult> {
|
|
|
40
110
|
reader.onload = async function (e) {
|
|
41
111
|
try {
|
|
42
112
|
const buffer = e.target?.result as ArrayBuffer;
|
|
113
|
+
const ExcelJS = await loadExcelJs();
|
|
43
114
|
const workbook = new ExcelJS.Workbook();
|
|
44
115
|
await workbook.xlsx.load(buffer);
|
|
45
116
|
const worksheet = workbook.worksheets[0];
|
|
@@ -57,17 +128,17 @@ export function convertFileToJson(file: File): Promise<ConversionResult> {
|
|
|
57
128
|
const obj: Record<string, any> = {};
|
|
58
129
|
row.eachCell({ includeEmpty: false }, (cell, colNumber) => {
|
|
59
130
|
const header = headers[colNumber - 1];
|
|
60
|
-
|
|
131
|
+
// A `__proto__` header would be the prototype setter
|
|
132
|
+
// here rather than a column; refused, as in `csv.ts`.
|
|
133
|
+
if (header && !isPrototypePollutingKey(header)) {
|
|
61
134
|
obj[header] = cell.value;
|
|
62
135
|
}
|
|
63
136
|
});
|
|
64
137
|
parsedData.push(obj);
|
|
65
138
|
});
|
|
66
139
|
|
|
67
|
-
const cleanedData = parsedData.map(mapJsonParse);
|
|
68
|
-
const jsonData = cleanedData.map(unflattenObject);
|
|
69
140
|
resolve({
|
|
70
|
-
data:
|
|
141
|
+
data: toImportRows(parsedData),
|
|
71
142
|
propertiesOrder: headers
|
|
72
143
|
});
|
|
73
144
|
} catch (err) {
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { CollectionAccessor, Entity, RebaseData } from "@rebasepro/types";
|
|
2
|
+
|
|
3
|
+
/** Rows written per request, and per transaction where the data source has one. */
|
|
4
|
+
export const IMPORT_BATCH_SIZE = 25;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* An import failure that says where it stopped.
|
|
8
|
+
*
|
|
9
|
+
* "Error saving data / null value in column "sku" violates not-null constraint"
|
|
10
|
+
* on a ten-thousand-row import named no row, and the Retry button restarted at
|
|
11
|
+
* zero — where the rows already written now conflicted, so it could never
|
|
12
|
+
* succeed. Both halves of that are what this carries.
|
|
13
|
+
*/
|
|
14
|
+
export interface ImportSaveError extends Error {
|
|
15
|
+
/** Rows written and confirmed before the failure. Where a retry resumes. */
|
|
16
|
+
committed: number;
|
|
17
|
+
/** Index of the first row that was not written, 0-based. */
|
|
18
|
+
failedFrom: number;
|
|
19
|
+
/** Index after the last row of the batch that was not written. */
|
|
20
|
+
failedTo: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** The row a bulk write takes: the values, with the chosen id as a column. */
|
|
24
|
+
function toBulkRow(entity: Partial<Entity<any>>): Record<string, unknown> {
|
|
25
|
+
const values = { ...(entity.values ?? {}) } as Record<string, unknown>;
|
|
26
|
+
if (entity.id !== undefined && entity.id !== null && entity.id !== "") {
|
|
27
|
+
// Exactly what single-row `create(values, id)` puts on the wire.
|
|
28
|
+
values.id = entity.id;
|
|
29
|
+
}
|
|
30
|
+
return values;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function isBulkUnsupported(error: unknown): boolean {
|
|
34
|
+
if ((error as { code?: string })?.code === "BULK_UNSUPPORTED") return true;
|
|
35
|
+
const message = (error as Error)?.message ?? "";
|
|
36
|
+
return /bulk writes are not supported|does not support bulk/i.test(message);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isConflict(error: unknown): boolean {
|
|
40
|
+
const status = (error as { status?: number; statusCode?: number })?.status
|
|
41
|
+
?? (error as { statusCode?: number })?.statusCode;
|
|
42
|
+
if (status === 409) return true;
|
|
43
|
+
const code = (error as { code?: string })?.code;
|
|
44
|
+
return code === "CONFLICT" || code === "23505";
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Write one batch, all-or-nothing wherever the data source can.
|
|
49
|
+
*
|
|
50
|
+
* `POST /<collection>/bulk` validates every row before the transaction opens,
|
|
51
|
+
* names a failure by row index, and takes the `upsert` flag the preview screen
|
|
52
|
+
* promises — none of which one `create` per row inside a `Promise.all` could
|
|
53
|
+
* offer, since `Promise.all` does not cancel its siblings and every request was
|
|
54
|
+
* its own transaction.
|
|
55
|
+
*/
|
|
56
|
+
async function writeBatch(accessor: CollectionAccessor,
|
|
57
|
+
batch: Partial<Entity<any>>[],
|
|
58
|
+
bulkUnsupported: { current: boolean },
|
|
59
|
+
written: { count: number }): Promise<void> {
|
|
60
|
+
|
|
61
|
+
if (accessor.createMany && !bulkUnsupported.current) {
|
|
62
|
+
try {
|
|
63
|
+
await accessor.createMany(batch.map(toBulkRow), { upsert: true });
|
|
64
|
+
written.count = batch.length;
|
|
65
|
+
return;
|
|
66
|
+
} catch (e) {
|
|
67
|
+
if (!isBulkUnsupported(e)) throw e;
|
|
68
|
+
// A data source without bulk writes: remember it, so the failed
|
|
69
|
+
// round trip is not paid again on every later batch.
|
|
70
|
+
bulkUnsupported.current = true;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Sequential, not `Promise.all`: a rejected write does not cancel its
|
|
75
|
+
// siblings, so a parallel batch commits an indeterminate subset of itself
|
|
76
|
+
// and nobody can say which rows landed.
|
|
77
|
+
for (const entity of batch) {
|
|
78
|
+
try {
|
|
79
|
+
await accessor.create(entity.values ?? {}, entity.id);
|
|
80
|
+
} catch (e) {
|
|
81
|
+
// Without `/bulk` there is no upsert, so the preview's promise is
|
|
82
|
+
// honoured the only way left: a row that collided on an id the user
|
|
83
|
+
// supplied is overwritten. A row with no id had nothing to collide
|
|
84
|
+
// on, and every other failure is still a failure.
|
|
85
|
+
if (entity.id === undefined || !isConflict(e)) throw e;
|
|
86
|
+
await accessor.update(entity.id, entity.values ?? {});
|
|
87
|
+
}
|
|
88
|
+
written.count++;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Write imported entities in batches, reporting what was committed.
|
|
94
|
+
*
|
|
95
|
+
* Resumable on purpose: a failure at row 4 000 leaves rows 1–3 999 in the table
|
|
96
|
+
* whatever this does, so the only useful thing to report is where to carry on
|
|
97
|
+
* from. `offset` is that point — the Retry button passes back the `committed`
|
|
98
|
+
* count of the error it is retrying, instead of starting from zero and
|
|
99
|
+
* conflicting on every row that already exists.
|
|
100
|
+
*
|
|
101
|
+
* @throws {ImportSaveError} naming the rows that were not written.
|
|
102
|
+
*/
|
|
103
|
+
export async function saveImportedEntities(dataClient: RebaseData,
|
|
104
|
+
path: string,
|
|
105
|
+
data: Partial<Entity<any>>[],
|
|
106
|
+
{
|
|
107
|
+
offset = 0,
|
|
108
|
+
batchSize = IMPORT_BATCH_SIZE,
|
|
109
|
+
onBatchCommitted = () => undefined,
|
|
110
|
+
bulkUnsupported = { current: false }
|
|
111
|
+
}: {
|
|
112
|
+
offset?: number;
|
|
113
|
+
batchSize?: number;
|
|
114
|
+
onBatchCommitted?: (rowsWritten: number) => void;
|
|
115
|
+
bulkUnsupported?: { current: boolean };
|
|
116
|
+
} = {}): Promise<void> {
|
|
117
|
+
|
|
118
|
+
console.debug("Saving imported data", offset, batchSize);
|
|
119
|
+
|
|
120
|
+
const accessor = dataClient.collection(path);
|
|
121
|
+
|
|
122
|
+
for (let start = offset; start < data.length; start += batchSize) {
|
|
123
|
+
const batch = data.slice(start, start + batchSize);
|
|
124
|
+
const written = { count: 0 };
|
|
125
|
+
try {
|
|
126
|
+
await writeBatch(accessor, batch, bulkUnsupported, written);
|
|
127
|
+
} catch (e) {
|
|
128
|
+
throw Object.assign(e as Error, {
|
|
129
|
+
committed: start + written.count,
|
|
130
|
+
failedFrom: start + written.count,
|
|
131
|
+
failedTo: start + batch.length
|
|
132
|
+
}) as ImportSaveError;
|
|
133
|
+
}
|
|
134
|
+
onBatchCommitted(start + batch.length);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
onBatchCommitted(data.length);
|
|
138
|
+
}
|
|
@@ -1,10 +1,27 @@
|
|
|
1
|
+
import { isPrototypePollutingKey, pathTraversesPrototype } from "@rebasepro/utils";
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Take an object with keys of type `address.street`, `address.city` and
|
|
3
5
|
* convert it to an object with nested objects like `{ address: { street: ..., city: ... } }`
|
|
6
|
+
*
|
|
7
|
+
* Keys here are the **header row of an uploaded file**, so they are attacker
|
|
8
|
+
* data. `res["__proto__"] = …` is the prototype setter rather than an own
|
|
9
|
+
* property, so a column named `__proto__.polluted` walked out of the accumulator
|
|
10
|
+
* and wrote onto `Object.prototype` for the life of the tab — and
|
|
11
|
+
* `constructor.prototype.x` threw a `TypeError` that surfaced as an unreadable
|
|
12
|
+
* file. Such a column is refused, not sanitised, the way `setIn` settled it
|
|
13
|
+
* (`docs/bug-classes.md` class 22).
|
|
14
|
+
*
|
|
4
15
|
* @param flatObj
|
|
5
16
|
*/
|
|
6
17
|
export function unflattenObject(flatObj: { [key: string]: any }) {
|
|
7
18
|
return Object.keys(flatObj).reduce((nestedObj, key) => {
|
|
19
|
+
// The string form, so `toPath` rewrites `__proto__[0]` to a `__proto__`
|
|
20
|
+
// segment too — the bracket branch below writes through the same setter.
|
|
21
|
+
if (pathTraversesPrototype(key)) {
|
|
22
|
+
console.warn(`Skipping column "${key}": a header may not reach the prototype chain`);
|
|
23
|
+
return nestedObj;
|
|
24
|
+
}
|
|
8
25
|
let currentObj = nestedObj;
|
|
9
26
|
const keyParts = key.split(".");
|
|
10
27
|
keyParts.forEach((keyPart, i) => {
|
|
@@ -37,6 +54,12 @@ export function unflattenObject(flatObj: { [key: string]: any }) {
|
|
|
37
54
|
|
|
38
55
|
export function mapJsonParse(obj: Record<string, any>) {
|
|
39
56
|
return Object.keys(obj).reduce((acc: Record<string, any>, key) => {
|
|
57
|
+
// Same header row, same setter: `acc["__proto__"] = value` replaces the
|
|
58
|
+
// accumulator's prototype instead of adding a column.
|
|
59
|
+
if (isPrototypePollutingKey(key)) {
|
|
60
|
+
console.warn(`Skipping column "${key}": a header may not reach the prototype chain`);
|
|
61
|
+
return acc;
|
|
62
|
+
}
|
|
40
63
|
try {
|
|
41
64
|
acc[key] = JSON.parse(obj[key]);
|
|
42
65
|
} catch (e) {
|
|
@@ -157,6 +157,7 @@ export const ImageBubble = forwardRef<HTMLDivElement, ImageBubbleProps>(
|
|
|
157
157
|
>
|
|
158
158
|
<TextField
|
|
159
159
|
size={"small"}
|
|
160
|
+
aria-label={t("alt_text")}
|
|
160
161
|
placeholder={t("alt_text")}
|
|
161
162
|
value={alt}
|
|
162
163
|
onChange={(e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
|
|
@@ -166,6 +167,7 @@ export const ImageBubble = forwardRef<HTMLDivElement, ImageBubbleProps>(
|
|
|
166
167
|
/>
|
|
167
168
|
<TextField
|
|
168
169
|
size={"small"}
|
|
170
|
+
aria-label={t("title")}
|
|
169
171
|
placeholder={t("title")}
|
|
170
172
|
value={title}
|
|
171
173
|
onChange={(e: React.ChangeEvent<HTMLInputElement | HTMLTextAreaElement>) => {
|
|
@@ -139,6 +139,7 @@ export function BinaryFieldBinding({
|
|
|
139
139
|
<div className={cls("flex flex-col gap-2 p-4 rounded-xl border", defaultBorderMixin)}>
|
|
140
140
|
<TextField
|
|
141
141
|
size={size}
|
|
142
|
+
aria-label={`${property.name ?? propertyKey} base64 value`}
|
|
142
143
|
value={text}
|
|
143
144
|
onChange={onChange}
|
|
144
145
|
autoFocus={autoFocus ?? true}
|
|
@@ -60,6 +60,10 @@ export function DateTimeFieldBinding({
|
|
|
60
60
|
required={property.validation?.required}
|
|
61
61
|
className={showError ? "text-red-500 dark:text-red-500" : "text-text-secondary dark:text-text-secondary-dark"}
|
|
62
62
|
title={property.name ?? propertyKey}/>}
|
|
63
|
+
// The entity form sets `hideLabel` for every field FieldBlock
|
|
64
|
+
// labels on its behalf, and FieldBlock's label is a <span> that
|
|
65
|
+
// points at nothing — so without this the control is nameless.
|
|
66
|
+
aria-label={hideLabel ? (property.name ?? propertyKey) : undefined}
|
|
63
67
|
/>
|
|
64
68
|
</PropertyIdCopyTooltip>
|
|
65
69
|
|
|
@@ -269,6 +269,7 @@ function MapKeyValueRow<T extends Record<string, unknown>>({
|
|
|
269
269
|
if (type === "string" || type === "number") {
|
|
270
270
|
return <TextField
|
|
271
271
|
key={type}
|
|
272
|
+
aria-label={fieldKey ? `${t("value")}: ${fieldKey}` : t("value")}
|
|
272
273
|
placeholder={"value"}
|
|
273
274
|
value={entryValue as string | number | undefined}
|
|
274
275
|
type={type === "number" ? "number" : "text"}
|
|
@@ -390,6 +391,7 @@ function MapKeyValueRow<T extends Record<string, unknown>>({
|
|
|
390
391
|
<div className="w-[300px] max-w-[30%]">
|
|
391
392
|
<TextField
|
|
392
393
|
value={fieldKey}
|
|
394
|
+
aria-label={t("key")}
|
|
393
395
|
placeholder={"key"}
|
|
394
396
|
disabled={disabled || (entryValue !== undefined && entryValue !== null && entryValue !== "")}
|
|
395
397
|
size={"medium"}
|
|
@@ -459,6 +461,7 @@ function ArrayKeyValueRow<T>({
|
|
|
459
461
|
function buildInput(entryValue: unknown, type: DataType) {
|
|
460
462
|
if (type === "string" || type === "number") {
|
|
461
463
|
return <TextField value={entryValue as string | number | undefined}
|
|
464
|
+
aria-label={`Value ${index + 1}`}
|
|
462
465
|
type={type === "number" ? "number" : "text"}
|
|
463
466
|
size={"medium"}
|
|
464
467
|
onChange={(event) => {
|
|
@@ -97,6 +97,22 @@ export function TextFieldBinding<T extends string | number>({
|
|
|
97
97
|
required={property.validation?.required || property.isId === true}
|
|
98
98
|
title={property.name ?? propertyKey}/>
|
|
99
99
|
);
|
|
100
|
+
|
|
101
|
+
// The control needs a name of its own in two cases, and between them they
|
|
102
|
+
// cover every text field in an entity form:
|
|
103
|
+
//
|
|
104
|
+
// - `hideLabel`, which the form sets for anything FieldBlock labels for it
|
|
105
|
+
// (`hideLabel: !selfLabelling`). FieldBlock draws the name in a <span>
|
|
106
|
+
// inside a <div> — not a <label>, and pointing at nothing — so suppressing
|
|
107
|
+
// the field's own label left the input with no accessible name at all and
|
|
108
|
+
// a screen reader fell back to reading its value back to you.
|
|
109
|
+
// - the multiline branch below, which never associated anything: its label
|
|
110
|
+
// is a decorative absolutely-positioned <div>, so it is unnamed whether or
|
|
111
|
+
// not the form also drew one.
|
|
112
|
+
//
|
|
113
|
+
// The single-line TextField is already handled by the kit when a label is
|
|
114
|
+
// present — it wires `aria-labelledby` — so it only needs this as a fallback.
|
|
115
|
+
const accessibleName = property.name ?? propertyKey;
|
|
100
116
|
return (<>
|
|
101
117
|
<PropertyIdCopyTooltip propertyKey={propertyKey}>
|
|
102
118
|
{isMultiline ? (
|
|
@@ -115,6 +131,7 @@ export function TextFieldBinding<T extends string | number>({
|
|
|
115
131
|
onChange={onChange}
|
|
116
132
|
autoFocus={autoFocus}
|
|
117
133
|
disabled={disabled}
|
|
134
|
+
aria-label={accessibleName}
|
|
118
135
|
className={cls(
|
|
119
136
|
"rounded-md resize-none w-full outline-none text-base bg-transparent min-h-[64px] px-3",
|
|
120
137
|
label ? "pt-8 pb-2" : "py-2",
|
|
@@ -138,6 +155,7 @@ export function TextFieldBinding<T extends string | number>({
|
|
|
138
155
|
onChange={onChange}
|
|
139
156
|
autoFocus={autoFocus}
|
|
140
157
|
label={label}
|
|
158
|
+
aria-label={label ? undefined : accessibleName}
|
|
141
159
|
type={inputType}
|
|
142
160
|
disabled={disabled}
|
|
143
161
|
endAdornment={
|
|
@@ -166,6 +166,7 @@ export function VectorFieldBinding({
|
|
|
166
166
|
<div className={cls("flex flex-col gap-2 p-4 rounded-xl border bg-surface-50/20 dark:bg-surface-800/10", defaultBorderMixin)}>
|
|
167
167
|
<TextField
|
|
168
168
|
size={size}
|
|
169
|
+
aria-label={`${property.name ?? propertyKey} vector values`}
|
|
169
170
|
value={textValue}
|
|
170
171
|
onChange={onChange}
|
|
171
172
|
autoFocus={true}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import React from "react";
|
|
2
2
|
|
|
3
3
|
import { format } from "date-fns";
|
|
4
|
-
import * as locales from "date-fns/locale";
|
|
5
4
|
import { useCustomizationController } from "@rebasepro/app";
|
|
6
5
|
import { defaultDateFormat } from "@rebasepro/utils";
|
|
6
|
+
import { getLoadedDateFnsLocale, loadDateFnsLocale } from "./date_fns_locales";
|
|
7
7
|
|
|
8
8
|
export interface DatePreviewProps {
|
|
9
9
|
date: Date;
|
|
@@ -28,7 +28,20 @@ export function DatePreview({
|
|
|
28
28
|
}: DatePreviewProps): React.ReactElement {
|
|
29
29
|
|
|
30
30
|
const customizationController = useCustomizationController();
|
|
31
|
-
const
|
|
31
|
+
const configuredLocale = customizationController?.locale;
|
|
32
|
+
|
|
33
|
+
// The locale arrives asynchronously now (see `date_fns_locales`). Seed from
|
|
34
|
+
// the cache so only the very first date on a page renders un-localised: by
|
|
35
|
+
// the time a table of them mounts, the module is already resolved and there
|
|
36
|
+
// is no second pass.
|
|
37
|
+
const [dateUtilsLocale, setDateUtilsLocale] = React.useState(() => getLoadedDateFnsLocale(configuredLocale));
|
|
38
|
+
React.useEffect(() => {
|
|
39
|
+
let cancelled = false;
|
|
40
|
+
loadDateFnsLocale(configuredLocale)
|
|
41
|
+
.then(loaded => { if (!cancelled) setDateUtilsLocale(loaded); })
|
|
42
|
+
.catch(error => console.error(`[rebase] could not load the "${configuredLocale}" date locale`, error));
|
|
43
|
+
return () => { cancelled = true; };
|
|
44
|
+
}, [configuredLocale]);
|
|
32
45
|
|
|
33
46
|
if (!date || !(date instanceof Date) || isNaN(date.getTime())) {
|
|
34
47
|
return <></>;
|