@rebasepro/admin 0.13.0 → 0.13.1-canary.g06dbe5b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{CollectionEditorDialog-CjXkWZ64.js → CollectionEditorDialog-C3DbFNqp.js} +3 -3
- package/dist/{CollectionEditorDialog-CjXkWZ64.js.map → CollectionEditorDialog-C3DbFNqp.js.map} +1 -1
- package/dist/{PropertyEditView-W8I3YV89.js → PropertyEditView-DFH8rXRN.js} +5 -5
- package/dist/PropertyEditView-DFH8rXRN.js.map +1 -0
- package/dist/{RouterCollectionsStudioView-BsLTNoqX.js → RouterCollectionsStudioView-DiUDOu4b.js} +9 -13
- package/dist/RouterCollectionsStudioView-DiUDOu4b.js.map +1 -0
- package/dist/collection_editor/serializable_types.d.ts +1 -0
- package/dist/collection_editor_ui.js +4 -4
- package/dist/components/AdditionalFieldValue.d.ts +25 -0
- package/dist/components/ClearFilterSortButton.d.ts +7 -1
- package/dist/components/CollectionViewBinding/CollectionViewBinding.d.ts +16 -0
- package/dist/components/CollectionViewBinding/EntityCardBinding.d.ts +3 -1
- package/dist/components/CollectionViewBinding/SearchExplanation.d.ts +48 -0
- package/dist/components/CollectionViewBinding/SearchHighlight.d.ts +101 -0
- package/dist/components/CollectionViewBinding/SlotValue.d.ts +63 -0
- package/dist/components/CollectionViewBinding/SortButton.d.ts +7 -1
- package/dist/components/CollectionViewBinding/SplitListCloseButton.d.ts +17 -4
- package/dist/components/CollectionViewBinding/SplitListShowButton.d.ts +12 -0
- package/dist/components/CollectionViewBinding/usePreviewSlots.d.ts +55 -5
- package/dist/components/DetailViewBinding.d.ts +15 -0
- package/dist/components/DrawerNavigationGroup.d.ts +9 -6
- package/dist/components/DrawerNavigationItem.d.ts +16 -6
- package/dist/components/EditViewBinding.d.ts +30 -1
- package/dist/components/EntityDisplayHeader.d.ts +53 -0
- package/dist/components/EntityIdentityBar.d.ts +43 -1
- package/dist/components/EntityViewBinding.d.ts +60 -6
- package/dist/components/RebaseLayout.d.ts +1 -1
- package/dist/components/RebaseShell.d.ts +1 -1
- package/dist/components/RelationSelector.d.ts +8 -1
- package/dist/components/UserSelector.d.ts +8 -1
- package/dist/components/app/Scaffold.d.ts +9 -3
- package/dist/components/index.d.ts +2 -0
- package/dist/data_export/export/export.d.ts +7 -0
- package/dist/data_export/export/fetch_export_data.d.ts +48 -0
- package/dist/data_export/export/index.d.ts +1 -0
- package/dist/data_import/utils/csv.d.ts +30 -0
- package/dist/data_import/utils/index.d.ts +2 -0
- package/dist/data_import/utils/save_entities.d.ts +38 -0
- package/dist/data_import/utils/transforms.d.ts +9 -0
- package/dist/{export-J79rMoNj.js → export-Bd5i6Uvm.js} +175 -39
- package/dist/export-Bd5i6Uvm.js.map +1 -0
- package/dist/form/components/FieldBlock.d.ts +58 -4
- package/dist/form/components/FormRail.d.ts +13 -0
- package/dist/form/components/FormSections.d.ts +15 -1
- package/dist/form/components/useRailVisible.d.ts +5 -4
- package/dist/form/form_utils.d.ts +38 -1
- package/dist/{history-D7L_JlLO.js → history-D9R6LUrH.js} +4 -4
- package/dist/{history-D7L_JlLO.js.map → history-D9R6LUrH.js.map} +1 -1
- package/dist/hooks/useEntityDisplay.d.ts +102 -0
- package/dist/hooks/useEntityDisplayTitle.d.ts +18 -9
- package/dist/{import-BvYl--U4.js → import-pt_j-0G3.js} +6 -5
- package/dist/import-pt_j-0G3.js.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +158 -60
- package/dist/index.js.map +1 -1
- package/dist/preview/compact_text.d.ts +23 -0
- package/dist/preview/components/BooleanPreview.d.ts +3 -1
- package/dist/preview/property_previews/ArrayOfStringsPreview.d.ts +1 -1
- package/dist/preview/property_previews/ArrayOneOfPreview.d.ts +1 -1
- package/dist/preview/property_previews/ArrayPropertyPreview.d.ts +1 -1
- package/dist/preview/property_previews/MapPropertyPreview.d.ts +1 -1
- package/dist/preview/property_previews/StringPropertyPreview.d.ts +1 -1
- package/dist/routes/RebaseRoute.d.ts +1 -1
- package/dist/types/components/PropertyPreviewProps.d.ts +19 -0
- package/dist/util/number_format.d.ts +14 -0
- package/dist/util/previews.d.ts +9 -2
- package/dist/util/view_constants.d.ts +24 -0
- package/dist/util/view_mode.d.ts +18 -0
- package/dist/{util-GiwPgxnL.js → util-nX_-JRuF.js} +3984 -1774
- package/dist/util-nX_-JRuF.js.map +1 -0
- package/package.json +10 -9
- package/src/collection_editor/serializable_types.ts +1 -0
- package/src/collection_editor/ui/collection_editor/CollectionsStudioView.tsx +9 -11
- package/src/collection_editor/ui/collection_editor/properties/MapPropertyField.tsx +1 -1
- package/src/collection_editor/ui/collection_editor/properties/advanced/AdvancedPropertyValidation.tsx +5 -2
- package/src/collection_editor/useLocalCollectionsConfigController.tsx +10 -3
- package/src/components/AdditionalFieldValue.tsx +68 -0
- package/src/components/ClearFilterSortButton.tsx +16 -7
- package/src/components/CollectionTableBinding/CollectionTableBinding.tsx +7 -8
- package/src/components/CollectionTableBinding/PropertyTableCell.tsx +2 -2
- package/src/components/CollectionTableBinding/column_utils.tsx +15 -6
- package/src/components/CollectionTableBinding/internal/popup_field/PopupFormField.tsx +2 -2
- package/src/components/CollectionTableBinding/table_bindings.tsx +7 -1
- package/src/components/CollectionViewBinding/BoardCardBinding.tsx +26 -27
- package/src/components/CollectionViewBinding/CollectionBoardViewBinding.tsx +13 -7
- package/src/components/CollectionViewBinding/CollectionCardViewBinding.tsx +2 -1
- package/src/components/CollectionViewBinding/CollectionListViewBinding.tsx +822 -354
- package/src/components/CollectionViewBinding/CollectionViewActions.tsx +1 -1
- package/src/components/CollectionViewBinding/CollectionViewBinding.tsx +51 -12
- package/src/components/CollectionViewBinding/CollectionViewStartActions.tsx +9 -0
- package/src/components/CollectionViewBinding/EntityCardBinding.tsx +51 -39
- package/src/components/CollectionViewBinding/SearchExplanation.tsx +106 -0
- package/src/components/CollectionViewBinding/SearchHighlight.tsx +311 -0
- package/src/components/CollectionViewBinding/SlotValue.tsx +156 -0
- package/src/components/CollectionViewBinding/SortButton.tsx +16 -4
- package/src/components/CollectionViewBinding/SplitListCloseButton.tsx +23 -7
- package/src/components/CollectionViewBinding/SplitListShowButton.tsx +28 -0
- package/src/components/CollectionViewBinding/SplitListView.tsx +47 -7
- package/src/components/CollectionViewBinding/ViewModeToggle.tsx +10 -5
- package/src/components/CollectionViewBinding/useBoardDataController.tsx +23 -6
- package/src/components/CollectionViewBinding/usePreviewSlots.ts +183 -64
- package/src/components/CollectionViewBinding/utils.ts +7 -4
- package/src/components/DefaultDrawer.tsx +19 -23
- package/src/components/DetailViewBinding.tsx +109 -62
- package/src/components/DrawerNavigationGroup.tsx +16 -39
- package/src/components/DrawerNavigationItem.tsx +44 -16
- package/src/components/EditViewBinding.tsx +233 -50
- package/src/components/EntityDisplayHeader.tsx +170 -0
- package/src/components/EntityIdentityBar.tsx +163 -23
- package/src/components/EntityInspector.tsx +24 -6
- package/src/components/EntityPreviewBinding.tsx +58 -18
- package/src/components/EntityViewBinding.tsx +344 -41
- package/src/components/InlineEntityPreview.tsx +10 -10
- package/src/components/RebaseLayout.tsx +2 -2
- package/src/components/RebaseShell.tsx +2 -2
- package/src/components/ReferenceTable/SelectionTableBinding.tsx +4 -2
- package/src/components/RelationSelector.tsx +35 -10
- package/src/components/SideDialogs.tsx +15 -3
- package/src/components/SidePanelBinding.tsx +122 -66
- package/src/components/UserSelector.tsx +67 -13
- package/src/components/app/Scaffold.tsx +112 -10
- package/src/components/history/LastEditedByIndicator.tsx +7 -12
- package/src/components/index.ts +3 -0
- package/src/data_export/export/ExportCollectionAction.tsx +101 -48
- package/src/data_export/export/export.ts +63 -16
- package/src/data_export/export/fetch_export_data.ts +90 -0
- package/src/data_export/export/index.ts +1 -0
- package/src/data_import/components/ImportSaveInProgress.tsx +35 -36
- package/src/data_import/import/ImportCollectionAction.tsx +13 -2
- package/src/data_import/utils/csv.ts +153 -0
- package/src/data_import/utils/data.ts +16 -2
- package/src/data_import/utils/file_to_json.ts +51 -5
- package/src/data_import/utils/index.ts +2 -0
- package/src/data_import/utils/save_entities.ts +138 -0
- package/src/data_import/utils/transforms.ts +23 -0
- package/src/form/EntityForm.tsx +74 -53
- package/src/form/EntityFormBinding.tsx +24 -11
- package/src/form/PropertyFieldBinding.tsx +2 -2
- package/src/form/components/FieldBlock.tsx +80 -10
- package/src/form/components/FormRail.tsx +19 -2
- package/src/form/components/FormSections.tsx +32 -5
- package/src/form/components/StorageUploadProgress.tsx +9 -3
- package/src/form/components/useRailVisible.tsx +5 -4
- package/src/form/field_bindings/ArrayCustomShapedFieldBinding.tsx +2 -2
- package/src/form/field_bindings/BinaryFieldBinding.tsx +10 -5
- package/src/form/field_bindings/BlockFieldBinding.tsx +3 -3
- package/src/form/field_bindings/GeopointFieldBinding.tsx +23 -16
- package/src/form/field_bindings/MapFieldBinding.tsx +2 -2
- package/src/form/field_bindings/MarkdownEditorFieldBinding.tsx +13 -7
- package/src/form/field_bindings/MultiSelectFieldBinding.tsx +7 -1
- package/src/form/field_bindings/ReadOnlyFieldBinding.tsx +21 -2
- package/src/form/field_bindings/ReferenceFieldBinding.tsx +1 -1
- package/src/form/field_bindings/RelationFieldBinding.tsx +5 -2
- package/src/form/field_bindings/RepeatFieldBinding.tsx +3 -3
- package/src/form/field_bindings/StorageUploadFieldBinding.tsx +2 -2
- package/src/form/field_bindings/UserSelectFieldBinding.tsx +4 -1
- package/src/form/field_bindings/VectorFieldBinding.tsx +6 -4
- package/src/form/form_utils.ts +78 -0
- package/src/hooks/navigation/useBuildCollectionRegistryController.tsx +29 -5
- package/src/hooks/useEntityDisplay.tsx +262 -0
- package/src/hooks/useEntityDisplayTitle.tsx +27 -38
- package/src/index.ts +9 -1
- package/src/preview/PropertyPreview.tsx +25 -5
- package/src/preview/compact_text.ts +68 -0
- package/src/preview/components/BooleanPreview.tsx +5 -2
- package/src/preview/components/ReferencePreview.tsx +5 -8
- package/src/preview/components/RelationPreview.tsx +8 -7
- package/src/preview/components/StorageThumbnail.tsx +5 -2
- package/src/preview/property_previews/ArrayOfRelationsPreview.tsx +6 -3
- package/src/preview/property_previews/ArrayOfStringsPreview.tsx +26 -1
- package/src/preview/property_previews/ArrayOneOfPreview.tsx +10 -1
- package/src/preview/property_previews/ArrayPropertyPreview.tsx +10 -1
- package/src/preview/property_previews/MapPropertyPreview.tsx +52 -1
- package/src/preview/property_previews/NumberPropertyPreview.tsx +9 -1
- package/src/preview/property_previews/StringPropertyPreview.tsx +8 -5
- package/src/routes/RebaseRoute.tsx +62 -4
- package/src/types/components/PropertyPreviewProps.tsx +21 -0
- package/src/util/navigation_utils.ts +4 -2
- package/src/util/number_format.ts +46 -0
- package/src/util/previews.ts +8 -55
- package/src/util/property_utils.tsx +6 -1
- package/src/util/view_constants.ts +27 -0
- package/src/util/view_mode.ts +36 -0
- package/dist/PropertyEditView-W8I3YV89.js.map +0 -1
- package/dist/RouterCollectionsStudioView-BsLTNoqX.js.map +0 -1
- package/dist/export-J79rMoNj.js.map +0 -1
- package/dist/import-BvYl--U4.js.map +0 -1
- package/dist/util-GiwPgxnL.js.map +0 -1
package/src/components/index.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
export type { EntityViewBindingProps } from "./EntityViewBinding";
|
|
2
2
|
export { EntityViewBinding } from "./EntityViewBinding";
|
|
3
3
|
|
|
4
|
+
export type { EntityDisplayHeaderProps } from "./EntityDisplayHeader";
|
|
5
|
+
export { EntityDisplayHeader, HEADER_DISPLAY_ROLES } from "./EntityDisplayHeader";
|
|
6
|
+
|
|
4
7
|
export * from "./DetailViewBinding";
|
|
5
8
|
|
|
6
9
|
export type { SelectionProps } from "./ReferenceTable/SelectionTableBinding";
|
|
@@ -29,9 +29,26 @@ import {
|
|
|
29
29
|
Tooltip
|
|
30
30
|
} from "@rebasepro/ui";
|
|
31
31
|
import { downloadEntitiesExport } from "./export";
|
|
32
|
+
import { fetchAllEntitiesForExport, MAX_EXPORT_ROWS } from "./fetch_export_data";
|
|
32
33
|
|
|
33
34
|
const DOCS_LIMIT = 500;
|
|
34
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Additional-field builders run per row and may do I/O, so a `Promise.all` over
|
|
38
|
+
* the whole export is one request per row all at once. That was bounded only by
|
|
39
|
+
* the 50-row cap the export used to have; now that the read is paginated, it is
|
|
40
|
+
* bounded here instead.
|
|
41
|
+
*/
|
|
42
|
+
const ADDITIONAL_FIELDS_CONCURRENCY = 50;
|
|
43
|
+
|
|
44
|
+
async function mapInChunks<T, R>(items: T[], fn: (item: T) => Promise<R>, chunkSize = ADDITIONAL_FIELDS_CONCURRENCY): Promise<R[]> {
|
|
45
|
+
const results: R[] = [];
|
|
46
|
+
for (let i = 0; i < items.length; i += chunkSize) {
|
|
47
|
+
results.push(...await Promise.all(items.slice(i, i + chunkSize).map(fn)));
|
|
48
|
+
}
|
|
49
|
+
return results;
|
|
50
|
+
}
|
|
51
|
+
|
|
35
52
|
export function ExportCollectionAction<M extends Record<string, unknown>, USER extends User>({
|
|
36
53
|
collection,
|
|
37
54
|
path,
|
|
@@ -59,6 +76,11 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
59
76
|
const context = useAdminContext<USER>();
|
|
60
77
|
const dataClient = useData();
|
|
61
78
|
|
|
79
|
+
// Said before the download starts, not discovered halfway through it: the
|
|
80
|
+
// walk refuses rather than writing a short file, so the dialog has to name
|
|
81
|
+
// the ceiling it is about to hit.
|
|
82
|
+
const tooManyToExport = collectionEntitiesCount !== undefined && collectionEntitiesCount > MAX_EXPORT_ROWS;
|
|
83
|
+
|
|
62
84
|
const canExport = !exportAllowed || exportAllowed({
|
|
63
85
|
collectionEntitiesCount: collectionEntitiesCount ?? 0,
|
|
64
86
|
path,
|
|
@@ -67,6 +89,7 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
67
89
|
|
|
68
90
|
const [dataLoading, setDataLoading] = React.useState<boolean>(false);
|
|
69
91
|
const [dataLoadingError, setDataLoadingError] = React.useState<Error | undefined>();
|
|
92
|
+
const [progress, setProgress] = React.useState<{ loaded: number, total?: number }>({ loaded: 0 });
|
|
70
93
|
|
|
71
94
|
const [open, setOpen] = React.useState(false);
|
|
72
95
|
|
|
@@ -84,7 +107,7 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
84
107
|
const additionalFields = collection.additionalFields;
|
|
85
108
|
|
|
86
109
|
const resolvedExportColumnsValues: Record<string, any>[] = additionalExportFields
|
|
87
|
-
? await
|
|
110
|
+
? await mapInChunks(entities, async (entity) => {
|
|
88
111
|
return (await Promise.all(additionalExportFields.map(async (column) => {
|
|
89
112
|
return {
|
|
90
113
|
[column.key]: await column.builder({
|
|
@@ -94,11 +117,11 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
94
117
|
};
|
|
95
118
|
}))).reduce((a, b) => ({ ...a,
|
|
96
119
|
...b }), {});
|
|
97
|
-
})
|
|
120
|
+
})
|
|
98
121
|
: [];
|
|
99
122
|
|
|
100
123
|
const resolvedColumnsValues: Record<string, any>[] = additionalFields
|
|
101
|
-
? await
|
|
124
|
+
? await mapInChunks(entities, async (entity) => {
|
|
102
125
|
return (await Promise.all(additionalFields
|
|
103
126
|
.map(async (field) => {
|
|
104
127
|
if (!field.value)
|
|
@@ -111,64 +134,74 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
111
134
|
};
|
|
112
135
|
}))).reduce((a, b) => ({ ...a,
|
|
113
136
|
...b }), {});
|
|
114
|
-
})
|
|
137
|
+
})
|
|
115
138
|
: [];
|
|
116
139
|
return [...resolvedExportColumnsValues, ...resolvedColumnsValues];
|
|
117
140
|
}, [exportConfig?.additionalFields]);
|
|
118
141
|
|
|
119
142
|
const doDownload = useCallback(async (collection: AdminCollection<M>,
|
|
120
|
-
exportConfig: ExportConfig<any> | undefined) => {
|
|
143
|
+
exportConfig: ExportConfig<any> | undefined): Promise<boolean> => {
|
|
121
144
|
|
|
122
145
|
onAnalyticsEvent?.("export_collection", {
|
|
123
146
|
collection: collection.slug
|
|
124
147
|
});
|
|
125
148
|
setDataLoading(true);
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
149
|
+
setDataLoadingError(undefined);
|
|
150
|
+
setProgress({ loaded: 0 });
|
|
151
|
+
try {
|
|
152
|
+
// Paginated, not `find({})`: an absent limit resolves to 50 rows
|
|
153
|
+
// server-side, so the export used to be the first page of the
|
|
154
|
+
// collection under a filename that read like all of it.
|
|
155
|
+
const data = await fetchAllEntitiesForExport<M>({
|
|
156
|
+
accessor: dataClient.collection(path) as { find: (params?: any) => Promise<any> },
|
|
157
|
+
onProgress: (loaded, total) => setProgress({ loaded,
|
|
158
|
+
total })
|
|
159
|
+
});
|
|
160
|
+
const additionalData = await fetchAdditionalFields(data);
|
|
161
|
+
const additionalHeaders = [
|
|
162
|
+
...exportConfig?.additionalFields?.map(column => column.key) ?? [],
|
|
163
|
+
...collection.additionalFields?.map(field => field.key) ?? []
|
|
164
|
+
];
|
|
165
|
+
|
|
166
|
+
const defaultValues = includeUndefinedValues ? getDefaultValuesFor(collection.properties) : undefined;
|
|
167
|
+
const dataWithDefaults = defaultValues
|
|
168
|
+
? data.map(entity => ({
|
|
169
|
+
...entity,
|
|
170
|
+
values: { ...defaultValues,
|
|
142
171
|
...entity.values }
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
.
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
172
|
+
}))
|
|
173
|
+
: data;
|
|
174
|
+
downloadEntitiesExport({
|
|
175
|
+
data: dataWithDefaults,
|
|
176
|
+
additionalData,
|
|
177
|
+
properties: collection.properties,
|
|
178
|
+
propertiesOrder: collection.propertiesOrder,
|
|
179
|
+
name: collection.name,
|
|
180
|
+
flattenArrays,
|
|
181
|
+
additionalHeaders,
|
|
182
|
+
exportType,
|
|
183
|
+
dateExportType
|
|
184
|
+
});
|
|
185
|
+
onAnalyticsEvent?.("export_collection_success", {
|
|
186
|
+
collection: collection.slug
|
|
187
|
+
});
|
|
188
|
+
return true;
|
|
189
|
+
} catch (e) {
|
|
190
|
+
console.error("Error loading export data", e);
|
|
191
|
+
setDataLoadingError(e as Error);
|
|
192
|
+
return false;
|
|
193
|
+
} finally {
|
|
194
|
+
setDataLoading(false);
|
|
195
|
+
}
|
|
166
196
|
|
|
167
197
|
}, [onAnalyticsEvent, dataClient, path, fetchAdditionalFields, includeUndefinedValues, flattenArrays, exportType, dateExportType]);
|
|
168
198
|
|
|
169
199
|
const onOkClicked = useCallback(() => {
|
|
170
|
-
|
|
171
|
-
|
|
200
|
+
// The dialog stays open until the walk finishes: it is the only place a
|
|
201
|
+
// multi-page export reports progress, or reports that it failed.
|
|
202
|
+
doDownload(collection, exportConfig).then((downloaded) => {
|
|
203
|
+
if (downloaded) handleClose();
|
|
204
|
+
});
|
|
172
205
|
}, [doDownload, collection, exportConfig, handleClose]);
|
|
173
206
|
|
|
174
207
|
return <>
|
|
@@ -194,13 +227,23 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
194
227
|
|
|
195
228
|
<div>{t("download_table_csv")}</div>
|
|
196
229
|
|
|
197
|
-
{collectionEntitiesCount !== undefined && collectionEntitiesCount > DOCS_LIMIT &&
|
|
230
|
+
{collectionEntitiesCount !== undefined && collectionEntitiesCount > DOCS_LIMIT && !tooManyToExport &&
|
|
198
231
|
<Alert color={"warning"}>
|
|
199
232
|
<div>
|
|
200
233
|
{t("large_number_of_documents", { count: collectionEntitiesCount.toString() })}
|
|
201
234
|
</div>
|
|
202
235
|
</Alert>}
|
|
203
236
|
|
|
237
|
+
{tooManyToExport &&
|
|
238
|
+
<Alert color={"error"}>
|
|
239
|
+
<div>
|
|
240
|
+
{t("too_many_documents_to_export", {
|
|
241
|
+
count: (collectionEntitiesCount ?? 0).toString(),
|
|
242
|
+
limit: MAX_EXPORT_ROWS.toString()
|
|
243
|
+
})}
|
|
244
|
+
</div>
|
|
245
|
+
</Alert>}
|
|
246
|
+
|
|
204
247
|
<div className={"flex flex-row gap-4"}>
|
|
205
248
|
<div className={"p-4 flex flex-col"}>
|
|
206
249
|
<RadioGroup value={exportType} onValueChange={(v) => setExportType(v as "csv" | "json")}>
|
|
@@ -242,6 +285,10 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
242
285
|
onValueChange={setIncludeUndefinedValues}
|
|
243
286
|
label={t("include_undefined_values")}/>
|
|
244
287
|
|
|
288
|
+
{dataLoadingError && <Alert color={"error"}>
|
|
289
|
+
<div>{dataLoadingError.message}</div>
|
|
290
|
+
</Alert>}
|
|
291
|
+
|
|
245
292
|
{!canExport && notAllowedView}
|
|
246
293
|
|
|
247
294
|
</DialogContent>
|
|
@@ -250,13 +297,19 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
|
|
|
250
297
|
|
|
251
298
|
{dataLoading && <CircularProgress size={"smallest"}/>}
|
|
252
299
|
|
|
300
|
+
{dataLoading && <Label>
|
|
301
|
+
{progress.total !== undefined
|
|
302
|
+
? `${progress.loaded} / ${progress.total}`
|
|
303
|
+
: `${progress.loaded}`}
|
|
304
|
+
</Label>}
|
|
305
|
+
|
|
253
306
|
<Button onClick={handleClose}
|
|
254
307
|
variant={"text"}>
|
|
255
308
|
{t("cancel")}
|
|
256
309
|
</Button>
|
|
257
310
|
|
|
258
311
|
<Button onClick={onOkClicked}
|
|
259
|
-
disabled={dataLoading || !canExport}>
|
|
312
|
+
disabled={dataLoading || !canExport || tooManyToExport}>
|
|
260
313
|
{t("download")}
|
|
261
314
|
</Button>
|
|
262
315
|
|
|
@@ -154,6 +154,24 @@ function getHeaders(property: Property, propertyKey: string, prefix = ""): Heade
|
|
|
154
154
|
}
|
|
155
155
|
}
|
|
156
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Read whatever a `date` column actually arrived as into a `Date`.
|
|
159
|
+
*
|
|
160
|
+
* The driver/WebSocket path revives `{ __type: "date" }` into a `Date`; the REST
|
|
161
|
+
* path — which is the one the admin is on — leaves an ISO string, and an epoch
|
|
162
|
+
* number is what a re-imported export carries. Returns `undefined` for anything
|
|
163
|
+
* that is not a date, so the caller can pass the raw value through.
|
|
164
|
+
*/
|
|
165
|
+
function toExportableDate(inputValue: unknown): Date | undefined {
|
|
166
|
+
if (inputValue instanceof Date) return Number.isNaN(inputValue.getTime()) ? undefined : inputValue;
|
|
167
|
+
if (typeof inputValue === "number" && Number.isFinite(inputValue)) return new Date(inputValue);
|
|
168
|
+
if (typeof inputValue === "string" && inputValue.trim() !== "") {
|
|
169
|
+
const parsed = new Date(inputValue);
|
|
170
|
+
return Number.isNaN(parsed.getTime()) ? undefined : parsed;
|
|
171
|
+
}
|
|
172
|
+
return undefined;
|
|
173
|
+
}
|
|
174
|
+
|
|
157
175
|
function processValueForExport(inputValue: unknown,
|
|
158
176
|
property: Property,
|
|
159
177
|
exportType: "csv" | "json",
|
|
@@ -181,8 +199,17 @@ function processValueForExport(inputValue: unknown,
|
|
|
181
199
|
} else if (property.type === "reference" && inputValue && typeof inputValue === "object" && "isEntityReference" in inputValue && typeof (inputValue as EntityReference).isEntityReference === "function" && (inputValue as EntityReference).isEntityReference()) {
|
|
182
200
|
const ref = inputValue ? inputValue as EntityReference : undefined;
|
|
183
201
|
value = ref ? ref.fullPath : null;
|
|
184
|
-
} else if (property.type === "date"
|
|
185
|
-
|
|
202
|
+
} else if (property.type === "date") {
|
|
203
|
+
// Not `instanceof Date`: the admin reads over REST, and the REST row
|
|
204
|
+
// pipeline hands dates back as the database wrote them — a string. That
|
|
205
|
+
// made this branch dead on every live export, so both radio options
|
|
206
|
+
// produced identical files. Normalise here, where the declared type says
|
|
207
|
+
// the column is a date, and leave anything unparseable untouched rather
|
|
208
|
+
// than exporting an invalid date as `null`.
|
|
209
|
+
const date = toExportableDate(inputValue);
|
|
210
|
+
value = date
|
|
211
|
+
? (dateExportType === "timestamp" ? date.getTime() : date.toISOString())
|
|
212
|
+
: inputValue;
|
|
186
213
|
} else {
|
|
187
214
|
value = inputValue;
|
|
188
215
|
}
|
|
@@ -209,15 +236,40 @@ function processValuesForExport<M extends Record<string, unknown>>
|
|
|
209
236
|
...updatedValues };
|
|
210
237
|
}
|
|
211
238
|
|
|
212
|
-
|
|
239
|
+
/**
|
|
240
|
+
* Characters that make a spreadsheet read a cell as a formula rather than text.
|
|
241
|
+
*
|
|
242
|
+
* Quoting is CSV *escaping* and is not enough: Excel, LibreOffice Calc and
|
|
243
|
+
* Sheets strip the CSV quoting at parse time and then evaluate a cell whose
|
|
244
|
+
* first character is one of these — `=HYPERLINK(…)` exfiltrates neighbouring
|
|
245
|
+
* cells on click, `=cmd|'…'!A0` is worse.
|
|
246
|
+
*/
|
|
247
|
+
const FORMULA_TRIGGERS = ["=", "+", "-", "@", "\t", "\r"];
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Neutralise a cell that a spreadsheet would otherwise evaluate, by prefixing a
|
|
251
|
+
* single quote — the standard OWASP mitigation. The apostrophe is consumed by
|
|
252
|
+
* the spreadsheet as "treat the rest as text", so the value reads back intact.
|
|
253
|
+
*/
|
|
254
|
+
export function escapeCsvFormula(value: string): string {
|
|
255
|
+
const first = value.charAt(0);
|
|
256
|
+
if (!FORMULA_TRIGGERS.includes(first)) return value;
|
|
257
|
+
// A signed number is not a formula, and prefixing it would turn every
|
|
258
|
+
// negative amount in the file into text. `-1+1` is still neutralised,
|
|
259
|
+
// because it is not a number.
|
|
260
|
+
if ((first === "-" || first === "+") && value.trim() !== "" && Number.isFinite(Number(value))) return value;
|
|
261
|
+
return "'" + value;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function toCSVCell(v: unknown): string {
|
|
265
|
+
if (v === null || v === undefined) return "";
|
|
266
|
+
const s = Array.isArray(v) ? JSON.stringify(v) : String(v);
|
|
267
|
+
return "\"" + escapeCsvFormula(s).replaceAll("\"", "\"\"") + "\"";
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
export function entryToCSVRow(entry: unknown[]) {
|
|
213
271
|
return entry
|
|
214
|
-
.map(
|
|
215
|
-
if (v === null || v === undefined) return "";
|
|
216
|
-
if (Array.isArray(v))
|
|
217
|
-
return "\"" + JSON.stringify(v).replaceAll("\"", "\"\"") + "\"";
|
|
218
|
-
const s = String(v);
|
|
219
|
-
return "\"" + s.replaceAll("\"", "\"\"") + "\"";
|
|
220
|
-
})
|
|
272
|
+
.map(toCSVCell)
|
|
221
273
|
.join(",") + "\r\n";
|
|
222
274
|
}
|
|
223
275
|
|
|
@@ -236,12 +288,7 @@ export function downloadDataAsCsv(data: object[], name: string) {
|
|
|
236
288
|
const headers = Object.keys(data[0]);
|
|
237
289
|
const csvContent = [
|
|
238
290
|
headers.join(","),
|
|
239
|
-
...data.map(row => headers.map(header =>
|
|
240
|
-
const value = (row as Record<string, unknown>)[header];
|
|
241
|
-
if (value === null || value === undefined) return "";
|
|
242
|
-
if (Array.isArray(value)) return `"${JSON.stringify(value).replace(/"/g, '""')}"`;
|
|
243
|
-
return `"${String(value).replace(/"/g, '""')}"`;
|
|
244
|
-
}).join(","))
|
|
291
|
+
...data.map(row => headers.map(header => toCSVCell((row as Record<string, unknown>)[header])).join(","))
|
|
245
292
|
].join("\r\n");
|
|
246
293
|
|
|
247
294
|
downloadBlob([csvContent], `${name}.csv`, "text/csv");
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { Entity, FindParams, FindResponse } from "@rebasepro/types";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Rows asked for per request.
|
|
5
|
+
*
|
|
6
|
+
* Kept at or below the server's `MAX_LIST_LIMIT`, which is now the largest page
|
|
7
|
+
* a read will serve *and refuses* anything larger — it used to clamp silently,
|
|
8
|
+
* and asking for 100 000 while receiving 1 000 looked exactly like a collection
|
|
9
|
+
* with 1 000 rows in it. The walk still never trusts the page size it asked for:
|
|
10
|
+
* it advances by the rows it was actually handed.
|
|
11
|
+
*/
|
|
12
|
+
export const EXPORT_PAGE_SIZE = 500;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Most rows one browser-side export will materialise.
|
|
16
|
+
*
|
|
17
|
+
* Everything here is built in memory and handed to a Blob, so there is a
|
|
18
|
+
* ceiling whether or not it is named. Naming it means going over it **throws**
|
|
19
|
+
* instead of writing a short file that reads like the whole collection.
|
|
20
|
+
*/
|
|
21
|
+
export const MAX_EXPORT_ROWS = 100_000;
|
|
22
|
+
|
|
23
|
+
export interface FetchAllForExportParams<M extends Record<string, unknown>> {
|
|
24
|
+
/** The collection accessor to read through — `dataClient.collection(path)`. */
|
|
25
|
+
accessor: { find(params?: FindParams<M>): Promise<FindResponse<M>> };
|
|
26
|
+
/** Query to export. The window (`limit` / `offset` / `page`) is owned by the walk. */
|
|
27
|
+
params?: Omit<FindParams<M>, "limit" | "offset" | "page">;
|
|
28
|
+
pageSize?: number;
|
|
29
|
+
maxRows?: number;
|
|
30
|
+
/** Called after every page with the rows so far, and the total when the server reports one. */
|
|
31
|
+
onProgress?: (loaded: number, total: number | undefined) => void;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Read **every** row matching a query, one page at a time.
|
|
36
|
+
*
|
|
37
|
+
* `find({})` sends no `limit`, and an absent limit is resolved server-side to
|
|
38
|
+
* `DEFAULT_LIST_LIMIT` — 50 rows. The export used to hand those 50 rows back as
|
|
39
|
+
* `<collection>.csv` with nothing saying it was a sample, next to a dialog
|
|
40
|
+
* warning that the collection had 100 000 documents.
|
|
41
|
+
*
|
|
42
|
+
* Termination is driven by the server's `meta.hasMore` where it is reported and
|
|
43
|
+
* by an empty page otherwise — never by comparing a page's length against the
|
|
44
|
+
* requested limit, since a final page that happens to be exactly full is
|
|
45
|
+
* indistinguishable that way and a walk that stops there drops rows.
|
|
46
|
+
*
|
|
47
|
+
* @throws When more than `maxRows` rows match. A truncated export is worse than
|
|
48
|
+
* no export: it is indistinguishable from a complete one.
|
|
49
|
+
*/
|
|
50
|
+
export async function fetchAllEntitiesForExport<M extends Record<string, unknown>>({
|
|
51
|
+
accessor,
|
|
52
|
+
params,
|
|
53
|
+
pageSize = EXPORT_PAGE_SIZE,
|
|
54
|
+
maxRows = MAX_EXPORT_ROWS,
|
|
55
|
+
onProgress
|
|
56
|
+
}: FetchAllForExportParams<M>): Promise<Entity<M>[]> {
|
|
57
|
+
|
|
58
|
+
const rows: Entity<M>[] = [];
|
|
59
|
+
let offset = 0;
|
|
60
|
+
|
|
61
|
+
for (;;) {
|
|
62
|
+
const res = await accessor.find({
|
|
63
|
+
...params,
|
|
64
|
+
limit: pageSize,
|
|
65
|
+
offset
|
|
66
|
+
} as FindParams<M>);
|
|
67
|
+
|
|
68
|
+
const page = res?.data ?? [];
|
|
69
|
+
rows.push(...page);
|
|
70
|
+
onProgress?.(rows.length, res?.meta?.total);
|
|
71
|
+
|
|
72
|
+
if (rows.length > maxRows) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`This export would contain more than ${maxRows.toLocaleString()} rows, which is more than the ` +
|
|
75
|
+
"browser can build in one file. Filter the collection down, or export it from the server."
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// An empty page ends the walk whatever the metadata says, so a server
|
|
80
|
+
// that keeps answering `hasMore` cannot spin forever.
|
|
81
|
+
if (page.length === 0) break;
|
|
82
|
+
if (res?.meta && !res.meta.hasMore) break;
|
|
83
|
+
|
|
84
|
+
// By rows received, not by `pageSize`: a server whose ceiling is lower
|
|
85
|
+
// than the page we asked for would otherwise skip everything it clamped.
|
|
86
|
+
offset += page.length;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return rows;
|
|
90
|
+
}
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { useData } from "@rebasepro/app";
|
|
2
|
-
import { Entity } from "@rebasepro/types";
|
|
3
|
-
import { RebaseData } from "@rebasepro/types";
|
|
4
2
|
import { Button, CenteredView, CircularProgress, Typography } from "@rebasepro/ui";
|
|
5
3
|
import { useEffect, useRef, useState } from "react";
|
|
6
4
|
import { ImportConfig } from "../types";
|
|
7
5
|
import type { AdminCollection } from "@rebasepro/admin-types";
|
|
6
|
+
import { IMPORT_BATCH_SIZE, ImportSaveError, saveImportedEntities } from "../utils/save_entities";
|
|
8
7
|
|
|
9
8
|
export function ImportSaveInProgress<C extends AdminCollection<any>>
|
|
10
9
|
({
|
|
@@ -20,11 +19,16 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
|
|
|
20
19
|
onImportSuccess: (collection: C) => void
|
|
21
20
|
}) {
|
|
22
21
|
|
|
23
|
-
const [errorSaving, setErrorSaving] = useState<
|
|
22
|
+
const [errorSaving, setErrorSaving] = useState<ImportSaveError | undefined>(undefined);
|
|
24
23
|
const dataClient = useData();
|
|
25
24
|
|
|
26
25
|
const savingRef = useRef<boolean>(false);
|
|
27
26
|
|
|
27
|
+
// Rows confirmed as written. A retry resumes here rather than re-sending
|
|
28
|
+
// rows that now exist and conflicting on its first batch, forever.
|
|
29
|
+
const committedRef = useRef<number>(0);
|
|
30
|
+
const bulkUnsupportedRef = useRef<{ current: boolean }>({ current: false });
|
|
31
|
+
|
|
28
32
|
const [processedEntities, setProcessedEntities] = useState<number>(0);
|
|
29
33
|
|
|
30
34
|
function save() {
|
|
@@ -33,20 +37,29 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
|
|
|
33
37
|
return;
|
|
34
38
|
|
|
35
39
|
savingRef.current = true;
|
|
40
|
+
setErrorSaving(undefined);
|
|
36
41
|
|
|
37
|
-
|
|
42
|
+
saveImportedEntities(
|
|
38
43
|
dataClient,
|
|
39
|
-
collection,
|
|
40
44
|
path,
|
|
41
45
|
importConfig.entities,
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
46
|
+
{
|
|
47
|
+
offset: committedRef.current,
|
|
48
|
+
batchSize: IMPORT_BATCH_SIZE,
|
|
49
|
+
onBatchCommitted: (written) => {
|
|
50
|
+
committedRef.current = written;
|
|
51
|
+
setProcessedEntities(written);
|
|
52
|
+
},
|
|
53
|
+
bulkUnsupported: bulkUnsupportedRef.current
|
|
54
|
+
}
|
|
45
55
|
).then(() => {
|
|
46
56
|
onImportSuccess(collection);
|
|
47
57
|
savingRef.current = false;
|
|
48
58
|
}).catch((e) => {
|
|
49
|
-
|
|
59
|
+
const error = e as ImportSaveError;
|
|
60
|
+
committedRef.current = error.committed ?? committedRef.current;
|
|
61
|
+
setProcessedEntities(committedRef.current);
|
|
62
|
+
setErrorSaving(error);
|
|
50
63
|
savingRef.current = false;
|
|
51
64
|
});
|
|
52
65
|
}
|
|
@@ -56,6 +69,10 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
|
|
|
56
69
|
}, []);
|
|
57
70
|
|
|
58
71
|
if (errorSaving) {
|
|
72
|
+
const total = importConfig.entities.length;
|
|
73
|
+
const failedRows = errorSaving.failedTo - errorSaving.failedFrom > 1
|
|
74
|
+
? `rows ${errorSaving.failedFrom + 1}–${errorSaving.failedTo}`
|
|
75
|
+
: `row ${errorSaving.failedFrom + 1}`;
|
|
59
76
|
return (
|
|
60
77
|
<CenteredView className={"flex flex-col gap-4 items-center"}>
|
|
61
78
|
<Typography variant={"h6"}>
|
|
@@ -65,10 +82,18 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
|
|
|
65
82
|
<Typography variant={"body2"} color={"error"}>
|
|
66
83
|
{errorSaving.message}
|
|
67
84
|
</Typography>
|
|
85
|
+
|
|
86
|
+
<Typography variant={"body2"}>
|
|
87
|
+
{errorSaving.committed} of {total} rows were imported.
|
|
88
|
+
The import stopped on {failedRows}, which {errorSaving.failedTo - errorSaving.failedFrom > 1 ? "were" : "was"} not written.
|
|
89
|
+
</Typography>
|
|
90
|
+
|
|
68
91
|
<Button
|
|
69
92
|
onClick={save}
|
|
70
93
|
>
|
|
71
|
-
|
|
94
|
+
{errorSaving.committed > 0
|
|
95
|
+
? `Retry from row ${errorSaving.committed + 1}`
|
|
96
|
+
: "Retry"}
|
|
72
97
|
</Button>
|
|
73
98
|
</CenteredView>
|
|
74
99
|
);
|
|
@@ -94,29 +119,3 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
|
|
|
94
119
|
);
|
|
95
120
|
|
|
96
121
|
}
|
|
97
|
-
|
|
98
|
-
function saveDataBatch(dataClient: RebaseData,
|
|
99
|
-
collection: AdminCollection,
|
|
100
|
-
path: string,
|
|
101
|
-
data: Partial<Entity<any>>[],
|
|
102
|
-
offset = 0,
|
|
103
|
-
batchSize = 25,
|
|
104
|
-
onProgressUpdate: (progress: number) => void): Promise<void> {
|
|
105
|
-
|
|
106
|
-
console.debug("Saving imported data", offset, batchSize);
|
|
107
|
-
|
|
108
|
-
const batch = data.slice(offset, offset + batchSize);
|
|
109
|
-
return Promise.all(batch.map(d =>
|
|
110
|
-
dataClient.collection(path).create(
|
|
111
|
-
d.values,
|
|
112
|
-
d.id
|
|
113
|
-
)))
|
|
114
|
-
.then(() => {
|
|
115
|
-
if (offset + batchSize < data.length) {
|
|
116
|
-
onProgressUpdate(offset + batchSize);
|
|
117
|
-
return saveDataBatch(dataClient, collection, path, data, offset + batchSize, batchSize, onProgressUpdate);
|
|
118
|
-
}
|
|
119
|
-
onProgressUpdate(data.length);
|
|
120
|
-
return Promise.resolve();
|
|
121
|
-
});
|
|
122
|
-
}
|
|
@@ -35,7 +35,7 @@ import { DataNewPropertiesMapping } from "../components/DataNewPropertiesMapping
|
|
|
35
35
|
import { ImportFileUpload } from "../components/ImportFileUpload";
|
|
36
36
|
import { ImportSaveInProgress } from "../components/ImportSaveInProgress";
|
|
37
37
|
import { ImportConfig } from "../types";
|
|
38
|
-
import { slugify } from "@rebasepro/utils";
|
|
38
|
+
import { isPrototypePollutingKey, slugify } from "@rebasepro/utils";
|
|
39
39
|
|
|
40
40
|
type ImportState = "initial" | "mapping" | "preview" | "import_data_saving";
|
|
41
41
|
|
|
@@ -400,7 +400,15 @@ export function ImportDataPreview<M extends Record<string, unknown>>({
|
|
|
400
400
|
return <CollectionTableBinding
|
|
401
401
|
title={<div>
|
|
402
402
|
<Typography variant={"subtitle2"}>Imported data preview</Typography>
|
|
403
|
-
|
|
403
|
+
{/* Conditional because it is only true when a column was chosen as
|
|
404
|
+
the id: without one, every row is created and nothing can be
|
|
405
|
+
overwritten. With one, the import asks for an upsert — which it
|
|
406
|
+
did not, back when this sentence was unconditional and false. */}
|
|
407
|
+
<Typography variant={"caption"}>
|
|
408
|
+
{importConfig.idColumn
|
|
409
|
+
? "Entities with the same id will be overwritten"
|
|
410
|
+
: "All rows will be imported as new entities"}
|
|
411
|
+
</Typography>
|
|
404
412
|
</div>}
|
|
405
413
|
tableController={{
|
|
406
414
|
data: importConfig.entities,
|
|
@@ -421,6 +429,9 @@ function buildHeadersMappingFromData(objArr: object[], properties?: Properties)
|
|
|
421
429
|
const headersMapping: Record<string, string> = {};
|
|
422
430
|
objArr.filter(Boolean).forEach((obj) => {
|
|
423
431
|
Object.keys(obj).forEach((key) => {
|
|
432
|
+
// The keys are the uploaded file's header row; `headersMapping[key] = …`
|
|
433
|
+
// is the same setter the import pipeline refuses elsewhere.
|
|
434
|
+
if (isPrototypePollutingKey(key)) return;
|
|
424
435
|
const child = (obj as Record<string, unknown>)[key];
|
|
425
436
|
if (child != null && typeof child === "object" && !Array.isArray(child)) {
|
|
426
437
|
const childProperty = properties?.[key];
|