@rebasepro/admin 0.13.0 → 0.13.1-canary.g06dbe5b

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/dist/{CollectionEditorDialog-CjXkWZ64.js → CollectionEditorDialog-C3DbFNqp.js} +3 -3
  2. package/dist/{CollectionEditorDialog-CjXkWZ64.js.map → CollectionEditorDialog-C3DbFNqp.js.map} +1 -1
  3. package/dist/{PropertyEditView-W8I3YV89.js → PropertyEditView-DFH8rXRN.js} +5 -5
  4. package/dist/PropertyEditView-DFH8rXRN.js.map +1 -0
  5. package/dist/{RouterCollectionsStudioView-BsLTNoqX.js → RouterCollectionsStudioView-DiUDOu4b.js} +9 -13
  6. package/dist/RouterCollectionsStudioView-DiUDOu4b.js.map +1 -0
  7. package/dist/collection_editor/serializable_types.d.ts +1 -0
  8. package/dist/collection_editor_ui.js +4 -4
  9. package/dist/components/AdditionalFieldValue.d.ts +25 -0
  10. package/dist/components/ClearFilterSortButton.d.ts +7 -1
  11. package/dist/components/CollectionViewBinding/CollectionViewBinding.d.ts +16 -0
  12. package/dist/components/CollectionViewBinding/EntityCardBinding.d.ts +3 -1
  13. package/dist/components/CollectionViewBinding/SearchExplanation.d.ts +48 -0
  14. package/dist/components/CollectionViewBinding/SearchHighlight.d.ts +101 -0
  15. package/dist/components/CollectionViewBinding/SlotValue.d.ts +63 -0
  16. package/dist/components/CollectionViewBinding/SortButton.d.ts +7 -1
  17. package/dist/components/CollectionViewBinding/SplitListCloseButton.d.ts +17 -4
  18. package/dist/components/CollectionViewBinding/SplitListShowButton.d.ts +12 -0
  19. package/dist/components/CollectionViewBinding/usePreviewSlots.d.ts +55 -5
  20. package/dist/components/DetailViewBinding.d.ts +15 -0
  21. package/dist/components/DrawerNavigationGroup.d.ts +9 -6
  22. package/dist/components/DrawerNavigationItem.d.ts +16 -6
  23. package/dist/components/EditViewBinding.d.ts +30 -1
  24. package/dist/components/EntityDisplayHeader.d.ts +53 -0
  25. package/dist/components/EntityIdentityBar.d.ts +43 -1
  26. package/dist/components/EntityViewBinding.d.ts +60 -6
  27. package/dist/components/RebaseLayout.d.ts +1 -1
  28. package/dist/components/RebaseShell.d.ts +1 -1
  29. package/dist/components/RelationSelector.d.ts +8 -1
  30. package/dist/components/UserSelector.d.ts +8 -1
  31. package/dist/components/app/Scaffold.d.ts +9 -3
  32. package/dist/components/index.d.ts +2 -0
  33. package/dist/data_export/export/export.d.ts +7 -0
  34. package/dist/data_export/export/fetch_export_data.d.ts +48 -0
  35. package/dist/data_export/export/index.d.ts +1 -0
  36. package/dist/data_import/utils/csv.d.ts +30 -0
  37. package/dist/data_import/utils/index.d.ts +2 -0
  38. package/dist/data_import/utils/save_entities.d.ts +38 -0
  39. package/dist/data_import/utils/transforms.d.ts +9 -0
  40. package/dist/{export-J79rMoNj.js → export-Bd5i6Uvm.js} +175 -39
  41. package/dist/export-Bd5i6Uvm.js.map +1 -0
  42. package/dist/form/components/FieldBlock.d.ts +58 -4
  43. package/dist/form/components/FormRail.d.ts +13 -0
  44. package/dist/form/components/FormSections.d.ts +15 -1
  45. package/dist/form/components/useRailVisible.d.ts +5 -4
  46. package/dist/form/form_utils.d.ts +38 -1
  47. package/dist/{history-D7L_JlLO.js → history-D9R6LUrH.js} +4 -4
  48. package/dist/{history-D7L_JlLO.js.map → history-D9R6LUrH.js.map} +1 -1
  49. package/dist/hooks/useEntityDisplay.d.ts +102 -0
  50. package/dist/hooks/useEntityDisplayTitle.d.ts +18 -9
  51. package/dist/{import-BvYl--U4.js → import-pt_j-0G3.js} +6 -5
  52. package/dist/import-pt_j-0G3.js.map +1 -0
  53. package/dist/index.d.ts +2 -2
  54. package/dist/index.js +158 -60
  55. package/dist/index.js.map +1 -1
  56. package/dist/preview/compact_text.d.ts +23 -0
  57. package/dist/preview/components/BooleanPreview.d.ts +3 -1
  58. package/dist/preview/property_previews/ArrayOfStringsPreview.d.ts +1 -1
  59. package/dist/preview/property_previews/ArrayOneOfPreview.d.ts +1 -1
  60. package/dist/preview/property_previews/ArrayPropertyPreview.d.ts +1 -1
  61. package/dist/preview/property_previews/MapPropertyPreview.d.ts +1 -1
  62. package/dist/preview/property_previews/StringPropertyPreview.d.ts +1 -1
  63. package/dist/routes/RebaseRoute.d.ts +1 -1
  64. package/dist/types/components/PropertyPreviewProps.d.ts +19 -0
  65. package/dist/util/number_format.d.ts +14 -0
  66. package/dist/util/previews.d.ts +9 -2
  67. package/dist/util/view_constants.d.ts +24 -0
  68. package/dist/util/view_mode.d.ts +18 -0
  69. package/dist/{util-GiwPgxnL.js → util-nX_-JRuF.js} +3984 -1774
  70. package/dist/util-nX_-JRuF.js.map +1 -0
  71. package/package.json +10 -9
  72. package/src/collection_editor/serializable_types.ts +1 -0
  73. package/src/collection_editor/ui/collection_editor/CollectionsStudioView.tsx +9 -11
  74. package/src/collection_editor/ui/collection_editor/properties/MapPropertyField.tsx +1 -1
  75. package/src/collection_editor/ui/collection_editor/properties/advanced/AdvancedPropertyValidation.tsx +5 -2
  76. package/src/collection_editor/useLocalCollectionsConfigController.tsx +10 -3
  77. package/src/components/AdditionalFieldValue.tsx +68 -0
  78. package/src/components/ClearFilterSortButton.tsx +16 -7
  79. package/src/components/CollectionTableBinding/CollectionTableBinding.tsx +7 -8
  80. package/src/components/CollectionTableBinding/PropertyTableCell.tsx +2 -2
  81. package/src/components/CollectionTableBinding/column_utils.tsx +15 -6
  82. package/src/components/CollectionTableBinding/internal/popup_field/PopupFormField.tsx +2 -2
  83. package/src/components/CollectionTableBinding/table_bindings.tsx +7 -1
  84. package/src/components/CollectionViewBinding/BoardCardBinding.tsx +26 -27
  85. package/src/components/CollectionViewBinding/CollectionBoardViewBinding.tsx +13 -7
  86. package/src/components/CollectionViewBinding/CollectionCardViewBinding.tsx +2 -1
  87. package/src/components/CollectionViewBinding/CollectionListViewBinding.tsx +822 -354
  88. package/src/components/CollectionViewBinding/CollectionViewActions.tsx +1 -1
  89. package/src/components/CollectionViewBinding/CollectionViewBinding.tsx +51 -12
  90. package/src/components/CollectionViewBinding/CollectionViewStartActions.tsx +9 -0
  91. package/src/components/CollectionViewBinding/EntityCardBinding.tsx +51 -39
  92. package/src/components/CollectionViewBinding/SearchExplanation.tsx +106 -0
  93. package/src/components/CollectionViewBinding/SearchHighlight.tsx +311 -0
  94. package/src/components/CollectionViewBinding/SlotValue.tsx +156 -0
  95. package/src/components/CollectionViewBinding/SortButton.tsx +16 -4
  96. package/src/components/CollectionViewBinding/SplitListCloseButton.tsx +23 -7
  97. package/src/components/CollectionViewBinding/SplitListShowButton.tsx +28 -0
  98. package/src/components/CollectionViewBinding/SplitListView.tsx +47 -7
  99. package/src/components/CollectionViewBinding/ViewModeToggle.tsx +10 -5
  100. package/src/components/CollectionViewBinding/useBoardDataController.tsx +23 -6
  101. package/src/components/CollectionViewBinding/usePreviewSlots.ts +183 -64
  102. package/src/components/CollectionViewBinding/utils.ts +7 -4
  103. package/src/components/DefaultDrawer.tsx +19 -23
  104. package/src/components/DetailViewBinding.tsx +109 -62
  105. package/src/components/DrawerNavigationGroup.tsx +16 -39
  106. package/src/components/DrawerNavigationItem.tsx +44 -16
  107. package/src/components/EditViewBinding.tsx +233 -50
  108. package/src/components/EntityDisplayHeader.tsx +170 -0
  109. package/src/components/EntityIdentityBar.tsx +163 -23
  110. package/src/components/EntityInspector.tsx +24 -6
  111. package/src/components/EntityPreviewBinding.tsx +58 -18
  112. package/src/components/EntityViewBinding.tsx +344 -41
  113. package/src/components/InlineEntityPreview.tsx +10 -10
  114. package/src/components/RebaseLayout.tsx +2 -2
  115. package/src/components/RebaseShell.tsx +2 -2
  116. package/src/components/ReferenceTable/SelectionTableBinding.tsx +4 -2
  117. package/src/components/RelationSelector.tsx +35 -10
  118. package/src/components/SideDialogs.tsx +15 -3
  119. package/src/components/SidePanelBinding.tsx +122 -66
  120. package/src/components/UserSelector.tsx +67 -13
  121. package/src/components/app/Scaffold.tsx +112 -10
  122. package/src/components/history/LastEditedByIndicator.tsx +7 -12
  123. package/src/components/index.ts +3 -0
  124. package/src/data_export/export/ExportCollectionAction.tsx +101 -48
  125. package/src/data_export/export/export.ts +63 -16
  126. package/src/data_export/export/fetch_export_data.ts +90 -0
  127. package/src/data_export/export/index.ts +1 -0
  128. package/src/data_import/components/ImportSaveInProgress.tsx +35 -36
  129. package/src/data_import/import/ImportCollectionAction.tsx +13 -2
  130. package/src/data_import/utils/csv.ts +153 -0
  131. package/src/data_import/utils/data.ts +16 -2
  132. package/src/data_import/utils/file_to_json.ts +51 -5
  133. package/src/data_import/utils/index.ts +2 -0
  134. package/src/data_import/utils/save_entities.ts +138 -0
  135. package/src/data_import/utils/transforms.ts +23 -0
  136. package/src/form/EntityForm.tsx +74 -53
  137. package/src/form/EntityFormBinding.tsx +24 -11
  138. package/src/form/PropertyFieldBinding.tsx +2 -2
  139. package/src/form/components/FieldBlock.tsx +80 -10
  140. package/src/form/components/FormRail.tsx +19 -2
  141. package/src/form/components/FormSections.tsx +32 -5
  142. package/src/form/components/StorageUploadProgress.tsx +9 -3
  143. package/src/form/components/useRailVisible.tsx +5 -4
  144. package/src/form/field_bindings/ArrayCustomShapedFieldBinding.tsx +2 -2
  145. package/src/form/field_bindings/BinaryFieldBinding.tsx +10 -5
  146. package/src/form/field_bindings/BlockFieldBinding.tsx +3 -3
  147. package/src/form/field_bindings/GeopointFieldBinding.tsx +23 -16
  148. package/src/form/field_bindings/MapFieldBinding.tsx +2 -2
  149. package/src/form/field_bindings/MarkdownEditorFieldBinding.tsx +13 -7
  150. package/src/form/field_bindings/MultiSelectFieldBinding.tsx +7 -1
  151. package/src/form/field_bindings/ReadOnlyFieldBinding.tsx +21 -2
  152. package/src/form/field_bindings/ReferenceFieldBinding.tsx +1 -1
  153. package/src/form/field_bindings/RelationFieldBinding.tsx +5 -2
  154. package/src/form/field_bindings/RepeatFieldBinding.tsx +3 -3
  155. package/src/form/field_bindings/StorageUploadFieldBinding.tsx +2 -2
  156. package/src/form/field_bindings/UserSelectFieldBinding.tsx +4 -1
  157. package/src/form/field_bindings/VectorFieldBinding.tsx +6 -4
  158. package/src/form/form_utils.ts +78 -0
  159. package/src/hooks/navigation/useBuildCollectionRegistryController.tsx +29 -5
  160. package/src/hooks/useEntityDisplay.tsx +262 -0
  161. package/src/hooks/useEntityDisplayTitle.tsx +27 -38
  162. package/src/index.ts +9 -1
  163. package/src/preview/PropertyPreview.tsx +25 -5
  164. package/src/preview/compact_text.ts +68 -0
  165. package/src/preview/components/BooleanPreview.tsx +5 -2
  166. package/src/preview/components/ReferencePreview.tsx +5 -8
  167. package/src/preview/components/RelationPreview.tsx +8 -7
  168. package/src/preview/components/StorageThumbnail.tsx +5 -2
  169. package/src/preview/property_previews/ArrayOfRelationsPreview.tsx +6 -3
  170. package/src/preview/property_previews/ArrayOfStringsPreview.tsx +26 -1
  171. package/src/preview/property_previews/ArrayOneOfPreview.tsx +10 -1
  172. package/src/preview/property_previews/ArrayPropertyPreview.tsx +10 -1
  173. package/src/preview/property_previews/MapPropertyPreview.tsx +52 -1
  174. package/src/preview/property_previews/NumberPropertyPreview.tsx +9 -1
  175. package/src/preview/property_previews/StringPropertyPreview.tsx +8 -5
  176. package/src/routes/RebaseRoute.tsx +62 -4
  177. package/src/types/components/PropertyPreviewProps.tsx +21 -0
  178. package/src/util/navigation_utils.ts +4 -2
  179. package/src/util/number_format.ts +46 -0
  180. package/src/util/previews.ts +8 -55
  181. package/src/util/property_utils.tsx +6 -1
  182. package/src/util/view_constants.ts +27 -0
  183. package/src/util/view_mode.ts +36 -0
  184. package/dist/PropertyEditView-W8I3YV89.js.map +0 -1
  185. package/dist/RouterCollectionsStudioView-BsLTNoqX.js.map +0 -1
  186. package/dist/export-J79rMoNj.js.map +0 -1
  187. package/dist/import-BvYl--U4.js.map +0 -1
  188. package/dist/util-GiwPgxnL.js.map +0 -1
@@ -1,6 +1,9 @@
1
1
  export type { EntityViewBindingProps } from "./EntityViewBinding";
2
2
  export { EntityViewBinding } from "./EntityViewBinding";
3
3
 
4
+ export type { EntityDisplayHeaderProps } from "./EntityDisplayHeader";
5
+ export { EntityDisplayHeader, HEADER_DISPLAY_ROLES } from "./EntityDisplayHeader";
6
+
4
7
  export * from "./DetailViewBinding";
5
8
 
6
9
  export type { SelectionProps } from "./ReferenceTable/SelectionTableBinding";
@@ -29,9 +29,26 @@ import {
29
29
  Tooltip
30
30
  } from "@rebasepro/ui";
31
31
  import { downloadEntitiesExport } from "./export";
32
+ import { fetchAllEntitiesForExport, MAX_EXPORT_ROWS } from "./fetch_export_data";
32
33
 
33
34
  const DOCS_LIMIT = 500;
34
35
 
36
+ /**
37
+ * Additional-field builders run per row and may do I/O, so a `Promise.all` over
38
+ * the whole export is one request per row all at once. That was bounded only by
39
+ * the 50-row cap the export used to have; now that the read is paginated, it is
40
+ * bounded here instead.
41
+ */
42
+ const ADDITIONAL_FIELDS_CONCURRENCY = 50;
43
+
44
+ async function mapInChunks<T, R>(items: T[], fn: (item: T) => Promise<R>, chunkSize = ADDITIONAL_FIELDS_CONCURRENCY): Promise<R[]> {
45
+ const results: R[] = [];
46
+ for (let i = 0; i < items.length; i += chunkSize) {
47
+ results.push(...await Promise.all(items.slice(i, i + chunkSize).map(fn)));
48
+ }
49
+ return results;
50
+ }
51
+
35
52
  export function ExportCollectionAction<M extends Record<string, unknown>, USER extends User>({
36
53
  collection,
37
54
  path,
@@ -59,6 +76,11 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
59
76
  const context = useAdminContext<USER>();
60
77
  const dataClient = useData();
61
78
 
79
+ // Said before the download starts, not discovered halfway through it: the
80
+ // walk refuses rather than writing a short file, so the dialog has to name
81
+ // the ceiling it is about to hit.
82
+ const tooManyToExport = collectionEntitiesCount !== undefined && collectionEntitiesCount > MAX_EXPORT_ROWS;
83
+
62
84
  const canExport = !exportAllowed || exportAllowed({
63
85
  collectionEntitiesCount: collectionEntitiesCount ?? 0,
64
86
  path,
@@ -67,6 +89,7 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
67
89
 
68
90
  const [dataLoading, setDataLoading] = React.useState<boolean>(false);
69
91
  const [dataLoadingError, setDataLoadingError] = React.useState<Error | undefined>();
92
+ const [progress, setProgress] = React.useState<{ loaded: number, total?: number }>({ loaded: 0 });
70
93
 
71
94
  const [open, setOpen] = React.useState(false);
72
95
 
@@ -84,7 +107,7 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
84
107
  const additionalFields = collection.additionalFields;
85
108
 
86
109
  const resolvedExportColumnsValues: Record<string, any>[] = additionalExportFields
87
- ? await Promise.all(entities.map(async (entity) => {
110
+ ? await mapInChunks(entities, async (entity) => {
88
111
  return (await Promise.all(additionalExportFields.map(async (column) => {
89
112
  return {
90
113
  [column.key]: await column.builder({
@@ -94,11 +117,11 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
94
117
  };
95
118
  }))).reduce((a, b) => ({ ...a,
96
119
  ...b }), {});
97
- }))
120
+ })
98
121
  : [];
99
122
 
100
123
  const resolvedColumnsValues: Record<string, any>[] = additionalFields
101
- ? await Promise.all(entities.map(async (entity) => {
124
+ ? await mapInChunks(entities, async (entity) => {
102
125
  return (await Promise.all(additionalFields
103
126
  .map(async (field) => {
104
127
  if (!field.value)
@@ -111,64 +134,74 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
111
134
  };
112
135
  }))).reduce((a, b) => ({ ...a,
113
136
  ...b }), {});
114
- }))
137
+ })
115
138
  : [];
116
139
  return [...resolvedExportColumnsValues, ...resolvedColumnsValues];
117
140
  }, [exportConfig?.additionalFields]);
118
141
 
119
142
  const doDownload = useCallback(async (collection: AdminCollection<M>,
120
- exportConfig: ExportConfig<any> | undefined) => {
143
+ exportConfig: ExportConfig<any> | undefined): Promise<boolean> => {
121
144
 
122
145
  onAnalyticsEvent?.("export_collection", {
123
146
  collection: collection.slug
124
147
  });
125
148
  setDataLoading(true);
126
- dataClient.collection(path).find({})
127
- .then(async (res) => {
128
- const data = res.data as Entity<M>[];
129
- setDataLoadingError(undefined);
130
- const additionalData = await fetchAdditionalFields(data);
131
- const additionalHeaders = [
132
- ...exportConfig?.additionalFields?.map(column => column.key) ?? [],
133
- ...collection.additionalFields?.map(field => field.key) ?? []
134
- ];
135
-
136
- const dataWithDefaults = includeUndefinedValues
137
- ? data.map(entity => {
138
- const defaultValues = getDefaultValuesFor(collection.properties);
139
- return {
140
- ...entity,
141
- values: { ...defaultValues,
149
+ setDataLoadingError(undefined);
150
+ setProgress({ loaded: 0 });
151
+ try {
152
+ // Paginated, not `find({})`: an absent limit resolves to 50 rows
153
+ // server-side, so the export used to be the first page of the
154
+ // collection under a filename that read like all of it.
155
+ const data = await fetchAllEntitiesForExport<M>({
156
+ accessor: dataClient.collection(path) as { find: (params?: any) => Promise<any> },
157
+ onProgress: (loaded, total) => setProgress({ loaded,
158
+ total })
159
+ });
160
+ const additionalData = await fetchAdditionalFields(data);
161
+ const additionalHeaders = [
162
+ ...exportConfig?.additionalFields?.map(column => column.key) ?? [],
163
+ ...collection.additionalFields?.map(field => field.key) ?? []
164
+ ];
165
+
166
+ const defaultValues = includeUndefinedValues ? getDefaultValuesFor(collection.properties) : undefined;
167
+ const dataWithDefaults = defaultValues
168
+ ? data.map(entity => ({
169
+ ...entity,
170
+ values: { ...defaultValues,
142
171
  ...entity.values }
143
- };
144
- })
145
- : data;
146
- downloadEntitiesExport({
147
- data: dataWithDefaults,
148
- additionalData,
149
- properties: collection.properties,
150
- propertiesOrder: collection.propertiesOrder,
151
- name: collection.name,
152
- flattenArrays,
153
- additionalHeaders,
154
- exportType,
155
- dateExportType
156
- });
157
- onAnalyticsEvent?.("export_collection_success", {
158
- collection: collection.slug
159
- });
160
- })
161
- .catch((e) => {
162
- console.error("Error loading export data", e);
163
- setDataLoadingError(e);
164
- })
165
- .finally(() => setDataLoading(false));
172
+ }))
173
+ : data;
174
+ downloadEntitiesExport({
175
+ data: dataWithDefaults,
176
+ additionalData,
177
+ properties: collection.properties,
178
+ propertiesOrder: collection.propertiesOrder,
179
+ name: collection.name,
180
+ flattenArrays,
181
+ additionalHeaders,
182
+ exportType,
183
+ dateExportType
184
+ });
185
+ onAnalyticsEvent?.("export_collection_success", {
186
+ collection: collection.slug
187
+ });
188
+ return true;
189
+ } catch (e) {
190
+ console.error("Error loading export data", e);
191
+ setDataLoadingError(e as Error);
192
+ return false;
193
+ } finally {
194
+ setDataLoading(false);
195
+ }
166
196
 
167
197
  }, [onAnalyticsEvent, dataClient, path, fetchAdditionalFields, includeUndefinedValues, flattenArrays, exportType, dateExportType]);
168
198
 
169
199
  const onOkClicked = useCallback(() => {
170
- doDownload(collection, exportConfig);
171
- handleClose();
200
+ // The dialog stays open until the walk finishes: it is the only place a
201
+ // multi-page export reports progress, or reports that it failed.
202
+ doDownload(collection, exportConfig).then((downloaded) => {
203
+ if (downloaded) handleClose();
204
+ });
172
205
  }, [doDownload, collection, exportConfig, handleClose]);
173
206
 
174
207
  return <>
@@ -194,13 +227,23 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
194
227
 
195
228
  <div>{t("download_table_csv")}</div>
196
229
 
197
- {collectionEntitiesCount !== undefined && collectionEntitiesCount > DOCS_LIMIT &&
230
+ {collectionEntitiesCount !== undefined && collectionEntitiesCount > DOCS_LIMIT && !tooManyToExport &&
198
231
  <Alert color={"warning"}>
199
232
  <div>
200
233
  {t("large_number_of_documents", { count: collectionEntitiesCount.toString() })}
201
234
  </div>
202
235
  </Alert>}
203
236
 
237
+ {tooManyToExport &&
238
+ <Alert color={"error"}>
239
+ <div>
240
+ {t("too_many_documents_to_export", {
241
+ count: (collectionEntitiesCount ?? 0).toString(),
242
+ limit: MAX_EXPORT_ROWS.toString()
243
+ })}
244
+ </div>
245
+ </Alert>}
246
+
204
247
  <div className={"flex flex-row gap-4"}>
205
248
  <div className={"p-4 flex flex-col"}>
206
249
  <RadioGroup value={exportType} onValueChange={(v) => setExportType(v as "csv" | "json")}>
@@ -242,6 +285,10 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
242
285
  onValueChange={setIncludeUndefinedValues}
243
286
  label={t("include_undefined_values")}/>
244
287
 
288
+ {dataLoadingError && <Alert color={"error"}>
289
+ <div>{dataLoadingError.message}</div>
290
+ </Alert>}
291
+
245
292
  {!canExport && notAllowedView}
246
293
 
247
294
  </DialogContent>
@@ -250,13 +297,19 @@ export function ExportCollectionAction<M extends Record<string, unknown>, USER e
250
297
 
251
298
  {dataLoading && <CircularProgress size={"smallest"}/>}
252
299
 
300
+ {dataLoading && <Label>
301
+ {progress.total !== undefined
302
+ ? `${progress.loaded} / ${progress.total}`
303
+ : `${progress.loaded}`}
304
+ </Label>}
305
+
253
306
  <Button onClick={handleClose}
254
307
  variant={"text"}>
255
308
  {t("cancel")}
256
309
  </Button>
257
310
 
258
311
  <Button onClick={onOkClicked}
259
- disabled={dataLoading || !canExport}>
312
+ disabled={dataLoading || !canExport || tooManyToExport}>
260
313
  {t("download")}
261
314
  </Button>
262
315
 
@@ -154,6 +154,24 @@ function getHeaders(property: Property, propertyKey: string, prefix = ""): Heade
154
154
  }
155
155
  }
156
156
 
157
+ /**
158
+ * Read whatever a `date` column actually arrived as into a `Date`.
159
+ *
160
+ * The driver/WebSocket path revives `{ __type: "date" }` into a `Date`; the REST
161
+ * path — which is the one the admin is on — leaves an ISO string, and an epoch
162
+ * number is what a re-imported export carries. Returns `undefined` for anything
163
+ * that is not a date, so the caller can pass the raw value through.
164
+ */
165
+ function toExportableDate(inputValue: unknown): Date | undefined {
166
+ if (inputValue instanceof Date) return Number.isNaN(inputValue.getTime()) ? undefined : inputValue;
167
+ if (typeof inputValue === "number" && Number.isFinite(inputValue)) return new Date(inputValue);
168
+ if (typeof inputValue === "string" && inputValue.trim() !== "") {
169
+ const parsed = new Date(inputValue);
170
+ return Number.isNaN(parsed.getTime()) ? undefined : parsed;
171
+ }
172
+ return undefined;
173
+ }
174
+
157
175
  function processValueForExport(inputValue: unknown,
158
176
  property: Property,
159
177
  exportType: "csv" | "json",
@@ -181,8 +199,17 @@ function processValueForExport(inputValue: unknown,
181
199
  } else if (property.type === "reference" && inputValue && typeof inputValue === "object" && "isEntityReference" in inputValue && typeof (inputValue as EntityReference).isEntityReference === "function" && (inputValue as EntityReference).isEntityReference()) {
182
200
  const ref = inputValue ? inputValue as EntityReference : undefined;
183
201
  value = ref ? ref.fullPath : null;
184
- } else if (property.type === "date" && inputValue instanceof Date) {
185
- value = inputValue ? (dateExportType === "timestamp" ? inputValue.getTime() : inputValue.toISOString()) : null;
202
+ } else if (property.type === "date") {
203
+ // Not `instanceof Date`: the admin reads over REST, and the REST row
204
+ // pipeline hands dates back as the database wrote them — a string. That
205
+ // made this branch dead on every live export, so both radio options
206
+ // produced identical files. Normalise here, where the declared type says
207
+ // the column is a date, and leave anything unparseable untouched rather
208
+ // than exporting an invalid date as `null`.
209
+ const date = toExportableDate(inputValue);
210
+ value = date
211
+ ? (dateExportType === "timestamp" ? date.getTime() : date.toISOString())
212
+ : inputValue;
186
213
  } else {
187
214
  value = inputValue;
188
215
  }
@@ -209,15 +236,40 @@ function processValuesForExport<M extends Record<string, unknown>>
209
236
  ...updatedValues };
210
237
  }
211
238
 
212
- function entryToCSVRow(entry: unknown[]) {
239
+ /**
240
+ * Characters that make a spreadsheet read a cell as a formula rather than text.
241
+ *
242
+ * Quoting is CSV *escaping* and is not enough: Excel, LibreOffice Calc and
243
+ * Sheets strip the CSV quoting at parse time and then evaluate a cell whose
244
+ * first character is one of these — `=HYPERLINK(…)` exfiltrates neighbouring
245
+ * cells on click, `=cmd|'…'!A0` is worse.
246
+ */
247
+ const FORMULA_TRIGGERS = ["=", "+", "-", "@", "\t", "\r"];
248
+
249
+ /**
250
+ * Neutralise a cell that a spreadsheet would otherwise evaluate, by prefixing a
251
+ * single quote — the standard OWASP mitigation. The apostrophe is consumed by
252
+ * the spreadsheet as "treat the rest as text", so the value reads back intact.
253
+ */
254
+ export function escapeCsvFormula(value: string): string {
255
+ const first = value.charAt(0);
256
+ if (!FORMULA_TRIGGERS.includes(first)) return value;
257
+ // A signed number is not a formula, and prefixing it would turn every
258
+ // negative amount in the file into text. `-1+1` is still neutralised,
259
+ // because it is not a number.
260
+ if ((first === "-" || first === "+") && value.trim() !== "" && Number.isFinite(Number(value))) return value;
261
+ return "'" + value;
262
+ }
263
+
264
+ function toCSVCell(v: unknown): string {
265
+ if (v === null || v === undefined) return "";
266
+ const s = Array.isArray(v) ? JSON.stringify(v) : String(v);
267
+ return "\"" + escapeCsvFormula(s).replaceAll("\"", "\"\"") + "\"";
268
+ }
269
+
270
+ export function entryToCSVRow(entry: unknown[]) {
213
271
  return entry
214
- .map((v: unknown) => {
215
- if (v === null || v === undefined) return "";
216
- if (Array.isArray(v))
217
- return "\"" + JSON.stringify(v).replaceAll("\"", "\"\"") + "\"";
218
- const s = String(v);
219
- return "\"" + s.replaceAll("\"", "\"\"") + "\"";
220
- })
272
+ .map(toCSVCell)
221
273
  .join(",") + "\r\n";
222
274
  }
223
275
 
@@ -236,12 +288,7 @@ export function downloadDataAsCsv(data: object[], name: string) {
236
288
  const headers = Object.keys(data[0]);
237
289
  const csvContent = [
238
290
  headers.join(","),
239
- ...data.map(row => headers.map(header => {
240
- const value = (row as Record<string, unknown>)[header];
241
- if (value === null || value === undefined) return "";
242
- if (Array.isArray(value)) return `"${JSON.stringify(value).replace(/"/g, '""')}"`;
243
- return `"${String(value).replace(/"/g, '""')}"`;
244
- }).join(","))
291
+ ...data.map(row => headers.map(header => toCSVCell((row as Record<string, unknown>)[header])).join(","))
245
292
  ].join("\r\n");
246
293
 
247
294
  downloadBlob([csvContent], `${name}.csv`, "text/csv");
@@ -0,0 +1,90 @@
1
+ import { Entity, FindParams, FindResponse } from "@rebasepro/types";
2
+
3
+ /**
4
+ * Rows asked for per request.
5
+ *
6
+ * Kept at or below the server's `MAX_LIST_LIMIT`, which is now the largest page
7
+ * a read will serve *and refuses* anything larger — it used to clamp silently,
8
+ * and asking for 100 000 while receiving 1 000 looked exactly like a collection
9
+ * with 1 000 rows in it. The walk still never trusts the page size it asked for:
10
+ * it advances by the rows it was actually handed.
11
+ */
12
+ export const EXPORT_PAGE_SIZE = 500;
13
+
14
+ /**
15
+ * Most rows one browser-side export will materialise.
16
+ *
17
+ * Everything here is built in memory and handed to a Blob, so there is a
18
+ * ceiling whether or not it is named. Naming it means going over it **throws**
19
+ * instead of writing a short file that reads like the whole collection.
20
+ */
21
+ export const MAX_EXPORT_ROWS = 100_000;
22
+
23
+ export interface FetchAllForExportParams<M extends Record<string, unknown>> {
24
+ /** The collection accessor to read through — `dataClient.collection(path)`. */
25
+ accessor: { find(params?: FindParams<M>): Promise<FindResponse<M>> };
26
+ /** Query to export. The window (`limit` / `offset` / `page`) is owned by the walk. */
27
+ params?: Omit<FindParams<M>, "limit" | "offset" | "page">;
28
+ pageSize?: number;
29
+ maxRows?: number;
30
+ /** Called after every page with the rows so far, and the total when the server reports one. */
31
+ onProgress?: (loaded: number, total: number | undefined) => void;
32
+ }
33
+
34
+ /**
35
+ * Read **every** row matching a query, one page at a time.
36
+ *
37
+ * `find({})` sends no `limit`, and an absent limit is resolved server-side to
38
+ * `DEFAULT_LIST_LIMIT` — 50 rows. The export used to hand those 50 rows back as
39
+ * `<collection>.csv` with nothing saying it was a sample, next to a dialog
40
+ * warning that the collection had 100 000 documents.
41
+ *
42
+ * Termination is driven by the server's `meta.hasMore` where it is reported and
43
+ * by an empty page otherwise — never by comparing a page's length against the
44
+ * requested limit, since a final page that happens to be exactly full is
45
+ * indistinguishable that way and a walk that stops there drops rows.
46
+ *
47
+ * @throws When more than `maxRows` rows match. A truncated export is worse than
48
+ * no export: it is indistinguishable from a complete one.
49
+ */
50
+ export async function fetchAllEntitiesForExport<M extends Record<string, unknown>>({
51
+ accessor,
52
+ params,
53
+ pageSize = EXPORT_PAGE_SIZE,
54
+ maxRows = MAX_EXPORT_ROWS,
55
+ onProgress
56
+ }: FetchAllForExportParams<M>): Promise<Entity<M>[]> {
57
+
58
+ const rows: Entity<M>[] = [];
59
+ let offset = 0;
60
+
61
+ for (;;) {
62
+ const res = await accessor.find({
63
+ ...params,
64
+ limit: pageSize,
65
+ offset
66
+ } as FindParams<M>);
67
+
68
+ const page = res?.data ?? [];
69
+ rows.push(...page);
70
+ onProgress?.(rows.length, res?.meta?.total);
71
+
72
+ if (rows.length > maxRows) {
73
+ throw new Error(
74
+ `This export would contain more than ${maxRows.toLocaleString()} rows, which is more than the ` +
75
+ "browser can build in one file. Filter the collection down, or export it from the server."
76
+ );
77
+ }
78
+
79
+ // An empty page ends the walk whatever the metadata says, so a server
80
+ // that keeps answering `hasMore` cannot spin forever.
81
+ if (page.length === 0) break;
82
+ if (res?.meta && !res.meta.hasMore) break;
83
+
84
+ // By rows received, not by `pageSize`: a server whose ceiling is lower
85
+ // than the page we asked for would otherwise skip everything it clamped.
86
+ offset += page.length;
87
+ }
88
+
89
+ return rows;
90
+ }
@@ -1,3 +1,4 @@
1
1
  export * from "./export";
2
+ export * from "./fetch_export_data";
2
3
  export * from "./BasicExportAction";
3
4
  export * from "./ExportCollectionAction";
@@ -1,10 +1,9 @@
1
1
  import { useData } from "@rebasepro/app";
2
- import { Entity } from "@rebasepro/types";
3
- import { RebaseData } from "@rebasepro/types";
4
2
  import { Button, CenteredView, CircularProgress, Typography } from "@rebasepro/ui";
5
3
  import { useEffect, useRef, useState } from "react";
6
4
  import { ImportConfig } from "../types";
7
5
  import type { AdminCollection } from "@rebasepro/admin-types";
6
+ import { IMPORT_BATCH_SIZE, ImportSaveError, saveImportedEntities } from "../utils/save_entities";
8
7
 
9
8
  export function ImportSaveInProgress<C extends AdminCollection<any>>
10
9
  ({
@@ -20,11 +19,16 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
20
19
  onImportSuccess: (collection: C) => void
21
20
  }) {
22
21
 
23
- const [errorSaving, setErrorSaving] = useState<Error | undefined>(undefined);
22
+ const [errorSaving, setErrorSaving] = useState<ImportSaveError | undefined>(undefined);
24
23
  const dataClient = useData();
25
24
 
26
25
  const savingRef = useRef<boolean>(false);
27
26
 
27
+ // Rows confirmed as written. A retry resumes here rather than re-sending
28
+ // rows that now exist and conflicting on its first batch, forever.
29
+ const committedRef = useRef<number>(0);
30
+ const bulkUnsupportedRef = useRef<{ current: boolean }>({ current: false });
31
+
28
32
  const [processedEntities, setProcessedEntities] = useState<number>(0);
29
33
 
30
34
  function save() {
@@ -33,20 +37,29 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
33
37
  return;
34
38
 
35
39
  savingRef.current = true;
40
+ setErrorSaving(undefined);
36
41
 
37
- saveDataBatch(
42
+ saveImportedEntities(
38
43
  dataClient,
39
- collection,
40
44
  path,
41
45
  importConfig.entities,
42
- 0,
43
- 25,
44
- setProcessedEntities
46
+ {
47
+ offset: committedRef.current,
48
+ batchSize: IMPORT_BATCH_SIZE,
49
+ onBatchCommitted: (written) => {
50
+ committedRef.current = written;
51
+ setProcessedEntities(written);
52
+ },
53
+ bulkUnsupported: bulkUnsupportedRef.current
54
+ }
45
55
  ).then(() => {
46
56
  onImportSuccess(collection);
47
57
  savingRef.current = false;
48
58
  }).catch((e) => {
49
- setErrorSaving(e);
59
+ const error = e as ImportSaveError;
60
+ committedRef.current = error.committed ?? committedRef.current;
61
+ setProcessedEntities(committedRef.current);
62
+ setErrorSaving(error);
50
63
  savingRef.current = false;
51
64
  });
52
65
  }
@@ -56,6 +69,10 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
56
69
  }, []);
57
70
 
58
71
  if (errorSaving) {
72
+ const total = importConfig.entities.length;
73
+ const failedRows = errorSaving.failedTo - errorSaving.failedFrom > 1
74
+ ? `rows ${errorSaving.failedFrom + 1}–${errorSaving.failedTo}`
75
+ : `row ${errorSaving.failedFrom + 1}`;
59
76
  return (
60
77
  <CenteredView className={"flex flex-col gap-4 items-center"}>
61
78
  <Typography variant={"h6"}>
@@ -65,10 +82,18 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
65
82
  <Typography variant={"body2"} color={"error"}>
66
83
  {errorSaving.message}
67
84
  </Typography>
85
+
86
+ <Typography variant={"body2"}>
87
+ {errorSaving.committed} of {total} rows were imported.
88
+ The import stopped on {failedRows}, which {errorSaving.failedTo - errorSaving.failedFrom > 1 ? "were" : "was"} not written.
89
+ </Typography>
90
+
68
91
  <Button
69
92
  onClick={save}
70
93
  >
71
- Retry
94
+ {errorSaving.committed > 0
95
+ ? `Retry from row ${errorSaving.committed + 1}`
96
+ : "Retry"}
72
97
  </Button>
73
98
  </CenteredView>
74
99
  );
@@ -94,29 +119,3 @@ export function ImportSaveInProgress<C extends AdminCollection<any>>
94
119
  );
95
120
 
96
121
  }
97
-
98
- function saveDataBatch(dataClient: RebaseData,
99
- collection: AdminCollection,
100
- path: string,
101
- data: Partial<Entity<any>>[],
102
- offset = 0,
103
- batchSize = 25,
104
- onProgressUpdate: (progress: number) => void): Promise<void> {
105
-
106
- console.debug("Saving imported data", offset, batchSize);
107
-
108
- const batch = data.slice(offset, offset + batchSize);
109
- return Promise.all(batch.map(d =>
110
- dataClient.collection(path).create(
111
- d.values,
112
- d.id
113
- )))
114
- .then(() => {
115
- if (offset + batchSize < data.length) {
116
- onProgressUpdate(offset + batchSize);
117
- return saveDataBatch(dataClient, collection, path, data, offset + batchSize, batchSize, onProgressUpdate);
118
- }
119
- onProgressUpdate(data.length);
120
- return Promise.resolve();
121
- });
122
- }
@@ -35,7 +35,7 @@ import { DataNewPropertiesMapping } from "../components/DataNewPropertiesMapping
35
35
  import { ImportFileUpload } from "../components/ImportFileUpload";
36
36
  import { ImportSaveInProgress } from "../components/ImportSaveInProgress";
37
37
  import { ImportConfig } from "../types";
38
- import { slugify } from "@rebasepro/utils";
38
+ import { isPrototypePollutingKey, slugify } from "@rebasepro/utils";
39
39
 
40
40
  type ImportState = "initial" | "mapping" | "preview" | "import_data_saving";
41
41
 
@@ -400,7 +400,15 @@ export function ImportDataPreview<M extends Record<string, unknown>>({
400
400
  return <CollectionTableBinding
401
401
  title={<div>
402
402
  <Typography variant={"subtitle2"}>Imported data preview</Typography>
403
- <Typography variant={"caption"}>Entities with the same id will be overwritten</Typography>
403
+ {/* Conditional because it is only true when a column was chosen as
404
+ the id: without one, every row is created and nothing can be
405
+ overwritten. With one, the import asks for an upsert — which it
406
+ did not, back when this sentence was unconditional and false. */}
407
+ <Typography variant={"caption"}>
408
+ {importConfig.idColumn
409
+ ? "Entities with the same id will be overwritten"
410
+ : "All rows will be imported as new entities"}
411
+ </Typography>
404
412
  </div>}
405
413
  tableController={{
406
414
  data: importConfig.entities,
@@ -421,6 +429,9 @@ function buildHeadersMappingFromData(objArr: object[], properties?: Properties)
421
429
  const headersMapping: Record<string, string> = {};
422
430
  objArr.filter(Boolean).forEach((obj) => {
423
431
  Object.keys(obj).forEach((key) => {
432
+ // The keys are the uploaded file's header row; `headersMapping[key] = …`
433
+ // is the same setter the import pipeline refuses elsewhere.
434
+ if (isPrototypePollutingKey(key)) return;
424
435
  const child = (obj as Record<string, unknown>)[key];
425
436
  if (child != null && typeof child === "object" && !Array.isArray(child)) {
426
437
  const childProperty = properties?.[key];