@commercetools-frontend-extensions/operations 3.0.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +72 -2
- package/dist/commercetools-frontend-extensions-operations.cjs.dev.js +50 -13
- package/dist/commercetools-frontend-extensions-operations.cjs.prod.js +50 -13
- package/dist/commercetools-frontend-extensions-operations.esm.js +50 -14
- package/dist/declarations/src/@api/file-import-jobs.d.ts +1 -1
- package/dist/declarations/src/@constants/import-limits.d.ts +1 -1
- package/dist/declarations/src/@hooks/index.d.ts +1 -0
- package/dist/declarations/src/@hooks/use-fetch-file-import-job-records.d.ts +18 -0
- package/dist/declarations/src/@hooks/use-file-import-job-upload.d.ts +1 -0
- package/dist/declarations/src/@hooks/use-file-upload.d.ts +2 -0
- package/dist/declarations/src/@types/file-import-job.d.ts +1 -0
- package/package.json +9 -9
- package/src/@api/file-import-jobs.ts +2 -0
- package/src/@constants/import-limits.ts +1 -1
- package/src/@hooks/index.ts +1 -0
- package/src/@hooks/use-fetch-file-import-job-records.ts +58 -0
- package/src/@hooks/use-file-import-job-upload.ts +2 -0
- package/src/@hooks/use-file-upload.ts +27 -17
- package/src/@types/file-import-job.ts +1 -0
- package/src/@utils/poll-job-until-validated.ts +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
# @commercetools-frontend-extensions/operations
|
|
2
2
|
|
|
3
|
+
## 3.1.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- [#1673](https://github.com/commercetools/merchant-center-operations/pull/1673) [`fda98c9`](https://github.com/commercetools/merchant-center-operations/commit/fda98c935fc45adfffe1247e30bea8c13df2abc8) Thanks [@yassinejebli](https://github.com/yassinejebli)! - feat: add `autoProcess` option for file import job flow
|
|
8
|
+
|
|
9
|
+
- [#1673](https://github.com/commercetools/merchant-center-operations/pull/1673) [`fda98c9`](https://github.com/commercetools/merchant-center-operations/commit/fda98c935fc45adfffe1247e30bea8c13df2abc8) Thanks [@yassinejebli](https://github.com/yassinejebli)! - feat: adjust the max file size for the new Import flow
|
|
10
|
+
|
|
11
|
+
## 3.1.0
|
|
12
|
+
|
|
13
|
+
### Minor Changes
|
|
14
|
+
|
|
15
|
+
- [#1669](https://github.com/commercetools/merchant-center-operations/pull/1669) [`a4a9c65`](https://github.com/commercetools/merchant-center-operations/commit/a4a9c6513e9a0b6da7d0681f044d7640ad8f0132) Thanks [@yassinejebli](https://github.com/yassinejebli)! - feat(useFileUpload): add `total` to `validationProgress` for tracking validation progress
|
|
16
|
+
|
|
17
|
+
The `validationProgress` object returned by `useFileUpload` now includes a `total` field representing the total number of unique resources in the CSV file (counted by unique keys). This allows consumers to display progress like "Validating X of Y resources" during the job-based validation flow.
|
|
18
|
+
|
|
19
|
+
- [#1669](https://github.com/commercetools/merchant-center-operations/pull/1669) [`a4a9c65`](https://github.com/commercetools/merchant-center-operations/commit/a4a9c6513e9a0b6da7d0681f044d7640ad8f0132) Thanks [@yassinejebli](https://github.com/yassinejebli)! - feat(import): server side pagination for new import flow errors
|
|
20
|
+
|
|
21
|
+
### Patch Changes
|
|
22
|
+
|
|
23
|
+
- [#1669](https://github.com/commercetools/merchant-center-operations/pull/1669) [`a4a9c65`](https://github.com/commercetools/merchant-center-operations/commit/a4a9c6513e9a0b6da7d0681f044d7640ad8f0132) Thanks [@yassinejebli](https://github.com/yassinejebli)! - feat: add `total` for tracking validation progress
|
|
24
|
+
|
|
3
25
|
## 3.0.0
|
|
4
26
|
|
|
5
27
|
### Major Changes
|
package/README.md
CHANGED
|
@@ -2,6 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
Shared functionality for import/export operations across multiple frontend applications and extensions.
|
|
4
4
|
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pnpm add @commercetools-frontend-extensions/operations
|
|
9
|
+
```
|
|
10
|
+
|
|
5
11
|
## Hooks
|
|
6
12
|
|
|
7
13
|
### `useFileUpload`
|
|
@@ -21,13 +27,27 @@ const { upload, isUploading, progress, validationProgress } = useFileUpload({
|
|
|
21
27
|
- `projectKey` (required): The commercetools project key
|
|
22
28
|
- `useJobBasedFlow` (optional): Whether to use the job-based flow. Default: `false`
|
|
23
29
|
- `pollingInterval` (optional): Polling interval in ms for job-based flow. Default: `5000`
|
|
24
|
-
- `maxPollingAttempts` (optional): Maximum polling attempts. Default: `
|
|
30
|
+
- `maxPollingAttempts` (optional): Maximum polling attempts. Default: `200`
|
|
25
31
|
|
|
26
32
|
**Returns:**
|
|
27
33
|
- `upload` - Function to start the upload
|
|
28
34
|
- `isUploading` - Whether upload is in progress
|
|
29
35
|
- `progress` - Upload progress (0-100)
|
|
30
|
-
- `validationProgress` - `{ processed: number, isValidating: boolean }` (job-based flow only)
|
|
36
|
+
- `validationProgress` - `{ processed: number, total: number, isValidating: boolean }` (job-based flow only)
|
|
37
|
+
- `processed`: Number of resources validated so far (from backend)
|
|
38
|
+
- `total`: Total number of unique resources in the file (counted by unique keys in the CSV)
|
|
39
|
+
- `isValidating`: Whether validation is in progress
|
|
40
|
+
|
|
41
|
+
**Upload config options:**
|
|
42
|
+
- `file` (required): The file to upload
|
|
43
|
+
- `resourceType` (required): The resource type
|
|
44
|
+
- `settings` (optional): Import settings (format, decimal separator...)
|
|
45
|
+
- `autoProcess` (optional): When `true`, the backend automatically starts processing after validation completes (job-based flow only). Default: `false`
|
|
46
|
+
- `abortSignal` (optional): AbortSignal for cancellation
|
|
47
|
+
- `onSuccess` (required): Callback when upload completes
|
|
48
|
+
- `onError` (optional): Callback for errors
|
|
49
|
+
- `onProgress` (optional): Callback for upload progress (0-100)
|
|
50
|
+
- `onValidationProgress` (optional): Callback for validation progress (job-based flow only)
|
|
31
51
|
|
|
32
52
|
**Usage:**
|
|
33
53
|
```typescript
|
|
@@ -52,6 +72,7 @@ await upload({
|
|
|
52
72
|
unpublishAllChanges?: boolean
|
|
53
73
|
}
|
|
54
74
|
},
|
|
75
|
+
autoProcess?: boolean, // job-based flow only, default: false
|
|
55
76
|
abortSignal: abortController.signal,
|
|
56
77
|
onSuccess: (result) => {
|
|
57
78
|
// result.containerKey - Import container key
|
|
@@ -149,6 +170,55 @@ const { data, error, isLoading, refetch } = useFetchFileImportJob({
|
|
|
149
170
|
|
|
150
171
|
---
|
|
151
172
|
|
|
173
|
+
### `useFetchFileImportJobRecords`
|
|
174
|
+
|
|
175
|
+
Fetches records (errors or valid entries) from a File Import Job with pagination support. Keeps previous data visible while loading new pages.
|
|
176
|
+
|
|
177
|
+
```typescript
|
|
178
|
+
const { data, error, isLoading, refetch } = useFetchFileImportJobRecords({
|
|
179
|
+
projectKey,
|
|
180
|
+
importContainerKey,
|
|
181
|
+
jobId,
|
|
182
|
+
limit?: number,
|
|
183
|
+
offset?: number,
|
|
184
|
+
isValid?: boolean,
|
|
185
|
+
skip?: boolean
|
|
186
|
+
})
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
**Parameters:**
|
|
190
|
+
- `projectKey`: The commercetools project key
|
|
191
|
+
- `importContainerKey`: The import container key
|
|
192
|
+
- `jobId`: The file import job ID
|
|
193
|
+
- `limit` (optional): Number of records to fetch per page
|
|
194
|
+
- `offset` (optional): Offset for pagination
|
|
195
|
+
- `isValid` (optional): Filter by valid (`true`) or invalid (`false`) records
|
|
196
|
+
- `skip` (optional): Skip fetching (useful for conditional fetching)
|
|
197
|
+
|
|
198
|
+
**Returns:**
|
|
199
|
+
- `data` - `{ results, total, limit, offset, count }` or `null`
|
|
200
|
+
- `error` - Error object if fetch failed
|
|
201
|
+
- `isLoading` - Whether fetch is in progress
|
|
202
|
+
- `refetch` - Function to manually trigger a refetch
|
|
203
|
+
|
|
204
|
+
**Usage example (paginated error table):**
|
|
205
|
+
```typescript
|
|
206
|
+
const pagination = usePaginationState()
|
|
207
|
+
const offset = (pagination.page.value - 1) * pagination.perPage.value
|
|
208
|
+
|
|
209
|
+
const { data, isLoading } = useFetchFileImportJobRecords({
|
|
210
|
+
projectKey,
|
|
211
|
+
importContainerKey: containerKey,
|
|
212
|
+
jobId,
|
|
213
|
+
offset,
|
|
214
|
+
limit: pagination.perPage.value,
|
|
215
|
+
isValid: false, // Fetch only invalid records (errors)
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
152
222
|
## Helper Functions
|
|
153
223
|
|
|
154
224
|
```typescript
|
|
@@ -742,7 +742,7 @@ const pollJobUntilValidated = async _ref => {
|
|
|
742
742
|
_ref$pollingInterval = _ref.pollingInterval,
|
|
743
743
|
pollingInterval = _ref$pollingInterval === void 0 ? 5000 : _ref$pollingInterval,
|
|
744
744
|
_ref$maxAttempts = _ref.maxAttempts,
|
|
745
|
-
maxAttempts = _ref$maxAttempts === void 0 ?
|
|
745
|
+
maxAttempts = _ref$maxAttempts === void 0 ? 200 : _ref$maxAttempts,
|
|
746
746
|
onJobUpdate = _ref.onJobUpdate,
|
|
747
747
|
abortSignal = _ref.abortSignal;
|
|
748
748
|
let attempts = 0;
|
|
@@ -941,6 +941,8 @@ function createFileImportJob(_ref) {
|
|
|
941
941
|
resourceType = _ref.resourceType,
|
|
942
942
|
importContainerKey = _ref.importContainerKey,
|
|
943
943
|
payload = _ref.payload,
|
|
944
|
+
_ref$autoProcess = _ref.autoProcess,
|
|
945
|
+
autoProcess = _ref$autoProcess === void 0 ? false : _ref$autoProcess,
|
|
944
946
|
onProgress = _ref.onProgress,
|
|
945
947
|
abortSignal = _ref.abortSignal;
|
|
946
948
|
const url = getFileImportJobsURL({
|
|
@@ -954,6 +956,7 @@ function createFileImportJob(_ref) {
|
|
|
954
956
|
formData.append('fileType', payload.fileType);
|
|
955
957
|
formData.append('fileName', payload.fileName);
|
|
956
958
|
formData.append('file', payload.file, payload.fileName);
|
|
959
|
+
formData.append('autoProcess', autoProcess ? 'true' : 'false');
|
|
957
960
|
fetchUsingXhr({
|
|
958
961
|
url,
|
|
959
962
|
payload: formData,
|
|
@@ -1371,7 +1374,7 @@ const COLUMN_DELIMITERS = [DELIMITERS.COMMA, DELIMITERS.SEMICOLON, DELIMITERS.PI
|
|
|
1371
1374
|
|
|
1372
1375
|
const FILE_IMPORT_JOB_POLLING_INTERVAL = 2000;
|
|
1373
1376
|
|
|
1374
|
-
const IMPORT_MAX_FILE_SIZE_MB =
|
|
1377
|
+
const IMPORT_MAX_FILE_SIZE_MB = 100;
|
|
1375
1378
|
const IMPORT_MAX_ITEM_COUNT = 500_000;
|
|
1376
1379
|
|
|
1377
1380
|
// =============================================================================
|
|
@@ -2679,6 +2682,39 @@ const useFetchFileImportJob = _ref => {
|
|
|
2679
2682
|
});
|
|
2680
2683
|
};
|
|
2681
2684
|
|
|
2685
|
+
const EMPTY_RESPONSE = {
|
|
2686
|
+
results: [],
|
|
2687
|
+
total: 0,
|
|
2688
|
+
limit: 0,
|
|
2689
|
+
offset: 0,
|
|
2690
|
+
count: 0
|
|
2691
|
+
};
|
|
2692
|
+
const useFetchFileImportJobRecords = _ref => {
|
|
2693
|
+
let projectKey = _ref.projectKey,
|
|
2694
|
+
importContainerKey = _ref.importContainerKey,
|
|
2695
|
+
jobId = _ref.jobId,
|
|
2696
|
+
limit = _ref.limit,
|
|
2697
|
+
offset = _ref.offset,
|
|
2698
|
+
isValid = _ref.isValid,
|
|
2699
|
+
_ref$skip = _ref.skip,
|
|
2700
|
+
skip = _ref$skip === void 0 ? false : _ref$skip;
|
|
2701
|
+
const shouldSkip = skip || !projectKey || !importContainerKey || !jobId;
|
|
2702
|
+
const fetchData = React__default["default"].useCallback(() => {
|
|
2703
|
+
if (shouldSkip) {
|
|
2704
|
+
return _Promise__default["default"].resolve(EMPTY_RESPONSE);
|
|
2705
|
+
}
|
|
2706
|
+
return getFileImportJobRecords({
|
|
2707
|
+
projectKey: projectKey,
|
|
2708
|
+
importContainerKey: importContainerKey,
|
|
2709
|
+
jobId: jobId,
|
|
2710
|
+
limit,
|
|
2711
|
+
offset,
|
|
2712
|
+
isValid
|
|
2713
|
+
});
|
|
2714
|
+
}, [shouldSkip, projectKey, importContainerKey, jobId, limit, offset, isValid]);
|
|
2715
|
+
return useFetch(fetchData);
|
|
2716
|
+
};
|
|
2717
|
+
|
|
2682
2718
|
const useFetchImportContainerDetails = _ref => {
|
|
2683
2719
|
let projectKey = _ref.projectKey,
|
|
2684
2720
|
importContainerKey = _ref.importContainerKey,
|
|
@@ -2786,6 +2822,7 @@ const useFileImportJobUpload = _ref => {
|
|
|
2786
2822
|
fileName: config.file.name,
|
|
2787
2823
|
file: config.file
|
|
2788
2824
|
},
|
|
2825
|
+
autoProcess: config.autoProcess,
|
|
2789
2826
|
onProgress: uploadProgress => {
|
|
2790
2827
|
setProgress(uploadProgress);
|
|
2791
2828
|
config.onProgress?.(uploadProgress);
|
|
@@ -2935,7 +2972,7 @@ const useFileUpload = _ref2 => {
|
|
|
2935
2972
|
_ref2$pollingInterval = _ref2.pollingInterval,
|
|
2936
2973
|
pollingInterval = _ref2$pollingInterval === void 0 ? 5000 : _ref2$pollingInterval,
|
|
2937
2974
|
_ref2$maxPollingAttem = _ref2.maxPollingAttempts,
|
|
2938
|
-
maxPollingAttempts = _ref2$maxPollingAttem === void 0 ?
|
|
2975
|
+
maxPollingAttempts = _ref2$maxPollingAttem === void 0 ? 200 : _ref2$maxPollingAttem;
|
|
2939
2976
|
const _React$useState = React__default["default"].useState(false),
|
|
2940
2977
|
_React$useState2 = _slicedToArray(_React$useState, 2),
|
|
2941
2978
|
isUploading = _React$useState2[0],
|
|
@@ -2946,6 +2983,7 @@ const useFileUpload = _ref2 => {
|
|
|
2946
2983
|
setProgress = _React$useState4[1];
|
|
2947
2984
|
const _React$useState5 = React__default["default"].useState({
|
|
2948
2985
|
processed: 0,
|
|
2986
|
+
total: 0,
|
|
2949
2987
|
isValidating: false
|
|
2950
2988
|
}),
|
|
2951
2989
|
_React$useState6 = _slicedToArray(_React$useState5, 2),
|
|
@@ -2962,6 +3000,7 @@ const useFileUpload = _ref2 => {
|
|
|
2962
3000
|
setProgress(0);
|
|
2963
3001
|
setValidationProgress({
|
|
2964
3002
|
processed: 0,
|
|
3003
|
+
total: 0,
|
|
2965
3004
|
isValidating: false
|
|
2966
3005
|
});
|
|
2967
3006
|
}, []);
|
|
@@ -2970,15 +3009,18 @@ const useFileUpload = _ref2 => {
|
|
|
2970
3009
|
setProgress(0);
|
|
2971
3010
|
try {
|
|
2972
3011
|
if (useJobBasedFlow) {
|
|
3012
|
+
const totalResources = await countUniqueResourcesInCsv(config.file);
|
|
2973
3013
|
await jobUpload.upload({
|
|
2974
3014
|
file: config.file,
|
|
2975
3015
|
resourceType: config.resourceType,
|
|
2976
3016
|
settings: config.settings,
|
|
3017
|
+
autoProcess: config.autoProcess,
|
|
2977
3018
|
abortSignal: config.abortSignal,
|
|
2978
3019
|
onSuccess: async (jobId, containerKey) => {
|
|
2979
3020
|
try {
|
|
2980
3021
|
setValidationProgress({
|
|
2981
3022
|
processed: 0,
|
|
3023
|
+
total: totalResources,
|
|
2982
3024
|
isValidating: true
|
|
2983
3025
|
});
|
|
2984
3026
|
const validatedJob = await pollJobUntilValidated({
|
|
@@ -2992,6 +3034,7 @@ const useFileUpload = _ref2 => {
|
|
|
2992
3034
|
const processed = job.summary?.total ?? 0;
|
|
2993
3035
|
setValidationProgress({
|
|
2994
3036
|
processed,
|
|
3037
|
+
total: totalResources,
|
|
2995
3038
|
isValidating: true
|
|
2996
3039
|
});
|
|
2997
3040
|
config.onValidationProgress?.(job);
|
|
@@ -3003,16 +3046,7 @@ const useFileUpload = _ref2 => {
|
|
|
3003
3046
|
if (validatedJob.jobError) {
|
|
3004
3047
|
throw new HttpError(400, validatedJob.jobError.message, validatedJob.jobError);
|
|
3005
3048
|
}
|
|
3006
|
-
let results = [];
|
|
3007
3049
|
if (validatedJob.summary.invalid > 0) {
|
|
3008
|
-
const recordsResponse = await getFileImportJobRecords({
|
|
3009
|
-
projectKey,
|
|
3010
|
-
importContainerKey: containerKey,
|
|
3011
|
-
jobId,
|
|
3012
|
-
limit: 500,
|
|
3013
|
-
isValid: false
|
|
3014
|
-
});
|
|
3015
|
-
results = recordsResponse.results;
|
|
3016
3050
|
await safeDeleteContainer({
|
|
3017
3051
|
projectKey,
|
|
3018
3052
|
containerKey
|
|
@@ -3021,7 +3055,8 @@ const useFileUpload = _ref2 => {
|
|
|
3021
3055
|
const result = {
|
|
3022
3056
|
containerKey,
|
|
3023
3057
|
summary: _objectSpread(_objectSpread({}, validatedJob.summary), {}, {
|
|
3024
|
-
|
|
3058
|
+
// TODO: Remove this once the old flow is fully removed
|
|
3059
|
+
results: []
|
|
3025
3060
|
}),
|
|
3026
3061
|
jobId,
|
|
3027
3062
|
job: validatedJob
|
|
@@ -3029,6 +3064,7 @@ const useFileUpload = _ref2 => {
|
|
|
3029
3064
|
setIsUploading(false);
|
|
3030
3065
|
setValidationProgress({
|
|
3031
3066
|
processed: 0,
|
|
3067
|
+
total: 0,
|
|
3032
3068
|
isValidating: false
|
|
3033
3069
|
});
|
|
3034
3070
|
config.onSuccess(result);
|
|
@@ -3258,6 +3294,7 @@ exports.toImportApiResourceType = toImportApiResourceType;
|
|
|
3258
3294
|
exports.uploadFileForImport = uploadFileForImport;
|
|
3259
3295
|
exports.useFetchExportOperations = useFetchExportOperations;
|
|
3260
3296
|
exports.useFetchFileImportJob = useFetchFileImportJob;
|
|
3297
|
+
exports.useFetchFileImportJobRecords = useFetchFileImportJobRecords;
|
|
3261
3298
|
exports.useFetchImportContainerDetails = useFetchImportContainerDetails;
|
|
3262
3299
|
exports.useFetchImportOperations = useFetchImportOperations;
|
|
3263
3300
|
exports.useFetchImportSummaries = useFetchImportSummaries;
|
|
@@ -742,7 +742,7 @@ const pollJobUntilValidated = async _ref => {
|
|
|
742
742
|
_ref$pollingInterval = _ref.pollingInterval,
|
|
743
743
|
pollingInterval = _ref$pollingInterval === void 0 ? 5000 : _ref$pollingInterval,
|
|
744
744
|
_ref$maxAttempts = _ref.maxAttempts,
|
|
745
|
-
maxAttempts = _ref$maxAttempts === void 0 ?
|
|
745
|
+
maxAttempts = _ref$maxAttempts === void 0 ? 200 : _ref$maxAttempts,
|
|
746
746
|
onJobUpdate = _ref.onJobUpdate,
|
|
747
747
|
abortSignal = _ref.abortSignal;
|
|
748
748
|
let attempts = 0;
|
|
@@ -941,6 +941,8 @@ function createFileImportJob(_ref) {
|
|
|
941
941
|
resourceType = _ref.resourceType,
|
|
942
942
|
importContainerKey = _ref.importContainerKey,
|
|
943
943
|
payload = _ref.payload,
|
|
944
|
+
_ref$autoProcess = _ref.autoProcess,
|
|
945
|
+
autoProcess = _ref$autoProcess === void 0 ? false : _ref$autoProcess,
|
|
944
946
|
onProgress = _ref.onProgress,
|
|
945
947
|
abortSignal = _ref.abortSignal;
|
|
946
948
|
const url = getFileImportJobsURL({
|
|
@@ -954,6 +956,7 @@ function createFileImportJob(_ref) {
|
|
|
954
956
|
formData.append('fileType', payload.fileType);
|
|
955
957
|
formData.append('fileName', payload.fileName);
|
|
956
958
|
formData.append('file', payload.file, payload.fileName);
|
|
959
|
+
formData.append('autoProcess', autoProcess ? 'true' : 'false');
|
|
957
960
|
fetchUsingXhr({
|
|
958
961
|
url,
|
|
959
962
|
payload: formData,
|
|
@@ -1371,7 +1374,7 @@ const COLUMN_DELIMITERS = [DELIMITERS.COMMA, DELIMITERS.SEMICOLON, DELIMITERS.PI
|
|
|
1371
1374
|
|
|
1372
1375
|
const FILE_IMPORT_JOB_POLLING_INTERVAL = 2000;
|
|
1373
1376
|
|
|
1374
|
-
const IMPORT_MAX_FILE_SIZE_MB =
|
|
1377
|
+
const IMPORT_MAX_FILE_SIZE_MB = 100;
|
|
1375
1378
|
const IMPORT_MAX_ITEM_COUNT = 500_000;
|
|
1376
1379
|
|
|
1377
1380
|
// =============================================================================
|
|
@@ -2671,6 +2674,39 @@ const useFetchFileImportJob = _ref => {
|
|
|
2671
2674
|
});
|
|
2672
2675
|
};
|
|
2673
2676
|
|
|
2677
|
+
const EMPTY_RESPONSE = {
|
|
2678
|
+
results: [],
|
|
2679
|
+
total: 0,
|
|
2680
|
+
limit: 0,
|
|
2681
|
+
offset: 0,
|
|
2682
|
+
count: 0
|
|
2683
|
+
};
|
|
2684
|
+
const useFetchFileImportJobRecords = _ref => {
|
|
2685
|
+
let projectKey = _ref.projectKey,
|
|
2686
|
+
importContainerKey = _ref.importContainerKey,
|
|
2687
|
+
jobId = _ref.jobId,
|
|
2688
|
+
limit = _ref.limit,
|
|
2689
|
+
offset = _ref.offset,
|
|
2690
|
+
isValid = _ref.isValid,
|
|
2691
|
+
_ref$skip = _ref.skip,
|
|
2692
|
+
skip = _ref$skip === void 0 ? false : _ref$skip;
|
|
2693
|
+
const shouldSkip = skip || !projectKey || !importContainerKey || !jobId;
|
|
2694
|
+
const fetchData = React__default["default"].useCallback(() => {
|
|
2695
|
+
if (shouldSkip) {
|
|
2696
|
+
return _Promise__default["default"].resolve(EMPTY_RESPONSE);
|
|
2697
|
+
}
|
|
2698
|
+
return getFileImportJobRecords({
|
|
2699
|
+
projectKey: projectKey,
|
|
2700
|
+
importContainerKey: importContainerKey,
|
|
2701
|
+
jobId: jobId,
|
|
2702
|
+
limit,
|
|
2703
|
+
offset,
|
|
2704
|
+
isValid
|
|
2705
|
+
});
|
|
2706
|
+
}, [shouldSkip, projectKey, importContainerKey, jobId, limit, offset, isValid]);
|
|
2707
|
+
return useFetch(fetchData);
|
|
2708
|
+
};
|
|
2709
|
+
|
|
2674
2710
|
const useFetchImportContainerDetails = _ref => {
|
|
2675
2711
|
let projectKey = _ref.projectKey,
|
|
2676
2712
|
importContainerKey = _ref.importContainerKey,
|
|
@@ -2778,6 +2814,7 @@ const useFileImportJobUpload = _ref => {
|
|
|
2778
2814
|
fileName: config.file.name,
|
|
2779
2815
|
file: config.file
|
|
2780
2816
|
},
|
|
2817
|
+
autoProcess: config.autoProcess,
|
|
2781
2818
|
onProgress: uploadProgress => {
|
|
2782
2819
|
setProgress(uploadProgress);
|
|
2783
2820
|
config.onProgress?.(uploadProgress);
|
|
@@ -2927,7 +2964,7 @@ const useFileUpload = _ref2 => {
|
|
|
2927
2964
|
_ref2$pollingInterval = _ref2.pollingInterval,
|
|
2928
2965
|
pollingInterval = _ref2$pollingInterval === void 0 ? 5000 : _ref2$pollingInterval,
|
|
2929
2966
|
_ref2$maxPollingAttem = _ref2.maxPollingAttempts,
|
|
2930
|
-
maxPollingAttempts = _ref2$maxPollingAttem === void 0 ?
|
|
2967
|
+
maxPollingAttempts = _ref2$maxPollingAttem === void 0 ? 200 : _ref2$maxPollingAttem;
|
|
2931
2968
|
const _React$useState = React__default["default"].useState(false),
|
|
2932
2969
|
_React$useState2 = _slicedToArray(_React$useState, 2),
|
|
2933
2970
|
isUploading = _React$useState2[0],
|
|
@@ -2938,6 +2975,7 @@ const useFileUpload = _ref2 => {
|
|
|
2938
2975
|
setProgress = _React$useState4[1];
|
|
2939
2976
|
const _React$useState5 = React__default["default"].useState({
|
|
2940
2977
|
processed: 0,
|
|
2978
|
+
total: 0,
|
|
2941
2979
|
isValidating: false
|
|
2942
2980
|
}),
|
|
2943
2981
|
_React$useState6 = _slicedToArray(_React$useState5, 2),
|
|
@@ -2954,6 +2992,7 @@ const useFileUpload = _ref2 => {
|
|
|
2954
2992
|
setProgress(0);
|
|
2955
2993
|
setValidationProgress({
|
|
2956
2994
|
processed: 0,
|
|
2995
|
+
total: 0,
|
|
2957
2996
|
isValidating: false
|
|
2958
2997
|
});
|
|
2959
2998
|
}, []);
|
|
@@ -2962,15 +3001,18 @@ const useFileUpload = _ref2 => {
|
|
|
2962
3001
|
setProgress(0);
|
|
2963
3002
|
try {
|
|
2964
3003
|
if (useJobBasedFlow) {
|
|
3004
|
+
const totalResources = await countUniqueResourcesInCsv(config.file);
|
|
2965
3005
|
await jobUpload.upload({
|
|
2966
3006
|
file: config.file,
|
|
2967
3007
|
resourceType: config.resourceType,
|
|
2968
3008
|
settings: config.settings,
|
|
3009
|
+
autoProcess: config.autoProcess,
|
|
2969
3010
|
abortSignal: config.abortSignal,
|
|
2970
3011
|
onSuccess: async (jobId, containerKey) => {
|
|
2971
3012
|
try {
|
|
2972
3013
|
setValidationProgress({
|
|
2973
3014
|
processed: 0,
|
|
3015
|
+
total: totalResources,
|
|
2974
3016
|
isValidating: true
|
|
2975
3017
|
});
|
|
2976
3018
|
const validatedJob = await pollJobUntilValidated({
|
|
@@ -2984,6 +3026,7 @@ const useFileUpload = _ref2 => {
|
|
|
2984
3026
|
const processed = job.summary?.total ?? 0;
|
|
2985
3027
|
setValidationProgress({
|
|
2986
3028
|
processed,
|
|
3029
|
+
total: totalResources,
|
|
2987
3030
|
isValidating: true
|
|
2988
3031
|
});
|
|
2989
3032
|
config.onValidationProgress?.(job);
|
|
@@ -2995,16 +3038,7 @@ const useFileUpload = _ref2 => {
|
|
|
2995
3038
|
if (validatedJob.jobError) {
|
|
2996
3039
|
throw new HttpError(400, validatedJob.jobError.message, validatedJob.jobError);
|
|
2997
3040
|
}
|
|
2998
|
-
let results = [];
|
|
2999
3041
|
if (validatedJob.summary.invalid > 0) {
|
|
3000
|
-
const recordsResponse = await getFileImportJobRecords({
|
|
3001
|
-
projectKey,
|
|
3002
|
-
importContainerKey: containerKey,
|
|
3003
|
-
jobId,
|
|
3004
|
-
limit: 500,
|
|
3005
|
-
isValid: false
|
|
3006
|
-
});
|
|
3007
|
-
results = recordsResponse.results;
|
|
3008
3042
|
await safeDeleteContainer({
|
|
3009
3043
|
projectKey,
|
|
3010
3044
|
containerKey
|
|
@@ -3013,7 +3047,8 @@ const useFileUpload = _ref2 => {
|
|
|
3013
3047
|
const result = {
|
|
3014
3048
|
containerKey,
|
|
3015
3049
|
summary: _objectSpread(_objectSpread({}, validatedJob.summary), {}, {
|
|
3016
|
-
|
|
3050
|
+
// TODO: Remove this once the old flow is fully removed
|
|
3051
|
+
results: []
|
|
3017
3052
|
}),
|
|
3018
3053
|
jobId,
|
|
3019
3054
|
job: validatedJob
|
|
@@ -3021,6 +3056,7 @@ const useFileUpload = _ref2 => {
|
|
|
3021
3056
|
setIsUploading(false);
|
|
3022
3057
|
setValidationProgress({
|
|
3023
3058
|
processed: 0,
|
|
3059
|
+
total: 0,
|
|
3024
3060
|
isValidating: false
|
|
3025
3061
|
});
|
|
3026
3062
|
config.onSuccess(result);
|
|
@@ -3250,6 +3286,7 @@ exports.toImportApiResourceType = toImportApiResourceType;
|
|
|
3250
3286
|
exports.uploadFileForImport = uploadFileForImport;
|
|
3251
3287
|
exports.useFetchExportOperations = useFetchExportOperations;
|
|
3252
3288
|
exports.useFetchFileImportJob = useFetchFileImportJob;
|
|
3289
|
+
exports.useFetchFileImportJobRecords = useFetchFileImportJobRecords;
|
|
3253
3290
|
exports.useFetchImportContainerDetails = useFetchImportContainerDetails;
|
|
3254
3291
|
exports.useFetchImportOperations = useFetchImportOperations;
|
|
3255
3292
|
exports.useFetchImportSummaries = useFetchImportSummaries;
|
|
@@ -702,7 +702,7 @@ const pollJobUntilValidated = async _ref => {
|
|
|
702
702
|
_ref$pollingInterval = _ref.pollingInterval,
|
|
703
703
|
pollingInterval = _ref$pollingInterval === void 0 ? 5000 : _ref$pollingInterval,
|
|
704
704
|
_ref$maxAttempts = _ref.maxAttempts,
|
|
705
|
-
maxAttempts = _ref$maxAttempts === void 0 ?
|
|
705
|
+
maxAttempts = _ref$maxAttempts === void 0 ? 200 : _ref$maxAttempts,
|
|
706
706
|
onJobUpdate = _ref.onJobUpdate,
|
|
707
707
|
abortSignal = _ref.abortSignal;
|
|
708
708
|
let attempts = 0;
|
|
@@ -901,6 +901,8 @@ function createFileImportJob(_ref) {
|
|
|
901
901
|
resourceType = _ref.resourceType,
|
|
902
902
|
importContainerKey = _ref.importContainerKey,
|
|
903
903
|
payload = _ref.payload,
|
|
904
|
+
_ref$autoProcess = _ref.autoProcess,
|
|
905
|
+
autoProcess = _ref$autoProcess === void 0 ? false : _ref$autoProcess,
|
|
904
906
|
onProgress = _ref.onProgress,
|
|
905
907
|
abortSignal = _ref.abortSignal;
|
|
906
908
|
const url = getFileImportJobsURL({
|
|
@@ -914,6 +916,7 @@ function createFileImportJob(_ref) {
|
|
|
914
916
|
formData.append('fileType', payload.fileType);
|
|
915
917
|
formData.append('fileName', payload.fileName);
|
|
916
918
|
formData.append('file', payload.file, payload.fileName);
|
|
919
|
+
formData.append('autoProcess', autoProcess ? 'true' : 'false');
|
|
917
920
|
fetchUsingXhr({
|
|
918
921
|
url,
|
|
919
922
|
payload: formData,
|
|
@@ -1331,7 +1334,7 @@ const COLUMN_DELIMITERS = [DELIMITERS.COMMA, DELIMITERS.SEMICOLON, DELIMITERS.PI
|
|
|
1331
1334
|
|
|
1332
1335
|
const FILE_IMPORT_JOB_POLLING_INTERVAL = 2000;
|
|
1333
1336
|
|
|
1334
|
-
const IMPORT_MAX_FILE_SIZE_MB =
|
|
1337
|
+
const IMPORT_MAX_FILE_SIZE_MB = 100;
|
|
1335
1338
|
const IMPORT_MAX_ITEM_COUNT = 500_000;
|
|
1336
1339
|
|
|
1337
1340
|
// =============================================================================
|
|
@@ -2639,6 +2642,39 @@ const useFetchFileImportJob = _ref => {
|
|
|
2639
2642
|
});
|
|
2640
2643
|
};
|
|
2641
2644
|
|
|
2645
|
+
const EMPTY_RESPONSE = {
|
|
2646
|
+
results: [],
|
|
2647
|
+
total: 0,
|
|
2648
|
+
limit: 0,
|
|
2649
|
+
offset: 0,
|
|
2650
|
+
count: 0
|
|
2651
|
+
};
|
|
2652
|
+
const useFetchFileImportJobRecords = _ref => {
|
|
2653
|
+
let projectKey = _ref.projectKey,
|
|
2654
|
+
importContainerKey = _ref.importContainerKey,
|
|
2655
|
+
jobId = _ref.jobId,
|
|
2656
|
+
limit = _ref.limit,
|
|
2657
|
+
offset = _ref.offset,
|
|
2658
|
+
isValid = _ref.isValid,
|
|
2659
|
+
_ref$skip = _ref.skip,
|
|
2660
|
+
skip = _ref$skip === void 0 ? false : _ref$skip;
|
|
2661
|
+
const shouldSkip = skip || !projectKey || !importContainerKey || !jobId;
|
|
2662
|
+
const fetchData = React.useCallback(() => {
|
|
2663
|
+
if (shouldSkip) {
|
|
2664
|
+
return _Promise.resolve(EMPTY_RESPONSE);
|
|
2665
|
+
}
|
|
2666
|
+
return getFileImportJobRecords({
|
|
2667
|
+
projectKey: projectKey,
|
|
2668
|
+
importContainerKey: importContainerKey,
|
|
2669
|
+
jobId: jobId,
|
|
2670
|
+
limit,
|
|
2671
|
+
offset,
|
|
2672
|
+
isValid
|
|
2673
|
+
});
|
|
2674
|
+
}, [shouldSkip, projectKey, importContainerKey, jobId, limit, offset, isValid]);
|
|
2675
|
+
return useFetch(fetchData);
|
|
2676
|
+
};
|
|
2677
|
+
|
|
2642
2678
|
const useFetchImportContainerDetails = _ref => {
|
|
2643
2679
|
let projectKey = _ref.projectKey,
|
|
2644
2680
|
importContainerKey = _ref.importContainerKey,
|
|
@@ -2746,6 +2782,7 @@ const useFileImportJobUpload = _ref => {
|
|
|
2746
2782
|
fileName: config.file.name,
|
|
2747
2783
|
file: config.file
|
|
2748
2784
|
},
|
|
2785
|
+
autoProcess: config.autoProcess,
|
|
2749
2786
|
onProgress: uploadProgress => {
|
|
2750
2787
|
setProgress(uploadProgress);
|
|
2751
2788
|
config.onProgress?.(uploadProgress);
|
|
@@ -2895,7 +2932,7 @@ const useFileUpload = _ref2 => {
|
|
|
2895
2932
|
_ref2$pollingInterval = _ref2.pollingInterval,
|
|
2896
2933
|
pollingInterval = _ref2$pollingInterval === void 0 ? 5000 : _ref2$pollingInterval,
|
|
2897
2934
|
_ref2$maxPollingAttem = _ref2.maxPollingAttempts,
|
|
2898
|
-
maxPollingAttempts = _ref2$maxPollingAttem === void 0 ?
|
|
2935
|
+
maxPollingAttempts = _ref2$maxPollingAttem === void 0 ? 200 : _ref2$maxPollingAttem;
|
|
2899
2936
|
const _React$useState = React.useState(false),
|
|
2900
2937
|
_React$useState2 = _slicedToArray(_React$useState, 2),
|
|
2901
2938
|
isUploading = _React$useState2[0],
|
|
@@ -2906,6 +2943,7 @@ const useFileUpload = _ref2 => {
|
|
|
2906
2943
|
setProgress = _React$useState4[1];
|
|
2907
2944
|
const _React$useState5 = React.useState({
|
|
2908
2945
|
processed: 0,
|
|
2946
|
+
total: 0,
|
|
2909
2947
|
isValidating: false
|
|
2910
2948
|
}),
|
|
2911
2949
|
_React$useState6 = _slicedToArray(_React$useState5, 2),
|
|
@@ -2922,6 +2960,7 @@ const useFileUpload = _ref2 => {
|
|
|
2922
2960
|
setProgress(0);
|
|
2923
2961
|
setValidationProgress({
|
|
2924
2962
|
processed: 0,
|
|
2963
|
+
total: 0,
|
|
2925
2964
|
isValidating: false
|
|
2926
2965
|
});
|
|
2927
2966
|
}, []);
|
|
@@ -2930,15 +2969,18 @@ const useFileUpload = _ref2 => {
|
|
|
2930
2969
|
setProgress(0);
|
|
2931
2970
|
try {
|
|
2932
2971
|
if (useJobBasedFlow) {
|
|
2972
|
+
const totalResources = await countUniqueResourcesInCsv(config.file);
|
|
2933
2973
|
await jobUpload.upload({
|
|
2934
2974
|
file: config.file,
|
|
2935
2975
|
resourceType: config.resourceType,
|
|
2936
2976
|
settings: config.settings,
|
|
2977
|
+
autoProcess: config.autoProcess,
|
|
2937
2978
|
abortSignal: config.abortSignal,
|
|
2938
2979
|
onSuccess: async (jobId, containerKey) => {
|
|
2939
2980
|
try {
|
|
2940
2981
|
setValidationProgress({
|
|
2941
2982
|
processed: 0,
|
|
2983
|
+
total: totalResources,
|
|
2942
2984
|
isValidating: true
|
|
2943
2985
|
});
|
|
2944
2986
|
const validatedJob = await pollJobUntilValidated({
|
|
@@ -2952,6 +2994,7 @@ const useFileUpload = _ref2 => {
|
|
|
2952
2994
|
const processed = job.summary?.total ?? 0;
|
|
2953
2995
|
setValidationProgress({
|
|
2954
2996
|
processed,
|
|
2997
|
+
total: totalResources,
|
|
2955
2998
|
isValidating: true
|
|
2956
2999
|
});
|
|
2957
3000
|
config.onValidationProgress?.(job);
|
|
@@ -2963,16 +3006,7 @@ const useFileUpload = _ref2 => {
|
|
|
2963
3006
|
if (validatedJob.jobError) {
|
|
2964
3007
|
throw new HttpError(400, validatedJob.jobError.message, validatedJob.jobError);
|
|
2965
3008
|
}
|
|
2966
|
-
let results = [];
|
|
2967
3009
|
if (validatedJob.summary.invalid > 0) {
|
|
2968
|
-
const recordsResponse = await getFileImportJobRecords({
|
|
2969
|
-
projectKey,
|
|
2970
|
-
importContainerKey: containerKey,
|
|
2971
|
-
jobId,
|
|
2972
|
-
limit: 500,
|
|
2973
|
-
isValid: false
|
|
2974
|
-
});
|
|
2975
|
-
results = recordsResponse.results;
|
|
2976
3010
|
await safeDeleteContainer({
|
|
2977
3011
|
projectKey,
|
|
2978
3012
|
containerKey
|
|
@@ -2981,7 +3015,8 @@ const useFileUpload = _ref2 => {
|
|
|
2981
3015
|
const result = {
|
|
2982
3016
|
containerKey,
|
|
2983
3017
|
summary: _objectSpread(_objectSpread({}, validatedJob.summary), {}, {
|
|
2984
|
-
|
|
3018
|
+
// TODO: Remove this once the old flow is fully removed
|
|
3019
|
+
results: []
|
|
2985
3020
|
}),
|
|
2986
3021
|
jobId,
|
|
2987
3022
|
job: validatedJob
|
|
@@ -2989,6 +3024,7 @@ const useFileUpload = _ref2 => {
|
|
|
2989
3024
|
setIsUploading(false);
|
|
2990
3025
|
setValidationProgress({
|
|
2991
3026
|
processed: 0,
|
|
3027
|
+
total: 0,
|
|
2992
3028
|
isValidating: false
|
|
2993
3029
|
});
|
|
2994
3030
|
config.onSuccess(result);
|
|
@@ -3071,4 +3107,4 @@ const useFileUpload = _ref2 => {
|
|
|
3071
3107
|
};
|
|
3072
3108
|
};
|
|
3073
3109
|
|
|
3074
|
-
export { ActiveDragDropArea, COLUMN_DELIMITERS, CT_API_DOCS_URL, DELIMITERS, DisabledDropArea, DropAreaWrapper, EnabledDropArea, FILE_IMPORT_JOB_POLLING_INTERVAL, FileDropArea, FileDroppedArea, FileIcon, HttpError, IMPORT_LEGACY_MAX_FILE_SIZE_MB, IMPORT_LEGACY_MAX_ROW_COUNT, IMPORT_MAX_FILE_SIZE_MB, IMPORT_MAX_ITEM_COUNT, IMPORT_TAG_KEYS, IMPORT_TAG_VALUES, ImportStates, InfoBox, InvalidResponseError, LockIcon, NoResourcesToExportError, PollingAbortedError, ProjectKeyNotAvailableError, QueryPredicateError, RESOURCE_TYPE_DOCUMENTATION_LINKS, RESOURCE_TYPE_TEMPLATE_DOWNLOAD_LINKS, TAG_KEY_SOURCE_FILE_UPLOAD, UnexpectedColumnError, UnexpectedOperationStateError, UnexpectedResourceTypeError, UploadSeparator, UploadSettings, UploadingModal, allAutomatedImportOperations, allAutomatedImportOperationsResponse, allFileUploadImportOperations, allFileUploadImportOperationsResponse, appendCsvOrJsonExtensionIfAbsent, assertCancelContainerResponse, assertExportOperationsDownloadFileResponse, assertFileImportJob, assertFileImportJobRecordsResponse, assertFileUploadResponse, assertImportContainer, assertImportContainerPagedResponse, assertImportOperationPagedResponse, assertImportSummary, assertListFileImportJobsResponse, assertPaginatedExportOperationResponse, assertProcessFileImportJobResponse, assertProcessFileResponse, assertResourceType, automatedImportContainerKey, automatedImports, cancelImportContainerByKey, checkIfFileUploadImport, convertFileSizeToKB, countJsonFileItems, countUniqueResourcesInCsv, createFileImportJob, createImportContainerForFileUpload, decodeFileNameFromImportContainerKey, deleteFileImportJob, deleteImportContainer, dropAreaStyles, encodeFileNameWithTimestampToContainerKey, exportOperationsCompleted, exportOperationsProcessing, extractErrorDescriptionFromValidationMessage, fetchExportOperations, fetchImportContainerByKey, fetchImportContainerDetails, fetchImportContainers, fetchImportOperations, fetchImportSummaries, fetchImportSummary, fetchUsingXhr, fetcher, fileUploadImportContainerKey, fileUploadMissingKeysResponse, formatErrorCode, formatKeys, formatQueryString, getCreateImportContainerURL, getDeleteImportContainerURL, getExportOperationsURL, getFileImportJob, getFileImportJobByIdURL, getFileImportJobDeleteURL, getFileImportJobFileType, getFileImportJobProcessURL, getFileImportJobRecords, getFileImportJobRecordsURL, getFileImportJobsListURL, getFileImportJobsURL, getFileUploadErrorsCount, getFileUploadURL, getImportContainerByKeyURL, getImportContainerTasksURL, getImportContainersURL, getImportOperationsURL, getImportState, getImportSummaryURL, getMissingRequiredFields, getProccessFileURL, getRowCount, getValidatedColumns, hasOwnProperty, hasRequiredFields, hasSingleKeyColumn, importContainers, importStatesMap, importsSummaries, invalidFileImportJobRecordsResponse, invalidFileImportJobValidated, invalidFileUploadResponse, isAbortError, isError, isImportJobInitializing, isImportJobProcessing, isImportJobQueued, isImportJobReady, isImportJobRejected, isImportJobTerminal, isImportJobValidated, isResourceType, listFileImportJobs, manualImports, mapFileUploadErrorsToUploadFileErrorRows, mapFormikErrors, mapUploadFileErrorsResponseToUploadFileErrorRows, pollJobUntilValidated, processFileImportJob, processFileImportJobResponse, processUploadedFile, shouldContinuePollingForImportValidation, successfulAutomatedImportOperations, successfulAutomatedImportOperationsResponse, successfulFileUploadImportOperations, successfulFileUploadImportOperationsResponse, toBytes, toImportApiResourceType, uploadFileForImport, useFetchExportOperations, useFetchFileImportJob, useFetchImportContainerDetails, useFetchImportOperations, useFetchImportSummaries, useFileImportJobUpload, useFileUpload, useImportContainerUpload, validFileImportJobProcessing, validFileImportJobQueued, validFileImportJobRecordsResponse, validFileImportJobValidated, validFileUploadResponse, validProcessFileResponse, validateDelimiter };
|
|
3110
|
+
export { ActiveDragDropArea, COLUMN_DELIMITERS, CT_API_DOCS_URL, DELIMITERS, DisabledDropArea, DropAreaWrapper, EnabledDropArea, FILE_IMPORT_JOB_POLLING_INTERVAL, FileDropArea, FileDroppedArea, FileIcon, HttpError, IMPORT_LEGACY_MAX_FILE_SIZE_MB, IMPORT_LEGACY_MAX_ROW_COUNT, IMPORT_MAX_FILE_SIZE_MB, IMPORT_MAX_ITEM_COUNT, IMPORT_TAG_KEYS, IMPORT_TAG_VALUES, ImportStates, InfoBox, InvalidResponseError, LockIcon, NoResourcesToExportError, PollingAbortedError, ProjectKeyNotAvailableError, QueryPredicateError, RESOURCE_TYPE_DOCUMENTATION_LINKS, RESOURCE_TYPE_TEMPLATE_DOWNLOAD_LINKS, TAG_KEY_SOURCE_FILE_UPLOAD, UnexpectedColumnError, UnexpectedOperationStateError, UnexpectedResourceTypeError, UploadSeparator, UploadSettings, UploadingModal, allAutomatedImportOperations, allAutomatedImportOperationsResponse, allFileUploadImportOperations, allFileUploadImportOperationsResponse, appendCsvOrJsonExtensionIfAbsent, assertCancelContainerResponse, assertExportOperationsDownloadFileResponse, assertFileImportJob, assertFileImportJobRecordsResponse, assertFileUploadResponse, assertImportContainer, assertImportContainerPagedResponse, assertImportOperationPagedResponse, assertImportSummary, assertListFileImportJobsResponse, assertPaginatedExportOperationResponse, assertProcessFileImportJobResponse, assertProcessFileResponse, assertResourceType, automatedImportContainerKey, automatedImports, cancelImportContainerByKey, checkIfFileUploadImport, convertFileSizeToKB, countJsonFileItems, countUniqueResourcesInCsv, createFileImportJob, createImportContainerForFileUpload, decodeFileNameFromImportContainerKey, deleteFileImportJob, deleteImportContainer, dropAreaStyles, encodeFileNameWithTimestampToContainerKey, exportOperationsCompleted, exportOperationsProcessing, extractErrorDescriptionFromValidationMessage, fetchExportOperations, fetchImportContainerByKey, fetchImportContainerDetails, fetchImportContainers, fetchImportOperations, fetchImportSummaries, fetchImportSummary, fetchUsingXhr, fetcher, fileUploadImportContainerKey, fileUploadMissingKeysResponse, formatErrorCode, formatKeys, formatQueryString, getCreateImportContainerURL, getDeleteImportContainerURL, getExportOperationsURL, getFileImportJob, getFileImportJobByIdURL, getFileImportJobDeleteURL, getFileImportJobFileType, getFileImportJobProcessURL, getFileImportJobRecords, getFileImportJobRecordsURL, getFileImportJobsListURL, getFileImportJobsURL, getFileUploadErrorsCount, getFileUploadURL, getImportContainerByKeyURL, getImportContainerTasksURL, getImportContainersURL, getImportOperationsURL, getImportState, getImportSummaryURL, getMissingRequiredFields, getProccessFileURL, getRowCount, getValidatedColumns, hasOwnProperty, hasRequiredFields, hasSingleKeyColumn, importContainers, importStatesMap, importsSummaries, invalidFileImportJobRecordsResponse, invalidFileImportJobValidated, invalidFileUploadResponse, isAbortError, isError, isImportJobInitializing, isImportJobProcessing, isImportJobQueued, isImportJobReady, isImportJobRejected, isImportJobTerminal, isImportJobValidated, isResourceType, listFileImportJobs, manualImports, mapFileUploadErrorsToUploadFileErrorRows, mapFormikErrors, mapUploadFileErrorsResponseToUploadFileErrorRows, pollJobUntilValidated, processFileImportJob, processFileImportJobResponse, processUploadedFile, shouldContinuePollingForImportValidation, successfulAutomatedImportOperations, successfulAutomatedImportOperationsResponse, successfulFileUploadImportOperations, successfulFileUploadImportOperationsResponse, toBytes, toImportApiResourceType, uploadFileForImport, useFetchExportOperations, useFetchFileImportJob, useFetchFileImportJobRecords, useFetchImportContainerDetails, useFetchImportOperations, useFetchImportSummaries, useFileImportJobUpload, useFileUpload, useImportContainerUpload, validFileImportJobProcessing, validFileImportJobQueued, validFileImportJobRecordsResponse, validFileImportJobValidated, validFileUploadResponse, validProcessFileResponse, validateDelimiter };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { FileImportJob, CreateFileImportJobParameters, GetFileImportJobParameters, GetFileImportJobRecordsParameters, ProcessFileImportJobParameters, ProcessFileImportJobResponse, DeleteFileImportJobParameters, ListFileImportJobsParameters, ListFileImportJobsResponse, FileImportJobRecordsResponse } from "../@types/index.js";
|
|
2
|
-
export declare function createFileImportJob({ projectKey, resourceType, importContainerKey, payload, onProgress, abortSignal, }: CreateFileImportJobParameters): Promise<FileImportJob>;
|
|
2
|
+
export declare function createFileImportJob({ projectKey, resourceType, importContainerKey, payload, autoProcess, onProgress, abortSignal, }: CreateFileImportJobParameters): Promise<FileImportJob>;
|
|
3
3
|
export declare function getFileImportJob({ projectKey, importContainerKey, jobId, }: GetFileImportJobParameters): Promise<FileImportJob>;
|
|
4
4
|
export declare function getFileImportJobRecords({ projectKey, importContainerKey, jobId, limit, offset, isValid, }: GetFileImportJobRecordsParameters): Promise<FileImportJobRecordsResponse>;
|
|
5
5
|
export declare function processFileImportJob({ projectKey, resourceType, importContainerKey, jobId, action, }: ProcessFileImportJobParameters): Promise<ProcessFileImportJobResponse>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const IMPORT_MAX_FILE_SIZE_MB =
|
|
1
|
+
export declare const IMPORT_MAX_FILE_SIZE_MB = 100;
|
|
2
2
|
export declare const IMPORT_MAX_ITEM_COUNT = 500000;
|
|
3
3
|
/** @deprecated Use IMPORT_MAX_FILE_SIZE_MB instead. Remove after migration. */
|
|
4
4
|
export declare const IMPORT_LEGACY_MAX_FILE_SIZE_MB = 35;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from "./use-fetch-export-operations.js";
|
|
2
2
|
export * from "./use-fetch-file-import-job.js";
|
|
3
|
+
export * from "./use-fetch-file-import-job-records.js";
|
|
3
4
|
export * from "./use-fetch-import-container-details.js";
|
|
4
5
|
export * from "./use-fetch-import-operations.js";
|
|
5
6
|
export * from "./use-fetch-import-summaries.js";
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { FileImportJobRecordsResponse } from "../@types/index.js";
|
|
2
|
+
type UseFetchFileImportJobRecordsConfig = {
|
|
3
|
+
projectKey?: string;
|
|
4
|
+
importContainerKey?: string;
|
|
5
|
+
jobId?: string;
|
|
6
|
+
limit?: number;
|
|
7
|
+
offset?: number;
|
|
8
|
+
isValid?: boolean;
|
|
9
|
+
skip?: boolean;
|
|
10
|
+
};
|
|
11
|
+
export declare const useFetchFileImportJobRecords: ({ projectKey, importContainerKey, jobId, limit, offset, isValid, skip, }: UseFetchFileImportJobRecordsConfig) => {
|
|
12
|
+
data: FileImportJobRecordsResponse | null;
|
|
13
|
+
error: Error | null;
|
|
14
|
+
isLoading: boolean;
|
|
15
|
+
refetch: () => void;
|
|
16
|
+
lastFetchTime: Date;
|
|
17
|
+
};
|
|
18
|
+
export {};
|
|
@@ -4,6 +4,7 @@ export type UseFileImportJobUploadConfig = {
|
|
|
4
4
|
file: File;
|
|
5
5
|
resourceType: ResourceTypeId;
|
|
6
6
|
settings?: ExtendedImportContainerDraft['settings'];
|
|
7
|
+
autoProcess?: boolean;
|
|
7
8
|
onSuccess: (jobId: string, importContainerKey: string) => void;
|
|
8
9
|
onError?: (error: unknown) => void;
|
|
9
10
|
onProgress?: (progress: number) => void;
|
|
@@ -2,12 +2,14 @@ import type { ResourceTypeId } from '@commercetools/importapi-sdk';
|
|
|
2
2
|
import type { ExtendedImportContainerDraft, FileUploadResult, FileImportJob } from "../@types/index.js";
|
|
3
3
|
export type ValidationProgress = {
|
|
4
4
|
processed: number;
|
|
5
|
+
total: number;
|
|
5
6
|
isValidating: boolean;
|
|
6
7
|
};
|
|
7
8
|
export type FileUploadConfig = {
|
|
8
9
|
file: File;
|
|
9
10
|
resourceType: ResourceTypeId;
|
|
10
11
|
settings?: ExtendedImportContainerDraft['settings'];
|
|
12
|
+
autoProcess?: boolean;
|
|
11
13
|
onSuccess: (result: FileUploadResult) => void;
|
|
12
14
|
onError?: (error: unknown) => void;
|
|
13
15
|
onProgress?: (progress: number) => void;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@commercetools-frontend-extensions/operations",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.1.1",
|
|
4
4
|
"license": "Proprietary",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -18,14 +18,14 @@
|
|
|
18
18
|
"react-dropzone": "14.3.8"
|
|
19
19
|
},
|
|
20
20
|
"devDependencies": {
|
|
21
|
-
"@commercetools-frontend/actions-global": "24.
|
|
22
|
-
"@commercetools-frontend/application-components": "24.
|
|
23
|
-
"@commercetools-frontend/application-shell": "24.
|
|
24
|
-
"@commercetools-frontend/application-shell-connectors": "24.
|
|
25
|
-
"@commercetools-frontend/constants": "24.
|
|
26
|
-
"@commercetools-frontend/jest-preset-mc-app": "24.
|
|
27
|
-
"@commercetools-frontend/permissions": "24.
|
|
28
|
-
"@commercetools-frontend/sentry": "24.
|
|
21
|
+
"@commercetools-frontend/actions-global": "24.12.0",
|
|
22
|
+
"@commercetools-frontend/application-components": "24.12.0",
|
|
23
|
+
"@commercetools-frontend/application-shell": "24.12.0",
|
|
24
|
+
"@commercetools-frontend/application-shell-connectors": "24.12.0",
|
|
25
|
+
"@commercetools-frontend/constants": "24.12.0",
|
|
26
|
+
"@commercetools-frontend/jest-preset-mc-app": "24.12.0",
|
|
27
|
+
"@commercetools-frontend/permissions": "24.12.0",
|
|
28
|
+
"@commercetools-frontend/sentry": "24.12.0",
|
|
29
29
|
"@commercetools-frontend/ui-kit": "20.3.0",
|
|
30
30
|
"@emotion/react": "11.14.0",
|
|
31
31
|
"@emotion/styled": "11.14.1",
|
|
@@ -33,6 +33,7 @@ export function createFileImportJob({
|
|
|
33
33
|
resourceType,
|
|
34
34
|
importContainerKey,
|
|
35
35
|
payload,
|
|
36
|
+
autoProcess = false,
|
|
36
37
|
onProgress,
|
|
37
38
|
abortSignal,
|
|
38
39
|
}: CreateFileImportJobParameters): Promise<FileImportJob> {
|
|
@@ -48,6 +49,7 @@ export function createFileImportJob({
|
|
|
48
49
|
formData.append('fileType', payload.fileType)
|
|
49
50
|
formData.append('fileName', payload.fileName)
|
|
50
51
|
formData.append('file', payload.file, payload.fileName)
|
|
52
|
+
formData.append('autoProcess', autoProcess ? 'true' : 'false')
|
|
51
53
|
|
|
52
54
|
fetchUsingXhr({
|
|
53
55
|
url,
|
package/src/@hooks/index.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export * from './use-fetch-export-operations'
|
|
2
2
|
export * from './use-fetch-file-import-job'
|
|
3
|
+
export * from './use-fetch-file-import-job-records'
|
|
3
4
|
export * from './use-fetch-import-container-details'
|
|
4
5
|
export * from './use-fetch-import-operations'
|
|
5
6
|
export * from './use-fetch-import-summaries'
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import React from 'react'
|
|
2
|
+
import { getFileImportJobRecords } from '../@api'
|
|
3
|
+
import type { FileImportJobRecordsResponse } from '../@types'
|
|
4
|
+
import { useFetch } from './use-fetch'
|
|
5
|
+
|
|
6
|
+
type UseFetchFileImportJobRecordsConfig = {
|
|
7
|
+
projectKey?: string
|
|
8
|
+
importContainerKey?: string
|
|
9
|
+
jobId?: string
|
|
10
|
+
limit?: number
|
|
11
|
+
offset?: number
|
|
12
|
+
isValid?: boolean
|
|
13
|
+
skip?: boolean
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
const EMPTY_RESPONSE: FileImportJobRecordsResponse = {
|
|
17
|
+
results: [],
|
|
18
|
+
total: 0,
|
|
19
|
+
limit: 0,
|
|
20
|
+
offset: 0,
|
|
21
|
+
count: 0,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export const useFetchFileImportJobRecords = ({
|
|
25
|
+
projectKey,
|
|
26
|
+
importContainerKey,
|
|
27
|
+
jobId,
|
|
28
|
+
limit,
|
|
29
|
+
offset,
|
|
30
|
+
isValid,
|
|
31
|
+
skip = false,
|
|
32
|
+
}: UseFetchFileImportJobRecordsConfig) => {
|
|
33
|
+
const shouldSkip = skip || !projectKey || !importContainerKey || !jobId
|
|
34
|
+
|
|
35
|
+
const fetchData = React.useCallback(() => {
|
|
36
|
+
if (shouldSkip) {
|
|
37
|
+
return Promise.resolve(EMPTY_RESPONSE)
|
|
38
|
+
}
|
|
39
|
+
return getFileImportJobRecords({
|
|
40
|
+
projectKey: projectKey!,
|
|
41
|
+
importContainerKey: importContainerKey!,
|
|
42
|
+
jobId: jobId!,
|
|
43
|
+
limit,
|
|
44
|
+
offset,
|
|
45
|
+
isValid,
|
|
46
|
+
})
|
|
47
|
+
}, [
|
|
48
|
+
shouldSkip,
|
|
49
|
+
projectKey,
|
|
50
|
+
importContainerKey,
|
|
51
|
+
jobId,
|
|
52
|
+
limit,
|
|
53
|
+
offset,
|
|
54
|
+
isValid,
|
|
55
|
+
])
|
|
56
|
+
|
|
57
|
+
return useFetch<FileImportJobRecordsResponse>(fetchData)
|
|
58
|
+
}
|
|
@@ -17,6 +17,7 @@ export type UseFileImportJobUploadConfig = {
|
|
|
17
17
|
file: File
|
|
18
18
|
resourceType: ResourceTypeId
|
|
19
19
|
settings?: ExtendedImportContainerDraft['settings']
|
|
20
|
+
autoProcess?: boolean
|
|
20
21
|
onSuccess: (jobId: string, importContainerKey: string) => void
|
|
21
22
|
onError?: (error: unknown) => void
|
|
22
23
|
onProgress?: (progress: number) => void
|
|
@@ -64,6 +65,7 @@ export const useFileImportJobUpload = ({
|
|
|
64
65
|
fileName: config.file.name,
|
|
65
66
|
file: config.file,
|
|
66
67
|
},
|
|
68
|
+
autoProcess: config.autoProcess,
|
|
67
69
|
onProgress: (uploadProgress) => {
|
|
68
70
|
setProgress(uploadProgress)
|
|
69
71
|
config.onProgress?.(uploadProgress)
|
|
@@ -2,9 +2,9 @@ import React from 'react'
|
|
|
2
2
|
import type { ResourceTypeId } from '@commercetools/importapi-sdk'
|
|
3
3
|
import { useImportContainerUpload } from './use-import-container-upload'
|
|
4
4
|
import { useFileImportJobUpload } from './use-file-import-job-upload'
|
|
5
|
-
import {
|
|
5
|
+
import { deleteImportContainer } from '../@api'
|
|
6
6
|
import { HttpError, PollingAbortedError } from '../@errors'
|
|
7
|
-
import { pollJobUntilValidated } from '../@utils'
|
|
7
|
+
import { pollJobUntilValidated, countUniqueResourcesInCsv } from '../@utils'
|
|
8
8
|
import type {
|
|
9
9
|
ExtendedImportContainerDraft,
|
|
10
10
|
FileUploadResult,
|
|
@@ -13,6 +13,7 @@ import type {
|
|
|
13
13
|
|
|
14
14
|
export type ValidationProgress = {
|
|
15
15
|
processed: number
|
|
16
|
+
total: number
|
|
16
17
|
isValidating: boolean
|
|
17
18
|
}
|
|
18
19
|
|
|
@@ -20,6 +21,7 @@ export type FileUploadConfig = {
|
|
|
20
21
|
file: File
|
|
21
22
|
resourceType: ResourceTypeId
|
|
22
23
|
settings?: ExtendedImportContainerDraft['settings']
|
|
24
|
+
autoProcess?: boolean
|
|
23
25
|
onSuccess: (result: FileUploadResult) => void
|
|
24
26
|
onError?: (error: unknown) => void
|
|
25
27
|
onProgress?: (progress: number) => void
|
|
@@ -53,13 +55,14 @@ export const useFileUpload = ({
|
|
|
53
55
|
projectKey,
|
|
54
56
|
useJobBasedFlow = false,
|
|
55
57
|
pollingInterval = 5000,
|
|
56
|
-
maxPollingAttempts =
|
|
58
|
+
maxPollingAttempts = 200,
|
|
57
59
|
}: FileUploadOptions) => {
|
|
58
60
|
const [isUploading, setIsUploading] = React.useState(false)
|
|
59
61
|
const [progress, setProgress] = React.useState(0)
|
|
60
62
|
const [validationProgress, setValidationProgress] =
|
|
61
63
|
React.useState<ValidationProgress>({
|
|
62
64
|
processed: 0,
|
|
65
|
+
total: 0,
|
|
63
66
|
isValidating: false,
|
|
64
67
|
})
|
|
65
68
|
|
|
@@ -69,7 +72,7 @@ export const useFileUpload = ({
|
|
|
69
72
|
const resetState = React.useCallback(() => {
|
|
70
73
|
setIsUploading(false)
|
|
71
74
|
setProgress(0)
|
|
72
|
-
setValidationProgress({ processed: 0, isValidating: false })
|
|
75
|
+
setValidationProgress({ processed: 0, total: 0, isValidating: false })
|
|
73
76
|
}, [])
|
|
74
77
|
|
|
75
78
|
const upload = React.useCallback(
|
|
@@ -79,14 +82,21 @@ export const useFileUpload = ({
|
|
|
79
82
|
|
|
80
83
|
try {
|
|
81
84
|
if (useJobBasedFlow) {
|
|
85
|
+
const totalResources = await countUniqueResourcesInCsv(config.file)
|
|
86
|
+
|
|
82
87
|
await jobUpload.upload({
|
|
83
88
|
file: config.file,
|
|
84
89
|
resourceType: config.resourceType,
|
|
85
90
|
settings: config.settings,
|
|
91
|
+
autoProcess: config.autoProcess,
|
|
86
92
|
abortSignal: config.abortSignal,
|
|
87
93
|
onSuccess: async (jobId, containerKey) => {
|
|
88
94
|
try {
|
|
89
|
-
setValidationProgress({
|
|
95
|
+
setValidationProgress({
|
|
96
|
+
processed: 0,
|
|
97
|
+
total: totalResources,
|
|
98
|
+
isValidating: true,
|
|
99
|
+
})
|
|
90
100
|
|
|
91
101
|
const validatedJob = await pollJobUntilValidated({
|
|
92
102
|
projectKey,
|
|
@@ -97,7 +107,11 @@ export const useFileUpload = ({
|
|
|
97
107
|
abortSignal: config.abortSignal,
|
|
98
108
|
onJobUpdate: (job) => {
|
|
99
109
|
const processed = job.summary?.total ?? 0
|
|
100
|
-
setValidationProgress({
|
|
110
|
+
setValidationProgress({
|
|
111
|
+
processed,
|
|
112
|
+
total: totalResources,
|
|
113
|
+
isValidating: true,
|
|
114
|
+
})
|
|
101
115
|
config.onValidationProgress?.(job)
|
|
102
116
|
},
|
|
103
117
|
})
|
|
@@ -112,16 +126,7 @@ export const useFileUpload = ({
|
|
|
112
126
|
)
|
|
113
127
|
}
|
|
114
128
|
|
|
115
|
-
let results: FileUploadResult['summary']['results'] = []
|
|
116
129
|
if (validatedJob.summary.invalid > 0) {
|
|
117
|
-
const recordsResponse = await getFileImportJobRecords({
|
|
118
|
-
projectKey,
|
|
119
|
-
importContainerKey: containerKey,
|
|
120
|
-
jobId,
|
|
121
|
-
limit: 500,
|
|
122
|
-
isValid: false,
|
|
123
|
-
})
|
|
124
|
-
results = recordsResponse.results
|
|
125
130
|
await safeDeleteContainer({ projectKey, containerKey })
|
|
126
131
|
}
|
|
127
132
|
|
|
@@ -129,14 +134,19 @@ export const useFileUpload = ({
|
|
|
129
134
|
containerKey,
|
|
130
135
|
summary: {
|
|
131
136
|
...validatedJob.summary,
|
|
132
|
-
|
|
137
|
+
// TODO: Remove this once the old flow is fully removed
|
|
138
|
+
results: [],
|
|
133
139
|
},
|
|
134
140
|
jobId,
|
|
135
141
|
job: validatedJob,
|
|
136
142
|
}
|
|
137
143
|
|
|
138
144
|
setIsUploading(false)
|
|
139
|
-
setValidationProgress({
|
|
145
|
+
setValidationProgress({
|
|
146
|
+
processed: 0,
|
|
147
|
+
total: 0,
|
|
148
|
+
isValidating: false,
|
|
149
|
+
})
|
|
140
150
|
config.onSuccess(result)
|
|
141
151
|
} catch (error) {
|
|
142
152
|
await safeDeleteContainer({ projectKey, containerKey })
|