@overmap-ai/core 1.0.58-asset-description.5 → 1.0.58-asset-description.7

package/dist/overmap-core.umd.cjs

@@ -5368,6 +5368,16 @@ var __publicField = (obj, key, value) => {
       store.dispatch(setCategories(result));
     }
   }
+  function chunkArray(arr, chunkSize) {
+    const chunks = [];
+    let index2 = 0;
+    const arrLength = arr.length;
+    while (index2 < arrLength) {
+      chunks.push(arr.slice(index2, index2 += chunkSize));
+    }
+    return chunks;
+  }
+  const BULK_ADD_ASSET_BATCH_SIZE = 1e3;
   class AssetService extends BaseApiService {
     // Basic CRUD functions
     add(asset, workspaceId) {
@@ -5444,36 +5454,49 @@ var __publicField = (obj, key, value) => {
         throw err;
       });
     }
-    async addBatch(assetsToCreate, workspaceId, assetTypeId) {
+    addBulk(assetsToCreate, workspaceId, assetTypeId) {
+      const { store } = this.client;
       const fullAssets = assetsToCreate.map((asset) => {
         return { ...offline(asset), submitted_at: (/* @__PURE__ */ new Date()).toISOString() };
       });
-      const { store } = this.client;
-      store.dispatch(addAssetsInBatches(fullAssets));
-      const promise = this.client.enqueueRequest({
-        description: "Batch create assets",
-        method: HttpMethod.POST,
-        url: `/assets/types/${assetTypeId}/add-assets/`,
-        queryParams: {
-          workspace_id: workspaceId.toString()
-        },
-        payload: {
-          assets: fullAssets
-        },
-        blockers: [assetTypeId],
-        blocks: fullAssets.map((c) => c.offline_id)
-      });
-      void promise.then((result) => {
-        for (const assets of Object.values(result)) {
-          store.dispatch(updateAsset(assets));
-        }
-      }).catch((e) => {
-        for (const asset of fullAssets) {
-          store.dispatch(removeAsset(asset.offline_id));
-        }
-        throw e;
+      const assetBatches = chunkArray(fullAssets, BULK_ADD_ASSET_BATCH_SIZE).map((assetBatch) => {
+        return {
+          batchId: uuid.v4(),
+          payload: {
+            assets: assetBatch
+          }
+        };
       });
-      return promise;
+      store.dispatch(addAssetsInBatches(fullAssets));
+      let prevBatchId = null;
+      const batchPromises = Promise.all(
+        assetBatches.map((assetBatch) => {
+          const { batchId, payload } = assetBatch;
+          const promise = this.client.enqueueRequest({
+            description: "Batch create assets",
+            method: HttpMethod.POST,
+            url: `/assets/types/${assetTypeId}/add-assets/`,
+            queryParams: {
+              workspace_id: workspaceId.toString()
+            },
+            payload,
+            blockers: prevBatchId ? [assetTypeId, prevBatchId] : [assetTypeId],
+            blocks: assetBatch.payload.assets.map((c) => c.offline_id).concat([batchId])
+          });
+          promise.then((result) => {
+            for (const assets of Object.values(result)) {
+              store.dispatch(updateAsset(assets));
+            }
+          }).catch(() => {
+            for (const asset of assetBatch.payload.assets) {
+              store.dispatch(removeAsset(asset.offline_id));
+            }
+          });
+          prevBatchId = assetBatch.batchId;
+          return promise;
+        })
+      );
+      return [fullAssets, batchPromises.then((result) => result)];
     }
     async refreshStore() {
       const { store } = this.client;
@@ -7259,6 +7282,7 @@ var __publicField = (obj, key, value) => {
     }
     return { values: newValues, files };
   };
+  const MAX_BULK_ADD_SUBMISSIONS = 1e3;
   class UserFormSubmissionService extends BaseApiService {
     constructor() {
       super(...arguments);
@@ -7345,103 +7369,129 @@ var __publicField = (obj, key, value) => {
       const offlineSubmissions = [];
       const offlineAttachments = [];
       const submissionOfflineIds = [];
-      const submissionsPayload = [];
-      const attachmentsPayload = [];
-      let files = {};
+      const allFilesRecord = {};
       const { values: fileSeperatedCommonFieldValues, files: commonFiles } = separateFilesFromValues(commonFieldValues);
-      files = commonFiles;
       const submittedAt = (/* @__PURE__ */ new Date()).toISOString();
       const createdBy = store.getState().userReducer.currentUser.id;
-      for (const assetId in fieldValuesByAsset) {
-        const { values: fileSeperatedSubmissionSpecificValues, files: submissionSpecificFiles } = separateFilesFromValues(fieldValuesByAsset[assetId] ?? {});
-        files = Object.assign(files, submissionSpecificFiles);
-        const submissionValues = { ...fileSeperatedCommonFieldValues, ...fileSeperatedSubmissionSpecificValues };
-        const submission = offline({
-          form_revision: formRevision,
-          values: submissionValues,
-          created_by: createdBy,
-          submitted_at: submittedAt,
-          asset: assetId
-        });
-        submissionOfflineIds.push(submission.offline_id);
-        submissionsPayload.push({
-          offline_id: submission.offline_id,
-          asset_id: assetId,
-          form_data: fileSeperatedSubmissionSpecificValues
-        });
-        offlineSubmissions.push(submission);
-        for (const [fieldIdentifier, fileArray] of Object.entries(files)) {
-          for (const file of fileArray) {
-            const sha1 = await hashFile(file);
-            await this.client.files.addCache(file, sha1);
-            const offlineAttachment = offline({
-              file_name: file.name,
-              file_sha1: sha1,
-              file: URL.createObjectURL(file),
-              submission: submission.offline_id,
-              field_identifier: fieldIdentifier
+      const assetIdBatches = chunkArray(Object.keys(fieldValuesByAsset), MAX_BULK_ADD_SUBMISSIONS);
+      const bulkAddBatches = await Promise.all(
+        assetIdBatches.map(async (assetIdBatch) => {
+          const batchId = uuid.v4();
+          const submissionsPayload = [];
+          const attachmentsPayload = [];
+          let files = { ...commonFiles };
+          for (const assetId of assetIdBatch) {
+            const { values: fileSeperatedSubmissionSpecificValues, files: submissionSpecificFiles } = separateFilesFromValues(fieldValuesByAsset[assetId] ?? {});
+            files = Object.assign(files, submissionSpecificFiles);
+            const submissionValues = {
+              ...fileSeperatedCommonFieldValues,
+              ...fileSeperatedSubmissionSpecificValues
+            };
+            const submission = offline({
+              form_revision: formRevision,
+              values: submissionValues,
+              created_by: createdBy,
+              submitted_at: submittedAt,
+              asset: assetId
             });
-            offlineAttachments.push(offlineAttachment);
-            attachmentsPayload.push({
-              offline_id: offlineAttachment.offline_id,
-              submission_id: submission.offline_id,
-              sha1,
-              name: file.name,
-              field_identifier: fieldIdentifier
+            submissionOfflineIds.push(submission.offline_id);
+            submissionsPayload.push({
+              offline_id: submission.offline_id,
+              asset_id: assetId,
+              form_data: fileSeperatedSubmissionSpecificValues
             });
+            offlineSubmissions.push(submission);
+            for (const [fieldIdentifier, fileArray] of Object.entries(files)) {
+              for (const file of fileArray) {
+                const sha1 = await hashFile(file);
+                await this.client.files.addCache(file, sha1);
+                const offlineAttachment = offline({
+                  file_name: file.name,
+                  file_sha1: sha1,
+                  file: URL.createObjectURL(file),
+                  submission: submission.offline_id,
+                  field_identifier: fieldIdentifier
+                });
+                offlineAttachments.push(offlineAttachment);
+                attachmentsPayload.push({
+                  offline_id: offlineAttachment.offline_id,
+                  submission_id: submission.offline_id,
+                  sha1,
+                  name: file.name,
+                  field_identifier: fieldIdentifier
+                });
+              }
+            }
           }
-        }
-      }
-      const filesRecord = {};
-      for (const file of Object.values(files).flat()) {
-        const sha1 = await hashFile(file);
-        filesRecord[sha1] = {
-          sha1,
-          extension: file.name.split(".").pop() || "",
-          file_type: file.type,
-          size: file.size
-        };
-      }
+          const filePaylods = [];
+          for (const file of Object.values(files).flat()) {
+            const sha1 = await hashFile(file);
+            const filePayload = {
+              sha1,
+              extension: file.name.split(".").pop() || "",
+              file_type: file.type,
+              size: file.size
+            };
+            allFilesRecord[sha1] = filePayload;
+            filePaylods.push(filePayload);
+          }
+          return {
+            batchId,
+            payload: {
+              form_data: fileSeperatedCommonFieldValues,
+              submitted_at: submittedAt,
+              submissions: submissionsPayload,
+              attachments: attachmentsPayload,
+              files: filePaylods
+            }
+          };
+        })
+      );
       store.dispatch(addFormSubmissions(offlineSubmissions));
       store.dispatch(addFormSubmissionAttachments(offlineAttachments));
-      const promise = this.client.enqueueRequest({
-        description: "Bulk add form submissions",
-        method: HttpMethod.POST,
-        url: `/forms/revisions/${formRevision}/bulk-respond/`,
-        payload: {
-          form_data: fileSeperatedCommonFieldValues,
-          submitted_at: submittedAt,
-          submissions: submissionsPayload,
-          attachments: attachmentsPayload,
-          files: Object.values(filesRecord)
-        },
-        blockers: Object.keys(fieldValuesByAsset),
-        blocks: submissionOfflineIds
-      });
-      promise.then(({ submissions, attachments, presigned_urls }) => {
-        store.dispatch(updateFormSubmissions(submissions));
-        store.dispatch(updateFormSubmissionAttachments(attachments));
-        for (const [sha1, presigned_url] of Object.entries(presigned_urls)) {
-          const file = filesRecord[sha1];
-          if (!file)
-            continue;
-          void this.client.enqueueRequest({
-            url: presigned_url.url,
-            description: "Upload file",
+      let prevBatchId = null;
+      const batchPromises = Promise.all(
+        bulkAddBatches.map((batch) => {
+          const { payload, batchId } = batch;
+          const batchAssetIds = payload.submissions.map((x) => x.asset_id);
+          const batchSubmissionOfflineIds = payload.submissions.map((x) => x.offline_id);
+          const batchAttachmentsOfflineIds = payload.attachments.map((x) => x.offline_id);
+          const promise = this.client.enqueueRequest({
+            description: "Bulk add form submissions",
             method: HttpMethod.POST,
-            isExternalUrl: true,
-            isAuthNeeded: false,
-            attachmentHash: sha1,
-            blockers: [`s3-${file.sha1}.${file.extension}`],
-            blocks: [sha1],
-            s3url: presigned_url
+            url: `/forms/revisions/${formRevision}/bulk-respond/`,
+            payload,
+            blockers: prevBatchId ? batchAssetIds.concat([prevBatchId]) : batchAssetIds,
+            blocks: batchSubmissionOfflineIds.concat([batchId])
           });
-        }
-      }).catch(() => {
-        store.dispatch(deleteFormSubmissions(submissionOfflineIds));
-        store.dispatch(deleteFormSubmissionAttachments(offlineAttachments.map((x) => x.offline_id)));
-      });
-      return [offlineSubmissions, promise.then(({ submissions }) => submissions)];
+          promise.then(({ submissions, attachments, presigned_urls }) => {
+            store.dispatch(updateFormSubmissions(submissions));
+            store.dispatch(updateFormSubmissionAttachments(attachments));
+            for (const [sha1, presigned_url] of Object.entries(presigned_urls)) {
+              const file = allFilesRecord[sha1];
+              if (!file)
+                continue;
+              void this.client.enqueueRequest({
+                url: presigned_url.url,
+                description: "Upload file",
+                method: HttpMethod.POST,
+                isExternalUrl: true,
+                isAuthNeeded: false,
+                attachmentHash: sha1,
+                blockers: [`s3-${file.sha1}.${file.extension}`],
+                blocks: [sha1],
+                s3url: presigned_url
+              });
+            }
+          }).catch(() => {
+            store.dispatch(deleteFormSubmissions(batchSubmissionOfflineIds));
+            store.dispatch(deleteFormSubmissionAttachments(batchAttachmentsOfflineIds));
+          });
+          prevBatchId = batchId;
+          return promise;
+        })
+      );
+      return [offlineSubmissions, batchPromises.then((results) => results.map(({ submissions }) => submissions))];
     }
     update(submission) {
       const { store } = this.client;