@overmap-ai/core 1.0.58-asset-description.5 → 1.0.58-asset-description.7

package/dist/overmap-core.js

@@ -5378,6 +5378,16 @@ class CategoryService extends BaseApiService {
     store.dispatch(setCategories(result));
   }
 }
+function chunkArray(arr, chunkSize) {
+  const chunks = [];
+  let index2 = 0;
+  const arrLength = arr.length;
+  while (index2 < arrLength) {
+    chunks.push(arr.slice(index2, index2 += chunkSize));
+  }
+  return chunks;
+}
+const BULK_ADD_ASSET_BATCH_SIZE = 1e3;
 class AssetService extends BaseApiService {
   // Basic CRUD functions
   add(asset, workspaceId) {
@@ -5454,36 +5464,49 @@ class AssetService extends BaseApiService {
       throw err;
     });
   }
-  async addBatch(assetsToCreate, workspaceId, assetTypeId) {
+  addBulk(assetsToCreate, workspaceId, assetTypeId) {
+    const { store } = this.client;
     const fullAssets = assetsToCreate.map((asset) => {
       return { ...offline(asset), submitted_at: (/* @__PURE__ */ new Date()).toISOString() };
     });
-    const { store } = this.client;
-    store.dispatch(addAssetsInBatches(fullAssets));
-    const promise = this.client.enqueueRequest({
-      description: "Batch create assets",
-      method: HttpMethod.POST,
-      url: `/assets/types/${assetTypeId}/add-assets/`,
-      queryParams: {
-        workspace_id: workspaceId.toString()
-      },
-      payload: {
-        assets: fullAssets
-      },
-      blockers: [assetTypeId],
-      blocks: fullAssets.map((c) => c.offline_id)
-    });
-    void promise.then((result) => {
-      for (const assets of Object.values(result)) {
-        store.dispatch(updateAsset(assets));
-      }
-    }).catch((e) => {
-      for (const asset of fullAssets) {
-        store.dispatch(removeAsset(asset.offline_id));
-      }
-      throw e;
+    const assetBatches = chunkArray(fullAssets, BULK_ADD_ASSET_BATCH_SIZE).map((assetBatch) => {
+      return {
+        batchId: v4(),
+        payload: {
+          assets: assetBatch
+        }
+      };
     });
-    return promise;
+    store.dispatch(addAssetsInBatches(fullAssets));
+    let prevBatchId = null;
+    const batchPromises = Promise.all(
+      assetBatches.map((assetBatch) => {
+        const { batchId, payload } = assetBatch;
+        const promise = this.client.enqueueRequest({
+          description: "Batch create assets",
+          method: HttpMethod.POST,
+          url: `/assets/types/${assetTypeId}/add-assets/`,
+          queryParams: {
+            workspace_id: workspaceId.toString()
+          },
+          payload,
+          blockers: prevBatchId ? [assetTypeId, prevBatchId] : [assetTypeId],
+          blocks: assetBatch.payload.assets.map((c) => c.offline_id).concat([batchId])
+        });
+        promise.then((result) => {
+          for (const assets of Object.values(result)) {
+            store.dispatch(updateAsset(assets));
+          }
+        }).catch(() => {
+          for (const asset of assetBatch.payload.assets) {
+            store.dispatch(removeAsset(asset.offline_id));
+          }
+        });
+        prevBatchId = assetBatch.batchId;
+        return promise;
+      })
+    );
+    return [fullAssets, batchPromises.then((result) => result)];
   }
   async refreshStore() {
     const { store } = this.client;
@@ -7269,6 +7292,7 @@ const separateFilesFromValues = (values) => {
   }
   return { values: newValues, files };
 };
+const MAX_BULK_ADD_SUBMISSIONS = 1e3;
 class UserFormSubmissionService extends BaseApiService {
   constructor() {
     super(...arguments);
@@ -7355,103 +7379,129 @@ class UserFormSubmissionService extends BaseApiService {
     const offlineSubmissions = [];
     const offlineAttachments = [];
     const submissionOfflineIds = [];
-    const submissionsPayload = [];
-    const attachmentsPayload = [];
-    let files = {};
+    const allFilesRecord = {};
     const { values: fileSeperatedCommonFieldValues, files: commonFiles } = separateFilesFromValues(commonFieldValues);
-    files = commonFiles;
     const submittedAt = (/* @__PURE__ */ new Date()).toISOString();
     const createdBy = store.getState().userReducer.currentUser.id;
-    for (const assetId in fieldValuesByAsset) {
-      const { values: fileSeperatedSubmissionSpecificValues, files: submissionSpecificFiles } = separateFilesFromValues(fieldValuesByAsset[assetId] ?? {});
-      files = Object.assign(files, submissionSpecificFiles);
-      const submissionValues = { ...fileSeperatedCommonFieldValues, ...fileSeperatedSubmissionSpecificValues };
-      const submission = offline({
-        form_revision: formRevision,
-        values: submissionValues,
-        created_by: createdBy,
-        submitted_at: submittedAt,
-        asset: assetId
-      });
-      submissionOfflineIds.push(submission.offline_id);
-      submissionsPayload.push({
-        offline_id: submission.offline_id,
-        asset_id: assetId,
-        form_data: fileSeperatedSubmissionSpecificValues
-      });
-      offlineSubmissions.push(submission);
-      for (const [fieldIdentifier, fileArray] of Object.entries(files)) {
-        for (const file of fileArray) {
-          const sha1 = await hashFile(file);
-          await this.client.files.addCache(file, sha1);
-          const offlineAttachment = offline({
-            file_name: file.name,
-            file_sha1: sha1,
-            file: URL.createObjectURL(file),
-            submission: submission.offline_id,
-            field_identifier: fieldIdentifier
+    const assetIdBatches = chunkArray(Object.keys(fieldValuesByAsset), MAX_BULK_ADD_SUBMISSIONS);
+    const bulkAddBatches = await Promise.all(
+      assetIdBatches.map(async (assetIdBatch) => {
+        const batchId = v4();
+        const submissionsPayload = [];
+        const attachmentsPayload = [];
+        let files = { ...commonFiles };
+        for (const assetId of assetIdBatch) {
+          const { values: fileSeperatedSubmissionSpecificValues, files: submissionSpecificFiles } = separateFilesFromValues(fieldValuesByAsset[assetId] ?? {});
+          files = Object.assign(files, submissionSpecificFiles);
+          const submissionValues = {
+            ...fileSeperatedCommonFieldValues,
+            ...fileSeperatedSubmissionSpecificValues
+          };
+          const submission = offline({
+            form_revision: formRevision,
+            values: submissionValues,
+            created_by: createdBy,
+            submitted_at: submittedAt,
+            asset: assetId
           });
-          offlineAttachments.push(offlineAttachment);
-          attachmentsPayload.push({
-            offline_id: offlineAttachment.offline_id,
-            submission_id: submission.offline_id,
-            sha1,
-            name: file.name,
-            field_identifier: fieldIdentifier
+          submissionOfflineIds.push(submission.offline_id);
+          submissionsPayload.push({
+            offline_id: submission.offline_id,
+            asset_id: assetId,
+            form_data: fileSeperatedSubmissionSpecificValues
           });
+          offlineSubmissions.push(submission);
+          for (const [fieldIdentifier, fileArray] of Object.entries(files)) {
+            for (const file of fileArray) {
+              const sha1 = await hashFile(file);
+              await this.client.files.addCache(file, sha1);
+              const offlineAttachment = offline({
+                file_name: file.name,
+                file_sha1: sha1,
+                file: URL.createObjectURL(file),
+                submission: submission.offline_id,
+                field_identifier: fieldIdentifier
+              });
+              offlineAttachments.push(offlineAttachment);
+              attachmentsPayload.push({
+                offline_id: offlineAttachment.offline_id,
+                submission_id: submission.offline_id,
+                sha1,
+                name: file.name,
+                field_identifier: fieldIdentifier
+              });
+            }
+          }
         }
-      }
-    }
-    const filesRecord = {};
-    for (const file of Object.values(files).flat()) {
-      const sha1 = await hashFile(file);
-      filesRecord[sha1] = {
-        sha1,
-        extension: file.name.split(".").pop() || "",
-        file_type: file.type,
-        size: file.size
-      };
-    }
+        const filePaylods = [];
+        for (const file of Object.values(files).flat()) {
+          const sha1 = await hashFile(file);
+          const filePayload = {
+            sha1,
+            extension: file.name.split(".").pop() || "",
+            file_type: file.type,
+            size: file.size
+          };
+          allFilesRecord[sha1] = filePayload;
+          filePaylods.push(filePayload);
+        }
+        return {
+          batchId,
+          payload: {
+            form_data: fileSeperatedCommonFieldValues,
+            submitted_at: submittedAt,
+            submissions: submissionsPayload,
+            attachments: attachmentsPayload,
+            files: filePaylods
+          }
+        };
+      })
+    );
     store.dispatch(addFormSubmissions(offlineSubmissions));
     store.dispatch(addFormSubmissionAttachments(offlineAttachments));
-    const promise = this.client.enqueueRequest({
-      description: "Bulk add form submissions",
-      method: HttpMethod.POST,
-      url: `/forms/revisions/${formRevision}/bulk-respond/`,
-      payload: {
-        form_data: fileSeperatedCommonFieldValues,
-        submitted_at: submittedAt,
-        submissions: submissionsPayload,
-        attachments: attachmentsPayload,
-        files: Object.values(filesRecord)
-      },
-      blockers: Object.keys(fieldValuesByAsset),
-      blocks: submissionOfflineIds
-    });
-    promise.then(({ submissions, attachments, presigned_urls }) => {
-      store.dispatch(updateFormSubmissions(submissions));
-      store.dispatch(updateFormSubmissionAttachments(attachments));
-      for (const [sha1, presigned_url] of Object.entries(presigned_urls)) {
-        const file = filesRecord[sha1];
-        if (!file)
-          continue;
-        void this.client.enqueueRequest({
-          url: presigned_url.url,
-          description: "Upload file",
+    let prevBatchId = null;
+    const batchPromises = Promise.all(
+      bulkAddBatches.map((batch) => {
+        const { payload, batchId } = batch;
+        const batchAssetIds = payload.submissions.map((x) => x.asset_id);
+        const batchSubmissionOfflineIds = payload.submissions.map((x) => x.offline_id);
+        const batchAttachmentsOfflineIds = payload.attachments.map((x) => x.offline_id);
+        const promise = this.client.enqueueRequest({
+          description: "Bulk add form submissions",
           method: HttpMethod.POST,
-          isExternalUrl: true,
-          isAuthNeeded: false,
-          attachmentHash: sha1,
-          blockers: [`s3-${file.sha1}.${file.extension}`],
-          blocks: [sha1],
-          s3url: presigned_url
+          url: `/forms/revisions/${formRevision}/bulk-respond/`,
+          payload,
+          blockers: prevBatchId ? batchAssetIds.concat([prevBatchId]) : batchAssetIds,
+          blocks: batchSubmissionOfflineIds.concat([batchId])
         });
-      }
-    }).catch(() => {
-      store.dispatch(deleteFormSubmissions(submissionOfflineIds));
-      store.dispatch(deleteFormSubmissionAttachments(offlineAttachments.map((x) => x.offline_id)));
-    });
-    return [offlineSubmissions, promise.then(({ submissions }) => submissions)];
+        promise.then(({ submissions, attachments, presigned_urls }) => {
+          store.dispatch(updateFormSubmissions(submissions));
+          store.dispatch(updateFormSubmissionAttachments(attachments));
+          for (const [sha1, presigned_url] of Object.entries(presigned_urls)) {
+            const file = allFilesRecord[sha1];
+            if (!file)
+              continue;
+            void this.client.enqueueRequest({
+              url: presigned_url.url,
+              description: "Upload file",
+              method: HttpMethod.POST,
+              isExternalUrl: true,
+              isAuthNeeded: false,
+              attachmentHash: sha1,
+              blockers: [`s3-${file.sha1}.${file.extension}`],
+              blocks: [sha1],
+              s3url: presigned_url
+            });
+          }
+        }).catch(() => {
+          store.dispatch(deleteFormSubmissions(batchSubmissionOfflineIds));
+          store.dispatch(deleteFormSubmissionAttachments(batchAttachmentsOfflineIds));
+        });
+        prevBatchId = batchId;
+        return promise;
+      })
+    );
+    return [offlineSubmissions, batchPromises.then((results) => results.map(({ submissions }) => submissions))];
   }
   update(submission) {
     const { store } = this.client;