@objectstack/rest 12.4.0 → 12.6.0

package/dist/index.js

@@ -519,6 +519,17 @@ async function coerceRow(rawRow, metaMap, ctx) {
 }
 
 // src/import-runner.ts
+import { bulkWrite, withTransientRetry } from "@objectstack/core";
+function extractRecordId(rec) {
+  const id = rec?.id ?? rec?.record?.id;
+  return id != null ? String(id) : void 0;
+}
+function toFailedResult(rowNo, err) {
+  const code = err?.code ?? "IMPORT_ROW_FAILED";
+  const message = typeof err?.message === "string" ? err.message.slice(0, 300) : "Row failed";
+  return { row: rowNo, ok: false, action: "failed", error: message, code };
+}
+var MAX_CREATE_BATCH_SIZE = 200;
 function runImport(opts) {
   const {
     p,
@@ -606,7 +617,7 @@ function runImport(opts) {
     return recs[0];
   };
   const writeCtx = { ...context ?? {}, skipAutomations: !runAutomations };
-  const results = [];
+  const results = new Array(rows.length);
   let okCount = 0, errCount = 0, created = 0, updated = 0, skipped = 0;
   let cancelled = false;
   const snapshot = (processed) => ({
@@ -617,6 +628,47 @@ function runImport(opts) {
     skipped,
     errors: errCount
   });
+  const canBulkCreate = typeof p.createManyData === "function";
+  const pendingCreates = [];
+  const flushPendingCreates = async () => {
+    if (pendingCreates.length === 0) return;
+    const batch = pendingCreates.splice(0, pendingCreates.length);
+    const writeResults = await bulkWrite(
+      batch.map((b) => b.data),
+      {
+        // Flush cadence follows progressEvery, but the write batch itself is
+        // capped independently — a caller-supplied progressEvery far above
+        // the issue's suggested 100-500 rows/batch must not translate into
+        // one oversized multi-row INSERT statement.
+        batchSize: Math.min(progressEvery, MAX_CREATE_BATCH_SIZE),
+        writeBatch: (chunk) => p.createManyData({
+          object: objectName,
+          records: chunk,
+          context: writeCtx,
+          ...environmentId ? { environmentId } : {}
+        }).then((r) => r.records),
+        writeOne: (row) => p.createData({
+          object: objectName,
+          data: row,
+          context: writeCtx,
+          ...environmentId ? { environmentId } : {}
+        })
+      }
+    );
+    for (const res of writeResults) {
+      const { index, rowNo } = batch[res.index];
+      if (res.ok) {
+        const id = extractRecordId(res.record);
+        okCount++;
+        created++;
+        if (collectUndo && id != null) undoLog.created.push(id);
+        results[index] = { row: rowNo, ok: true, action: "created", id };
+      } else {
+        errCount++;
+        results[index] = toFailedResult(rowNo, res.error);
+      }
+    }
+  };
   return (async () => {
     for (let i = 0; i < rows.length; i++) {
       const rowNo = i + 1;
@@ -630,7 +682,7 @@ function runImport(opts) {
         if (errors.length > 0) {
           const first2 = errors[0];
           errCount++;
-          results.push({ row: rowNo, ok: false, action: "failed", field: first2.field, code: first2.code, error: first2.message });
+          results[i] = { row: rowNo, ok: false, action: "failed", field: first2.field, code: first2.code, error: first2.message };
         } else {
           let existing = "none";
           let handled = false;
@@ -638,11 +690,11 @@ function runImport(opts) {
             existing = await findExisting(data);
             if (existing === "ambiguous") {
               errCount++;
-              results.push({ row: rowNo, ok: false, action: "failed", code: "AMBIGUOUS_MATCH", error: `matchFields matched more than one ${objectName} record` });
+              results[i] = { row: rowNo, ok: false, action: "failed", code: "AMBIGUOUS_MATCH", error: `matchFields matched more than one ${objectName} record` };
               handled = true;
             } else if (existing === "blank" && (skipBlankMatchKey || writeMode === "update")) {
               skipped++;
-              results.push({ row: rowNo, ok: true, action: "skipped", code: "BLANK_MATCH_KEY" });
+              results[i] = { row: rowNo, ok: true, action: "skipped", code: "BLANK_MATCH_KEY" };
               handled = true;
             }
           }
@@ -651,45 +703,46 @@ function runImport(opts) {
             const willCreate = !willUpdate && (writeMode === "insert" || writeMode === "upsert");
             if (!willUpdate && !willCreate) {
               skipped++;
-              results.push({ row: rowNo, ok: true, action: "skipped", code: "NO_MATCH" });
+              results[i] = { row: rowNo, ok: true, action: "skipped", code: "NO_MATCH" };
             } else if (dryRun) {
               okCount++;
               if (willUpdate) {
                 updated++;
-                results.push({ row: rowNo, ok: true, action: "updated", id: String(existing.id ?? "") || void 0 });
+                results[i] = { row: rowNo, ok: true, action: "updated", id: String(existing.id ?? "") || void 0 };
               } else {
                 created++;
-                results.push({ row: rowNo, ok: true, action: "created" });
+                results[i] = { row: rowNo, ok: true, action: "created" };
               }
             } else if (willUpdate) {
               const target = existing;
-              const res2 = await p.updateData({ object: objectName, id: target.id, data, context: writeCtx, ...environmentId ? { environmentId } : {} });
-              const id = res2?.id ?? res2?.record?.id ?? target.id;
+              const res2 = await withTransientRetry(() => p.updateData({ object: objectName, id: target.id, data, context: writeCtx, ...environmentId ? { environmentId } : {} }));
+              const id = extractRecordId(res2) ?? String(target.id);
               okCount++;
               updated++;
               if (collectUndo && target.id != null) {
                 undoLog.updated.push({ id: String(target.id), before: captureBefore(target, data) });
               }
-              results.push({ row: rowNo, ok: true, action: "updated", id: id != null ? String(id) : void 0 });
+              results[i] = { row: rowNo, ok: true, action: "updated", id };
+            } else if (canBulkCreate) {
+              pendingCreates.push({ index: i, rowNo, data });
             } else {
               const res2 = await p.createData({ object: objectName, data, context: writeCtx, ...environmentId ? { environmentId } : {} });
-              const id = res2?.id ?? res2?.record?.id;
+              const id = extractRecordId(res2);
               okCount++;
               created++;
-              if (collectUndo && id != null) undoLog.created.push(String(id));
-              results.push({ row: rowNo, ok: true, action: "created", id: id != null ? String(id) : void 0 });
+              if (collectUndo && id != null) undoLog.created.push(id);
+              results[i] = { row: rowNo, ok: true, action: "created", id };
             }
           }
         }
       } catch (err) {
         errCount++;
-        const code = err?.code ?? "IMPORT_ROW_FAILED";
-        const message = typeof err?.message === "string" ? err.message.slice(0, 300) : "Row failed";
-        results.push({ row: rowNo, ok: false, action: "failed", error: message, code });
+        results[i] = toFailedResult(rowNo, err);
       }
       const processed = i + 1;
-      if (onProgress && (processed % progressEvery === 0 || processed === rows.length)) {
-        await onProgress(snapshot(processed));
+      if (processed % progressEvery === 0 || processed === rows.length) {
+        await flushPendingCreates();
+        if (onProgress) await onProgress(snapshot(processed));
       }
       if (shouldCancel && processed < rows.length && processed % progressEvery === 0) {
         if (await shouldCancel()) {
@@ -698,10 +751,12 @@ function runImport(opts) {
         }
       }
     }
+    await flushPendingCreates();
+    const compacted = results.filter((r) => r !== void 0);
     return {
-      ...snapshot(results.length),
+      ...snapshot(compacted.length),
       ok: okCount,
-      results,
+      results: compacted,
       cancelled,
       ...collectUndo ? { undoLog } : {}
     };