@wix/bex-core 2.327.0 → 2.329.0

package/dist/esm/state/ImportState/ImportState.js

@@ -1,22 +1,36 @@
 import { action, computed, makeObservable, observable, runInAction, } from 'mobx';
 import { queryStatus, } from '../../model';
 import { ConditionalModalState } from '../ConditionalModalState';
+import { JobStatus, WritePolicy, } from '@wix/bex-utils/@wix/ambassador-hub-v1-job/types';
 import { parseCsv, unparseCsv } from './parseCsv';
+import { rewriteParsedCsvForImport } from './rewriteCsv';
+import { createHubJobClient } from './hubJobClient';
+import { pollImportJob } from './importJobPoller';
 import { detectColumnType, TYPE_SAMPLE_SIZE, } from './detectFieldType';
-import set from 'lodash/set';
-const DEFAULT_BATCH_SIZE = 100;
+import { toFieldEdit } from './fieldEdit';
+// `NewFieldType`, `detectColumnType`, etc. are re-exported from the package via
+// the `ImportState` barrel (`./index`), which also exports `./detectFieldType`.
+// Re-export the write-policy enum so consumers (and the cairo UI) reference it
+// from `@wix/bex-core` rather than the bundled ambassador path.
+export { WritePolicy } from '@wix/bex-utils/@wix/ambassador-hub-v1-job/types';
 // How long to keep the completed progress bar (100%) on screen before closing
 // the modal on a clean import. Without it, an import of very few rows finishes
 // near-instantly and the modal appears to vanish without ever showing the bar
 // reaching completion.
 const COMPLETION_LINGER_MS = 600;
+// The Wix Data collection's id field key. A CSV column mapped to this field lets
+// the user choose how rows that collide on `_id` are written (writePolicy).
+const ID_FIELD_KEY = '_id';
 function normalize(value) {
     return value.trim().toLowerCase();
 }
 // camelCase key for a field created from a column header
 // (e.g. "Product Name" -> "productName").
 function toFieldKey(header) {
-    const words = header.trim().split(/[^a-zA-Z0-9]+/).filter(Boolean);
+    const words = header
+        .trim()
+        .split(/[^a-zA-Z0-9]+/)
+        .filter(Boolean);
     if (words.length === 0) {
         return 'field';
     }
@@ -44,19 +58,6 @@ function uniqueFieldNameAndKey(header, usedNames, usedKeys) {
         displayName = `${header}-${suffix}`;
     }
 }
-function toMappedRows(rows, headers, mappingByHeader) {
-    return rows.map((row) => {
-        const mapped = {};
-        headers.forEach((header, index) => {
-            const mappedField = mappingByHeader[header];
-            if (mappedField == null) {
-                return;
-            }
-            set(mapped, mappedField, row[index] ?? '');
-        });
-        return mapped;
-    });
-}
 export class ImportState {
     constructor(props) {
         this.discardChangesModal = new ConditionalModalState();
@@ -68,25 +69,36 @@ export class ImportState {
         this.csvHeaders = [];
         this.csvRows = [];
         this.mappingByHeader = {};
+        // How rows that collide on `_id` are written. Only meaningful (and only sent
+        // to the server) when a CSV column is mapped to the `_id` field — see
+        // `idMappedHeader` and `_destination`. Defaults to the server default
+        // (`OVERWRITE` = upsert by `_id`).
+        //
+        // Retention: the choice is deliberately kept while editing mappings within a
+        // file — remapping `_id` away and back keeps the user's selection rather than
+        // silently reverting it. It resets to the default only on a *new file* upload
+        // (`applyParsedCsv`) or a full `reset()`.
+        this.writePolicy = WritePolicy.OVERWRITE;
         this.processedCount = 0;
         this.totalCount = 0;
         this.result = {
             succeeded: 0,
             failed: 0,
-            errors: [],
         };
         this.isBackingUp = false;
         this.isRestoring = false;
         this.restoreFn = null;
         this._cancelled = false;
         this._abortController = null;
-        // Global indices (into `csvRows`) of every row that was not imported: rows
-        // the server rejected via a batch's returned `errors`, plus — when a batch
-        // throws after some rows succeeded — the throwing batch and all un-attempted
-        // rows. Offered to the user as a downloadable CSV via the partial toast.
-        this._failedRowIndices = [];
+        // Id of the running data-movement job, set once submitted — used to cancel it
+        // (`terminateJob`) if the user closes mid-import.
+        this._jobId = null;
+        // The column whose pending new field the field editor is currently editing.
+        // Not observable — transient wiring between `beginFieldEdit` and the editor's
+        // `applyFieldEdit` callback, never rendered.
+        this._editingHeader = null;
         this.fields = (props.fields ?? []);
-        this.bulkUploadItems = props.bulkUploadItems;
+        this.collectionId = props.collectionId;
         this.backup = props.backup;
         this.updateSchema = props.updateSchema;
         this.collection = props.collection;
@@ -95,8 +107,12 @@ export class ImportState {
         this.errorHandler = props.errorHandler;
         this.showToast = props.showToast;
         this.downloadCsv = props.downloadCsv;
-        this.batchSize = props.batchSize ?? DEFAULT_BATCH_SIZE;
+        this.onJobError = props.onJobError;
         this._detectColumnType = props.detectColumnType ?? detectColumnType;
+        this.hubJobClient = createHubJobClient({
+            httpClient: props.httpClient,
+            errorHandler: props.errorHandler,
+        });
         makeObservable(this, {
             currentStep: observable.ref,
             importStatus: observable.ref,
@@ -104,6 +120,7 @@ export class ImportState {
             csvHeaders: observable.ref,
             csvRows: observable.ref,
             mappingByHeader: observable.ref,
+            writePolicy: observable.ref,
             processedCount: observable.ref,
             totalCount: observable.ref,
             result: observable.ref,
@@ -113,10 +130,15 @@ export class ImportState {
             hasMappings: computed,
             hasNoMappedData: computed,
             selectedFieldIds: computed,
+            idMappedHeader: computed,
+            setWritePolicy: action.bound,
             onUploadContinue: action.bound,
             goToUpload: action.bound,
             setMapping: action.bound,
             setNewFieldMapping: action.bound,
+            setNewFieldConfig: action.bound,
+            beginFieldEdit: action.bound,
+            applyFieldEdit: action.bound,
             close: action.bound,
             onFileSelected: action.bound,
             startImport: action.bound,
@@ -124,7 +146,15 @@ export class ImportState {
         });
     }
     init() {
-        return () => undefined;
+        return () => {
+            // Unmounting mid-import stops the poll loop (it checks `_cancelled`) so it
+            // doesn't run to terminal holding references. We deliberately do NOT abort
+            // the AbortController: during a cancel-triggered restore it holds the
+            // restore controller, and aborting it would cancel an in-progress restore.
+            // The server-side job is also left running — an unmount isn't an explicit
+            // user cancel.
+            this._cancelled = true;
+        };
     }
     get hasParsedRows() {
         return this.csvHeaders.length > 0 && this.csvRows.length > 0;
@@ -156,6 +186,19 @@ export class ImportState {
             .filter((m) => m?.kind === 'existing')
             .map((m) => m.fieldId));
     }
+    /**
+     * The CSV header currently mapped to the collection's `_id` field, or `null`
+     * when no column targets it. Drives both the "When items share the same ID"
+     * control (only shown on this row) and whether `writePolicy` is sent at all —
+     * with no `_id` mapped, every row is a fresh insert and collisions can't occur.
+     */
+    get idMappedHeader() {
+        const entry = Object.entries(this.mappingByHeader).find(([, mapping]) => mapping?.kind === 'existing' && mapping.fieldId === ID_FIELD_KEY);
+        return entry ? entry[0] : null;
+    }
+    setWritePolicy(policy) {
+        this.writePolicy = policy;
+    }
     /**
      * Whether columns may be mapped to a brand-new field created during import.
      * Driven purely by the presence of the `updateSchema` callback — providing the
@@ -176,23 +219,39 @@ export class ImportState {
         this.csvHeaders = [];
         this.csvRows = [];
         this.mappingByHeader = {};
+        this.writePolicy = WritePolicy.OVERWRITE;
         this.processedCount = 0;
         this.totalCount = 0;
         this.result = {
             succeeded: 0,
             failed: 0,
-            errors: [],
         };
         this.isBackingUp = false;
         this.isRestoring = false;
         this.restoreFn = null;
         this._cancelled = false;
-        this._failedRowIndices = [];
+        this._jobId = null;
     }
     async close() {
-        if (this.currentStep === 'progress') {
+        // While on the progress step we treat close as a cancel — terminate the job
+        // and restore the backup — UNLESS the import already succeeded. On the
+        // success path `startImport` lingers ~600ms on the completed bar with the
+        // step still 'progress'; closing during that window must NOT restore (it
+        // would undo a clean import). A successful close just closes the modal and
+        // lets the success path do its collection refresh.
+        if (this.currentStep === 'progress' && !this.importStatus.isSuccess) {
             this._cancelled = true;
             this._abortController?.abort();
+            // Stop the server-side job. It does not roll back rows already written —
+            // that's what the backup restore below is for.
+            if (this._jobId) {
+                try {
+                    await this.hubJobClient.terminateJob(this._jobId);
+                }
+                catch (error) {
+                    console.error('Terminate import job failed', error);
+                }
+            }
             if (!this.isBackingUp) {
                 await this._restoreBackup();
             }
@@ -283,6 +342,10 @@ export class ImportState {
         this.csvHeaders = parsedCsv.headers;
         this.csvRows = parsedCsv.rows;
         this.mappingByHeader = this.getDefaultMapping(parsedCsv.headers);
+        // A new file is a fresh import: start from the default policy so a prior
+        // file's Skip choice doesn't silently carry over (only the new-file path
+        // resets — remapping `_id` within the same file keeps the user's choice).
+        this.writePolicy = WritePolicy.OVERWRITE;
         this.currentStep = 'configuration';
     }
     async onFileSelected(file) {
@@ -354,12 +417,11 @@ export class ImportState {
         });
     }
     _showPartialSuccessToast() {
-        const failed = this._failedRowIndices.length;
         this.showToast?.({
             type: 'WARNING',
             message: this.translate('cairo.import.partialSuccessToast', {
-                failed: String(failed),
-                imported: String(this.totalCount - failed),
+                failed: String(this.result.failed),
+                imported: String(this.result.succeeded),
             }),
             action: {
                 text: this.translate('cairo.import.downloadFailedRows'),
@@ -368,11 +430,66 @@ export class ImportState {
             },
         });
     }
-    _downloadFailedRows() {
-        const failedRows = this._failedRowIndices.map((index) => this.csvRows[index]);
+    /**
+     * Builds a plain, re-importable CSV of the rows the server rejected — the
+     * original headers plus just the failed rows — and hands it to `downloadCsv`.
+     * The failed rows come from the job's movement logs (which row failed), mapped
+     * back to the parsed `csvRows` we still hold in memory.
+     */
+    async _downloadFailedRows() {
+        const jobId = this._jobId;
+        if (!jobId) {
+            return;
+        }
+        let indices;
+        try {
+            indices = await this._fetchFailedRowIndices(jobId);
+        }
+        catch (error) {
+            console.error('Fetching failed rows report failed', error);
+            return;
+        }
+        const failedRows = indices
+            .map((index) => this.csvRows[index])
+            .filter((row) => row != null);
+        if (failedRows.length === 0) {
+            return;
+        }
         const content = unparseCsv(this.csvHeaders, failedRows);
         this.downloadCsv?.(content, `failed-${this.fileName || 'import.csv'}`);
     }
+    // Pages through the job's movement logs and returns the `csvRows` indices of
+    // the rows that failed. data-hub's CSV decoder
+    // (wix-data-hub: packages/wix-data-hub-executor-c/src/plugins/formats/csv/
+    // csv-binary-decoder.ts) parses with the `csv-parse` library (`columns: true`)
+    // and sets `sourceItemId = "row ${++index + 1}"`, incrementing once **per
+    // record** — not per physical line. So N is the 1-based record number with
+    // the header consumed as columns: the first data record is `"row 2"` →
+    // `csvRows[0]`, i.e. `csvRows[N - 2]`. Because `csv-parse` is RFC-4180,
+    // quoted cells containing embedded newlines are a single record, so they do
+    // NOT shift the numbering. Out-of-range indices are guarded below.
+    async _fetchFailedRowIndices(jobId) {
+        const indices = [];
+        let cursor;
+        do {
+            const { logs, pagingMetadata } = await this.hubJobClient.listMovementLogs(jobId, cursor ? { cursor } : undefined);
+            for (const log of logs ?? []) {
+                if (!log.failure || log.sourceItemId == null) {
+                    continue;
+                }
+                const fileLine = Number(/\d+/.exec(log.sourceItemId)?.[0]);
+                const index = fileLine - 2;
+                if (Number.isInteger(index) &&
+                    index >= 0 &&
+                    index < this.csvRows.length) {
+                    indices.push(index);
+                }
+            }
+            cursor =
+                (pagingMetadata?.hasNext && pagingMetadata.cursors?.next) || undefined;
+        } while (cursor);
+        return indices;
+    }
     setMapping(header, fieldId) {
         this.mappingByHeader = {
             ...this.mappingByHeader,
@@ -387,53 +504,133 @@ export class ImportState {
             [header]: { kind: 'new', fieldType },
         };
     }
-    async _uploadBatches(batches) {
-        let processedRows = 0;
-        let receivedOkResponse = false;
-        const failedRowIndices = [];
-        for (const batch of batches) {
-            if (this._cancelled) {
-                break;
-            }
-            const batchStart = processedRows;
-            try {
-                const batchResult = await this.bulkUploadItems(batch);
-                receivedOkResponse = true;
-                this.result = {
-                    succeeded: this.result.succeeded + batchResult.succeeded,
-                    failed: this.result.failed + batchResult.failed,
-                    errors: [...this.result.errors, ...batchResult.errors],
-                };
-                // Rows the server rejected within an otherwise-OK response. `rowIndex`
-                // is relative to the batch, so offset it to a global `csvRows` index.
-                for (const { rowIndex } of batchResult.errors) {
-                    failedRowIndices.push(batchStart + rowIndex);
-                }
-            }
-            catch (error) {
-                // Full batch failure: a batch threw before any OK response came back.
-                // Surface it as an error state with a "Try again" action by rejecting
-                // up to `startImport`, instead of reporting a partial success.
-                if (!receivedOkResponse) {
-                    throw error;
-                }
-                // A batch threw after some rows were already imported. Every row from
-                // the throwing batch onward (including the batches we won't attempt)
-                // counts as failed, on top of any rows already rejected via returned
-                // errors in earlier batches.
-                for (let index = batchStart; index < this.totalCount; index++) {
-                    failedRowIndices.push(index);
-                }
-                break;
+    /**
+     * Persists the result of the "Edit field" editor for a column's pending new
+     * field: the (possibly overridden) type, an optional display-name override,
+     * and the per-type configuration. No-op if the column isn't currently mapped
+     * to a new field.
+     */
+    setNewFieldConfig(header, edit) {
+        const mapping = this.mappingByHeader[header];
+        if (mapping?.kind !== 'new') {
+            return;
+        }
+        this.mappingByHeader = {
+            ...this.mappingByHeader,
+            [header]: {
+                kind: 'new',
+                fieldType: edit.fieldType,
+                ...(edit.displayName != null && { displayName: edit.displayName }),
+                ...(edit.config && { config: edit.config }),
+            },
+        };
+    }
+    /**
+     * Marks which column's pending new field the field editor is currently
+     * editing, so a later `applyFieldEdit` (the editor's `onSubmit`) knows where to
+     * write the result.
+     */
+    beginFieldEdit(header) {
+        this._editingHeader = header;
+    }
+    /**
+     * `onSubmit` handler for the field editor: maps the edited field spec into the
+     * column being edited (set via `beginFieldEdit`). No-op if nothing is being
+     * edited.
+     */
+    applyFieldEdit(field) {
+        if (this._editingHeader == null) {
+            return;
+        }
+        this.setNewFieldConfig(this._editingHeader, toFieldEdit(field));
+    }
+    /**
+     * Runs the server-side import: align the CSV to the resolved field keys,
+     * upload it, submit a data-movement job, then poll until terminal. Throws on
+     * a failed job so `startImport` surfaces the error state; partial success is
+     * reported via `result.failed` and the partial toast.
+     */
+    async _runImportJob(resolvedMapping) {
+        const csv = rewriteParsedCsvForImport(this.csvHeaders, this.csvRows, resolvedMapping);
+        const file = new File([csv], this.fileName || 'import.csv', {
+            type: 'text/csv',
+        });
+        const fileId = await this.hubJobClient.uploadFile(file);
+        if (this._cancelled) {
+            return this.result;
+        }
+        const jobId = await this.hubJobClient.submitImportJob({
+            fileId,
+            name: this.fileName,
+            destination: this._destination(),
+        });
+        runInAction(() => {
+            this._jobId = jobId;
+        });
+        if (this._cancelled) {
+            // The user cancelled while `submitImportJob` was in flight, so `close()`
+            // saw no `_jobId` and couldn't terminate it. Terminate it here, otherwise
+            // the job keeps importing server-side after cancel (and would overwrite a
+            // restored backup).
+            await this.hubJobClient.terminateJob(jobId).catch(() => undefined);
+            return this.result;
+        }
+        const job = await pollImportJob((id) => this.hubJobClient.getJob(id), jobId, {
+            isCancelled: () => this._cancelled,
+            onProgress: (polled) => this._applyJobProgress(polled),
+        });
+        // `null` means cancelled mid-poll — leave whatever progress we have.
+        if (!job) {
+            return this.result;
+        }
+        if (job.status === JobStatus.FAILED) {
+            // The job ran server-side, so a failure arrives on a 200 (the poll
+            // succeeded; the job didn't) with no HTTP error to carry the application
+            // error. Build a synthetic, HTTP-error-shaped error (the error handler
+            // reads the code from `response.data.details.applicationError`) and hand
+            // the throwing operation to the consumer's `onJobError` so THEY wrap it
+            // with their own `withErrorHandler` + error-code map — `job.error` is the
+            // consumer's application error (their import business logic), so only they
+            // can map its codes, and the map stays coupled to the `withErrorHandler`
+            // call as everywhere else (e.g. `fetchData`). The map is attached to the
+            // error object, which the progress step resolves via `getResolvedError`.
+            // Without a handler the same error still throws, falling back to the
+            // generic message.
+            const error = Object.assign(new Error(`ImportJob.${job.error?.code ?? 'failed'}`), { response: { data: { details: { applicationError: job.error } } } });
+            const run = () => Promise.reject(error);
+            if (this.onJobError) {
+                // The consumer wraps `run` with `withErrorHandler`; it re-rejects with
+                // `error` (now carrying their map), which propagates from here.
+                await this.onJobError(run);
             }
-            processedRows += batch.length;
-            runInAction(() => {
-                this.processedCount = Math.min(processedRows, this.totalCount);
-            });
+            throw error;
         }
-        this._failedRowIndices = failedRowIndices;
+        this._applyJobProgress(job);
         return this.result;
     }
+    // Destination: the consumer's Wix Data collection. `writePolicy` is sent only
+    // when a CSV column is mapped to `_id` (the only case collisions can occur);
+    // otherwise it's omitted and the server applies its default (upsert by `_id`).
+    _destination() {
+        return {
+            wixDataCollection: {
+                collectionId: this.collectionId,
+                ...(this.idMappedHeader != null && { writePolicy: this.writePolicy }),
+            },
+        };
+    }
+    _applyJobProgress(job) {
+        const progress = job.progress;
+        const succeeded = progress?.itemsSuccessful ?? 0;
+        const failed = progress?.itemsRejected ?? 0;
+        const current = progress?.current ?? succeeded + failed;
+        // `totalCount` is fixed up front in `startImport` from the CSV row count
+        // (one uploaded row per CSV row); polling only advances `processedCount`.
+        runInAction(() => {
+            this.processedCount = Math.min(current, this.totalCount);
+            this.result = { succeeded, failed };
+        });
+    }
     async _runBackup() {
         const backup = this.backup;
         if (!backup) {
@@ -494,10 +691,18 @@ export class ImportState {
                 resolved[header] = mapping.fieldId;
             }
             else if (this.updateSchema) {
-                const { displayName, key } = uniqueFieldNameAndKey(header, usedNames, usedKeys);
+                // The user's name override (from the field editor) seeds the unique
+                // name/key derivation; otherwise fall back to the column header.
+                const baseName = mapping.displayName?.trim() || header;
+                const { displayName, key } = uniqueFieldNameAndKey(baseName, usedNames, usedKeys);
                 usedKeys.add(key);
                 usedNames.add(displayName);
-                newFields.push({ key, type: mapping.fieldType, displayName });
+                newFields.push({
+                    key,
+                    type: mapping.fieldType,
+                    displayName,
+                    ...(mapping.config && { config: mapping.config }),
+                });
                 resolved[header] = key;
             }
             else {
@@ -510,23 +715,14 @@ export class ImportState {
         }
         return resolved;
     }
-    _createRowsBatches(rows) {
-        return rows.reduce((acc, row, index) => {
-            if (index % this.batchSize === 0) {
-                acc.push([]);
-            }
-            acc[acc.length - 1].push(row);
-            return acc;
-        }, []);
-    }
     async startImport() {
         if (!this.hasMappings) {
             return;
         }
         this._cancelled = false;
         this._abortController = new AbortController();
-        // One mapped row is produced per CSV row, so the total is known up front and
-        // doesn't depend on field creation below.
+        // One uploaded row per CSV row (the rewrite drops columns, not rows), so the
+        // total is known up front from the parsed CSV and stays fixed through polling.
         const totalCount = this.csvRows.length;
         runInAction(() => {
             this.currentStep = 'progress';
@@ -539,9 +735,8 @@ export class ImportState {
             this.result = {
                 succeeded: 0,
                 failed: 0,
-                errors: [],
             };
-            this._failedRowIndices = [];
+            this._jobId = null;
         });
         const backupFailed = await this._runBackup();
         if (backupFailed) {
@@ -551,8 +746,8 @@ export class ImportState {
             return;
         }
         // Create any new fields and resolve every column to its destination key
-        // before building the payload. A creation failure surfaces as the error
-        // state with a "Try again" CTA, like a first-batch upload failure.
+        // before uploading. A creation failure surfaces as the error state with a
+        // "Try again" CTA, like an upload/job failure.
         let resolvedMapping;
         try {
             resolvedMapping = await this._resolveFieldMapping();
@@ -569,9 +764,7 @@ export class ImportState {
         if (this._cancelled) {
             return;
         }
-        const rows = toMappedRows(this.csvRows, this.csvHeaders, resolvedMapping);
-        const batches = this._createRowsBatches(rows);
-        const promise = this._uploadBatches(batches);
+        const promise = this._runImportJob(resolvedMapping);
         this.importStatus = {
             ...queryStatus.loading,
             promise,
@@ -591,17 +784,16 @@ export class ImportState {
             // briefly so the completion is perceptible even for very few rows. Skip
             // the wait when there are failed rows — that path closes and surfaces a
             // toast instead.
-            if (this._failedRowIndices.length === 0) {
+            if (this.result.failed === 0) {
                 await this._lingerOnCompletion();
                 if (this._cancelled) {
                     return;
                 }
             }
             runInAction(() => {
-                // Any rows that didn't make it in — whether rejected via a batch's
-                // returned `errors` or left unimported when a batch threw — are
-                // reported via a toast that offers them as a downloadable CSV.
-                if (this._failedRowIndices.length > 0) {
+                // Rows the server rejected are reported via a toast (the downloadable
+                // failed-rows report is added by the failed-rows-report task).
+                if (this.result.failed > 0) {
                     this._showPartialSuccessToast();
                 }
                 this.importModalState.close();