@haus-tech/product-import-export-plugin 3.6.12 → 3.6.14

package/src/providers/import-providers/product-importer.js

@@ -18,6 +18,7 @@ exports.ProductImporter = void 0;
 const common_1 = require("@nestjs/common");
 const core_1 = require("@vendure/core");
 const core_2 = require("@vendure/core");
+const sync_1 = require("csv-parse/sync");
 const rxjs_1 = require("rxjs");
 const shared_utils_1 = require("@vendure/common/lib/shared-utils");
 const generated_types_1 = require("@vendure/common/lib/generated-types");
@@ -30,12 +31,20 @@ const typeorm_1 = require("typeorm");
 const generate_asset_hash_1 = require("../../helpers/generate-asset-hash");
 const crypto_1 = require("crypto");
 const remove_update_at_fields_1 = require("../../helpers/remove-update-at-fields");
-const sku_relation_resolver_1 = require("../../services/sku-relation-resolver");
+const relation_lookup_resolver_1 = require("../../services/relation-lookup-resolver");
+const import_row_validation_strategy_1 = require("../../strategies/import-row-validation-strategy");
+const import_row_groups_1 = require("../../services/import-row-groups");
+const channel_column_resolver_1 = require("../../services/channel-column-resolver");
 const import_csv_lines_1 = require("../../helpers/import-csv-lines");
-const NO_SKU_LOOKUP = { fields: new Set(), variantIdsBySku: new Map() };
+const NO_RELATION_LOOKUPS = { fields: new Map(), idsByLookup: new Map() };
+/** The row an error names (`Row 12: ...`), 0 for an error about the whole file. */
+function errorRow(error) {
+    const match = /^Row (\d+):/.exec(error);
+    return match ? Number(match[1]) : 0;
+}
 /**
- * Splits a typed custom field header (`product:field:type`, or with a fourth `:sku` segment that
- * SkuRelationResolver reads) into the `<owner>:<field>` key the parser checks and the type.
+ * Splits a typed custom field header (`product:field:type`, or with a fourth lookup segment that
+ * RelationLookupResolver reads) into the `<owner>:<field>` key the parser checks and the type.
  */
 function splitTypedCustomFieldHeader(header) {
     if (!header.startsWith('product:') && !header.startsWith('variant:')) {
@@ -63,17 +72,20 @@ let ProductImporter = class ProductImporter {
     assetService;
     optionGroupService;
     pluginOptions;
-    skuRelationResolver;
+    relationLookupResolver;
+    rowValidationStrategy;
+    channelColumnResolver;
     taxCategoryMatches = {};
     facetMap = new Map();
     facetValueMap = new Map();
     customFieldTypes = {};
     allAssets = [];
     allAssetsById = new Map();
+    allAssetsBySource = new Map();
     facetIdCache = new Map();
     facetValueIdCache = new Map();
     slugStrategy;
-    constructor(connection, configService, importParser, assetImporter, fastImporter, translatableSaver, productService, channelService, facetService, facetValueService, taxCategoryService, assetService, optionGroupService, pluginOptions, skuRelationResolver) {
+    constructor(connection, configService, importParser, assetImporter, fastImporter, translatableSaver, productService, channelService, facetService, facetValueService, taxCategoryService, assetService, optionGroupService, pluginOptions, relationLookupResolver, rowValidationStrategy, channelColumnResolver) {
         this.connection = connection;
         this.configService = configService;
         this.importParser = importParser;
@@ -88,7 +100,9 @@ let ProductImporter = class ProductImporter {
         this.assetService = assetService;
         this.optionGroupService = optionGroupService;
         this.pluginOptions = pluginOptions;
-        this.skuRelationResolver = skuRelationResolver;
+        this.relationLookupResolver = relationLookupResolver;
+        this.rowValidationStrategy = rowValidationStrategy;
+        this.channelColumnResolver = channelColumnResolver;
         this.slugStrategy = this.configService.entityOptions.slugStrategy;
     }
     parseAndImport(input, ctx, updateProductSlug = true, mainLanguage, updatingStrategy) {
@@ -98,28 +112,31 @@ let ProductImporter = class ProductImporter {
             }).then((value) => {
                 subscriber.next({ ...value, currentProduct: 'Complete' });
                 subscriber.complete();
-            });
+            }, (error) => subscriber.error(error));
         });
     }
+    /**
+     * Column 0 of every data record as a product id, null when empty or not a number. Index 0 is
+     * line 2, counted in records as the parser counts them, so a quoted cell that spans several
+     * physical lines does not shift the ids.
+     */
     extractProductIdsFromCSV(csvString) {
-        // Split the CSV string into lines
-        const lines = csvString.split('\n');
-        // Initialize an array to hold the product IDs
-        const productIds = [];
-        // Loop through each line in the CSV
-        lines.forEach((line, index) => {
-            if (index === 0)
-                return; // Skip header row
-            const columns = line.split(',');
-            const productId = columns[0].trim();
-            if (productId) {
-                productIds.push(parseInt(productId, 10));
-            }
-            else {
-                productIds.push(null); // Push null or undefined if productId is missing
-            }
+        let records;
+        try {
+            records = (0, sync_1.parse)(csvString, {
+                trim: true,
+                relax_column_count: true,
+                relax_quotes: true,
+            });
+        }
+        catch {
+            // The parser cannot read the file either, so there are no products to give ids to.
+            return [];
+        }
+        return records.slice(1).map((record) => {
+            const productId = parseInt(record[0] ?? '', 10);
+            return Number.isNaN(productId) ? null : productId;
         });
-        return productIds;
     }
     /**
      * The problems in the file's header row, checked the way the import job checks it. Empty when
@@ -129,9 +146,57 @@ let ProductImporter = class ProductImporter {
         const headers = (0, import_csv_lines_1.importCsvHeaders)((0, import_csv_lines_1.importCsvLines)(csv)).map((header) => splitTypedCustomFieldHeader(header)?.fullFieldName ?? header);
         return this.importParser.validateHeader(headers);
     }
+    /**
+     * Everything that is checked before an import writes anything: relation lookups, then the parsed
+     * rows through the row validation strategy. The upload runs this before the file is stored, and
+     * the import job runs it again before its first write, since data can change while the job waits.
+     * With the default strategy and no `channels` column the file is parsed only when `parse` asks
+     * for it (the job does), so such an upload costs no more than the lookups. `uploader` is the
+     * context whose user needs rights in the channels the `channels` column names; the job passes
+     * the uploader's, since it imports with a context of its own.
+     */
+    async checkFile(ctx, csv, mainLanguage, updateProductSlug, { parse = false, uploader = ctx } = {}) {
+        const unchecked = { customFieldTypes: {}, channelsByToken: new Map() };
+        const relationLookups = await this.relationLookupResolver.resolve(ctx, csv);
+        if (!relationLookups.rowsChecked) {
+            return { errors: relationLookups.errors, relationLookups, ...unchecked };
+        }
+        // Only the default itself is skipped: a subclass of it has rules of its own.
+        const validatesRows = this.rowValidationStrategy.constructor !== import_row_validation_strategy_1.DefaultImportRowValidationStrategy;
+        const namesChannels = (0, import_csv_lines_1.importCsvHeaders)((0, import_csv_lines_1.importCsvLines)(csv)).includes(channel_column_resolver_1.CHANNELS_COLUMN);
+        if (!validatesRows && !namesChannels && !parse) {
+            return { errors: relationLookups.errors, relationLookups, ...unchecked };
+        }
+        const { cleanedCsv, customFieldTypes } = this.preprocessCsv(csv, updateProductSlug);
+        const parsed = (await this.importParser.parseProducts(cleanedCsv, mainLanguage));
+        const productIds = this.extractProductIdsFromCSV(cleanedCsv);
+        for (const result of parsed.results) {
+            // The id on the product's own first row, not on the line with the product's index.
+            const productId = result.product.line ? productIds[result.product.line - 2] : null;
+            if (productId != null) {
+                result.product.productId = productId;
+            }
+        }
+        const rowErrors = validatesRows
+            ? await this.rowValidationStrategy.validate(ctx, (0, import_row_groups_1.toImportRowGroups)(parsed.results, relationLookups, this.configService.customFields))
+            : [];
+        const channelColumn = await this.channelColumnResolver.resolve(ctx, uploader, parsed.results);
+        const errors = [
+            ...relationLookups.errors,
+            ...channelColumn.errors,
+            ...rowErrors.map(({ line, message }) => `Row ${line}: ${message}`),
+        ].sort((a, b) => errorRow(a) - errorRow(b));
+        return {
+            errors,
+            relationLookups,
+            parsed,
+            customFieldTypes,
+            channelsByToken: channelColumn.channelsByToken,
+        };
+    }
     preprocessCsv(input, updateProductSlug) {
         const customFieldTypes = {};
-        // Without blank and comma-only lines, as SkuRelationResolver reads the file
+        // Without blank and comma-only lines, as RelationLookupResolver reads the file
         const filteredLines = (0, import_csv_lines_1.importCsvLines)(input);
         const headers = (0, import_csv_lines_1.splitCsvLine)(filteredLines[0]);
         const cleanedHeaders = headers.map((rawHeader) => {
@@ -199,13 +264,13 @@ let ProductImporter = class ProductImporter {
         }
         this.allAssets = assets;
         this.allAssetsById = new Map(assets.map((asset) => [asset.id, asset]));
+        this.allAssetsBySource = new Map(assets.map((asset) => [asset.source, asset]));
     }
     async doParseAndImport(input, updateProductSlug, mainLanguage, updatingStrategy, reqCtx, onProgress) {
         // Per-import caches to avoid stale cross-job state.
         this.facetIdCache.clear();
         this.facetValueIdCache.clear();
         const ctx = await this.getRequestContext(reqCtx, mainLanguage);
-        await this.setMissingAssetHashes(ctx);
         let csvContent;
         if (typeof input === 'string') {
             csvContent = input;
@@ -214,32 +279,28 @@ let ProductImporter = class ProductImporter {
             // If input is a Stream, you need to read it fully into memory
             csvContent = await this.streamToString(input);
         }
-        // Checked again here, not only on upload: variants can change while the job waits. SKUs are
-        // resolved once, here: a variant deleted while the import runs is still linked to.
-        const skuRelations = await this.skuRelationResolver.resolve(ctx, csvContent);
-        if (skuRelations.errors.length > 0) {
-            return { errors: skuRelations.errors, imported: 0, processed: 0 };
+        // Checked again here, before the first write, not only on upload: data can change while the
+        // job waits. Lookup values are resolved once, here: a row deleted while the import runs is
+        // still linked to.
+        const { errors, relationLookups, parsed, customFieldTypes, channelsByToken } = await this.checkFile(ctx, csvContent, mainLanguage, updateProductSlug, {
+            parse: true,
+            uploader: reqCtx,
+        });
+        if (errors.length > 0 || !parsed) {
+            // Never fail a job without saying why.
+            const reasons = errors.length > 0 ? errors : ['The import file could not be read'];
+            return { errors: reasons, imported: 0, processed: 0, rejected: true };
         }
-        // Preprocess the CSV to extract types and clean headers
-        const { cleanedCsv, customFieldTypes } = this.preprocessCsv(csvContent, updateProductSlug);
+        await this.setMissingAssetHashes(ctx);
         this.customFieldTypes = customFieldTypes;
-        const parsed = (await this.importParser.parseProducts(cleanedCsv, ctx.languageCode));
-        const productIds = this.extractProductIdsFromCSV(cleanedCsv);
-        parsed.results.forEach((result, index) => {
-            const productId = productIds[index];
-            if (productId !== null) {
-                result.product.productId = productId;
-            }
-            // Handle the case where productId is null if necessary
-        });
-        if (parsed && parsed.results.length) {
+        if (parsed.results.length) {
             try {
                 const importErrors = await this.importProducts(ctx, parsed.results, updatingStrategy, (progess) => {
                     onProgress({
                         ...progess,
                         processed: parsed.processed,
                     });
-                }, skuRelations);
+                }, relationLookups, channelsByToken);
                 return {
                     errors: parsed.errors.concat(importErrors),
                     imported: parsed.results.length,
@@ -277,41 +338,31 @@ let ProductImporter = class ProductImporter {
             throw new core_2.InternalServerError('No RequestContext provided');
         }
     }
-    async importProducts(ctx, rows, updatingStrategy, onProgress, skuLookup = NO_SKU_LOOKUP) {
+    async importProducts(ctx, rows, updatingStrategy, onProgress, relationLookups = NO_RELATION_LOOKUPS, channelsByToken = new Map()) {
         let errors = [];
         let imported = 0;
         const languageCode = ctx.languageCode;
-        const taxCategories = await this.taxCategoryService.findAll(ctx);
-        // Create taxCategory if none exists
+        let taxCategories = await this.taxCategoryService.findAll(ctx);
+        // Create taxCategory if none exists, and read the list again so the rows can match it
         if (taxCategories.totalItems === 0) {
             await this.taxCategoryService.create(ctx, {
                 name: 'Standard Tax',
                 isDefault: true,
             });
+            taxCategories = await this.taxCategoryService.findAll(ctx);
         }
         await this.fastImporter.initialize(ctx.channel);
         const shouldRestore = this.pluginOptions?.importOptions?.defaultOptions?.restoreSoftDeleted !== false;
         const sharedGroupMap = await this.buildSharedOptionGroupMap(ctx, rows);
         for (const { product, variants } of rows) {
+            this.fastImporter.setProductChannels((0, lodash_1.compact)(product.channelTokens.map((token) => channelsByToken.get(token))));
             const productMainTranslation = this.getTranslationByCodeOrFirst(product.translations, ctx.languageCode);
-            const assetsToImport = (0, lodash_1.compact)(product.assetPaths.map((assetPath, idx) => {
-                const assetID = product.assetsJson?.[idx]?.id;
-                if (assetPath.startsWith('http://') || assetPath.startsWith('https://')) {
-                    return {
-                        url: assetPath,
-                        name: product.assetsJson?.[idx]?.name || undefined,
-                        id: assetID && this.allAssetsById.get(assetID)
-                            ? product.assetsJson?.[idx]?.id
-                            : undefined,
-                    };
-                }
-            }));
-            const createProductAssets = await this.assetImporter.getAssets(assetsToImport, ctx);
+            const createProductAssets = await this.resolveAssets(product.assetPaths, product.assetsJson, ctx);
             const productAssets = createProductAssets.assets;
             if (createProductAssets.errors.length) {
                 errors = errors.concat(createProductAssets.errors);
             }
-            const customFields = await this.processCustomFieldValues(product.translations[0].customFields, this.configService.customFields.Product, ctx, 'product', skuLookup);
+            const customFields = await this.processCustomFieldValues(product.translations[0].customFields, this.configService.customFields.Product, ctx, 'product', relationLookups);
             let checkExistingProduct;
             if (product.productId) {
                 checkExistingProduct = await this.productService.findOne(ctx, product.productId);
@@ -387,7 +438,7 @@ let ProductImporter = class ProductImporter {
                             ? { description: translation.description }
                             : {}),
                         slug: existingTranslation?.slug || translation.slug,
-                        customFields: await this.processCustomFieldValues(translation.customFields, this.configService.customFields.Product, ctx, 'product', skuLookup),
+                        customFields: await this.processCustomFieldValues(translation.customFields, this.configService.customFields.Product, ctx, 'product', relationLookups),
                     };
                 })),
                 customFields,
@@ -495,19 +546,7 @@ let ProductImporter = class ProductImporter {
             }
             for (const variant of variants) {
                 const variantMainTranslation = this.getTranslationByCodeOrFirst(variant.translations, ctx.languageCode);
-                const assetsToImport = (0, lodash_1.compact)(variant.assetPaths.map((assetPath, idx) => {
-                    const assetID = variant.assetsJson?.[idx]?.id;
-                    if (assetPath.startsWith('http://') || assetPath.startsWith('https://')) {
-                        return {
-                            url: assetPath,
-                            name: variant.assetsJson?.[idx]?.name || undefined,
-                            id: assetID && this.allAssetsById.get(assetID)
-                                ? variant.assetsJson?.[idx]?.id
-                                : undefined,
-                        };
-                    }
-                }));
-                const createVariantAssets = await this.assetImporter.getAssets(assetsToImport);
+                const createVariantAssets = await this.resolveAssets(variant.assetPaths, variant.assetsJson);
                 const variantAssets = createVariantAssets.assets;
                 if (createVariantAssets.errors.length) {
                     errors = errors.concat(createVariantAssets.errors);
@@ -516,7 +555,7 @@ let ProductImporter = class ProductImporter {
                 // if (0 < variant.facets.length) {
                 //   facetValueIds = await this.getFacetValueIds(ctx, variant.facets, languageCode)
                 // }
-                const variantCustomFields = await this.processCustomFieldValues(variantMainTranslation.customFields, this.configService.customFields.ProductVariant, ctx, 'variant', skuLookup);
+                const variantCustomFields = await this.processCustomFieldValues(variantMainTranslation.customFields, this.configService.customFields.ProductVariant, ctx, 'variant', relationLookups);
                 const optionIds = shouldUpdateOptions
                     ? variantMainTranslation.optionValues.map((v, index) => optionsMap[`${index}_${v}`])
                     : undefined;
@@ -577,7 +616,7 @@ let ProductImporter = class ProductImporter {
                         return {
                             languageCode: translation.languageCode,
                             name: [productTranslation.name, ...translation.optionValues].join(' '),
-                            customFields: await this.processCustomFieldValues(translation.customFields, this.configService.customFields.ProductVariant, ctx, 'variant', skuLookup),
+                            customFields: await this.processCustomFieldValues(translation.customFields, this.configService.customFields.ProductVariant, ctx, 'variant', relationLookups),
                         };
                     })),
                     price: (0, lodash_1.isUndefined)(variant.price) ? undefined : Math.round(variant.price * 100),
@@ -748,24 +787,24 @@ let ProductImporter = class ProductImporter {
         }
         return out;
     }
-    /** Swaps SKUs for variant ids; `SkuRelationResolver` has already checked each SKU. */
-    variantIdsForSkus(variantIdsBySku, value, list) {
-        const idFor = (sku) => {
-            const id = variantIdsBySku.get(sku);
+    /** Swaps lookup values for ids; `RelationLookupResolver` has already checked each value. */
+    idsForLookupValues(idsByLookup, lookup, value, list) {
+        const idFor = (lookupValue) => {
+            const id = idsByLookup.get((0, relation_lookup_resolver_1.relationLookupKey)(lookup, lookupValue));
             if (id === undefined) {
-                throw new core_2.InternalServerError(`No variant with SKU ${sku}`);
+                throw new core_2.InternalServerError(`No ${lookup.entityName} with ${lookup.column} ${lookupValue}`);
             }
             return String(id);
         };
         if (!list) {
             return idFor(value.trim());
         }
-        return (0, sku_relation_resolver_1.splitSkuList)(value).map(idFor);
+        return (0, relation_lookup_resolver_1.splitLookupList)(value).map(idFor);
     }
     isValidRelationIdentifier(value) {
         return /^[A-Za-z0-9_-]+$/.test(value);
     }
-    async processCustomFieldValues(customFields, config, ctx, owner, skuLookup = NO_SKU_LOOKUP) {
+    async processCustomFieldValues(customFields, config, ctx, owner, relationLookups = NO_RELATION_LOOKUPS) {
         function parseBoolean(input) {
             if (input == null) {
                 return false;
@@ -843,6 +882,15 @@ let ProductImporter = class ProductImporter {
             if (fieldDef.type === 'relation') {
                 const inputKey = (0, shared_utils_1.getGraphQlInputName)(fieldDef);
                 const relDef = fieldDef;
+                const lookup = relationLookups.fields.get(`${owner}:${fieldDef.name}`);
+                if (lookup) {
+                    const ids = this.idsForLookupValues(relationLookups.idsByLookup, lookup, value, fieldDef.list === true);
+                    if (relDef.nullable === false && Array.isArray(ids) && ids.length === 0) {
+                        throw new core_2.InternalServerError(`Required relation custom field "${fieldDef.name}" is missing or empty`);
+                    }
+                    processed[inputKey] = ids;
+                    continue;
+                }
                 if (relDef.entity === core_2.Asset) {
                     let resolvedAssetRelation;
                     if ((0, lodash_1.startsWith)(value, '{') || (0, lodash_1.startsWith)(value, '[')) {
@@ -901,14 +949,6 @@ let ProductImporter = class ProductImporter {
                     }
                     continue;
                 }
-                if (skuLookup.fields.has(`${owner}:${fieldDef.name}`)) {
-                    const variantIds = this.variantIdsForSkus(skuLookup.variantIdsBySku, value, fieldDef.list === true);
-                    if (relDef.nullable === false && Array.isArray(variantIds) && variantIds.length === 0) {
-                        throw new core_2.InternalServerError(`Required relation custom field "${fieldDef.name}" is missing or empty`);
-                    }
-                    processed[inputKey] = variantIds;
-                    continue;
-                }
                 if (fieldDef.list === true) {
                     const relationIds = value
                         .split('|')
@@ -1113,6 +1153,34 @@ let ProductImporter = class ProductImporter {
         this.facetValueIdCache.set(facetValueCacheKey, facetValueId);
         return facetValueId;
     }
+    /**
+     * Resolves the assets of an `assets` or `variantAssets` cell, in cell order. A URL goes through
+     * the asset importer. Any other value names an existing asset the way the export writes it: by
+     * the `id` of the JSON format or by its `source`. An existing asset is linked as it is, never
+     * uploaded again, and a value that matches no asset is reported.
+     */
+    async resolveAssets(assetPaths, assetsJson, ctx) {
+        const assets = [];
+        const errors = [];
+        for (const [idx, assetPath] of assetPaths.entries()) {
+            const assetID = assetsJson?.[idx]?.id;
+            const knownAsset = assetID ? this.allAssetsById.get(assetID) : undefined;
+            if (assetPath.startsWith('http://') || assetPath.startsWith('https://')) {
+                const imported = await this.assetImporter.getAssets([{ url: assetPath, name: assetsJson?.[idx]?.name || undefined, id: knownAsset?.id }], ctx);
+                assets.push(...imported.assets);
+                errors.push(...imported.errors);
+                continue;
+            }
+            const existingAsset = knownAsset ?? this.allAssetsBySource.get(assetPath);
+            if (existingAsset) {
+                assets.push(existingAsset);
+            }
+            else {
+                errors.push(`Could not find an existing asset with source "${assetPath}"`);
+            }
+        }
+        return { assets, errors };
+    }
     resolveFeaturedAssetId(importedAssets, existingFeaturedAsset, updatingStrategy) {
         const firstFeaturedCandidate = importedAssets.find((a) => a.type === generated_types_1.AssetType.IMAGE) ?? importedAssets[0];
         if (importedAssets.length > 0 && firstFeaturedCandidate) {
@@ -1169,6 +1237,7 @@ exports.ProductImporter = ProductImporter;
 exports.ProductImporter = ProductImporter = __decorate([
     (0, common_1.Injectable)(),
     __param(13, (0, common_1.Inject)(constants_1.PRODUCT_IMPORT_EXPORT_PLUGIN_OPTIONS)),
+    __param(15, (0, common_1.Inject)(constants_1.IMPORT_ROW_VALIDATION_STRATEGY)),
     __metadata("design:paramtypes", [core_1.TransactionalConnection,
         core_1.ConfigService,
         import_parser_1.ImportParser,
@@ -1181,6 +1250,6 @@ exports.ProductImporter = ProductImporter = __decorate([
         core_1.FacetValueService,
         core_1.TaxCategoryService,
         core_1.AssetService,
-        core_1.ProductOptionGroupService, Object, sku_relation_resolver_1.SkuRelationResolver])
+        core_1.ProductOptionGroupService, Object, relation_lookup_resolver_1.RelationLookupResolver, Object, channel_column_resolver_1.ChannelColumnResolver])
 ], ProductImporter);
 //# sourceMappingURL=product-importer.js.map