@softwear/latestcollectioncore 1.0.202 → 1.0.204

package/src/imageBinder.ts

@@ -25,11 +25,89 @@ const GLOBAL_PATTERNS = [
   ['articleDescription', 'articleCode|articleCodeSupplier', 'colorCodeSupplier|colorSupplier'],
   ['colorCodeSupplier|colorSupplier', 'articleCode|articleCodeSupplier'],
   ['brand', 'collectionSupplier', 'colorCodeSupplier', 'articleCode|articleCodeSupplier'],
-  ['brand', 'collectionSupplier', 'colorCodeSupplier', 'articleCode|articleCodeSupplier', 'buyPrice|valuePrice'],
   ['brand', 'articleDescription', 'colorSupplier'],
 ]
 
-function prepFilterSkusImages(filteredSkus: SkuI[], merge = true, fileNames: Array<string>) {
+const FIELDS_TO_SKIP: Array<string> = ['__created', '__modified', 'images', 'brandhash', 'active', 'PRODUCT', 'colorFamily', 'price', 'sizeIndex', '_customsize', 'SELECTED']
+
+const MERGE_GROUPS: Record<string, string> = {
+  'id|barcode': 'id',
+  'articleCode|articleCodeSupplier': 'articleCode',
+  'colorCode|colorCodeSupplier|colorSupplier': 'color',
+  'size|sizeSupplier|mainSize|subSize|subSizeSupplier|_customsize': 'size',
+  'articleGroup|articleGroupSupplier': 'articleGroup',
+  'collection|collectionSupplier': 'collection',
+  'brand|brandSupplier': 'brand',
+  '_gender|genderSupplier': 'gender',
+}
+
+// Letters, digits, and apostrophe stay; everything else is a delimiter.
+const DELIMITERS = /[^\p{L}\p{N}']+/gu
+
+/** Shared stem: decode URI, then strip from the first known image extension onward. */
+function filenameStem(fileName: string): string {
+  let s: string
+  try {
+    s = decodeURIComponent(fileName)
+  } catch {
+    s = fileName
+  }
+  s = s.toLowerCase()
+  const extIdx = s.search(/\.(?:jpe?g|png|gif)/i)
+  if (extIdx !== -1) s = s.slice(0, extIdx)
+  return s
+}
+
+function checkPattern(fieldVals: Array<string>, fileName: string) {
+
+  const fieldIndexes2: Record<string, Array<number>> = {}
+
+  const sortedFields = fieldVals.sort((a, b) => {
+    return b.length - a.length
+  })
+
+  // Return quickly if one of the fields in the pattern is not in the fileName
+  for (const fieldVal of sortedFields) {
+    const idx = fileName.indexOf(fieldVal)
+    if (idx == -1) {
+      return false
+    }
+    fieldIndexes2[fieldVal] = [idx, idx+fieldVal.length]
+  }
+
+  // Check if all fields have a specific place in file name, with no overlap
+  for (let i = 1; i < sortedFields.length; i++) {
+    const startIndex = fieldIndexes2[sortedFields[i]][0]
+    const endIndex = fieldIndexes2[sortedFields[i]][1]
+    for (let j = 0; j < i; j++) {
+      const startIndex2 = fieldIndexes2[sortedFields[j]][0]
+      const endIndex2 = fieldIndexes2[sortedFields[j]][1]
+      if (startIndex < endIndex2 && endIndex > startIndex2) {
+        const nextIndex = fileName.indexOf(sortedFields[i], endIndex2)
+        if (nextIndex == -1) {
+          return false
+        } else {
+          fieldIndexes2[sortedFields[i]] = [nextIndex, nextIndex+sortedFields[i].length]
+          break
+        }
+      }
+    }
+  }
+
+  return true
+}
+
+function normalizeSkuField(sku: SkuI, fieldName: string) {
+  let fieldVal = sku[fieldName]
+  if (fieldVal == null || fieldVal == '') return ''
+  if (typeof fieldVal == 'string') {
+    return fieldVal.replace(DELIMITERS, '').toLowerCase()
+  } else {
+    return fieldVal.toString()
+  }
+}
+
+function prepFilterSkusImagesAuto(filteredSkus: SkuI[], fileNames: Array<string>) {
   const skusCopy = deepCopy(filteredSkus)
   const preppedSkus: Array<Record<string, any>> = []
   const fileNames2skip: Array<string> = []
@@ -47,41 +125,92 @@ function prepFilterSkusImages(filteredSkus: SkuI[], merge = true, fileNames: Arr
 
     const fieldsToBeMerged = {}
     for (const fieldName in sku) {
-      if (fieldName == '__created' || fieldName == '__modified' || fieldName == 'images' || fieldName == 'brandhash' || (merge == true && fieldName == 'colorFamily')) continue
-      let fieldVal = sku[fieldName]
-      if (fieldVal == '') continue
+      if (FIELDS_TO_SKIP.indexOf(fieldName) != -1 || fieldName.includes('Price')) continue
+      if (!(fieldName in sku) || sku[fieldName] == null || sku[fieldName] == undefined) continue
+      sku[fieldName] = normalizeSkuField(sku, fieldName)
 
-      if (typeof fieldVal == 'string') {
-        sku[fieldName] = fieldVal.replace(/[^A-Za-zÀ-ÖØ-öø-ÿ0-9]/g, '').toLowerCase()
+      const fieldVal = sku[fieldName]
+      if (!fieldVal) continue
+
+      if (fieldVal in fieldsToBeMerged) {
+        fieldsToBeMerged[fieldVal].push(fieldName)
       } else {
-        sku[fieldName] = fieldVal.toString()
+        fieldsToBeMerged[fieldVal] = [fieldName]
       }
-
-      if (merge) {
-        fieldVal = sku[fieldName]
-
-        if (fieldVal in fieldsToBeMerged) {
-          fieldsToBeMerged[fieldVal].push(fieldName)
-        } else {
-          fieldsToBeMerged[fieldVal] = [fieldName]
+    }
+    const newSku = {}
+    newSku['originalSku'] = filteredSkus[idx]
+    for (const fieldVal in fieldsToBeMerged) {
+      const fieldNameArray = fieldsToBeMerged[fieldVal].sort()
+      if (fieldNameArray.length > 1) {
+        const groups = {}
+        for (const fieldName of fieldNameArray) {
+          let included = false
+          Object.keys(MERGE_GROUPS).forEach((group) => {
+            if (group.includes(fieldName)) {
+              included = true
+              if (!groups[MERGE_GROUPS[group]]) groups[MERGE_GROUPS[group]] = fieldName
+              else groups[MERGE_GROUPS[group]] = groups[MERGE_GROUPS[group]] + '|' + fieldName
+            }
+          })
+          if (!included) {
+            groups[fieldName] = fieldName
+          }
         }
+        Object.keys(groups).forEach((group) => {
+          newSku[groups[group]] = fieldVal
+        })
+      } else {
+        newSku[fieldNameArray[0]] = fieldVal
       }
     }
-    if (!merge) {
+    preppedSkus.push(newSku)
+  })
+  const fileNames2Match = fileNames.filter((fileName) => {
+    return fileNames2skip.indexOf(fileName) == -1
+  })
+  return { preppedSkus: preppedSkus, fileNames2Match: fileNames2Match }
+}
+
+function prepFilterSkusImagesManual(filteredSkus: SkuI[], fileNames: Array<string>, pattern: Array<string>) {
+  const skusCopy = deepCopy(filteredSkus)
+  const preppedSkus: Array<Record<string, any>> = []
+  const fileNames2skip: Array<string> = []
+  skusCopy.forEach((sku, idx) => {
+    if (sku.images && sku.images.length > 0) {
+      // If a fileName is already matched, we skip it
+      fileNames.forEach((fileName) => {
+        if (sku.images?.includes(fileName) && fileNames2skip.indexOf(fileName) == -1) fileNames2skip.push(fileName)
+      })
+      // If a sku has images, we skip it
+      sku['skip'] = true
       preppedSkus.push(sku)
-    } else {
-      const newSku = {}
-      newSku['originalSku'] = filteredSkus[idx]
-      for (const fieldVal in fieldsToBeMerged) {
-        const fieldNameArray = fieldsToBeMerged[fieldVal].sort()
-        if (fieldNameArray.length > 1) {
-          newSku[fieldNameArray.join('|')] = fieldVal
-        } else {
-          newSku[fieldNameArray[0]] = fieldVal
+      return
+    }
+
+    const foundValues: Array<string> = []
+    for (const fieldName of pattern) {
+      if (!(fieldName in sku) || sku[fieldName] == null || sku[fieldName] == undefined) {
+        sku['skip'] = true
+        break
+      }
+      sku[fieldName] = normalizeSkuField(sku, fieldName)
+      if (!sku[fieldName]) {
+        sku['skip'] = true
+        break
+      }
+      for (const value of foundValues) {
+        if (value.includes(sku[fieldName]) || sku[fieldName] == value) {
+          if (!sku['subStrings']) sku['subStrings'] = []
+          sku['subStrings'].push([value, sku[fieldName]])
+        } else if (sku[fieldName].includes(value)) {
+          if (!sku['subStrings']) sku['subStrings'] = []
+          sku['subStrings'].push([sku[fieldName], value])
         }
       }
-      preppedSkus.push(newSku)
+      foundValues.push(sku[fieldName])
     }
+    preppedSkus.push(sku)
   })
   const fileNames2Match = fileNames.filter((fileName) => {
     return fileNames2skip.indexOf(fileName) == -1
@@ -90,19 +219,10 @@ function prepFilterSkusImages(filteredSkus: SkuI[], merge = true, fileNames: Arr
 }
 
 function cleanFilename(fileName: string) {
-  return (
-    fileName
-      // ── 1. strip any query string (everything from the first “?” to the end)
-      .replace(/\?.*$/, '')
-      // ── 2. strip a terminal image extension (case‑insensitive) .jpeg, .jpg, .png or .gif
-      .replace(/\.(?:jpe?g|png|gif)$/i, '')
-      // ── 3. drop every character that is NOT a letter (Unicode!) or a digit
-      .replace(/[^\p{L}\p{N}]+/gu, '')
-      .toLowerCase()
-  )
+  return filenameStem(fileName).replace(DELIMITERS, '')
 }
 
-function checkWithoutLast(fileName: string) {
+function removeLastToken(fileName: string) {
   // If last token equals any of the following patterns, we will check the fileName without the last token
   // 1) includes front, back, side or sole
   // 2) is 1 alphanumerical character
@@ -112,11 +232,9 @@ function checkWithoutLast(fileName: string) {
   // 5) ends with a number
   // 6) ends with two numbers
   // If any of those checks are true, we return the fileName without the last token
-  // Last token here is defined as the last token in the fileName, split by underscores, dashes, dots, spaces or URL encoded characters (like %20)
-  const tokens = fileName
-    .toLowerCase()
-    .split(/[_\-.\s()]+|%[0-9A-Fa-f]{2}/)
-    .slice(0, -1)
+  // Tokens come from the shared stem (decode + extension strip), split on anything that is
+  // not a letter, digit, or apostrophe — same rule as cleanFilename.
+  const tokens = filenameStem(fileName).split(DELIMITERS).filter(Boolean)
   const lastToken = tokens[tokens.length - 1]
   let returnWithoutLast = false
   if (lastToken) {
@@ -127,37 +245,22 @@ function checkWithoutLast(fileName: string) {
     const tokenLength = tokens.length
     if (returnWithoutLast) {
       let sliceIndex = -1
-      if (tokenLength > 3 && tokens[tokenLength - 3] == 'press' && ((tokens[tokenLength - 2] == 'pack') || (tokens[tokenLength - 2] == 'model'))) sliceIndex = -3
+      if (tokenLength > 3 && tokens[tokenLength - 3] == 'press' && (tokens[tokenLength - 2] == 'pack' || tokens[tokenLength - 2] == 'model')) sliceIndex = -3
       else if (tokenLength > 2 && tokens[tokenLength - 2] == 'packshot') sliceIndex = -2
-      return tokens
-        .slice(0, sliceIndex)
-        .join('')
-        .replace(/[^\p{L}\p{N}]+/gu, '')
+      return tokens.slice(0, sliceIndex).join('')
     }
-    if (tokenLength > 2 && tokens[tokenLength - 2] == 'press' && ((tokens[tokenLength - 1] == 'pack') || (tokens[tokenLength - 1] == 'model'))) {
-      return tokens
-        .slice(0, -2)
-        .join('')
-        .replace(/[^\p{L}\p{N}]+/gu, '')
+    if (tokenLength > 2 && tokens[tokenLength - 2] == 'press' && (tokens[tokenLength - 1] == 'pack' || tokens[tokenLength - 1] == 'model')) {
+      return tokens.slice(0, -2).join('')
     } else if (tokenLength > 1 && tokens[tokenLength - 1] == 'packshot') {
-      return tokens
-        .slice(0, -1)
-        .join('')
-        .replace(/[^\p{L}\p{N}]+/gu, '')
+      return tokens.slice(0, -1).join('')
     }
     if (lastToken.includes('product')) {
-      const joinedCleanFileName = tokens.join('').replace(/[^\p{L}\p{N}]+/gu, '')
+      const joinedCleanFileName = tokens.join('')
       return joinedCleanFileName.slice(0, joinedCleanFileName.indexOf('product'))
     } else if (Number(lastToken.slice(-1))) {
-      return tokens
-        .join('')
-        .replace(/[^\p{L}\p{N}]+/gu, '')
-        .slice(0, -1)
+      return tokens.join('').slice(0, -1)
     } else if (Number(lastToken.slice(-2))) {
-      return tokens
-        .join('')
-        .replace(/[^\p{L}\p{N}]+/gu, '')
-        .substring(0, -2)
+      return tokens.join('').substring(0, -2)
     }
   }
   return ''
@@ -364,7 +467,6 @@ function fallbackSearch(cleanedFileName: string, fileNameWithoutLast: string, pr
 }
 
 function imageBinder(fileNames: Array<string>, filteredSkus: SkuI[], brand: string, specifiedPattern: Array<string>, test = false) {
-  const totalLength = fileNames.length
   let count = 1
   const fileNameIndexMatch = {}
   let quickCount = 0
@@ -373,32 +475,96 @@ function imageBinder(fileNames: Array<string>, filteredSkus: SkuI[], brand: stri
   let newPatterns = {}
   if (specifiedPattern[0] != 'auto') {
     // If the user/administrator provides a pattern, we use that by default
-    const prepObject = prepFilterSkusImages(filteredSkus, false, fileNames)
+    const prepObject = prepFilterSkusImagesManual(filteredSkus, fileNames, specifiedPattern)
     const preppedSkus = prepObject.preppedSkus
     const fileNames2Match = prepObject.fileNames2Match
     fileNames2Match.forEach((fileName) => {
       const cleanedFileName = cleanFilename(fileName)
-      const fileNameWithoutLast = checkWithoutLast(fileName)
       fileNameIndexMatch[fileName] = []
+      const matchingCombinations = {}
+      // Check which skus match the specified pattern and store matching fields (combinations)
       preppedSkus.forEach((sku, idx) => {
         if (sku['skip']) return
         let skuStillMatches = true
+        let combinationArray: Array<string> = []
         specifiedPattern.every((fieldName) => {
-          if (!(fieldName in sku)) skuStillMatches = false
-          else {
-            const fieldVal = sku[fieldName]
-            if (!cleanedFileName.includes(fieldVal) && !fileNameWithoutLast.includes(fieldVal)) skuStillMatches = false
-          }
+          const fieldVal = sku[fieldName]
+          if (!cleanedFileName.includes(fieldVal)) skuStillMatches = false
+          else combinationArray.push(fieldVal)
           return skuStillMatches
         })
         if (skuStillMatches) {
-          fileNameIndexMatch[fileName].push(idx)
+          let combination = combinationArray.join('|')
+          if (combination in matchingCombinations) {
+            matchingCombinations[combination].push(idx)
+          } else {
+            matchingCombinations[combination] = [idx]
+          }
         }
       })
+      let matchingIndices: Array<number> = []
+      // If only one combination of specified pattern matches, we use it
+      if (Object.keys(matchingCombinations).length == 1) { 
+        matchingIndices = matchingCombinations[Object.keys(matchingCombinations)[0]] }
+      else {
+        // In case of multiple combinations, do substring check
+        // -> Checks if all fields have a specific place in file name, with no overlap
+        const matchesAfterSubstringCheck = {}
+        Object.keys(matchingCombinations).forEach((combination) => {
+          const combinationArray = combination.split('|')
+          const isMatch = checkPattern(combinationArray, cleanedFileName)
+          if (isMatch) {
+            matchesAfterSubstringCheck[combination] = matchingCombinations[combination]
+          }
+        })
+        // If only one combination is left after substring check, we use it
+        if (Object.keys(matchesAfterSubstringCheck).length == 1) { 
+          matchingIndices = matchesAfterSubstringCheck[Object.keys(matchesAfterSubstringCheck)[0]] }
+        else {
+          // In case there are still multiple combinations, we check longest combinations
+          // -> Longest combinations are more specific and therefore preferred
+          let lengthLongest = 0
+          let longestCombinations: Array<string> = []
+          Object.keys(matchesAfterSubstringCheck).forEach((combination) => {
+            if (combination.length > lengthLongest) {
+              lengthLongest = combination.length
+              longestCombinations = [combination]
+            } else if (combination.length == lengthLongest) {
+              longestCombinations.push(combination)
+            }
+          })
+          // If only one combination is left after longest combination check, we use it
+          if (longestCombinations.length == 1) { 
+            matchingIndices = matchesAfterSubstringCheck[longestCombinations[0]] }
+          else {
+            // In case there are still multiple combinations, we check fileNameWithoutLast
+            // -> If any combination matches fileNameWithoutLast, we prefer it, as last tokens can be junk
+            const fileNameWithoutLast = removeLastToken(fileName)
+            const matchesWithoutLast: Record<string, number[]> = {}
+            if (fileNameWithoutLast) {
+              longestCombinations.forEach((combination) => {
+                const combinationArray = combination.split('|')
+                if (checkPattern(combinationArray, fileNameWithoutLast)) {
+                  matchesWithoutLast[combination] = matchesAfterSubstringCheck[combination]
+                }
+              })
+            }
+            // If only one combination is left after fileNameWithoutLast check, we use it
+            if (Object.keys(matchesWithoutLast).length == 1) { matchingIndices = matchesWithoutLast[Object.keys(matchesWithoutLast)[0]] }
+            else {
+              Object.keys(matchesAfterSubstringCheck).forEach((combination) => {
+                matchingIndices.push(...matchesAfterSubstringCheck[combination])
+              })
+            }
+          }
+        }
+      }
+      if (matchingIndices.length > 0) fileNameIndexMatch[fileName] = matchingIndices
+      if (!fileNameIndexMatch[fileName]) fileNameIndexMatch[fileName] = []
     })
   } else {
     // Else, we look for patterns ourselves using a couple of methods, prioritized on efficiency
-    const prepObject = prepFilterSkusImages(filteredSkus, true, fileNames)
+    const prepObject = prepFilterSkusImagesAuto(filteredSkus, fileNames)
     const preppedSkus = prepObject.preppedSkus
     const fileNames2Match = prepObject.fileNames2Match
     let quickSearch = prepQuickSearch(preppedSkus)
@@ -406,7 +572,7 @@ function imageBinder(fileNames: Array<string>, filteredSkus: SkuI[], brand: stri
     fileNames2Match.forEach((fileName) => {
       let matchFound = false
       const cleanedFileName = cleanFilename(fileName)
-      const fileNameWithoutLast = checkWithoutLast(fileName)
+      const fileNameWithoutLast = removeLastToken(fileName)
       let indicesForFileName: Array<number> = []
       // Method #1: try quicksearch, an object with patterns as keys and arrays of [proposedFilename, sku] as values
       // based on existing patterns in GLOBAL_PATTERNS
@@ -469,7 +635,7 @@ function onlySaveIDFromSkus(matchDict: {}): Record<string, number[]> {
   return out
 }
 
-async function testBatch(brand: string) {
+async function testBatch(brand: string, specifiedPattern: Array<string> = ['auto']) {
   const [skusModule, imagesModule, matchedModule] = await Promise.all([
     import(`../test/imageBinderTestData/brandSkus/${brand}.json`),
     import(`../test/imageBinderTestData/brandImages/${brand}.json`),
@@ -482,7 +648,7 @@ async function testBatch(brand: string) {
 
   // 1) Call imageBinder
   const t0 = Date.now()
-  const imageBinderResult = imageBinder(fileNames, filteredSkus, brand, ['auto'], true)
+  const imageBinderResult = imageBinder(fileNames, filteredSkus, brand, specifiedPattern, true)
   const t1 = Date.now()
   const matchIndices = imageBinderResult.match || {}
 

package/test/imageBinder.spec.js

@@ -15,12 +15,12 @@ describe('imageBinder function', function () {
     expect(result.Time).toBeLessThan(10)
   }, 300000)
   it('should deal with large batches', async function () {
-    // Tests on 1594 images and 18196 skus, 1593 are quick search
+    // Tests on 1594 images and 18196 skus; most are quick search
     const result = await testBatch('lowa')
     expect(Object.keys(result).length).toBe(8)
     expect(result.Mismatches).toBe(0)
-    expect(result.QuickSearch).toBe(1587)
-    expect(result.Patterns['articleCode|articleCodeSupplier-colorCodeSupplier|colorSupplier']).toBe(1586)
+    expect(result.QuickSearch).toBe(1588)
+    expect(result.Patterns['articleCode|articleCodeSupplier-colorCodeSupplier|colorSupplier']).toBe(1588)
     // Time can vary, so we use a loose check (within 5 seconds)
     expect(result.Time).toBeGreaterThan(0)
     expect(result.Time).toBeLessThan(5)
@@ -48,4 +48,53 @@ describe('imageBinder function', function () {
     expect(result.Time).toBeGreaterThan(0)
     expect(result.Time).toBeLessThan(5)
   })
+  it('should prefer the longest matching combination for specified patterns (lingadore 8501)', async function () {
+    // When articleCodeSupplier values share prefixes (8501 vs 8501B vs 8501BD),
+    // images must match only the most specific article code.
+    const result = await testBatch('lingadore', ['articleCodeSupplier', 'colorCodeSupplier'])
+    expect(Object.keys(result).length).toBe(8)
+    expect(result.Brand).toBe('lingadore')
+    expect(result.Mismatches).toBe(0)
+    expect(result.QuickSearch).toBe(0)
+    expect(result.FallbackSearch).toBe(0)
+    expect(result.Time).toBeGreaterThan(0)
+    expect(result.Time).toBeLessThan(5)
+  })
+  it('should not match if substrings do not appear in separate positions (pennAndInk)', async function () {
+    // e.g. article W26B422 with colors 15/90 must not also match color 42 just because "42"
+    // appears inside the article code.
+    const result = await testBatch('pennAndInk', ['articleCodeSupplier', 'colorCodeSupplier'])
+    expect(Object.keys(result).length).toBe(8)
+    expect(result.Brand).toBe('pennAndInk')
+    expect(result.Mismatches).toBe(0)
+    expect(result.QuickSearch).toBe(0)
+    expect(result.FallbackSearch).toBe(0)
+    expect(result.Time).toBeGreaterThan(0)
+    expect(result.Time).toBeLessThan(5)
+  })
+  it('should allow adjacent article and colour spans in specified patterns (lowa)', async function () {
+    // When article and colour abut in the cleaned filename (e.g. LA830581 + black-grey),
+    // checkPattern must treat touching spans as non-overlapping so the real colour wins
+    // over shorter substring colours (black).
+    const result = await testBatch('lowa', ['articleCodeSupplier', 'colorCodeSupplier'])
+    expect(Object.keys(result).length).toBe(8)
+    expect(result.Brand).toBe('lowa')
+    expect(result.Mismatches).toBe(0)
+    expect(result.QuickSearch).toBe(0)
+    expect(result.FallbackSearch).toBe(0)
+    expect(result.Time).toBeGreaterThan(0)
+    expect(result.Time).toBeLessThan(5)
+  }, 140000)
+  it('should prefer combinations that still match after dropping a junk last token (exxcellent)', async function () {
+    // Equal-length colours can both match the full stem (e.g. 001 vs 014 on …001_4).
+    // After removeLastToken, only the true colour remains — keep that combination.
+    const result = await testBatch('exxcellent', ['articleCodeSupplier', 'colorCodeSupplier'])
+    expect(Object.keys(result).length).toBe(8)
+    expect(result.Brand).toBe('exxcellent')
+    expect(result.Mismatches).toBe(0)
+    expect(result.QuickSearch).toBe(0)
+    expect(result.FallbackSearch).toBe(0)
+    expect(result.Time).toBeGreaterThan(0)
+    expect(result.Time).toBeLessThan(5)
+  })
 })