@open-mercato/shared 0.7.1-develop.7112.1.d486e6f1b2 → 0.7.1-develop.7120.1.a0154ac412

package/dist/lib/search/tokenize.js

@@ -1,7 +1,40 @@
 import crypto from "crypto";
 import { resolveSearchConfig, resolveSearchTokenLimits } from "./config.js";
+const NON_DECOMPOSING_FOLDS = {
+  "\u0142": "l",
+  "\u0141": "L",
+  "\xF8": "o",
+  "\xD8": "O",
+  "\u0111": "d",
+  "\u0110": "D",
+  "\xF0": "d",
+  "\xD0": "D",
+  "\u0127": "h",
+  "\u0126": "H",
+  "\u0131": "i",
+  "\u0138": "k",
+  "\u014B": "n",
+  "\u014A": "N",
+  "\u0167": "t",
+  "\u0166": "T",
+  "\xE6": "ae",
+  "\xC6": "AE",
+  "\u0153": "oe",
+  "\u0152": "OE",
+  "\xFE": "th",
+  "\xDE": "TH",
+  "\xDF": "ss",
+  "\u1E9E": "SS"
+};
+const NON_DECOMPOSING_PATTERN = new RegExp(
+  `[${Object.keys(NON_DECOMPOSING_FOLDS).join("").replace(/[\\\]^-]/g, "\\$&")}]`,
+  "g"
+);
+function foldNonDecomposingLetters(text) {
+  return text.replace(NON_DECOMPOSING_PATTERN, (char) => NON_DECOMPOSING_FOLDS[char]);
+}
 function normalizeText(text) {
-  return text.normalize("NFKD").replace(/[\u0300-\u036f]/g, "").replace(/[%_]/g, " ").toLowerCase();
+  return foldNonDecomposingLetters(text.normalize("NFKD").replace(/[\u0300-\u036f]/g, "")).replace(/[%_]/g, " ").toLowerCase();
 }
 function splitTokens(text, minLength) {
   return normalizeText(text).split(/[^a-z0-9]+/i).filter((token) => token.length >= minLength);

package/dist/lib/search/tokenize.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../src/lib/search/tokenize.ts"],
-  "sourcesContent": ["import crypto from 'crypto'\nimport { resolveSearchConfig, resolveSearchTokenLimits, type SearchConfig } from './config'\n\nexport type TokenizationResult = {\n  tokens: string[]\n  hashes: string[]\n}\n\nfunction normalizeText(text: string): string {\n  return text\n    .normalize('NFKD')\n    .replace(/[\\u0300-\\u036f]/g, '')\n    .replace(/[%_]/g, ' ')\n    .toLowerCase()\n}\n\nfunction splitTokens(text: string, minLength: number): string[] {\n  return normalizeText(text)\n    .split(/[^a-z0-9]+/i)\n    .filter((token) => token.length >= minLength)\n}\n\nfunction appendExpandedToken(\n  token: string,\n  config: SearchConfig,\n  seen: Set<string>,\n  tokens: string[],\n  limit: number,\n): void {\n  const append = (candidate: string): boolean => {\n    if (seen.has(candidate)) return tokens.length < limit\n    seen.add(candidate)\n    tokens.push(candidate)\n    return tokens.length < limit\n  }\n\n  if (!config.enablePartials) {\n    append(token)\n    return\n  }\n\n  for (let length = config.minTokenLength; length <= token.length; length += 1) {\n    if (!append(token.slice(0, length))) return\n  }\n}\n\nexport function hashToken(token: string, config?: SearchConfig): string {\n  const cfg = config ?? resolveSearchConfig()\n  return crypto.createHash(cfg.hashAlgorithm).update(token).digest('hex')\n}\n\nexport function tokenizeText(text: string, config?: SearchConfig): TokenizationResult {\n  const cfg = config ?? resolveSearchConfig()\n  const limits = resolveSearchTokenLimits(cfg)\n  const boundedText = limits.maxFieldChars > 0 ? text.slice(0, limits.maxFieldChars) : text\n  const tokenLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY\n  const seen = new Set<string>()\n  const tokens: string[] = []\n\n  for (const token of splitTokens(boundedText, cfg.minTokenLength)) {\n    if (tokens.length >= tokenLimit) break\n    appendExpandedToken(token, cfg, seen, tokens, tokenLimit)\n  }\n\n  const hashes = tokens.map((token) => hashToken(token, cfg))\n  return { tokens, hashes }\n}\n"],
-  "mappings": "AAAA,OAAO,YAAY;AACnB,SAAS,qBAAqB,gCAAmD;AAOjF,SAAS,cAAc,MAAsB;AAC3C,SAAO,KACJ,UAAU,MAAM,EAChB,QAAQ,oBAAoB,EAAE,EAC9B,QAAQ,SAAS,GAAG,EACpB,YAAY;AACjB;AAEA,SAAS,YAAY,MAAc,WAA6B;AAC9D,SAAO,cAAc,IAAI,EACtB,MAAM,aAAa,EACnB,OAAO,CAAC,UAAU,MAAM,UAAU,SAAS;AAChD;AAEA,SAAS,oBACP,OACA,QACA,MACA,QACA,OACM;AACN,QAAM,SAAS,CAAC,cAA+B;AAC7C,QAAI,KAAK,IAAI,SAAS,EAAG,QAAO,OAAO,SAAS;AAChD,SAAK,IAAI,SAAS;AAClB,WAAO,KAAK,SAAS;AACrB,WAAO,OAAO,SAAS;AAAA,EACzB;AAEA,MAAI,CAAC,OAAO,gBAAgB;AAC1B,WAAO,KAAK;AACZ;AAAA,EACF;AAEA,WAAS,SAAS,OAAO,gBAAgB,UAAU,MAAM,QAAQ,UAAU,GAAG;AAC5E,QAAI,CAAC,OAAO,MAAM,MAAM,GAAG,MAAM,CAAC,EAAG;AAAA,EACvC;AACF;AAEO,SAAS,UAAU,OAAe,QAA+B;AACtE,QAAM,MAAM,UAAU,oBAAoB;AAC1C,SAAO,OAAO,WAAW,IAAI,aAAa,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxE;AAEO,SAAS,aAAa,MAAc,QAA2C;AACpF,QAAM,MAAM,UAAU,oBAAoB;AAC1C,QAAM,SAAS,yBAAyB,GAAG;AAC3C,QAAM,cAAc,OAAO,gBAAgB,IAAI,KAAK,MAAM,GAAG,OAAO,aAAa,IAAI;AACrF,QAAM,aAAa,OAAO,oBAAoB,IAAI,OAAO,oBAAoB,OAAO;AACpF,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,SAAmB,CAAC;AAE1B,aAAW,SAAS,YAAY,aAAa,IAAI,cAAc,GAAG;AAChE,QAAI,OAAO,UAAU,WAAY;AACjC,wBAAoB,OAAO,KAAK,MAAM,QAAQ,UAAU;AAAA,EAC1D;AAEA,QAAM,SAAS,OAAO,IAAI,CAAC,UAAU,UAAU,OAAO,GAAG,CAAC;AAC1D,SAAO,EAAE,QAAQ,OAAO;AAC1B;",
+  "sourcesContent": ["import crypto from 'crypto'\nimport { resolveSearchConfig, resolveSearchTokenLimits, type SearchConfig } from './config'\n\nexport type TokenizationResult = {\n  tokens: string[]\n  hashes: string[]\n}\n\n/**\n * Latin letters that NFKD leaves intact because they are atomic codepoints rather than a\n * base letter plus a combining mark. Stripping combining marks therefore never folds them\n * to ASCII, and `splitTokens` then consumes them as separators \\u2014 truncating `\\u0141ukasz` to\n * `ukasz` and cutting `Za\\u017c\\u00f3\\u0142\\u0107` down to `zazo`. Because the same tokenizer runs at index\n * time and at query time, such a record becomes unreachable from every spelling. Only\n * characters with a single unambiguous ASCII fold belong here; anything language-dependent\n * must stay out.\n *\n * `normalizeText` applies this fold AFTER NFKD and mark stripping, and the order is\n * load-bearing: folding first would miss the characters that decompose *into* one of these\n * letters, such as `\\u01ff` (U+01FF \\u2192 `\\u00f8` + U+0301), `\\u01fd` (U+01FD), `\\u01e3` (U+01E3) and `\\u210f` (U+210F).\n * The letters below have no decomposition of their own, so NFKD passes them through\n * untouched and they still fold correctly in this position.\n *\n * The table covers every letter in Latin-1 Supplement (U+00C0-U+00FF) and Latin Extended-A\n * (U+0100-U+017F) that NFKD leaves un-folded; `tokenize.test.ts` pins that range so a gap\n * cannot silently reopen. Two entries look redundant and are not: `\\u0110` (U+0110, D with\n * stroke) and `\\u00d0` (U+00D0, Eth) are visually indistinguishable in uppercase and are\n * routinely substituted for one another in Croatian, Serbian and Vietnamese text, so both\n * must fold to `D` or the same rendered name yields two disjoint token sets. Do not delete\n * either as a duplicate of the other.\n *\n * Letters outside those two blocks are deliberately out of scope \\u2014 `\\u0259`/`\\u018f` (U+0259/U+018F,\n * common in Azerbaijani names such as `\\u018fliyev`) fold to `e` under ICU and belong here on\n * the same reasoning, but each addition forces operators through another `search_tokens`\n * reindex, so extending the range is tracked separately rather than done piecemeal.\n */\nconst NON_DECOMPOSING_FOLDS: Record<string, string> = {\n  '\\u0142': 'l',\n  '\\u0141': 'L',\n  '\\u00f8': 'o',\n  '\\u00d8': 'O',\n  '\\u0111': 'd',\n  '\\u0110': 'D',\n  '\\u00f0': 'd',\n  '\\u00d0': 'D',\n  '\\u0127': 'h',\n  '\\u0126': 'H',\n  '\\u0131': 'i',\n  '\\u0138': 'k',\n  '\\u014b': 'n',\n  '\\u014a': 'N',\n  '\\u0167': 't',\n  '\\u0166': 'T',\n  '\\u00e6': 'ae',\n  '\\u00c6': 'AE',\n  '\\u0153': 'oe',\n  '\\u0152': 'OE',\n  '\\u00fe': 'th',\n  '\\u00de': 'TH',\n  '\\u00df': 'ss',\n  '\\u1e9e': 'SS',\n}\n\nconst NON_DECOMPOSING_PATTERN = new RegExp(\n  `[${Object.keys(NON_DECOMPOSING_FOLDS)\n    .join('')\n    .replace(/[\\\\\\]^-]/g, '\\\\$&')}]`,\n  'g',\n)\n\nfunction foldNonDecomposingLetters(text: string): string {\n  return text.replace(NON_DECOMPOSING_PATTERN, (char) => NON_DECOMPOSING_FOLDS[char])\n}\n\nfunction normalizeText(text: string): string {\n  return foldNonDecomposingLetters(text.normalize('NFKD').replace(/[\\u0300-\\u036f]/g, ''))\n    .replace(/[%_]/g, ' ')\n    .toLowerCase()\n}\n\nfunction splitTokens(text: string, minLength: number): string[] {\n  return normalizeText(text)\n    .split(/[^a-z0-9]+/i)\n    .filter((token) => token.length >= minLength)\n}\n\nfunction appendExpandedToken(\n  token: string,\n  config: SearchConfig,\n  seen: Set<string>,\n  tokens: string[],\n  limit: number,\n): void {\n  const append = (candidate: string): boolean => {\n    if (seen.has(candidate)) return tokens.length < limit\n    seen.add(candidate)\n    tokens.push(candidate)\n    return tokens.length < limit\n  }\n\n  if (!config.enablePartials) {\n    append(token)\n    return\n  }\n\n  for (let length = config.minTokenLength; length <= token.length; length += 1) {\n    if (!append(token.slice(0, length))) return\n  }\n}\n\nexport function hashToken(token: string, config?: SearchConfig): string {\n  const cfg = config ?? resolveSearchConfig()\n  return crypto.createHash(cfg.hashAlgorithm).update(token).digest('hex')\n}\n\nexport function tokenizeText(text: string, config?: SearchConfig): TokenizationResult {\n  const cfg = config ?? resolveSearchConfig()\n  const limits = resolveSearchTokenLimits(cfg)\n  const boundedText = limits.maxFieldChars > 0 ? text.slice(0, limits.maxFieldChars) : text\n  const tokenLimit = limits.maxTokensPerField > 0 ? limits.maxTokensPerField : Number.POSITIVE_INFINITY\n  const seen = new Set<string>()\n  const tokens: string[] = []\n\n  for (const token of splitTokens(boundedText, cfg.minTokenLength)) {\n    if (tokens.length >= tokenLimit) break\n    appendExpandedToken(token, cfg, seen, tokens, tokenLimit)\n  }\n\n  const hashes = tokens.map((token) => hashToken(token, cfg))\n  return { tokens, hashes }\n}\n"],
+  "mappings": "AAAA,OAAO,YAAY;AACnB,SAAS,qBAAqB,gCAAmD;AAmCjF,MAAM,wBAAgD;AAAA,EACpD,UAAU;AAAA,EACV,UAAU;AAAA,EACV,QAAU;AAAA,EACV,QAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,QAAU;AAAA,EACV,QAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,QAAU;AAAA,EACV,QAAU;AAAA,EACV,UAAU;AAAA,EACV,UAAU;AAAA,EACV,QAAU;AAAA,EACV,QAAU;AAAA,EACV,QAAU;AAAA,EACV,UAAU;AACZ;AAEA,MAAM,0BAA0B,IAAI;AAAA,EAClC,IAAI,OAAO,KAAK,qBAAqB,EAClC,KAAK,EAAE,EACP,QAAQ,aAAa,MAAM,CAAC;AAAA,EAC/B;AACF;AAEA,SAAS,0BAA0B,MAAsB;AACvD,SAAO,KAAK,QAAQ,yBAAyB,CAAC,SAAS,sBAAsB,IAAI,CAAC;AACpF;AAEA,SAAS,cAAc,MAAsB;AAC3C,SAAO,0BAA0B,KAAK,UAAU,MAAM,EAAE,QAAQ,oBAAoB,EAAE,CAAC,EACpF,QAAQ,SAAS,GAAG,EACpB,YAAY;AACjB;AAEA,SAAS,YAAY,MAAc,WAA6B;AAC9D,SAAO,cAAc,IAAI,EACtB,MAAM,aAAa,EACnB,OAAO,CAAC,UAAU,MAAM,UAAU,SAAS;AAChD;AAEA,SAAS,oBACP,OACA,QACA,MACA,QACA,OACM;AACN,QAAM,SAAS,CAAC,cAA+B;AAC7C,QAAI,KAAK,IAAI,SAAS,EAAG,QAAO,OAAO,SAAS;AAChD,SAAK,IAAI,SAAS;AAClB,WAAO,KAAK,SAAS;AACrB,WAAO,OAAO,SAAS;AAAA,EACzB;AAEA,MAAI,CAAC,OAAO,gBAAgB;AAC1B,WAAO,KAAK;AACZ;AAAA,EACF;AAEA,WAAS,SAAS,OAAO,gBAAgB,UAAU,MAAM,QAAQ,UAAU,GAAG;AAC5E,QAAI,CAAC,OAAO,MAAM,MAAM,GAAG,MAAM,CAAC,EAAG;AAAA,EACvC;AACF;AAEO,SAAS,UAAU,OAAe,QAA+B;AACtE,QAAM,MAAM,UAAU,oBAAoB;AAC1C,SAAO,OAAO,WAAW,IAAI,aAAa,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxE;AAEO,SAAS,aAAa,MAAc,QAA2C;AACpF,QAAM,MAAM,UAAU,oBAAoB;AAC1C,QAAM,SAAS,yBAAyB,GAAG;AAC3C,QAAM,cAAc,OAAO,gBAAgB,IAAI,KAAK,MAAM,GAAG,OAAO,aAAa,IAAI;AACrF,QAAM,aAAa,OAAO,oBAAoB,IAAI,OAAO,oBAAoB,OAAO;AACpF,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,SAAmB,CAAC;AAE1B,aAAW,SAAS,YAAY,aAAa,IAAI,cAAc,GAAG;AAChE,QAAI,OAAO,UAAU,WAAY;AACjC,wBAAoB,OAAO,KAAK,MAAM,QAAQ,UAAU;AAAA,EAC1D;AAEA,QAAM,SAAS,OAAO,IAAI,CAAC,UAAU,UAAU,OAAO,GAAG,CAAC;AAC1D,SAAO,EAAE,QAAQ,OAAO;AAC1B;",
   "names": []
 }

package/dist/lib/version.js

@@ -1,4 +1,4 @@
-const APP_VERSION = "0.7.1-develop.7112.1.d486e6f1b2";
+const APP_VERSION = "0.7.1-develop.7120.1.a0154ac412";
 const appVersion = APP_VERSION;
 export {
   APP_VERSION,

package/dist/lib/version.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/lib/version.ts"],
-  "sourcesContent": ["// Build-time generated version\nexport const APP_VERSION = '0.7.1-develop.7112.1.d486e6f1b2';\nexport const appVersion = APP_VERSION;\n"],
+  "sourcesContent": ["// Build-time generated version\nexport const APP_VERSION = '0.7.1-develop.7120.1.a0154ac412';\nexport const appVersion = APP_VERSION;\n"],
   "mappings": "AACO,MAAM,cAAc;AACpB,MAAM,aAAa;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@open-mercato/shared",
-  "version": "0.7.1-develop.7112.1.d486e6f1b2",
+  "version": "0.7.1-develop.7120.1.a0154ac412",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.js",
@@ -109,7 +109,7 @@
     "@mikro-orm/core": "^7.1.8",
     "@mikro-orm/decorators": "^7.1.8",
     "@mikro-orm/postgresql": "^7.1.8",
-    "@open-mercato/cache": "0.7.1-develop.7112.1.d486e6f1b2",
+    "@open-mercato/cache": "0.7.1-develop.7120.1.a0154ac412",
     "@types/html-to-text": "^9.0.4",
     "@types/sanitize-html": "^2.16.1",
     "dotenv": "^17.4.2",

package/src/lib/search/__tests__/tokenize.test.ts

@@ -10,6 +10,114 @@ const baseConfig: SearchConfig = {
   blocklistedFields: [],
 }
 
+describe('tokenizeText diacritic folding', () => {
+  const wholeWordConfig: SearchConfig = { ...baseConfig, enablePartials: false }
+
+  test.each([
+    ['Łukasz', ['lukasz']],
+    ['lukasz', ['lukasz']],
+    ['Zażółć', ['zazolc']],
+    ['Łódź', ['lodz']],
+    ['Lodz', ['lodz']],
+  ])('folds non-decomposing Polish letters in %s', (input, expected) => {
+    const { tokens } = tokenizeText(input, wholeWordConfig)
+
+    expect(tokens).toEqual(expected)
+  })
+
+  test.each([
+    ['Bąk', ['bak']],
+    ['Wróbel', ['wrobel']],
+    ['Piotr Świątek', ['piotr', 'swiatek']],
+  ])('keeps folding NFKD-decomposable diacritics in %s', (input, expected) => {
+    const { tokens } = tokenizeText(input, wholeWordConfig)
+
+    expect(tokens).toEqual(expected)
+  })
+
+  test.each([
+    ['Jørgensen', ['jorgensen']],
+    ['Đurić', ['duric']],
+    ['Ħamrun', ['hamrun']],
+    ['Işık', ['isik']],
+    ['Æther', ['aether']],
+    ['Œuvre', ['oeuvre']],
+    ['Straße', ['strasse']],
+    ['Þórsdóttir', ['thorsdottir']],
+    ['Guðmundsdóttir', ['gudmundsdottir']],
+    ['Sæþór', ['saethor']],
+    ['Ŋoma', ['noma']],
+    ['Ŧorvald', ['torvald']],
+  ])('folds non-decomposing letters beyond Polish in %s', (input, expected) => {
+    const { tokens } = tokenizeText(input, wholeWordConfig)
+
+    expect(tokens).toEqual(expected)
+  })
+
+  test('folds Eth and D-with-stroke identically, since the two are visually indistinguishable', () => {
+    const dWithStroke = tokenizeText('Đurić', wholeWordConfig)
+    const eth = tokenizeText('Ðurić', wholeWordConfig)
+
+    expect(dWithStroke.tokens).toEqual(['duric'])
+    expect(eth.tokens).toEqual(['duric'])
+    expect(eth.hashes).toEqual(dWithStroke.hashes)
+  })
+
+  test.each([
+    ['Ǿrnulf', ['ornulf']],
+    ['Ǽlfric', ['aelfric']],
+    ['ǣrest', ['aerest']],
+    ['ℏbar', ['hbar']],
+  ])('folds %s, which NFKD decomposes into a non-decomposing letter', (input, expected) => {
+    const { tokens } = tokenizeText(input, wholeWordConfig)
+
+    expect(tokens).toEqual(expected)
+  })
+
+  test('produces identical hashes for the diacritic and ASCII spellings of a name', () => {
+    const indexed = tokenizeText('Łukasz Wałęsa', wholeWordConfig)
+    const queried = tokenizeText('lukasz walesa', wholeWordConfig)
+
+    expect(indexed.tokens).toEqual(queried.tokens)
+    expect(indexed.hashes).toEqual(queried.hashes)
+  })
+
+  test('expands prefixes from the folded token rather than the truncated one', () => {
+    const { tokens } = tokenizeText('Łódź', baseConfig)
+
+    expect(tokens).toEqual(['lod', 'lodz'])
+  })
+
+  // The property NON_DECOMPOSING_FOLDS actually exists to guarantee: no letter in the two
+  // blocks it draws from may split or truncate the word it sits in. Asserting the range
+  // directly is what catches a gap; enumerating characters by hand is what let nine of them
+  // through in the first place.
+  //
+  // The three exclusions are a different defect class, not missing table rows. NFKD *does*
+  // decompose them — into a base letter plus a non-combining separator (U+00B7 middle dot
+  // for the two L-with-middle-dot letters, U+02BC modifier apostrophe for U+0149) — which
+  // `splitTokens` then cuts the word at. A table entry for them would be dead code, because
+  // the fold runs after NFKD and the codepoint no longer exists by then. Fixing them means
+  // deciding whether that separator residue should survive tokenization at all, which also
+  // governs the far commoner ASCII spelling (`Paral·lel` normalizes identically to
+  // `Paraŀlel`), so it is tracked as its own change rather than smuggled in here.
+  const SEPARATOR_RESIDUE_LETTERS = ['U+013F Ŀ', 'U+0140 ŀ', 'U+0149 ʼn']
+
+  test('keeps every Latin-1 Supplement and Latin Extended-A letter inside a single token', () => {
+    const lost: string[] = []
+
+    for (let codePoint = 0xc0; codePoint <= 0x17f; codePoint += 1) {
+      const char = String.fromCodePoint(codePoint)
+      if (!/\p{L}/u.test(char)) continue
+      if (tokenizeText(`a${char}b`, wholeWordConfig).tokens.length !== 1) {
+        lost.push(`U+${codePoint.toString(16).toUpperCase().padStart(4, '0')} ${char}`)
+      }
+    }
+
+    expect(lost).toEqual(SEPARATOR_RESIDUE_LETTERS)
+  })
+})
+
 describe('tokenizeText limits', () => {
   test('truncates oversized field text before tokenizing', () => {
     const config = { ...baseConfig, enablePartials: false, maxFieldChars: 10 }

package/src/lib/search/tokenize.ts

@@ -6,10 +6,74 @@ export type TokenizationResult = {
   hashes: string[]
 }
 
+/**
+ * Latin letters that NFKD leaves intact because they are atomic codepoints rather than a
+ * base letter plus a combining mark. Stripping combining marks therefore never folds them
+ * to ASCII, and `splitTokens` then consumes them as separators \u2014 truncating `\u0141ukasz` to
+ * `ukasz` and cutting `Za\u017c\u00f3\u0142\u0107` down to `zazo`. Because the same tokenizer runs at index
+ * time and at query time, such a record becomes unreachable from every spelling. Only
+ * characters with a single unambiguous ASCII fold belong here; anything language-dependent
+ * must stay out.
+ *
+ * `normalizeText` applies this fold AFTER NFKD and mark stripping, and the order is
+ * load-bearing: folding first would miss the characters that decompose *into* one of these
+ * letters, such as `\u01ff` (U+01FF \u2192 `\u00f8` + U+0301), `\u01fd` (U+01FD), `\u01e3` (U+01E3) and `\u210f` (U+210F).
+ * The letters below have no decomposition of their own, so NFKD passes them through
+ * untouched and they still fold correctly in this position.
+ *
+ * The table covers every letter in Latin-1 Supplement (U+00C0-U+00FF) and Latin Extended-A
+ * (U+0100-U+017F) that NFKD leaves un-folded; `tokenize.test.ts` pins that range so a gap
+ * cannot silently reopen. Two entries look redundant and are not: `\u0110` (U+0110, D with
+ * stroke) and `\u00d0` (U+00D0, Eth) are visually indistinguishable in uppercase and are
+ * routinely substituted for one another in Croatian, Serbian and Vietnamese text, so both
+ * must fold to `D` or the same rendered name yields two disjoint token sets. Do not delete
+ * either as a duplicate of the other.
+ *
+ * Letters outside those two blocks are deliberately out of scope \u2014 `\u0259`/`\u018f` (U+0259/U+018F,
+ * common in Azerbaijani names such as `\u018fliyev`) fold to `e` under ICU and belong here on
+ * the same reasoning, but each addition forces operators through another `search_tokens`
+ * reindex, so extending the range is tracked separately rather than done piecemeal.
+ */
+const NON_DECOMPOSING_FOLDS: Record<string, string> = {
+  '\u0142': 'l',
+  '\u0141': 'L',
+  '\u00f8': 'o',
+  '\u00d8': 'O',
+  '\u0111': 'd',
+  '\u0110': 'D',
+  '\u00f0': 'd',
+  '\u00d0': 'D',
+  '\u0127': 'h',
+  '\u0126': 'H',
+  '\u0131': 'i',
+  '\u0138': 'k',
+  '\u014b': 'n',
+  '\u014a': 'N',
+  '\u0167': 't',
+  '\u0166': 'T',
+  '\u00e6': 'ae',
+  '\u00c6': 'AE',
+  '\u0153': 'oe',
+  '\u0152': 'OE',
+  '\u00fe': 'th',
+  '\u00de': 'TH',
+  '\u00df': 'ss',
+  '\u1e9e': 'SS',
+}
+
+const NON_DECOMPOSING_PATTERN = new RegExp(
+  `[${Object.keys(NON_DECOMPOSING_FOLDS)
+    .join('')
+    .replace(/[\\\]^-]/g, '\\$&')}]`,
+  'g',
+)
+
+function foldNonDecomposingLetters(text: string): string {
+  return text.replace(NON_DECOMPOSING_PATTERN, (char) => NON_DECOMPOSING_FOLDS[char])
+}
+
 function normalizeText(text: string): string {
-  return text
-    .normalize('NFKD')
-    .replace(/[\u0300-\u036f]/g, '')
+  return foldNonDecomposingLetters(text.normalize('NFKD').replace(/[\u0300-\u036f]/g, ''))
     .replace(/[%_]/g, ' ')
     .toLowerCase()
 }