@lde/text-normalization 0.0.0

package/README.md

@@ -0,0 +1,39 @@
+# @lde/text-normalization
+
+Zero-dependency text folding for search index and query normalization.
+
+`fold()` produces a diacritic- and case-insensitive form of a string, applied
+**identically at index time and query time** so that a search index never
+diverges from the queries run against it (divergence = silent search misses).
+
+```ts
+import { fold } from '@lde/text-normalization';
+
+fold('Møhlmann'); // 'mohlmann'
+fold('Coöperatieve'); // 'cooperatieve'
+fold('Straße'); // 'strasse'
+```
+
+It combines Unicode NFKD decomposition + combining-mark stripping (which folds
+é, ö, å, ç, …) with an explicit transliteration map for letters that do **not**
+decompose under NFKD (ø, æ, œ, ß, ð, þ, ł, đ, …).
+
+## When it’s needed
+
+A search engine on its default locale often folds case and diacritics for you –
+Typesense v30 (verified) even folds the non-decomposing `ø`/`æ`/`ß` – so on the
+default locale `fold()` is redundant for _search_. It becomes necessary when:
+
+- **Sorting** – engines sort strings by raw code-point order with no collation,
+  so a `fold()`-ed companion field is the only way to sort case- and
+  diacritic-insensitively.
+- **Stemming** – enabling a language’s stemmer requires a non-default
+  `locale`, which switches the tokenizer (Typesense → ICU) to one that
+  _preserves_ diacritics; the default folding is lost, and `fold()` restores
+  diacritic-insensitive matching.
+
+`fold()` is idempotent (`fold(fold(x)) === fold(x)`). Punctuation and word
+boundaries are preserved; tokenization is left to the search engine.
+
+Because folded values are stored in the search index, the same `fold()` must be
+used at index time and query time, and any change to it requires a full rebuild.

package/dist/fold.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Fold a string into a diacritic- and case-insensitive normalized form, applied
+ * IDENTICALLY at index time and query time (divergence = silent search misses).
+ *
+ * Steps: lowercase → transliterate non-decomposing letters → NFKD → strip
+ * combining marks. Idempotent: `fold(fold(x)) === fold(x)`. Punctuation and
+ * word boundaries are preserved; tokenization is left to the search engine.
+ */
+export declare function fold(input: string): string;
+//# sourceMappingURL=fold.d.ts.map

package/dist/fold.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fold.d.ts","sourceRoot":"","sources":["../src/fold.ts"],"names":[],"mappings":"AAiCA;;;;;;;GAOG;AACH,wBAAgB,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAM1C"}

package/dist/fold.js

@@ -0,0 +1,42 @@
+/**
+ * Transliteration map for letters that do NOT decompose under Unicode NFKD
+ * normalization, so a plain combining-mark strip cannot fold them. Keys are
+ * lowercase; uppercase variants are handled by lowercasing before lookup.
+ *
+ * The flagship case is ø (#1661: “Møhlmann” must be found by “Mohlmann”).
+ * Decomposing letters (é, ö, å, ç, …) are intentionally NOT listed here —
+ * NFKD + the combining-mark strip already fold them.
+ */
+const TRANSLITERATION_MAP = {
+    ø: 'o',
+    æ: 'ae',
+    œ: 'oe',
+    ß: 'ss',
+    ð: 'd',
+    þ: 'th',
+    ł: 'l',
+    đ: 'd',
+    ħ: 'h',
+    ŋ: 'ng',
+    ı: 'i',
+    ĸ: 'k',
+};
+const TRANSLITERATION_PATTERN = new RegExp(`[${Object.keys(TRANSLITERATION_MAP).join('')}]`, 'g');
+// Strip all Unicode nonspacing combining marks left behind by NFKD
+// decomposition (e.g. the diaeresis in ö, the acute in é).
+const COMBINING_MARKS_PATTERN = /\p{Mn}/gu;
+/**
+ * Fold a string into a diacritic- and case-insensitive normalized form, applied
+ * IDENTICALLY at index time and query time (divergence = silent search misses).
+ *
+ * Steps: lowercase → transliterate non-decomposing letters → NFKD → strip
+ * combining marks. Idempotent: `fold(fold(x)) === fold(x)`. Punctuation and
+ * word boundaries are preserved; tokenization is left to the search engine.
+ */
+export function fold(input) {
+    return input
+        .toLowerCase()
+        .replace(TRANSLITERATION_PATTERN, (letter) => TRANSLITERATION_MAP[letter])
+        .normalize('NFKD')
+        .replace(COMBINING_MARKS_PATTERN, '');
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { fold } from './fold.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC"}

package/dist/index.js

@@ -0,0 +1 @@
+export { fold } from './fold.js';

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@lde/text-normalization",
+  "version": "0.0.0",
+  "description": "Zero-dependency text folding (diacritic stripping and transliteration) for search index and query normalization",
+  "repository": {
+    "url": "git+https://github.com/ldelements/lde.git",
+    "directory": "packages/text-normalization"
+  },
+  "license": "MIT",
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "development": "./src/index.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo"
+  ],
+  "dependencies": {
+    "tslib": "^2.3.0"
+  }
+}