@stll/text-normalize 0.3.0 → 0.4.0

package/README.md

@@ -61,8 +61,18 @@ generated with.
 
 ## Substring matching with highlight offsets
 
-`foldSearchMatchText` is the client-side match key for interactive
-filtering: NFKD, the `unaccent()`-parity ASCII folds, mark strip, Arabic
+`createSearchQueryMatcher(query)` (from `@stll/text-normalize/search-query-matcher`) is the client-side answer to "does this
+text match what the user typed": it trims and folds the query once and returns
+a predicate over candidate texts (a blank query matches everything, a missing
+candidate nothing). `filterBySearchQuery({ items, query, fields })` applies it
+to a list, keeping the items any field of which matches. Use these for every
+picker, list filter and typeahead, so a query typed without accents (`dvorak`,
+`strasse`, `lodz`) finds the same items everywhere and in every UI locale. An
+`Intl.Collator` with `sensitivity: "base"` is not an alternative: it compares
+equal-length windows (`strasse` misses `Straße`), its verdict depends on the
+runtime locale, and it keeps Arabic tashkeel significant.
+
+`foldSearchMatchText` is the match key both sides are folded with: NFKD, the `unaccent()`-parity ASCII folds, mark strip, Arabic
 folds, lowercase — `capek` matches `Čapek` and `wroclaw` matches
 `Wrocław`, both ways. `findSearchMatchRanges` finds a folded query in
 folded content and returns ranges into the **original** string, so
@@ -80,9 +90,9 @@ import {
 ```
 
 `containsSearchMatchKey(content, query)` checks a literal substring in keys
-already prepared by the caller. It does not trim or fold either input, so a
-picker can retain locale-specific casing or accent-sensitive matching. Empty
-queries match every key. For accent-insensitive highlight ranges into original
+already prepared by the caller, such as fixed tokens in a message or keys a
+caller folded itself. It does not trim or fold either input; a user-typed query
+goes through `createSearchQueryMatcher` instead. Empty queries match every key. For accent-insensitive highlight ranges into original
 text, use `findSearchMatchRanges` from `@stll/text-normalize` instead.
 
 `findSearchMatchKeyRanges(content, query, { start?, end?, maxMatches? })`

package/dist/search-query-matcher.d.ts

@@ -0,0 +1,38 @@
+/**
+ * The client-side match for user-typed queries, kept out of the package root
+ * so the corpus parsers that read the root do not depend on it.
+ */
+/** Whether one candidate text contains the query a matcher was built for. */
+export type SearchQueryMatcher = (candidate: string | null | undefined) => boolean;
+/**
+ * The client-side answer to "does this text match what the user typed".
+ *
+ * Both sides go through `foldSearchMatchText`, so case, accents, letters
+ * without a decomposition (`ł`, `ø`, `ß` → `ss`), compatibility ligatures and
+ * Arabic orthographic variants are ignored identically in every UI locale,
+ * and a filtered list agrees with the highlights `findSearchMatchRanges`
+ * draws in it. The query is trimmed and folded once and candidate keys are
+ * cached; a blank query matches every candidate, and a missing candidate
+ * matches none. Lone surrogates become U+FFFD on both sides, so a query can
+ * never match half of an astral character.
+ *
+ * Preferred over an `Intl.Collator` with `sensitivity: "base"`: a collator
+ * compares equal-length windows, so `strasse` never finds `Straße`; its
+ * verdict depends on the runtime locale (Turkish `ISTANBUL` misses
+ * `istanbul`, German `Müller` misses `muller`); it keeps Arabic tashkeel
+ * significant; and it compares every window of every item per keystroke,
+ * where a fold key is computed once per text.
+ */
+export declare const createSearchQueryMatcher: (query: string) => SearchQueryMatcher;
+type FilterBySearchQueryOptions<T> = {
+    /** The texts an item is found by; it matches when any of them does. */
+    fields: (item: T) => readonly (string | null | undefined)[];
+    items: readonly T[];
+    query: string;
+};
+/**
+ * The items any of whose fields matches the query, in their given order,
+ * through `createSearchQueryMatcher`. A blank query keeps every item.
+ */
+export declare const filterBySearchQuery: <T>({ fields, items, query, }: FilterBySearchQueryOptions<T>) => T[];
+export {};

package/dist/search-query-matcher.js

@@ -0,0 +1,61 @@
+/**
+ * The client-side match for user-typed queries, kept out of the package root
+ * so the corpus parsers that read the root do not depend on it.
+ */
+import { foldSearchMatchText } from "./search-match.js";
+// A list filter folds the same names on every keystroke. Remembering their
+// keys turns a keystroke over 2,000 three-field rows from about 3 ms into
+// about 0.2 ms, level with a bare `toLowerCase().includes()`. The fold is
+// pure, so the cache is unobservable; clearing it whole when full bounds its
+// memory without an eviction policy.
+const CANDIDATE_KEY_CACHE_LIMIT = 10_000;
+const candidateKeys = new Map();
+const candidateKey = (candidate) => {
+    const cached = candidateKeys.get(candidate);
+    if (cached !== undefined) {
+        return cached;
+    }
+    if (candidateKeys.size >= CANDIDATE_KEY_CACHE_LIMIT) {
+        candidateKeys.clear();
+    }
+    const key = foldSearchMatchText(candidate.toWellFormed());
+    candidateKeys.set(candidate, key);
+    return key;
+};
+/**
+ * The client-side answer to "does this text match what the user typed".
+ *
+ * Both sides go through `foldSearchMatchText`, so case, accents, letters
+ * without a decomposition (`ł`, `ø`, `ß` → `ss`), compatibility ligatures and
+ * Arabic orthographic variants are ignored identically in every UI locale,
+ * and a filtered list agrees with the highlights `findSearchMatchRanges`
+ * draws in it. The query is trimmed and folded once and candidate keys are
+ * cached; a blank query matches every candidate, and a missing candidate
+ * matches none. Lone surrogates become U+FFFD on both sides, so a query can
+ * never match half of an astral character.
+ *
+ * Preferred over an `Intl.Collator` with `sensitivity: "base"`: a collator
+ * compares equal-length windows, so `strasse` never finds `Straße`; its
+ * verdict depends on the runtime locale (Turkish `ISTANBUL` misses
+ * `istanbul`, German `Müller` misses `muller`); it keeps Arabic tashkeel
+ * significant; and it compares every window of every item per keystroke,
+ * where a fold key is computed once per text.
+ */
+export const createSearchQueryMatcher = (query) => {
+    const foldedQuery = foldSearchMatchText(query.trim().toWellFormed());
+    if (foldedQuery === "") {
+        return () => true;
+    }
+    return (candidate) => candidate !== null &&
+        candidate !== undefined &&
+        candidateKey(candidate).includes(foldedQuery);
+};
+/**
+ * The items any of whose fields matches the query, in their given order,
+ * through `createSearchQueryMatcher`. A blank query keeps every item.
+ */
+export const filterBySearchQuery = ({ fields, items, query, }) => {
+    const matches = createSearchQueryMatcher(query);
+    return items.filter((item) => fields(item).some(matches));
+};
+//# sourceMappingURL=search-query-matcher.js.map

package/dist/search-query-matcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-query-matcher.js","sourceRoot":"","sources":["../src/search-query-matcher.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExD,2EAA2E;AAC3E,0EAA0E;AAC1E,0EAA0E;AAC1E,6EAA6E;AAC7E,qCAAqC;AACrC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AACzC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;AAEhD,MAAM,YAAY,GAAG,CAAC,SAAiB,EAAU,EAAE;IACjD,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,IAAI,yBAAyB,EAAE,CAAC;QACpD,aAAa,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IACD,MAAM,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;IAC1D,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAOF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,KAAa,EAAsB,EAAE;IAC5E,MAAM,WAAW,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;IACrE,IAAI,WAAW,KAAK,EAAE,EAAE,CAAC;QACvB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IACpB,CAAC;IACD,OAAO,CAAC,SAAS,EAAE,EAAE,CACnB,SAAS,KAAK,IAAI;QAClB,SAAS,KAAK,SAAS;QACvB,YAAY,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AAClD,CAAC,CAAC;AASF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAI,EACrC,MAAM,EACN,KAAK,EACL,KAAK,GACyB,EAAO,EAAE;IACvC,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5D,CAAC,CAAC"}

package/dist/slug.d.ts

@@ -7,10 +7,11 @@
  * characters into one separator, clip, trim a trailing separator, fall back
  * when nothing is left — was identical in all four and is fixed here.
  *
- * Deliberately does NOT transliterate. `slugify("café")` is `"caf"` under the
- * `ascii` charset, not `"cafe"`: slugs are persisted, and folding here would
- * quietly start minting different keys for the same labels. The folding
- * primitives in this package are for search text, where nothing is stored.
+ * The `ascii` charset transliterates Latin letters before filtering:
+ * `slugify("Łódź")` is `"lodz"`, not `"odz"`. A key minted before
+ * transliteration is not recomputed by its owner; a caller that re-derives a
+ * stored key at lookup (the case-law routes) recognises the earlier fold
+ * itself.
  *
  * Single-pass by construction: a regex pipeline over untrusted label text is
  * what the slow-regex guard exists to prevent.
@@ -19,7 +20,7 @@
  * Which characters survive.
  *
  * - `ascii` keeps `[a-z0-9]`, for keys that end up in URLs and columns
- *   validated against that charset.
+ *   validated against that charset. Latin letters are transliterated first.
  * - `unicode` keeps any Unicode letter or number, for paths shown back to the
  *   person who typed them.
  */

package/dist/slug.js

@@ -1,3 +1,4 @@
+// parser-output-unchanged: no case-law or legislation adapter or parser calls slugify; adapters reach this module only through the package index.
 /**
  * One slug generator for the key-shaped strings the apps derive from a label.
  *
@@ -7,15 +8,28 @@
  * characters into one separator, clip, trim a trailing separator, fall back
  * when nothing is left — was identical in all four and is fixed here.
  *
- * Deliberately does NOT transliterate. `slugify("café")` is `"caf"` under the
- * `ascii` charset, not `"cafe"`: slugs are persisted, and folding here would
- * quietly start minting different keys for the same labels. The folding
- * primitives in this package are for search text, where nothing is stored.
+ * The `ascii` charset transliterates Latin letters before filtering:
+ * `slugify("Łódź")` is `"lodz"`, not `"odz"`. A key minted before
+ * transliteration is not recomputed by its owner; a caller that re-derives a
+ * stored key at lookup (the case-law routes) recognises the earlier fold
+ * itself.
  *
  * Single-pass by construction: a regex pipeline over untrusted label text is
  * what the slow-regex guard exists to prevent.
  */
+import { applyAsciiFolds } from "./ascii-fold.js";
 const UNICODE_SLUG_CHARACTER = /[\p{L}\p{N}]/u;
+/** Compatibility decomposition, the search fold's mark strip and the table. */
+const spellInAscii = (text) => applyAsciiFolds(text.normalize("NFKD"));
+/**
+ * Latin letters spelled in ASCII: compatibility decomposition (`fi`, `Ａ`), the
+ * search fold's mark strip (`č`), and the `unaccent()` table for letters no
+ * decomposition reaches (`ł`, `ø`, `ß`, `þ`, `ı`). The table is not closed
+ * under case (`Ɩ` folds and its small letter `ɩ` does not; `ɡ` folds and its
+ * capital `Ɡ` does not), so it applies on both sides of the lowercase. Other
+ * scripts pass through and are filtered out.
+ */
+const transliterateToAscii = (value) => spellInAscii(spellInAscii(value).toLowerCase()).toLowerCase();
 const isSlugCharacter = (character, charset) => {
     if (charset === "unicode") {
         return UNICODE_SLUG_CHARACTER.test(character);
@@ -24,9 +38,10 @@ const isSlugCharacter = (character, charset) => {
         (character >= "0" && character <= "9"));
 };
 export const slugify = (value, { charset, separator, maxLength, fallback }) => {
+    const source = charset === "ascii" ? transliterateToAscii(value) : value.toLowerCase();
     let buffer = "";
     let lastWasSeparator = true;
-    for (const character of value.toLowerCase()) {
+    for (const character of source) {
         if (isSlugCharacter(character, charset)) {
             buffer += character;
             lastWasSeparator = false;

package/dist/slug.js.map

@@ -1 +1 @@
-{"version":3,"file":"slug.js","sourceRoot":"","sources":["../src/slug.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAgCH,MAAM,sBAAsB,GAAG,eAAe,CAAC;AAE/C,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,OAAoB,EAAW,EAAE;IAC3E,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,CACL,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC;QACtC,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC,CACvC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,KAAa,EACb,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAkB,EACnD,EAAE;IACV,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAE5B,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5C,IAAI,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,KAAK,CAAC;YACzB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAClD,MAAM;QACR,CAAC;QACD,OAAO,IAAI,SAAS,CAAC;IACvB,CAAC;IACD,0EAA0E;IAC1E,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,OAAO,IAAI,QAAQ,CAAC;AAC7B,CAAC,CAAC"}
+{"version":3,"file":"slug.js","sourceRoot":"","sources":["../src/slug.ts"],"names":[],"mappings":"AAAA,kJAAkJ;AAClJ;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAgClD,MAAM,sBAAsB,GAAG,eAAe,CAAC;AAE/C,+EAA+E;AAC/E,MAAM,YAAY,GAAG,CAAC,IAAY,EAAU,EAAE,CAC5C,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;AAE1C;;;;;;;GAOG;AACH,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAU,EAAE,CACrD,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAEhE,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,OAAoB,EAAW,EAAE;IAC3E,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,CACL,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC;QACtC,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC,CACvC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,KAAa,EACb,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAkB,EACnD,EAAE;IACV,MAAM,MAAM,GACV,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IAC1E,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAE5B,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,IAAI,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,KAAK,CAAC;YACzB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAClD,MAAM;QACR,CAAC;QACD,OAAO,IAAI,SAAS,CAAC;IACvB,CAAC;IACD,0EAA0E;IAC1E,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,OAAO,IAAI,QAAQ,CAAC;AAC7B,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stll/text-normalize",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Search match-key text normalization — Arabic orthographic variant folding (alef/hamza/teh-marbuta/yeh, tashkeel, tatweel, digits) with per-language fold tables",
   "keywords": [
     "arabic",
@@ -36,6 +36,10 @@
     "./search-match-key": {
       "types": "./dist/search-match-key.d.ts",
       "import": "./dist/search-match-key.js"
+    },
+    "./search-query-matcher": {
+      "types": "./dist/search-query-matcher.d.ts",
+      "import": "./dist/search-query-matcher.js"
     }
   },
   "publishConfig": {
@@ -50,13 +54,12 @@
     "typecheck": "bun check --no-pretty --all --project=tsconfig.json",
     "lint": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --report-unused-disable-directives-severity=error --type-aware --type-check packages/text-normalize",
     "lint:fix": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --type-aware --fix packages/text-normalize",
-    "format": "bun ../../scripts/run-oxfmt.ts .",
     "prepack": "bun run build"
   },
   "devDependencies": {
     "@stll/property-testing": "0.0.0",
     "@stll/typescript-config": "0.0.0",
-    "bun-types": "1.4.3",
+    "bun-types": "1.4.4",
     "fast-check": "^4.10.2"
   }
 }