@stll/text-normalize 0.2.0 → 0.4.0

package/README.md

@@ -61,8 +61,18 @@ generated with.
 
 ## Substring matching with highlight offsets
 
-`foldSearchMatchText` is the client-side match key for interactive
-filtering: NFKD, the `unaccent()`-parity ASCII folds, mark strip, Arabic
+`createSearchQueryMatcher(query)` (from `@stll/text-normalize/search-query-matcher`) is the client-side answer to "does this
+text match what the user typed": it trims and folds the query once and returns
+a predicate over candidate texts (a blank query matches everything, a missing
+candidate nothing). `filterBySearchQuery({ items, query, fields })` applies it
+to a list, keeping the items any field of which matches. Use these for every
+picker, list filter and typeahead, so a query typed without accents (`dvorak`,
+`strasse`, `lodz`) finds the same items everywhere and in every UI locale. An
+`Intl.Collator` with `sensitivity: "base"` is not an alternative: it compares
+equal-length windows (`strasse` misses `Straße`), its verdict depends on the
+runtime locale, and it keeps Arabic tashkeel significant.
+
+`foldSearchMatchText` is the match key both sides are folded with: NFKD, the `unaccent()`-parity ASCII folds, mark strip, Arabic
 folds, lowercase — `capek` matches `Čapek` and `wroclaw` matches
 `Wrocław`, both ways. `findSearchMatchRanges` finds a folded query in
 folded content and returns ranges into the **original** string, so
@@ -70,6 +80,35 @@ highlights wrap the text the user sees even where folding changes string
 length; it accepts content pre-folded with `foldSearchMatchTextWithOffsets`
 so repeated queries against the same text fold it once.
 
+Import the literal helpers from `@stll/text-normalize/search-match-key`:
+
+```ts
+import {
+  containsSearchMatchKey,
+  findSearchMatchKeyRanges,
+} from "@stll/text-normalize/search-match-key";
+```
+
+`containsSearchMatchKey(content, query)` checks a literal substring in keys
+already prepared by the caller, such as fixed tokens in a message or keys a
+caller folded itself. It does not trim or fold either input; a user-typed query
+goes through `createSearchQueryMatcher` instead. Empty queries match every key. For accent-insensitive highlight ranges into original
+text, use `findSearchMatchRanges` from `@stll/text-normalize` instead.
+
+`findSearchMatchKeyRanges(content, query, { start?, end?, maxMatches? })`
+returns non-overlapping literal matches as UTF-16 ranges into `content`.
+It applies no trimming, case conversion, or diacritic folding. Empty queries return no ranges; this differs from
+`containsSearchMatchKey`, where an empty query passes a list filter.
+
+`start` is inclusive and `end` exclusive; both default to the content's
+bounds, are truncated to integers, and clamped to `[0, content.length]`.
+Only complete matches inside that window are returned. `maxMatches` is
+floored; zero or negative limits, reversed windows, and NaN options return
+no ranges. Infinite bounds clamp to the content and an infinite limit
+returns all matches. Use `{ maxMatches: 1 }` for the first occurrence of a
+scope, then pass its `start` and `end` to search within that scope. Existing
+callers retain their own casing or folding before calling either helper.
+
 ## Spaced letters
 
 `collapseSpacedLetters` joins letter-spaced court headings
@@ -79,3 +118,9 @@ lists (`a b c`) are left intact. `spacedLetterRunRegex()` returns the
 underlying matcher for callers that need per-match offsets (find-in-page
 highlighting). Index-time and query-time collapse share this one
 threshold so a heading collapses identically on both sides.
+
+Both match-key helpers replace lone surrogates with U+FFFD on both sides via
+`toWellFormed()` before matching Unicode scalars. A lone surrogate can match
+U+FFFD or another lone surrogate, and cannot match half of an astral character.
+Offsets remain UTF-16 offsets into the well-formed content; replacement does not
+change its length. Caller-owned case, accent, and whitespace policies remain.

package/dist/ascii-fold.js

@@ -1,3 +1,4 @@
+// parser-output-unchanged: scripts/parser-output-diff.ts reports every case-law adapter identical over its recorded fixtures with this change; search folds keep non-Latin spacing marks while slug and judge-key folds stay pinned.
 /**
  * Latin-ASCII folding for search keys whose counterpart is a PostgreSQL
  * `unaccent()` index or a Tantivy `ascii_folding` index.
@@ -10,12 +11,13 @@
  * `unaccent` applies that decomposition cannot express.
  */
 import { ASCII_FOLD_TABLE } from "./ascii-fold-table.js";
-// The table is matched before `\p{Diacritic}` so modifier letters that Unicode
+import { SEARCH_DIACRITIC_SOURCE } from "./diacritics.js";
+// The table is matched before the diacritic strip so modifier letters that Unicode
 // classifies as diacritics (`ʰ`, `ᴬ`) fold to their base letter the way
 // `unaccent` folds them, instead of being deleted by the mark strip. Every key
 // is a single code point outside the character-class metacharacter set, so the
 // class needs no escaping.
-const ASCII_FOLD_RE = new RegExp(`[${Object.keys(ASCII_FOLD_TABLE).join("")}]|\\p{Diacritic}`, "gu");
+const ASCII_FOLD_RE = new RegExp(`[${Object.keys(ASCII_FOLD_TABLE).join("")}]|${SEARCH_DIACRITIC_SOURCE}`, "gu");
 /**
  * The replace step of `foldToAscii`, for callers that decompose themselves
  * (`foldSearchMatchText` decomposes with NFKD and per character). The input

package/dist/ascii-fold.js.map

@@ -1 +1 @@
-{"version":3,"file":"ascii-fold.js","sourceRoot":"","sources":["../src/ascii-fold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,+EAA+E;AAC/E,wEAAwE;AACxE,+EAA+E;AAC/E,+EAA+E;AAC/E,2BAA2B;AAC3B,MAAM,aAAa,GAAG,IAAI,MAAM,CAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,kBAAkB,EAC5D,IAAI,CACL,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,UAAkB,EAAU,EAAE,CAC5D,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAE5E;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAU,EAAE,CAClD,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC"}
+{"version":3,"file":"ascii-fold.js","sourceRoot":"","sources":["../src/ascii-fold.ts"],"names":[],"mappings":"AAAA,qOAAqO;AACrO;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAE1D,mFAAmF;AACnF,wEAAwE;AACxE,+EAA+E;AAC/E,+EAA+E;AAC/E,2BAA2B;AAC3B,MAAM,aAAa,GAAG,IAAI,MAAM,CAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,uBAAuB,EAAE,EACxE,IAAI,CACL,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,UAAkB,EAAU,EAAE,CAC5D,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAE5E;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAU,EAAE,CAClD,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC"}

package/dist/diacritics.d.ts

@@ -1,27 +1,43 @@
 /**
  * Latin/combining-mark diacritic stripping for search keys and slugs.
  *
- * Both variants decompose the input and then remove every character
- * Unicode classifies as a diacritic (`\p{Diacritic}`). `\p{Diacritic}`
- * is deliberate: it covers combining marks in every block, not only the
+ * Both variants decompose the input and then remove characters Unicode
+ * classifies as diacritics (`\p{Diacritic}`). `\p{Diacritic}` is
+ * deliberate: it covers combining marks in every block, not only the
  * Combining Diacritical Marks block (U+0300–U+036F). A range like
  * `[̀-ͯ]` misses marks in the Extended (U+1AB0–U+1AFF) and
  * Supplement (U+1DC0–U+1DFF) blocks, so two callers using different
  * classes would disagree on the same text.
  *
- * The variants differ only in the decomposition form:
+ * The variants differ in decomposition form and in what they remove:
  *
- * - `stripDiacritics` uses NFD (canonical decomposition). Use it for
- *   search keys, where compatibility characters must survive unchanged.
+ * - `stripDiacritics` uses NFD (canonical decomposition) and removes
+ *   `SEARCH_DIACRITIC_SOURCE`. Use it for search keys, where compatibility
+ *   characters and non-Latin spacing marks (`ー`, `·`) survive unchanged.
  * - `stripDiacriticsForSlug` uses NFKD (compatibility decomposition), so
  *   ligatures, full-width forms, superscripts, and similar fold to their
  *   ASCII base before the `[a-z0-9]` slug filter runs. Slugs are
  *   persisted, public URL segments, so this variant pins the exact form
- *   the existing slugs were generated with; do not switch it to NFD.
+ *   the existing slugs were generated with, spacing diacritics removed
+ *   too; do not switch it to NFD or narrow its class.
  */
+/**
+ * The diacritics a search fold removes: combining marks, plus spacing ones in
+ * Latin script (modifier letters such as `ʰ`).
+ *
+ * `\p{Diacritic}` alone also covers spacing characters in other scripts that
+ * are text, not marks on a letter: the Japanese prolonged sound mark
+ * (`コーヒー`), the middle dot (`l·l`), `^`, and `` ` ``. Removing those turns
+ * `コーヒー` into `コヒ`, a different word, and PostgreSQL `unaccent()` keeps
+ * them, so a query folded that way misses the indexed text. Latin script keeps
+ * the full class because `foldToAscii` is pinned to `unaccent()` parity there.
+ * Shared as a source string so `foldToAscii` strips the identical class.
+ */
+export declare const SEARCH_DIACRITIC_SOURCE = "(?=\\p{M}|\\p{Script=Latin})\\p{Diacritic}";
 /**
  * Strip combining diacritics from search text (NFD). Non-diacritic
- * characters, including compatibility characters, pass through unchanged.
+ * characters, including compatibility characters and spacing marks such as
+ * `ー`, pass through unchanged.
  */
 export declare const stripDiacritics: (text: string) => string;
 /**

package/dist/diacritics.js

@@ -1,34 +1,56 @@
+// parser-output-unchanged: scripts/parser-output-diff.ts reports every case-law adapter identical over its recorded fixtures with this change; search folds keep non-Latin spacing marks while slug and judge-key folds stay pinned.
 /**
  * Latin/combining-mark diacritic stripping for search keys and slugs.
  *
- * Both variants decompose the input and then remove every character
- * Unicode classifies as a diacritic (`\p{Diacritic}`). `\p{Diacritic}`
- * is deliberate: it covers combining marks in every block, not only the
+ * Both variants decompose the input and then remove characters Unicode
+ * classifies as diacritics (`\p{Diacritic}`). `\p{Diacritic}` is
+ * deliberate: it covers combining marks in every block, not only the
  * Combining Diacritical Marks block (U+0300–U+036F). A range like
  * `[̀-ͯ]` misses marks in the Extended (U+1AB0–U+1AFF) and
  * Supplement (U+1DC0–U+1DFF) blocks, so two callers using different
  * classes would disagree on the same text.
  *
- * The variants differ only in the decomposition form:
+ * The variants differ in decomposition form and in what they remove:
  *
- * - `stripDiacritics` uses NFD (canonical decomposition). Use it for
- *   search keys, where compatibility characters must survive unchanged.
+ * - `stripDiacritics` uses NFD (canonical decomposition) and removes
+ *   `SEARCH_DIACRITIC_SOURCE`. Use it for search keys, where compatibility
+ *   characters and non-Latin spacing marks (`ー`, `·`) survive unchanged.
  * - `stripDiacriticsForSlug` uses NFKD (compatibility decomposition), so
  *   ligatures, full-width forms, superscripts, and similar fold to their
  *   ASCII base before the `[a-z0-9]` slug filter runs. Slugs are
  *   persisted, public URL segments, so this variant pins the exact form
- *   the existing slugs were generated with; do not switch it to NFD.
+ *   the existing slugs were generated with, spacing diacritics removed
+ *   too; do not switch it to NFD or narrow its class.
  */
-const DIACRITIC_RE = /\p{Diacritic}/gu;
+/**
+ * The diacritics a search fold removes: combining marks, plus spacing ones in
+ * Latin script (modifier letters such as `ʰ`).
+ *
+ * `\p{Diacritic}` alone also covers spacing characters in other scripts that
+ * are text, not marks on a letter: the Japanese prolonged sound mark
+ * (`コーヒー`), the middle dot (`l·l`), `^`, and `` ` ``. Removing those turns
+ * `コーヒー` into `コヒ`, a different word, and PostgreSQL `unaccent()` keeps
+ * them, so a query folded that way misses the indexed text. Latin script keeps
+ * the full class because `foldToAscii` is pinned to `unaccent()` parity there.
+ * Shared as a source string so `foldToAscii` strips the identical class.
+ */
+export const SEARCH_DIACRITIC_SOURCE = "(?=\\p{M}|\\p{Script=Latin})\\p{Diacritic}";
+const SEARCH_DIACRITIC_RE = new RegExp(SEARCH_DIACRITIC_SOURCE, "gu");
+/**
+ * Pinned to the class the persisted slugs were minted with, spacing
+ * characters included; see `stripDiacriticsForSlug`.
+ */
+const SLUG_DIACRITIC_RE = /\p{Diacritic}/gu;
 /**
  * Strip combining diacritics from search text (NFD). Non-diacritic
- * characters, including compatibility characters, pass through unchanged.
+ * characters, including compatibility characters and spacing marks such as
+ * `ー`, pass through unchanged.
  */
-export const stripDiacritics = (text) => text.normalize("NFD").replace(DIACRITIC_RE, "");
+export const stripDiacritics = (text) => text.normalize("NFD").replace(SEARCH_DIACRITIC_RE, "");
 /**
  * Strip combining diacritics for slug generation (NFKD). Callers apply
  * their own case folding and `[a-z0-9]` filtering around this; the NFKD
  * form is load-bearing for byte-stable slugs and must not change.
  */
-export const stripDiacriticsForSlug = (text) => text.normalize("NFKD").replace(DIACRITIC_RE, "");
+export const stripDiacriticsForSlug = (text) => text.normalize("NFKD").replace(SLUG_DIACRITIC_RE, "");
 //# sourceMappingURL=diacritics.js.map

package/dist/diacritics.js.map

@@ -1 +1 @@
-{"version":3,"file":"diacritics.js","sourceRoot":"","sources":["../src/diacritics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,YAAY,GAAG,iBAAiB,CAAC;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CACtD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC7D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC"}
+{"version":3,"file":"diacritics.js","sourceRoot":"","sources":["../src/diacritics.ts"],"names":[],"mappings":"AAAA,qOAAqO;AACrO;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAClC,4CAA4C,CAAC;AAE/C,MAAM,mBAAmB,GAAG,IAAI,MAAM,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;AAEtE;;;GAGG;AACH,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CACtD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;AAEzD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC7D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC"}

package/dist/search-match-key.d.ts

@@ -0,0 +1,29 @@
+/** UTF-16 range into the well-formed content searched by literal key helpers. */
+type SearchMatchKeyRange = {
+    end: number;
+    start: number;
+};
+/**
+ * Literal substring search over caller-owned match keys. No trimming or
+ * folding is applied: callers whose policy differs from the highlighter's
+ * fold keep control over case, accents, whitespace, and field boundaries.
+ * Lone surrogates become U+FFFD in both inputs. Empty keys match every value.
+ */
+export declare const containsSearchMatchKey: (content: string, query: string) => boolean;
+type FindSearchMatchKeyRangesOptions = {
+    /** Inclusive UTF-16 offset; defaults to the start of the content. */
+    start?: number;
+    /** Exclusive UTF-16 offset; defaults to the end of the content. */
+    end?: number;
+    maxMatches?: number;
+};
+/**
+ * Non-overlapping literal occurrences in caller-owned keys, with UTF-16
+ * offsets into content after replacing lone surrogates with U+FFFD. Neither
+ * input is trimmed or folded. Well-formed literals match whole Unicode scalars.
+ * Only matches wholly inside [start, end) are returned. Bounds are truncated
+ * and clamped to content; maxMatches is floored. NaN options, empty queries,
+ * reversed windows, and nonpositive limits produce no matches.
+ */
+export declare const findSearchMatchKeyRanges: (content: string, query: string, options?: FindSearchMatchKeyRangesOptions) => SearchMatchKeyRange[];
+export {};

package/dist/search-match-key.js

@@ -0,0 +1,44 @@
+/**
+ * Literal substring search over caller-owned match keys. No trimming or
+ * folding is applied: callers whose policy differs from the highlighter's
+ * fold keep control over case, accents, whitespace, and field boundaries.
+ * Lone surrogates become U+FFFD in both inputs. Empty keys match every value.
+ */
+export const containsSearchMatchKey = (content, query) => content.toWellFormed().includes(query.toWellFormed());
+/**
+ * Non-overlapping literal occurrences in caller-owned keys, with UTF-16
+ * offsets into content after replacing lone surrogates with U+FFFD. Neither
+ * input is trimmed or folded. Well-formed literals match whole Unicode scalars.
+ * Only matches wholly inside [start, end) are returned. Bounds are truncated
+ * and clamped to content; maxMatches is floored. NaN options, empty queries,
+ * reversed windows, and nonpositive limits produce no matches.
+ */
+export const findSearchMatchKeyRanges = (content, query, options = {}) => {
+    const start = Math.min(content.length, Math.max(0, Math.trunc(options.start ?? 0)));
+    const end = Math.min(content.length, Math.max(0, Math.trunc(options.end ?? content.length)));
+    const maxMatches = Math.max(0, Math.floor(options.maxMatches ?? Number.MAX_SAFE_INTEGER));
+    if (query.length === 0 ||
+        !(start < end) ||
+        maxMatches === 0 ||
+        Number.isNaN(maxMatches)) {
+        return [];
+    }
+    const haystack = content.toWellFormed();
+    const needle = query.toWellFormed();
+    const matches = [];
+    let searchFrom = start;
+    while (searchFrom <= end - needle.length) {
+        const matchStart = haystack.indexOf(needle, searchFrom);
+        const matchEnd = matchStart + needle.length;
+        if (matchStart === -1 || matchEnd > end) {
+            break;
+        }
+        matches.push({ start: matchStart, end: matchEnd });
+        if (matches.length >= maxMatches) {
+            break;
+        }
+        searchFrom = matchEnd;
+    }
+    return matches;
+};
+//# sourceMappingURL=search-match-key.js.map

package/dist/search-match-key.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-match-key.js","sourceRoot":"","sources":["../src/search-match-key.ts"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,OAAe,EACf,KAAa,EACJ,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;AAUpE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACtC,OAAe,EACf,KAAa,EACb,OAAO,GAAoC,EAAE,EACtB,EAAE;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,OAAO,CAAC,MAAM,EACd,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAC5C,CAAC;IACF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,OAAO,CAAC,MAAM,EACd,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CACvD,CAAC;IACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAC1D,CAAC;IACF,IACE,KAAK,CAAC,MAAM,KAAK,CAAC;QAClB,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC;QACd,UAAU,KAAK,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EACxB,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,EAAE,CAAC;IACpC,MAAM,OAAO,GAA0B,EAAE,CAAC;IAC1C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,OAAO,UAAU,IAAI,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5C,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YACxC,MAAM;QACR,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QACD,UAAU,GAAG,QAAQ,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}

package/dist/search-query-matcher.d.ts

@@ -0,0 +1,38 @@
+/**
+ * The client-side match for user-typed queries, kept out of the package root
+ * so the corpus parsers that read the root do not depend on it.
+ */
+/** Whether one candidate text contains the query a matcher was built for. */
+export type SearchQueryMatcher = (candidate: string | null | undefined) => boolean;
+/**
+ * The client-side answer to "does this text match what the user typed".
+ *
+ * Both sides go through `foldSearchMatchText`, so case, accents, letters
+ * without a decomposition (`ł`, `ø`, `ß` → `ss`), compatibility ligatures and
+ * Arabic orthographic variants are ignored identically in every UI locale,
+ * and a filtered list agrees with the highlights `findSearchMatchRanges`
+ * draws in it. The query is trimmed and folded once and candidate keys are
+ * cached; a blank query matches every candidate, and a missing candidate
+ * matches none. Lone surrogates become U+FFFD on both sides, so a query can
+ * never match half of an astral character.
+ *
+ * Preferred over an `Intl.Collator` with `sensitivity: "base"`: a collator
+ * compares equal-length windows, so `strasse` never finds `Straße`; its
+ * verdict depends on the runtime locale (Turkish `ISTANBUL` misses
+ * `istanbul`, German `Müller` misses `muller`); it keeps Arabic tashkeel
+ * significant; and it compares every window of every item per keystroke,
+ * where a fold key is computed once per text.
+ */
+export declare const createSearchQueryMatcher: (query: string) => SearchQueryMatcher;
+type FilterBySearchQueryOptions<T> = {
+    /** The texts an item is found by; it matches when any of them does. */
+    fields: (item: T) => readonly (string | null | undefined)[];
+    items: readonly T[];
+    query: string;
+};
+/**
+ * The items any of whose fields matches the query, in their given order,
+ * through `createSearchQueryMatcher`. A blank query keeps every item.
+ */
+export declare const filterBySearchQuery: <T>({ fields, items, query, }: FilterBySearchQueryOptions<T>) => T[];
+export {};

package/dist/search-query-matcher.js

@@ -0,0 +1,61 @@
+/**
+ * The client-side match for user-typed queries, kept out of the package root
+ * so the corpus parsers that read the root do not depend on it.
+ */
+import { foldSearchMatchText } from "./search-match.js";
+// A list filter folds the same names on every keystroke. Remembering their
+// keys turns a keystroke over 2,000 three-field rows from about 3 ms into
+// about 0.2 ms, level with a bare `toLowerCase().includes()`. The fold is
+// pure, so the cache is unobservable; clearing it whole when full bounds its
+// memory without an eviction policy.
+const CANDIDATE_KEY_CACHE_LIMIT = 10_000;
+const candidateKeys = new Map();
+const candidateKey = (candidate) => {
+    const cached = candidateKeys.get(candidate);
+    if (cached !== undefined) {
+        return cached;
+    }
+    if (candidateKeys.size >= CANDIDATE_KEY_CACHE_LIMIT) {
+        candidateKeys.clear();
+    }
+    const key = foldSearchMatchText(candidate.toWellFormed());
+    candidateKeys.set(candidate, key);
+    return key;
+};
+/**
+ * The client-side answer to "does this text match what the user typed".
+ *
+ * Both sides go through `foldSearchMatchText`, so case, accents, letters
+ * without a decomposition (`ł`, `ø`, `ß` → `ss`), compatibility ligatures and
+ * Arabic orthographic variants are ignored identically in every UI locale,
+ * and a filtered list agrees with the highlights `findSearchMatchRanges`
+ * draws in it. The query is trimmed and folded once and candidate keys are
+ * cached; a blank query matches every candidate, and a missing candidate
+ * matches none. Lone surrogates become U+FFFD on both sides, so a query can
+ * never match half of an astral character.
+ *
+ * Preferred over an `Intl.Collator` with `sensitivity: "base"`: a collator
+ * compares equal-length windows, so `strasse` never finds `Straße`; its
+ * verdict depends on the runtime locale (Turkish `ISTANBUL` misses
+ * `istanbul`, German `Müller` misses `muller`); it keeps Arabic tashkeel
+ * significant; and it compares every window of every item per keystroke,
+ * where a fold key is computed once per text.
+ */
+export const createSearchQueryMatcher = (query) => {
+    const foldedQuery = foldSearchMatchText(query.trim().toWellFormed());
+    if (foldedQuery === "") {
+        return () => true;
+    }
+    return (candidate) => candidate !== null &&
+        candidate !== undefined &&
+        candidateKey(candidate).includes(foldedQuery);
+};
+/**
+ * The items any of whose fields matches the query, in their given order,
+ * through `createSearchQueryMatcher`. A blank query keeps every item.
+ */
+export const filterBySearchQuery = ({ fields, items, query, }) => {
+    const matches = createSearchQueryMatcher(query);
+    return items.filter((item) => fields(item).some(matches));
+};
+//# sourceMappingURL=search-query-matcher.js.map

package/dist/search-query-matcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-query-matcher.js","sourceRoot":"","sources":["../src/search-query-matcher.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExD,2EAA2E;AAC3E,0EAA0E;AAC1E,0EAA0E;AAC1E,6EAA6E;AAC7E,qCAAqC;AACrC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AACzC,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;AAEhD,MAAM,YAAY,GAAG,CAAC,SAAiB,EAAU,EAAE;IACjD,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,IAAI,aAAa,CAAC,IAAI,IAAI,yBAAyB,EAAE,CAAC;QACpD,aAAa,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;IACD,MAAM,GAAG,GAAG,mBAAmB,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC,CAAC;IAC1D,aAAa,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAOF;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,KAAa,EAAsB,EAAE;IAC5E,MAAM,WAAW,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;IACrE,IAAI,WAAW,KAAK,EAAE,EAAE,CAAC;QACvB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IACpB,CAAC;IACD,OAAO,CAAC,SAAS,EAAE,EAAE,CACnB,SAAS,KAAK,IAAI;QAClB,SAAS,KAAK,SAAS;QACvB,YAAY,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AAClD,CAAC,CAAC;AASF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAI,EACrC,MAAM,EACN,KAAK,EACL,KAAK,GACyB,EAAO,EAAE;IACvC,MAAM,OAAO,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AAC5D,CAAC,CAAC"}

package/dist/slug.d.ts

@@ -7,10 +7,11 @@
  * characters into one separator, clip, trim a trailing separator, fall back
  * when nothing is left — was identical in all four and is fixed here.
  *
- * Deliberately does NOT transliterate. `slugify("café")` is `"caf"` under the
- * `ascii` charset, not `"cafe"`: slugs are persisted, and folding here would
- * quietly start minting different keys for the same labels. The folding
- * primitives in this package are for search text, where nothing is stored.
+ * The `ascii` charset transliterates Latin letters before filtering:
+ * `slugify("Łódź")` is `"lodz"`, not `"odz"`. A key minted before
+ * transliteration is not recomputed by its owner; a caller that re-derives a
+ * stored key at lookup (the case-law routes) recognises the earlier fold
+ * itself.
  *
  * Single-pass by construction: a regex pipeline over untrusted label text is
  * what the slow-regex guard exists to prevent.
@@ -19,7 +20,7 @@
  * Which characters survive.
  *
  * - `ascii` keeps `[a-z0-9]`, for keys that end up in URLs and columns
- *   validated against that charset.
+ *   validated against that charset. Latin letters are transliterated first.
  * - `unicode` keeps any Unicode letter or number, for paths shown back to the
  *   person who typed them.
  */

package/dist/slug.js

@@ -1,3 +1,4 @@
+// parser-output-unchanged: no case-law or legislation adapter or parser calls slugify; adapters reach this module only through the package index.
 /**
  * One slug generator for the key-shaped strings the apps derive from a label.
  *
@@ -7,15 +8,28 @@
  * characters into one separator, clip, trim a trailing separator, fall back
  * when nothing is left — was identical in all four and is fixed here.
  *
- * Deliberately does NOT transliterate. `slugify("café")` is `"caf"` under the
- * `ascii` charset, not `"cafe"`: slugs are persisted, and folding here would
- * quietly start minting different keys for the same labels. The folding
- * primitives in this package are for search text, where nothing is stored.
+ * The `ascii` charset transliterates Latin letters before filtering:
+ * `slugify("Łódź")` is `"lodz"`, not `"odz"`. A key minted before
+ * transliteration is not recomputed by its owner; a caller that re-derives a
+ * stored key at lookup (the case-law routes) recognises the earlier fold
+ * itself.
  *
  * Single-pass by construction: a regex pipeline over untrusted label text is
  * what the slow-regex guard exists to prevent.
  */
+import { applyAsciiFolds } from "./ascii-fold.js";
 const UNICODE_SLUG_CHARACTER = /[\p{L}\p{N}]/u;
+/** Compatibility decomposition, the search fold's mark strip and the table. */
+const spellInAscii = (text) => applyAsciiFolds(text.normalize("NFKD"));
+/**
+ * Latin letters spelled in ASCII: compatibility decomposition (`fi`, `Ａ`), the
+ * search fold's mark strip (`č`), and the `unaccent()` table for letters no
+ * decomposition reaches (`ł`, `ø`, `ß`, `þ`, `ı`). The table is not closed
+ * under case (`Ɩ` folds and its small letter `ɩ` does not; `ɡ` folds and its
+ * capital `Ɡ` does not), so it applies on both sides of the lowercase. Other
+ * scripts pass through and are filtered out.
+ */
+const transliterateToAscii = (value) => spellInAscii(spellInAscii(value).toLowerCase()).toLowerCase();
 const isSlugCharacter = (character, charset) => {
     if (charset === "unicode") {
         return UNICODE_SLUG_CHARACTER.test(character);
@@ -24,9 +38,10 @@ const isSlugCharacter = (character, charset) => {
         (character >= "0" && character <= "9"));
 };
 export const slugify = (value, { charset, separator, maxLength, fallback }) => {
+    const source = charset === "ascii" ? transliterateToAscii(value) : value.toLowerCase();
     let buffer = "";
     let lastWasSeparator = true;
-    for (const character of value.toLowerCase()) {
+    for (const character of source) {
         if (isSlugCharacter(character, charset)) {
             buffer += character;
             lastWasSeparator = false;

package/dist/slug.js.map

@@ -1 +1 @@
-{"version":3,"file":"slug.js","sourceRoot":"","sources":["../src/slug.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAgCH,MAAM,sBAAsB,GAAG,eAAe,CAAC;AAE/C,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,OAAoB,EAAW,EAAE;IAC3E,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,CACL,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC;QACtC,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC,CACvC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,KAAa,EACb,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAkB,EACnD,EAAE;IACV,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAE5B,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5C,IAAI,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,KAAK,CAAC;YACzB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAClD,MAAM;QACR,CAAC;QACD,OAAO,IAAI,SAAS,CAAC;IACvB,CAAC;IACD,0EAA0E;IAC1E,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,OAAO,IAAI,QAAQ,CAAC;AAC7B,CAAC,CAAC"}
+{"version":3,"file":"slug.js","sourceRoot":"","sources":["../src/slug.ts"],"names":[],"mappings":"AAAA,kJAAkJ;AAClJ;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAgClD,MAAM,sBAAsB,GAAG,eAAe,CAAC;AAE/C,+EAA+E;AAC/E,MAAM,YAAY,GAAG,CAAC,IAAY,EAAU,EAAE,CAC5C,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;AAE1C;;;;;;;GAOG;AACH,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAU,EAAE,CACrD,YAAY,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAEhE,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,OAAoB,EAAW,EAAE;IAC3E,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,CACL,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC;QACtC,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC,CACvC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,KAAa,EACb,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAkB,EACnD,EAAE;IACV,MAAM,MAAM,GACV,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IAC1E,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAE5B,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,IAAI,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,KAAK,CAAC;YACzB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAClD,MAAM;QACR,CAAC;QACD,OAAO,IAAI,SAAS,CAAC;IACvB,CAAC;IACD,0EAA0E;IAC1E,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,OAAO,IAAI,QAAQ,CAAC;AAC7B,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stll/text-normalize",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Search match-key text normalization — Arabic orthographic variant folding (alef/hamza/teh-marbuta/yeh, tashkeel, tatweel, digits) with per-language fold tables",
   "keywords": [
     "arabic",
@@ -32,6 +32,14 @@
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
+    },
+    "./search-match-key": {
+      "types": "./dist/search-match-key.d.ts",
+      "import": "./dist/search-match-key.js"
+    },
+    "./search-query-matcher": {
+      "types": "./dist/search-query-matcher.d.ts",
+      "import": "./dist/search-query-matcher.js"
     }
   },
   "publishConfig": {
@@ -46,13 +54,12 @@
     "typecheck": "bun check --no-pretty --all --project=tsconfig.json",
     "lint": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --report-unused-disable-directives-severity=error --type-aware --type-check packages/text-normalize",
     "lint:fix": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --type-aware --fix packages/text-normalize",
-    "format": "bun ../../scripts/run-oxfmt.ts .",
     "prepack": "bun run build"
   },
   "devDependencies": {
     "@stll/property-testing": "0.0.0",
     "@stll/typescript-config": "0.0.0",
-    "bun-types": "1.4.3",
+    "bun-types": "1.4.4",
     "fast-check": "^4.10.2"
   }
 }