@stll/text-normalize 0.2.0 → 0.3.0

package/README.md

@@ -70,6 +70,35 @@ highlights wrap the text the user sees even where folding changes string
 length; it accepts content pre-folded with `foldSearchMatchTextWithOffsets`
 so repeated queries against the same text fold it once.
 
+Import the literal helpers from `@stll/text-normalize/search-match-key`:
+
+```ts
+import {
+  containsSearchMatchKey,
+  findSearchMatchKeyRanges,
+} from "@stll/text-normalize/search-match-key";
+```
+
+`containsSearchMatchKey(content, query)` checks a literal substring in keys
+already prepared by the caller. It does not trim or fold either input, so a
+picker can retain locale-specific casing or accent-sensitive matching. Empty
+queries match every key. For accent-insensitive highlight ranges into original
+text, use `findSearchMatchRanges` from `@stll/text-normalize` instead.
+
+`findSearchMatchKeyRanges(content, query, { start?, end?, maxMatches? })`
+returns non-overlapping literal matches as UTF-16 ranges into `content`.
+It applies no trimming, case conversion, or diacritic folding. Empty queries return no ranges; this differs from
+`containsSearchMatchKey`, where an empty query passes a list filter.
+
+`start` is inclusive and `end` exclusive; both default to the content's
+bounds, are truncated to integers, and clamped to `[0, content.length]`.
+Only complete matches inside that window are returned. `maxMatches` is
+floored; zero or negative limits, reversed windows, and NaN options return
+no ranges. Infinite bounds clamp to the content and an infinite limit
+returns all matches. Use `{ maxMatches: 1 }` for the first occurrence of a
+scope, then pass its `start` and `end` to search within that scope. Existing
+callers retain their own casing or folding before calling either helper.
+
 ## Spaced letters
 
 `collapseSpacedLetters` joins letter-spaced court headings
@@ -79,3 +108,9 @@ lists (`a b c`) are left intact. `spacedLetterRunRegex()` returns the
 underlying matcher for callers that need per-match offsets (find-in-page
 highlighting). Index-time and query-time collapse share this one
 threshold so a heading collapses identically on both sides.
+
+Both match-key helpers replace lone surrogates with U+FFFD on both sides via
+`toWellFormed()` before matching Unicode scalars. A lone surrogate can match
+U+FFFD or another lone surrogate, and cannot match half of an astral character.
+Offsets remain UTF-16 offsets into the well-formed content; replacement does not
+change its length. Caller-owned case, accent, and whitespace policies remain.

package/dist/ascii-fold.js

@@ -1,3 +1,4 @@
+// parser-output-unchanged: scripts/parser-output-diff.ts reports every case-law adapter identical over its recorded fixtures with this change; search folds keep non-Latin spacing marks while slug and judge-key folds stay pinned.
 /**
  * Latin-ASCII folding for search keys whose counterpart is a PostgreSQL
  * `unaccent()` index or a Tantivy `ascii_folding` index.
@@ -10,12 +11,13 @@
  * `unaccent` applies that decomposition cannot express.
  */
 import { ASCII_FOLD_TABLE } from "./ascii-fold-table.js";
-// The table is matched before `\p{Diacritic}` so modifier letters that Unicode
+import { SEARCH_DIACRITIC_SOURCE } from "./diacritics.js";
+// The table is matched before the diacritic strip so modifier letters that Unicode
 // classifies as diacritics (`ʰ`, `ᴬ`) fold to their base letter the way
 // `unaccent` folds them, instead of being deleted by the mark strip. Every key
 // is a single code point outside the character-class metacharacter set, so the
 // class needs no escaping.
-const ASCII_FOLD_RE = new RegExp(`[${Object.keys(ASCII_FOLD_TABLE).join("")}]|\\p{Diacritic}`, "gu");
+const ASCII_FOLD_RE = new RegExp(`[${Object.keys(ASCII_FOLD_TABLE).join("")}]|${SEARCH_DIACRITIC_SOURCE}`, "gu");
 /**
  * The replace step of `foldToAscii`, for callers that decompose themselves
  * (`foldSearchMatchText` decomposes with NFKD and per character). The input

package/dist/ascii-fold.js.map

@@ -1 +1 @@
-{"version":3,"file":"ascii-fold.js","sourceRoot":"","sources":["../src/ascii-fold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAEzD,+EAA+E;AAC/E,wEAAwE;AACxE,+EAA+E;AAC/E,+EAA+E;AAC/E,2BAA2B;AAC3B,MAAM,aAAa,GAAG,IAAI,MAAM,CAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,kBAAkB,EAC5D,IAAI,CACL,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,UAAkB,EAAU,EAAE,CAC5D,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAE5E;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAU,EAAE,CAClD,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC"}
+{"version":3,"file":"ascii-fold.js","sourceRoot":"","sources":["../src/ascii-fold.ts"],"names":[],"mappings":"AAAA,qOAAqO;AACrO;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iBAAiB,CAAC;AAE1D,mFAAmF;AACnF,wEAAwE;AACxE,+EAA+E;AAC/E,+EAA+E;AAC/E,2BAA2B;AAC3B,MAAM,aAAa,GAAG,IAAI,MAAM,CAC9B,IAAI,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,uBAAuB,EAAE,EACxE,IAAI,CACL,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,UAAkB,EAAU,EAAE,CAC5D,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAE5E;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAU,EAAE,CAClD,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC"}

package/dist/diacritics.d.ts

@@ -1,27 +1,43 @@
 /**
  * Latin/combining-mark diacritic stripping for search keys and slugs.
  *
- * Both variants decompose the input and then remove every character
- * Unicode classifies as a diacritic (`\p{Diacritic}`). `\p{Diacritic}`
- * is deliberate: it covers combining marks in every block, not only the
+ * Both variants decompose the input and then remove characters Unicode
+ * classifies as diacritics (`\p{Diacritic}`). `\p{Diacritic}` is
+ * deliberate: it covers combining marks in every block, not only the
  * Combining Diacritical Marks block (U+0300–U+036F). A range like
  * `[̀-ͯ]` misses marks in the Extended (U+1AB0–U+1AFF) and
  * Supplement (U+1DC0–U+1DFF) blocks, so two callers using different
  * classes would disagree on the same text.
  *
- * The variants differ only in the decomposition form:
+ * The variants differ in decomposition form and in what they remove:
  *
- * - `stripDiacritics` uses NFD (canonical decomposition). Use it for
- *   search keys, where compatibility characters must survive unchanged.
+ * - `stripDiacritics` uses NFD (canonical decomposition) and removes
+ *   `SEARCH_DIACRITIC_SOURCE`. Use it for search keys, where compatibility
+ *   characters and non-Latin spacing marks (`ー`, `·`) survive unchanged.
  * - `stripDiacriticsForSlug` uses NFKD (compatibility decomposition), so
  *   ligatures, full-width forms, superscripts, and similar fold to their
  *   ASCII base before the `[a-z0-9]` slug filter runs. Slugs are
  *   persisted, public URL segments, so this variant pins the exact form
- *   the existing slugs were generated with; do not switch it to NFD.
+ *   the existing slugs were generated with, spacing diacritics removed
+ *   too; do not switch it to NFD or narrow its class.
  */
+/**
+ * The diacritics a search fold removes: combining marks, plus spacing ones in
+ * Latin script (modifier letters such as `ʰ`).
+ *
+ * `\p{Diacritic}` alone also covers spacing characters in other scripts that
+ * are text, not marks on a letter: the Japanese prolonged sound mark
+ * (`コーヒー`), the middle dot (`l·l`), `^`, and `` ` ``. Removing those turns
+ * `コーヒー` into `コヒ`, a different word, and PostgreSQL `unaccent()` keeps
+ * them, so a query folded that way misses the indexed text. Latin script keeps
+ * the full class because `foldToAscii` is pinned to `unaccent()` parity there.
+ * Shared as a source string so `foldToAscii` strips the identical class.
+ */
+export declare const SEARCH_DIACRITIC_SOURCE = "(?=\\p{M}|\\p{Script=Latin})\\p{Diacritic}";
 /**
  * Strip combining diacritics from search text (NFD). Non-diacritic
- * characters, including compatibility characters, pass through unchanged.
+ * characters, including compatibility characters and spacing marks such as
+ * `ー`, pass through unchanged.
  */
 export declare const stripDiacritics: (text: string) => string;
 /**

package/dist/diacritics.js

@@ -1,34 +1,56 @@
+// parser-output-unchanged: scripts/parser-output-diff.ts reports every case-law adapter identical over its recorded fixtures with this change; search folds keep non-Latin spacing marks while slug and judge-key folds stay pinned.
 /**
  * Latin/combining-mark diacritic stripping for search keys and slugs.
  *
- * Both variants decompose the input and then remove every character
- * Unicode classifies as a diacritic (`\p{Diacritic}`). `\p{Diacritic}`
- * is deliberate: it covers combining marks in every block, not only the
+ * Both variants decompose the input and then remove characters Unicode
+ * classifies as diacritics (`\p{Diacritic}`). `\p{Diacritic}` is
+ * deliberate: it covers combining marks in every block, not only the
  * Combining Diacritical Marks block (U+0300–U+036F). A range like
  * `[̀-ͯ]` misses marks in the Extended (U+1AB0–U+1AFF) and
  * Supplement (U+1DC0–U+1DFF) blocks, so two callers using different
  * classes would disagree on the same text.
  *
- * The variants differ only in the decomposition form:
+ * The variants differ in decomposition form and in what they remove:
  *
- * - `stripDiacritics` uses NFD (canonical decomposition). Use it for
- *   search keys, where compatibility characters must survive unchanged.
+ * - `stripDiacritics` uses NFD (canonical decomposition) and removes
+ *   `SEARCH_DIACRITIC_SOURCE`. Use it for search keys, where compatibility
+ *   characters and non-Latin spacing marks (`ー`, `·`) survive unchanged.
  * - `stripDiacriticsForSlug` uses NFKD (compatibility decomposition), so
  *   ligatures, full-width forms, superscripts, and similar fold to their
  *   ASCII base before the `[a-z0-9]` slug filter runs. Slugs are
  *   persisted, public URL segments, so this variant pins the exact form
- *   the existing slugs were generated with; do not switch it to NFD.
+ *   the existing slugs were generated with, spacing diacritics removed
+ *   too; do not switch it to NFD or narrow its class.
  */
-const DIACRITIC_RE = /\p{Diacritic}/gu;
+/**
+ * The diacritics a search fold removes: combining marks, plus spacing ones in
+ * Latin script (modifier letters such as `ʰ`).
+ *
+ * `\p{Diacritic}` alone also covers spacing characters in other scripts that
+ * are text, not marks on a letter: the Japanese prolonged sound mark
+ * (`コーヒー`), the middle dot (`l·l`), `^`, and `` ` ``. Removing those turns
+ * `コーヒー` into `コヒ`, a different word, and PostgreSQL `unaccent()` keeps
+ * them, so a query folded that way misses the indexed text. Latin script keeps
+ * the full class because `foldToAscii` is pinned to `unaccent()` parity there.
+ * Shared as a source string so `foldToAscii` strips the identical class.
+ */
+export const SEARCH_DIACRITIC_SOURCE = "(?=\\p{M}|\\p{Script=Latin})\\p{Diacritic}";
+const SEARCH_DIACRITIC_RE = new RegExp(SEARCH_DIACRITIC_SOURCE, "gu");
+/**
+ * Pinned to the class the persisted slugs were minted with, spacing
+ * characters included; see `stripDiacriticsForSlug`.
+ */
+const SLUG_DIACRITIC_RE = /\p{Diacritic}/gu;
 /**
  * Strip combining diacritics from search text (NFD). Non-diacritic
- * characters, including compatibility characters, pass through unchanged.
+ * characters, including compatibility characters and spacing marks such as
+ * `ー`, pass through unchanged.
  */
-export const stripDiacritics = (text) => text.normalize("NFD").replace(DIACRITIC_RE, "");
+export const stripDiacritics = (text) => text.normalize("NFD").replace(SEARCH_DIACRITIC_RE, "");
 /**
  * Strip combining diacritics for slug generation (NFKD). Callers apply
  * their own case folding and `[a-z0-9]` filtering around this; the NFKD
  * form is load-bearing for byte-stable slugs and must not change.
  */
-export const stripDiacriticsForSlug = (text) => text.normalize("NFKD").replace(DIACRITIC_RE, "");
+export const stripDiacriticsForSlug = (text) => text.normalize("NFKD").replace(SLUG_DIACRITIC_RE, "");
 //# sourceMappingURL=diacritics.js.map

package/dist/diacritics.js.map

@@ -1 +1 @@
-{"version":3,"file":"diacritics.js","sourceRoot":"","sources":["../src/diacritics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,YAAY,GAAG,iBAAiB,CAAC;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CACtD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC7D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC"}
+{"version":3,"file":"diacritics.js","sourceRoot":"","sources":["../src/diacritics.ts"],"names":[],"mappings":"AAAA,qOAAqO;AACrO;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAClC,4CAA4C,CAAC;AAE/C,MAAM,mBAAmB,GAAG,IAAI,MAAM,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;AAEtE;;;GAGG;AACH,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CACtD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;AAEzD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC7D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC"}

package/dist/search-match-key.d.ts

@@ -0,0 +1,29 @@
+/** UTF-16 range into the well-formed content searched by literal key helpers. */
+type SearchMatchKeyRange = {
+    end: number;
+    start: number;
+};
+/**
+ * Literal substring search over caller-owned match keys. No trimming or
+ * folding is applied: callers whose policy differs from the highlighter's
+ * fold keep control over case, accents, whitespace, and field boundaries.
+ * Lone surrogates become U+FFFD in both inputs. Empty keys match every value.
+ */
+export declare const containsSearchMatchKey: (content: string, query: string) => boolean;
+type FindSearchMatchKeyRangesOptions = {
+    /** Inclusive UTF-16 offset; defaults to the start of the content. */
+    start?: number;
+    /** Exclusive UTF-16 offset; defaults to the end of the content. */
+    end?: number;
+    maxMatches?: number;
+};
+/**
+ * Non-overlapping literal occurrences in caller-owned keys, with UTF-16
+ * offsets into content after replacing lone surrogates with U+FFFD. Neither
+ * input is trimmed or folded. Well-formed literals match whole Unicode scalars.
+ * Only matches wholly inside [start, end) are returned. Bounds are truncated
+ * and clamped to content; maxMatches is floored. NaN options, empty queries,
+ * reversed windows, and nonpositive limits produce no matches.
+ */
+export declare const findSearchMatchKeyRanges: (content: string, query: string, options?: FindSearchMatchKeyRangesOptions) => SearchMatchKeyRange[];
+export {};

package/dist/search-match-key.js

@@ -0,0 +1,44 @@
+/**
+ * Literal substring search over caller-owned match keys. No trimming or
+ * folding is applied: callers whose policy differs from the highlighter's
+ * fold keep control over case, accents, whitespace, and field boundaries.
+ * Lone surrogates become U+FFFD in both inputs. Empty keys match every value.
+ */
+export const containsSearchMatchKey = (content, query) => content.toWellFormed().includes(query.toWellFormed());
+/**
+ * Non-overlapping literal occurrences in caller-owned keys, with UTF-16
+ * offsets into content after replacing lone surrogates with U+FFFD. Neither
+ * input is trimmed or folded. Well-formed literals match whole Unicode scalars.
+ * Only matches wholly inside [start, end) are returned. Bounds are truncated
+ * and clamped to content; maxMatches is floored. NaN options, empty queries,
+ * reversed windows, and nonpositive limits produce no matches.
+ */
+export const findSearchMatchKeyRanges = (content, query, options = {}) => {
+    const start = Math.min(content.length, Math.max(0, Math.trunc(options.start ?? 0)));
+    const end = Math.min(content.length, Math.max(0, Math.trunc(options.end ?? content.length)));
+    const maxMatches = Math.max(0, Math.floor(options.maxMatches ?? Number.MAX_SAFE_INTEGER));
+    if (query.length === 0 ||
+        !(start < end) ||
+        maxMatches === 0 ||
+        Number.isNaN(maxMatches)) {
+        return [];
+    }
+    const haystack = content.toWellFormed();
+    const needle = query.toWellFormed();
+    const matches = [];
+    let searchFrom = start;
+    while (searchFrom <= end - needle.length) {
+        const matchStart = haystack.indexOf(needle, searchFrom);
+        const matchEnd = matchStart + needle.length;
+        if (matchStart === -1 || matchEnd > end) {
+            break;
+        }
+        matches.push({ start: matchStart, end: matchEnd });
+        if (matches.length >= maxMatches) {
+            break;
+        }
+        searchFrom = matchEnd;
+    }
+    return matches;
+};
+//# sourceMappingURL=search-match-key.js.map

package/dist/search-match-key.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-match-key.js","sourceRoot":"","sources":["../src/search-match-key.ts"],"names":[],"mappings":"AAMA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,OAAe,EACf,KAAa,EACJ,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;AAUpE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACtC,OAAe,EACf,KAAa,EACb,OAAO,GAAoC,EAAE,EACtB,EAAE;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,OAAO,CAAC,MAAM,EACd,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAC5C,CAAC;IACF,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,OAAO,CAAC,MAAM,EACd,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CACvD,CAAC;IACF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAC1D,CAAC;IACF,IACE,KAAK,CAAC,MAAM,KAAK,CAAC;QAClB,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC;QACd,UAAU,KAAK,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EACxB,CAAC;QACD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IACxC,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,EAAE,CAAC;IACpC,MAAM,OAAO,GAA0B,EAAE,CAAC;IAC1C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,OAAO,UAAU,IAAI,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5C,IAAI,UAAU,KAAK,CAAC,CAAC,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;YACxC,MAAM;QACR,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QACnD,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QACD,UAAU,GAAG,QAAQ,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stll/text-normalize",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Search match-key text normalization — Arabic orthographic variant folding (alef/hamza/teh-marbuta/yeh, tashkeel, tatweel, digits) with per-language fold tables",
   "keywords": [
     "arabic",
@@ -32,6 +32,10 @@
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
+    },
+    "./search-match-key": {
+      "types": "./dist/search-match-key.d.ts",
+      "import": "./dist/search-match-key.js"
     }
   },
   "publishConfig": {