@stll/text-normalize 0.0.1-placeholder.0 → 0.1.1

package/dist/diacritics.js

@@ -0,0 +1,34 @@
+/**
+ * Latin/combining-mark diacritic stripping for search keys and slugs.
+ *
+ * Both variants decompose the input and then remove every character
+ * Unicode classifies as a diacritic (`\p{Diacritic}`). `\p{Diacritic}`
+ * is deliberate: it covers combining marks in every block, not only the
+ * Combining Diacritical Marks block (U+0300–U+036F). A range like
+ * `[̀-ͯ]` misses marks in the Extended (U+1AB0–U+1AFF) and
+ * Supplement (U+1DC0–U+1DFF) blocks, so two callers using different
+ * classes would disagree on the same text.
+ *
+ * The variants differ only in the decomposition form:
+ *
+ * - `stripDiacritics` uses NFD (canonical decomposition). Use it for
+ *   search keys, where compatibility characters must survive unchanged.
+ * - `stripDiacriticsForSlug` uses NFKD (compatibility decomposition), so
+ *   ligatures, full-width forms, superscripts, and similar fold to their
+ *   ASCII base before the `[a-z0-9]` slug filter runs. Slugs are
+ *   persisted, public URL segments, so this variant pins the exact form
+ *   the existing slugs were generated with; do not switch it to NFD.
+ */
+const DIACRITIC_RE = /\p{Diacritic}/gu;
+/**
+ * Strip combining diacritics from search text (NFD). Non-diacritic
+ * characters, including compatibility characters, pass through unchanged.
+ */
+export const stripDiacritics = (text) => text.normalize("NFD").replace(DIACRITIC_RE, "");
+/**
+ * Strip combining diacritics for slug generation (NFKD). Callers apply
+ * their own case folding and `[a-z0-9]` filtering around this; the NFKD
+ * form is load-bearing for byte-stable slugs and must not change.
+ */
+export const stripDiacriticsForSlug = (text) => text.normalize("NFKD").replace(DIACRITIC_RE, "");
+//# sourceMappingURL=diacritics.js.map

package/dist/diacritics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diacritics.js","sourceRoot":"","sources":["../src/diacritics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,MAAM,YAAY,GAAG,iBAAiB,CAAC;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CACtD,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC7D,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC"}

package/dist/escape-regexp.d.ts

@@ -0,0 +1,10 @@
+/**
+ * The one escaper for text that becomes part of a pattern.
+ *
+ * Every call site builds a regular expression around something a document or
+ * a user wrote, so each needed the same escape and each had written it out.
+ * Two spellings of this is one spelling too many: the character class is the
+ * contract, and a call site that escapes one character fewer matches things
+ * its author never meant it to.
+ */
+export declare const escapeRegExp: (value: string) => string;

package/dist/escape-regexp.js

@@ -0,0 +1,11 @@
+/**
+ * The one escaper for text that becomes part of a pattern.
+ *
+ * Every call site builds a regular expression around something a document or
+ * a user wrote, so each needed the same escape and each had written it out.
+ * Two spellings of this is one spelling too many: the character class is the
+ * contract, and a call site that escapes one character fewer matches things
+ * its author never meant it to.
+ */
+export const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+//# sourceMappingURL=escape-regexp.js.map

package/dist/escape-regexp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"escape-regexp.js","sourceRoot":"","sources":["../src/escape-regexp.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,KAAa,EAAU,EAAE,CACpD,KAAK,CAAC,OAAO,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+export { applyArabicFolds, applyArabicFoldsWithOffsets, ARABIC_DIGIT_FOLDS, ARABIC_LETTER_FOLDS, ARABIC_REMOVED, } from "./arabic.js";
+export type { FoldedText } from "./arabic.js";
+export { foldToAscii } from "./ascii-fold.js";
+export { ASCII_FOLD_TABLE } from "./ascii-fold-table.js";
+export { stripDiacritics, stripDiacriticsForSlug } from "./diacritics.js";
+export { escapeRegExp } from "./escape-regexp.js";
+export { arabicNormalize } from "./normalize.js";
+export { findSearchMatchRanges, foldSearchMatchText, foldSearchMatchTextWithOffsets, } from "./search-match.js";
+export type { FoldedSearchText, SearchMatchRange } from "./search-match.js";
+export { collapseSpacedLetters, spacedLetterRunRegex, } from "./spaced-letters.js";
+export { slugify } from "./slug.js";
+export type { SlugCharset, SlugifyOptions, SlugSeparator } from "./slug.js";

package/dist/index.js

@@ -0,0 +1,10 @@
+export { applyArabicFolds, applyArabicFoldsWithOffsets, ARABIC_DIGIT_FOLDS, ARABIC_LETTER_FOLDS, ARABIC_REMOVED, } from "./arabic.js";
+export { foldToAscii } from "./ascii-fold.js";
+export { ASCII_FOLD_TABLE } from "./ascii-fold-table.js";
+export { stripDiacritics, stripDiacriticsForSlug } from "./diacritics.js";
+export { escapeRegExp } from "./escape-regexp.js";
+export { arabicNormalize } from "./normalize.js";
+export { findSearchMatchRanges, foldSearchMatchText, foldSearchMatchTextWithOffsets, } from "./search-match.js";
+export { collapseSpacedLetters, spacedLetterRunRegex, } from "./spaced-letters.js";
+export { slugify } from "./slug.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,2BAA2B,EAC3B,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,8BAA8B,GAC/B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC"}

package/dist/normalize.d.ts

@@ -0,0 +1,16 @@
+/**
+ * TypeScript mirror of the SQL `arabic_normalize()` function. Folds Arabic
+ * orthographic variants so a query matches regardless of how the text was
+ * typed (alef/hamza/teh-marbuta/yeh variants, tashkeel, tatweel,
+ * Arabic-Indic digits), then applies locale-stable ASCII case folding for
+ * mixed-script names. It must NOT strip Latin diacritics — the stored match
+ * keys don't; client-side diacritic-insensitive matching is
+ * `foldSearchMatchText` (search-match.ts).
+ *
+ * NFKC runs first so presentation forms (U+FB50–FDFF, U+FE70–FEFF) fold
+ * to their canonical letters before the explicit folds apply.
+ *
+ * This MUST stay consistent with the SQL `arabic_normalize()` function;
+ * the golden vectors in normalize.test.ts pin the contract for both.
+ */
+export declare const arabicNormalize: (text: string) => string;

package/dist/normalize.js

@@ -0,0 +1,32 @@
+import { applyArabicFolds } from "./arabic.js";
+const ASCII_UPPERCASE_RE = /[A-Zİ]/gu;
+const SEARCH_WHITESPACE_RE = /[ \t\n\v\f\r\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/gu;
+const foldSearchCase = (text) => text.replace(ASCII_UPPERCASE_RE, (char) => {
+    if (char === "İ") {
+        return "i";
+    }
+    const codePoint = char.codePointAt(0);
+    if (codePoint === undefined) {
+        return char;
+    }
+    return String.fromCodePoint(codePoint + 32);
+});
+/**
+ * TypeScript mirror of the SQL `arabic_normalize()` function. Folds Arabic
+ * orthographic variants so a query matches regardless of how the text was
+ * typed (alef/hamza/teh-marbuta/yeh variants, tashkeel, tatweel,
+ * Arabic-Indic digits), then applies locale-stable ASCII case folding for
+ * mixed-script names. It must NOT strip Latin diacritics — the stored match
+ * keys don't; client-side diacritic-insensitive matching is
+ * `foldSearchMatchText` (search-match.ts).
+ *
+ * NFKC runs first so presentation forms (U+FB50–FDFF, U+FE70–FEFF) fold
+ * to their canonical letters before the explicit folds apply.
+ *
+ * This MUST stay consistent with the SQL `arabic_normalize()` function;
+ * the golden vectors in normalize.test.ts pin the contract for both.
+ */
+export const arabicNormalize = (text) => foldSearchCase(applyArabicFolds(text.normalize("NFKC")))
+    .replace(SEARCH_WHITESPACE_RE, " ")
+    .trim();
+//# sourceMappingURL=normalize.js.map

package/dist/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../src/normalize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,MAAM,kBAAkB,GAAG,UAAU,CAAC;AACtC,MAAM,oBAAoB,GACxB,+EAA+E,CAAC;AAElF,MAAM,cAAc,GAAG,CAAC,IAAY,EAAU,EAAE,CAC9C,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,IAAI,EAAE,EAAE;IACxC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;QACjB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IACtC,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,MAAM,CAAC,aAAa,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEL;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAU,EAAE,CACtD,cAAc,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;KACrD,OAAO,CAAC,oBAAoB,EAAE,GAAG,CAAC;KAClC,IAAI,EAAE,CAAC"}

package/dist/search-match.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Diacritic-insensitive substring matching with original-index mapping.
+ *
+ * `foldSearchMatchText` builds the client-side match key: NFKD
+ * decomposition, the `unaccent()`-parity ASCII folds (letters with no
+ * decomposition: `ł`, `ø`, `đ`, `ß`), combining-mark strip, Arabic
+ * orthographic folds, then lowercase — so a `capek` query matches `Čapek`
+ * and `wroclaw` matches `Wrocław`, both ways. It is deliberately distinct
+ * from `arabicNormalize` (normalize.ts), which is pinned to the SQL
+ * `arabic_normalize()` contract and must not strip Latin diacritics.
+ *
+ * `findSearchMatchRanges` locates a folded query inside folded content and
+ * reports matches as ranges into the ORIGINAL string, so highlights wrap the
+ * text the user actually sees. Folding changes string length (per-character
+ * decomposition, mark removal, contextual lowercasing), so the folded form
+ * carries a per-code-unit map back to the originating character's range.
+ */
+export type SearchMatchRange = {
+    end: number;
+    start: number;
+};
+export type FoldedSearchText = {
+    /** Original-string character range for each folded UTF-16 code unit. */
+    originalRanges: SearchMatchRange[];
+    text: string;
+};
+export declare const foldSearchMatchText: (value: string) => string;
+export declare const foldSearchMatchTextWithOffsets: (content: string) => FoldedSearchText;
+type FindSearchMatchRangesOptions = {
+    maxMatches?: number;
+};
+/**
+ * Non-overlapping occurrences of the folded query in the content, as ranges
+ * into the original string. Accepts pre-folded content so callers matching
+ * many queries against the same text fold it once.
+ */
+export declare const findSearchMatchRanges: (content: string | FoldedSearchText, query: string, options?: FindSearchMatchRangesOptions) => SearchMatchRange[];
+export {};

package/dist/search-match.js

@@ -0,0 +1,118 @@
+/**
+ * Diacritic-insensitive substring matching with original-index mapping.
+ *
+ * `foldSearchMatchText` builds the client-side match key: NFKD
+ * decomposition, the `unaccent()`-parity ASCII folds (letters with no
+ * decomposition: `ł`, `ø`, `đ`, `ß`), combining-mark strip, Arabic
+ * orthographic folds, then lowercase — so a `capek` query matches `Čapek`
+ * and `wroclaw` matches `Wrocław`, both ways. It is deliberately distinct
+ * from `arabicNormalize` (normalize.ts), which is pinned to the SQL
+ * `arabic_normalize()` contract and must not strip Latin diacritics.
+ *
+ * `findSearchMatchRanges` locates a folded query inside folded content and
+ * reports matches as ranges into the ORIGINAL string, so highlights wrap the
+ * text the user actually sees. Folding changes string length (per-character
+ * decomposition, mark removal, contextual lowercasing), so the folded form
+ * carries a per-code-unit map back to the originating character's range.
+ */
+import { applyArabicFolds } from "./arabic.js";
+import { applyAsciiFolds } from "./ascii-fold.js";
+const COMBINING_MARKS = /\p{M}+/gu;
+// `toLowerCase` keeps the positional final sigma that Unicode case folding
+// maps to σ; a lone `σ` query would otherwise miss the end of `ΟΣ`. Same
+// length, so offsets are unaffected; a full `toUpperCase().toLowerCase()`
+// fold is not used because it changes lengths (ß → SS) and the ASCII table
+// already covers those letters.
+const FINAL_SIGMA = /ς/gu;
+const foldSearchMatchTextBeforeCase = (value) => applyArabicFolds(applyAsciiFolds(value.normalize("NFKD")).replace(COMBINING_MARKS, ""));
+const caseFold = (value) => value.toLowerCase().replace(FINAL_SIGMA, "σ");
+export const foldSearchMatchText = (value) => caseFold(foldSearchMatchTextBeforeCase(value));
+export const foldSearchMatchTextWithOffsets = (content) => {
+    const units = [];
+    let originalOffset = 0;
+    for (const character of content) {
+        const start = originalOffset;
+        originalOffset += character.length;
+        units.push({
+            range: { start, end: originalOffset },
+            text: foldSearchMatchTextBeforeCase(character),
+        });
+    }
+    const beforeCase = units.map(({ text }) => text).join("");
+    const contextualLowercase = caseFold(beforeCase);
+    if (contextualLowercase.length === beforeCase.length) {
+        const originalRanges = [];
+        for (const unit of units) {
+            let codeUnit = 0;
+            while (codeUnit < unit.text.length) {
+                originalRanges.push(unit.range);
+                codeUnit += 1;
+            }
+        }
+        return {
+            text: contextualLowercase,
+            originalRanges,
+        };
+    }
+    // Lowercasing changed the length (e.g. İ gains a combining dot), so the
+    // whole-string result cannot be aligned with the per-character units;
+    // lowercase each unit separately instead.
+    const loweredParts = [];
+    const originalRanges = [];
+    for (const unit of units) {
+        const lowered = caseFold(unit.text);
+        loweredParts.push(lowered);
+        let codeUnit = 0;
+        while (codeUnit < lowered.length) {
+            originalRanges.push(unit.range);
+            codeUnit += 1;
+        }
+    }
+    return { text: loweredParts.join(""), originalRanges };
+};
+/**
+ * Non-overlapping occurrences of the folded query in the content, as ranges
+ * into the original string. Accepts pre-folded content so callers matching
+ * many queries against the same text fold it once.
+ */
+export const findSearchMatchRanges = (content, query, options = {}) => {
+    const maxMatches = Math.max(0, Math.floor(options.maxMatches ?? Number.MAX_SAFE_INTEGER));
+    if (maxMatches === 0) {
+        return [];
+    }
+    const foldedContent = typeof content === "string"
+        ? foldSearchMatchTextWithOffsets(content)
+        : content;
+    const foldedQuery = foldSearchMatchText(query.trim());
+    if (foldedQuery.length === 0) {
+        return [];
+    }
+    const matches = [];
+    let searchFrom = 0;
+    while (searchFrom <= foldedContent.text.length - foldedQuery.length) {
+        const foldedStart = foldedContent.text.indexOf(foldedQuery, searchFrom);
+        if (foldedStart === -1) {
+            break;
+        }
+        const foldedEnd = foldedStart + foldedQuery.length;
+        const firstRange = foldedContent.originalRanges.at(foldedStart);
+        const lastRange = foldedContent.originalRanges.at(foldedEnd - 1);
+        if (!firstRange || !lastRange) {
+            break;
+        }
+        matches.push({ start: firstRange.start, end: lastRange.end });
+        if (matches.length >= maxMatches) {
+            break;
+        }
+        // An expanding fold (ß → ss) maps several folded units to one original
+        // character; resume after the last unit inside the matched characters so
+        // that character is reported once and the budget reaches later hits.
+        searchFrom = foldedEnd;
+        while ((foldedContent.originalRanges.at(searchFrom)?.start ?? Infinity) <
+            lastRange.end) {
+            searchFrom += 1;
+        }
+    }
+    return matches;
+};
+//# sourceMappingURL=search-match.js.map

package/dist/search-match.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-match.js","sourceRoot":"","sources":["../src/search-match.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,eAAe,GAAG,UAAU,CAAC;AACnC,2EAA2E;AAC3E,yEAAyE;AACzE,0EAA0E;AAC1E,2EAA2E;AAC3E,gCAAgC;AAChC,MAAM,WAAW,GAAG,KAAK,CAAC;AAa1B,MAAM,6BAA6B,GAAG,CAAC,KAAa,EAAU,EAAE,CAC9D,gBAAgB,CACd,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CACtE,CAAC;AAEJ,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAU,EAAE,CACzC,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;AAEhD,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,KAAa,EAAU,EAAE,CAC3D,QAAQ,CAAC,6BAA6B,CAAC,KAAK,CAAC,CAAC,CAAC;AAEjD,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,OAAe,EACG,EAAE;IACpB,MAAM,KAAK,GAAgD,EAAE,CAAC;IAC9D,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,SAAS,IAAI,OAAO,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,cAAc,CAAC;QAC7B,cAAc,IAAI,SAAS,CAAC,MAAM,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,cAAc,EAAE;YACrC,IAAI,EAAE,6BAA6B,CAAC,SAAS,CAAC;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1D,MAAM,mBAAmB,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,mBAAmB,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,cAAc,GAAuB,EAAE,CAAC;QAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,OAAO,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACnC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAChC,QAAQ,IAAI,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;QACD,OAAO;YACL,IAAI,EAAE,mBAAmB;YACzB,cAAc;SACf,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,sEAAsE;IACtE,0CAA0C;IAC1C,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,cAAc,GAAuB,EAAE,CAAC;IAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,OAAO,QAAQ,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACjC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChC,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC;AACzD,CAAC,CAAC;AAMF;;;;GAIG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,OAAkC,EAClC,KAAa,EACb,OAAO,GAAiC,EAAE,EACtB,EAAE;IACtB,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,gBAAgB,CAAC,CAC1D,CAAC;IACF,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,aAAa,GACjB,OAAO,OAAO,KAAK,QAAQ;QACzB,CAAC,CAAC,8BAA8B,CAAC,OAAO,CAAC;QACzC,CAAC,CAAC,OAAO,CAAC;IACd,MAAM,WAAW,GAAG,mBAAmB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,UAAU,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;QACpE,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QACxE,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,MAAM;QACR,CAAC;QACD,MAAM,SAAS,GAAG,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC;QACnD,MAAM,UAAU,GAAG,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9B,MAAM;QACR,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC;QAC9D,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QACD,uEAAuE;QACvE,yEAAyE;QACzE,qEAAqE;QACrE,UAAU,GAAG,SAAS,CAAC;QACvB,OACE,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,KAAK,IAAI,QAAQ,CAAC;YAChE,SAAS,CAAC,GAAG,EACb,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC"}

package/dist/slug.d.ts

@@ -0,0 +1,45 @@
+/**
+ * One slug generator for the key-shaped strings the apps derive from a label.
+ *
+ * Four copies of this loop existed, differing only in their separator, their
+ * budget, their fallback, and whether a non-ASCII letter survives. Those four
+ * are the parameters; everything else — lowercase, collapse a run of rejected
+ * characters into one separator, clip, trim a trailing separator, fall back
+ * when nothing is left — was identical in all four and is fixed here.
+ *
+ * Deliberately does NOT transliterate. `slugify("café")` is `"caf"` under the
+ * `ascii` charset, not `"cafe"`: slugs are persisted, and folding here would
+ * quietly start minting different keys for the same labels. The folding
+ * primitives in this package are for search text, where nothing is stored.
+ *
+ * Single-pass by construction: a regex pipeline over untrusted label text is
+ * what the slow-regex guard exists to prevent.
+ */
+/**
+ * Which characters survive.
+ *
+ * - `ascii` keeps `[a-z0-9]`, for keys that end up in URLs and columns
+ *   validated against that charset.
+ * - `unicode` keeps any Unicode letter or number, for paths shown back to the
+ *   person who typed them.
+ */
+export type SlugCharset = "ascii" | "unicode";
+/**
+ * The delimiters callers use: a hyphen for URL and column keys, an underscore
+ * for field paths.
+ *
+ * A closed union rather than a `string`, because an empty separator would make
+ * the trim loop below spin forever — `"".endsWith("")` is true and
+ * `slice(0, -0)` is the whole string. A caller cannot reach that state now.
+ */
+export type SlugSeparator = "-" | "_";
+export type SlugifyOptions = {
+    readonly charset: SlugCharset;
+    /** Joins the surviving runs. */
+    readonly separator: SlugSeparator;
+    /** Budget the result is clipped to, before the trailing separator is cut. */
+    readonly maxLength: number;
+    /** Returned when nothing survives, so a caller always gets a usable key. */
+    readonly fallback: string;
+};
+export declare const slugify: (value: string, { charset, separator, maxLength, fallback }: SlugifyOptions) => string;

package/dist/slug.js

@@ -0,0 +1,53 @@
+/**
+ * One slug generator for the key-shaped strings the apps derive from a label.
+ *
+ * Four copies of this loop existed, differing only in their separator, their
+ * budget, their fallback, and whether a non-ASCII letter survives. Those four
+ * are the parameters; everything else — lowercase, collapse a run of rejected
+ * characters into one separator, clip, trim a trailing separator, fall back
+ * when nothing is left — was identical in all four and is fixed here.
+ *
+ * Deliberately does NOT transliterate. `slugify("café")` is `"caf"` under the
+ * `ascii` charset, not `"cafe"`: slugs are persisted, and folding here would
+ * quietly start minting different keys for the same labels. The folding
+ * primitives in this package are for search text, where nothing is stored.
+ *
+ * Single-pass by construction: a regex pipeline over untrusted label text is
+ * what the slow-regex guard exists to prevent.
+ */
+const UNICODE_SLUG_CHARACTER = /[\p{L}\p{N}]/u;
+const isSlugCharacter = (character, charset) => {
+    if (charset === "unicode") {
+        return UNICODE_SLUG_CHARACTER.test(character);
+    }
+    return ((character >= "a" && character <= "z") ||
+        (character >= "0" && character <= "9"));
+};
+export const slugify = (value, { charset, separator, maxLength, fallback }) => {
+    let buffer = "";
+    let lastWasSeparator = true;
+    for (const character of value.toLowerCase()) {
+        if (isSlugCharacter(character, charset)) {
+            buffer += character;
+            lastWasSeparator = false;
+            continue;
+        }
+        if (!lastWasSeparator) {
+            buffer += separator;
+            lastWasSeparator = true;
+        }
+    }
+    let clipped = "";
+    for (const character of buffer) {
+        if (clipped.length + character.length > maxLength) {
+            break;
+        }
+        clipped += character;
+    }
+    // Every separator is one character, so the trim shortens by one per pass.
+    while (clipped.endsWith(separator)) {
+        clipped = clipped.slice(0, -1);
+    }
+    return clipped || fallback;
+};
+//# sourceMappingURL=slug.js.map

package/dist/slug.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"slug.js","sourceRoot":"","sources":["../src/slug.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAgCH,MAAM,sBAAsB,GAAG,eAAe,CAAC;AAE/C,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,OAAoB,EAAW,EAAE;IAC3E,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,CACL,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC;QACtC,CAAC,SAAS,IAAI,GAAG,IAAI,SAAS,IAAI,GAAG,CAAC,CACvC,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CACrB,KAAa,EACb,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAkB,EACnD,EAAE;IACV,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAE5B,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5C,IAAI,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,KAAK,CAAC;YACzB,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,MAAM,IAAI,SAAS,CAAC;YACpB,gBAAgB,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAClD,MAAM;QACR,CAAC;QACD,OAAO,IAAI,SAAS,CAAC;IACvB,CAAC;IACD,0EAA0E;IAC1E,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACnC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,OAAO,IAAI,QAAQ,CAAC;AAC7B,CAAC,CAAC"}

package/dist/spaced-letters.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Collapse spaced-out letters used for emphasis in court PDFs.
+ *
+ * Slovak and Czech courts format words with letter-spacing:
+ *   `r o z h o d o l :` -> `rozhodol:`
+ *   `o d ô v o d n e n i e :` -> `odôvodnenie:`
+ *   `z a m i e t a` -> `zamieta`
+ *
+ * These break full-text search ("rozhodol" won't match
+ * "r o z h o d o l"). We collapse runs of single Unicode letters
+ * separated by single spaces, optionally followed by punctuation.
+ *
+ * Requires at least FOUR letters in the run. Czech/Slovak have many
+ * single-letter words (prepositions a, i, k, o, s, u, v, z), so a 2–3
+ * letter run like `u a v` ("u", "a", "v") is far more likely to be real
+ * words than letter-spaced emphasis; collapsing it to `uav` would corrupt
+ * the text. Genuine spaced words ("z a m i e t a", "r o z h o d o l") are
+ * always longer, so the floor loses nothing in practice.
+ *
+ * This is the single source of the threshold: the ingestion pipeline
+ * (index-time) and the case-viewer find-in-page normalizer (query-time)
+ * both consume it, so a spaced heading collapses identically on both
+ * sides and highlight offsets stay aligned.
+ */
+/**
+ * A fresh global RegExp matching one spaced-letter run. Returned from a
+ * factory (not a shared constant) because callers use it with both
+ * `String.replace` and `String.matchAll`; a fresh instance avoids any
+ * shared `lastIndex` surprises.
+ */
+export declare const spacedLetterRunRegex: () => RegExp;
+/**
+ * Collapse every spaced-letter run in `text` to its concatenated letters,
+ * normalizing multi-spaces to a single space.
+ *
+ * Applied to a fixpoint rather than once, so `f(f(x)) === f(x)` holds for
+ * every input. A single pass did not settle: the pattern matches letters
+ * separated by exactly one space, while the multi-space squeeze runs
+ * afterwards, so a run written with two spaces survived the pass and came
+ * out in the very shape the pattern was looking for. The next call then
+ * collapsed what the previous one had not.
+ *
+ * That mattered because the two sides of a search comparison are folded
+ * at different times — index time during ingestion, query time in the
+ * case-viewer's find-in-page — and the query side is handed text the
+ * index side already collapsed. A function that collapsed more on its
+ * second application made those two sides disagree, which is exactly the
+ * alignment this module exists to guarantee.
+ *
+ * Widening the pattern's separator to `+` would also settle it in one
+ * pass, and is wrong: a single space separates the letters of one
+ * emphasized word, and a wider gap separates two of them. Accepting both
+ * as letter separators collapses `Ž a l o b a  s e  z a m í t á` to
+ * `Žalobasezamítá` — the whole verdict as one word. The loop keeps that
+ * distinction inside each pass.
+ */
+export declare const collapseSpacedLetters: (text: string) => string;

package/dist/spaced-letters.js

@@ -0,0 +1,101 @@
+/**
+ * Collapse spaced-out letters used for emphasis in court PDFs.
+ *
+ * Slovak and Czech courts format words with letter-spacing:
+ *   `r o z h o d o l :` -> `rozhodol:`
+ *   `o d ô v o d n e n i e :` -> `odôvodnenie:`
+ *   `z a m i e t a` -> `zamieta`
+ *
+ * These break full-text search ("rozhodol" won't match
+ * "r o z h o d o l"). We collapse runs of single Unicode letters
+ * separated by single spaces, optionally followed by punctuation.
+ *
+ * Requires at least FOUR letters in the run. Czech/Slovak have many
+ * single-letter words (prepositions a, i, k, o, s, u, v, z), so a 2–3
+ * letter run like `u a v` ("u", "a", "v") is far more likely to be real
+ * words than letter-spaced emphasis; collapsing it to `uav` would corrupt
+ * the text. Genuine spaced words ("z a m i e t a", "r o z h o d o l") are
+ * always longer, so the floor loses nothing in practice.
+ *
+ * This is the single source of the threshold: the ingestion pipeline
+ * (index-time) and the case-viewer find-in-page normalizer (query-time)
+ * both consume it, so a spaced heading collapses identically on both
+ * sides and highlight offsets stay aligned.
+ */
+// `\p{L} (?:\p{L} ){2,}\p{L}` is four or more spaced letters: one leading
+// letter, at least two interior letters, one trailing letter. Anchored by
+// whitespace/string boundaries so it never touches normal words, digits,
+// or case references.
+const buildSpacedLetterRunSource = () => "(?<=\\s|^)(?:\\p{L} (?:\\p{L} ){2,}\\p{L})(?: ?[,:;.!?])?(?=\\s|$)";
+/**
+ * A fresh global RegExp matching one spaced-letter run. Returned from a
+ * factory (not a shared constant) because callers use it with both
+ * `String.replace` and `String.matchAll`; a fresh instance avoids any
+ * shared `lastIndex` surprises.
+ */
+export const spacedLetterRunRegex = () => new RegExp(buildSpacedLetterRunSource(), "gu");
+const MULTI_SPACE_RE = / {2,}/gu;
+const SPACE_RE = / /gu;
+// Single-entry cache: sk-courts.ts's classifiers (isHoldingMarker,
+// isReasoningMarker, isInstructionMarker) each re-normalize the same line
+// text in sequence while classifying a heading, so consecutive calls on an
+// identical string are common during ingestion. Deterministic pure
+// function, so caching only the most recent call is always correct — a
+// cache miss just falls through to the normal computation.
+let lastInput;
+let lastResult;
+/**
+ * One collapse pass: join every spaced-letter run, then normalize any
+ * resulting multi-spaces to a single space.
+ *
+ * A pass only ever deletes space characters, so it either leaves the
+ * text alone or makes it strictly shorter. That is what bounds the loop
+ * below.
+ */
+const collapsePass = (text) => text
+    .replace(spacedLetterRunRegex(), (match) => match.replace(SPACE_RE, ""))
+    .replace(MULTI_SPACE_RE, " ");
+/**
+ * Collapse every spaced-letter run in `text` to its concatenated letters,
+ * normalizing multi-spaces to a single space.
+ *
+ * Applied to a fixpoint rather than once, so `f(f(x)) === f(x)` holds for
+ * every input. A single pass did not settle: the pattern matches letters
+ * separated by exactly one space, while the multi-space squeeze runs
+ * afterwards, so a run written with two spaces survived the pass and came
+ * out in the very shape the pattern was looking for. The next call then
+ * collapsed what the previous one had not.
+ *
+ * That mattered because the two sides of a search comparison are folded
+ * at different times — index time during ingestion, query time in the
+ * case-viewer's find-in-page — and the query side is handed text the
+ * index side already collapsed. A function that collapsed more on its
+ * second application made those two sides disagree, which is exactly the
+ * alignment this module exists to guarantee.
+ *
+ * Widening the pattern's separator to `+` would also settle it in one
+ * pass, and is wrong: a single space separates the letters of one
+ * emphasized word, and a wider gap separates two of them. Accepting both
+ * as letter separators collapses `Ž a l o b a  s e  z a m í t á` to
+ * `Žalobasezamítá` — the whole verdict as one word. The loop keeps that
+ * distinction inside each pass.
+ */
+export const collapseSpacedLetters = (text) => {
+    if (text === lastInput && lastResult !== undefined) {
+        return lastResult;
+    }
+    let result = text;
+    for (;;) {
+        const next = collapsePass(result);
+        // Terminates: a pass that changes anything strictly shortens the
+        // text, and a pass that changes nothing ends the loop.
+        if (next === result) {
+            break;
+        }
+        result = next;
+    }
+    lastInput = text;
+    lastResult = result;
+    return result;
+};
+//# sourceMappingURL=spaced-letters.js.map

package/dist/spaced-letters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"spaced-letters.js","sourceRoot":"","sources":["../src/spaced-letters.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,0EAA0E;AAC1E,0EAA0E;AAC1E,yEAAyE;AACzE,sBAAsB;AACtB,MAAM,0BAA0B,GAAG,GAAG,EAAE,CACtC,oEAAoE,CAAC;AAEvE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAW,EAAE,CAC/C,IAAI,MAAM,CAAC,0BAA0B,EAAE,EAAE,IAAI,CAAC,CAAC;AAEjD,MAAM,cAAc,GAAG,SAAS,CAAC;AACjC,MAAM,QAAQ,GAAG,KAAK,CAAC;AAEvB,mEAAmE;AACnE,0EAA0E;AAC1E,2EAA2E;AAC3E,mEAAmE;AACnE,uEAAuE;AACvE,2DAA2D;AAC3D,IAAI,SAA6B,CAAC;AAClC,IAAI,UAA8B,CAAC;AAEnC;;;;;;;GAOG;AACH,MAAM,YAAY,GAAG,CAAC,IAAY,EAAU,EAAE,CAC5C,IAAI;KACD,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;KACvE,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;AAElC;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAU,EAAE;IAC5D,IAAI,IAAI,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACnD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;QAClC,iEAAiE;QACjE,uDAAuD;QACvD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACpB,MAAM;QACR,CAAC;QACD,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,SAAS,GAAG,IAAI,CAAC;IACjB,UAAU,GAAG,MAAM,CAAC;IACpB,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC"}

package/package.json

@@ -1,9 +1,58 @@
 {
   "name": "@stll/text-normalize",
-  "version": "0.0.1-placeholder.0",
-  "description": "Placeholder to bootstrap npm trusted publishing. Not for use.",
-  "license": "Apache-2.0",
+  "version": "0.1.1",
+  "description": "Search match-key text normalization — Arabic orthographic variant folding (alef/hamza/teh-marbuta/yeh, tashkeel, tatweel, digits) with per-language fold tables",
+  "keywords": [
+    "arabic",
+    "i18n",
+    "normalization",
+    "search",
+    "text-search",
+    "unicode"
+  ],
   "homepage": "https://github.com/stella/stella/tree/main/packages/text-normalize",
-  "bugs": { "url": "https://github.com/stella/stella/issues" },
-  "repository": { "type": "git", "url": "git+https://github.com/stella/stella.git", "directory": "packages/text-normalize" }
+  "bugs": {
+    "url": "https://github.com/stella/stella/issues"
+  },
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/stella/stella.git",
+    "directory": "packages/text-normalize"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "type": "module",
+  "sideEffects": false,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "clean": "git clean -xdf dist .cache .turbo node_modules",
+    "build": "rm -rf dist && bun ../../packages/scripts/src/tsc-native.ts -p tsconfig.build.json",
+    "pack:dry-run": "bun pm pack --dry-run",
+    "test": "bun test src",
+    "test:property": "bun test --preload @stll/property-testing/preload $(grep -rlF 'fc.assert' src --include='*.test.ts')",
+    "typecheck": "bun ../../packages/scripts/src/tsc-native.ts --noEmit",
+    "lint": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --report-unused-disable-directives-severity=error --type-aware --type-check packages/text-normalize",
+    "lint:fix": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --type-aware --fix packages/text-normalize",
+    "format": "bun ../../scripts/run-oxfmt.ts .",
+    "prepack": "bun run build"
+  },
+  "devDependencies": {
+    "@stll/property-testing": "0.0.0",
+    "@stll/typescript-config": "0.0.0",
+    "bun-types": "1.4.2",
+    "fast-check": "^4.10.1"
+  }
 }