@stll/text-normalize 0.1.1 → 0.2.0

package/dist/index.d.ts

@@ -10,3 +10,5 @@ export type { FoldedSearchText, SearchMatchRange } from "./search-match.js";
 export { collapseSpacedLetters, spacedLetterRunRegex, } from "./spaced-letters.js";
 export { slugify } from "./slug.js";
 export type { SlugCharset, SlugifyOptions, SlugSeparator } from "./slug.js";
+export { normalizeUnicode, stripUnicodeMarks, UNICODE_NORMALIZATION_FORMS, } from "./unicode.js";
+export type { UnicodeMarkClass, UnicodeNormalizationForm } from "./unicode.js";

package/dist/index.js

@@ -1,3 +1,4 @@
+// parser-output-unchanged: Unicode owner migration retains native normalization forms, mark classes and operation order; no folding is added.
 export { applyArabicFolds, applyArabicFoldsWithOffsets, ARABIC_DIGIT_FOLDS, ARABIC_LETTER_FOLDS, ARABIC_REMOVED, } from "./arabic.js";
 export { foldToAscii } from "./ascii-fold.js";
 export { ASCII_FOLD_TABLE } from "./ascii-fold-table.js";
@@ -7,4 +8,5 @@ export { arabicNormalize } from "./normalize.js";
 export { findSearchMatchRanges, foldSearchMatchText, foldSearchMatchTextWithOffsets, } from "./search-match.js";
 export { collapseSpacedLetters, spacedLetterRunRegex, } from "./spaced-letters.js";
 export { slugify } from "./slug.js";
+export { normalizeUnicode, stripUnicodeMarks, UNICODE_NORMALIZATION_FORMS, } from "./unicode.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,2BAA2B,EAC3B,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,8BAA8B,GAC/B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,8IAA8I;AAC9I,OAAO,EACL,gBAAgB,EAChB,2BAA2B,EAC3B,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,8BAA8B,GAC/B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,GAC5B,MAAM,cAAc,CAAC"}

package/dist/unicode.d.ts

@@ -0,0 +1,12 @@
+export declare const UNICODE_NORMALIZATION_FORMS: readonly ["NFC", "NFD", "NFKC", "NFKD"];
+export type UnicodeNormalizationForm = (typeof UNICODE_NORMALIZATION_FORMS)[number];
+export type UnicodeMarkClass = "combining" | "diacritic" | "basic-combining";
+type StripUnicodeMarksOptions = {
+    form: UnicodeNormalizationForm;
+    markClass: UnicodeMarkClass;
+};
+/** Applies one explicit Unicode normalization form without additional folding. */
+export declare const normalizeUnicode: (text: string, form: UnicodeNormalizationForm) => string;
+/** Normalizes text, then removes the caller's exact historical mark class. */
+export declare const stripUnicodeMarks: (text: string, { form, markClass }: StripUnicodeMarksOptions) => string;
+export {};

package/dist/unicode.js

@@ -0,0 +1,23 @@
+// parser-output-unchanged: Unicode owner migration retains native normalization forms, mark classes and operation order; no folding is added.
+export const UNICODE_NORMALIZATION_FORMS = [
+    "NFC",
+    "NFD",
+    "NFKC",
+    "NFKD",
+];
+const COMBINING_MARKS = /\p{M}/gu;
+const DIACRITICS = /\p{Diacritic}/gu;
+const BASIC_COMBINING_MARKS = /[\u0300-\u036f]/gu;
+const MARK_PATTERNS = {
+    combining: COMBINING_MARKS,
+    diacritic: DIACRITICS,
+    "basic-combining": BASIC_COMBINING_MARKS,
+};
+/** Applies one explicit Unicode normalization form without additional folding. */
+export const normalizeUnicode = (text, form) => text.normalize(form);
+/** Normalizes text, then removes the caller's exact historical mark class. */
+export const stripUnicodeMarks = (text, { form, markClass }) => {
+    const normalized = normalizeUnicode(text, form);
+    return normalized.replaceAll(MARK_PATTERNS[markClass], "");
+};
+//# sourceMappingURL=unicode.js.map

package/dist/unicode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"unicode.js","sourceRoot":"","sources":["../src/unicode.ts"],"names":[],"mappings":"AAAA,8IAA8I;AAC9I,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;CACE,CAAC;AAYX,MAAM,eAAe,GAAG,SAAS,CAAC;AAClC,MAAM,UAAU,GAAG,iBAAiB,CAAC;AACrC,MAAM,qBAAqB,GAAG,mBAAmB,CAAC;AAClD,MAAM,aAAa,GAAG;IACpB,SAAS,EAAE,eAAe;IAC1B,SAAS,EAAE,UAAU;IACrB,iBAAiB,EAAE,qBAAqB;CACW,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,IAAY,EACZ,IAA8B,EACtB,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AAElC,8EAA8E;AAC9E,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,IAAY,EACZ,EAAE,IAAI,EAAE,SAAS,EAA4B,EACrC,EAAE;IACV,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChD,OAAO,UAAU,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;AAC7D,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stll/text-normalize",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Search match-key text normalization — Arabic orthographic variant folding (alef/hamza/teh-marbuta/yeh, tashkeel, tatweel, digits) with per-language fold tables",
   "keywords": [
     "arabic",
@@ -42,8 +42,8 @@
     "build": "rm -rf dist && bun ../../packages/scripts/src/tsc-native.ts -p tsconfig.build.json",
     "pack:dry-run": "bun pm pack --dry-run",
     "test": "bun test src",
-    "test:property": "bun test --preload @stll/property-testing/preload $(grep -rlF 'fc.assert' src --include='*.test.ts')",
-    "typecheck": "bun ../../packages/scripts/src/tsc-native.ts --noEmit",
+    "test:property": "bun test --preload @stll/property-testing/preload $(grep -rlE 'fc\\.assert|assertProperty' src --include='*.test.ts')",
+    "typecheck": "bun check --no-pretty --all --project=tsconfig.json",
     "lint": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --report-unused-disable-directives-severity=error --type-aware --type-check packages/text-normalize",
     "lint:fix": "cd ../.. && bun --bun oxlint -c oxlint.config.ts --type-aware --fix packages/text-normalize",
     "format": "bun ../../scripts/run-oxfmt.ts .",
@@ -52,7 +52,7 @@
   "devDependencies": {
     "@stll/property-testing": "0.0.0",
     "@stll/typescript-config": "0.0.0",
-    "bun-types": "1.4.2",
-    "fast-check": "^4.10.1"
+    "bun-types": "1.4.3",
+    "fast-check": "^4.10.2"
   }
 }