@stll/text-normalize 0.5.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/index.d.ts +1 -0
  2. package/dist/index.js +26 -0
  3. package/dist/index.js.map +1 -1
  4. package/dist/numeric-indices.d.ts +6 -0
  5. package/dist/numeric-indices.js +36 -0
  6. package/dist/numeric-indices.js.map +1 -0
  7. package/package.json +1 -1
package/dist/index.d.ts CHANGED
@@ -12,3 +12,4 @@ export { slugify } from "./slug.js";
12
12
  export type { SlugCharset, SlugifyOptions, SlugSeparator } from "./slug.js";
13
13
  export { normalizeUnicode, stripUnicodeMarks, UNICODE_NORMALIZATION_FORMS, } from "./unicode.js";
14
14
  export type { UnicodeMarkClass, UnicodeNormalizationForm } from "./unicode.js";
15
+ export { normalizeNumericIndices } from "./numeric-indices.js";
package/dist/index.js CHANGED
@@ -1,3 +1,28 @@
1
+ // parser-output-unchanged: [cz-uohs] the reader supplies fulltext and every block plainText directly; its stored footnote inlines and replay fixture fields are unchanged.
2
+ // parser-output-unchanged: [at-asylgh] registered replay fixture fields are unchanged by numeric index normalization.
3
+ // parser-output-unchanged: [at-bks] registered replay fixture fields are unchanged by numeric index normalization.
4
+ // parser-output-unchanged: [at-bvwg] registered replay fixture fields are unchanged by numeric index normalization.
5
+ // parser-output-unchanged: [at-courts] registered replay fixture fields are unchanged by numeric index normalization.
6
+ // parser-output-unchanged: [at-findok] registered replay fixture fields are unchanged by numeric index normalization.
7
+ // parser-output-unchanged: [at-lvwg] registered replay fixture fields are unchanged by numeric index normalization.
8
+ // parser-output-unchanged: [at-ubas] registered replay fixture fields are unchanged by numeric index normalization.
9
+ // parser-output-unchanged: [at-umse] registered replay fixture fields are unchanged by numeric index normalization.
10
+ // parser-output-unchanged: [at-uvs] registered replay fixture fields are unchanged by numeric index normalization.
11
+ // parser-output-unchanged: [at-verg] registered replay fixture fields are unchanged by numeric index normalization.
12
+ // parser-output-unchanged: [at-vfgh] registered replay fixture fields are unchanged by numeric index normalization.
13
+ // parser-output-unchanged: [at-vwgh] registered replay fixture fields are unchanged by numeric index normalization.
14
+ // parser-output-unchanged: [cz-ns] registered replay fixture fields are unchanged by numeric index normalization.
15
+ // parser-output-unchanged: [cz-nss] registered replay fixture fields are unchanged by numeric index normalization.
16
+ // parser-output-unchanged: [cz-regional] registered replay fixture fields are unchanged by numeric index normalization.
17
+ // parser-output-unchanged: [cz-us] registered replay fixture fields are unchanged by numeric index normalization.
18
+ // parser-output-unchanged: [hu-bhgy] registered replay fixture fields are unchanged by numeric index normalization.
19
+ // parser-output-unchanged: [pl-kis] registered replay fixture fields are unchanged by numeric index normalization.
20
+ // parser-output-unchanged: [pl-nsa] registered replay fixture fields are unchanged by numeric index normalization.
21
+ // parser-output-unchanged: [pl-uokik] registered replay fixture fields are unchanged by numeric index normalization.
22
+ // parser-output-unchanged: [sk-courts] registered replay fixture fields are unchanged by numeric index normalization.
23
+ // parser-output-unchanged: [sk-us] registered replay fixture fields are unchanged by numeric index normalization.
24
+ // parser-output-unchanged: [uk-fcl] registered replay fixture fields are unchanged by numeric index normalization.
25
+ // parser-output-unchanged: [us-courtlistener] registered replay fixture fields are unchanged by numeric index normalization.
1
26
  // parser-output-unchanged: Unicode owner migration retains native normalization forms, mark classes and operation order; no folding is added.
2
27
  export { applyArabicFolds, applyArabicFoldsWithOffsets, ARABIC_DIGIT_FOLDS, ARABIC_LETTER_FOLDS, ARABIC_REMOVED, } from "./arabic.js";
3
28
  export { foldToAscii } from "./ascii-fold.js";
@@ -9,4 +34,5 @@ export { findSearchMatchRanges, foldSearchMatchText, foldSearchMatchTextWithOffs
9
34
  export { collapseSpacedLetters, spacedLetterRunRegex, } from "./spaced-letters.js";
10
35
  export { slugify } from "./slug.js";
11
36
  export { normalizeUnicode, stripUnicodeMarks, UNICODE_NORMALIZATION_FORMS, } from "./unicode.js";
37
+ export { normalizeNumericIndices } from "./numeric-indices.js";
12
38
  //# sourceMappingURL=index.js.map
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,8IAA8I;AAC9I,OAAO,EACL,gBAAgB,EAChB,2BAA2B,EAC3B,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,8BAA8B,GAC/B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,GAC5B,MAAM,cAAc,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,2KAA2K;AAC3K,sHAAsH;AACtH,mHAAmH;AACnH,oHAAoH;AACpH,sHAAsH;AACtH,sHAAsH;AACtH,oHAAoH;AACpH,oHAAoH;AACpH,oHAAoH;AACpH,mHAAmH;AACnH,oHAAoH;AACpH,oHAAoH;AACpH,oHAAoH;AACpH,kHAAkH;AAClH,mHAAmH;AACnH,wHAAwH;AACxH,kHAAkH;AAClH,oHAAoH;AACpH,mHAAmH;AACnH,mHAAmH;AACnH,qHAAqH;AACrH,sHAAsH;AACtH,kHAAkH;AAClH,mHAAmH;AACnH,6HAA6H;AAC7H,8IAA8I;AAC9I,OAAO,EACL,gBAAgB,EAChB,2BAA2B,EAC3B,kBAAkB,EAClB,mBAAmB,EACnB,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC1E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,8BAA8B,GAC/B,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EACL,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,2BAA2B,GAC5B,MAAM,cAAc,CAAC;AAGtB,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC"}
package/dist/numeric-indices.d.ts ADDED
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Keep a numeric superscript attached to a number distinct from its digits.
3
+ * Run before compatibility folding: NFKC alone turns `505⁹` into `5059`.
4
+ * Caret and parenthesized indices already have an ordinary token boundary.
5
+ */
6
+ export declare const normalizeNumericIndices: (text: string) => string;
package/dist/numeric-indices.js ADDED
@@ -0,0 +1,36 @@
1
+ // parser-output-unchanged: [cz-uohs] the reader supplies fulltext and every block plainText directly; its stored footnote inlines and replay fixture fields are unchanged.
2
+ // parser-output-unchanged: [at-asylgh] registered replay fixture fields are unchanged by numeric index normalization.
3
+ // parser-output-unchanged: [at-bks] registered replay fixture fields are unchanged by numeric index normalization.
4
+ // parser-output-unchanged: [at-bvwg] registered replay fixture fields are unchanged by numeric index normalization.
5
+ // parser-output-unchanged: [at-courts] registered replay fixture fields are unchanged by numeric index normalization.
6
+ // parser-output-unchanged: [at-findok] registered replay fixture fields are unchanged by numeric index normalization.
7
+ // parser-output-unchanged: [at-lvwg] registered replay fixture fields are unchanged by numeric index normalization.
8
+ // parser-output-unchanged: [at-ubas] registered replay fixture fields are unchanged by numeric index normalization.
9
+ // parser-output-unchanged: [at-umse] registered replay fixture fields are unchanged by numeric index normalization.
10
+ // parser-output-unchanged: [at-uvs] registered replay fixture fields are unchanged by numeric index normalization.
11
+ // parser-output-unchanged: [at-verg] registered replay fixture fields are unchanged by numeric index normalization.
12
+ // parser-output-unchanged: [at-vfgh] registered replay fixture fields are unchanged by numeric index normalization.
13
+ // parser-output-unchanged: [at-vwgh] registered replay fixture fields are unchanged by numeric index normalization.
14
+ // parser-output-unchanged: [cz-ns] registered replay fixture fields are unchanged by numeric index normalization.
15
+ // parser-output-unchanged: [cz-nss] registered replay fixture fields are unchanged by numeric index normalization.
16
+ // parser-output-unchanged: [cz-regional] registered replay fixture fields are unchanged by numeric index normalization.
17
+ // parser-output-unchanged: [cz-us] registered replay fixture fields are unchanged by numeric index normalization.
18
+ // parser-output-unchanged: [hu-bhgy] registered replay fixture fields are unchanged by numeric index normalization.
19
+ // parser-output-unchanged: [pl-kis] registered replay fixture fields are unchanged by numeric index normalization.
20
+ // parser-output-unchanged: [pl-nsa] registered replay fixture fields are unchanged by numeric index normalization.
21
+ // parser-output-unchanged: [pl-uokik] registered replay fixture fields are unchanged by numeric index normalization.
22
+ // parser-output-unchanged: [sk-courts] registered replay fixture fields are unchanged by numeric index normalization.
23
+ // parser-output-unchanged: [sk-us] registered replay fixture fields are unchanged by numeric index normalization.
24
+ // parser-output-unchanged: [uk-fcl] registered replay fixture fields are unchanged by numeric index normalization.
25
+ // parser-output-unchanged: [us-courtlistener] registered replay fixture fields are unchanged by numeric index normalization.
26
+ import { normalizeUnicode } from "./unicode.js";
27
+ // Any decimal digit, not only ASCII: NFKC folds full-width `５０５⁹` to `5059`
28
+ // just as it folds `505⁹`.
29
+ const NUMERIC_SUPERSCRIPT_INDEX = /(?<=\p{Nd})[⁰¹²³⁴⁵⁶⁷⁸⁹]+/gu;
30
+ /**
31
+ * Keep a numeric superscript attached to a number distinct from its digits.
32
+ * Run before compatibility folding: NFKC alone turns `505⁹` into `5059`.
33
+ * Caret and parenthesized indices already have an ordinary token boundary.
34
+ */
35
+ export const normalizeNumericIndices = (text) => text.replace(NUMERIC_SUPERSCRIPT_INDEX, (index) => `^${normalizeUnicode(index, "NFKC")}`);
36
+ //# sourceMappingURL=numeric-indices.js.map
package/dist/numeric-indices.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"numeric-indices.js","sourceRoot":"","sources":["../src/numeric-indices.ts"],"names":[],"mappings":"AAAA,2KAA2K;AAC3K,sHAAsH;AACtH,mHAAmH;AACnH,oHAAoH;AACpH,sHAAsH;AACtH,sHAAsH;AACtH,oHAAoH;AACpH,oHAAoH;AACpH,oHAAoH;AACpH,mHAAmH;AACnH,oHAAoH;AACpH,oHAAoH;AACpH,oHAAoH;AACpH,kHAAkH;AAClH,mHAAmH;AACnH,wHAAwH;AACxH,kHAAkH;AAClH,oHAAoH;AACpH,mHAAmH;AACnH,mHAAmH;AACnH,qHAAqH;AACrH,sHAAsH;AACtH,kHAAkH;AAClH,mHAAmH;AACnH,6HAA6H;AAC7H,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,4EAA4E;AAC5E,2BAA2B;AAC3B,MAAM,yBAAyB,GAAG,4BAA4B,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC9D,IAAI,CAAC,OAAO,CACV,yBAAyB,EACzB,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CACjD,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@stll/text-normalize",
3
- "version": "0.5.0",
3
+ "version": "0.5.2",
4
4
  "description": "Search match-key text normalization — Arabic orthographic variant folding (alef/hamza/teh-marbuta/yeh, tashkeel, tatweel, digits) with per-language fold tables",
5
5
  "keywords": [
6
6
  "arabic",