@mailwoman/normalize 8.1.0 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abbreviations.ts +5 -0
- package/cjk.ts +27 -7
- package/compute.ts +1 -1
- package/nfc.ts +7 -5
- package/offset-map.ts +3 -1
- package/out/abbreviations.d.ts.map +1 -1
- package/out/abbreviations.js.map +1 -1
- package/out/cjk.d.ts +6 -2
- package/out/cjk.d.ts.map +1 -1
- package/out/cjk.js +17 -5
- package/out/cjk.js.map +1 -1
- package/out/compute.js +1 -1
- package/out/compute.js.map +1 -1
- package/out/nfc.d.ts +3 -1
- package/out/nfc.d.ts.map +1 -1
- package/out/nfc.js +4 -4
- package/out/nfc.js.map +1 -1
- package/out/offset-map.d.ts +3 -1
- package/out/offset-map.d.ts.map +1 -1
- package/out/offset-map.js +3 -1
- package/out/offset-map.js.map +1 -1
- package/out/punctuation.d.ts.map +1 -1
- package/out/punctuation.js +1 -0
- package/out/punctuation.js.map +1 -1
- package/out/types.d.ts +30 -10
- package/out/types.d.ts.map +1 -1
- package/out/whitespace.d.ts.map +1 -1
- package/out/whitespace.js +5 -3
- package/out/whitespace.js.map +1 -1
- package/package.json +1 -1
- package/punctuation.ts +1 -0
- package/types.ts +30 -10
- package/whitespace.ts +10 -3
package/abbreviations.ts
CHANGED
|
@@ -129,13 +129,16 @@ export function expandAbbreviations(input: string, locale?: string): Abbreviatio
|
|
|
129
129
|
out.push(ch)
|
|
130
130
|
map.push(i)
|
|
131
131
|
i += 1
|
|
132
|
+
|
|
132
133
|
continue
|
|
133
134
|
}
|
|
135
|
+
|
|
134
136
|
const start = i
|
|
135
137
|
|
|
136
138
|
while (i < input.length && isTokenChar(input[i]!)) {
|
|
137
139
|
i += 1
|
|
138
140
|
}
|
|
141
|
+
|
|
139
142
|
const token = input.slice(start, i)
|
|
140
143
|
const tokenWithTrailingDot = i < input.length && input[i] === "." ? `${token}.` : token
|
|
141
144
|
const lookupKey = token.replace(/\.$/, "").toLowerCase()
|
|
@@ -146,6 +149,7 @@ export function expandAbbreviations(input: string, locale?: string): Abbreviatio
|
|
|
146
149
|
out.push(token[k]!)
|
|
147
150
|
map.push(start + k)
|
|
148
151
|
}
|
|
152
|
+
|
|
149
153
|
continue
|
|
150
154
|
}
|
|
151
155
|
|
|
@@ -154,6 +158,7 @@ export function expandAbbreviations(input: string, locale?: string): Abbreviatio
|
|
|
154
158
|
out.push(expansion[k]!)
|
|
155
159
|
map.push(start + Math.min(k, token.length - 1))
|
|
156
160
|
}
|
|
161
|
+
|
|
157
162
|
expansions.push({
|
|
158
163
|
from: tokenWithTrailingDot,
|
|
159
164
|
to: expansion,
|
package/cjk.ts
CHANGED
|
@@ -32,17 +32,33 @@ import { identityMap } from "./offset-map.ts"
|
|
|
32
32
|
export interface CjkResult {
|
|
33
33
|
text: string
|
|
34
34
|
map: number[]
|
|
35
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* Count of characters folded in place (full-width → ASCII, ideographic space → ' ').
|
|
37
|
+
*/
|
|
36
38
|
folded: number
|
|
37
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Count of characters dropped (the postal mark).
|
|
41
|
+
*/
|
|
38
42
|
stripped: number
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
const
|
|
45
|
-
|
|
45
|
+
/**
|
|
46
|
+
* ！.
|
|
47
|
+
*/
|
|
48
|
+
const FULLWIDTH_START = 0xff_01
|
|
49
|
+
/**
|
|
50
|
+
* ～.
|
|
51
|
+
*/
|
|
52
|
+
const FULLWIDTH_END = 0xff_5e
|
|
53
|
+
/**
|
|
54
|
+
* U+FFxx − 0xFEE0 = U+00xx.
|
|
55
|
+
*/
|
|
56
|
+
const FULLWIDTH_TO_ASCII = 0xfe_e0
|
|
57
|
+
const IDEOGRAPHIC_SPACE = 0x30_00
|
|
58
|
+
/**
|
|
59
|
+
* 〒.
|
|
60
|
+
*/
|
|
61
|
+
const POSTAL_MARK = 0x30_12
|
|
46
62
|
|
|
47
63
|
export function applyCjkNormalization(input: string): CjkResult {
|
|
48
64
|
let folded = 0
|
|
@@ -57,6 +73,7 @@ export function applyCjkNormalization(input: string): CjkResult {
|
|
|
57
73
|
|
|
58
74
|
if (code === POSTAL_MARK) {
|
|
59
75
|
stripped += 1
|
|
76
|
+
|
|
60
77
|
continue // drop — no addressing content; whitespace collapse later tidies any gap
|
|
61
78
|
}
|
|
62
79
|
|
|
@@ -64,6 +81,7 @@ export function applyCjkNormalization(input: string): CjkResult {
|
|
|
64
81
|
out.push(String.fromCharCode(code - FULLWIDTH_TO_ASCII))
|
|
65
82
|
map.push(i)
|
|
66
83
|
folded += 1
|
|
84
|
+
|
|
67
85
|
continue
|
|
68
86
|
}
|
|
69
87
|
|
|
@@ -71,8 +89,10 @@ export function applyCjkNormalization(input: string): CjkResult {
|
|
|
71
89
|
out.push(" ")
|
|
72
90
|
map.push(i)
|
|
73
91
|
folded += 1
|
|
92
|
+
|
|
74
93
|
continue
|
|
75
94
|
}
|
|
95
|
+
|
|
76
96
|
out.push(input[i]!)
|
|
77
97
|
map.push(i)
|
|
78
98
|
}
|
package/compute.ts
CHANGED
|
@@ -68,7 +68,7 @@ export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
|
|
|
68
68
|
if (opts?.expandAbbreviations) {
|
|
69
69
|
const r = expandAbbreviations(text, opts.locale)
|
|
70
70
|
|
|
71
|
-
if (r.expansions.length
|
|
71
|
+
if (r.expansions.length) {
|
|
72
72
|
text = r.text
|
|
73
73
|
map = composeMaps(map, r.map)
|
|
74
74
|
|
package/nfc.ts
CHANGED
|
@@ -16,7 +16,9 @@ import { identityMap } from "./offset-map.ts"
|
|
|
16
16
|
|
|
17
17
|
export interface NFCResult {
|
|
18
18
|
text: string
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* `text[i]` came from `input[map[i]]`.
|
|
21
|
+
*/
|
|
20
22
|
map: number[]
|
|
21
23
|
changed: boolean
|
|
22
24
|
}
|
|
@@ -43,18 +45,18 @@ function estimateNFCMap(input: string, output: string): number[] {
|
|
|
43
45
|
for (let outIdx = 0; outIdx < output.length; outIdx++) {
|
|
44
46
|
map.push(inIdx)
|
|
45
47
|
const outCp = output.codePointAt(outIdx)!
|
|
46
|
-
const outStep = outCp >
|
|
48
|
+
const outStep = outCp > 0xff_ff ? 2 : 1
|
|
47
49
|
|
|
48
50
|
// Walk the input forward by at least one codepoint; absorb any combining marks (0x0300–0x036f).
|
|
49
51
|
if (inIdx < input.length) {
|
|
50
52
|
const inCp = input.codePointAt(inIdx)!
|
|
51
|
-
inIdx += inCp >
|
|
53
|
+
inIdx += inCp > 0xff_ff ? 2 : 1
|
|
52
54
|
|
|
53
55
|
while (inIdx < input.length) {
|
|
54
56
|
const nextCp = input.codePointAt(inIdx)!
|
|
55
57
|
|
|
56
|
-
if (nextCp >=
|
|
57
|
-
inIdx += nextCp >
|
|
58
|
+
if (nextCp >= 0x03_00 && nextCp <= 0x03_6f) {
|
|
59
|
+
inIdx += nextCp > 0xff_ff ? 2 : 1
|
|
58
60
|
} else {
|
|
59
61
|
break
|
|
60
62
|
}
|
package/offset-map.ts
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* Utilities for composing per-transform offset maps into the final `raw → normalized` map.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
/**
|
|
9
|
+
/**
|
|
10
|
+
* Identity map for an input of length `n`: `[0, 1, 2, ..., n-1]`.
|
|
11
|
+
*/
|
|
10
12
|
export function identityMap(n: number): number[] {
|
|
11
13
|
const m = new Array<number>(n)
|
|
12
14
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE3C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AAkED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,
|
|
1
|
+
{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE3C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AAkED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAmEtF"}
|
package/out/abbreviations.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YAED,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1C,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
|
package/out/cjk.d.ts
CHANGED
|
@@ -29,9 +29,13 @@
|
|
|
29
29
|
export interface CjkResult {
|
|
30
30
|
text: string;
|
|
31
31
|
map: number[];
|
|
32
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Count of characters folded in place (full-width → ASCII, ideographic space → ' ').
|
|
34
|
+
*/
|
|
33
35
|
folded: number;
|
|
34
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* Count of characters dropped (the postal mark).
|
|
38
|
+
*/
|
|
35
39
|
stripped: number;
|
|
36
40
|
}
|
|
37
41
|
export declare function applyCjkNormalization(input: string): CjkResult;
|
package/out/cjk.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb
|
|
1
|
+
{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAoBD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CA0C9D"}
|
package/out/cjk.js
CHANGED
|
@@ -27,11 +27,23 @@
|
|
|
27
27
|
* untouched.
|
|
28
28
|
*/
|
|
29
29
|
import { identityMap } from "./offset-map.js";
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
const
|
|
34
|
-
|
|
30
|
+
/**
|
|
31
|
+
* ！.
|
|
32
|
+
*/
|
|
33
|
+
const FULLWIDTH_START = 0xff_01;
|
|
34
|
+
/**
|
|
35
|
+
* ～.
|
|
36
|
+
*/
|
|
37
|
+
const FULLWIDTH_END = 0xff_5e;
|
|
38
|
+
/**
|
|
39
|
+
* U+FFxx − 0xFEE0 = U+00xx.
|
|
40
|
+
*/
|
|
41
|
+
const FULLWIDTH_TO_ASCII = 0xfe_e0;
|
|
42
|
+
const IDEOGRAPHIC_SPACE = 0x30_00;
|
|
43
|
+
/**
|
|
44
|
+
* 〒.
|
|
45
|
+
*/
|
|
46
|
+
const POSTAL_MARK = 0x30_12;
|
|
35
47
|
export function applyCjkNormalization(input) {
|
|
36
48
|
let folded = 0;
|
|
37
49
|
let stripped = 0;
|
package/out/cjk.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAe7C;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAE3B,MAAM,UAAU,qBAAqB,CAAC,KAAa;IAClD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC1B,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}
|
package/out/compute.js
CHANGED
|
@@ -56,7 +56,7 @@ export function normalize(raw, opts) {
|
|
|
56
56
|
// expansion form (e.g. "Street") gives a consistent final case.
|
|
57
57
|
if (opts?.expandAbbreviations) {
|
|
58
58
|
const r = expandAbbreviations(text, opts.locale);
|
|
59
|
-
if (r.expansions.length
|
|
59
|
+
if (r.expansions.length) {
|
|
60
60
|
text = r.text;
|
|
61
61
|
map = composeMaps(map, r.map);
|
|
62
62
|
for (const e of r.expansions) {
|
package/out/compute.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compute.js","sourceRoot":"","sources":["../compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,IAAoB;IAC1D,MAAM,UAAU,GAA6B,EAAE,CAAA;IAC/C,IAAI,IAAI,GAAG,GAAG,CAAA;IACd,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEjC,SAAS;IACT,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;QACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;QAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,+FAA+F;IAC/F,+FAA+F;IAC/F,iGAAiG;IACjG,CAAC;QACA,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAA;QAErC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;QACnF,CAAC;IACF,CAAC;IAED,iBAAiB;IACjB,CAAC;QACA,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAEhC,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAA;QACjF,CAAC;IACF,CAAC;IAED,gBAAgB;IAChB,CAAC;QACA,MAAM,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;QAElC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACjD,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,2FAA2F;IAC3F,gEAAgE;IAChE,IAAI,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhD,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"compute.js","sourceRoot":"","sources":["../compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,IAAoB;IAC1D,MAAM,UAAU,GAA6B,EAAE,CAAA;IAC/C,IAAI,IAAI,GAAG,GAAG,CAAA;IACd,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEjC,SAAS;IACT,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;QACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;QAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,+FAA+F;IAC/F,+FAA+F;IAC/F,iGAAiG;IACjG,CAAC;QACA,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAA;QAErC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;QACnF,CAAC;IACF,CAAC;IAED,iBAAiB;IACjB,CAAC;QACA,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAEhC,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAA;QACjF,CAAC;IACF,CAAC;IAED,gBAAgB;IAChB,CAAC;QACA,MAAM,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;QAElC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACjD,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,2FAA2F;IAC3F,gEAAgE;IAChE,IAAI,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhD,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAE7B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YACnF,CAAC;QACF,CAAC;IACF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAE9C,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,EAAE,CAAA;YACT,sEAAsE;YACtE,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAA;QACrE,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC;QACpB,GAAG;QACH,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAA6B;QACjE,SAAS,EAAE,GAAG;QACd,aAAa,EAAE,IAAI,EAAE,MAAM;KAC3B,CAA2B,CAAA;AAC7B,CAAC"}
|
package/out/nfc.d.ts
CHANGED
package/out/nfc.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nfc.d.ts","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ
|
|
1
|
+
{"version":3,"file":"nfc.d.ts","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;CAChB;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAQjD"}
|
package/out/nfc.js
CHANGED
|
@@ -30,15 +30,15 @@ function estimateNFCMap(input, output) {
|
|
|
30
30
|
for (let outIdx = 0; outIdx < output.length; outIdx++) {
|
|
31
31
|
map.push(inIdx);
|
|
32
32
|
const outCp = output.codePointAt(outIdx);
|
|
33
|
-
const outStep = outCp >
|
|
33
|
+
const outStep = outCp > 0xff_ff ? 2 : 1;
|
|
34
34
|
// Walk the input forward by at least one codepoint; absorb any combining marks (0x0300–0x036f).
|
|
35
35
|
if (inIdx < input.length) {
|
|
36
36
|
const inCp = input.codePointAt(inIdx);
|
|
37
|
-
inIdx += inCp >
|
|
37
|
+
inIdx += inCp > 0xff_ff ? 2 : 1;
|
|
38
38
|
while (inIdx < input.length) {
|
|
39
39
|
const nextCp = input.codePointAt(inIdx);
|
|
40
|
-
if (nextCp >=
|
|
41
|
-
inIdx += nextCp >
|
|
40
|
+
if (nextCp >= 0x03_00 && nextCp <= 0x03_6f) {
|
|
41
|
+
inIdx += nextCp > 0xff_ff ? 2 : 1;
|
|
42
42
|
}
|
|
43
43
|
else {
|
|
44
44
|
break;
|
package/out/nfc.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nfc.js","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"nfc.js","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAW7C,MAAM,UAAU,QAAQ,CAAC,KAAa;IACrC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAEzC,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;IACvE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,KAAa,EAAE,MAAc;IACpD,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAE,CAAA;QACzC,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAEvC,gGAAgG;QAChG,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;YACtC,KAAK,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/B,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;gBAExC,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC5C,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACP,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
package/out/offset-map.d.ts
CHANGED
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Utilities for composing per-transform offset maps into the final `raw → normalized` map.
|
|
7
7
|
*/
|
|
8
|
-
/**
|
|
8
|
+
/**
|
|
9
|
+
* Identity map for an input of length `n`: `[0, 1, 2, ..., n-1]`.
|
|
10
|
+
*/
|
|
9
11
|
export declare function identityMap(n: number): number[];
|
|
10
12
|
/**
|
|
11
13
|
* Compose `inputMap` (input → raw) with `transformMap` (output → input) to produce `outputMap` (output → raw).
|
package/out/offset-map.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"offset-map.d.ts","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH
|
|
1
|
+
{"version":3,"file":"offset-map.d.ts","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAShF"}
|
package/out/offset-map.js
CHANGED
|
@@ -5,7 +5,9 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Utilities for composing per-transform offset maps into the final `raw → normalized` map.
|
|
7
7
|
*/
|
|
8
|
-
/**
|
|
8
|
+
/**
|
|
9
|
+
* Identity map for an input of length `n`: `[0, 1, 2, ..., n-1]`.
|
|
10
|
+
*/
|
|
9
11
|
export function identityMap(n) {
|
|
10
12
|
const m = new Array(n);
|
|
11
13
|
for (let i = 0; i < n; i++) {
|
package/out/offset-map.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"offset-map.js","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH
|
|
1
|
+
{"version":3,"file":"offset-map.js","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,CAAS;IACpC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACT,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,YAAsB;IACrE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAS,YAAY,CAAC,MAAM,CAAC,CAAA;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAE,CAAA;QAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAC1B,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
package/out/punctuation.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAiBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,
|
|
1
|
+
{"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAiBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA8BjE"}
|
package/out/punctuation.js
CHANGED
|
@@ -34,6 +34,7 @@ export function applyPunctuation(input) {
|
|
|
34
34
|
else {
|
|
35
35
|
changed = true;
|
|
36
36
|
replacements += 1;
|
|
37
|
+
// oxlint-disable-next-line typescript/prefer-for-of -- code units, so the offset map stays aligned
|
|
37
38
|
for (let k = 0; k < sub.length; k++) {
|
|
38
39
|
out.push(sub[k]);
|
|
39
40
|
map.push(i);
|
package/out/punctuation.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,+EAA+E;IAC3F,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8DAA8D;IAC1E,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,+BAA+B;IAC7C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qBAAqB;CACjC,CAAC,CAAA;AAQF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEhC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,IAAI,CAAC,CAAA;YAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC"}
|
|
1
|
+
{"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,+EAA+E;IAC3F,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8DAA8D;IAC1E,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,+BAA+B;IAC7C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qBAAqB;CACjC,CAAC,CAAA;AAQF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEhC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,IAAI,CAAC,CAAA;YAEjB,mGAAmG;YACnG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC"}
|
package/out/types.d.ts
CHANGED
|
@@ -8,7 +8,9 @@ export interface SpanRange {
|
|
|
8
8
|
end: number;
|
|
9
9
|
body: string;
|
|
10
10
|
}
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* A single normalization step, as recorded on `NormalizedInput.transforms`.
|
|
13
|
+
*/
|
|
12
14
|
export type NormalizationTransform = {
|
|
13
15
|
kind: "nfc";
|
|
14
16
|
changed: boolean;
|
|
@@ -39,25 +41,43 @@ export type NormalizationTransform = {
|
|
|
39
41
|
* convention.
|
|
40
42
|
*/
|
|
41
43
|
export interface NormalizedInput {
|
|
42
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* The input as the caller sent it.
|
|
46
|
+
*/
|
|
43
47
|
raw: string;
|
|
44
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Canonical form, all transforms applied.
|
|
50
|
+
*/
|
|
45
51
|
normalized: string;
|
|
46
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Ordered record of what was done.
|
|
54
|
+
*/
|
|
47
55
|
transforms: NormalizationTransform[];
|
|
48
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* `normalized[i]` came from `raw[offsetMap[i]]`. Length === normalized.length.
|
|
58
|
+
*/
|
|
49
59
|
offsetMap: number[];
|
|
50
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* The locale used for case-folding + abbreviation rules.
|
|
62
|
+
*/
|
|
51
63
|
appliedLocale?: string;
|
|
52
64
|
}
|
|
53
65
|
export interface NormalizeOpts {
|
|
54
|
-
/**
|
|
66
|
+
/**
|
|
67
|
+
* Locale hint for case-folding + abbreviation dictionaries.
|
|
68
|
+
*/
|
|
55
69
|
locale?: string;
|
|
56
|
-
/**
|
|
70
|
+
/**
|
|
71
|
+
* Apply locale-aware lowercasing. Default: false (preserve case for downstream consumers).
|
|
72
|
+
*/
|
|
57
73
|
caseFold?: boolean;
|
|
58
|
-
/**
|
|
74
|
+
/**
|
|
75
|
+
* Expand known abbreviations (`St` → `Street`, `NW` → `Northwest`, etc.). Default: false.
|
|
76
|
+
*/
|
|
59
77
|
expandAbbreviations?: boolean;
|
|
60
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* Skip Unicode NFC. Only use for debugging — production callers should leave on.
|
|
80
|
+
*/
|
|
61
81
|
skipNFC?: boolean;
|
|
62
82
|
}
|
|
63
83
|
//# sourceMappingURL=types.d.ts.map
|
package/out/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACZ;AAED
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAC/B;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,SAAS,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAA;AAE9D;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,UAAU,EAAE,sBAAsB,EAAE,CAAA;IAEpC;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAA;IAEnB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;IAElB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CACjB"}
|
package/out/whitespace.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;
|
|
1
|
+
{"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAYH,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAwElE"}
|
package/out/whitespace.js
CHANGED
|
@@ -14,8 +14,10 @@
|
|
|
14
14
|
import { identityMap } from "./offset-map.js";
|
|
15
15
|
const INLINE_SPACE = /[ \t]/;
|
|
16
16
|
const ANY_SPACE = /[ \t\n\r]/;
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
/**
|
|
18
|
+
* Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user commonly appends.
|
|
19
|
+
* NOT leading (a leading token is load-bearing) and NOT quotes/brackets/parens.
|
|
20
|
+
*/
|
|
19
21
|
const TRAILING_NOISE = /[ \t\n\r.,;:]/;
|
|
20
22
|
export function collapseWhitespace(input) {
|
|
21
23
|
let changed = false;
|
|
@@ -47,7 +49,7 @@ export function collapseWhitespace(input) {
|
|
|
47
49
|
continue;
|
|
48
50
|
}
|
|
49
51
|
// Collapse \r\n into one
|
|
50
|
-
if (ch === "\n" && out
|
|
52
|
+
if (ch === "\n" && out.at(-1) === "\r") {
|
|
51
53
|
// Already handled in CR branch above by emitting both; skip combiner check
|
|
52
54
|
}
|
|
53
55
|
out.push(ch);
|
package/out/whitespace.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,OAAO,CAAA;AAC5B,MAAM,SAAS,GAAG,WAAW,CAAA;AAC7B
|
|
1
|
+
{"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,OAAO,CAAA;AAC5B,MAAM,SAAS,GAAG,WAAW,CAAA;AAC7B;;;GAGG;AACH,MAAM,cAAc,GAAG,eAAe,CAAA;AAQtC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChC,2CAA2C;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,KAAK,GAAG,CAAC,CAAA;YACf,CAAC,IAAI,CAAC,CAAA;YAEN,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACzD,CAAC,IAAI,CAAC,CAAA;YACP,CAAC;YAED,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO,GAAG,IAAI,CAAA;gBACd,IAAI,IAAI,CAAC,CAAA;YACV,CAAC;YAED,SAAQ;QACT,CAAC;QAED,yBAAyB;QACzB,IAAI,EAAE,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,2EAA2E;QAC5E,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,CAAC,IAAI,CAAC,CAAA;IACP,CAAC;IAED,6FAA6F;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,GAAG,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,CAAA;IACV,CAAC;IAED,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;IAEtB,OAAO,KAAK,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;QAC7D,KAAK,IAAI,CAAC,CAAA;IACX,CAAC;IAED,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACpC,OAAO,GAAG,IAAI,CAAA;IACf,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEzC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IAChE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AAC5D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/normalize",
|
|
3
|
-
"version": "8.
|
|
3
|
+
"version": "8.3.0",
|
|
4
4
|
"description": "Stage 1 of the runtime pipeline — deterministic input preprocessing (Unicode NFC, punctuation, whitespace, abbreviation). Pure functions, no ML.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
package/punctuation.ts
CHANGED
|
@@ -45,6 +45,7 @@ export function applyPunctuation(input: string): PunctuationResult {
|
|
|
45
45
|
changed = true
|
|
46
46
|
replacements += 1
|
|
47
47
|
|
|
48
|
+
// oxlint-disable-next-line typescript/prefer-for-of -- code units, so the offset map stays aligned
|
|
48
49
|
for (let k = 0; k < sub.length; k++) {
|
|
49
50
|
out.push(sub[k]!)
|
|
50
51
|
map.push(i)
|
package/types.ts
CHANGED
|
@@ -10,7 +10,9 @@ export interface SpanRange {
|
|
|
10
10
|
body: string
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* A single normalization step, as recorded on `NormalizedInput.transforms`.
|
|
15
|
+
*/
|
|
14
16
|
export type NormalizationTransform =
|
|
15
17
|
| { kind: "nfc"; changed: boolean }
|
|
16
18
|
| { kind: "case_fold"; locale: string }
|
|
@@ -27,32 +29,50 @@ export type NormalizationTransform =
|
|
|
27
29
|
* convention.
|
|
28
30
|
*/
|
|
29
31
|
export interface NormalizedInput {
|
|
30
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* The input as the caller sent it.
|
|
34
|
+
*/
|
|
31
35
|
raw: string
|
|
32
36
|
|
|
33
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Canonical form, all transforms applied.
|
|
39
|
+
*/
|
|
34
40
|
normalized: string
|
|
35
41
|
|
|
36
|
-
/**
|
|
42
|
+
/**
|
|
43
|
+
* Ordered record of what was done.
|
|
44
|
+
*/
|
|
37
45
|
transforms: NormalizationTransform[]
|
|
38
46
|
|
|
39
|
-
/**
|
|
47
|
+
/**
|
|
48
|
+
* `normalized[i]` came from `raw[offsetMap[i]]`. Length === normalized.length.
|
|
49
|
+
*/
|
|
40
50
|
offsetMap: number[]
|
|
41
51
|
|
|
42
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* The locale used for case-folding + abbreviation rules.
|
|
54
|
+
*/
|
|
43
55
|
appliedLocale?: string
|
|
44
56
|
}
|
|
45
57
|
|
|
46
58
|
export interface NormalizeOpts {
|
|
47
|
-
/**
|
|
59
|
+
/**
|
|
60
|
+
* Locale hint for case-folding + abbreviation dictionaries.
|
|
61
|
+
*/
|
|
48
62
|
locale?: string
|
|
49
63
|
|
|
50
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* Apply locale-aware lowercasing. Default: false (preserve case for downstream consumers).
|
|
66
|
+
*/
|
|
51
67
|
caseFold?: boolean
|
|
52
68
|
|
|
53
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* Expand known abbreviations (`St` → `Street`, `NW` → `Northwest`, etc.). Default: false.
|
|
71
|
+
*/
|
|
54
72
|
expandAbbreviations?: boolean
|
|
55
73
|
|
|
56
|
-
/**
|
|
74
|
+
/**
|
|
75
|
+
* Skip Unicode NFC. Only use for debugging — production callers should leave on.
|
|
76
|
+
*/
|
|
57
77
|
skipNFC?: boolean
|
|
58
78
|
}
|
package/whitespace.ts
CHANGED
|
@@ -16,8 +16,10 @@ import { identityMap } from "./offset-map.ts"
|
|
|
16
16
|
|
|
17
17
|
const INLINE_SPACE = /[ \t]/
|
|
18
18
|
const ANY_SPACE = /[ \t\n\r]/
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
/**
|
|
20
|
+
* Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user commonly appends.
|
|
21
|
+
* NOT leading (a leading token is load-bearing) and NOT quotes/brackets/parens.
|
|
22
|
+
*/
|
|
21
23
|
const TRAILING_NOISE = /[ \t\n\r.,;:]/
|
|
22
24
|
|
|
23
25
|
export interface WhitespaceResult {
|
|
@@ -41,6 +43,7 @@ export function collapseWhitespace(input: string): WhitespaceResult {
|
|
|
41
43
|
out.push(ch)
|
|
42
44
|
map.push(i)
|
|
43
45
|
i += 1
|
|
46
|
+
|
|
44
47
|
continue
|
|
45
48
|
}
|
|
46
49
|
|
|
@@ -58,13 +61,15 @@ export function collapseWhitespace(input: string): WhitespaceResult {
|
|
|
58
61
|
changed = true
|
|
59
62
|
runs += 1
|
|
60
63
|
}
|
|
64
|
+
|
|
61
65
|
continue
|
|
62
66
|
}
|
|
63
67
|
|
|
64
68
|
// Collapse \r\n into one
|
|
65
|
-
if (ch === "\n" && out
|
|
69
|
+
if (ch === "\n" && out.at(-1) === "\r") {
|
|
66
70
|
// Already handled in CR branch above by emitting both; skip combiner check
|
|
67
71
|
}
|
|
72
|
+
|
|
68
73
|
out.push(ch)
|
|
69
74
|
map.push(i)
|
|
70
75
|
i += 1
|
|
@@ -76,6 +81,7 @@ export function collapseWhitespace(input: string): WhitespaceResult {
|
|
|
76
81
|
while (lead < out.length && ANY_SPACE.test(out[lead]!)) {
|
|
77
82
|
lead += 1
|
|
78
83
|
}
|
|
84
|
+
|
|
79
85
|
let trail = out.length
|
|
80
86
|
|
|
81
87
|
while (trail > lead && TRAILING_NOISE.test(out[trail - 1]!)) {
|
|
@@ -85,6 +91,7 @@ export function collapseWhitespace(input: string): WhitespaceResult {
|
|
|
85
91
|
if (lead > 0 || trail < out.length) {
|
|
86
92
|
changed = true
|
|
87
93
|
}
|
|
94
|
+
|
|
88
95
|
const trimmedOut = out.slice(lead, trail)
|
|
89
96
|
const trimmedMap = map.slice(lead, trail)
|
|
90
97
|
|