@mailwoman/normalize 9.2.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/{abbreviations.ts → lib/abbreviations.ts} +3 -3
- package/{cjk.ts → lib/cjk.ts} +5 -4
- package/lib/comma-spacing.ts +68 -0
- package/{compute.ts → lib/compute.ts} +25 -9
- package/lib/fold.ts +30 -0
- package/{index.ts → lib/index.ts} +9 -8
- package/{nfc.ts → lib/nfc.ts} +1 -1
- package/{punctuation.ts → lib/punctuation.ts} +1 -1
- package/{types.ts → lib/types.ts} +8 -0
- package/{whitespace.ts → lib/whitespace.ts} +15 -11
- package/out/abbreviations.d.ts +2 -2
- package/out/abbreviations.d.ts.map +1 -1
- package/out/abbreviations.js +2 -2
- package/out/abbreviations.js.map +1 -1
- package/out/cjk.d.ts +4 -2
- package/out/cjk.d.ts.map +1 -1
- package/out/cjk.js +5 -4
- package/out/cjk.js.map +1 -1
- package/out/comma-spacing.d.ts +30 -0
- package/out/comma-spacing.d.ts.map +1 -0
- package/out/comma-spacing.js +52 -0
- package/out/comma-spacing.js.map +1 -0
- package/out/compute.d.ts +1 -1
- package/out/compute.d.ts.map +1 -1
- package/out/compute.js +22 -8
- package/out/compute.js.map +1 -1
- package/out/fold.d.ts +22 -0
- package/out/fold.d.ts.map +1 -0
- package/out/fold.js +28 -0
- package/out/fold.js.map +1 -0
- package/out/index.d.ts +9 -8
- package/out/index.d.ts.map +1 -1
- package/out/index.js +8 -7
- package/out/index.js.map +1 -1
- package/out/nfc.d.ts.map +1 -1
- package/out/nfc.js +1 -1
- package/out/nfc.js.map +1 -1
- package/out/offset-map.d.ts.map +1 -1
- package/out/offset-map.js.map +1 -1
- package/out/punctuation.d.ts.map +1 -1
- package/out/punctuation.js +1 -1
- package/out/punctuation.js.map +1 -1
- package/out/types.d.ts +10 -0
- package/out/types.d.ts.map +1 -1
- package/out/types.js.map +1 -1
- package/out/whitespace.d.ts +9 -3
- package/out/whitespace.d.ts.map +1 -1
- package/out/whitespace.js +11 -10
- package/out/whitespace.js.map +1 -1
- package/package.json +52 -1
- /package/{offset-map.ts → lib/offset-map.ts} +0 -0
package/README.md
CHANGED
|
@@ -46,7 +46,7 @@ identityMap(length: number): OffsetMap
|
|
|
46
46
|
## Pipeline position
|
|
47
47
|
|
|
48
48
|
```
|
|
49
|
-
raw string → normalize → query-shape → locale-
|
|
49
|
+
raw string → normalize → query-shape → locale-hint → kind-classifier → phrase-grouper → ...
|
|
50
50
|
```
|
|
51
51
|
|
|
52
52
|
Stage 1 in the [Staged Pipeline Contract](https://github.com/sister-software/mailwoman/blob/main/docs/engineering/reference/STAGES.mdx). No runtime dependencies.
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* intentionally — refactoring sharing is a separate task.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import type { SpanRange } from "
|
|
15
|
+
import type { SpanRange } from "#types"
|
|
16
16
|
|
|
17
17
|
export interface AbbreviationEntry {
|
|
18
18
|
from: string // short form (case-insensitive match)
|
|
@@ -80,7 +80,7 @@ const ES_ES_DICT: ReadonlyArray<AbbreviationEntry> = [
|
|
|
80
80
|
* Dropping the entry is NOT a table edit: `fr-op3-halles-market-bonneuil` is a passing row that asserts street "Avenue
|
|
81
81
|
* de la Convention" and an `address_point` tier, so it pins the current behaviour and a removal has to be measured on a
|
|
82
82
|
* resolver-gauntlet run. The real repair is upstream — the geocode path hardcodes `locale: "und"` because Stage 1
|
|
83
|
-
* precedes the parse, and `@mailwoman/locale-
|
|
83
|
+
* precedes the parse, and `@mailwoman/locale-hint` cannot presently detect Spanish (it scores script class + known
|
|
84
84
|
* postcode formats, and a 5-digit ES/MX code is indistinguishable from a US ZIP).
|
|
85
85
|
*/
|
|
86
86
|
const LOCALE_UNKNOWN_DICT: ReadonlyArray<AbbreviationEntry> = [
|
|
@@ -112,7 +112,7 @@ function getDictionary(locale: string | undefined): ReadonlyArray<AbbreviationEn
|
|
|
112
112
|
/**
|
|
113
113
|
* The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
|
|
114
114
|
* it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
|
|
115
|
-
* "no
|
|
115
|
+
* "no required trivia" rule means that data lives in exactly one place — here.
|
|
116
116
|
*/
|
|
117
117
|
export function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry> {
|
|
118
118
|
return getDictionary(locale)
|
package/{cjk.ts → lib/cjk.ts}
RENAMED
|
@@ -26,11 +26,11 @@
|
|
|
26
26
|
* normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
|
|
27
27
|
* deferred.
|
|
28
28
|
*
|
|
29
|
-
* Self-
|
|
29
|
+
* Self-limiting: a string with none of these characters returns identity, so Latin input is
|
|
30
30
|
* untouched.
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
|
-
import { identityMap } from "
|
|
33
|
+
import { identityMap } from "#offset-map"
|
|
34
34
|
|
|
35
35
|
export interface CjkResult {
|
|
36
36
|
text: string
|
|
@@ -70,18 +70,19 @@ function isHalfwidthKatakana(code: number): boolean {
|
|
|
70
70
|
return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
export function applyCjkNormalization(input: string): CjkResult {
|
|
73
|
+
export function applyCjkNormalization(input: string, opts: { postalMark?: "strip" | "keep" } = {}): CjkResult {
|
|
74
74
|
let folded = 0
|
|
75
75
|
let stripped = 0
|
|
76
76
|
const out: string[] = []
|
|
77
77
|
const map: number[] = []
|
|
78
|
+
const stripPostalMark = opts.postalMark !== "keep"
|
|
78
79
|
|
|
79
80
|
// All transformed code points are in the BMP (single UTF-16 unit), and every other character is
|
|
80
81
|
// passed through verbatim, so a per-unit walk is safe for surrogate-pair input too.
|
|
81
82
|
for (let i = 0; i < input.length; i++) {
|
|
82
83
|
const code = input.charCodeAt(i)
|
|
83
84
|
|
|
84
|
-
if (code === POSTAL_MARK) {
|
|
85
|
+
if (code === POSTAL_MARK && stripPostalMark) {
|
|
85
86
|
stripped += 1
|
|
86
87
|
|
|
87
88
|
continue // drop — no addressing content; whitespace collapse later tidies any gap
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
|
|
7
|
+
* reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
|
|
8
|
+
* is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
|
|
9
|
+
* the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
|
|
10
|
+
*
|
|
11
|
+
* A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
|
|
12
|
+
* BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
|
|
13
|
+
* the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
|
|
14
|
+
* comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
|
|
15
|
+
* expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { identityMap } from "#offset-map"
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Any letter or decimal digit in any script: the Unicode property classes `\p{L}` and `\p{Nd}`.
|
|
22
|
+
*/
|
|
23
|
+
const LETTER_OR_DIGIT = /[\p{L}\p{Nd}]/u
|
|
24
|
+
const DIGIT = /\p{Nd}/u
|
|
25
|
+
|
|
26
|
+
export interface CommaSpacingResult {
|
|
27
|
+
text: string
|
|
28
|
+
map: number[]
|
|
29
|
+
/**
|
|
30
|
+
* How many spaces were inserted.
|
|
31
|
+
*/
|
|
32
|
+
inserted: number
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* A comma with a decimal digit on both sides: `12,5`, `1,000`.
|
|
37
|
+
*/
|
|
38
|
+
function isNumericSeparator(input: string, commaIndex: number): boolean {
|
|
39
|
+
return commaIndex > 0 && DIGIT.test(input[commaIndex - 1]!) && DIGIT.test(input[commaIndex + 1]!)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
|
|
44
|
+
* separator. Offset-map-correct: the inserted space maps to the comma.
|
|
45
|
+
*/
|
|
46
|
+
export function spaceAfterComma(input: string): CommaSpacingResult {
|
|
47
|
+
let inserted = 0
|
|
48
|
+
const out: string[] = []
|
|
49
|
+
const map: number[] = []
|
|
50
|
+
|
|
51
|
+
for (let i = 0; i < input.length; i++) {
|
|
52
|
+
const ch = input[i]!
|
|
53
|
+
out.push(ch)
|
|
54
|
+
map.push(i)
|
|
55
|
+
|
|
56
|
+
if (ch === "," && i + 1 < input.length && LETTER_OR_DIGIT.test(input[i + 1]!) && !isNumericSeparator(input, i)) {
|
|
57
|
+
out.push(" ")
|
|
58
|
+
map.push(i)
|
|
59
|
+
inserted += 1
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (!inserted) {
|
|
64
|
+
return { text: input, map: identityMap(input.length), inserted: 0 }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return { text: out.join(""), map, inserted }
|
|
68
|
+
}
|
|
@@ -7,13 +7,14 @@
|
|
|
7
7
|
* (always) with case-fold + abbreviation expansion (opt-in).
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
-
import { expandAbbreviations } from "
|
|
11
|
-
import { applyCjkNormalization } from "
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import
|
|
16
|
-
import {
|
|
10
|
+
import { expandAbbreviations } from "#abbreviations"
|
|
11
|
+
import { applyCjkNormalization } from "#cjk"
|
|
12
|
+
import { spaceAfterComma } from "#comma-spacing"
|
|
13
|
+
import { applyNFC } from "#nfc"
|
|
14
|
+
import { composeMaps, identityMap } from "#offset-map"
|
|
15
|
+
import { applyPunctuation } from "#punctuation"
|
|
16
|
+
import type { NormalizationTransform, NormalizedInput, NormalizeOpts } from "#types"
|
|
17
|
+
import { collapseWhitespace } from "#whitespace"
|
|
17
18
|
|
|
18
19
|
export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
|
|
19
20
|
const transforms: NormalizationTransform[] = []
|
|
@@ -32,7 +33,7 @@ export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
|
|
|
32
33
|
// parse) and fold full-width ASCII + the ideographic space. Runs after NFC so it sees composed
|
|
33
34
|
// forms, before punctuation/whitespace so any gap left by 〒 is then collapsed. No-op off-script.
|
|
34
35
|
{
|
|
35
|
-
const r = applyCjkNormalization(text)
|
|
36
|
+
const r = applyCjkNormalization(text, opts?.postalMark ? { postalMark: opts.postalMark } : {})
|
|
36
37
|
|
|
37
38
|
if (r.folded > 0 || r.stripped > 0) {
|
|
38
39
|
text = r.text
|
|
@@ -52,11 +53,26 @@ export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
|
|
|
52
53
|
}
|
|
53
54
|
}
|
|
54
55
|
|
|
56
|
+
// 2.5 Comma spacing — a comma glued to a letter gains a space. Runs after punctuation (so a folded
|
|
57
|
+
// full-width comma is seen) and before whitespace collapse (so a comma already followed by a space is
|
|
58
|
+
// never doubled).
|
|
59
|
+
{
|
|
60
|
+
const r = spaceAfterComma(text)
|
|
61
|
+
|
|
62
|
+
if (r.inserted > 0) {
|
|
63
|
+
text = r.text
|
|
64
|
+
map = composeMaps(map, r.map)
|
|
65
|
+
transforms.push({ kind: "space_after_comma", inserted: r.inserted })
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
55
69
|
// 3. Whitespace
|
|
56
70
|
{
|
|
57
71
|
const r = collapseWhitespace(text)
|
|
58
72
|
|
|
59
|
-
|
|
73
|
+
// Compare the TEXT, not its length: folding a lone tab to a space is length-preserving, and a
|
|
74
|
+
// length test reads that edit as no edit at all.
|
|
75
|
+
if (r.text !== text) {
|
|
60
76
|
text = r.text
|
|
61
77
|
map = composeMaps(map, r.map)
|
|
62
78
|
transforms.push({ kind: "collapse_whitespace", runs: r.runs })
|
package/lib/fold.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Match-key folds: the loss-y transforms a comparison applies to both sides.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Lower-case, whitespace runs collapsed to one space, ends trimmed. The fold every string comparison applies before it
|
|
10
|
+
* compares; it keeps punctuation and diacritics, so `Saint-Étienne` and `Saint-Etienne` stay distinct.
|
|
11
|
+
*/
|
|
12
|
+
export function foldCaseWhitespace(input: string): string {
|
|
13
|
+
return input.toLowerCase().replaceAll(/\s+/gu, " ").trim()
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Combining marks removed after NFD decomposition: `é` → `e`, `ł` unchanged (it is not a base plus a mark). Case and
|
|
18
|
+
* whitespace are untouched.
|
|
19
|
+
*/
|
|
20
|
+
export function stripCombiningMarks(input: string): string {
|
|
21
|
+
return input.normalize("NFD").replaceAll(/\p{M}/gu, "")
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* {@link foldCaseWhitespace} over the NFKC composition, with locale-aware lower-casing: fullwidth and compatibility
|
|
26
|
+
* forms fold together before comparison.
|
|
27
|
+
*/
|
|
28
|
+
export function foldNFKCWhitespace(input: string): string {
|
|
29
|
+
return input.normalize("NFKC").toLocaleLowerCase().replaceAll(/\s+/gu, " ").trim()
|
|
30
|
+
}
|
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
* See `docs/engineering/reference/STAGES.md` § Stage 1 for the contract.
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "
|
|
17
|
-
export { applyCjkNormalization, type CjkResult } from "
|
|
18
|
-
export { normalize } from "
|
|
19
|
-
export {
|
|
20
|
-
export {
|
|
21
|
-
export {
|
|
22
|
-
export
|
|
23
|
-
export {
|
|
16
|
+
export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "#abbreviations"
|
|
17
|
+
export { applyCjkNormalization, type CjkResult } from "#cjk"
|
|
18
|
+
export { normalize } from "#compute"
|
|
19
|
+
export { foldCaseWhitespace, foldNFKCWhitespace, stripCombiningMarks } from "#fold"
|
|
20
|
+
export { applyNFC } from "#nfc"
|
|
21
|
+
export { composeMaps, identityMap } from "#offset-map"
|
|
22
|
+
export { applyPunctuation } from "#punctuation"
|
|
23
|
+
export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "#types"
|
|
24
|
+
export { collapseWhitespace } from "#whitespace"
|
package/{nfc.ts → lib/nfc.ts}
RENAMED
|
@@ -19,6 +19,7 @@ export type NormalizationTransform =
|
|
|
19
19
|
| { kind: "expand_abbreviation"; from: string; to: string; at: SpanRange }
|
|
20
20
|
| { kind: "collapse_whitespace"; runs: number }
|
|
21
21
|
| { kind: "normalize_punctuation"; replacements: number }
|
|
22
|
+
| { kind: "space_after_comma"; inserted: number }
|
|
22
23
|
| { kind: "normalize_cjk"; folded: number; stripped: number }
|
|
23
24
|
|
|
24
25
|
/**
|
|
@@ -75,4 +76,11 @@ export interface NormalizeOpts {
|
|
|
75
76
|
* Skip Unicode NFC. Only use for debugging — production callers should leave on.
|
|
76
77
|
*/
|
|
77
78
|
skipNFC?: boolean
|
|
79
|
+
/**
|
|
80
|
+
* What the CJK pass does with the postal mark 〒 (U+3012). `strip` (the default) drops it: it is byte-fallback OOV for
|
|
81
|
+
* the SentencePiece tokenizer and fragments the digits after it. `keep` leaves it in place for a classifier whose
|
|
82
|
+
* vocabulary carries it — the character-path CJK model was trained with the mark in front of every postcode, and
|
|
83
|
+
* without it misreads the prefecture boundary (`885-0061 宮崎県都城市…` → prefecture `崎県都`).
|
|
84
|
+
*/
|
|
85
|
+
postalMark?: "strip" | "keep"
|
|
78
86
|
}
|
|
@@ -3,28 +3,34 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* Whitespace collapse —
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* Whitespace collapse — a run of inline whitespace (`[ \t]`) becomes one ASCII space AT EVERY RUN
|
|
7
|
+
* LENGTH, so a lone tab normalizes exactly as a doubled one does. Newlines (`\n`/`\r`) are preserved:
|
|
8
|
+
* QueryShape's segmentation grammar reads one as a segment separator on a par with a comma, and it
|
|
9
|
+
* reads a RAW tab the same way — folding every tab here is what keeps a tab-separated export from
|
|
10
|
+
* re-segmenting the query. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
|
|
9
11
|
* trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
|
|
10
12
|
* tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
|
|
11
13
|
* quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
|
|
12
14
|
* as the whitespace trim, so span alignment survives.
|
|
13
15
|
*/
|
|
14
16
|
|
|
15
|
-
import { identityMap } from "
|
|
17
|
+
import { identityMap } from "#offset-map"
|
|
16
18
|
|
|
17
19
|
const INLINE_SPACE = /[ \t]/
|
|
18
20
|
const ANY_SPACE = /[ \t\n\r]/
|
|
19
21
|
/**
|
|
20
22
|
* Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user commonly appends.
|
|
21
|
-
* NOT leading (a leading token is
|
|
23
|
+
* NOT leading (a leading token is required) and NOT quotes/brackets/parens.
|
|
22
24
|
*/
|
|
23
25
|
const TRAILING_NOISE = /[ \t\n\r.,;:]/
|
|
24
26
|
|
|
25
27
|
export interface WhitespaceResult {
|
|
26
28
|
text: string
|
|
27
29
|
map: number[]
|
|
30
|
+
/**
|
|
31
|
+
* How many inline-whitespace runs were REWRITTEN — a run longer than one character, or a one-character run that was
|
|
32
|
+
* not already an ASCII space. A run that was already a single space is not one of them.
|
|
33
|
+
*/
|
|
28
34
|
runs: number
|
|
29
35
|
}
|
|
30
36
|
|
|
@@ -57,7 +63,10 @@ export function collapseWhitespace(input: string): WhitespaceResult {
|
|
|
57
63
|
i += 1
|
|
58
64
|
}
|
|
59
65
|
|
|
60
|
-
|
|
66
|
+
// A one-character run counts too when the character is not already an ASCII space: the tab→space
|
|
67
|
+
// rewrite emitted above is a real edit, and `changed` is what decides whether the caller ever
|
|
68
|
+
// receives it — the early return below hands back the untouched input otherwise.
|
|
69
|
+
if (i - start > 1 || ch !== " ") {
|
|
61
70
|
changed = true
|
|
62
71
|
runs += 1
|
|
63
72
|
}
|
|
@@ -65,11 +74,6 @@ export function collapseWhitespace(input: string): WhitespaceResult {
|
|
|
65
74
|
continue
|
|
66
75
|
}
|
|
67
76
|
|
|
68
|
-
// Collapse \r\n into one
|
|
69
|
-
if (ch === "\n" && out.at(-1) === "\r") {
|
|
70
|
-
// Already handled in CR branch above by emitting both; skip combiner check
|
|
71
|
-
}
|
|
72
|
-
|
|
73
77
|
out.push(ch)
|
|
74
78
|
map.push(i)
|
|
75
79
|
i += 1
|
package/out/abbreviations.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* augmentation). Both sides should eventually share dictionaries; for v1 this dict is duplicated
|
|
12
12
|
* intentionally — refactoring sharing is a separate task.
|
|
13
13
|
*/
|
|
14
|
-
import type { SpanRange } from "
|
|
14
|
+
import type { SpanRange } from "#types";
|
|
15
15
|
export interface AbbreviationEntry {
|
|
16
16
|
from: string;
|
|
17
17
|
to: string;
|
|
@@ -19,7 +19,7 @@ export interface AbbreviationEntry {
|
|
|
19
19
|
/**
|
|
20
20
|
* The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
|
|
21
21
|
* it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
|
|
22
|
-
* "no
|
|
22
|
+
* "no required trivia" rule means that data lives in exactly one place — here.
|
|
23
23
|
*/
|
|
24
24
|
export declare function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry>;
|
|
25
25
|
export interface AbbreviationResult {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../lib/abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAEvC,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AA4FD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAmEtF"}
|
package/out/abbreviations.js
CHANGED
|
@@ -69,7 +69,7 @@ const ES_ES_DICT = [
|
|
|
69
69
|
* Dropping the entry is NOT a table edit: `fr-op3-halles-market-bonneuil` is a passing row that asserts street "Avenue
|
|
70
70
|
* de la Convention" and an `address_point` tier, so it pins the current behaviour and a removal has to be measured on a
|
|
71
71
|
* resolver-gauntlet run. The real repair is upstream — the geocode path hardcodes `locale: "und"` because Stage 1
|
|
72
|
-
* precedes the parse, and `@mailwoman/locale-
|
|
72
|
+
* precedes the parse, and `@mailwoman/locale-hint` cannot presently detect Spanish (it scores script class + known
|
|
73
73
|
* postcode formats, and a 5-digit ES/MX code is indistinguishable from a US ZIP).
|
|
74
74
|
*/
|
|
75
75
|
const LOCALE_UNKNOWN_DICT = [
|
|
@@ -98,7 +98,7 @@ function getDictionary(locale) {
|
|
|
98
98
|
/**
|
|
99
99
|
* The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
|
|
100
100
|
* it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
|
|
101
|
-
* "no
|
|
101
|
+
* "no required trivia" rule means that data lives in exactly one place — here.
|
|
102
102
|
*/
|
|
103
103
|
export function abbreviationDictionary(locale) {
|
|
104
104
|
return getDictionary(locale);
|
package/out/abbreviations.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,kGAAkG;IAClG,8FAA8F;IAC9F,iGAAiG;IACjG,2FAA2F;IAC3F,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,+FAA+F;IAC/F,oGAAoG;IACpG,qGAAqG;IACrG,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YAED,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1C,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
|
|
1
|
+
{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../lib/abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,kGAAkG;IAClG,8FAA8F;IAC9F,iGAAiG;IACjG,2FAA2F;IAC3F,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,+FAA+F;IAC/F,oGAAoG;IACpG,qGAAqG;IACrG,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YAED,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1C,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
|
package/out/cjk.d.ts
CHANGED
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
|
|
27
27
|
* deferred.
|
|
28
28
|
*
|
|
29
|
-
* Self-
|
|
29
|
+
* Self-limiting: a string with none of these characters returns identity, so Latin input is
|
|
30
30
|
* untouched.
|
|
31
31
|
*/
|
|
32
32
|
export interface CjkResult {
|
|
@@ -41,5 +41,7 @@ export interface CjkResult {
|
|
|
41
41
|
*/
|
|
42
42
|
stripped: number;
|
|
43
43
|
}
|
|
44
|
-
export declare function applyCjkNormalization(input: string
|
|
44
|
+
export declare function applyCjkNormalization(input: string, opts?: {
|
|
45
|
+
postalMark?: "strip" | "keep";
|
|
46
|
+
}): CjkResult;
|
|
45
47
|
//# sourceMappingURL=cjk.d.ts.map
|
package/out/cjk.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AA2BD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,
|
|
1
|
+
{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../lib/cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AA2BD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAAO,GAAG,SAAS,CAmE5G"}
|
package/out/cjk.js
CHANGED
|
@@ -26,10 +26,10 @@
|
|
|
26
26
|
* normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
|
|
27
27
|
* deferred.
|
|
28
28
|
*
|
|
29
|
-
* Self-
|
|
29
|
+
* Self-limiting: a string with none of these characters returns identity, so Latin input is
|
|
30
30
|
* untouched.
|
|
31
31
|
*/
|
|
32
|
-
import { identityMap } from "
|
|
32
|
+
import { identityMap } from "#offset-map";
|
|
33
33
|
/**
|
|
34
34
|
* !.
|
|
35
35
|
*/
|
|
@@ -53,16 +53,17 @@ const HALFWIDTH_VOICING_START = 0xff_9e;
|
|
|
53
53
|
function isHalfwidthKatakana(code) {
|
|
54
54
|
return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END;
|
|
55
55
|
}
|
|
56
|
-
export function applyCjkNormalization(input) {
|
|
56
|
+
export function applyCjkNormalization(input, opts = {}) {
|
|
57
57
|
let folded = 0;
|
|
58
58
|
let stripped = 0;
|
|
59
59
|
const out = [];
|
|
60
60
|
const map = [];
|
|
61
|
+
const stripPostalMark = opts.postalMark !== "keep";
|
|
61
62
|
// All transformed code points are in the BMP (single UTF-16 unit), and every other character is
|
|
62
63
|
// passed through verbatim, so a per-unit walk is safe for surrogate-pair input too.
|
|
63
64
|
for (let i = 0; i < input.length; i++) {
|
|
64
65
|
const code = input.charCodeAt(i);
|
|
65
|
-
if (code === POSTAL_MARK) {
|
|
66
|
+
if (code === POSTAL_MARK && stripPostalMark) {
|
|
66
67
|
stripped += 1;
|
|
67
68
|
continue; // drop — no addressing content; whitespace collapse later tidies any gap
|
|
68
69
|
}
|
package/out/cjk.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../lib/cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAezC;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAC3B,MAAM,wBAAwB,GAAG,OAAO,CAAA;AACxC,MAAM,sBAAsB,GAAG,OAAO,CAAA;AACtC,MAAM,uBAAuB,GAAG,OAAO,CAAA;AAEvC,SAAS,mBAAmB,CAAC,IAAY;IACxC,OAAO,IAAI,IAAI,wBAAwB,IAAI,IAAI,IAAI,sBAAsB,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAa,EAAE,OAA0C,EAAE;IAChG,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,KAAK,MAAM,CAAA;IAElD,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,IAAI,eAAe,EAAE,CAAC;YAC7C,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEpC,MAAM,mBAAmB,GACxB,IAAI,GAAG,uBAAuB,IAAI,IAAI,IAAI,uBAAuB,IAAI,IAAI,IAAI,sBAAsB,CAAA;YAEpG,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAEzC,MAAM,UAAU,GAAG,KAAK;iBACtB,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;iBACnB,SAAS,CAAC,MAAM,CAAC;iBACjB,SAAS,CAAC,KAAK,CAAC,CAAA;YAElB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC/B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;YAED,MAAM,IAAI,KAAK,CAAA;YACf,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;YAEd,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
|
|
7
|
+
* reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
|
|
8
|
+
* is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
|
|
9
|
+
* the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
|
|
10
|
+
*
|
|
11
|
+
* A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
|
|
12
|
+
* BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
|
|
13
|
+
* the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
|
|
14
|
+
* comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
|
|
15
|
+
* expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
|
|
16
|
+
*/
|
|
17
|
+
export interface CommaSpacingResult {
|
|
18
|
+
text: string;
|
|
19
|
+
map: number[];
|
|
20
|
+
/**
|
|
21
|
+
* How many spaces were inserted.
|
|
22
|
+
*/
|
|
23
|
+
inserted: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
|
|
27
|
+
* separator. Offset-map-correct: the inserted space maps to the comma.
|
|
28
|
+
*/
|
|
29
|
+
export declare function spaceAfterComma(input: string): CommaSpacingResult;
|
|
30
|
+
//# sourceMappingURL=comma-spacing.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"comma-spacing.d.ts","sourceRoot":"","sources":["../lib/comma-spacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAUH,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AASD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAsBjE"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
|
|
7
|
+
* reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
|
|
8
|
+
* is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
|
|
9
|
+
* the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
|
|
10
|
+
*
|
|
11
|
+
* A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
|
|
12
|
+
* BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
|
|
13
|
+
* the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
|
|
14
|
+
* comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
|
|
15
|
+
* expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
|
|
16
|
+
*/
|
|
17
|
+
import { identityMap } from "#offset-map";
|
|
18
|
+
/**
|
|
19
|
+
* Any letter or decimal digit in any script: the Unicode property classes `\p{L}` and `\p{Nd}`.
|
|
20
|
+
*/
|
|
21
|
+
const LETTER_OR_DIGIT = /[\p{L}\p{Nd}]/u;
|
|
22
|
+
const DIGIT = /\p{Nd}/u;
|
|
23
|
+
/**
|
|
24
|
+
* A comma with a decimal digit on both sides: `12,5`, `1,000`.
|
|
25
|
+
*/
|
|
26
|
+
function isNumericSeparator(input, commaIndex) {
|
|
27
|
+
return commaIndex > 0 && DIGIT.test(input[commaIndex - 1]) && DIGIT.test(input[commaIndex + 1]);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
|
|
31
|
+
* separator. Offset-map-correct: the inserted space maps to the comma.
|
|
32
|
+
*/
|
|
33
|
+
export function spaceAfterComma(input) {
|
|
34
|
+
let inserted = 0;
|
|
35
|
+
const out = [];
|
|
36
|
+
const map = [];
|
|
37
|
+
for (let i = 0; i < input.length; i++) {
|
|
38
|
+
const ch = input[i];
|
|
39
|
+
out.push(ch);
|
|
40
|
+
map.push(i);
|
|
41
|
+
if (ch === "," && i + 1 < input.length && LETTER_OR_DIGIT.test(input[i + 1]) && !isNumericSeparator(input, i)) {
|
|
42
|
+
out.push(" ");
|
|
43
|
+
map.push(i);
|
|
44
|
+
inserted += 1;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (!inserted) {
|
|
48
|
+
return { text: input, map: identityMap(input.length), inserted: 0 };
|
|
49
|
+
}
|
|
50
|
+
return { text: out.join(""), map, inserted };
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=comma-spacing.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"comma-spacing.js","sourceRoot":"","sources":["../lib/comma-spacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC;;GAEG;AACH,MAAM,eAAe,GAAG,gBAAgB,CAAA;AACxC,MAAM,KAAK,GAAG,SAAS,CAAA;AAWvB;;GAEG;AACH,SAAS,kBAAkB,CAAC,KAAa,EAAE,UAAkB;IAC5D,OAAO,UAAU,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAE,CAAC,CAAA;AAClG,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC5C,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAEX,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC;YAChH,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,QAAQ,IAAI,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IACpE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAA;AAC7C,CAAC"}
|
package/out/compute.d.ts
CHANGED
|
@@ -6,6 +6,6 @@
|
|
|
6
6
|
* `normalize(raw, opts)` — the Stage 1 entry point. Composes NFC + punctuation + whitespace
|
|
7
7
|
* (always) with case-fold + abbreviation expansion (opt-in).
|
|
8
8
|
*/
|
|
9
|
-
import type { NormalizedInput, NormalizeOpts } from "
|
|
9
|
+
import type { NormalizedInput, NormalizeOpts } from "#types";
|
|
10
10
|
export declare function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput;
|
|
11
11
|
//# sourceMappingURL=compute.d.ts.map
|
package/out/compute.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compute.d.ts","sourceRoot":"","sources":["../compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;
|
|
1
|
+
{"version":3,"file":"compute.d.ts","sourceRoot":"","sources":["../lib/compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAQH,OAAO,KAAK,EAA0B,eAAe,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAGpF,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,eAAe,CAgG5E"}
|
package/out/compute.js
CHANGED
|
@@ -6,12 +6,13 @@
|
|
|
6
6
|
* `normalize(raw, opts)` — the Stage 1 entry point. Composes NFC + punctuation + whitespace
|
|
7
7
|
* (always) with case-fold + abbreviation expansion (opt-in).
|
|
8
8
|
*/
|
|
9
|
-
import { expandAbbreviations } from "
|
|
10
|
-
import { applyCjkNormalization } from "
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
9
|
+
import { expandAbbreviations } from "#abbreviations";
|
|
10
|
+
import { applyCjkNormalization } from "#cjk";
|
|
11
|
+
import { spaceAfterComma } from "#comma-spacing";
|
|
12
|
+
import { applyNFC } from "#nfc";
|
|
13
|
+
import { composeMaps, identityMap } from "#offset-map";
|
|
14
|
+
import { applyPunctuation } from "#punctuation";
|
|
15
|
+
import { collapseWhitespace } from "#whitespace";
|
|
15
16
|
export function normalize(raw, opts) {
|
|
16
17
|
const transforms = [];
|
|
17
18
|
let text = raw;
|
|
@@ -27,7 +28,7 @@ export function normalize(raw, opts) {
|
|
|
27
28
|
// parse) and fold full-width ASCII + the ideographic space. Runs after NFC so it sees composed
|
|
28
29
|
// forms, before punctuation/whitespace so any gap left by 〒 is then collapsed. No-op off-script.
|
|
29
30
|
{
|
|
30
|
-
const r = applyCjkNormalization(text);
|
|
31
|
+
const r = applyCjkNormalization(text, opts?.postalMark ? { postalMark: opts.postalMark } : {});
|
|
31
32
|
if (r.folded > 0 || r.stripped > 0) {
|
|
32
33
|
text = r.text;
|
|
33
34
|
map = composeMaps(map, r.map);
|
|
@@ -43,10 +44,23 @@ export function normalize(raw, opts) {
|
|
|
43
44
|
transforms.push({ kind: "normalize_punctuation", replacements: r.replacements });
|
|
44
45
|
}
|
|
45
46
|
}
|
|
47
|
+
// 2.5 Comma spacing — a comma glued to a letter gains a space. Runs after punctuation (so a folded
|
|
48
|
+
// full-width comma is seen) and before whitespace collapse (so a comma already followed by a space is
|
|
49
|
+
// never doubled).
|
|
50
|
+
{
|
|
51
|
+
const r = spaceAfterComma(text);
|
|
52
|
+
if (r.inserted > 0) {
|
|
53
|
+
text = r.text;
|
|
54
|
+
map = composeMaps(map, r.map);
|
|
55
|
+
transforms.push({ kind: "space_after_comma", inserted: r.inserted });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
46
58
|
// 3. Whitespace
|
|
47
59
|
{
|
|
48
60
|
const r = collapseWhitespace(text);
|
|
49
|
-
|
|
61
|
+
// Compare the TEXT, not its length: folding a lone tab to a space is length-preserving, and a
|
|
62
|
+
// length test reads that edit as no edit at all.
|
|
63
|
+
if (r.text !== text) {
|
|
50
64
|
text = r.text;
|
|
51
65
|
map = composeMaps(map, r.map);
|
|
52
66
|
transforms.push({ kind: "collapse_whitespace", runs: r.runs });
|
package/out/compute.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compute.js","sourceRoot":"","sources":["../compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"compute.js","sourceRoot":"","sources":["../lib/compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,MAAM,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAEhD,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,IAAoB;IAC1D,MAAM,UAAU,GAA6B,EAAE,CAAA;IAC/C,IAAI,IAAI,GAAG,GAAG,CAAA;IACd,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEjC,SAAS;IACT,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;QACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;QAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,+FAA+F;IAC/F,+FAA+F;IAC/F,iGAAiG;IACjG,CAAC;QACA,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAE9F,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;QACnF,CAAC;IACF,CAAC;IAED,iBAAiB;IACjB,CAAC;QACA,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAEhC,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAA;QACjF,CAAC;IACF,CAAC;IAED,mGAAmG;IACnG,sGAAsG;IACtG,kBAAkB;IAClB,CAAC;QACA,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;QAE/B,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;QACrE,CAAC;IACF,CAAC;IAED,gBAAgB;IAChB,CAAC;QACA,MAAM,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;QAElC,8FAA8F;QAC9F,iDAAiD;QACjD,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACrB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,2FAA2F;IAC3F,gEAAgE;IAChE,IAAI,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhD,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAE7B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YACnF,CAAC;QACF,CAAC;IACF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAE9C,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,EAAE,CAAA;YACT,sEAAsE;YACtE,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAA;QACrE,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC;QACpB,GAAG;QACH,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAA6B;QACjE,SAAS,EAAE,GAAG;QACd,aAAa,EAAE,IAAI,EAAE,MAAM;KAC3B,CAA2B,CAAA;AAC7B,CAAC"}
|
package/out/fold.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Match-key folds: the loss-y transforms a comparison applies to both sides.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Lower-case, whitespace runs collapsed to one space, ends trimmed. The fold every string comparison applies before it
|
|
9
|
+
* compares; it keeps punctuation and diacritics, so `Saint-Étienne` and `Saint-Etienne` stay distinct.
|
|
10
|
+
*/
|
|
11
|
+
export declare function foldCaseWhitespace(input: string): string;
|
|
12
|
+
/**
|
|
13
|
+
* Combining marks removed after NFD decomposition: `é` → `e`, `ł` unchanged (it is not a base plus a mark). Case and
|
|
14
|
+
* whitespace are untouched.
|
|
15
|
+
*/
|
|
16
|
+
export declare function stripCombiningMarks(input: string): string;
|
|
17
|
+
/**
|
|
18
|
+
* {@link foldCaseWhitespace} over the NFKC composition, with locale-aware lower-casing: fullwidth and compatibility
|
|
19
|
+
* forms fold together before comparison.
|
|
20
|
+
*/
|
|
21
|
+
export declare function foldNFKCWhitespace(input: string): string;
|
|
22
|
+
//# sourceMappingURL=fold.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fold.d.ts","sourceRoot":"","sources":["../lib/fold.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAExD"}
|
package/out/fold.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
* @file Match-key folds: the loss-y transforms a comparison applies to both sides.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Lower-case, whitespace runs collapsed to one space, ends trimmed. The fold every string comparison applies before it
|
|
9
|
+
* compares; it keeps punctuation and diacritics, so `Saint-Étienne` and `Saint-Etienne` stay distinct.
|
|
10
|
+
*/
|
|
11
|
+
export function foldCaseWhitespace(input) {
|
|
12
|
+
return input.toLowerCase().replaceAll(/\s+/gu, " ").trim();
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Combining marks removed after NFD decomposition: `é` → `e`, `ł` unchanged (it is not a base plus a mark). Case and
|
|
16
|
+
* whitespace are untouched.
|
|
17
|
+
*/
|
|
18
|
+
export function stripCombiningMarks(input) {
|
|
19
|
+
return input.normalize("NFD").replaceAll(/\p{M}/gu, "");
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* {@link foldCaseWhitespace} over the NFKC composition, with locale-aware lower-casing: fullwidth and compatibility
|
|
23
|
+
* forms fold together before comparison.
|
|
24
|
+
*/
|
|
25
|
+
export function foldNFKCWhitespace(input) {
|
|
26
|
+
return input.normalize("NFKC").toLocaleLowerCase().replaceAll(/\s+/gu, " ").trim();
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=fold.js.map
|
package/out/fold.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fold.js","sourceRoot":"","sources":["../lib/fold.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAChD,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,OAAO,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACnF,CAAC"}
|
package/out/index.d.ts
CHANGED
|
@@ -12,12 +12,13 @@
|
|
|
12
12
|
*
|
|
13
13
|
* See `docs/engineering/reference/STAGES.md` § Stage 1 for the contract.
|
|
14
14
|
*/
|
|
15
|
-
export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "
|
|
16
|
-
export { applyCjkNormalization, type CjkResult } from "
|
|
17
|
-
export { normalize } from "
|
|
18
|
-
export {
|
|
19
|
-
export {
|
|
20
|
-
export {
|
|
21
|
-
export
|
|
22
|
-
export {
|
|
15
|
+
export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "#abbreviations";
|
|
16
|
+
export { applyCjkNormalization, type CjkResult } from "#cjk";
|
|
17
|
+
export { normalize } from "#compute";
|
|
18
|
+
export { foldCaseWhitespace, foldNFKCWhitespace, stripCombiningMarks } from "#fold";
|
|
19
|
+
export { applyNFC } from "#nfc";
|
|
20
|
+
export { composeMaps, identityMap } from "#offset-map";
|
|
21
|
+
export { applyPunctuation } from "#punctuation";
|
|
22
|
+
export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "#types";
|
|
23
|
+
export { collapseWhitespace } from "#whitespace";
|
|
23
24
|
//# sourceMappingURL=index.d.ts.map
|
package/out/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,KAAK,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,KAAK,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpG,OAAO,EAAE,qBAAqB,EAAE,KAAK,SAAS,EAAE,MAAM,MAAM,CAAA;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAA;AACnF,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC/C,YAAY,EAAE,sBAAsB,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAC/F,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA"}
|
package/out/index.js
CHANGED
|
@@ -12,11 +12,12 @@
|
|
|
12
12
|
*
|
|
13
13
|
* See `docs/engineering/reference/STAGES.md` § Stage 1 for the contract.
|
|
14
14
|
*/
|
|
15
|
-
export { abbreviationDictionary, expandAbbreviations } from "
|
|
16
|
-
export { applyCjkNormalization } from "
|
|
17
|
-
export { normalize } from "
|
|
18
|
-
export {
|
|
19
|
-
export {
|
|
20
|
-
export {
|
|
21
|
-
export {
|
|
15
|
+
export { abbreviationDictionary, expandAbbreviations } from "#abbreviations";
|
|
16
|
+
export { applyCjkNormalization } from "#cjk";
|
|
17
|
+
export { normalize } from "#compute";
|
|
18
|
+
export { foldCaseWhitespace, foldNFKCWhitespace, stripCombiningMarks } from "#fold";
|
|
19
|
+
export { applyNFC } from "#nfc";
|
|
20
|
+
export { composeMaps, identityMap } from "#offset-map";
|
|
21
|
+
export { applyPunctuation } from "#punctuation";
|
|
22
|
+
export { collapseWhitespace } from "#whitespace";
|
|
22
23
|
//# sourceMappingURL=index.js.map
|
package/out/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAA0B,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAA0B,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpG,OAAO,EAAE,qBAAqB,EAAkB,MAAM,MAAM,CAAA;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAA;AACnF,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA"}
|
package/out/nfc.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nfc.d.ts","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;CAChB;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAQjD"}
|
|
1
|
+
{"version":3,"file":"nfc.d.ts","sourceRoot":"","sources":["../lib/nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;CAChB;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAQjD"}
|
package/out/nfc.js
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* map each output index to the start of its source sequence. Rare CJK edge cases involving
|
|
12
12
|
* variant selectors may produce off-by-one offsets — acceptable for v1.
|
|
13
13
|
*/
|
|
14
|
-
import { identityMap } from "
|
|
14
|
+
import { identityMap } from "#offset-map";
|
|
15
15
|
export function applyNFC(input) {
|
|
16
16
|
const normalized = input.normalize("NFC");
|
|
17
17
|
if (normalized === input) {
|
package/out/nfc.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nfc.js","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"nfc.js","sourceRoot":"","sources":["../lib/nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAWzC,MAAM,UAAU,QAAQ,CAAC,KAAa;IACrC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAEzC,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;IACvE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,KAAa,EAAE,MAAc;IACpD,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAE,CAAA;QACzC,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAEvC,gGAAgG;QAChG,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;YACtC,KAAK,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/B,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;gBAExC,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC5C,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACP,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
package/out/offset-map.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"offset-map.d.ts","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAShF"}
|
|
1
|
+
{"version":3,"file":"offset-map.d.ts","sourceRoot":"","sources":["../lib/offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAShF"}
|
package/out/offset-map.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"offset-map.js","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,CAAS;IACpC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACT,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,YAAsB;IACrE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAS,YAAY,CAAC,MAAM,CAAC,CAAA;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAE,CAAA;QAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAC1B,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
|
1
|
+
{"version":3,"file":"offset-map.js","sourceRoot":"","sources":["../lib/offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,CAAS;IACpC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACT,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,YAAsB;IACrE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAS,YAAY,CAAC,MAAM,CAAC,CAAA;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAE,CAAA;QAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAC1B,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
package/out/punctuation.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAmBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA8BjE"}
|
|
1
|
+
{"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../lib/punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAmBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA8BjE"}
|
package/out/punctuation.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* Punctuation normalization — fancy quotes / dashes to ASCII equivalents. Identity-length: every
|
|
7
7
|
* fancy character is a single codepoint that maps to a single ASCII char.
|
|
8
8
|
*/
|
|
9
|
-
import { identityMap } from "
|
|
9
|
+
import { identityMap } from "#offset-map";
|
|
10
10
|
const REPLACEMENTS = new Map([
|
|
11
11
|
["‘", "'"], // ‘
|
|
12
12
|
["’", "'"], // ’
|
package/out/punctuation.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../lib/punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,kBAAkB;IAC9B,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,+EAA+E;IAC3F,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8DAA8D;IAC1E,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8BAA8B;IAC1C,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,+BAA+B;IAC7C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qBAAqB;CACjC,CAAC,CAAA;AAQF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEhC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,IAAI,CAAC,CAAA;YAEjB,mGAAmG;YACnG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC"}
|
package/out/types.d.ts
CHANGED
|
@@ -28,6 +28,9 @@ export type NormalizationTransform = {
|
|
|
28
28
|
} | {
|
|
29
29
|
kind: "normalize_punctuation";
|
|
30
30
|
replacements: number;
|
|
31
|
+
} | {
|
|
32
|
+
kind: "space_after_comma";
|
|
33
|
+
inserted: number;
|
|
31
34
|
} | {
|
|
32
35
|
kind: "normalize_cjk";
|
|
33
36
|
folded: number;
|
|
@@ -79,5 +82,12 @@ export interface NormalizeOpts {
|
|
|
79
82
|
* Skip Unicode NFC. Only use for debugging — production callers should leave on.
|
|
80
83
|
*/
|
|
81
84
|
skipNFC?: boolean;
|
|
85
|
+
/**
|
|
86
|
+
* What the CJK pass does with the postal mark 〒 (U+3012). `strip` (the default) drops it: it is byte-fallback OOV for
|
|
87
|
+
* the SentencePiece tokenizer and fragments the digits after it. `keep` leaves it in place for a classifier whose
|
|
88
|
+
* vocabulary carries it — the character-path CJK model was trained with the mark in front of every postcode, and
|
|
89
|
+
* without it misreads the prefecture boundary (`885-0061 宮崎県都城市…` → prefecture `崎県都`).
|
|
90
|
+
*/
|
|
91
|
+
postalMark?: "strip" | "keep";
|
|
82
92
|
}
|
|
83
93
|
//# sourceMappingURL=types.d.ts.map
|
package/out/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAC/B;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,SAAS,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAA;AAE9D;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,UAAU,EAAE,sBAAsB,EAAE,CAAA;IAEpC;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAA;IAEnB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;IAElB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAC/B;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,SAAS,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAA;AAE9D;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,UAAU,EAAE,sBAAsB,EAAE,CAAA;IAEpC;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAA;IAEnB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;IAElB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC7B"}
|
package/out/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
package/out/whitespace.d.ts
CHANGED
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* Whitespace collapse —
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* Whitespace collapse — a run of inline whitespace (`[ \t]`) becomes one ASCII space AT EVERY RUN
|
|
7
|
+
* LENGTH, so a lone tab normalizes exactly as a doubled one does. Newlines (`\n`/`\r`) are preserved:
|
|
8
|
+
* QueryShape's segmentation grammar reads one as a segment separator on a par with a comma, and it
|
|
9
|
+
* reads a RAW tab the same way — folding every tab here is what keeps a tab-separated export from
|
|
10
|
+
* re-segmenting the query. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
|
|
9
11
|
* trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
|
|
10
12
|
* tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
|
|
11
13
|
* quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
|
|
@@ -14,6 +16,10 @@
|
|
|
14
16
|
export interface WhitespaceResult {
|
|
15
17
|
text: string;
|
|
16
18
|
map: number[];
|
|
19
|
+
/**
|
|
20
|
+
* How many inline-whitespace runs were REWRITTEN — a run longer than one character, or a one-character run that was
|
|
21
|
+
* not already an ASCII space. A run that was already a single space is not one of them.
|
|
22
|
+
*/
|
|
17
23
|
runs: number;
|
|
18
24
|
}
|
|
19
25
|
export declare function collapseWhitespace(input: string): WhitespaceResult;
|
package/out/whitespace.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../lib/whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAYH,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAsElE"}
|
package/out/whitespace.js
CHANGED
|
@@ -3,20 +3,22 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* Whitespace collapse —
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* Whitespace collapse — a run of inline whitespace (`[ \t]`) becomes one ASCII space AT EVERY RUN
|
|
7
|
+
* LENGTH, so a lone tab normalizes exactly as a doubled one does. Newlines (`\n`/`\r`) are preserved:
|
|
8
|
+
* QueryShape's segmentation grammar reads one as a segment separator on a par with a comma, and it
|
|
9
|
+
* reads a RAW tab the same way — folding every tab here is what keeps a tab-separated export from
|
|
10
|
+
* re-segmenting the query. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
|
|
9
11
|
* trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
|
|
10
12
|
* tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
|
|
11
13
|
* quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
|
|
12
14
|
* as the whitespace trim, so span alignment survives.
|
|
13
15
|
*/
|
|
14
|
-
import { identityMap } from "
|
|
16
|
+
import { identityMap } from "#offset-map";
|
|
15
17
|
const INLINE_SPACE = /[ \t]/;
|
|
16
18
|
const ANY_SPACE = /[ \t\n\r]/;
|
|
17
19
|
/**
|
|
18
20
|
* Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user commonly appends.
|
|
19
|
-
* NOT leading (a leading token is
|
|
21
|
+
* NOT leading (a leading token is required) and NOT quotes/brackets/parens.
|
|
20
22
|
*/
|
|
21
23
|
const TRAILING_NOISE = /[ \t\n\r.,;:]/;
|
|
22
24
|
export function collapseWhitespace(input) {
|
|
@@ -42,16 +44,15 @@ export function collapseWhitespace(input) {
|
|
|
42
44
|
while (i < input.length && INLINE_SPACE.test(input[i])) {
|
|
43
45
|
i += 1;
|
|
44
46
|
}
|
|
45
|
-
|
|
47
|
+
// A one-character run counts too when the character is not already an ASCII space: the tab→space
|
|
48
|
+
// rewrite emitted above is a real edit, and `changed` is what decides whether the caller ever
|
|
49
|
+
// receives it — the early return below hands back the untouched input otherwise.
|
|
50
|
+
if (i - start > 1 || ch !== " ") {
|
|
46
51
|
changed = true;
|
|
47
52
|
runs += 1;
|
|
48
53
|
}
|
|
49
54
|
continue;
|
|
50
55
|
}
|
|
51
|
-
// Collapse \r\n into one
|
|
52
|
-
if (ch === "\n" && out.at(-1) === "\r") {
|
|
53
|
-
// Already handled in CR branch above by emitting both; skip combiner check
|
|
54
|
-
}
|
|
55
56
|
out.push(ch);
|
|
56
57
|
map.push(i);
|
|
57
58
|
i += 1;
|
package/out/whitespace.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../lib/whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,MAAM,YAAY,GAAG,OAAO,CAAA;AAC5B,MAAM,SAAS,GAAG,WAAW,CAAA;AAC7B;;;GAGG;AACH,MAAM,cAAc,GAAG,eAAe,CAAA;AAYtC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChC,2CAA2C;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,KAAK,GAAG,CAAC,CAAA;YACf,CAAC,IAAI,CAAC,CAAA;YAEN,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACzD,CAAC,IAAI,CAAC,CAAA;YACP,CAAC;YAED,iGAAiG;YACjG,8FAA8F;YAC9F,iFAAiF;YACjF,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,OAAO,GAAG,IAAI,CAAA;gBACd,IAAI,IAAI,CAAC,CAAA;YACV,CAAC;YAED,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,CAAC,IAAI,CAAC,CAAA;IACP,CAAC;IAED,6FAA6F;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,GAAG,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,CAAA;IACV,CAAC;IAED,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;IAEtB,OAAO,KAAK,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;QAC7D,KAAK,IAAI,CAAC,CAAA;IACX,CAAC;IAED,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACpC,OAAO,GAAG,IAAI,CAAA;IACf,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEzC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IAChE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AAC5D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/normalize",
|
|
3
|
-
"version": "9.
|
|
3
|
+
"version": "9.3.0",
|
|
4
4
|
"description": "Stage 1 of the runtime pipeline — deterministic input preprocessing (Unicode NFC, punctuation, whitespace, abbreviation). Pure functions, no ML.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
@@ -24,38 +24,64 @@
|
|
|
24
24
|
"!test/**"
|
|
25
25
|
],
|
|
26
26
|
"type": "module",
|
|
27
|
+
"sideEffects": false,
|
|
28
|
+
"imports": {
|
|
29
|
+
"#*": {
|
|
30
|
+
"types": "./out/*.d.ts",
|
|
31
|
+
"node": "./out/*.js",
|
|
32
|
+
"default": "./out/*.js"
|
|
33
|
+
}
|
|
34
|
+
},
|
|
27
35
|
"exports": {
|
|
28
36
|
"./package.json": "./package.json",
|
|
29
37
|
".": {
|
|
30
38
|
"types": "./out/index.d.ts",
|
|
39
|
+
"node": "./out/index.js",
|
|
31
40
|
"default": "./out/index.js"
|
|
32
41
|
},
|
|
33
42
|
"./abbreviations": {
|
|
34
43
|
"types": "./out/abbreviations.d.ts",
|
|
44
|
+
"node": "./out/abbreviations.js",
|
|
35
45
|
"default": "./out/abbreviations.js"
|
|
36
46
|
},
|
|
37
47
|
"./cjk": {
|
|
38
48
|
"types": "./out/cjk.d.ts",
|
|
49
|
+
"node": "./out/cjk.js",
|
|
39
50
|
"default": "./out/cjk.js"
|
|
40
51
|
},
|
|
41
52
|
"./compute": {
|
|
42
53
|
"types": "./out/compute.d.ts",
|
|
54
|
+
"node": "./out/compute.js",
|
|
43
55
|
"default": "./out/compute.js"
|
|
44
56
|
},
|
|
45
57
|
"./nfc": {
|
|
46
58
|
"types": "./out/nfc.d.ts",
|
|
59
|
+
"node": "./out/nfc.js",
|
|
47
60
|
"default": "./out/nfc.js"
|
|
48
61
|
},
|
|
49
62
|
"./offset-map": {
|
|
50
63
|
"types": "./out/offset-map.d.ts",
|
|
64
|
+
"node": "./out/offset-map.js",
|
|
51
65
|
"default": "./out/offset-map.js"
|
|
52
66
|
},
|
|
53
67
|
"./punctuation": {
|
|
54
68
|
"types": "./out/punctuation.d.ts",
|
|
69
|
+
"node": "./out/punctuation.js",
|
|
55
70
|
"default": "./out/punctuation.js"
|
|
56
71
|
},
|
|
72
|
+
"./comma-spacing": {
|
|
73
|
+
"types": "./out/comma-spacing.d.ts",
|
|
74
|
+
"node": "./out/comma-spacing.js",
|
|
75
|
+
"default": "./out/comma-spacing.js"
|
|
76
|
+
},
|
|
77
|
+
"./fold": {
|
|
78
|
+
"types": "./out/fold.d.ts",
|
|
79
|
+
"node": "./out/fold.js",
|
|
80
|
+
"default": "./out/fold.js"
|
|
81
|
+
},
|
|
57
82
|
"./whitespace": {
|
|
58
83
|
"types": "./out/whitespace.d.ts",
|
|
84
|
+
"node": "./out/whitespace.js",
|
|
59
85
|
"default": "./out/whitespace.js"
|
|
60
86
|
}
|
|
61
87
|
},
|
|
@@ -65,36 +91,61 @@
|
|
|
65
91
|
"./package.json": "./package.json",
|
|
66
92
|
".": {
|
|
67
93
|
"types": "./out/index.d.ts",
|
|
94
|
+
"node": "./out/index.js",
|
|
68
95
|
"default": "./out/index.js"
|
|
69
96
|
},
|
|
70
97
|
"./abbreviations": {
|
|
71
98
|
"types": "./out/abbreviations.d.ts",
|
|
99
|
+
"node": "./out/abbreviations.js",
|
|
72
100
|
"default": "./out/abbreviations.js"
|
|
73
101
|
},
|
|
74
102
|
"./cjk": {
|
|
75
103
|
"types": "./out/cjk.d.ts",
|
|
104
|
+
"node": "./out/cjk.js",
|
|
76
105
|
"default": "./out/cjk.js"
|
|
77
106
|
},
|
|
78
107
|
"./compute": {
|
|
79
108
|
"types": "./out/compute.d.ts",
|
|
109
|
+
"node": "./out/compute.js",
|
|
80
110
|
"default": "./out/compute.js"
|
|
81
111
|
},
|
|
82
112
|
"./nfc": {
|
|
83
113
|
"types": "./out/nfc.d.ts",
|
|
114
|
+
"node": "./out/nfc.js",
|
|
84
115
|
"default": "./out/nfc.js"
|
|
85
116
|
},
|
|
86
117
|
"./offset-map": {
|
|
87
118
|
"types": "./out/offset-map.d.ts",
|
|
119
|
+
"node": "./out/offset-map.js",
|
|
88
120
|
"default": "./out/offset-map.js"
|
|
89
121
|
},
|
|
90
122
|
"./punctuation": {
|
|
91
123
|
"types": "./out/punctuation.d.ts",
|
|
124
|
+
"node": "./out/punctuation.js",
|
|
92
125
|
"default": "./out/punctuation.js"
|
|
93
126
|
},
|
|
127
|
+
"./comma-spacing": {
|
|
128
|
+
"types": "./out/comma-spacing.d.ts",
|
|
129
|
+
"node": "./out/comma-spacing.js",
|
|
130
|
+
"default": "./out/comma-spacing.js"
|
|
131
|
+
},
|
|
132
|
+
"./fold": {
|
|
133
|
+
"types": "./out/fold.d.ts",
|
|
134
|
+
"node": "./out/fold.js",
|
|
135
|
+
"default": "./out/fold.js"
|
|
136
|
+
},
|
|
94
137
|
"./whitespace": {
|
|
95
138
|
"types": "./out/whitespace.d.ts",
|
|
139
|
+
"node": "./out/whitespace.js",
|
|
96
140
|
"default": "./out/whitespace.js"
|
|
97
141
|
}
|
|
142
|
+
},
|
|
143
|
+
"imports": {
|
|
144
|
+
"#*": {
|
|
145
|
+
"types": "./out/*.d.ts",
|
|
146
|
+
"node": "./out/*.js",
|
|
147
|
+
"default": "./out/*.js"
|
|
148
|
+
}
|
|
98
149
|
}
|
|
99
150
|
}
|
|
100
151
|
}
|
|
File without changes
|