@mailwoman/normalize 5.4.0 → 5.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/abbreviations.d.ts +10 -0
- package/out/abbreviations.d.ts.map +1 -1
- package/out/abbreviations.js +28 -0
- package/out/abbreviations.js.map +1 -1
- package/out/index.d.ts +1 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +1 -1
- package/out/index.js.map +1 -1
- package/out/whitespace.d.ts +5 -1
- package/out/whitespace.d.ts.map +1 -1
- package/out/whitespace.js +10 -3
- package/out/whitespace.js.map +1 -1
- package/package.json +1 -1
package/out/abbreviations.d.ts
CHANGED
|
@@ -11,6 +11,16 @@
|
|
|
11
11
|
* intentionally — refactoring sharing is a separate task.
|
|
12
12
|
*/
|
|
13
13
|
import type { SpanRange } from "./types.js";
|
|
14
|
+
export interface AbbreviationEntry {
|
|
15
|
+
from: string;
|
|
16
|
+
to: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
|
|
20
|
+
* it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
|
|
21
|
+
* "no load-bearing trivia" rule means that data lives in exactly one place — here.
|
|
22
|
+
*/
|
|
23
|
+
export declare function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry>;
|
|
14
24
|
export interface AbbreviationResult {
|
|
15
25
|
text: string;
|
|
16
26
|
map: number[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;
|
|
1
|
+
{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE3C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AAkED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAwDtF"}
|
package/out/abbreviations.js
CHANGED
|
@@ -43,12 +43,40 @@ const FR_FR_DICT = [
|
|
|
43
43
|
{ from: "Imp", to: "Impasse" },
|
|
44
44
|
{ from: "Sq", to: "Square" },
|
|
45
45
|
];
|
|
46
|
+
/**
|
|
47
|
+
* #1002: the locale-UNKNOWN expansion set — the entries safe to apply when the input's locale hasn't been established
|
|
48
|
+
* yet (the geocode path expands BEFORE the parse, which is what determines the locale). Safe = multi-char,
|
|
49
|
+
* collision-free across the locale dictionaries, and never a plausible standalone token in the other locale (FR
|
|
50
|
+
* `Bd`/`Bvd`/`Imp` have no EN reading; `Av` reads Avenue in both). Deliberately EXCLUDED: the FR single letters (`R` →
|
|
51
|
+
* Rue would fire on Washington DC's literal "R St") and the EN suffixes (`St`, `Ave`, `Dr`, … — the model is
|
|
52
|
+
* trained-robust on those, and `St`/`Dr` are ambiguous with Saint/Doctor).
|
|
53
|
+
*/
|
|
54
|
+
const LOCALE_UNKNOWN_DICT = [
|
|
55
|
+
{ from: "Bd", to: "Boulevard" },
|
|
56
|
+
{ from: "Bvd", to: "Boulevard" },
|
|
57
|
+
{ from: "Boul", to: "Boulevard" },
|
|
58
|
+
{ from: "Av", to: "Avenue" },
|
|
59
|
+
{ from: "Imp", to: "Impasse" },
|
|
60
|
+
];
|
|
46
61
|
function getDictionary(locale) {
|
|
47
62
|
const lc = (locale ?? "en-US").toLowerCase();
|
|
63
|
+
// BCP-47 "und" (undetermined) — the caller knows it does NOT know the locale yet (the geocode path
|
|
64
|
+
// expands before the parse). Only the collision-free multi-locale set applies; `undefined` keeps its
|
|
65
|
+
// historical en-US default.
|
|
66
|
+
if (lc === "und")
|
|
67
|
+
return LOCALE_UNKNOWN_DICT;
|
|
48
68
|
if (lc.startsWith("fr"))
|
|
49
69
|
return FR_FR_DICT;
|
|
50
70
|
return EN_US_DICT;
|
|
51
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
|
|
74
|
+
* it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
|
|
75
|
+
* "no load-bearing trivia" rule means that data lives in exactly one place — here.
|
|
76
|
+
*/
|
|
77
|
+
export function abbreviationDictionary(locale) {
|
|
78
|
+
return getDictionary(locale);
|
|
79
|
+
}
|
|
52
80
|
/**
|
|
53
81
|
* Expand known abbreviations. Walks the input token-by-token (whitespace-delimited) and rewrites matching tokens to
|
|
54
82
|
* their canonical long form. The output map points every char of the expanded form to its position in the original
|
package/out/abbreviations.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI;QAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAExE,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YACN,SAAQ;QACT,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YAAE,CAAC,IAAI,CAAC,CAAA;QACzD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YACD,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAAE,CAAC,IAAI,CAAC,CAAA;IACjD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
|
|
1
|
+
{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI;QAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAExE,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YACN,SAAQ;QACT,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YAAE,CAAC,IAAI,CAAC,CAAA;QACzD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YACD,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QACD,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAAE,CAAC,IAAI,CAAC,CAAA;IACjD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
|
package/out/index.d.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* See `docs/articles/plan/reference/STAGES.md` § Stage 1 for the contract.
|
|
14
14
|
*/
|
|
15
|
-
export { expandAbbreviations } from "./abbreviations.js";
|
|
15
|
+
export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "./abbreviations.js";
|
|
16
16
|
export { applyCjkNormalization, type CjkResult } from "./cjk.js";
|
|
17
17
|
export { normalize } from "./compute.js";
|
|
18
18
|
export { applyNFC } from "./nfc.js";
|
package/out/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,KAAK,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxG,OAAO,EAAE,qBAAqB,EAAE,KAAK,SAAS,EAAE,MAAM,UAAU,CAAA;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,YAAY,EAAE,sBAAsB,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACnG,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
|
package/out/index.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* See `docs/articles/plan/reference/STAGES.md` § Stage 1 for the contract.
|
|
14
14
|
*/
|
|
15
|
-
export { expandAbbreviations } from "./abbreviations.js";
|
|
15
|
+
export { abbreviationDictionary, expandAbbreviations } from "./abbreviations.js";
|
|
16
16
|
export { applyCjkNormalization } from "./cjk.js";
|
|
17
17
|
export { normalize } from "./compute.js";
|
|
18
18
|
export { applyNFC } from "./nfc.js";
|
package/out/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAA0B,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxG,OAAO,EAAE,qBAAqB,EAAkB,MAAM,UAAU,CAAA;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
|
package/out/whitespace.d.ts
CHANGED
|
@@ -5,7 +5,11 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Whitespace collapse — runs of whitespace become a single ASCII space. Newlines and tabs are
|
|
7
7
|
* preserved as-is (segmentation grammar in QueryShape uses them); inline runs of spaces
|
|
8
|
-
* collapse.
|
|
8
|
+
* collapse. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
|
|
9
|
+
* trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
|
|
10
|
+
* tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
|
|
11
|
+
* quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
|
|
12
|
+
* as the whitespace trim, so span alignment survives.
|
|
9
13
|
*/
|
|
10
14
|
export interface WhitespaceResult {
|
|
11
15
|
text: string;
|
package/out/whitespace.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAUH,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CA6DlE"}
|
package/out/whitespace.js
CHANGED
|
@@ -5,11 +5,18 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Whitespace collapse — runs of whitespace become a single ASCII space. Newlines and tabs are
|
|
7
7
|
* preserved as-is (segmentation grammar in QueryShape uses them); inline runs of spaces
|
|
8
|
-
* collapse.
|
|
8
|
+
* collapse. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
|
|
9
|
+
* trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
|
|
10
|
+
* tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
|
|
11
|
+
* quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
|
|
12
|
+
* as the whitespace trim, so span alignment survives.
|
|
9
13
|
*/
|
|
10
14
|
import { identityMap } from "./offset-map.js";
|
|
11
15
|
const INLINE_SPACE = /[ \t]/;
|
|
12
16
|
const ANY_SPACE = /[ \t\n\r]/;
|
|
17
|
+
// Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user
|
|
18
|
+
// commonly appends. NOT leading (a leading token is load-bearing) and NOT quotes/brackets/parens.
|
|
19
|
+
const TRAILING_NOISE = /[ \t\n\r.,;:]/;
|
|
13
20
|
export function collapseWhitespace(input) {
|
|
14
21
|
let changed = false;
|
|
15
22
|
let runs = 0;
|
|
@@ -46,12 +53,12 @@ export function collapseWhitespace(input) {
|
|
|
46
53
|
map.push(i);
|
|
47
54
|
i += 1;
|
|
48
55
|
}
|
|
49
|
-
// Trim leading and trailing whitespace.
|
|
56
|
+
// Trim leading whitespace, and trailing whitespace + sentence-punctuation noise (#829 tail).
|
|
50
57
|
let lead = 0;
|
|
51
58
|
while (lead < out.length && ANY_SPACE.test(out[lead]))
|
|
52
59
|
lead += 1;
|
|
53
60
|
let trail = out.length;
|
|
54
|
-
while (trail > lead &&
|
|
61
|
+
while (trail > lead && TRAILING_NOISE.test(out[trail - 1]))
|
|
55
62
|
trail -= 1;
|
|
56
63
|
if (lead > 0 || trail < out.length) {
|
|
57
64
|
changed = true;
|
package/out/whitespace.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,OAAO,CAAA;AAC5B,MAAM,SAAS,GAAG,WAAW,CAAA;AAC7B,qGAAqG;AACrG,kGAAkG;AAClG,MAAM,cAAc,GAAG,eAAe,CAAA;AAQtC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChC,2CAA2C;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YACN,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,KAAK,GAAG,CAAC,CAAA;YACf,CAAC,IAAI,CAAC,CAAA;YAEN,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;gBAAE,CAAC,IAAI,CAAC,CAAA;YAE/D,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO,GAAG,IAAI,CAAA;gBACd,IAAI,IAAI,CAAC,CAAA;YACV,CAAC;YACD,SAAQ;QACT,CAAC;QAED,yBAAyB;QACzB,IAAI,EAAE,KAAK,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACjD,2EAA2E;QAC5E,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,CAAC,IAAI,CAAC,CAAA;IACP,CAAC;IAED,6FAA6F;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,GAAG,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QAAE,IAAI,IAAI,CAAC,CAAA;IACjE,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;IAEtB,OAAO,KAAK,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC;QAAE,KAAK,IAAI,CAAC,CAAA;IAEvE,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACpC,OAAO,GAAG,IAAI,CAAA;IACf,CAAC;IACD,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEzC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IAChE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AAC5D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/normalize",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.6.0",
|
|
4
4
|
"description": "Stage 1 of the runtime pipeline — deterministic input preprocessing (Unicode NFC, punctuation, whitespace, abbreviation). Pure functions, no ML.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|