@mailwoman/normalize 9.2.0 → 9.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +1 -1
  2. package/{abbreviations.ts → lib/abbreviations.ts} +3 -3
  3. package/{cjk.ts → lib/cjk.ts} +5 -4
  4. package/lib/comma-spacing.ts +68 -0
  5. package/{compute.ts → lib/compute.ts} +25 -9
  6. package/lib/fold.ts +30 -0
  7. package/{index.ts → lib/index.ts} +9 -8
  8. package/{nfc.ts → lib/nfc.ts} +1 -1
  9. package/{punctuation.ts → lib/punctuation.ts} +1 -1
  10. package/{types.ts → lib/types.ts} +8 -0
  11. package/{whitespace.ts → lib/whitespace.ts} +15 -11
  12. package/out/abbreviations.d.ts +2 -2
  13. package/out/abbreviations.d.ts.map +1 -1
  14. package/out/abbreviations.js +2 -2
  15. package/out/abbreviations.js.map +1 -1
  16. package/out/cjk.d.ts +4 -2
  17. package/out/cjk.d.ts.map +1 -1
  18. package/out/cjk.js +5 -4
  19. package/out/cjk.js.map +1 -1
  20. package/out/comma-spacing.d.ts +30 -0
  21. package/out/comma-spacing.d.ts.map +1 -0
  22. package/out/comma-spacing.js +52 -0
  23. package/out/comma-spacing.js.map +1 -0
  24. package/out/compute.d.ts +1 -1
  25. package/out/compute.d.ts.map +1 -1
  26. package/out/compute.js +22 -8
  27. package/out/compute.js.map +1 -1
  28. package/out/fold.d.ts +22 -0
  29. package/out/fold.d.ts.map +1 -0
  30. package/out/fold.js +28 -0
  31. package/out/fold.js.map +1 -0
  32. package/out/index.d.ts +9 -8
  33. package/out/index.d.ts.map +1 -1
  34. package/out/index.js +8 -7
  35. package/out/index.js.map +1 -1
  36. package/out/nfc.d.ts.map +1 -1
  37. package/out/nfc.js +1 -1
  38. package/out/nfc.js.map +1 -1
  39. package/out/offset-map.d.ts.map +1 -1
  40. package/out/offset-map.js.map +1 -1
  41. package/out/punctuation.d.ts.map +1 -1
  42. package/out/punctuation.js +1 -1
  43. package/out/punctuation.js.map +1 -1
  44. package/out/types.d.ts +10 -0
  45. package/out/types.d.ts.map +1 -1
  46. package/out/types.js.map +1 -1
  47. package/out/whitespace.d.ts +9 -3
  48. package/out/whitespace.d.ts.map +1 -1
  49. package/out/whitespace.js +11 -10
  50. package/out/whitespace.js.map +1 -1
  51. package/package.json +52 -1
  52. /package/{offset-map.ts → lib/offset-map.ts} +0 -0
package/README.md CHANGED
@@ -46,7 +46,7 @@ identityMap(length: number): OffsetMap
46
46
  ## Pipeline position
47
47
 
48
48
  ```
49
- raw string → normalize → query-shape → locale-gate → kind-classifier → phrase-grouper → ...
49
+ raw string → normalize → query-shape → locale-hint → kind-classifier → phrase-grouper → ...
50
50
  ```
51
51
 
52
52
  Stage 1 in the [Staged Pipeline Contract](https://github.com/sister-software/mailwoman/blob/main/docs/engineering/reference/STAGES.mdx). No runtime dependencies.
@@ -12,7 +12,7 @@
12
12
  * intentionally — refactoring sharing is a separate task.
13
13
  */
14
14
 
15
- import type { SpanRange } from "./types.ts"
15
+ import type { SpanRange } from "#types"
16
16
 
17
17
  export interface AbbreviationEntry {
18
18
  from: string // short form (case-insensitive match)
@@ -80,7 +80,7 @@ const ES_ES_DICT: ReadonlyArray<AbbreviationEntry> = [
80
80
  * Dropping the entry is NOT a table edit: `fr-op3-halles-market-bonneuil` is a passing row that asserts street "Avenue
81
81
  * de la Convention" and an `address_point` tier, so it pins the current behaviour and a removal has to be measured on a
82
82
  * resolver-gauntlet run. The real repair is upstream — the geocode path hardcodes `locale: "und"` because Stage 1
83
- * precedes the parse, and `@mailwoman/locale-gate` cannot presently detect Spanish (it scores script class + known
83
+ * precedes the parse, and `@mailwoman/locale-hint` cannot presently detect Spanish (it scores script class + known
84
84
  * postcode formats, and a 5-digit ES/MX code is indistinguishable from a US ZIP).
85
85
  */
86
86
  const LOCALE_UNKNOWN_DICT: ReadonlyArray<AbbreviationEntry> = [
@@ -112,7 +112,7 @@ function getDictionary(locale: string | undefined): ReadonlyArray<AbbreviationEn
112
112
  /**
113
113
  * The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
114
114
  * it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
115
- * "no load-bearing trivia" rule means that data lives in exactly one place — here.
115
+ * "no required trivia" rule means that data lives in exactly one place — here.
116
116
  */
117
117
  export function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry> {
118
118
  return getDictionary(locale)
@@ -26,11 +26,11 @@
26
26
  * normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
27
27
  * deferred.
28
28
  *
29
- * Self-gating: a string with none of these characters returns identity, so Latin input is
29
+ * Self-limiting: a string with none of these characters returns identity, so Latin input is
30
30
  * untouched.
31
31
  */
32
32
 
33
- import { identityMap } from "./offset-map.ts"
33
+ import { identityMap } from "#offset-map"
34
34
 
35
35
  export interface CjkResult {
36
36
  text: string
@@ -70,18 +70,19 @@ function isHalfwidthKatakana(code: number): boolean {
70
70
  return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END
71
71
  }
72
72
 
73
- export function applyCjkNormalization(input: string): CjkResult {
73
+ export function applyCjkNormalization(input: string, opts: { postalMark?: "strip" | "keep" } = {}): CjkResult {
74
74
  let folded = 0
75
75
  let stripped = 0
76
76
  const out: string[] = []
77
77
  const map: number[] = []
78
+ const stripPostalMark = opts.postalMark !== "keep"
78
79
 
79
80
  // All transformed code points are in the BMP (single UTF-16 unit), and every other character is
80
81
  // passed through verbatim, so a per-unit walk is safe for surrogate-pair input too.
81
82
  for (let i = 0; i < input.length; i++) {
82
83
  const code = input.charCodeAt(i)
83
84
 
84
- if (code === POSTAL_MARK) {
85
+ if (code === POSTAL_MARK && stripPostalMark) {
85
86
  stripped += 1
86
87
 
87
88
  continue // drop — no addressing content; whitespace collapse later tidies any gap
@@ -0,0 +1,68 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
7
+ * reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
8
+ * is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
9
+ * the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
10
+ *
11
+ * A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
12
+ * BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
13
+ * the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
14
+ * comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
15
+ * expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
16
+ */
17
+
18
+ import { identityMap } from "#offset-map"
19
+
20
+ /**
21
+ * Any letter or decimal digit in any script: the Unicode property classes `\p{L}` and `\p{Nd}`.
22
+ */
23
+ const LETTER_OR_DIGIT = /[\p{L}\p{Nd}]/u
24
+ const DIGIT = /\p{Nd}/u
25
+
26
+ export interface CommaSpacingResult {
27
+ text: string
28
+ map: number[]
29
+ /**
30
+ * How many spaces were inserted.
31
+ */
32
+ inserted: number
33
+ }
34
+
35
+ /**
36
+ * A comma with a decimal digit on both sides: `12,5`, `1,000`.
37
+ */
38
+ function isNumericSeparator(input: string, commaIndex: number): boolean {
39
+ return commaIndex > 0 && DIGIT.test(input[commaIndex - 1]!) && DIGIT.test(input[commaIndex + 1]!)
40
+ }
41
+
42
+ /**
43
+ * Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
44
+ * separator. Offset-map-correct: the inserted space maps to the comma.
45
+ */
46
+ export function spaceAfterComma(input: string): CommaSpacingResult {
47
+ let inserted = 0
48
+ const out: string[] = []
49
+ const map: number[] = []
50
+
51
+ for (let i = 0; i < input.length; i++) {
52
+ const ch = input[i]!
53
+ out.push(ch)
54
+ map.push(i)
55
+
56
+ if (ch === "," && i + 1 < input.length && LETTER_OR_DIGIT.test(input[i + 1]!) && !isNumericSeparator(input, i)) {
57
+ out.push(" ")
58
+ map.push(i)
59
+ inserted += 1
60
+ }
61
+ }
62
+
63
+ if (!inserted) {
64
+ return { text: input, map: identityMap(input.length), inserted: 0 }
65
+ }
66
+
67
+ return { text: out.join(""), map, inserted }
68
+ }
@@ -7,13 +7,14 @@
7
7
  * (always) with case-fold + abbreviation expansion (opt-in).
8
8
  */
9
9
 
10
- import { expandAbbreviations } from "./abbreviations.ts"
11
- import { applyCjkNormalization } from "./cjk.ts"
12
- import { applyNFC } from "./nfc.ts"
13
- import { composeMaps, identityMap } from "./offset-map.ts"
14
- import { applyPunctuation } from "./punctuation.ts"
15
- import type { NormalizationTransform, NormalizedInput, NormalizeOpts } from "./types.ts"
16
- import { collapseWhitespace } from "./whitespace.ts"
10
+ import { expandAbbreviations } from "#abbreviations"
11
+ import { applyCjkNormalization } from "#cjk"
12
+ import { spaceAfterComma } from "#comma-spacing"
13
+ import { applyNFC } from "#nfc"
14
+ import { composeMaps, identityMap } from "#offset-map"
15
+ import { applyPunctuation } from "#punctuation"
16
+ import type { NormalizationTransform, NormalizedInput, NormalizeOpts } from "#types"
17
+ import { collapseWhitespace } from "#whitespace"
17
18
 
18
19
  export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
19
20
  const transforms: NormalizationTransform[] = []
@@ -32,7 +33,7 @@ export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
32
33
  // parse) and fold full-width ASCII + the ideographic space. Runs after NFC so it sees composed
33
34
  // forms, before punctuation/whitespace so any gap left by 〒 is then collapsed. No-op off-script.
34
35
  {
35
- const r = applyCjkNormalization(text)
36
+ const r = applyCjkNormalization(text, opts?.postalMark ? { postalMark: opts.postalMark } : {})
36
37
 
37
38
  if (r.folded > 0 || r.stripped > 0) {
38
39
  text = r.text
@@ -52,11 +53,26 @@ export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
52
53
  }
53
54
  }
54
55
 
56
+ // 2.5 Comma spacing — a comma glued to a letter gains a space. Runs after punctuation (so a folded
57
+ // full-width comma is seen) and before whitespace collapse (so a comma already followed by a space is
58
+ // never doubled).
59
+ {
60
+ const r = spaceAfterComma(text)
61
+
62
+ if (r.inserted > 0) {
63
+ text = r.text
64
+ map = composeMaps(map, r.map)
65
+ transforms.push({ kind: "space_after_comma", inserted: r.inserted })
66
+ }
67
+ }
68
+
55
69
  // 3. Whitespace
56
70
  {
57
71
  const r = collapseWhitespace(text)
58
72
 
59
- if (r.runs > 0 || r.text.length !== text.length) {
73
+ // Compare the TEXT, not its length: folding a lone tab to a space is length-preserving, and a
74
+ // length test reads that edit as no edit at all.
75
+ if (r.text !== text) {
60
76
  text = r.text
61
77
  map = composeMaps(map, r.map)
62
78
  transforms.push({ kind: "collapse_whitespace", runs: r.runs })
package/lib/fold.ts ADDED
@@ -0,0 +1,30 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file Match-key folds: the loss-y transforms a comparison applies to both sides.
6
+ */
7
+
8
+ /**
9
+ * Lower-case, whitespace runs collapsed to one space, ends trimmed. The fold every string comparison applies before it
10
+ * compares; it keeps punctuation and diacritics, so `Saint-Étienne` and `Saint-Etienne` stay distinct.
11
+ */
12
+ export function foldCaseWhitespace(input: string): string {
13
+ return input.toLowerCase().replaceAll(/\s+/gu, " ").trim()
14
+ }
15
+
16
+ /**
17
+ * Combining marks removed after NFD decomposition: `é` → `e`, `ł` unchanged (it is not a base plus a mark). Case and
18
+ * whitespace are untouched.
19
+ */
20
+ export function stripCombiningMarks(input: string): string {
21
+ return input.normalize("NFD").replaceAll(/\p{M}/gu, "")
22
+ }
23
+
24
+ /**
25
+ * {@link foldCaseWhitespace} over the NFKC composition, with locale-aware lower-casing: fullwidth and compatibility
26
+ * forms fold together before comparison.
27
+ */
28
+ export function foldNFKCWhitespace(input: string): string {
29
+ return input.normalize("NFKC").toLocaleLowerCase().replaceAll(/\s+/gu, " ").trim()
30
+ }
@@ -13,11 +13,12 @@
13
13
  * See `docs/engineering/reference/STAGES.md` § Stage 1 for the contract.
14
14
  */
15
15
 
16
- export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "./abbreviations.ts"
17
- export { applyCjkNormalization, type CjkResult } from "./cjk.ts"
18
- export { normalize } from "./compute.ts"
19
- export { applyNFC } from "./nfc.ts"
20
- export { composeMaps, identityMap } from "./offset-map.ts"
21
- export { applyPunctuation } from "./punctuation.ts"
22
- export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "./types.ts"
23
- export { collapseWhitespace } from "./whitespace.ts"
16
+ export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "#abbreviations"
17
+ export { applyCjkNormalization, type CjkResult } from "#cjk"
18
+ export { normalize } from "#compute"
19
+ export { foldCaseWhitespace, foldNFKCWhitespace, stripCombiningMarks } from "#fold"
20
+ export { applyNFC } from "#nfc"
21
+ export { composeMaps, identityMap } from "#offset-map"
22
+ export { applyPunctuation } from "#punctuation"
23
+ export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "#types"
24
+ export { collapseWhitespace } from "#whitespace"
@@ -12,7 +12,7 @@
12
12
  * variant selectors may produce off-by-one offsets — acceptable for v1.
13
13
  */
14
14
 
15
- import { identityMap } from "./offset-map.ts"
15
+ import { identityMap } from "#offset-map"
16
16
 
17
17
  export interface NFCResult {
18
18
  text: string
@@ -7,7 +7,7 @@
7
7
  * fancy character is a single codepoint that maps to a single ASCII char.
8
8
  */
9
9
 
10
- import { identityMap } from "./offset-map.ts"
10
+ import { identityMap } from "#offset-map"
11
11
 
12
12
  const REPLACEMENTS = new Map<string, string>([
13
13
  ["‘", "'"], // ‘
@@ -19,6 +19,7 @@ export type NormalizationTransform =
19
19
  | { kind: "expand_abbreviation"; from: string; to: string; at: SpanRange }
20
20
  | { kind: "collapse_whitespace"; runs: number }
21
21
  | { kind: "normalize_punctuation"; replacements: number }
22
+ | { kind: "space_after_comma"; inserted: number }
22
23
  | { kind: "normalize_cjk"; folded: number; stripped: number }
23
24
 
24
25
  /**
@@ -75,4 +76,11 @@ export interface NormalizeOpts {
75
76
  * Skip Unicode NFC. Only use for debugging — production callers should leave on.
76
77
  */
77
78
  skipNFC?: boolean
79
+ /**
80
+ * What the CJK pass does with the postal mark 〒 (U+3012). `strip` (the default) drops it: it is byte-fallback OOV for
81
+ * the SentencePiece tokenizer and fragments the digits after it. `keep` leaves it in place for a classifier whose
82
+ * vocabulary carries it — the character-path CJK model was trained with the mark in front of every postcode, and
83
+ * without it misreads the prefecture boundary (`885-0061 宮崎県都城市…` → prefecture `崎県都`).
84
+ */
85
+ postalMark?: "strip" | "keep"
78
86
  }
@@ -3,28 +3,34 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Whitespace collapse — runs of whitespace become a single ASCII space. Newlines and tabs are
7
- * preserved as-is (segmentation grammar in QueryShape uses them); inline runs of spaces
8
- * collapse. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
6
+ * Whitespace collapse — a run of inline whitespace (`[ \t]`) becomes one ASCII space AT EVERY RUN
7
+ * LENGTH, so a lone tab normalizes exactly as a doubled one does. Newlines (`\n`/`\r`) are preserved:
8
+ * QueryShape's segmentation grammar reads one as a segment separator on a par with a comma, and it
9
+ * reads a RAW tab the same way — folding every tab here is what keeps a tab-separated export from
10
+ * re-segmenting the query. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
9
11
  * trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
10
12
  * tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
11
13
  * quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
12
14
  * as the whitespace trim, so span alignment survives.
13
15
  */
14
16
 
15
- import { identityMap } from "./offset-map.ts"
17
+ import { identityMap } from "#offset-map"
16
18
 
17
19
  const INLINE_SPACE = /[ \t]/
18
20
  const ANY_SPACE = /[ \t\n\r]/
19
21
  /**
20
22
  * Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user commonly appends.
21
- * NOT leading (a leading token is load-bearing) and NOT quotes/brackets/parens.
23
+ * NOT leading (a leading token is required) and NOT quotes/brackets/parens.
22
24
  */
23
25
  const TRAILING_NOISE = /[ \t\n\r.,;:]/
24
26
 
25
27
  export interface WhitespaceResult {
26
28
  text: string
27
29
  map: number[]
30
+ /**
31
+ * How many inline-whitespace runs were REWRITTEN — a run longer than one character, or a one-character run that was
32
+ * not already an ASCII space. A run that was already a single space is not one of them.
33
+ */
28
34
  runs: number
29
35
  }
30
36
 
@@ -57,7 +63,10 @@ export function collapseWhitespace(input: string): WhitespaceResult {
57
63
  i += 1
58
64
  }
59
65
 
60
- if (i - start > 1) {
66
+ // A one-character run counts too when the character is not already an ASCII space: the tab→space
67
+ // rewrite emitted above is a real edit, and `changed` is what decides whether the caller ever
68
+ // receives it — the early return below hands back the untouched input otherwise.
69
+ if (i - start > 1 || ch !== " ") {
61
70
  changed = true
62
71
  runs += 1
63
72
  }
@@ -65,11 +74,6 @@ export function collapseWhitespace(input: string): WhitespaceResult {
65
74
  continue
66
75
  }
67
76
 
68
- // Collapse \r\n into one
69
- if (ch === "\n" && out.at(-1) === "\r") {
70
- // Already handled in CR branch above by emitting both; skip combiner check
71
- }
72
-
73
77
  out.push(ch)
74
78
  map.push(i)
75
79
  i += 1
@@ -11,7 +11,7 @@
11
11
  * augmentation). Both sides should eventually share dictionaries; for v1 this dict is duplicated
12
12
  * intentionally — refactoring sharing is a separate task.
13
13
  */
14
- import type { SpanRange } from "./types.ts";
14
+ import type { SpanRange } from "#types";
15
15
  export interface AbbreviationEntry {
16
16
  from: string;
17
17
  to: string;
@@ -19,7 +19,7 @@ export interface AbbreviationEntry {
19
19
  /**
20
20
  * The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
21
21
  * it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
22
- * "no load-bearing trivia" rule means that data lives in exactly one place — here.
22
+ * "no required trivia" rule means that data lives in exactly one place — here.
23
23
  */
24
24
  export declare function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry>;
25
25
  export interface AbbreviationResult {
@@ -1 +1 @@
1
- {"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE3C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AA4FD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAmEtF"}
1
+ {"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../lib/abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAEvC,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AA4FD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAmEtF"}
@@ -69,7 +69,7 @@ const ES_ES_DICT = [
69
69
  * Dropping the entry is NOT a table edit: `fr-op3-halles-market-bonneuil` is a passing row that asserts street "Avenue
70
70
  * de la Convention" and an `address_point` tier, so it pins the current behaviour and a removal has to be measured on a
71
71
  * resolver-gauntlet run. The real repair is upstream — the geocode path hardcodes `locale: "und"` because Stage 1
72
- * precedes the parse, and `@mailwoman/locale-gate` cannot presently detect Spanish (it scores script class + known
72
+ * precedes the parse, and `@mailwoman/locale-hint` cannot presently detect Spanish (it scores script class + known
73
73
  * postcode formats, and a 5-digit ES/MX code is indistinguishable from a US ZIP).
74
74
  */
75
75
  const LOCALE_UNKNOWN_DICT = [
@@ -98,7 +98,7 @@ function getDictionary(locale) {
98
98
  /**
99
99
  * The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
100
100
  * it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
101
- * "no load-bearing trivia" rule means that data lives in exactly one place — here.
101
+ * "no required trivia" rule means that data lives in exactly one place — here.
102
102
  */
103
103
  export function abbreviationDictionary(locale) {
104
104
  return getDictionary(locale);
@@ -1 +1 @@
1
- {"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,kGAAkG;IAClG,8FAA8F;IAC9F,iGAAiG;IACjG,2FAA2F;IAC3F,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,+FAA+F;IAC/F,oGAAoG;IACpG,qGAAqG;IACrG,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YAED,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1C,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
1
+ {"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../lib/abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,kGAAkG;IAClG,8FAA8F;IAC9F,iGAAiG;IACjG,2FAA2F;IAC3F,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,+FAA+F;IAC/F,oGAAoG;IACpG,qGAAqG;IACrG,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YAED,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1C,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
package/out/cjk.d.ts CHANGED
@@ -26,7 +26,7 @@
26
26
  * normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
27
27
  * deferred.
28
28
  *
29
- * Self-gating: a string with none of these characters returns identity, so Latin input is
29
+ * Self-limiting: a string with none of these characters returns identity, so Latin input is
30
30
  * untouched.
31
31
  */
32
32
  export interface CjkResult {
@@ -41,5 +41,7 @@ export interface CjkResult {
41
41
  */
42
42
  stripped: number;
43
43
  }
44
- export declare function applyCjkNormalization(input: string): CjkResult;
44
+ export declare function applyCjkNormalization(input: string, opts?: {
45
+ postalMark?: "strip" | "keep";
46
+ }): CjkResult;
45
47
  //# sourceMappingURL=cjk.d.ts.map
package/out/cjk.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AA2BD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAkE9D"}
1
+ {"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../lib/cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AA2BD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAAO,GAAG,SAAS,CAmE5G"}
package/out/cjk.js CHANGED
@@ -26,10 +26,10 @@
26
26
  * normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
27
27
  * deferred.
28
28
  *
29
- * Self-gating: a string with none of these characters returns identity, so Latin input is
29
+ * Self-limiting: a string with none of these characters returns identity, so Latin input is
30
30
  * untouched.
31
31
  */
32
- import { identityMap } from "./offset-map.js";
32
+ import { identityMap } from "#offset-map";
33
33
  /**
34
34
  * !.
35
35
  */
@@ -53,16 +53,17 @@ const HALFWIDTH_VOICING_START = 0xff_9e;
53
53
  function isHalfwidthKatakana(code) {
54
54
  return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END;
55
55
  }
56
- export function applyCjkNormalization(input) {
56
+ export function applyCjkNormalization(input, opts = {}) {
57
57
  let folded = 0;
58
58
  let stripped = 0;
59
59
  const out = [];
60
60
  const map = [];
61
+ const stripPostalMark = opts.postalMark !== "keep";
61
62
  // All transformed code points are in the BMP (single UTF-16 unit), and every other character is
62
63
  // passed through verbatim, so a per-unit walk is safe for surrogate-pair input too.
63
64
  for (let i = 0; i < input.length; i++) {
64
65
  const code = input.charCodeAt(i);
65
- if (code === POSTAL_MARK) {
66
+ if (code === POSTAL_MARK && stripPostalMark) {
66
67
  stripped += 1;
67
68
  continue; // drop — no addressing content; whitespace collapse later tidies any gap
68
69
  }
package/out/cjk.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cjk.js","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAe7C;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAC3B,MAAM,wBAAwB,GAAG,OAAO,CAAA;AACxC,MAAM,sBAAsB,GAAG,OAAO,CAAA;AACtC,MAAM,uBAAuB,GAAG,OAAO,CAAA;AAEvC,SAAS,mBAAmB,CAAC,IAAY;IACxC,OAAO,IAAI,IAAI,wBAAwB,IAAI,IAAI,IAAI,sBAAsB,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAa;IAClD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC1B,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEpC,MAAM,mBAAmB,GACxB,IAAI,GAAG,uBAAuB,IAAI,IAAI,IAAI,uBAAuB,IAAI,IAAI,IAAI,sBAAsB,CAAA;YAEpG,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAEzC,MAAM,UAAU,GAAG,KAAK;iBACtB,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;iBACnB,SAAS,CAAC,MAAM,CAAC;iBACjB,SAAS,CAAC,KAAK,CAAC,CAAA;YAElB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC/B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;YAED,MAAM,IAAI,KAAK,CAAA;YACf,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;YAEd,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}
1
+ {"version":3,"file":"cjk.js","sourceRoot":"","sources":["../lib/cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAezC;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAC3B,MAAM,wBAAwB,GAAG,OAAO,CAAA;AACxC,MAAM,sBAAsB,GAAG,OAAO,CAAA;AACtC,MAAM,uBAAuB,GAAG,OAAO,CAAA;AAEvC,SAAS,mBAAmB,CAAC,IAAY;IACxC,OAAO,IAAI,IAAI,wBAAwB,IAAI,IAAI,IAAI,sBAAsB,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAa,EAAE,OAA0C,EAAE;IAChG,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,KAAK,MAAM,CAAA;IAElD,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,IAAI,eAAe,EAAE,CAAC;YAC7C,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEpC,MAAM,mBAAmB,GACxB,IAAI,GAAG,uBAAuB,IAAI,IAAI,IAAI,uBAAuB,IAAI,IAAI,IAAI,sBAAsB,CAAA;YAEpG,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAEzC,MAAM,UAAU,GAAG,KAAK;iBACtB,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;iBACnB,SAAS,CAAC,MAAM,CAAC;iBACjB,SAAS,CAAC,KAAK,CAAC,CAAA;YAElB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC/B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;YAED,MAAM,IAAI,KAAK,CAAA;YACf,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;YAEd,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
7
+ * reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
8
+ * is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
9
+ * the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
10
+ *
11
+ * A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
12
+ * BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
13
+ * the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
14
+ * comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
15
+ * expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
16
+ */
17
+ export interface CommaSpacingResult {
18
+ text: string;
19
+ map: number[];
20
+ /**
21
+ * How many spaces were inserted.
22
+ */
23
+ inserted: number;
24
+ }
25
+ /**
26
+ * Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
27
+ * separator. Offset-map-correct: the inserted space maps to the comma.
28
+ */
29
+ export declare function spaceAfterComma(input: string): CommaSpacingResult;
30
+ //# sourceMappingURL=comma-spacing.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"comma-spacing.d.ts","sourceRoot":"","sources":["../lib/comma-spacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAUH,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AASD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,kBAAkB,CAsBjE"}
@@ -0,0 +1,52 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
7
+ * reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
8
+ * is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
9
+ * the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
10
+ *
11
+ * A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
12
+ * BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
13
+ * the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
14
+ * comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
15
+ * expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
16
+ */
17
+ import { identityMap } from "#offset-map";
18
+ /**
19
+ * Any letter or decimal digit in any script: the Unicode property classes `\p{L}` and `\p{Nd}`.
20
+ */
21
+ const LETTER_OR_DIGIT = /[\p{L}\p{Nd}]/u;
22
+ const DIGIT = /\p{Nd}/u;
23
+ /**
24
+ * A comma with a decimal digit on both sides: `12,5`, `1,000`.
25
+ */
26
+ function isNumericSeparator(input, commaIndex) {
27
+ return commaIndex > 0 && DIGIT.test(input[commaIndex - 1]) && DIGIT.test(input[commaIndex + 1]);
28
+ }
29
+ /**
30
+ * Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
31
+ * separator. Offset-map-correct: the inserted space maps to the comma.
32
+ */
33
+ export function spaceAfterComma(input) {
34
+ let inserted = 0;
35
+ const out = [];
36
+ const map = [];
37
+ for (let i = 0; i < input.length; i++) {
38
+ const ch = input[i];
39
+ out.push(ch);
40
+ map.push(i);
41
+ if (ch === "," && i + 1 < input.length && LETTER_OR_DIGIT.test(input[i + 1]) && !isNumericSeparator(input, i)) {
42
+ out.push(" ");
43
+ map.push(i);
44
+ inserted += 1;
45
+ }
46
+ }
47
+ if (!inserted) {
48
+ return { text: input, map: identityMap(input.length), inserted: 0 };
49
+ }
50
+ return { text: out.join(""), map, inserted };
51
+ }
52
+ //# sourceMappingURL=comma-spacing.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"comma-spacing.js","sourceRoot":"","sources":["../lib/comma-spacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC;;GAEG;AACH,MAAM,eAAe,GAAG,gBAAgB,CAAA;AACxC,MAAM,KAAK,GAAG,SAAS,CAAA;AAWvB;;GAEG;AACH,SAAS,kBAAkB,CAAC,KAAa,EAAE,UAAkB;IAC5D,OAAO,UAAU,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAE,CAAC,CAAA;AAClG,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC5C,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAEX,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC;YAChH,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,QAAQ,IAAI,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IACpE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAA;AAC7C,CAAC"}
package/out/compute.d.ts CHANGED
@@ -6,6 +6,6 @@
6
6
  * `normalize(raw, opts)` — the Stage 1 entry point. Composes NFC + punctuation + whitespace
7
7
  * (always) with case-fold + abbreviation expansion (opt-in).
8
8
  */
9
- import type { NormalizedInput, NormalizeOpts } from "./types.ts";
9
+ import type { NormalizedInput, NormalizeOpts } from "#types";
10
10
  export declare function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput;
11
11
  //# sourceMappingURL=compute.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"compute.d.ts","sourceRoot":"","sources":["../compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAOH,OAAO,KAAK,EAA0B,eAAe,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAGxF,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,eAAe,CAiF5E"}
1
+ {"version":3,"file":"compute.d.ts","sourceRoot":"","sources":["../lib/compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAQH,OAAO,KAAK,EAA0B,eAAe,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAGpF,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,aAAa,GAAG,eAAe,CAgG5E"}
package/out/compute.js CHANGED
@@ -6,12 +6,13 @@
6
6
  * `normalize(raw, opts)` — the Stage 1 entry point. Composes NFC + punctuation + whitespace
7
7
  * (always) with case-fold + abbreviation expansion (opt-in).
8
8
  */
9
- import { expandAbbreviations } from "./abbreviations.js";
10
- import { applyCjkNormalization } from "./cjk.js";
11
- import { applyNFC } from "./nfc.js";
12
- import { composeMaps, identityMap } from "./offset-map.js";
13
- import { applyPunctuation } from "./punctuation.js";
14
- import { collapseWhitespace } from "./whitespace.js";
9
+ import { expandAbbreviations } from "#abbreviations";
10
+ import { applyCjkNormalization } from "#cjk";
11
+ import { spaceAfterComma } from "#comma-spacing";
12
+ import { applyNFC } from "#nfc";
13
+ import { composeMaps, identityMap } from "#offset-map";
14
+ import { applyPunctuation } from "#punctuation";
15
+ import { collapseWhitespace } from "#whitespace";
15
16
  export function normalize(raw, opts) {
16
17
  const transforms = [];
17
18
  let text = raw;
@@ -27,7 +28,7 @@ export function normalize(raw, opts) {
27
28
  // parse) and fold full-width ASCII + the ideographic space. Runs after NFC so it sees composed
28
29
  // forms, before punctuation/whitespace so any gap left by 〒 is then collapsed. No-op off-script.
29
30
  {
30
- const r = applyCjkNormalization(text);
31
+ const r = applyCjkNormalization(text, opts?.postalMark ? { postalMark: opts.postalMark } : {});
31
32
  if (r.folded > 0 || r.stripped > 0) {
32
33
  text = r.text;
33
34
  map = composeMaps(map, r.map);
@@ -43,10 +44,23 @@ export function normalize(raw, opts) {
43
44
  transforms.push({ kind: "normalize_punctuation", replacements: r.replacements });
44
45
  }
45
46
  }
47
+ // 2.5 Comma spacing — a comma glued to a letter gains a space. Runs after punctuation (so a folded
48
+ // full-width comma is seen) and before whitespace collapse (so a comma already followed by a space is
49
+ // never doubled).
50
+ {
51
+ const r = spaceAfterComma(text);
52
+ if (r.inserted > 0) {
53
+ text = r.text;
54
+ map = composeMaps(map, r.map);
55
+ transforms.push({ kind: "space_after_comma", inserted: r.inserted });
56
+ }
57
+ }
46
58
  // 3. Whitespace
47
59
  {
48
60
  const r = collapseWhitespace(text);
49
- if (r.runs > 0 || r.text.length !== text.length) {
61
+ // Compare the TEXT, not its length: folding a lone tab to a space is length-preserving, and a
62
+ // length test reads that edit as no edit at all.
63
+ if (r.text !== text) {
50
64
  text = r.text;
51
65
  map = composeMaps(map, r.map);
52
66
  transforms.push({ kind: "collapse_whitespace", runs: r.runs });
@@ -1 +1 @@
1
- {"version":3,"file":"compute.js","sourceRoot":"","sources":["../compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAEpD,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,IAAoB;IAC1D,MAAM,UAAU,GAA6B,EAAE,CAAA;IAC/C,IAAI,IAAI,GAAG,GAAG,CAAA;IACd,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEjC,SAAS;IACT,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;QACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;QAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,+FAA+F;IAC/F,+FAA+F;IAC/F,iGAAiG;IACjG,CAAC;QACA,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAA;QAErC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;QACnF,CAAC;IACF,CAAC;IAED,iBAAiB;IACjB,CAAC;QACA,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAEhC,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAA;QACjF,CAAC;IACF,CAAC;IAED,gBAAgB;IAChB,CAAC;QACA,MAAM,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;QAElC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACjD,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,2FAA2F;IAC3F,gEAAgE;IAChE,IAAI,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhD,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAE7B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YACnF,CAAC;QACF,CAAC;IACF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAE9C,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,EAAE,CAAA;YACT,sEAAsE;YACtE,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAA;QACrE,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC;QACpB,GAAG;QACH,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAA6B;QACjE,SAAS,EAAE,GAAG;QACd,aAAa,EAAE,IAAI,EAAE,MAAM;KAC3B,CAA2B,CAAA;AAC7B,CAAC"}
1
+ {"version":3,"file":"compute.js","sourceRoot":"","sources":["../lib/compute.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,MAAM,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAEhD,MAAM,UAAU,SAAS,CAAC,GAAW,EAAE,IAAoB;IAC1D,MAAM,UAAU,GAA6B,EAAE,CAAA;IAC/C,IAAI,IAAI,GAAG,GAAG,CAAA;IACd,IAAI,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEjC,SAAS;IACT,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;QACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;QAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,+FAA+F;IAC/F,+FAA+F;IAC/F,iGAAiG;IACjG,CAAC;QACA,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAE9F,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;QACnF,CAAC;IACF,CAAC;IAED,iBAAiB;IACjB,CAAC;QACA,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QAEhC,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,uBAAuB,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAA;QACjF,CAAC;IACF,CAAC;IAED,mGAAmG;IACnG,sGAAsG;IACtG,kBAAkB;IAClB,CAAC;QACA,MAAM,CAAC,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;QAE/B,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;YACpB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;QACrE,CAAC;IACF,CAAC;IAED,gBAAgB;IAChB,CAAC;QACA,MAAM,CAAC,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;QAElC,8FAA8F;QAC9F,iDAAiD;QACjD,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACrB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,2FAA2F;IAC3F,gEAAgE;IAChE,IAAI,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAEhD,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACzB,IAAI,GAAG,CAAC,CAAC,IAAI,CAAA;YACb,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YAE7B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC9B,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YACnF,CAAC;QACF,CAAC;IACF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAE9C,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,GAAG,EAAE,CAAA;YACT,sEAAsE;YACtE,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK,EAAE,CAAC,CAAA;QACrE,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC;QACpB,GAAG;QACH,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAA6B;QACjE,SAAS,EAAE,GAAG;QACd,aAAa,EAAE,IAAI,EAAE,MAAM;KAC3B,CAA2B,CAAA;AAC7B,CAAC"}
package/out/fold.d.ts ADDED
@@ -0,0 +1,22 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file Match-key folds: the loss-y transforms a comparison applies to both sides.
6
+ */
7
+ /**
8
+ * Lower-case, whitespace runs collapsed to one space, ends trimmed. The fold every string comparison applies before it
9
+ * compares; it keeps punctuation and diacritics, so `Saint-Étienne` and `Saint-Etienne` stay distinct.
10
+ */
11
+ export declare function foldCaseWhitespace(input: string): string;
12
+ /**
13
+ * Combining marks removed after NFD decomposition: `é` → `e`, `ł` unchanged (it is not a base plus a mark). Case and
14
+ * whitespace are untouched.
15
+ */
16
+ export declare function stripCombiningMarks(input: string): string;
17
+ /**
18
+ * {@link foldCaseWhitespace} over the NFKC composition, with locale-aware lower-casing: fullwidth and compatibility
19
+ * forms fold together before comparison.
20
+ */
21
+ export declare function foldNFKCWhitespace(input: string): string;
22
+ //# sourceMappingURL=fold.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fold.d.ts","sourceRoot":"","sources":["../lib/fold.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAExD"}
package/out/fold.js ADDED
@@ -0,0 +1,28 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ * @file Match-key folds: the loss-y transforms a comparison applies to both sides.
6
+ */
7
+ /**
8
+ * Lower-case, whitespace runs collapsed to one space, ends trimmed. The fold every string comparison applies before it
9
+ * compares; it keeps punctuation and diacritics, so `Saint-Étienne` and `Saint-Etienne` stay distinct.
10
+ */
11
+ export function foldCaseWhitespace(input) {
12
+ return input.toLowerCase().replaceAll(/\s+/gu, " ").trim();
13
+ }
14
+ /**
15
+ * Combining marks removed after NFD decomposition: `é` → `e`, `ł` unchanged (it is not a base plus a mark). Case and
16
+ * whitespace are untouched.
17
+ */
18
+ export function stripCombiningMarks(input) {
19
+ return input.normalize("NFD").replaceAll(/\p{M}/gu, "");
20
+ }
21
+ /**
22
+ * {@link foldCaseWhitespace} over the NFKC composition, with locale-aware lower-casing: fullwidth and compatibility
23
+ * forms fold together before comparison.
24
+ */
25
+ export function foldNFKCWhitespace(input) {
26
+ return input.normalize("NFKC").toLocaleLowerCase().replaceAll(/\s+/gu, " ").trim();
27
+ }
28
+ //# sourceMappingURL=fold.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fold.js","sourceRoot":"","sources":["../lib/fold.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAChD,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,OAAO,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACnF,CAAC"}
package/out/index.d.ts CHANGED
@@ -12,12 +12,13 @@
12
12
  *
13
13
  * See `docs/engineering/reference/STAGES.md` § Stage 1 for the contract.
14
14
  */
15
- export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "./abbreviations.ts";
16
- export { applyCjkNormalization, type CjkResult } from "./cjk.ts";
17
- export { normalize } from "./compute.ts";
18
- export { applyNFC } from "./nfc.ts";
19
- export { composeMaps, identityMap } from "./offset-map.ts";
20
- export { applyPunctuation } from "./punctuation.ts";
21
- export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "./types.ts";
22
- export { collapseWhitespace } from "./whitespace.ts";
15
+ export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "#abbreviations";
16
+ export { applyCjkNormalization, type CjkResult } from "#cjk";
17
+ export { normalize } from "#compute";
18
+ export { foldCaseWhitespace, foldNFKCWhitespace, stripCombiningMarks } from "#fold";
19
+ export { applyNFC } from "#nfc";
20
+ export { composeMaps, identityMap } from "#offset-map";
21
+ export { applyPunctuation } from "#punctuation";
22
+ export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "#types";
23
+ export { collapseWhitespace } from "#whitespace";
23
24
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,KAAK,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxG,OAAO,EAAE,qBAAqB,EAAE,KAAK,SAAS,EAAE,MAAM,UAAU,CAAA;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,YAAY,EAAE,sBAAsB,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACnG,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,KAAK,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpG,OAAO,EAAE,qBAAqB,EAAE,KAAK,SAAS,EAAE,MAAM,MAAM,CAAA;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAA;AACnF,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC/C,YAAY,EAAE,sBAAsB,EAAE,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAC/F,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA"}
package/out/index.js CHANGED
@@ -12,11 +12,12 @@
12
12
  *
13
13
  * See `docs/engineering/reference/STAGES.md` § Stage 1 for the contract.
14
14
  */
15
- export { abbreviationDictionary, expandAbbreviations } from "./abbreviations.js";
16
- export { applyCjkNormalization } from "./cjk.js";
17
- export { normalize } from "./compute.js";
18
- export { applyNFC } from "./nfc.js";
19
- export { composeMaps, identityMap } from "./offset-map.js";
20
- export { applyPunctuation } from "./punctuation.js";
21
- export { collapseWhitespace } from "./whitespace.js";
15
+ export { abbreviationDictionary, expandAbbreviations } from "#abbreviations";
16
+ export { applyCjkNormalization } from "#cjk";
17
+ export { normalize } from "#compute";
18
+ export { foldCaseWhitespace, foldNFKCWhitespace, stripCombiningMarks } from "#fold";
19
+ export { applyNFC } from "#nfc";
20
+ export { composeMaps, identityMap } from "#offset-map";
21
+ export { applyPunctuation } from "#punctuation";
22
+ export { collapseWhitespace } from "#whitespace";
22
23
  //# sourceMappingURL=index.js.map
package/out/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAA0B,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxG,OAAO,EAAE,qBAAqB,EAAkB,MAAM,UAAU,CAAA;AAChE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAA0B,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACpG,OAAO,EAAE,qBAAqB,EAAkB,MAAM,MAAM,CAAA;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AACpC,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAA;AACnF,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAE/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA"}
package/out/nfc.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"nfc.d.ts","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;CAChB;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAQjD"}
1
+ {"version":3,"file":"nfc.d.ts","sourceRoot":"","sources":["../lib/nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;CAChB;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAQjD"}
package/out/nfc.js CHANGED
@@ -11,7 +11,7 @@
11
11
  * map each output index to the start of its source sequence. Rare CJK edge cases involving
12
12
  * variant selectors may produce off-by-one offsets — acceptable for v1.
13
13
  */
14
- import { identityMap } from "./offset-map.js";
14
+ import { identityMap } from "#offset-map";
15
15
  export function applyNFC(input) {
16
16
  const normalized = input.normalize("NFC");
17
17
  if (normalized === input) {
package/out/nfc.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"nfc.js","sourceRoot":"","sources":["../nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAW7C,MAAM,UAAU,QAAQ,CAAC,KAAa;IACrC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAEzC,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;IACvE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,KAAa,EAAE,MAAc;IACpD,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAE,CAAA;QACzC,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAEvC,gGAAgG;QAChG,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;YACtC,KAAK,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/B,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;gBAExC,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC5C,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACP,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
1
+ {"version":3,"file":"nfc.js","sourceRoot":"","sources":["../lib/nfc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAWzC,MAAM,UAAU,QAAQ,CAAC,KAAa;IACrC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAEzC,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAA;IACvE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,cAAc,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,KAAa,EAAE,MAAc;IACpD,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACvD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACf,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAE,CAAA;QACzC,MAAM,OAAO,GAAG,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAEvC,gGAAgG;QAChG,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;YACtC,KAAK,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/B,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,KAAK,CAAE,CAAA;gBAExC,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC5C,KAAK,IAAI,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;gBAClC,CAAC;qBAAM,CAAC;oBACP,MAAK;gBACN,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,CAAA;QACZ,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"offset-map.d.ts","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAShF"}
1
+ {"version":3,"file":"offset-map.d.ts","sourceRoot":"","sources":["../lib/offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ/C;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAShF"}
@@ -1 +1 @@
1
- {"version":3,"file":"offset-map.js","sourceRoot":"","sources":["../offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,CAAS;IACpC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACT,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,YAAsB;IACrE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAS,YAAY,CAAC,MAAM,CAAC,CAAA;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAE,CAAA;QAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAC1B,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
1
+ {"version":3,"file":"offset-map.js","sourceRoot":"","sources":["../lib/offset-map.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,CAAS;IACpC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IACT,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,QAAkB,EAAE,YAAsB;IACrE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAS,YAAY,CAAC,MAAM,CAAC,CAAA;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAE,CAAA;QAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IAC1B,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAmBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA8BjE"}
1
+ {"version":3,"file":"punctuation.d.ts","sourceRoot":"","sources":["../lib/punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAmBH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;CACpB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,iBAAiB,CA8BjE"}
@@ -6,7 +6,7 @@
6
6
  * Punctuation normalization — fancy quotes / dashes to ASCII equivalents. Identity-length: every
7
7
  * fancy character is a single codepoint that maps to a single ASCII char.
8
8
  */
9
- import { identityMap } from "./offset-map.js";
9
+ import { identityMap } from "#offset-map";
10
10
  const REPLACEMENTS = new Map([
11
11
  ["‘", "'"], // ‘
12
12
  ["’", "'"], // ’
@@ -1 +1 @@
1
- {"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,kBAAkB;IAC9B,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,+EAA+E;IAC3F,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8DAA8D;IAC1E,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8BAA8B;IAC1C,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,+BAA+B;IAC7C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qBAAqB;CACjC,CAAC,CAAA;AAQF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEhC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,IAAI,CAAC,CAAA;YAEjB,mGAAmG;YACnG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC"}
1
+ {"version":3,"file":"punctuation.js","sourceRoot":"","sources":["../lib/punctuation.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAiB;IAC5C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI;IAChB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,YAAY;IACxB,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,kBAAkB;IAC9B,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,+EAA+E;IAC3F,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8DAA8D;IAC1E,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,8BAA8B;IAC1C,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,+BAA+B;IAC7C,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,qBAAqB;CACjC,CAAC,CAAA;AAQF,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEhC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,IAAI,CAAA;YACd,YAAY,IAAI,CAAC,CAAA;YAEjB,mGAAmG;YACnG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;gBACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;QACF,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IACxE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,CAAA;AACjD,CAAC"}
package/out/types.d.ts CHANGED
@@ -28,6 +28,9 @@ export type NormalizationTransform = {
28
28
  } | {
29
29
  kind: "normalize_punctuation";
30
30
  replacements: number;
31
+ } | {
32
+ kind: "space_after_comma";
33
+ inserted: number;
31
34
  } | {
32
35
  kind: "normalize_cjk";
33
36
  folded: number;
@@ -79,5 +82,12 @@ export interface NormalizeOpts {
79
82
  * Skip Unicode NFC. Only use for debugging — production callers should leave on.
80
83
  */
81
84
  skipNFC?: boolean;
85
+ /**
86
+ * What the CJK pass does with the postal mark 〒 (U+3012). `strip` (the default) drops it: it is byte-fallback OOV for
87
+ * the SentencePiece tokenizer and fragments the digits after it. `keep` leaves it in place for a classifier whose
88
+ * vocabulary carries it — the character-path CJK model was trained with the mark in front of every postcode, and
89
+ * without it misreads the prefecture boundary (`885-0061 宮崎県都城市…` → prefecture `崎県都`).
90
+ */
91
+ postalMark?: "strip" | "keep";
82
92
  }
83
93
  //# sourceMappingURL=types.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAC/B;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,SAAS,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAA;AAE9D;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,UAAU,EAAE,sBAAsB,EAAE,CAAA;IAEpC;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAA;IAEnB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;IAElB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CACjB"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,SAAS;IACzB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAC/B;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,GACjC;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,SAAS,CAAA;CAAE,GACxE;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC7C;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,mBAAmB,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAA;AAE9D;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,UAAU,EAAE,sBAAsB,EAAE,CAAA;IAEpC;;OAEG;IACH,SAAS,EAAE,MAAM,EAAE,CAAA;IAEnB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAA;IAElB;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;IAE7B;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAC7B"}
package/out/types.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
@@ -3,9 +3,11 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Whitespace collapse — runs of whitespace become a single ASCII space. Newlines and tabs are
7
- * preserved as-is (segmentation grammar in QueryShape uses them); inline runs of spaces
8
- * collapse. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
6
+ * Whitespace collapse — a run of inline whitespace (`[ \t]`) becomes one ASCII space AT EVERY RUN
7
+ * LENGTH, so a lone tab normalizes exactly as a doubled one does. Newlines (`\n`/`\r`) are preserved:
8
+ * QueryShape's segmentation grammar reads one as a segment separator on a par with a comma, and it
9
+ * reads a RAW tab the same way — folding every tab here is what keeps a tab-separated export from
10
+ * re-segmenting the query. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
9
11
  * trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
10
12
  * tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
11
13
  * quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
@@ -14,6 +16,10 @@
14
16
  export interface WhitespaceResult {
15
17
  text: string;
16
18
  map: number[];
19
+ /**
20
+ * How many inline-whitespace runs were REWRITTEN — a run longer than one character, or a one-character run that was
21
+ * not already an ASCII space. A run that was already a single space is not one of them.
22
+ */
17
23
  runs: number;
18
24
  }
19
25
  export declare function collapseWhitespace(input: string): WhitespaceResult;
@@ -1 +1 @@
1
- {"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAYH,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAwElE"}
1
+ {"version":3,"file":"whitespace.d.ts","sourceRoot":"","sources":["../lib/whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAYH,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAA;CACZ;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAsElE"}
package/out/whitespace.js CHANGED
@@ -3,20 +3,22 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Whitespace collapse — runs of whitespace become a single ASCII space. Newlines and tabs are
7
- * preserved as-is (segmentation grammar in QueryShape uses them); inline runs of spaces
8
- * collapse. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
6
+ * Whitespace collapse — a run of inline whitespace (`[ \t]`) becomes one ASCII space AT EVERY RUN
7
+ * LENGTH, so a lone tab normalizes exactly as a doubled one does. Newlines (`\n`/`\r`) are preserved:
8
+ * QueryShape's segmentation grammar reads one as a segment separator on a par with a comma, and it
9
+ * reads a RAW tab the same way — folding every tab here is what keeps a tab-separated export from
10
+ * re-segmenting the query. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
9
11
  * trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
10
12
  * tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
11
13
  * quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
12
14
  * as the whitespace trim, so span alignment survives.
13
15
  */
14
- import { identityMap } from "./offset-map.js";
16
+ import { identityMap } from "#offset-map";
15
17
  const INLINE_SPACE = /[ \t]/;
16
18
  const ANY_SPACE = /[ \t\n\r]/;
17
19
  /**
18
20
  * Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user commonly appends.
19
- * NOT leading (a leading token is load-bearing) and NOT quotes/brackets/parens.
21
+ * NOT leading (a leading token is required) and NOT quotes/brackets/parens.
20
22
  */
21
23
  const TRAILING_NOISE = /[ \t\n\r.,;:]/;
22
24
  export function collapseWhitespace(input) {
@@ -42,16 +44,15 @@ export function collapseWhitespace(input) {
42
44
  while (i < input.length && INLINE_SPACE.test(input[i])) {
43
45
  i += 1;
44
46
  }
45
- if (i - start > 1) {
47
+ // A one-character run counts too when the character is not already an ASCII space: the tab→space
48
+ // rewrite emitted above is a real edit, and `changed` is what decides whether the caller ever
49
+ // receives it — the early return below hands back the untouched input otherwise.
50
+ if (i - start > 1 || ch !== " ") {
46
51
  changed = true;
47
52
  runs += 1;
48
53
  }
49
54
  continue;
50
55
  }
51
- // Collapse \r\n into one
52
- if (ch === "\n" && out.at(-1) === "\r") {
53
- // Already handled in CR branch above by emitting both; skip combiner check
54
- }
55
56
  out.push(ch);
56
57
  map.push(i);
57
58
  i += 1;
@@ -1 +1 @@
1
- {"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAE7C,MAAM,YAAY,GAAG,OAAO,CAAA;AAC5B,MAAM,SAAS,GAAG,WAAW,CAAA;AAC7B;;;GAGG;AACH,MAAM,cAAc,GAAG,eAAe,CAAA;AAQtC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChC,2CAA2C;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,KAAK,GAAG,CAAC,CAAA;YACf,CAAC,IAAI,CAAC,CAAA;YAEN,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACzD,CAAC,IAAI,CAAC,CAAA;YACP,CAAC;YAED,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO,GAAG,IAAI,CAAA;gBACd,IAAI,IAAI,CAAC,CAAA;YACV,CAAC;YAED,SAAQ;QACT,CAAC;QAED,yBAAyB;QACzB,IAAI,EAAE,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,2EAA2E;QAC5E,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,CAAC,IAAI,CAAC,CAAA;IACP,CAAC;IAED,6FAA6F;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,GAAG,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,CAAA;IACV,CAAC;IAED,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;IAEtB,OAAO,KAAK,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;QAC7D,KAAK,IAAI,CAAC,CAAA;IACX,CAAC;IAED,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACpC,OAAO,GAAG,IAAI,CAAA;IACf,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEzC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IAChE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AAC5D,CAAC"}
1
+ {"version":3,"file":"whitespace.js","sourceRoot":"","sources":["../lib/whitespace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,MAAM,YAAY,GAAG,OAAO,CAAA;AAC5B,MAAM,SAAS,GAAG,WAAW,CAAA;AAC7B;;;GAGG;AACH,MAAM,cAAc,GAAG,eAAe,CAAA;AAYtC,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC/C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChC,2CAA2C;YAC3C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,KAAK,GAAG,CAAC,CAAA;YACf,CAAC,IAAI,CAAC,CAAA;YAEN,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACzD,CAAC,IAAI,CAAC,CAAA;YACP,CAAC;YAED,iGAAiG;YACjG,8FAA8F;YAC9F,iFAAiF;YACjF,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,OAAO,GAAG,IAAI,CAAA;gBACd,IAAI,IAAI,CAAC,CAAA;YACV,CAAC;YAED,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,CAAC,IAAI,CAAC,CAAA;IACP,CAAC;IAED,6FAA6F;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,GAAG,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,EAAE,CAAC;QACxD,IAAI,IAAI,CAAC,CAAA;IACV,CAAC;IAED,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,CAAA;IAEtB,OAAO,KAAK,GAAG,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;QAC7D,KAAK,IAAI,CAAC,CAAA;IACX,CAAC;IAED,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;QACpC,OAAO,GAAG,IAAI,CAAA;IACf,CAAC;IAED,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAEzC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IAChE,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;AAC5D,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mailwoman/normalize",
3
- "version": "9.2.0",
3
+ "version": "9.3.0",
4
4
  "description": "Stage 1 of the runtime pipeline — deterministic input preprocessing (Unicode NFC, punctuation, whitespace, abbreviation). Pure functions, no ML.",
5
5
  "license": "AGPL-3.0-only OR LicenseRef-Commercial",
6
6
  "repository": {
@@ -24,38 +24,64 @@
24
24
  "!test/**"
25
25
  ],
26
26
  "type": "module",
27
+ "sideEffects": false,
28
+ "imports": {
29
+ "#*": {
30
+ "types": "./out/*.d.ts",
31
+ "node": "./out/*.js",
32
+ "default": "./out/*.js"
33
+ }
34
+ },
27
35
  "exports": {
28
36
  "./package.json": "./package.json",
29
37
  ".": {
30
38
  "types": "./out/index.d.ts",
39
+ "node": "./out/index.js",
31
40
  "default": "./out/index.js"
32
41
  },
33
42
  "./abbreviations": {
34
43
  "types": "./out/abbreviations.d.ts",
44
+ "node": "./out/abbreviations.js",
35
45
  "default": "./out/abbreviations.js"
36
46
  },
37
47
  "./cjk": {
38
48
  "types": "./out/cjk.d.ts",
49
+ "node": "./out/cjk.js",
39
50
  "default": "./out/cjk.js"
40
51
  },
41
52
  "./compute": {
42
53
  "types": "./out/compute.d.ts",
54
+ "node": "./out/compute.js",
43
55
  "default": "./out/compute.js"
44
56
  },
45
57
  "./nfc": {
46
58
  "types": "./out/nfc.d.ts",
59
+ "node": "./out/nfc.js",
47
60
  "default": "./out/nfc.js"
48
61
  },
49
62
  "./offset-map": {
50
63
  "types": "./out/offset-map.d.ts",
64
+ "node": "./out/offset-map.js",
51
65
  "default": "./out/offset-map.js"
52
66
  },
53
67
  "./punctuation": {
54
68
  "types": "./out/punctuation.d.ts",
69
+ "node": "./out/punctuation.js",
55
70
  "default": "./out/punctuation.js"
56
71
  },
72
+ "./comma-spacing": {
73
+ "types": "./out/comma-spacing.d.ts",
74
+ "node": "./out/comma-spacing.js",
75
+ "default": "./out/comma-spacing.js"
76
+ },
77
+ "./fold": {
78
+ "types": "./out/fold.d.ts",
79
+ "node": "./out/fold.js",
80
+ "default": "./out/fold.js"
81
+ },
57
82
  "./whitespace": {
58
83
  "types": "./out/whitespace.d.ts",
84
+ "node": "./out/whitespace.js",
59
85
  "default": "./out/whitespace.js"
60
86
  }
61
87
  },
@@ -65,36 +91,61 @@
65
91
  "./package.json": "./package.json",
66
92
  ".": {
67
93
  "types": "./out/index.d.ts",
94
+ "node": "./out/index.js",
68
95
  "default": "./out/index.js"
69
96
  },
70
97
  "./abbreviations": {
71
98
  "types": "./out/abbreviations.d.ts",
99
+ "node": "./out/abbreviations.js",
72
100
  "default": "./out/abbreviations.js"
73
101
  },
74
102
  "./cjk": {
75
103
  "types": "./out/cjk.d.ts",
104
+ "node": "./out/cjk.js",
76
105
  "default": "./out/cjk.js"
77
106
  },
78
107
  "./compute": {
79
108
  "types": "./out/compute.d.ts",
109
+ "node": "./out/compute.js",
80
110
  "default": "./out/compute.js"
81
111
  },
82
112
  "./nfc": {
83
113
  "types": "./out/nfc.d.ts",
114
+ "node": "./out/nfc.js",
84
115
  "default": "./out/nfc.js"
85
116
  },
86
117
  "./offset-map": {
87
118
  "types": "./out/offset-map.d.ts",
119
+ "node": "./out/offset-map.js",
88
120
  "default": "./out/offset-map.js"
89
121
  },
90
122
  "./punctuation": {
91
123
  "types": "./out/punctuation.d.ts",
124
+ "node": "./out/punctuation.js",
92
125
  "default": "./out/punctuation.js"
93
126
  },
127
+ "./comma-spacing": {
128
+ "types": "./out/comma-spacing.d.ts",
129
+ "node": "./out/comma-spacing.js",
130
+ "default": "./out/comma-spacing.js"
131
+ },
132
+ "./fold": {
133
+ "types": "./out/fold.d.ts",
134
+ "node": "./out/fold.js",
135
+ "default": "./out/fold.js"
136
+ },
94
137
  "./whitespace": {
95
138
  "types": "./out/whitespace.d.ts",
139
+ "node": "./out/whitespace.js",
96
140
  "default": "./out/whitespace.js"
97
141
  }
142
+ },
143
+ "imports": {
144
+ "#*": {
145
+ "types": "./out/*.d.ts",
146
+ "node": "./out/*.js",
147
+ "default": "./out/*.js"
148
+ }
98
149
  }
99
150
  }
100
151
  }
File without changes