@mailwoman/normalize 9.1.0 → 9.3.0

package/README.md

@@ -46,7 +46,7 @@ identityMap(length: number): OffsetMap
 ## Pipeline position
 
 ```
-raw string → normalize → query-shape → locale-gate → kind-classifier → phrase-grouper → ...
+raw string → normalize → query-shape → locale-hint → kind-classifier → phrase-grouper → ...
 ```
 
 Stage 1 in the [Staged Pipeline Contract](https://github.com/sister-software/mailwoman/blob/main/docs/engineering/reference/STAGES.mdx). No runtime dependencies.
@@ -61,7 +61,7 @@ Stage 1 in the [Staged Pipeline Contract](https://github.com/sister-software/mai
 
 - [`@mailwoman/query-shape`](../query-shape) — Stage 1.5, structural priors that consume the normalized output
 - [Staged Pipeline Contract](https://github.com/sister-software/mailwoman/blob/main/docs/engineering/reference/STAGES.mdx)
-- [Tokenization concepts](https://mailwoman.sister.software/articles/concepts/tokenization/)
+- [Tokenization concepts](https://mailwoman.ai/articles/concepts/tokenization/)
 
 ## License
 

package/{abbreviations.ts → lib/abbreviations.ts}

@@ -12,7 +12,7 @@
  *   intentionally — refactoring sharing is a separate task.
  */
 
-import type { SpanRange } from "./types.ts"
+import type { SpanRange } from "#types"
 
 export interface AbbreviationEntry {
 	from: string // short form (case-insensitive match)
@@ -80,7 +80,7 @@ const ES_ES_DICT: ReadonlyArray<AbbreviationEntry> = [
  * Dropping the entry is NOT a table edit: `fr-op3-halles-market-bonneuil` is a passing row that asserts street "Avenue
  * de la Convention" and an `address_point` tier, so it pins the current behaviour and a removal has to be measured on a
  * resolver-gauntlet run. The real repair is upstream — the geocode path hardcodes `locale: "und"` because Stage 1
- * precedes the parse, and `@mailwoman/locale-gate` cannot presently detect Spanish (it scores script class + known
+ * precedes the parse, and `@mailwoman/locale-hint` cannot presently detect Spanish (it scores script class + known
  * postcode formats, and a 5-digit ES/MX code is indistinguishable from a US ZIP).
  */
 const LOCALE_UNKNOWN_DICT: ReadonlyArray<AbbreviationEntry> = [
@@ -112,7 +112,7 @@ function getDictionary(locale: string | undefined): ReadonlyArray<AbbreviationEn
 /**
  * The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
  * it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
- * "no load-bearing trivia" rule means that data lives in exactly one place — here.
+ * "no required trivia" rule means that data lives in exactly one place — here.
  */
 export function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry> {
 	return getDictionary(locale)

package/{cjk.ts → lib/cjk.ts}

@@ -16,6 +16,9 @@
  *       1, a full-width `－` always a hyphen — keyboards and copy-paste produce these constantly.
  *       Folding them to ASCII makes `１０４−００６１` and `104-0061` the same input.
  *   - **Fold the ideographic space (U+3000 → ' ').**
+ *   - **Fold half-width katakana (U+FF61–U+FF9F).** This matches the JP corpus builder. A voiced
+ *       pair such as `ﾃﾞ` contracts from two UTF-16 units to one `デ`; the output offset maps to the
+ *       first raw unit in the pair.
  *
  *   It deliberately does NOT convert **kanji numerals** (一二三…): place names carry numeral kanji as
  *   ordinary characters (三田 _Mita_, 四谷 _Yotsuya_), so a blind 三→3 would corrupt them.
@@ -23,11 +26,11 @@
  *   normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
  *   deferred.
  *
- *   Self-gating: a string with none of these characters returns identity, so Latin input is
+ *   Self-limiting: a string with none of these characters returns identity, so Latin input is
  *   untouched.
  */
 
-import { identityMap } from "./offset-map.ts"
+import { identityMap } from "#offset-map"
 
 export interface CjkResult {
 	text: string
@@ -59,19 +62,27 @@ const IDEOGRAPHIC_SPACE = 0x30_00
  * 〒.
  */
 const POSTAL_MARK = 0x30_12
+const HALFWIDTH_KATAKANA_START = 0xff_61
+const HALFWIDTH_KATAKANA_END = 0xff_9f
+const HALFWIDTH_VOICING_START = 0xff_9e
 
-export function applyCjkNormalization(input: string): CjkResult {
+function isHalfwidthKatakana(code: number): boolean {
+	return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END
+}
+
+export function applyCjkNormalization(input: string, opts: { postalMark?: "strip" | "keep" } = {}): CjkResult {
 	let folded = 0
 	let stripped = 0
 	const out: string[] = []
 	const map: number[] = []
+	const stripPostalMark = opts.postalMark !== "keep"
 
 	// All transformed code points are in the BMP (single UTF-16 unit), and every other character is
 	// passed through verbatim, so a per-unit walk is safe for surrogate-pair input too.
 	for (let i = 0; i < input.length; i++) {
 		const code = input.charCodeAt(i)
 
-		if (code === POSTAL_MARK) {
+		if (code === POSTAL_MARK && stripPostalMark) {
 			stripped += 1
 
 			continue // drop — no addressing content; whitespace collapse later tidies any gap
@@ -93,6 +104,30 @@ export function applyCjkNormalization(input: string): CjkResult {
 			continue
 		}
 
+		if (isHalfwidthKatakana(code)) {
+			const next = input.charCodeAt(i + 1)
+
+			const consumesVoicingMark =
+				code < HALFWIDTH_VOICING_START && next >= HALFWIDTH_VOICING_START && next <= HALFWIDTH_KATAKANA_END
+
+			const width = consumesVoicingMark ? 2 : 1
+
+			const normalized = input
+				.slice(i, i + width)
+				.normalize("NFKC")
+				.normalize("NFC")
+
+			for (const unit of normalized) {
+				out.push(unit)
+				map.push(i)
+			}
+
+			folded += width
+			i += width - 1
+
+			continue
+		}
+
 		out.push(input[i]!)
 		map.push(i)
 	}

package/lib/comma-spacing.ts

@@ -0,0 +1,68 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
+ *   reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
+ *   is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
+ *   the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
+ *
+ *   A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
+ *   BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
+ *   the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
+ *   comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
+ *   expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
+ */
+
+import { identityMap } from "#offset-map"
+
+/**
+ * Any letter or decimal digit in any script: the Unicode property classes `\p{L}` and `\p{Nd}`.
+ */
+const LETTER_OR_DIGIT = /[\p{L}\p{Nd}]/u
+const DIGIT = /\p{Nd}/u
+
+export interface CommaSpacingResult {
+	text: string
+	map: number[]
+	/**
+	 * How many spaces were inserted.
+	 */
+	inserted: number
+}
+
+/**
+ * A comma with a decimal digit on both sides: `12,5`, `1,000`.
+ */
+function isNumericSeparator(input: string, commaIndex: number): boolean {
+	return commaIndex > 0 && DIGIT.test(input[commaIndex - 1]!) && DIGIT.test(input[commaIndex + 1]!)
+}
+
+/**
+ * Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
+ * separator. Offset-map-correct: the inserted space maps to the comma.
+ */
+export function spaceAfterComma(input: string): CommaSpacingResult {
+	let inserted = 0
+	const out: string[] = []
+	const map: number[] = []
+
+	for (let i = 0; i < input.length; i++) {
+		const ch = input[i]!
+		out.push(ch)
+		map.push(i)
+
+		if (ch === "," && i + 1 < input.length && LETTER_OR_DIGIT.test(input[i + 1]!) && !isNumericSeparator(input, i)) {
+			out.push(" ")
+			map.push(i)
+			inserted += 1
+		}
+	}
+
+	if (!inserted) {
+		return { text: input, map: identityMap(input.length), inserted: 0 }
+	}
+
+	return { text: out.join(""), map, inserted }
+}

package/{compute.ts → lib/compute.ts}

@@ -7,13 +7,14 @@
  *   (always) with case-fold + abbreviation expansion (opt-in).
  */
 
-import { expandAbbreviations } from "./abbreviations.ts"
-import { applyCjkNormalization } from "./cjk.ts"
-import { applyNFC } from "./nfc.ts"
-import { composeMaps, identityMap } from "./offset-map.ts"
-import { applyPunctuation } from "./punctuation.ts"
-import type { NormalizationTransform, NormalizedInput, NormalizeOpts } from "./types.ts"
-import { collapseWhitespace } from "./whitespace.ts"
+import { expandAbbreviations } from "#abbreviations"
+import { applyCjkNormalization } from "#cjk"
+import { spaceAfterComma } from "#comma-spacing"
+import { applyNFC } from "#nfc"
+import { composeMaps, identityMap } from "#offset-map"
+import { applyPunctuation } from "#punctuation"
+import type { NormalizationTransform, NormalizedInput, NormalizeOpts } from "#types"
+import { collapseWhitespace } from "#whitespace"
 
 export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
 	const transforms: NormalizationTransform[] = []
@@ -32,7 +33,7 @@ export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
 	// parse) and fold full-width ASCII + the ideographic space. Runs after NFC so it sees composed
 	// forms, before punctuation/whitespace so any gap left by 〒 is then collapsed. No-op off-script.
 	{
-		const r = applyCjkNormalization(text)
+		const r = applyCjkNormalization(text, opts?.postalMark ? { postalMark: opts.postalMark } : {})
 
 		if (r.folded > 0 || r.stripped > 0) {
 			text = r.text
@@ -52,11 +53,26 @@ export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
 		}
 	}
 
+	// 2.5 Comma spacing — a comma glued to a letter gains a space. Runs after punctuation (so a folded
+	// full-width comma is seen) and before whitespace collapse (so a comma already followed by a space is
+	// never doubled).
+	{
+		const r = spaceAfterComma(text)
+
+		if (r.inserted > 0) {
+			text = r.text
+			map = composeMaps(map, r.map)
+			transforms.push({ kind: "space_after_comma", inserted: r.inserted })
+		}
+	}
+
 	// 3. Whitespace
 	{
 		const r = collapseWhitespace(text)
 
-		if (r.runs > 0 || r.text.length !== text.length) {
+		// Compare the TEXT, not its length: folding a lone tab to a space is length-preserving, and a
+		// length test reads that edit as no edit at all.
+		if (r.text !== text) {
 			text = r.text
 			map = composeMaps(map, r.map)
 			transforms.push({ kind: "collapse_whitespace", runs: r.runs })

package/lib/fold.ts

@@ -0,0 +1,30 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Match-key folds: the loss-y transforms a comparison applies to both sides.
+ */
+
+/**
+ * Lower-case, whitespace runs collapsed to one space, ends trimmed. The fold every string comparison applies before it
+ * compares; it keeps punctuation and diacritics, so `Saint-Étienne` and `Saint-Etienne` stay distinct.
+ */
+export function foldCaseWhitespace(input: string): string {
+	return input.toLowerCase().replaceAll(/\s+/gu, " ").trim()
+}
+
+/**
+ * Combining marks removed after NFD decomposition: `é` → `e`, `ł` unchanged (it is not a base plus a mark). Case and
+ * whitespace are untouched.
+ */
+export function stripCombiningMarks(input: string): string {
+	return input.normalize("NFD").replaceAll(/\p{M}/gu, "")
+}
+
+/**
+ * {@link foldCaseWhitespace} over the NFKC composition, with locale-aware lower-casing: fullwidth and compatibility
+ * forms fold together before comparison.
+ */
+export function foldNFKCWhitespace(input: string): string {
+	return input.normalize("NFKC").toLocaleLowerCase().replaceAll(/\s+/gu, " ").trim()
+}

package/{index.ts → lib/index.ts}

@@ -13,11 +13,12 @@
  *   See `docs/engineering/reference/STAGES.md` § Stage 1 for the contract.
  */
 
-export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "./abbreviations.ts"
-export { applyCjkNormalization, type CjkResult } from "./cjk.ts"
-export { normalize } from "./compute.ts"
-export { applyNFC } from "./nfc.ts"
-export { composeMaps, identityMap } from "./offset-map.ts"
-export { applyPunctuation } from "./punctuation.ts"
-export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "./types.ts"
-export { collapseWhitespace } from "./whitespace.ts"
+export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "#abbreviations"
+export { applyCjkNormalization, type CjkResult } from "#cjk"
+export { normalize } from "#compute"
+export { foldCaseWhitespace, foldNFKCWhitespace, stripCombiningMarks } from "#fold"
+export { applyNFC } from "#nfc"
+export { composeMaps, identityMap } from "#offset-map"
+export { applyPunctuation } from "#punctuation"
+export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "#types"
+export { collapseWhitespace } from "#whitespace"

package/{nfc.ts → lib/nfc.ts}

@@ -12,7 +12,7 @@
  *   variant selectors may produce off-by-one offsets — acceptable for v1.
  */
 
-import { identityMap } from "./offset-map.ts"
+import { identityMap } from "#offset-map"
 
 export interface NFCResult {
 	text: string

package/{punctuation.ts → lib/punctuation.ts}

@@ -7,7 +7,7 @@
  *   fancy character is a single codepoint that maps to a single ASCII char.
  */
 
-import { identityMap } from "./offset-map.ts"
+import { identityMap } from "#offset-map"
 
 const REPLACEMENTS = new Map<string, string>([
 	["‘", "'"], // ‘
@@ -16,8 +16,10 @@ const REPLACEMENTS = new Map<string, string>([
 	["”", '"'], // ”
 	["–", "-"], // – en dash
 	["—", "-"], // — em dash
+	["‐", "-"], // ‐ U+2010 hyphen
 	["−", "-"], // − U+2212 minus sign — Japanese IMEs emit this as the block separator (1−2−3)
 	["―", "-"], // ― U+2015 horizontal bar — another common JP block separator
+	["﹣", "-"], // ﹣ U+FE63 small hyphen-minus
 	["…", "..."], // … expands; tracked specially
 	[" ", " "], // non-breaking space
 ])

package/{types.ts → lib/types.ts}

@@ -19,6 +19,7 @@ export type NormalizationTransform =
 	| { kind: "expand_abbreviation"; from: string; to: string; at: SpanRange }
 	| { kind: "collapse_whitespace"; runs: number }
 	| { kind: "normalize_punctuation"; replacements: number }
+	| { kind: "space_after_comma"; inserted: number }
 	| { kind: "normalize_cjk"; folded: number; stripped: number }
 
 /**
@@ -75,4 +76,11 @@ export interface NormalizeOpts {
 	 * Skip Unicode NFC. Only use for debugging — production callers should leave on.
 	 */
 	skipNFC?: boolean
+	/**
+	 * What the CJK pass does with the postal mark 〒 (U+3012). `strip` (the default) drops it: it is byte-fallback OOV for
+	 * the SentencePiece tokenizer and fragments the digits after it. `keep` leaves it in place for a classifier whose
+	 * vocabulary carries it — the character-path CJK model was trained with the mark in front of every postcode, and
+	 * without it misreads the prefecture boundary (`885-0061 宮崎県都城市…` → prefecture `崎県都`).
+	 */
+	postalMark?: "strip" | "keep"
 }

package/{whitespace.ts → lib/whitespace.ts}

@@ -3,28 +3,34 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Whitespace collapse — runs of whitespace become a single ASCII space. Newlines and tabs are
- *   preserved as-is (segmentation grammar in QueryShape uses them); inline runs of spaces
- *   collapse. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
+ *   Whitespace collapse — a run of inline whitespace (`[ \t]`) becomes one ASCII space AT EVERY RUN
+ *   LENGTH, so a lone tab normalizes exactly as a doubled one does. Newlines (`\n`/`\r`) are preserved:
+ *   QueryShape's segmentation grammar reads one as a segment separator on a par with a comma, and it
+ *   reads a RAW tab the same way — folding every tab here is what keeps a tab-separated export from
+ *   re-segmenting the query. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
  *   trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
  *   tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
  *   quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
  *   as the whitespace trim, so span alignment survives.
  */
 
-import { identityMap } from "./offset-map.ts"
+import { identityMap } from "#offset-map"
 
 const INLINE_SPACE = /[ \t]/
 const ANY_SPACE = /[ \t\n\r]/
 /**
  * Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user commonly appends.
- * NOT leading (a leading token is load-bearing) and NOT quotes/brackets/parens.
+ * NOT leading (a leading token is required) and NOT quotes/brackets/parens.
  */
 const TRAILING_NOISE = /[ \t\n\r.,;:]/
 
 export interface WhitespaceResult {
 	text: string
 	map: number[]
+	/**
+	 * How many inline-whitespace runs were REWRITTEN — a run longer than one character, or a one-character run that was
+	 * not already an ASCII space. A run that was already a single space is not one of them.
+	 */
 	runs: number
 }
 
@@ -57,7 +63,10 @@ export function collapseWhitespace(input: string): WhitespaceResult {
 				i += 1
 			}
 
-			if (i - start > 1) {
+			// A one-character run counts too when the character is not already an ASCII space: the tab→space
+			// rewrite emitted above is a real edit, and `changed` is what decides whether the caller ever
+			// receives it — the early return below hands back the untouched input otherwise.
+			if (i - start > 1 || ch !== " ") {
 				changed = true
 				runs += 1
 			}
@@ -65,11 +74,6 @@ export function collapseWhitespace(input: string): WhitespaceResult {
 			continue
 		}
 
-		// Collapse \r\n into one
-		if (ch === "\n" && out.at(-1) === "\r") {
-			// Already handled in CR branch above by emitting both; skip combiner check
-		}
-
 		out.push(ch)
 		map.push(i)
 		i += 1

package/out/abbreviations.d.ts

@@ -11,7 +11,7 @@
  *   augmentation). Both sides should eventually share dictionaries; for v1 this dict is duplicated
  *   intentionally — refactoring sharing is a separate task.
  */
-import type { SpanRange } from "./types.ts";
+import type { SpanRange } from "#types";
 export interface AbbreviationEntry {
     from: string;
     to: string;
@@ -19,7 +19,7 @@ export interface AbbreviationEntry {
 /**
  * The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
  * it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
- * "no load-bearing trivia" rule means that data lives in exactly one place — here.
+ * "no required trivia" rule means that data lives in exactly one place — here.
  */
 export declare function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry>;
 export interface AbbreviationResult {

package/out/abbreviations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE3C,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AA4FD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAmEtF"}
+{"version":3,"file":"abbreviations.d.ts","sourceRoot":"","sources":["../lib/abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAEvC,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;CACV;AA4FD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAExF;AAED,MAAM,WAAW,kBAAkB;IAClC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,UAAU,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,SAAS,CAAA;KAAE,CAAC,CAAA;CAC9D;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAmEtF"}

package/out/abbreviations.js

@@ -69,7 +69,7 @@ const ES_ES_DICT = [
  * Dropping the entry is NOT a table edit: `fr-op3-halles-market-bonneuil` is a passing row that asserts street "Avenue
  * de la Convention" and an `address_point` tier, so it pins the current behaviour and a removal has to be measured on a
  * resolver-gauntlet run. The real repair is upstream — the geocode path hardcodes `locale: "und"` because Stage 1
- * precedes the parse, and `@mailwoman/locale-gate` cannot presently detect Spanish (it scores script class + known
+ * precedes the parse, and `@mailwoman/locale-hint` cannot presently detect Spanish (it scores script class + known
  * postcode formats, and a 5-digit ES/MX code is indistinguishable from a US ZIP).
  */
 const LOCALE_UNKNOWN_DICT = [
@@ -98,7 +98,7 @@ function getDictionary(locale) {
 /**
  * The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
  * it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
- * "no load-bearing trivia" rule means that data lives in exactly one place — here.
+ * "no required trivia" rule means that data lives in exactly one place — here.
  */
 export function abbreviationDictionary(locale) {
     return getDictionary(locale);

package/out/abbreviations.js.map

@@ -1 +1 @@
-{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,kGAAkG;IAClG,8FAA8F;IAC9F,iGAAiG;IACjG,2FAA2F;IAC3F,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,+FAA+F;IAC/F,oGAAoG;IACpG,qGAAqG;IACrG,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YAED,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1C,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}
+{"version":3,"file":"abbreviations.js","sourceRoot":"","sources":["../lib/abbreviations.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,MAAM,UAAU,GAAqC;IACpD,kCAAkC;IAClC,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE;IAC1B,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE;IACzB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,kBAAkB;IAClB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE;IAC1B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE;IACxB,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE;IAC3B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;IAC9B,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;CAC5B,CAAA;AAED,MAAM,UAAU,GAAqC;IACpD,kGAAkG;IAClG,8FAA8F;IAC9F,iGAAiG;IACjG,2FAA2F;IAC3F,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE;IAC7B,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,mBAAmB,GAAqC;IAC7D,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,WAAW,EAAE;IAC/B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,WAAW,EAAE;IAChC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE;IACjC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE;IAC5B,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE;CAC9B,CAAA;AAED,SAAS,aAAa,CAAC,MAA0B;IAChD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;IAE5C,mGAAmG;IACnG,qGAAqG;IACrG,4BAA4B;IAC5B,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,mBAAmB,CAAA;IAE5C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,+FAA+F;IAC/F,oGAAoG;IACpG,qGAAqG;IACrG,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAA;IAE1C,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAe;IACrD,OAAO,aAAa,CAAC,MAAM,CAAC,CAAA;AAC7B,CAAC;AAQD;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,MAAe;IACjE,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,UAAU,GAAuD,EAAE,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QACpB,kFAAkF;QAClF,8DAA8D;QAC9D,MAAM,WAAW,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7D,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,CAAC,IAAI,CAAC,CAAA;YAEN,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,CAAA;QAEf,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YACnD,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QACnC,MAAM,oBAAoB,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACvF,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAEvC,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;gBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;YACpB,CAAC;YAED,SAAQ;QACT,CAAC;QAED,gEAAgE;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,CAAA;YACvB,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,UAAU,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,oBAAoB;YAC1B,EAAE,EAAE,SAAS;YACb,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE;SAClC,CAAC,CAAA;QAEF,4FAA4F;QAC5F,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1C,CAAC,IAAI,CAAC,CAAA;QACP,CAAC;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC/C,CAAC"}

package/out/cjk.d.ts

@@ -16,6 +16,9 @@
  *       1, a full-width `－` always a hyphen — keyboards and copy-paste produce these constantly.
  *       Folding them to ASCII makes `１０４−００６１` and `104-0061` the same input.
  *   - **Fold the ideographic space (U+3000 → ' ').**
+ *   - **Fold half-width katakana (U+FF61–U+FF9F).** This matches the JP corpus builder. A voiced
+ *       pair such as `ﾃﾞ` contracts from two UTF-16 units to one `デ`; the output offset maps to the
+ *       first raw unit in the pair.
  *
  *   It deliberately does NOT convert **kanji numerals** (一二三…): place names carry numeral kanji as
  *   ordinary characters (三田 _Mita_, 四谷 _Yotsuya_), so a blind 三→3 would corrupt them.
@@ -23,7 +26,7 @@
  *   normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
  *   deferred.
  *
- *   Self-gating: a string with none of these characters returns identity, so Latin input is
+ *   Self-limiting: a string with none of these characters returns identity, so Latin input is
  *   untouched.
  */
 export interface CjkResult {
@@ -38,5 +41,7 @@ export interface CjkResult {
      */
     stripped: number;
 }
-export declare function applyCjkNormalization(input: string): CjkResult;
+export declare function applyCjkNormalization(input: string, opts?: {
+    postalMark?: "strip" | "keep";
+}): CjkResult;
 //# sourceMappingURL=cjk.d.ts.map

package/out/cjk.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAoBD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CA0C9D"}
+{"version":3,"file":"cjk.d.ts","sourceRoot":"","sources":["../lib/cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,MAAM,WAAW,SAAS;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,EAAE,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AA2BD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,OAAO,GAAG,MAAM,CAAA;CAAO,GAAG,SAAS,CAmE5G"}

package/out/cjk.js

@@ -16,6 +16,9 @@
  *       1, a full-width `－` always a hyphen — keyboards and copy-paste produce these constantly.
  *       Folding them to ASCII makes `１０４−００６１` and `104-0061` the same input.
  *   - **Fold the ideographic space (U+3000 → ' ').**
+ *   - **Fold half-width katakana (U+FF61–U+FF9F).** This matches the JP corpus builder. A voiced
+ *       pair such as `ﾃﾞ` contracts from two UTF-16 units to one `デ`; the output offset maps to the
+ *       first raw unit in the pair.
  *
  *   It deliberately does NOT convert **kanji numerals** (一二三…): place names carry numeral kanji as
  *   ordinary characters (三田 _Mita_, 四谷 _Yotsuya_), so a blind 三→3 would corrupt them.
@@ -23,10 +26,10 @@
  *   normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
  *   deferred.
  *
- *   Self-gating: a string with none of these characters returns identity, so Latin input is
+ *   Self-limiting: a string with none of these characters returns identity, so Latin input is
  *   untouched.
  */
-import { identityMap } from "./offset-map.js";
+import { identityMap } from "#offset-map";
 /**
  * ！.
  */
@@ -44,16 +47,23 @@ const IDEOGRAPHIC_SPACE = 0x30_00;
  * 〒.
  */
 const POSTAL_MARK = 0x30_12;
-export function applyCjkNormalization(input) {
+const HALFWIDTH_KATAKANA_START = 0xff_61;
+const HALFWIDTH_KATAKANA_END = 0xff_9f;
+const HALFWIDTH_VOICING_START = 0xff_9e;
+function isHalfwidthKatakana(code) {
+    return code >= HALFWIDTH_KATAKANA_START && code <= HALFWIDTH_KATAKANA_END;
+}
+export function applyCjkNormalization(input, opts = {}) {
     let folded = 0;
     let stripped = 0;
     const out = [];
     const map = [];
+    const stripPostalMark = opts.postalMark !== "keep";
     // All transformed code points are in the BMP (single UTF-16 unit), and every other character is
     // passed through verbatim, so a per-unit walk is safe for surrogate-pair input too.
     for (let i = 0; i < input.length; i++) {
         const code = input.charCodeAt(i);
-        if (code === POSTAL_MARK) {
+        if (code === POSTAL_MARK && stripPostalMark) {
             stripped += 1;
             continue; // drop — no addressing content; whitespace collapse later tidies any gap
         }
@@ -69,6 +79,22 @@ export function applyCjkNormalization(input) {
             folded += 1;
             continue;
         }
+        if (isHalfwidthKatakana(code)) {
+            const next = input.charCodeAt(i + 1);
+            const consumesVoicingMark = code < HALFWIDTH_VOICING_START && next >= HALFWIDTH_VOICING_START && next <= HALFWIDTH_KATAKANA_END;
+            const width = consumesVoicingMark ? 2 : 1;
+            const normalized = input
+                .slice(i, i + width)
+                .normalize("NFKC")
+                .normalize("NFC");
+            for (const unit of normalized) {
+                out.push(unit);
+                map.push(i);
+            }
+            folded += width;
+            i += width - 1;
+            continue;
+        }
         out.push(input[i]);
         map.push(i);
     }

package/out/cjk.js.map

@@ -1 +1 @@
-{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAe7C;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAE3B,MAAM,UAAU,qBAAqB,CAAC,KAAa;IAClD,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YAC1B,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}
+{"version":3,"file":"cjk.js","sourceRoot":"","sources":["../lib/cjk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAezC;;GAEG;AACH,MAAM,eAAe,GAAG,OAAO,CAAA;AAC/B;;GAEG;AACH,MAAM,aAAa,GAAG,OAAO,CAAA;AAC7B;;GAEG;AACH,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAClC,MAAM,iBAAiB,GAAG,OAAO,CAAA;AACjC;;GAEG;AACH,MAAM,WAAW,GAAG,OAAO,CAAA;AAC3B,MAAM,wBAAwB,GAAG,OAAO,CAAA;AACxC,MAAM,sBAAsB,GAAG,OAAO,CAAA;AACtC,MAAM,uBAAuB,GAAG,OAAO,CAAA;AAEvC,SAAS,mBAAmB,CAAC,IAAY;IACxC,OAAO,IAAI,IAAI,wBAAwB,IAAI,IAAI,IAAI,sBAAsB,CAAA;AAC1E,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,KAAa,EAAE,OAA0C,EAAE;IAChG,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,KAAK,MAAM,CAAA;IAElD,gGAAgG;IAChG,oFAAoF;IACpF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAEhC,IAAI,IAAI,KAAK,WAAW,IAAI,eAAe,EAAE,CAAC;YAC7C,QAAQ,IAAI,CAAC,CAAA;YAEb,SAAQ,CAAC,yEAAyE;QACnF,CAAC;QAED,IAAI,IAAI,IAAI,eAAe,IAAI,IAAI,IAAI,aAAa,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC,CAAA;YACxD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAChC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACX,MAAM,IAAI,CAAC,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEpC,MAAM,mBAAmB,GACxB,IAAI,GAAG,uBAAuB,IAAI,IAAI,IAAI,uBAAuB,IAAI,IAAI,IAAI,sBAAsB,CAAA;YAEpG,MAAM,KAAK,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;YAEzC,MAAM,UAAU,GAAG,KAAK;iBACtB,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC;iBACnB,SAAS,CAAC,MAAM,CAAC;iBACjB,SAAS,CAAC,KAAK,CAAC,CAAA;YAElB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC/B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACZ,CAAC;YAED,MAAM,IAAI,KAAK,CAAA;YACf,CAAC,IAAI,KAAK,GAAG,CAAC,CAAA;YAEd,SAAQ;QACT,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QACnB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,MAAM,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAA;IAC/E,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACrD,CAAC"}

package/out/comma-spacing.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Comma spacing — a comma directly followed by a letter gains one space, so `Biggin Hill,United Kingdom`
+ *   reaches stage 2 as `Biggin Hill, United Kingdom`. A tight comma is an ordinary typing pattern, and the parse
+ *   is the first stage that reads it differently: with no space, the tokenizer glues `,United` into one piece,
+ *   the segmenter still splits, and the decoder labels the glued piece as a street or a locality it is not.
+ *
+ *   A letter or a digit after the comma triggers the insertion, with one exception: a comma with a digit on
+ *   BOTH sides is a numeric separator (`12,5`, `1,000`) and is left as typed. `Köln,50733` has a letter before
+ *   the comma, so it is a list separator and gains the space. A space, punctuation or end of input after the
+ *   comma is left as typed. The inserted space maps to the comma's own offset, the same rule the `…`
+ *   expansion in `punctuation.ts` follows, so every span that starts after it still points into the raw input.
+ */
+export interface CommaSpacingResult {
+    text: string;
+    map: number[];
+    /**
+     * How many spaces were inserted.
+     */
+    inserted: number;
+}
+/**
+ * Insert one space after every comma that is directly followed by a letter or digit, unless the comma is a numeric
+ * separator. Offset-map-correct: the inserted space maps to the comma.
+ */
+export declare function spaceAfterComma(input: string): CommaSpacingResult;
+//# sourceMappingURL=comma-spacing.d.ts.map