@mailwoman/normalize 7.2.0 → 7.3.0

package/abbreviations.ts

@@ -0,0 +1,170 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Abbreviation expansion — a small bounded dictionary per locale. Initial dict covers en-US street
+ *   suffixes + directional prefixes. fr-FR + others added as needed.
+ *
+ *   This is the INVERSE of the corpus synthesis pass (which produces `Ave` from `Avenue` for
+ *   augmentation). Both sides should eventually share dictionaries; for v1 this dict is duplicated
+ *   intentionally — refactoring sharing is a separate task.
+ */
+
+import type { SpanRange } from "./types.ts"
+
+export interface AbbreviationEntry {
+	from: string // short form (case-insensitive match)
+	to: string // canonical long form
+}
+
+const EN_US_DICT: ReadonlyArray<AbbreviationEntry> = [
+	// Directional prefixes / suffixes
+	{ from: "N", to: "North" },
+	{ from: "S", to: "South" },
+	{ from: "E", to: "East" },
+	{ from: "W", to: "West" },
+	{ from: "NE", to: "Northeast" },
+	{ from: "NW", to: "Northwest" },
+	{ from: "SE", to: "Southeast" },
+	{ from: "SW", to: "Southwest" },
+	// Street suffixes
+	{ from: "St", to: "Street" },
+	{ from: "Ave", to: "Avenue" },
+	{ from: "Blvd", to: "Boulevard" },
+	{ from: "Rd", to: "Road" },
+	{ from: "Dr", to: "Drive" },
+	{ from: "Ct", to: "Court" },
+	{ from: "Ln", to: "Lane" },
+	{ from: "Pl", to: "Place" },
+	{ from: "Pkwy", to: "Parkway" },
+	{ from: "Hwy", to: "Highway" },
+	{ from: "Sq", to: "Square" },
+	{ from: "Ter", to: "Terrace" },
+]
+
+const FR_FR_DICT: ReadonlyArray<AbbreviationEntry> = [
+	{ from: "R", to: "Rue" },
+	{ from: "Bd", to: "Boulevard" },
+	{ from: "Av", to: "Avenue" },
+	{ from: "Bvd", to: "Boulevard" },
+	{ from: "Pl", to: "Place" },
+	{ from: "Imp", to: "Impasse" },
+	{ from: "Sq", to: "Square" },
+]
+
+/**
+ * #1002: the locale-UNKNOWN expansion set — the entries safe to apply when the input's locale hasn't been established
+ * yet (the geocode path expands BEFORE the parse, which is what determines the locale). Safe = multi-char,
+ * collision-free across the locale dictionaries, and never a plausible standalone token in the other locale (FR
+ * `Bd`/`Bvd`/`Imp` have no EN reading; `Av` reads Avenue in both). Deliberately EXCLUDED: the FR single letters (`R` →
+ * Rue would fire on Washington DC's literal "R St") and the EN suffixes (`St`, `Ave`, `Dr`, … — the model is
+ * trained-robust on those, and `St`/`Dr` are ambiguous with Saint/Doctor).
+ */
+const LOCALE_UNKNOWN_DICT: ReadonlyArray<AbbreviationEntry> = [
+	{ from: "Bd", to: "Boulevard" },
+	{ from: "Bvd", to: "Boulevard" },
+	{ from: "Boul", to: "Boulevard" },
+	{ from: "Av", to: "Avenue" },
+	{ from: "Imp", to: "Impasse" },
+]
+
+function getDictionary(locale: string | undefined): ReadonlyArray<AbbreviationEntry> {
+	const lc = (locale ?? "en-US").toLowerCase()
+
+	// BCP-47 "und" (undetermined) — the caller knows it does NOT know the locale yet (the geocode path
+	// expands before the parse). Only the collision-free multi-locale set applies; `undefined` keeps its
+	// historical en-US default.
+	if (lc === "und") return LOCALE_UNKNOWN_DICT
+
+	if (lc.startsWith("fr")) return FR_FR_DICT
+
+	return EN_US_DICT
+}
+
+/**
+ * The per-locale abbreviation table (short↔long), exposed so consumers can reuse the SAME data instead of duplicating
+ * it. The metamorphic gauntlet inverts this table to generate expanded→abbreviated perturbations (`Avenue`→`Ave`); the
+ * "no load-bearing trivia" rule means that data lives in exactly one place — here.
+ */
+export function abbreviationDictionary(locale?: string): ReadonlyArray<AbbreviationEntry> {
+	return getDictionary(locale)
+}
+
+export interface AbbreviationResult {
+	text: string
+	map: number[]
+	expansions: Array<{ from: string; to: string; at: SpanRange }>
+}
+
+/**
+ * Expand known abbreviations. Walks the input token-by-token (whitespace-delimited) and rewrites matching tokens to
+ * their canonical long form. The output map points every char of the expanded form to its position in the original
+ * short form (first char of input token).
+ *
+ * Case rules: match case-insensitively. Output form preserves the dictionary's canonical casing (`St` → `Street`, `st`
+ * → `Street`, `ST` → `Street`).
+ */
+export function expandAbbreviations(input: string, locale?: string): AbbreviationResult {
+	const dict = getDictionary(locale)
+	const lookup = new Map<string, string>()
+
+	for (const entry of dict) {
+		lookup.set(entry.from.toLowerCase(), entry.to)
+	}
+
+	const out: string[] = []
+	const map: number[] = []
+	const expansions: Array<{ from: string; to: string; at: SpanRange }> = []
+
+	let i = 0
+
+	while (i < input.length) {
+		const ch = input[i]!
+		// Walk to end of token (non-whitespace, non-punctuation). Unicode-letter-aware so
+		// "République" stays one token instead of fragmenting on 'é'.
+		const isTokenChar = (c: string) => /[\p{L}\p{N}'_-]/u.test(c)
+
+		if (!isTokenChar(ch)) {
+			out.push(ch)
+			map.push(i)
+			i += 1
+			continue
+		}
+		const start = i
+
+		while (i < input.length && isTokenChar(input[i]!)) {
+			i += 1
+		}
+		const token = input.slice(start, i)
+		const tokenWithTrailingDot = i < input.length && input[i] === "." ? `${token}.` : token
+		const lookupKey = token.replace(/\.$/, "").toLowerCase()
+		const expansion = lookup.get(lookupKey)
+
+		if (!expansion) {
+			for (let k = 0; k < token.length; k++) {
+				out.push(token[k]!)
+				map.push(start + k)
+			}
+			continue
+		}
+
+		// Emit expansion; map every char back to start of source token.
+		for (let k = 0; k < expansion.length; k++) {
+			out.push(expansion[k]!)
+			map.push(start + Math.min(k, token.length - 1))
+		}
+		expansions.push({
+			from: tokenWithTrailingDot,
+			to: expansion,
+			at: { start, end: i, body: token },
+		})
+
+		// Skip the trailing period if we consumed an abbreviation with one (e.g. "St." → "Street").
+		if (i < input.length && input[i] === ".") {
+			i += 1
+		}
+	}
+
+	return { text: out.join(""), map, expansions }
+}

package/cjk.ts

@@ -0,0 +1,85 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   CJK input normalization (Direction E, #291) — a conservative, character-level pass that runs as
+ *   part of `normalize()` so the parser sees a stable form of CJK addresses. It does only the
+ *   transformations that are UNAMBIGUOUS in any context:
+ *
+ *   - **Strip the postal mark 〒 (U+3012).** The JP cheap-probe found 〒 is byte-fallback OOV for the
+ *       SentencePiece tokenizer — it fragments into raw UTF-8 byte pieces and poisons the parse of
+ *       the digits right after it (the postcode gets mislabeled as a house number). It's a
+ *       "postcode follows" marker with no addressing content of its own, so dropping it is safe and
+ *       fixes the bug.
+ *   - **Fold full-width ASCII (U+FF01–U+FF5E → U+0021–U+007E).** A full-width `１` is always the digit
+ *       1, a full-width `－` always a hyphen — keyboards and copy-paste produce these constantly.
+ *       Folding them to ASCII makes `１０４−００６１` and `104-0061` the same input.
+ *   - **Fold the ideographic space (U+3000 → ' ').**
+ *
+ *   It deliberately does NOT convert **kanji numerals** (一二三…): place names carry numeral kanji as
+ *   ordinary characters (三田 _Mita_, 四谷 _Yotsuya_), so a blind 三→3 would corrupt them.
+ *   Disambiguating "this 三 is a block number, that one is part of a name" is parsing, not
+ *   normalization — deferred. Kana→kanji transliteration (ちょうめ→丁目) is dictionary work and likewise
+ *   deferred.
+ *
+ *   Self-gating: a string with none of these characters returns identity, so Latin input is
+ *   untouched.
+ */
+
+import { identityMap } from "./offset-map.ts"
+
+export interface CjkResult {
+	text: string
+	map: number[]
+	/** Count of characters folded in place (full-width → ASCII, ideographic space → ' '). */
+	folded: number
+	/** Count of characters dropped (the postal mark). */
+	stripped: number
+}
+
+const FULLWIDTH_START = 0xff01 // ！
+const FULLWIDTH_END = 0xff5e // ～
+const FULLWIDTH_TO_ASCII = 0xfee0 // U+FFxx − 0xFEE0 = U+00xx
+const IDEOGRAPHIC_SPACE = 0x3000
+const POSTAL_MARK = 0x3012 // 〒
+
+export function applyCjkNormalization(input: string): CjkResult {
+	let folded = 0
+	let stripped = 0
+	const out: string[] = []
+	const map: number[] = []
+
+	// All transformed code points are in the BMP (single UTF-16 unit), and every other character is
+	// passed through verbatim, so a per-unit walk is safe for surrogate-pair input too.
+	for (let i = 0; i < input.length; i++) {
+		const code = input.charCodeAt(i)
+
+		if (code === POSTAL_MARK) {
+			stripped += 1
+			continue // drop — no addressing content; whitespace collapse later tidies any gap
+		}
+
+		if (code >= FULLWIDTH_START && code <= FULLWIDTH_END) {
+			out.push(String.fromCharCode(code - FULLWIDTH_TO_ASCII))
+			map.push(i)
+			folded += 1
+			continue
+		}
+
+		if (code === IDEOGRAPHIC_SPACE) {
+			out.push(" ")
+			map.push(i)
+			folded += 1
+			continue
+		}
+		out.push(input[i]!)
+		map.push(i)
+	}
+
+	if (folded === 0 && stripped === 0) {
+		return { text: input, map: identityMap(input.length), folded: 0, stripped: 0 }
+	}
+
+	return { text: out.join(""), map, folded, stripped }
+}

package/compute.ts

@@ -0,0 +1,99 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `normalize(raw, opts)` — the Stage 1 entry point. Composes NFC + punctuation + whitespace
+ *   (always) with case-fold + abbreviation expansion (opt-in).
+ */
+
+import { expandAbbreviations } from "./abbreviations.ts"
+import { applyCjkNormalization } from "./cjk.ts"
+import { applyNFC } from "./nfc.ts"
+import { composeMaps, identityMap } from "./offset-map.ts"
+import { applyPunctuation } from "./punctuation.ts"
+import type { NormalizationTransform, NormalizedInput, NormalizeOpts } from "./types.ts"
+import { collapseWhitespace } from "./whitespace.ts"
+
+export function normalize(raw: string, opts?: NormalizeOpts): NormalizedInput {
+	const transforms: NormalizationTransform[] = []
+	let text = raw
+	let map = identityMap(raw.length)
+
+	// 1. NFC
+	if (!opts?.skipNFC) {
+		const r = applyNFC(text)
+		text = r.text
+		map = composeMaps(map, r.map)
+		transforms.push({ kind: "nfc", changed: r.changed })
+	}
+
+	// 1.5 CJK normalization — strip the postal mark 〒 (byte-fallback OOV that poisons the postcode
+	// parse) and fold full-width ASCII + the ideographic space. Runs after NFC so it sees composed
+	// forms, before punctuation/whitespace so any gap left by 〒 is then collapsed. No-op off-script.
+	{
+		const r = applyCjkNormalization(text)
+
+		if (r.folded > 0 || r.stripped > 0) {
+			text = r.text
+			map = composeMaps(map, r.map)
+			transforms.push({ kind: "normalize_cjk", folded: r.folded, stripped: r.stripped })
+		}
+	}
+
+	// 2. Punctuation
+	{
+		const r = applyPunctuation(text)
+
+		if (r.replacements > 0) {
+			text = r.text
+			map = composeMaps(map, r.map)
+			transforms.push({ kind: "normalize_punctuation", replacements: r.replacements })
+		}
+	}
+
+	// 3. Whitespace
+	{
+		const r = collapseWhitespace(text)
+
+		if (r.runs > 0 || r.text.length !== text.length) {
+			text = r.text
+			map = composeMaps(map, r.map)
+			transforms.push({ kind: "collapse_whitespace", runs: r.runs })
+		}
+	}
+
+	// 4. Abbreviation expansion (opt-in) — runs BEFORE case-fold so case-folding the canonical
+	// expansion form (e.g. "Street") gives a consistent final case.
+	if (opts?.expandAbbreviations) {
+		const r = expandAbbreviations(text, opts.locale)
+
+		if (r.expansions.length > 0) {
+			text = r.text
+			map = composeMaps(map, r.map)
+
+			for (const e of r.expansions) {
+				transforms.push({ kind: "expand_abbreviation", from: e.from, to: e.to, at: e.at })
+			}
+		}
+	}
+
+	// 5. Case fold (opt-in)
+	if (opts?.caseFold) {
+		const lc = text.toLocaleLowerCase(opts.locale)
+
+		if (lc !== text) {
+			text = lc
+			// Case-fold is identity-length for ASCII + most Latin; map unchanged.
+			transforms.push({ kind: "case_fold", locale: opts.locale ?? "und" })
+		}
+	}
+
+	return Object.freeze({
+		raw,
+		normalized: text,
+		transforms: Object.freeze(transforms) as NormalizationTransform[],
+		offsetMap: map,
+		appliedLocale: opts?.locale,
+	}) satisfies NormalizedInput
+}

package/index.ts

@@ -0,0 +1,23 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `@mailwoman/normalize` — Stage 1 of the runtime pipeline.
+ *
+ *   Deterministic input preprocessing: NFC, punctuation, whitespace, optional case-fold +
+ *   abbreviation expansion. Pure functions. Produces a `NormalizedInput` with a critical
+ *   `offsetMap` so downstream stages can map normalized-string spans back to raw-string character
+ *   offsets.
+ *
+ *   See `docs/articles/plan/reference/STAGES.md` § Stage 1 for the contract.
+ */
+
+export { type AbbreviationEntry, abbreviationDictionary, expandAbbreviations } from "./abbreviations.ts"
+export { applyCjkNormalization, type CjkResult } from "./cjk.ts"
+export { normalize } from "./compute.ts"
+export { applyNFC } from "./nfc.ts"
+export { composeMaps, identityMap } from "./offset-map.ts"
+export { applyPunctuation } from "./punctuation.ts"
+export type { NormalizationTransform, NormalizeOpts, NormalizedInput, SpanRange } from "./types.ts"
+export { collapseWhitespace } from "./whitespace.ts"

package/nfc.ts

@@ -0,0 +1,70 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Unicode NFC normalization. For inputs already in NFC (the common case) this is a no-op. When the
+ *   input has combining characters (`e` + `́` → `é`), NFC composes them — the normalized string can
+ *   be shorter than the raw.
+ *
+ *   Approximation: we walk the input grapheme-by-grapheme (best effort via codepoint stepping) and
+ *   map each output index to the start of its source sequence. Rare CJK edge cases involving
+ *   variant selectors may produce off-by-one offsets — acceptable for v1.
+ */
+
+import { identityMap } from "./offset-map.ts"
+
+export interface NFCResult {
+	text: string
+	/** `text[i]` came from `input[map[i]]`. */
+	map: number[]
+	changed: boolean
+}
+
+export function applyNFC(input: string): NFCResult {
+	const normalized = input.normalize("NFC")
+
+	if (normalized === input) {
+		return { text: input, map: identityMap(input.length), changed: false }
+	}
+
+	return { text: normalized, map: estimateNFCMap(input, normalized), changed: true }
+}
+
+/**
+ * Estimate per-output-codepoint offsets. Walks both strings in parallel; emits the next source index for each output
+ * position. Imprecise for combining sequences but correct for length-equal NFC outputs (the common length-changing case
+ * is when a sequence shortens).
+ */
+function estimateNFCMap(input: string, output: string): number[] {
+	const map: number[] = []
+	let inIdx = 0
+
+	for (let outIdx = 0; outIdx < output.length; outIdx++) {
+		map.push(inIdx)
+		const outCp = output.codePointAt(outIdx)!
+		const outStep = outCp > 0xffff ? 2 : 1
+
+		// Walk the input forward by at least one codepoint; absorb any combining marks (0x0300–0x036f).
+		if (inIdx < input.length) {
+			const inCp = input.codePointAt(inIdx)!
+			inIdx += inCp > 0xffff ? 2 : 1
+
+			while (inIdx < input.length) {
+				const nextCp = input.codePointAt(inIdx)!
+
+				if (nextCp >= 0x0300 && nextCp <= 0x036f) {
+					inIdx += nextCp > 0xffff ? 2 : 1
+				} else {
+					break
+				}
+			}
+		}
+
+		if (outStep === 2) {
+			outIdx += 1
+		}
+	}
+
+	return map
+}

package/offset-map.ts

@@ -0,0 +1,38 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Utilities for composing per-transform offset maps into the final `raw → normalized` map.
+ */
+
+/** Identity map for an input of length `n`: `[0, 1, 2, ..., n-1]`. */
+export function identityMap(n: number): number[] {
+	const m = new Array<number>(n)
+
+	for (let i = 0; i < n; i++) {
+		m[i] = i
+	}
+
+	return m
+}
+
+/**
+ * Compose `inputMap` (input → raw) with `transformMap` (output → input) to produce `outputMap` (output → raw).
+ *
+ * @example
+ * 	// raw = "350 5th" (chars 0..7, double space at 3-4) // input = "350 5th" (identity from
+ * 	raw, length 8) // output = "350 5th" (whitespace collapsed, length 7) // inputMap =
+ * 	[0,1,2,3,4,5,6,7] // transformMap = [0,1,2,3,5,6,7] (output[3]=' ' came from input[3];
+ * 	output[4]='5' from input[5]) // composed = [0,1,2,3,5,6,7]
+ */
+export function composeMaps(inputMap: number[], transformMap: number[]): number[] {
+	const out = new Array<number>(transformMap.length)
+
+	for (let i = 0; i < transformMap.length; i++) {
+		const j = transformMap[i]!
+		out[i] = inputMap[j] ?? j
+	}
+
+	return out
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/normalize",
-  "version": "7.2.0",
+  "version": "7.3.0",
   "description": "Stage 1 of the runtime pipeline — deterministic input preprocessing (Unicode NFC, punctuation, whitespace, abbreviation). Pure functions, no ML.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "repository": {
@@ -12,18 +12,32 @@
     "out/**/*.js",
     "out/**/*.js.map",
     "out/**/*.d.ts",
-    "out/**/*.d.ts.map"
+    "out/**/*.d.ts.map",
+    "*.ts",
+    "*.tsx",
+    "**/*.ts",
+    "**/*.tsx",
+    "!*.test.ts",
+    "!*.test.tsx",
+    "!**/*.test.ts",
+    "!**/*.test.tsx"
   ],
   "type": "module",
   "exports": {
     "./package.json": "./package.json",
     ".": {
-      "node": "./index.ts",
-      "default": "./out/index.js",
-      "types": "./out/index.d.ts"
+      "types": "./out/index.d.ts",
+      "default": "./out/index.js"
     }
   },
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "exports": {
+      "./package.json": "./package.json",
+      ".": {
+        "types": "./out/index.d.ts",
+        "default": "./out/index.js"
+      }
+    }
   }
 }

package/punctuation.ts

@@ -0,0 +1,60 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Punctuation normalization — fancy quotes / dashes to ASCII equivalents. Identity-length: every
+ *   fancy character is a single codepoint that maps to a single ASCII char.
+ */
+
+import { identityMap } from "./offset-map.ts"
+
+const REPLACEMENTS = new Map<string, string>([
+	["‘", "'"], // ‘
+	["’", "'"], // ’
+	["“", '"'], // “
+	["”", '"'], // ”
+	["–", "-"], // – en dash
+	["—", "-"], // — em dash
+	["−", "-"], // − U+2212 minus sign — Japanese IMEs emit this as the block separator (1−2−3)
+	["―", "-"], // ― U+2015 horizontal bar — another common JP block separator
+	["…", "..."], // … expands; tracked specially
+	[" ", " "], // non-breaking space
+])
+
+export interface PunctuationResult {
+	text: string
+	map: number[]
+	replacements: number
+}
+
+export function applyPunctuation(input: string): PunctuationResult {
+	let changed = false
+	let replacements = 0
+	const out: string[] = []
+	const map: number[] = []
+
+	for (let i = 0; i < input.length; i++) {
+		const ch = input[i]!
+		const sub = REPLACEMENTS.get(ch)
+
+		if (sub === undefined) {
+			out.push(ch)
+			map.push(i)
+		} else {
+			changed = true
+			replacements += 1
+
+			for (let k = 0; k < sub.length; k++) {
+				out.push(sub[k]!)
+				map.push(i)
+			}
+		}
+	}
+
+	if (!changed) {
+		return { text: input, map: identityMap(input.length), replacements: 0 }
+	}
+
+	return { text: out.join(""), map, replacements }
+}

package/types.ts

@@ -0,0 +1,58 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+export interface SpanRange {
+	start: number
+	end: number
+	body: string
+}
+
+/** A single normalization step, as recorded on `NormalizedInput.transforms`. */
+export type NormalizationTransform =
+	| { kind: "nfc"; changed: boolean }
+	| { kind: "case_fold"; locale: string }
+	| { kind: "expand_abbreviation"; from: string; to: string; at: SpanRange }
+	| { kind: "collapse_whitespace"; runs: number }
+	| { kind: "normalize_punctuation"; replacements: number }
+	| { kind: "normalize_cjk"; folded: number; stripped: number }
+
+/**
+ * Result of running `normalize()` on a raw input string.
+ *
+ * `offsetMap[i]` is the index in `raw` from which `normalized[i]` came. For multi-character source sequences (NFC
+ * composition, whitespace collapse, abbreviation expansion), each output char points to the FIRST source char by
+ * convention.
+ */
+export interface NormalizedInput {
+	/** The input as the caller sent it. */
+	raw: string
+
+	/** Canonical form, all transforms applied. */
+	normalized: string
+
+	/** Ordered record of what was done. */
+	transforms: NormalizationTransform[]
+
+	/** `normalized[i]` came from `raw[offsetMap[i]]`. Length === normalized.length. */
+	offsetMap: number[]
+
+	/** The locale used for case-folding + abbreviation rules. */
+	appliedLocale?: string
+}
+
+export interface NormalizeOpts {
+	/** Locale hint for case-folding + abbreviation dictionaries. */
+	locale?: string
+
+	/** Apply locale-aware lowercasing. Default: false (preserve case for downstream consumers). */
+	caseFold?: boolean
+
+	/** Expand known abbreviations (`St` → `Street`, `NW` → `Northwest`, etc.). Default: false. */
+	expandAbbreviations?: boolean
+
+	/** Skip Unicode NFC. Only use for debugging — production callers should leave on. */
+	skipNFC?: boolean
+}

package/whitespace.ts

@@ -0,0 +1,96 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Whitespace collapse — runs of whitespace become a single ASCII space. Newlines and tabs are
+ *   preserved as-is (segmentation grammar in QueryShape uses them); inline runs of spaces
+ *   collapse. The trailing trim also drops trailing sentence-punctuation NOISE (#829 tail): a
+ *   trailing `.`/`,`/`;`/`:` (e.g. `…Washington DC.`) glues onto the last token and drops the street
+ *   tier (`address_point`→`admin`). Trailing only + a conservative set — leading punctuation and
+ *   quotes/brackets are never touched (they can be meaningful). Offset-map-correct via the same slice
+ *   as the whitespace trim, so span alignment survives.
+ */
+
+import { identityMap } from "./offset-map.ts"
+
+const INLINE_SPACE = /[ \t]/
+const ANY_SPACE = /[ \t\n\r]/
+// Trailing NOISE trimmed off the END of the input: whitespace + the sentence-punctuation that a user
+// commonly appends. NOT leading (a leading token is load-bearing) and NOT quotes/brackets/parens.
+const TRAILING_NOISE = /[ \t\n\r.,;:]/
+
+export interface WhitespaceResult {
+	text: string
+	map: number[]
+	runs: number
+}
+
+export function collapseWhitespace(input: string): WhitespaceResult {
+	let changed = false
+	let runs = 0
+	const out: string[] = []
+	const map: number[] = []
+	let i = 0
+
+	while (i < input.length) {
+		const ch = input[i]!
+
+		if (ch === "\n" || ch === "\r") {
+			// Preserve newlines as segment separators.
+			out.push(ch)
+			map.push(i)
+			i += 1
+			continue
+		}
+
+		if (INLINE_SPACE.test(ch)) {
+			out.push(" ")
+			map.push(i)
+			const start = i
+			i += 1
+
+			while (i < input.length && INLINE_SPACE.test(input[i]!)) {
+				i += 1
+			}
+
+			if (i - start > 1) {
+				changed = true
+				runs += 1
+			}
+			continue
+		}
+
+		// Collapse \r\n into one
+		if (ch === "\n" && out[out.length - 1] === "\r") {
+			// Already handled in CR branch above by emitting both; skip combiner check
+		}
+		out.push(ch)
+		map.push(i)
+		i += 1
+	}
+
+	// Trim leading whitespace, and trailing whitespace + sentence-punctuation noise (#829 tail).
+	let lead = 0
+
+	while (lead < out.length && ANY_SPACE.test(out[lead]!)) {
+		lead += 1
+	}
+	let trail = out.length
+
+	while (trail > lead && TRAILING_NOISE.test(out[trail - 1]!)) {
+		trail -= 1
+	}
+
+	if (lead > 0 || trail < out.length) {
+		changed = true
+	}
+	const trimmedOut = out.slice(lead, trail)
+	const trimmedMap = map.slice(lead, trail)
+
+	if (!changed && trimmedOut.length === input.length) {
+		return { text: input, map: identityMap(input.length), runs: 0 }
+	}
+
+	return { text: trimmedOut.join(""), map: trimmedMap, runs }
+}