@mailwoman/phrase-grouper 9.4.0 → 10.1.0

package/README.md

@@ -25,7 +25,7 @@ const groups = groupPhrases(normalizedInput, queryShape, localeHint)
 | --------------------- | ----------------------------------------------------------- |
 | `street_phrase`       | Number + capitalized words, hyphenated street names         |
 | `locality_phrase`     | Capitalized word sequence after comma, near region/postcode |
-| `venue_phrase`        | Leading capitalized word sequence before a street phrase    |
+| `venue_phrase`        | Capitalized word sequence at the start of a street phrase   |
 | `postcode`            | Known postcode format (ZIP5, UK outward, etc.)              |
 | `region_abbreviation` | US state / CA province / AU state abbreviations             |
 | `numeric`             | Standalone number (potential house number)                  |
@@ -55,7 +55,7 @@ kind-classifier → phrase-grouper → classifier (neural/rule-based) → ...
 
 ## Design
 
-- **Boundary discovery, not classification.** The phrase grouper answers "where
+- **Boundary discovery rather than classification.** The phrase grouper answers "where
   are the coherent units?" — the classifier answers "what _type_ is each unit?"
   This separation makes both problems easier.
 - **Bitter-lesson-safe:** uses only universal structural cues (proximity,
@@ -70,8 +70,8 @@ kind-classifier → phrase-grouper → classifier (neural/rule-based) → ...
 - [`@mailwoman/core`](../core) — pipeline coordinator that consumes phrase groups
 - [`@mailwoman/kind-classifier`](../kind-classifier) — preceding stage
 - [The Knowledge Ladder](https://mailwoman.ai/articles/concepts/the-knowledge-ladder/) — design rationale
-- [Staged Pipeline Contract](https://github.com/sister-software/mailwoman/blob/main/docs/engineering/reference/STAGES.mdx)
+- [Staged Pipeline Interface](https://github.com/sister-software/mailwoman/blob/main/docs/records/plan/reference/STAGES.mdx)
 
 ## License
 
-[AGPL-3.0-only](https://www.gnu.org/licenses/agpl-3.0.html)
+[AGPL-3.0-only](https://www.gnu.org/licenses/AGPL-3.0.html)

package/lib/group.ts

@@ -5,11 +5,12 @@
  *
  *   `groupPhrases` — Stage 2.7 entry point.
  *
- *   Composes per-kind rules over the normalized input + QueryShape and emits one `PhraseProposal` per
- *   fired rule. Overlapping proposals are expected — the reconciler (Stage 5) picks the best
- *   non-overlapping subset.
+ *   The function composes per-kind rules over normalized input and `QueryShape`. Each rule that fires emits
+ *   one `PhraseProposal`. Proposals can overlap. The reconciler (Stage 5) selects the best non-overlapping
+ *   subset.
  *
- *   See `docs/articles/concepts/the-knowledge-ladder.md` § Phrase grouper for the design rationale,
+ *   See `docs/records/site-2026-08/understanding/our-approach/the-knowledge-ladder.mdx` § Phrase grouper
+ *   for the design rationale.
  *   and `phrase-grouper/rules.ts` for per-rule documentation.
  */
 
@@ -30,9 +31,10 @@ import {
 import type { GroupPhrasesOpts } from "#types"
 
 /**
- * Walk every QueryShape segment and emit one `tokens-by-segment` list. Falls back to treating the whole input as a
- * single segment when QueryShape didn't supply segmentation (e.g. callers wiring the grouper into a path that bypasses
- * QueryShape).
+ * Walk every QueryShape segment and emit one `tokens-by-segment` list.
+ *
+ * Falls back to treating the whole input as a single segment when QueryShape didn't supply
+ * segmentation (e.g. Callers wiring the grouper into a path that bypasses QueryShape).
  */
 function tokensPerSegment(
 	text: string,
@@ -57,14 +59,18 @@ function tokensPerSegment(
 }
 
 /**
- * Synchronous, pure rule-based implementation. The async wrapper matches the pipeline contract.
+ * Synchronous, pure rule-based implementation.
+ *
+ * The async wrapper matches the pipeline interface.
  *
- * Emits overlapping proposals freely — the consumer (Stage 5 reconcile) picks the best non-overlapping subset under
- * semantic+hierarchical constraints. Confidence is a [0,1] score per proposal; relative ordering is what matters more
- * than absolute calibration at v0.5.0.
+ * Emits overlapping proposals freely — the consumer (Stage 5 reconcile) picks the best
+ * non-overlapping subset under semantic+hierarchical constraints.
+ * Confidence is a [0,1] score per proposal.
  *
- * The `_locale` parameter is reserved for future locale-aware rule packs (Japanese postcode/honorific patterns, French
- * preposition-bound localities) — currently unused.
+ * Relative ordering is what matters more than absolute calibration at v0.5.0.
+ *
+ * The `_locale` parameter is reserved for future locale-aware rule packs
+ * (Japanese postcode/honorific patterns, French preposition-bound localities) — currently unused.
  */
 export function groupPhrasesSync(
 	input: NormalizedInputLite,
@@ -90,8 +96,8 @@ export function groupPhrasesSync(
 		proposals.push(...scoreVenuePhrase(tokens, text, isFirst))
 	}
 
-	// Sort: descending confidence, ties broken by span start (left-to-right). Downstream Stage 5
-	// can rely on this ordering for top-k selection without re-sorting.
+	// Sort: descending confidence, ties broken by span start (left-to-right).
+	// Downstream Stage 5 can rely on this ordering for top-k selection without re-sorting.
 	proposals.sort((a, b) => {
 		if (a.confidence !== b.confidence) return b.confidence - a.confidence
 
@@ -102,8 +108,9 @@ export function groupPhrasesSync(
 }
 
 /**
- * Async variant matching `RuntimePipelineStages.groupPhrases`. Wraps the sync impl so the pipeline coordinator can use
- * it as-is.
+ * Async variant matching `RuntimePipelineStages.groupPhrases`.
+ *
+ * Wraps the sync impl so the pipeline coordinator can use it as-is.
  */
 export async function groupPhrases(
 	input: NormalizedInputLite,

package/lib/index.ts

@@ -13,10 +13,11 @@
  *
  *   Bitter-lesson-safe: only universal structural cues (proximity, punctuation, capitalization,
  *   hyphenation, format-shape repetition) — never place-name dictionaries. v0.5.0 ships the
- *   rule-based v1; learned 1-2M-param span proposer reserved for v0.5.1.
+ *   rule-based v1. learned 1-2M-param span proposer reserved for v0.5.1.
  *
- *   See `docs/articles/concepts/the-knowledge-ladder.md` § Phrase grouper for the design rationale
- *   and `docs/articles/plan/phases/PHASE_8_v0_5_0_fresh_slate.md` § E for the v0.5.0 thread.
+ *   See `docs/records/site-2026-08/understanding/our-approach/the-knowledge-ladder.mdx` § Phrase grouper
+ *   for the design rationale and `docs/records/plan/phases/PHASE_8_v0_5_0_fresh_slate.mdx` § E for the
+ *   v0.5.0 thread.
  */
 
 export { groupPhrases, groupPhrasesSync } from "#group"

package/lib/rules/tokens.ts

@@ -0,0 +1,132 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @file Phrase-grouping token spans.
+ */
+
+import { US_STATE_NAMES } from "@mailwoman/codex/us/state"
+import { Span } from "@mailwoman/core/tokenization"
+
+/**
+ * One token within a segment — absolute offsets into the normalized input.
+ *
+ * Built by `tokenizeSegment` from a (segment-text, segment-start) pair.
+ */
+export interface SegmentToken {
+	body: string
+	start: number
+	end: number
+}
+
+const WHITESPACE = /\s+/
+
+/**
+ * Neutral baseline confidence for phrase proposals when no structural cue
+ * (position, length, known suffix/prefix/marker, format-hit) lifts or penalizes the score.
+ *
+ * Each rule adds bonuses on top of this base (e.g. +0.15 for 2-token locality runs, +0.1 for tail-of-last-segment)
+ * and subtracts penalties (e.g. −0.2 for a known US region name that isn't at segment-tail).
+ */
+export const NEUTRAL_PROPOSAL_CONFIDENCE = 0.55
+
+/**
+ * Single-token US state/territory names, derived from the codex roster.
+ *
+ * Single-token scope is deliberate: the non-tail region-name penalty below reads one token at a
+ * time. and a multi-word name ("New York", "North Carolina") can never match a single token —
+ * deriving only the single-token names keeps the set equal to what the check can ever see.
+ */
+export const US_REGION_NAMES: ReadonlySet<string> = new Set(
+	US_STATE_NAMES.filter((name) => !name.includes(" ")).map((name) => name.toLowerCase())
+)
+
+/**
+ * Split a segment body into whitespace-separated tokens.
+ *
+ * Offsets are absolute into the original input (caller supplies the segment's `start` offset).
+ * Deliberately not `@mailwoman/query-shape`'s tokenizer: that one yields code-point
+ * class runs for the whole input, while this one handles segment-relative → absolute span math for the proposal spans —
+ * the two disagree on what a token boundary is.
+ */
+/**
+ * Digit count above which a pure-numeric token stops being unambiguously a house
+ * number. 1-4 digits are clearly numeric. 5 and up collide with postcodes,
+ * so the proposal is emitted at neutral confidence and the reconciler decides.
+ */
+export const MAX_UNAMBIGUOUS_HOUSE_NUMBER_DIGITS = 4
+
+/**
+ * Confidence for a pure-numeric token short enough to be unambiguous.
+ */
+export const UNAMBIGUOUS_NUMERIC_CONFIDENCE = 0.95
+
+/**
+ * Token count at which a run reads as a venue name in its own right rather than a stray pair.
+ */
+export const VENUE_RUN_MIN_TOKENS = 3
+
+/**
+ * Confidence for a venue run too short to clear {@link VENUE_RUN_MIN_TOKENS}.
+ */
+export const SHORT_VENUE_RUN_CONFIDENCE = 0.5
+
+/**
+ * Confidence added to a place-name run by its token count.
+ *
+ * Longer runs are less likely to be a coincidental adjacency, so they warrant more —
+ * the curve flattens past four tokens.
+ */
+export const PLACE_RUN_LENGTH_BONUS: ReadonlyMap<number, number> = new Map([
+	[2, 0.15],
+	[3, 0.12],
+])
+
+/**
+ * Bonus applied to place-name runs at or beyond {@link VENUE_RUN_MIN_TOKENS} + 1 tokens.
+ */
+export const LONG_PLACE_RUN_BONUS = 0.08
+
+/**
+ * Penalty for a US region name appearing away from the tail, where it is more likely a locality.
+ */
+export const NON_TAIL_REGION_NAME_PENALTY = 0.2
+
+export function tokenizeSegment(segmentBody: string, segmentStart: number): SegmentToken[] {
+	const tokens: SegmentToken[] = []
+	let i = 0
+
+	while (i < segmentBody.length) {
+		while (i < segmentBody.length && WHITESPACE.test(segmentBody[i]!)) {
+			i++
+		}
+
+		if (i >= segmentBody.length) break
+		const start = i
+
+		while (i < segmentBody.length && !WHITESPACE.test(segmentBody[i]!)) {
+			i++
+		}
+
+		tokens.push({
+			body: segmentBody.slice(start, i),
+			start: segmentStart + start,
+			end: segmentStart + i,
+		})
+	}
+
+	return tokens
+}
+
+/**
+ * Build a `Section` (Span instance) from absolute offsets into the original text.
+ */
+export function makeSection(text: string, start: number, end: number): Span {
+	return Span.from(text.slice(start, end), { start })
+}
+
+/**
+ * True when token body is non-empty digits only.
+ */
+export function isAllDigit(s: string): boolean {
+	return s.length > 0 && /^[0-9]+$/.test(s)
+}