@mailwoman/corpus 7.2.0 → 7.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +59 -27
  2. package/src/adapter.ts +153 -0
  3. package/src/adapters/ban/adapter.ts +189 -0
  4. package/src/adapters/ban/street-decompose.ts +98 -0
  5. package/src/adapters/fcc-bdc/adapter.ts +188 -0
  6. package/src/adapters/geonames/adapter.ts +185 -0
  7. package/src/adapters/geonames-postal/adapter.ts +111 -0
  8. package/src/adapters/gnaf/adapter.ts +139 -0
  9. package/src/adapters/gnaf/assemble.ts +200 -0
  10. package/src/adapters/index.ts +135 -0
  11. package/src/adapters/openaddresses/adapter.ts +237 -0
  12. package/src/adapters/overture/adapter.ts +149 -0
  13. package/src/adapters/state-hi-schools/adapter.ts +171 -0
  14. package/src/adapters/state-ia-contractors/adapter.ts +148 -0
  15. package/src/adapters/state-ny-notaries/adapter.ts +169 -0
  16. package/src/adapters/state-tx-notaries/adapter.ts +160 -0
  17. package/src/adapters/synth-po-box/adapter.ts +197 -0
  18. package/src/adapters/tiger/adapter.ts +232 -0
  19. package/src/adapters/tiger/street-decompose.ts +120 -0
  20. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
  21. package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
  22. package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
  23. package/src/adapters/usgov-nad/adapter.ts +338 -0
  24. package/src/adapters/usgov-nppes/adapter.ts +169 -0
  25. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
  26. package/src/adapters/wof-admin-jp/adapter.ts +176 -0
  27. package/src/adapters/wof-admin-json/adapter.ts +280 -0
  28. package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
  29. package/src/align.ts +317 -0
  30. package/src/build.ts +348 -0
  31. package/src/codex/us-fips-state.ts +109 -0
  32. package/src/format.ts +14 -0
  33. package/src/golden.ts +147 -0
  34. package/src/index.ts +28 -0
  35. package/src/license.ts +48 -0
  36. package/src/parquet-wrapper/index.ts +9 -0
  37. package/src/parquet-wrapper/reader.ts +67 -0
  38. package/src/parquet-wrapper/schema.ts +80 -0
  39. package/src/parquet-wrapper/writer.ts +102 -0
  40. package/src/parquet.ts +360 -0
  41. package/src/runner.ts +283 -0
  42. package/src/shard-recipes/anchor-absorption.ts +86 -0
  43. package/src/shard-recipes/boundary-stress.ts +87 -0
  44. package/src/shard-recipes/country-balanced.ts +471 -0
  45. package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
  46. package/src/shard-recipes/fr-admin-split.ts +212 -0
  47. package/src/shard-recipes/fr-bare-street.ts +85 -0
  48. package/src/shard-recipes/fr-fragment.ts +304 -0
  49. package/src/shard-recipes/fr-order.ts +283 -0
  50. package/src/shard-recipes/german.ts +206 -0
  51. package/src/shard-recipes/house-venue.ts +78 -0
  52. package/src/shard-recipes/index.ts +76 -0
  53. package/src/shard-recipes/intersection.ts +617 -0
  54. package/src/shard-recipes/locale.ts +415 -0
  55. package/src/shard-recipes/nl-postcode.ts +105 -0
  56. package/src/shard-recipes/no-fragment.ts +206 -0
  57. package/src/shard-recipes/no-street-led.ts +130 -0
  58. package/src/shard-recipes/no-street.ts +78 -0
  59. package/src/shard-recipes/po-box-cedex.ts +853 -0
  60. package/src/shard-recipes/po-box.ts +111 -0
  61. package/src/shard-recipes/scaffold.ts +187 -0
  62. package/src/shard-recipes/si-bare-village.ts +88 -0
  63. package/src/shard-recipes/street-affix.ts +549 -0
  64. package/src/shard-recipes/street-bare.ts +90 -0
  65. package/src/shard-recipes/street.ts +74 -0
  66. package/src/shard-recipes/unit.ts +315 -0
  67. package/src/split.ts +251 -0
  68. package/src/synthesize-anchor-absorption.ts +263 -0
  69. package/src/synthesize-boundary-stress.ts +504 -0
  70. package/src/synthesize-german.ts +227 -0
  71. package/src/synthesize-house-venue.ts +192 -0
  72. package/src/synthesize-intersection.ts +205 -0
  73. package/src/synthesize-no-street.ts +341 -0
  74. package/src/synthesize-po-box.ts +307 -0
  75. package/src/synthesize-street.ts +264 -0
  76. package/src/synthesize.ts +872 -0
  77. package/src/tokenize.ts +68 -0
  78. package/src/tools/align-shard.ts +67 -0
  79. package/src/tools/audit.ts +352 -0
  80. package/src/tools/corpus-stats.ts +183 -0
  81. package/src/tools/fetch/ban.ts +288 -0
  82. package/src/tools/fetch/download.ts +119 -0
  83. package/src/tools/fetch/hrsa.ts +65 -0
  84. package/src/tools/fetch/imls-pls.ts +162 -0
  85. package/src/tools/fetch/index.ts +121 -0
  86. package/src/tools/fetch/nad.ts +304 -0
  87. package/src/tools/fetch/nppes.ts +177 -0
  88. package/src/tools/fetch/openaddresses.ts +380 -0
  89. package/src/tools/fetch/state-hi-schools.ts +217 -0
  90. package/src/tools/fetch/state-sources.ts +142 -0
  91. package/src/tools/fetch/tiger-full.ts +318 -0
  92. package/src/tools/golden-expand.ts +524 -0
  93. package/src/tools/golden-promote.ts +304 -0
  94. package/src/tools/index.ts +24 -0
  95. package/src/tools/ingest-csv.ts +420 -0
  96. package/src/tools/jsonl-to-parquet.ts +200 -0
  97. package/src/tools/lint-shard-vocab.ts +357 -0
  98. package/src/tools/lint-shard.ts +502 -0
  99. package/src/tools/overlay-manifest.ts +141 -0
  100. package/src/tools/shard-kryptonite.ts +131 -0
  101. package/src/tools/shard-translit.ts +244 -0
  102. package/src/types.ts +213 -0
  103. package/src/wof-json.ts +236 -0
@@ -0,0 +1,227 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * German address synthesizer — multi-locale coverage (night-shift 2026-06-02, DE-1).
7
+ *
8
+ * The neural model is out-of-distribution on German: it truncates `Straußstraße`→`Strau` (exits at
9
+ * the ß-piece boundary), absorbs the house number into the street (`Hauptstraße 5` → one span),
10
+ * and mis-tags the native-order house number as a postcode (`Prenzlauer Allee 36, 10405 Berlin` →
11
+ * postcode `36`). The cause is ORDER: the model was trained US+FR (house-number-FIRST,
12
+ * postcode-AFTER-city), and never saw the German convention (house-number-AFTER-street,
13
+ * postcode-BEFORE-city). DE-0 confirmed the tokenizer round-trips German orthography cleanly, so
14
+ * this is a coverage gap, not a tokenizer ceiling.
15
+ *
16
+ * This generator produces the missing signal as a small targeted supplement shard
17
+ * (synthesis-as-supplement discipline: weight < 0.25, one-and-done). It does NOT synthesize
18
+ * German street names (German morphology is hard to fake) — it takes REAL German component tuples
19
+ * (from OpenAddresses Berlin/Saxony) and renders them in idiomatic German order via the OpenCage
20
+ * `DE` template (`formatAddress(..., "DE")` → `"Straußstraße 27, 12623 Berlin"`). The corpus
21
+ * aligner turns the row into BIO labels; every emitted component surface form occurs verbatim in
22
+ * `raw` so alignment lands.
23
+ */
24
+
25
+ import { formatAddress } from "./format.ts"
26
+ import type { CanonicalRow } from "./types.ts"
27
+
28
+ /** A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional. */
29
+ export interface LocaleBaseTuple {
30
+ house_number?: string
31
+ street: string
32
+ locality: string
33
+ /**
34
+ * A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
35
+ * column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
36
+ * Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
37
+ */
38
+ dependent_locality?: string
39
+ region?: string
40
+ postcode?: string
41
+ }
42
+ /** @deprecated Alias — use LocaleBaseTuple. */
43
+ export type GermanBaseTuple = LocaleBaseTuple
44
+
45
+ export interface SynthesizedLocaleRow {
46
+ raw: string
47
+ components: CanonicalRow["components"]
48
+ locale: string
49
+ }
50
+ /** @deprecated Alias — use SynthesizedLocaleRow. */
51
+ export type SynthesizedGermanRow = SynthesizedLocaleRow
52
+
53
+ export interface LocaleSynthesisOpts {
54
+ random?: () => number
55
+ /**
56
+ * Rendering order for the SAME components. `"native"` (default) uses the country's own template (DE →
57
+ * house-AFTER-street, postcode-BEFORE-city). `"international"` renders house-FIRST, postcode-AFTER-city — the US/GB
58
+ * layout that international feeds, US-centric systems, and our own OpenAddresses de-sample impose on non-US
59
+ * addresses. Training both teaches the model that a German address can arrive either way, so the eval's US-order
60
+ * rendering stops reading as a collapse. See `docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md` (the
61
+ * order-artifact correction).
62
+ */
63
+ order?: "native" | "international"
64
+ /**
65
+ * Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
66
+ * `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
67
+ * evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
68
+ * either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
69
+ * currently glues the suffix onto the city).
70
+ */
71
+ postcodeShape?: "conventional" | "as-source"
72
+ /**
73
+ * How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
74
+ * — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
75
+ * form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
76
+ * <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
77
+ * (DE/IT/NL) render identically under both. Mixing both stops an ES shard from teaching the comma as THE street→house
78
+ * boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
79
+ * house-first space-joined).
80
+ */
81
+ nativeHouseJoin?: "template" | "space"
82
+ }
83
+ /** @deprecated Alias — use LocaleSynthesisOpts. */
84
+ export type GermanSynthesisOpts = LocaleSynthesisOpts
85
+
86
+ /** ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country). */
87
+ const LOCALE_TAG: Record<string, string> = {
88
+ DE: "de-DE",
89
+ ES: "es-ES",
90
+ IT: "it-IT",
91
+ NL: "nl-NL",
92
+ GB: "en-GB",
93
+ FR: "fr-FR",
94
+ US: "en-US",
95
+ NZ: "en-NZ",
96
+ }
97
+
98
+ /**
99
+ * Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
100
+ * `raw`. NL is the case that needs it: OA stores `1011AB` but the OpenCage NL template emits the conventional spaced
101
+ * `1011 AB` (4 digits + space + 2 letters), which otherwise fails verbatim alignment and drops the row. Other countries
102
+ * pass through unchanged.
103
+ */
104
+ function normalizePostcode(postcode: string, country: string): string {
105
+ if (country === "NL") {
106
+ const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
107
+
108
+ if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
109
+ }
110
+
111
+ return postcode
112
+ }
113
+
114
+ /** True when `value` appears verbatim AND as a standalone token (so BIO alignment lands cleanly). */
115
+ function tokenPresent(raw: string, value: string): boolean {
116
+ if (!raw.includes(value)) return false
117
+ // Reject substring-of-a-larger-number collisions (e.g. house "2" inside postcode "12623").
118
+ const i = raw.indexOf(value)
119
+ const before = raw[i - 1]
120
+ const after = raw[i + value.length]
121
+ const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
122
+
123
+ if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
124
+
125
+ return true
126
+ }
127
+
128
+ /**
129
+ * Render one real tuple into an idiomatic, locale-ordered `{raw, components}` row via the OpenCage `country` template
130
+ * (DE → house-after-street + postcode-before-city; ES/IT the same; GB house-first; NL carries the `1012 LM` postcode),
131
+ * with light variation (drop house number / postcode some of the time). Returns `null` when the tuple is too thin or a
132
+ * component wouldn't align cleanly.
133
+ *
134
+ * Region handling is order-dependent: NATIVE order omits it (the native template absorbs the admin region into the
135
+ * postcode/city line, so it rarely renders verbatim and would break BIO alignment), while INTERNATIONAL order includes
136
+ * it in the tail ("City, Region Postcode" — the US/feed layout the eval uses; v0.9.3 / #327).
137
+ *
138
+ * Pass `opts.order: "international"` to render the same components house-first / postcode-after-city instead (see
139
+ * {@link LocaleSynthesisOpts.order}) — the layout international feeds impose on foreign addresses, and the one a
140
+ * native-order-trained model treats as a "collapse."
141
+ */
142
+ export function synthesizeLocaleRow(
143
+ base: LocaleBaseTuple,
144
+ country: string,
145
+ opts: LocaleSynthesisOpts = {}
146
+ ): SynthesizedLocaleRow | null {
147
+ const random = opts.random ?? Math.random
148
+ const order = opts.order ?? "native"
149
+
150
+ if (!base.street || !base.locality) return null
151
+
152
+ const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
153
+
154
+ // Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
155
+ // of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
156
+ // the envelope). The tokenPresent gate below drops the row if the template didn't surface it verbatim.
157
+ if (base.dependent_locality) {
158
+ components.dependent_locality = base.dependent_locality
159
+ }
160
+
161
+ // ~80% keep the house number (the rest are street-only forms, also idiomatic).
162
+ if (base.house_number && random() < 0.8) {
163
+ components.house_number = base.house_number
164
+ }
165
+
166
+ // ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
167
+ // `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
168
+ // normalizes the postcode itself, so a glued input can't survive rendering directly.
169
+ if (base.postcode && random() < 0.85) {
170
+ components.postcode = normalizePostcode(base.postcode, country)
171
+ }
172
+
173
+ // International order carries the REGION in the tail ("City, Region Postcode") — the layout real
174
+ // US/feed renderings (and our OA eval) use. v0.9.2 rendered international order WITHOUT the region,
175
+ // so the model never learned to segment the tail and mangled it at eval (region absorbed into the
176
+ // locality / locality dropped); v0.9.3 closes that gap (#327). Native order still drops the region
177
+ // (the native template absorbs it into the city line, which would break verbatim alignment).
178
+ if (order === "international" && base.region) {
179
+ components.region = base.region
180
+ }
181
+
182
+ // Native order uses the address's own country template; international order uses the US template —
183
+ // house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
184
+ // `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
185
+ const renderCountry = order === "international" ? "US" : country
186
+ let raw = formatAddress(components, renderCountry, { separator: ", " })
187
+
188
+ if (!raw) return null
189
+
190
+ // Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
191
+ // `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
192
+ // space-join (the substring isn't present), and skipped when the house number was dropped above.
193
+ if (order === "native" && opts.nativeHouseJoin === "space" && components.house_number) {
194
+ raw = raw.replace(
195
+ `${components.street}, ${components.house_number}`,
196
+ `${components.street} ${components.house_number}`
197
+ )
198
+ }
199
+
200
+ // `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
201
+ // the component back to the source's own surface (NL glued `1011AB`). Post-render because the
202
+ // OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
203
+ if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
204
+ const sourceForm = base.postcode.trim()
205
+
206
+ if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
207
+ raw = raw.replace(components.postcode, sourceForm)
208
+ components.postcode = sourceForm
209
+ }
210
+ }
211
+
212
+ // Every component must align — drop the row if the template didn't surface one verbatim, or a
213
+ // numeric component collides with a neighbouring digit run.
214
+ for (const value of Object.values(components)) {
215
+ if (!value || !tokenPresent(raw, value)) return null
216
+ }
217
+
218
+ return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
219
+ }
220
+
221
+ /** German wrapper over {@link synthesizeLocaleRow}. Kept for the build-german-shard caller + tests. */
222
+ export function synthesizeGermanRow(
223
+ base: LocaleBaseTuple,
224
+ opts: LocaleSynthesisOpts = {}
225
+ ): SynthesizedLocaleRow | null {
226
+ return synthesizeLocaleRow(base, "DE", opts)
227
+ }
@@ -0,0 +1,192 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * House-number + venue + street co-occurrence synthesizer. The v0.6.3 corrective shard.
7
+ *
8
+ * The v0.6.2 step-20K diagnostic showed that adding synth-no-street counter-distribution regressed
9
+ * house_number recall by ~4-5pp. DeepSeek's turn-8 root-cause:
10
+ *
11
+ * 1. Direct: `5th Avenue Theatre`-style adversarial venues teach the model that tokens like "5th"
12
+ * belong to venues, not house_numbers. (Fixed in `synthesize-no-street.ts` by removing
13
+ * digit+ordinal venue patterns.)
14
+ * 2. Distributional dilution: synth-no-street adds 122K rows where house_number is absent. The model's
15
+ * training distribution shifts toward "house_number is rare," and it under-emits the tag at
16
+ * inference.
17
+ *
18
+ * This synthesizer fixes #2 directly. Each emitted row has ALL of: house_number, street, venue,
19
+ * locality, region, postcode — a counter-example to "house_number is rare." Used as a companion
20
+ * shard to synth-no-street; the v0.6.3 config weights synth-no-street at 0.5 and
21
+ * synth-house-venue at 1.0 to recover the lost house_number signal.
22
+ *
23
+ * Real-world shape: business cards, mailing labels, store directories — `"123 Main St, Sunrise
24
+ * Bakery, Springfield, IL 62701"` is a perfectly ordinary address form.
25
+ *
26
+ * Venue pool: PLAIN_VENUES from `synthesize-no-street.ts` (re-exported here). Adversarial venues
27
+ * are deliberately NOT used here — the point is to teach co-occurrence, not to re-introduce
28
+ * decompose-mode pressure.
29
+ */
30
+
31
+ import type { CanonicalRow } from "./types.ts"
32
+
33
+ export interface HouseVenueBaseTuple {
34
+ locality: string
35
+ region: string
36
+ postcode: string
37
+ country: string
38
+ street?: string
39
+ houseNumber?: string
40
+ }
41
+
42
+ export type HouseVenueTemplate =
43
+ | "venue-after-street" // "123 Main St, Sunrise Bakery, Springfield, IL 02101"
44
+ | "venue-before-street" // "Sunrise Bakery, 123 Main St, Springfield, IL 02101"
45
+
46
+ export interface HouseVenueSynthesisOpts {
47
+ random?: () => number
48
+ forceTemplate?: HouseVenueTemplate
49
+ }
50
+
51
+ export interface SynthesizedHouseVenueRow {
52
+ raw: string
53
+ components: CanonicalRow["components"]
54
+ locale: string
55
+ template: HouseVenueTemplate
56
+ }
57
+
58
+ // ---------------------------------------------------------------------------------------------
59
+ // Venue pool — PLAIN, no street-typing tokens. The point of this shard is to teach
60
+ // house_number + venue coexistence, NOT to re-introduce decompose-mode pressure.
61
+ // Adversarial venue names live in `synthesize-no-street.ts`.
62
+ // ---------------------------------------------------------------------------------------------
63
+
64
+ const PLAIN_VENUES: ReadonlyArray<string> = [
65
+ "Bob's Pizza",
66
+ "Acme Corporation",
67
+ "Joe's Diner",
68
+ "Sunrise Bakery",
69
+ "Maple Leaf Cafe",
70
+ "Riverside Garden Center",
71
+ "Tech Solutions Inc",
72
+ "Pacific Industries",
73
+ "Atlantic Holdings",
74
+ "Stellar Consulting",
75
+ "Greenfield Partners",
76
+ "Mountain View Studio",
77
+ "The Daily Grind",
78
+ "Sunset Bistro",
79
+ "Harvest Moon Florist",
80
+ "Iron Forge Brewing",
81
+ "Crescent City Bookstore",
82
+ "Lighthouse Insurance Group",
83
+ "Pinecrest Veterinary",
84
+ "Westwood Realty",
85
+ ]
86
+
87
+ // ---------------------------------------------------------------------------------------------
88
+ // Fallback street pool for tuples that didn't carry a `street` field. Plain street names
89
+ // without typing-token ambiguity.
90
+ // ---------------------------------------------------------------------------------------------
91
+
92
+ const FALLBACK_STREETS: ReadonlyArray<string> = [
93
+ "Main St",
94
+ "Oak Ave",
95
+ "Pine Rd",
96
+ "Elm Dr",
97
+ "Cedar Ln",
98
+ "Maple Blvd",
99
+ "Birch Ct",
100
+ "Walnut Pl",
101
+ "Cherry Way",
102
+ "Spruce St",
103
+ "Park Ave",
104
+ "Lake Dr",
105
+ "Hill Rd",
106
+ "River Ln",
107
+ "Forest Blvd",
108
+ ]
109
+
110
+ // ---------------------------------------------------------------------------------------------
111
+ // House-number generator
112
+ // ---------------------------------------------------------------------------------------------
113
+
114
+ function randomHouseNumber(random: () => number): string {
115
+ // Generate a plain numeric house number 1-9999. No fractions/ranges — those land in
116
+ // `data/eval/falsehoods/numbers.jsonl` as known edge cases, not training material.
117
+ const digits = Math.floor(random() * 4) + 1
118
+ const max = Math.pow(10, digits)
119
+ const n = Math.floor(random() * max) + 1
120
+
121
+ return String(n)
122
+ }
123
+
124
+ function pick<T>(arr: ReadonlyArray<T>, random: () => number): T {
125
+ return arr[Math.floor(random() * arr.length)]!
126
+ }
127
+
128
+ function countryToLocale(country: string): string {
129
+ switch (country) {
130
+ case "US":
131
+ return "en-US"
132
+ case "CA":
133
+ return "en-CA"
134
+ case "GB":
135
+ return "en-GB"
136
+ case "AU":
137
+ return "en-AU"
138
+ case "FR":
139
+ return "fr-FR"
140
+ case "DE":
141
+ return "de-DE"
142
+ default:
143
+ return "en-US"
144
+ }
145
+ }
146
+
147
+ // ---------------------------------------------------------------------------------------------
148
+ // Synthesis
149
+ // ---------------------------------------------------------------------------------------------
150
+
151
+ export function synthesizeHouseVenueRow(
152
+ base: HouseVenueBaseTuple,
153
+ opts: HouseVenueSynthesisOpts = {}
154
+ ): SynthesizedHouseVenueRow | null {
155
+ const random = opts.random ?? Math.random
156
+ const locale = countryToLocale(base.country)
157
+ const template = opts.forceTemplate ?? (random() < 0.5 ? "venue-after-street" : "venue-before-street")
158
+
159
+ const venue = pick(PLAIN_VENUES, random)
160
+ const street = base.street ?? pick(FALLBACK_STREETS, random)
161
+ const houseNumber = base.houseNumber ?? randomHouseNumber(random)
162
+
163
+ const components: CanonicalRow["components"] = {
164
+ house_number: houseNumber,
165
+ street,
166
+ venue,
167
+ locality: base.locality,
168
+ region: base.region,
169
+ postcode: base.postcode,
170
+ }
171
+
172
+ let raw: string
173
+
174
+ switch (template) {
175
+ case "venue-after-street":
176
+ raw = `${houseNumber} ${street}, ${venue}, ${base.locality}, ${base.region} ${base.postcode}`
177
+ break
178
+ case "venue-before-street":
179
+ raw = `${venue}, ${houseNumber} ${street}, ${base.locality}, ${base.region} ${base.postcode}`
180
+ break
181
+ }
182
+
183
+ return { raw, components, locale, template }
184
+ }
185
+
186
+ /**
187
+ * Contract: every synthesized row carries BOTH house_number AND venue (the co-occurrence signal that synth-no-street's
188
+ * distributional shift cost the model). Used by tests + downstream consumers.
189
+ */
190
+ export function hasHouseNumberAndVenue(components: CanonicalRow["components"]): boolean {
191
+ return components.house_number !== undefined && components.venue !== undefined
192
+ }
@@ -0,0 +1,205 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Intersection synthesizer — v0.7 coverage fix (night-3, DeepSeek-decided).
7
+ *
8
+ * The 2026-05-29 harness diagnostic found the neural model emits `intersection_a`/`intersection_b`
9
+ * with ~0.0001 probability on canonical intersections ("Broadway & W 42nd St") — it never learned
10
+ * the tags, because the corpus has NO intersection training signal (no generator, and real-data
11
+ * adapters don't emit intersection-formatted rows). Intersections are 65 of the 376 harness
12
+ * assertions (17%), all 0% neural. This generator produces the missing signal as a small targeted
13
+ * supplement shard (synthesis-as-supplement discipline: weight < 0.25, one-and-done).
14
+ *
15
+ * Output is a `CanonicalRow` ({raw, components}); the corpus aligner turns it into BIO labels
16
+ * (B-/I-intersection_a, O on the connector, B-/I-intersection_b). Surface forms of both streets
17
+ * MUST occur verbatim in `raw` so alignment lands.
18
+ *
19
+ * US-idiomatic only (the harness intersection cases are US: "X & Y, City, ST ZIP").
20
+ */
21
+
22
+ import type { CanonicalRow } from "./types.ts"
23
+
24
+ /** Street name cores (no suffix) — proper-noun streets that often appear bare. */
25
+ const STREET_CORES = [
26
+ "Main",
27
+ "Oak",
28
+ "Elm",
29
+ "Maple",
30
+ "Pine",
31
+ "Cedar",
32
+ "Park",
33
+ "Lake",
34
+ "Hill",
35
+ "Washington",
36
+ "Lincoln",
37
+ "Jefferson",
38
+ "Madison",
39
+ "Franklin",
40
+ "Market",
41
+ "Broad",
42
+ "Church",
43
+ "Mill",
44
+ "Highland",
45
+ "Sunset",
46
+ "Union",
47
+ "Spring",
48
+ ] as const
49
+
50
+ /** Bare proper-noun streets that idiomatically take NO suffix. */
51
+ const BARE_NAMES = ["Broadway", "Wall", "Bourbon", "Esplanade", "Riverside", "Lakeshore"] as const
52
+
53
+ const ORDINALS = [
54
+ "1st",
55
+ "2nd",
56
+ "3rd",
57
+ "4th",
58
+ "5th",
59
+ "6th",
60
+ "7th",
61
+ "8th",
62
+ "9th",
63
+ "10th",
64
+ "42nd",
65
+ "23rd",
66
+ "34th",
67
+ ] as const
68
+
69
+ const SUFFIXES = ["St", "Ave", "Blvd", "Rd", "Dr", "Ln", "Way", "Pl", "Ct", "Pkwy", "Ter", "Cir"] as const
70
+
71
+ const DIRECTIONALS = ["N", "S", "E", "W", "NE", "NW", "SE", "SW"] as const
72
+
73
+ /**
74
+ * Connectors between the two streets. Whitespace-padded forms keep tokens clean for alignment. `@` added in v0.7.2 —
75
+ * the harness uses it ("Main St @ Second Ave") and v0.7.1 had never seen it.
76
+ */
77
+ const CONNECTORS = [" & ", " and ", " at ", " / ", " @ "] as const
78
+
79
+ export interface IntersectionBaseTuple {
80
+ locality: string
81
+ region: string
82
+ /** ZIP — optional; ~30% of synthetic intersections omit it (idiomatic). */
83
+ postcode?: string
84
+ country: string
85
+ }
86
+
87
+ export interface SynthesizedIntersectionRow {
88
+ raw: string
89
+ components: CanonicalRow["components"]
90
+ locale: string
91
+ }
92
+
93
+ export interface IntersectionSynthesisOpts {
94
+ random?: () => number
95
+ }
96
+
97
+ function pick<T>(arr: ReadonlyArray<T>, random: () => number): T {
98
+ return arr[Math.floor(random() * arr.length)]!
99
+ }
100
+
101
+ /** Build a single street surface form, e.g. "W 42nd St", "Broadway", "Main St", "N Oak Ave". */
102
+ function buildStreetName(random: () => number): string {
103
+ // ~20% bare proper-noun street (no suffix), else directional? + core/ordinal + suffix.
104
+ if (random() < 0.2) return pick(BARE_NAMES, random)
105
+
106
+ const parts: string[] = []
107
+
108
+ if (random() < 0.35) {
109
+ parts.push(pick(DIRECTIONALS, random))
110
+ }
111
+ parts.push(random() < 0.45 ? pick(ORDINALS, random) : pick(STREET_CORES, random))
112
+ parts.push(pick(SUFFIXES, random))
113
+
114
+ return parts.join(" ")
115
+ }
116
+
117
+ /**
118
+ * Synthesize one US intersection row. Returns null on the rare degenerate case where the two streets collide (so
119
+ * alignment never has two identical surface forms to disambiguate).
120
+ */
121
+ export function synthesizeIntersectionRow(
122
+ base: IntersectionBaseTuple,
123
+ opts: IntersectionSynthesisOpts = {}
124
+ ): SynthesizedIntersectionRow | null {
125
+ const random = opts.random ?? Math.random
126
+
127
+ if (base.country !== "US") return null
128
+
129
+ const a = buildStreetName(random)
130
+ let b = buildStreetName(random)
131
+ // Ensure distinct surface forms (and not a substring of each other — alignment needs unambiguous spans).
132
+ let tries = 0
133
+
134
+ while ((b === a || a.includes(b) || b.includes(a)) && tries++ < 8) {
135
+ b = buildStreetName(random)
136
+ }
137
+
138
+ if (b === a || a.includes(b) || b.includes(a)) return null
139
+
140
+ const connector = pick(CONNECTORS, random)
141
+ // "corner of" prefix variant (~20%) — still labels the two streets identically.
142
+ const cornerPrefix = random() < 0.2 ? "corner of " : ""
143
+
144
+ const components: CanonicalRow["components"] = { intersection_a: a, intersection_b: b }
145
+
146
+ // v0.7.2: ~60% BARE (no locality tail). v0.7.1 always appended ", City, ST", so the model learned
147
+ // to read post-intersection text as a locality and fumbled the harness's bare "X & Y" cases
148
+ // (mislabeling the second street as a locality). Match the eval distribution.
149
+ const bare = random() < 0.6
150
+ let raw: string
151
+
152
+ if (bare) {
153
+ raw = `${cornerPrefix}${a}${connector}${b}`
154
+ } else {
155
+ const includePostcode = base.postcode != null && random() < 0.7
156
+ const tail = includePostcode
157
+ ? `, ${base.locality}, ${base.region} ${base.postcode}`
158
+ : `, ${base.locality}, ${base.region}`
159
+ raw = `${cornerPrefix}${a}${connector}${b}${tail}`
160
+ components.locality = base.locality
161
+ components.region = base.region
162
+
163
+ if (includePostcode) {
164
+ components.postcode = base.postcode
165
+ }
166
+ }
167
+
168
+ return { raw, components, locale: "en-US" }
169
+ }
170
+
171
+ /** A small built-in US city/region/zip pool for standalone shard generation + tests. */
172
+ export const DEFAULT_US_BASES: ReadonlyArray<IntersectionBaseTuple> = [
173
+ { locality: "New York", region: "NY", postcode: "10036", country: "US" },
174
+ { locality: "Chicago", region: "IL", postcode: "60613", country: "US" },
175
+ { locality: "Los Angeles", region: "CA", postcode: "90012", country: "US" },
176
+ { locality: "Seattle", region: "WA", postcode: "98109", country: "US" },
177
+ { locality: "Austin", region: "TX", postcode: "78701", country: "US" },
178
+ { locality: "Portland", region: "OR", postcode: "97205", country: "US" },
179
+ { locality: "Denver", region: "CO", postcode: "80202", country: "US" },
180
+ { locality: "Boston", region: "MA", postcode: "02116", country: "US" },
181
+ { locality: "Miami", region: "FL", postcode: "33130", country: "US" },
182
+ { locality: "Atlanta", region: "GA", postcode: "30303", country: "US" },
183
+ ]
184
+
185
+ /** Generate `count` intersection rows over the provided bases (round-robin). */
186
+ export function generateIntersectionRows(
187
+ count: number,
188
+ bases: ReadonlyArray<IntersectionBaseTuple> = DEFAULT_US_BASES,
189
+ opts: IntersectionSynthesisOpts = {}
190
+ ): SynthesizedIntersectionRow[] {
191
+ const random = opts.random ?? Math.random
192
+ const out: SynthesizedIntersectionRow[] = []
193
+ let guard = 0
194
+
195
+ while (out.length < count && guard++ < count * 4) {
196
+ const base = bases[out.length % bases.length]!
197
+ const row = synthesizeIntersectionRow(base, { random })
198
+
199
+ if (row) {
200
+ out.push(row)
201
+ }
202
+ }
203
+
204
+ return out
205
+ }