@mailwoman/corpus 7.2.0 → 7.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +59 -27
- package/src/adapter.ts +153 -0
- package/src/adapters/ban/adapter.ts +189 -0
- package/src/adapters/ban/street-decompose.ts +98 -0
- package/src/adapters/fcc-bdc/adapter.ts +188 -0
- package/src/adapters/geonames/adapter.ts +185 -0
- package/src/adapters/geonames-postal/adapter.ts +111 -0
- package/src/adapters/gnaf/adapter.ts +139 -0
- package/src/adapters/gnaf/assemble.ts +200 -0
- package/src/adapters/index.ts +135 -0
- package/src/adapters/openaddresses/adapter.ts +237 -0
- package/src/adapters/overture/adapter.ts +149 -0
- package/src/adapters/state-hi-schools/adapter.ts +171 -0
- package/src/adapters/state-ia-contractors/adapter.ts +148 -0
- package/src/adapters/state-ny-notaries/adapter.ts +169 -0
- package/src/adapters/state-tx-notaries/adapter.ts +160 -0
- package/src/adapters/synth-po-box/adapter.ts +197 -0
- package/src/adapters/tiger/adapter.ts +232 -0
- package/src/adapters/tiger/street-decompose.ts +120 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
- package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
- package/src/adapters/usgov-nad/adapter.ts +338 -0
- package/src/adapters/usgov-nppes/adapter.ts +169 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
- package/src/adapters/wof-admin-jp/adapter.ts +176 -0
- package/src/adapters/wof-admin-json/adapter.ts +280 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
- package/src/align.ts +317 -0
- package/src/build.ts +348 -0
- package/src/codex/us-fips-state.ts +109 -0
- package/src/format.ts +14 -0
- package/src/golden.ts +147 -0
- package/src/index.ts +28 -0
- package/src/license.ts +48 -0
- package/src/parquet-wrapper/index.ts +9 -0
- package/src/parquet-wrapper/reader.ts +67 -0
- package/src/parquet-wrapper/schema.ts +80 -0
- package/src/parquet-wrapper/writer.ts +102 -0
- package/src/parquet.ts +360 -0
- package/src/runner.ts +283 -0
- package/src/shard-recipes/anchor-absorption.ts +86 -0
- package/src/shard-recipes/boundary-stress.ts +87 -0
- package/src/shard-recipes/country-balanced.ts +471 -0
- package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
- package/src/shard-recipes/fr-admin-split.ts +212 -0
- package/src/shard-recipes/fr-bare-street.ts +85 -0
- package/src/shard-recipes/fr-fragment.ts +304 -0
- package/src/shard-recipes/fr-order.ts +283 -0
- package/src/shard-recipes/german.ts +206 -0
- package/src/shard-recipes/house-venue.ts +78 -0
- package/src/shard-recipes/index.ts +76 -0
- package/src/shard-recipes/intersection.ts +617 -0
- package/src/shard-recipes/locale.ts +415 -0
- package/src/shard-recipes/nl-postcode.ts +105 -0
- package/src/shard-recipes/no-fragment.ts +206 -0
- package/src/shard-recipes/no-street-led.ts +130 -0
- package/src/shard-recipes/no-street.ts +78 -0
- package/src/shard-recipes/po-box-cedex.ts +853 -0
- package/src/shard-recipes/po-box.ts +111 -0
- package/src/shard-recipes/scaffold.ts +187 -0
- package/src/shard-recipes/si-bare-village.ts +88 -0
- package/src/shard-recipes/street-affix.ts +549 -0
- package/src/shard-recipes/street-bare.ts +90 -0
- package/src/shard-recipes/street.ts +74 -0
- package/src/shard-recipes/unit.ts +315 -0
- package/src/split.ts +251 -0
- package/src/synthesize-anchor-absorption.ts +263 -0
- package/src/synthesize-boundary-stress.ts +504 -0
- package/src/synthesize-german.ts +227 -0
- package/src/synthesize-house-venue.ts +192 -0
- package/src/synthesize-intersection.ts +205 -0
- package/src/synthesize-no-street.ts +341 -0
- package/src/synthesize-po-box.ts +307 -0
- package/src/synthesize-street.ts +264 -0
- package/src/synthesize.ts +872 -0
- package/src/tokenize.ts +68 -0
- package/src/tools/align-shard.ts +67 -0
- package/src/tools/audit.ts +352 -0
- package/src/tools/corpus-stats.ts +183 -0
- package/src/tools/fetch/ban.ts +288 -0
- package/src/tools/fetch/download.ts +119 -0
- package/src/tools/fetch/hrsa.ts +65 -0
- package/src/tools/fetch/imls-pls.ts +162 -0
- package/src/tools/fetch/index.ts +121 -0
- package/src/tools/fetch/nad.ts +304 -0
- package/src/tools/fetch/nppes.ts +177 -0
- package/src/tools/fetch/openaddresses.ts +380 -0
- package/src/tools/fetch/state-hi-schools.ts +217 -0
- package/src/tools/fetch/state-sources.ts +142 -0
- package/src/tools/fetch/tiger-full.ts +318 -0
- package/src/tools/golden-expand.ts +524 -0
- package/src/tools/golden-promote.ts +304 -0
- package/src/tools/index.ts +24 -0
- package/src/tools/ingest-csv.ts +420 -0
- package/src/tools/jsonl-to-parquet.ts +200 -0
- package/src/tools/lint-shard-vocab.ts +357 -0
- package/src/tools/lint-shard.ts +502 -0
- package/src/tools/overlay-manifest.ts +141 -0
- package/src/tools/shard-kryptonite.ts +131 -0
- package/src/tools/shard-translit.ts +244 -0
- package/src/types.ts +213 -0
- package/src/wof-json.ts +236 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* German address synthesizer — multi-locale coverage (night-shift 2026-06-02, DE-1).
|
|
7
|
+
*
|
|
8
|
+
* The neural model is out-of-distribution on German: it truncates `Straußstraße`→`Strau` (exits at
|
|
9
|
+
* the ß-piece boundary), absorbs the house number into the street (`Hauptstraße 5` → one span),
|
|
10
|
+
* and mis-tags the native-order house number as a postcode (`Prenzlauer Allee 36, 10405 Berlin` →
|
|
11
|
+
* postcode `36`). The cause is ORDER: the model was trained US+FR (house-number-FIRST,
|
|
12
|
+
* postcode-AFTER-city), and never saw the German convention (house-number-AFTER-street,
|
|
13
|
+
* postcode-BEFORE-city). DE-0 confirmed the tokenizer round-trips German orthography cleanly, so
|
|
14
|
+
* this is a coverage gap, not a tokenizer ceiling.
|
|
15
|
+
*
|
|
16
|
+
* This generator produces the missing signal as a small targeted supplement shard
|
|
17
|
+
* (synthesis-as-supplement discipline: weight < 0.25, one-and-done). It does NOT synthesize
|
|
18
|
+
* German street names (German morphology is hard to fake) — it takes REAL German component tuples
|
|
19
|
+
* (from OpenAddresses Berlin/Saxony) and renders them in idiomatic German order via the OpenCage
|
|
20
|
+
* `DE` template (`formatAddress(..., "DE")` → `"Straußstraße 27, 12623 Berlin"`). The corpus
|
|
21
|
+
* aligner turns the row into BIO labels; every emitted component surface form occurs verbatim in
|
|
22
|
+
* `raw` so alignment lands.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { formatAddress } from "./format.ts"
|
|
26
|
+
import type { CanonicalRow } from "./types.ts"
|
|
27
|
+
|
|
28
|
+
/** A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional. */
|
|
29
|
+
export interface LocaleBaseTuple {
|
|
30
|
+
house_number?: string
|
|
31
|
+
street: string
|
|
32
|
+
locality: string
|
|
33
|
+
/**
|
|
34
|
+
* A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
|
|
35
|
+
* column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
|
|
36
|
+
* Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
|
|
37
|
+
*/
|
|
38
|
+
dependent_locality?: string
|
|
39
|
+
region?: string
|
|
40
|
+
postcode?: string
|
|
41
|
+
}
|
|
42
|
+
/** @deprecated Alias — use LocaleBaseTuple. */
|
|
43
|
+
export type GermanBaseTuple = LocaleBaseTuple
|
|
44
|
+
|
|
45
|
+
export interface SynthesizedLocaleRow {
|
|
46
|
+
raw: string
|
|
47
|
+
components: CanonicalRow["components"]
|
|
48
|
+
locale: string
|
|
49
|
+
}
|
|
50
|
+
/** @deprecated Alias — use SynthesizedLocaleRow. */
|
|
51
|
+
export type SynthesizedGermanRow = SynthesizedLocaleRow
|
|
52
|
+
|
|
53
|
+
export interface LocaleSynthesisOpts {
|
|
54
|
+
random?: () => number
|
|
55
|
+
/**
|
|
56
|
+
* Rendering order for the SAME components. `"native"` (default) uses the country's own template (DE →
|
|
57
|
+
* house-AFTER-street, postcode-BEFORE-city). `"international"` renders house-FIRST, postcode-AFTER-city — the US/GB
|
|
58
|
+
* layout that international feeds, US-centric systems, and our own OpenAddresses de-sample impose on non-US
|
|
59
|
+
* addresses. Training both teaches the model that a German address can arrive either way, so the eval's US-order
|
|
60
|
+
* rendering stops reading as a collapse. See `docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md` (the
|
|
61
|
+
* order-artifact correction).
|
|
62
|
+
*/
|
|
63
|
+
order?: "native" | "international"
|
|
64
|
+
/**
|
|
65
|
+
* Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
|
|
66
|
+
* `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
|
|
67
|
+
* evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
|
|
68
|
+
* either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
|
|
69
|
+
* currently glues the suffix onto the city).
|
|
70
|
+
*/
|
|
71
|
+
postcodeShape?: "conventional" | "as-source"
|
|
72
|
+
/**
|
|
73
|
+
* How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
|
|
74
|
+
* — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
|
|
75
|
+
* form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
|
|
76
|
+
* <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
|
|
77
|
+
* (DE/IT/NL) render identically under both. Mixing both stops an ES shard from teaching the comma as THE street→house
|
|
78
|
+
* boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
|
|
79
|
+
* house-first space-joined).
|
|
80
|
+
*/
|
|
81
|
+
nativeHouseJoin?: "template" | "space"
|
|
82
|
+
}
|
|
83
|
+
/** @deprecated Alias — use LocaleSynthesisOpts. */
|
|
84
|
+
export type GermanSynthesisOpts = LocaleSynthesisOpts
|
|
85
|
+
|
|
86
|
+
/** ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country). */
|
|
87
|
+
const LOCALE_TAG: Record<string, string> = {
|
|
88
|
+
DE: "de-DE",
|
|
89
|
+
ES: "es-ES",
|
|
90
|
+
IT: "it-IT",
|
|
91
|
+
NL: "nl-NL",
|
|
92
|
+
GB: "en-GB",
|
|
93
|
+
FR: "fr-FR",
|
|
94
|
+
US: "en-US",
|
|
95
|
+
NZ: "en-NZ",
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
|
|
100
|
+
* `raw`. NL is the case that needs it: OA stores `1011AB` but the OpenCage NL template emits the conventional spaced
|
|
101
|
+
* `1011 AB` (4 digits + space + 2 letters), which otherwise fails verbatim alignment and drops the row. Other countries
|
|
102
|
+
* pass through unchanged.
|
|
103
|
+
*/
|
|
104
|
+
function normalizePostcode(postcode: string, country: string): string {
|
|
105
|
+
if (country === "NL") {
|
|
106
|
+
const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
|
|
107
|
+
|
|
108
|
+
if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return postcode
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** True when `value` appears verbatim AND as a standalone token (so BIO alignment lands cleanly). */
|
|
115
|
+
function tokenPresent(raw: string, value: string): boolean {
|
|
116
|
+
if (!raw.includes(value)) return false
|
|
117
|
+
// Reject substring-of-a-larger-number collisions (e.g. house "2" inside postcode "12623").
|
|
118
|
+
const i = raw.indexOf(value)
|
|
119
|
+
const before = raw[i - 1]
|
|
120
|
+
const after = raw[i + value.length]
|
|
121
|
+
const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
|
|
122
|
+
|
|
123
|
+
if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
|
|
124
|
+
|
|
125
|
+
return true
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Render one real tuple into an idiomatic, locale-ordered `{raw, components}` row via the OpenCage `country` template
|
|
130
|
+
* (DE → house-after-street + postcode-before-city; ES/IT the same; GB house-first; NL carries the `1012 LM` postcode),
|
|
131
|
+
* with light variation (drop house number / postcode some of the time). Returns `null` when the tuple is too thin or a
|
|
132
|
+
* component wouldn't align cleanly.
|
|
133
|
+
*
|
|
134
|
+
* Region handling is order-dependent: NATIVE order omits it (the native template absorbs the admin region into the
|
|
135
|
+
* postcode/city line, so it rarely renders verbatim and would break BIO alignment), while INTERNATIONAL order includes
|
|
136
|
+
* it in the tail ("City, Region Postcode" — the US/feed layout the eval uses; v0.9.3 / #327).
|
|
137
|
+
*
|
|
138
|
+
* Pass `opts.order: "international"` to render the same components house-first / postcode-after-city instead (see
|
|
139
|
+
* {@link LocaleSynthesisOpts.order}) — the layout international feeds impose on foreign addresses, and the one a
|
|
140
|
+
* native-order-trained model treats as a "collapse."
|
|
141
|
+
*/
|
|
142
|
+
export function synthesizeLocaleRow(
|
|
143
|
+
base: LocaleBaseTuple,
|
|
144
|
+
country: string,
|
|
145
|
+
opts: LocaleSynthesisOpts = {}
|
|
146
|
+
): SynthesizedLocaleRow | null {
|
|
147
|
+
const random = opts.random ?? Math.random
|
|
148
|
+
const order = opts.order ?? "native"
|
|
149
|
+
|
|
150
|
+
if (!base.street || !base.locality) return null
|
|
151
|
+
|
|
152
|
+
const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
|
|
153
|
+
|
|
154
|
+
// Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
|
|
155
|
+
// of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
|
|
156
|
+
// the envelope). The tokenPresent gate below drops the row if the template didn't surface it verbatim.
|
|
157
|
+
if (base.dependent_locality) {
|
|
158
|
+
components.dependent_locality = base.dependent_locality
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// ~80% keep the house number (the rest are street-only forms, also idiomatic).
|
|
162
|
+
if (base.house_number && random() < 0.8) {
|
|
163
|
+
components.house_number = base.house_number
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
|
|
167
|
+
// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
|
|
168
|
+
// normalizes the postcode itself, so a glued input can't survive rendering directly.
|
|
169
|
+
if (base.postcode && random() < 0.85) {
|
|
170
|
+
components.postcode = normalizePostcode(base.postcode, country)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
|
|
174
|
+
// US/feed renderings (and our OA eval) use. v0.9.2 rendered international order WITHOUT the region,
|
|
175
|
+
// so the model never learned to segment the tail and mangled it at eval (region absorbed into the
|
|
176
|
+
// locality / locality dropped); v0.9.3 closes that gap (#327). Native order still drops the region
|
|
177
|
+
// (the native template absorbs it into the city line, which would break verbatim alignment).
|
|
178
|
+
if (order === "international" && base.region) {
|
|
179
|
+
components.region = base.region
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// Native order uses the address's own country template; international order uses the US template —
|
|
183
|
+
// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
|
|
184
|
+
// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
|
|
185
|
+
const renderCountry = order === "international" ? "US" : country
|
|
186
|
+
let raw = formatAddress(components, renderCountry, { separator: ", " })
|
|
187
|
+
|
|
188
|
+
if (!raw) return null
|
|
189
|
+
|
|
190
|
+
// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
|
|
191
|
+
// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
|
|
192
|
+
// space-join (the substring isn't present), and skipped when the house number was dropped above.
|
|
193
|
+
if (order === "native" && opts.nativeHouseJoin === "space" && components.house_number) {
|
|
194
|
+
raw = raw.replace(
|
|
195
|
+
`${components.street}, ${components.house_number}`,
|
|
196
|
+
`${components.street} ${components.house_number}`
|
|
197
|
+
)
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
|
|
201
|
+
// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
|
|
202
|
+
// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
|
|
203
|
+
if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
|
|
204
|
+
const sourceForm = base.postcode.trim()
|
|
205
|
+
|
|
206
|
+
if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
|
|
207
|
+
raw = raw.replace(components.postcode, sourceForm)
|
|
208
|
+
components.postcode = sourceForm
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Every component must align — drop the row if the template didn't surface one verbatim, or a
|
|
213
|
+
// numeric component collides with a neighbouring digit run.
|
|
214
|
+
for (const value of Object.values(components)) {
|
|
215
|
+
if (!value || !tokenPresent(raw, value)) return null
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** German wrapper over {@link synthesizeLocaleRow}. Kept for the build-german-shard caller + tests. */
|
|
222
|
+
export function synthesizeGermanRow(
|
|
223
|
+
base: LocaleBaseTuple,
|
|
224
|
+
opts: LocaleSynthesisOpts = {}
|
|
225
|
+
): SynthesizedLocaleRow | null {
|
|
226
|
+
return synthesizeLocaleRow(base, "DE", opts)
|
|
227
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* House-number + venue + street co-occurrence synthesizer. The v0.6.3 corrective shard.
|
|
7
|
+
*
|
|
8
|
+
* The v0.6.2 step-20K diagnostic showed that adding synth-no-street counter-distribution regressed
|
|
9
|
+
* house_number recall by ~4-5pp. DeepSeek's turn-8 root-cause:
|
|
10
|
+
*
|
|
11
|
+
* 1. Direct: `5th Avenue Theatre`-style adversarial venues teach the model that tokens like "5th"
|
|
12
|
+
* belong to venues, not house_numbers. (Fixed in `synthesize-no-street.ts` by removing
|
|
13
|
+
* digit+ordinal venue patterns.)
|
|
14
|
+
* 2. Distributional dilution: synth-no-street adds 122K rows where house_number is absent. The model's
|
|
15
|
+
* training distribution shifts toward "house_number is rare," and it under-emits the tag at
|
|
16
|
+
* inference.
|
|
17
|
+
*
|
|
18
|
+
* This synthesizer fixes #2 directly. Each emitted row has ALL of: house_number, street, venue,
|
|
19
|
+
* locality, region, postcode — a counter-example to "house_number is rare." Used as a companion
|
|
20
|
+
* shard to synth-no-street; the v0.6.3 config weights synth-no-street at 0.5 and
|
|
21
|
+
* synth-house-venue at 1.0 to recover the lost house_number signal.
|
|
22
|
+
*
|
|
23
|
+
* Real-world shape: business cards, mailing labels, store directories — `"123 Main St, Sunrise
|
|
24
|
+
* Bakery, Springfield, IL 62701"` is a perfectly ordinary address form.
|
|
25
|
+
*
|
|
26
|
+
* Venue pool: PLAIN_VENUES from `synthesize-no-street.ts` (re-exported here). Adversarial venues
|
|
27
|
+
* are deliberately NOT used here — the point is to teach co-occurrence, not to re-introduce
|
|
28
|
+
* decompose-mode pressure.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import type { CanonicalRow } from "./types.ts"
|
|
32
|
+
|
|
33
|
+
export interface HouseVenueBaseTuple {
|
|
34
|
+
locality: string
|
|
35
|
+
region: string
|
|
36
|
+
postcode: string
|
|
37
|
+
country: string
|
|
38
|
+
street?: string
|
|
39
|
+
houseNumber?: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type HouseVenueTemplate =
|
|
43
|
+
| "venue-after-street" // "123 Main St, Sunrise Bakery, Springfield, IL 02101"
|
|
44
|
+
| "venue-before-street" // "Sunrise Bakery, 123 Main St, Springfield, IL 02101"
|
|
45
|
+
|
|
46
|
+
export interface HouseVenueSynthesisOpts {
|
|
47
|
+
random?: () => number
|
|
48
|
+
forceTemplate?: HouseVenueTemplate
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface SynthesizedHouseVenueRow {
|
|
52
|
+
raw: string
|
|
53
|
+
components: CanonicalRow["components"]
|
|
54
|
+
locale: string
|
|
55
|
+
template: HouseVenueTemplate
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// ---------------------------------------------------------------------------------------------
|
|
59
|
+
// Venue pool — PLAIN, no street-typing tokens. The point of this shard is to teach
|
|
60
|
+
// house_number + venue coexistence, NOT to re-introduce decompose-mode pressure.
|
|
61
|
+
// Adversarial venue names live in `synthesize-no-street.ts`.
|
|
62
|
+
// ---------------------------------------------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
const PLAIN_VENUES: ReadonlyArray<string> = [
|
|
65
|
+
"Bob's Pizza",
|
|
66
|
+
"Acme Corporation",
|
|
67
|
+
"Joe's Diner",
|
|
68
|
+
"Sunrise Bakery",
|
|
69
|
+
"Maple Leaf Cafe",
|
|
70
|
+
"Riverside Garden Center",
|
|
71
|
+
"Tech Solutions Inc",
|
|
72
|
+
"Pacific Industries",
|
|
73
|
+
"Atlantic Holdings",
|
|
74
|
+
"Stellar Consulting",
|
|
75
|
+
"Greenfield Partners",
|
|
76
|
+
"Mountain View Studio",
|
|
77
|
+
"The Daily Grind",
|
|
78
|
+
"Sunset Bistro",
|
|
79
|
+
"Harvest Moon Florist",
|
|
80
|
+
"Iron Forge Brewing",
|
|
81
|
+
"Crescent City Bookstore",
|
|
82
|
+
"Lighthouse Insurance Group",
|
|
83
|
+
"Pinecrest Veterinary",
|
|
84
|
+
"Westwood Realty",
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
// ---------------------------------------------------------------------------------------------
|
|
88
|
+
// Fallback street pool for tuples that didn't carry a `street` field. Plain street names
|
|
89
|
+
// without typing-token ambiguity.
|
|
90
|
+
// ---------------------------------------------------------------------------------------------
|
|
91
|
+
|
|
92
|
+
const FALLBACK_STREETS: ReadonlyArray<string> = [
|
|
93
|
+
"Main St",
|
|
94
|
+
"Oak Ave",
|
|
95
|
+
"Pine Rd",
|
|
96
|
+
"Elm Dr",
|
|
97
|
+
"Cedar Ln",
|
|
98
|
+
"Maple Blvd",
|
|
99
|
+
"Birch Ct",
|
|
100
|
+
"Walnut Pl",
|
|
101
|
+
"Cherry Way",
|
|
102
|
+
"Spruce St",
|
|
103
|
+
"Park Ave",
|
|
104
|
+
"Lake Dr",
|
|
105
|
+
"Hill Rd",
|
|
106
|
+
"River Ln",
|
|
107
|
+
"Forest Blvd",
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
// ---------------------------------------------------------------------------------------------
|
|
111
|
+
// House-number generator
|
|
112
|
+
// ---------------------------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
function randomHouseNumber(random: () => number): string {
|
|
115
|
+
// Generate a plain numeric house number 1-9999. No fractions/ranges — those land in
|
|
116
|
+
// `data/eval/falsehoods/numbers.jsonl` as known edge cases, not training material.
|
|
117
|
+
const digits = Math.floor(random() * 4) + 1
|
|
118
|
+
const max = Math.pow(10, digits)
|
|
119
|
+
const n = Math.floor(random() * max) + 1
|
|
120
|
+
|
|
121
|
+
return String(n)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function pick<T>(arr: ReadonlyArray<T>, random: () => number): T {
|
|
125
|
+
return arr[Math.floor(random() * arr.length)]!
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function countryToLocale(country: string): string {
|
|
129
|
+
switch (country) {
|
|
130
|
+
case "US":
|
|
131
|
+
return "en-US"
|
|
132
|
+
case "CA":
|
|
133
|
+
return "en-CA"
|
|
134
|
+
case "GB":
|
|
135
|
+
return "en-GB"
|
|
136
|
+
case "AU":
|
|
137
|
+
return "en-AU"
|
|
138
|
+
case "FR":
|
|
139
|
+
return "fr-FR"
|
|
140
|
+
case "DE":
|
|
141
|
+
return "de-DE"
|
|
142
|
+
default:
|
|
143
|
+
return "en-US"
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// ---------------------------------------------------------------------------------------------
|
|
148
|
+
// Synthesis
|
|
149
|
+
// ---------------------------------------------------------------------------------------------
|
|
150
|
+
|
|
151
|
+
export function synthesizeHouseVenueRow(
|
|
152
|
+
base: HouseVenueBaseTuple,
|
|
153
|
+
opts: HouseVenueSynthesisOpts = {}
|
|
154
|
+
): SynthesizedHouseVenueRow | null {
|
|
155
|
+
const random = opts.random ?? Math.random
|
|
156
|
+
const locale = countryToLocale(base.country)
|
|
157
|
+
const template = opts.forceTemplate ?? (random() < 0.5 ? "venue-after-street" : "venue-before-street")
|
|
158
|
+
|
|
159
|
+
const venue = pick(PLAIN_VENUES, random)
|
|
160
|
+
const street = base.street ?? pick(FALLBACK_STREETS, random)
|
|
161
|
+
const houseNumber = base.houseNumber ?? randomHouseNumber(random)
|
|
162
|
+
|
|
163
|
+
const components: CanonicalRow["components"] = {
|
|
164
|
+
house_number: houseNumber,
|
|
165
|
+
street,
|
|
166
|
+
venue,
|
|
167
|
+
locality: base.locality,
|
|
168
|
+
region: base.region,
|
|
169
|
+
postcode: base.postcode,
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
let raw: string
|
|
173
|
+
|
|
174
|
+
switch (template) {
|
|
175
|
+
case "venue-after-street":
|
|
176
|
+
raw = `${houseNumber} ${street}, ${venue}, ${base.locality}, ${base.region} ${base.postcode}`
|
|
177
|
+
break
|
|
178
|
+
case "venue-before-street":
|
|
179
|
+
raw = `${venue}, ${houseNumber} ${street}, ${base.locality}, ${base.region} ${base.postcode}`
|
|
180
|
+
break
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
return { raw, components, locale, template }
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Contract: every synthesized row carries BOTH house_number AND venue (the co-occurrence signal that synth-no-street's
|
|
188
|
+
* distributional shift cost the model). Used by tests + downstream consumers.
|
|
189
|
+
*/
|
|
190
|
+
export function hasHouseNumberAndVenue(components: CanonicalRow["components"]): boolean {
|
|
191
|
+
return components.house_number !== undefined && components.venue !== undefined
|
|
192
|
+
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Intersection synthesizer — v0.7 coverage fix (night-3, DeepSeek-decided).
|
|
7
|
+
*
|
|
8
|
+
* The 2026-05-29 harness diagnostic found the neural model emits `intersection_a`/`intersection_b`
|
|
9
|
+
* with ~0.0001 probability on canonical intersections ("Broadway & W 42nd St") — it never learned
|
|
10
|
+
* the tags, because the corpus has NO intersection training signal (no generator, and real-data
|
|
11
|
+
* adapters don't emit intersection-formatted rows). Intersections are 65 of the 376 harness
|
|
12
|
+
* assertions (17%), all 0% neural. This generator produces the missing signal as a small targeted
|
|
13
|
+
* supplement shard (synthesis-as-supplement discipline: weight < 0.25, one-and-done).
|
|
14
|
+
*
|
|
15
|
+
* Output is a `CanonicalRow` ({raw, components}); the corpus aligner turns it into BIO labels
|
|
16
|
+
* (B-/I-intersection_a, O on the connector, B-/I-intersection_b). Surface forms of both streets
|
|
17
|
+
* MUST occur verbatim in `raw` so alignment lands.
|
|
18
|
+
*
|
|
19
|
+
* US-idiomatic only (the harness intersection cases are US: "X & Y, City, ST ZIP").
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import type { CanonicalRow } from "./types.ts"
|
|
23
|
+
|
|
24
|
+
/** Street name cores (no suffix) — proper-noun streets that often appear bare. */
|
|
25
|
+
const STREET_CORES = [
|
|
26
|
+
"Main",
|
|
27
|
+
"Oak",
|
|
28
|
+
"Elm",
|
|
29
|
+
"Maple",
|
|
30
|
+
"Pine",
|
|
31
|
+
"Cedar",
|
|
32
|
+
"Park",
|
|
33
|
+
"Lake",
|
|
34
|
+
"Hill",
|
|
35
|
+
"Washington",
|
|
36
|
+
"Lincoln",
|
|
37
|
+
"Jefferson",
|
|
38
|
+
"Madison",
|
|
39
|
+
"Franklin",
|
|
40
|
+
"Market",
|
|
41
|
+
"Broad",
|
|
42
|
+
"Church",
|
|
43
|
+
"Mill",
|
|
44
|
+
"Highland",
|
|
45
|
+
"Sunset",
|
|
46
|
+
"Union",
|
|
47
|
+
"Spring",
|
|
48
|
+
] as const
|
|
49
|
+
|
|
50
|
+
/** Bare proper-noun streets that idiomatically take NO suffix. */
|
|
51
|
+
const BARE_NAMES = ["Broadway", "Wall", "Bourbon", "Esplanade", "Riverside", "Lakeshore"] as const
|
|
52
|
+
|
|
53
|
+
const ORDINALS = [
|
|
54
|
+
"1st",
|
|
55
|
+
"2nd",
|
|
56
|
+
"3rd",
|
|
57
|
+
"4th",
|
|
58
|
+
"5th",
|
|
59
|
+
"6th",
|
|
60
|
+
"7th",
|
|
61
|
+
"8th",
|
|
62
|
+
"9th",
|
|
63
|
+
"10th",
|
|
64
|
+
"42nd",
|
|
65
|
+
"23rd",
|
|
66
|
+
"34th",
|
|
67
|
+
] as const
|
|
68
|
+
|
|
69
|
+
const SUFFIXES = ["St", "Ave", "Blvd", "Rd", "Dr", "Ln", "Way", "Pl", "Ct", "Pkwy", "Ter", "Cir"] as const
|
|
70
|
+
|
|
71
|
+
const DIRECTIONALS = ["N", "S", "E", "W", "NE", "NW", "SE", "SW"] as const
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Connectors between the two streets. Whitespace-padded forms keep tokens clean for alignment. `@` added in v0.7.2 —
|
|
75
|
+
* the harness uses it ("Main St @ Second Ave") and v0.7.1 had never seen it.
|
|
76
|
+
*/
|
|
77
|
+
const CONNECTORS = [" & ", " and ", " at ", " / ", " @ "] as const
|
|
78
|
+
|
|
79
|
+
export interface IntersectionBaseTuple {
|
|
80
|
+
locality: string
|
|
81
|
+
region: string
|
|
82
|
+
/** ZIP — optional; ~30% of synthetic intersections omit it (idiomatic). */
|
|
83
|
+
postcode?: string
|
|
84
|
+
country: string
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export interface SynthesizedIntersectionRow {
|
|
88
|
+
raw: string
|
|
89
|
+
components: CanonicalRow["components"]
|
|
90
|
+
locale: string
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface IntersectionSynthesisOpts {
|
|
94
|
+
random?: () => number
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function pick<T>(arr: ReadonlyArray<T>, random: () => number): T {
|
|
98
|
+
return arr[Math.floor(random() * arr.length)]!
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Build a single street surface form, e.g. "W 42nd St", "Broadway", "Main St", "N Oak Ave". */
|
|
102
|
+
function buildStreetName(random: () => number): string {
|
|
103
|
+
// ~20% bare proper-noun street (no suffix), else directional? + core/ordinal + suffix.
|
|
104
|
+
if (random() < 0.2) return pick(BARE_NAMES, random)
|
|
105
|
+
|
|
106
|
+
const parts: string[] = []
|
|
107
|
+
|
|
108
|
+
if (random() < 0.35) {
|
|
109
|
+
parts.push(pick(DIRECTIONALS, random))
|
|
110
|
+
}
|
|
111
|
+
parts.push(random() < 0.45 ? pick(ORDINALS, random) : pick(STREET_CORES, random))
|
|
112
|
+
parts.push(pick(SUFFIXES, random))
|
|
113
|
+
|
|
114
|
+
return parts.join(" ")
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Synthesize one US intersection row. Returns null on the rare degenerate case where the two streets collide (so
|
|
119
|
+
* alignment never has two identical surface forms to disambiguate).
|
|
120
|
+
*/
|
|
121
|
+
export function synthesizeIntersectionRow(
|
|
122
|
+
base: IntersectionBaseTuple,
|
|
123
|
+
opts: IntersectionSynthesisOpts = {}
|
|
124
|
+
): SynthesizedIntersectionRow | null {
|
|
125
|
+
const random = opts.random ?? Math.random
|
|
126
|
+
|
|
127
|
+
if (base.country !== "US") return null
|
|
128
|
+
|
|
129
|
+
const a = buildStreetName(random)
|
|
130
|
+
let b = buildStreetName(random)
|
|
131
|
+
// Ensure distinct surface forms (and not a substring of each other — alignment needs unambiguous spans).
|
|
132
|
+
let tries = 0
|
|
133
|
+
|
|
134
|
+
while ((b === a || a.includes(b) || b.includes(a)) && tries++ < 8) {
|
|
135
|
+
b = buildStreetName(random)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if (b === a || a.includes(b) || b.includes(a)) return null
|
|
139
|
+
|
|
140
|
+
const connector = pick(CONNECTORS, random)
|
|
141
|
+
// "corner of" prefix variant (~20%) — still labels the two streets identically.
|
|
142
|
+
const cornerPrefix = random() < 0.2 ? "corner of " : ""
|
|
143
|
+
|
|
144
|
+
const components: CanonicalRow["components"] = { intersection_a: a, intersection_b: b }
|
|
145
|
+
|
|
146
|
+
// v0.7.2: ~60% BARE (no locality tail). v0.7.1 always appended ", City, ST", so the model learned
|
|
147
|
+
// to read post-intersection text as a locality and fumbled the harness's bare "X & Y" cases
|
|
148
|
+
// (mislabeling the second street as a locality). Match the eval distribution.
|
|
149
|
+
const bare = random() < 0.6
|
|
150
|
+
let raw: string
|
|
151
|
+
|
|
152
|
+
if (bare) {
|
|
153
|
+
raw = `${cornerPrefix}${a}${connector}${b}`
|
|
154
|
+
} else {
|
|
155
|
+
const includePostcode = base.postcode != null && random() < 0.7
|
|
156
|
+
const tail = includePostcode
|
|
157
|
+
? `, ${base.locality}, ${base.region} ${base.postcode}`
|
|
158
|
+
: `, ${base.locality}, ${base.region}`
|
|
159
|
+
raw = `${cornerPrefix}${a}${connector}${b}${tail}`
|
|
160
|
+
components.locality = base.locality
|
|
161
|
+
components.region = base.region
|
|
162
|
+
|
|
163
|
+
if (includePostcode) {
|
|
164
|
+
components.postcode = base.postcode
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
return { raw, components, locale: "en-US" }
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** A small built-in US city/region/zip pool for standalone shard generation + tests. */
|
|
172
|
+
export const DEFAULT_US_BASES: ReadonlyArray<IntersectionBaseTuple> = [
|
|
173
|
+
{ locality: "New York", region: "NY", postcode: "10036", country: "US" },
|
|
174
|
+
{ locality: "Chicago", region: "IL", postcode: "60613", country: "US" },
|
|
175
|
+
{ locality: "Los Angeles", region: "CA", postcode: "90012", country: "US" },
|
|
176
|
+
{ locality: "Seattle", region: "WA", postcode: "98109", country: "US" },
|
|
177
|
+
{ locality: "Austin", region: "TX", postcode: "78701", country: "US" },
|
|
178
|
+
{ locality: "Portland", region: "OR", postcode: "97205", country: "US" },
|
|
179
|
+
{ locality: "Denver", region: "CO", postcode: "80202", country: "US" },
|
|
180
|
+
{ locality: "Boston", region: "MA", postcode: "02116", country: "US" },
|
|
181
|
+
{ locality: "Miami", region: "FL", postcode: "33130", country: "US" },
|
|
182
|
+
{ locality: "Atlanta", region: "GA", postcode: "30303", country: "US" },
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
/** Generate `count` intersection rows over the provided bases (round-robin). */
|
|
186
|
+
export function generateIntersectionRows(
|
|
187
|
+
count: number,
|
|
188
|
+
bases: ReadonlyArray<IntersectionBaseTuple> = DEFAULT_US_BASES,
|
|
189
|
+
opts: IntersectionSynthesisOpts = {}
|
|
190
|
+
): SynthesizedIntersectionRow[] {
|
|
191
|
+
const random = opts.random ?? Math.random
|
|
192
|
+
const out: SynthesizedIntersectionRow[] = []
|
|
193
|
+
let guard = 0
|
|
194
|
+
|
|
195
|
+
while (out.length < count && guard++ < count * 4) {
|
|
196
|
+
const base = bases[out.length % bases.length]!
|
|
197
|
+
const row = synthesizeIntersectionRow(base, { random })
|
|
198
|
+
|
|
199
|
+
if (row) {
|
|
200
|
+
out.push(row)
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
return out
|
|
205
|
+
}
|