@mailwoman/corpus 7.2.0 → 7.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +59 -27
  2. package/src/adapter.ts +153 -0
  3. package/src/adapters/ban/adapter.ts +189 -0
  4. package/src/adapters/ban/street-decompose.ts +98 -0
  5. package/src/adapters/fcc-bdc/adapter.ts +188 -0
  6. package/src/adapters/geonames/adapter.ts +185 -0
  7. package/src/adapters/geonames-postal/adapter.ts +111 -0
  8. package/src/adapters/gnaf/adapter.ts +139 -0
  9. package/src/adapters/gnaf/assemble.ts +200 -0
  10. package/src/adapters/index.ts +135 -0
  11. package/src/adapters/openaddresses/adapter.ts +237 -0
  12. package/src/adapters/overture/adapter.ts +149 -0
  13. package/src/adapters/state-hi-schools/adapter.ts +171 -0
  14. package/src/adapters/state-ia-contractors/adapter.ts +148 -0
  15. package/src/adapters/state-ny-notaries/adapter.ts +169 -0
  16. package/src/adapters/state-tx-notaries/adapter.ts +160 -0
  17. package/src/adapters/synth-po-box/adapter.ts +197 -0
  18. package/src/adapters/tiger/adapter.ts +232 -0
  19. package/src/adapters/tiger/street-decompose.ts +120 -0
  20. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
  21. package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
  22. package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
  23. package/src/adapters/usgov-nad/adapter.ts +338 -0
  24. package/src/adapters/usgov-nppes/adapter.ts +169 -0
  25. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
  26. package/src/adapters/wof-admin-jp/adapter.ts +176 -0
  27. package/src/adapters/wof-admin-json/adapter.ts +280 -0
  28. package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
  29. package/src/align.ts +317 -0
  30. package/src/build.ts +348 -0
  31. package/src/codex/us-fips-state.ts +109 -0
  32. package/src/format.ts +14 -0
  33. package/src/golden.ts +147 -0
  34. package/src/index.ts +28 -0
  35. package/src/license.ts +48 -0
  36. package/src/parquet-wrapper/index.ts +9 -0
  37. package/src/parquet-wrapper/reader.ts +67 -0
  38. package/src/parquet-wrapper/schema.ts +80 -0
  39. package/src/parquet-wrapper/writer.ts +102 -0
  40. package/src/parquet.ts +360 -0
  41. package/src/runner.ts +283 -0
  42. package/src/shard-recipes/anchor-absorption.ts +86 -0
  43. package/src/shard-recipes/boundary-stress.ts +87 -0
  44. package/src/shard-recipes/country-balanced.ts +471 -0
  45. package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
  46. package/src/shard-recipes/fr-admin-split.ts +212 -0
  47. package/src/shard-recipes/fr-bare-street.ts +85 -0
  48. package/src/shard-recipes/fr-fragment.ts +304 -0
  49. package/src/shard-recipes/fr-order.ts +283 -0
  50. package/src/shard-recipes/german.ts +206 -0
  51. package/src/shard-recipes/house-venue.ts +78 -0
  52. package/src/shard-recipes/index.ts +76 -0
  53. package/src/shard-recipes/intersection.ts +617 -0
  54. package/src/shard-recipes/locale.ts +415 -0
  55. package/src/shard-recipes/nl-postcode.ts +105 -0
  56. package/src/shard-recipes/no-fragment.ts +206 -0
  57. package/src/shard-recipes/no-street-led.ts +130 -0
  58. package/src/shard-recipes/no-street.ts +78 -0
  59. package/src/shard-recipes/po-box-cedex.ts +853 -0
  60. package/src/shard-recipes/po-box.ts +111 -0
  61. package/src/shard-recipes/scaffold.ts +187 -0
  62. package/src/shard-recipes/si-bare-village.ts +88 -0
  63. package/src/shard-recipes/street-affix.ts +549 -0
  64. package/src/shard-recipes/street-bare.ts +90 -0
  65. package/src/shard-recipes/street.ts +74 -0
  66. package/src/shard-recipes/unit.ts +315 -0
  67. package/src/split.ts +251 -0
  68. package/src/synthesize-anchor-absorption.ts +263 -0
  69. package/src/synthesize-boundary-stress.ts +504 -0
  70. package/src/synthesize-german.ts +227 -0
  71. package/src/synthesize-house-venue.ts +192 -0
  72. package/src/synthesize-intersection.ts +205 -0
  73. package/src/synthesize-no-street.ts +341 -0
  74. package/src/synthesize-po-box.ts +307 -0
  75. package/src/synthesize-street.ts +264 -0
  76. package/src/synthesize.ts +872 -0
  77. package/src/tokenize.ts +68 -0
  78. package/src/tools/align-shard.ts +67 -0
  79. package/src/tools/audit.ts +352 -0
  80. package/src/tools/corpus-stats.ts +183 -0
  81. package/src/tools/fetch/ban.ts +288 -0
  82. package/src/tools/fetch/download.ts +119 -0
  83. package/src/tools/fetch/hrsa.ts +65 -0
  84. package/src/tools/fetch/imls-pls.ts +162 -0
  85. package/src/tools/fetch/index.ts +121 -0
  86. package/src/tools/fetch/nad.ts +304 -0
  87. package/src/tools/fetch/nppes.ts +177 -0
  88. package/src/tools/fetch/openaddresses.ts +380 -0
  89. package/src/tools/fetch/state-hi-schools.ts +217 -0
  90. package/src/tools/fetch/state-sources.ts +142 -0
  91. package/src/tools/fetch/tiger-full.ts +318 -0
  92. package/src/tools/golden-expand.ts +524 -0
  93. package/src/tools/golden-promote.ts +304 -0
  94. package/src/tools/index.ts +24 -0
  95. package/src/tools/ingest-csv.ts +420 -0
  96. package/src/tools/jsonl-to-parquet.ts +200 -0
  97. package/src/tools/lint-shard-vocab.ts +357 -0
  98. package/src/tools/lint-shard.ts +502 -0
  99. package/src/tools/overlay-manifest.ts +141 -0
  100. package/src/tools/shard-kryptonite.ts +131 -0
  101. package/src/tools/shard-translit.ts +244 -0
  102. package/src/types.ts +213 -0
  103. package/src/wof-json.ts +236 -0
@@ -0,0 +1,549 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `street-affix` shard recipe — the US street-affix coverage shard (the v0-parity `street_prefix` /
7
+ * `street_suffix` gap — both ~0% F1 in the #15 assessment, collapsed into `street`). Raises
8
+ * PREVALENCE of affix-split streets with format diversity so the model learns to split "N Main
9
+ * St" → street_prefix="N" + street="Main" + street_suffix="St", and (negative space) sharpens
10
+ * `street` itself. Ported from scripts/build-street-affix-shard.mjs.
11
+ *
12
+ * Reads REAL US OpenAddresses tuples and SPLITS the OA `street` field via the codex:
13
+ * `matchLeadingDirectional` (USPS Pub-28 C1) for the prefix, `matchTrailingSuffix` (Pub-28 C2
14
+ * street suffixes) for the suffix. OA streets nearly all carry a suffix; only ~10-20% carry a
15
+ * directional, so we INJECT a directional prefix onto a fraction of prefix-less streets to give
16
+ * `street_prefix` real signal. Each row varies surface form per affix — abbreviated ("N", "St")
17
+ * vs expanded ("North", "Street") — and varies the layout (full address / bare / street-only /
18
+ * venue-prefixed).
19
+ *
20
+ * LEAKAGE-SAFE EVAL (`--golden`): held-out eval uses the VERMONT source only (the corpus
21
+ * defaultHoldout), a different seed, and emits {raw, components} for per-locale-f1. Train uses
22
+ * every NON-Vermont US source.
23
+ *
24
+ * Multi-locale BALANCE (`--multilocale-count`, opts.multilocaleCount > 0): appends NO-affix
25
+ * native-order rows (FR/DE/IT/NL) AFTER the US affix rows, riding the same source weight, purely
26
+ * to keep the postcode-ORDER distribution multi-locale so a US-heavy affix shard doesn't dilute
27
+ * FR/DE postcode (the v0.9.8 blemish).
28
+ */
29
+
30
+ import { spawnSync } from "node:child_process"
31
+
32
+ import {
33
+ DirectionalAbbreviation,
34
+ lookupDirectional,
35
+ matchCase,
36
+ matchLeadingDirectional,
37
+ matchTrailingSuffix,
38
+ renderDirectional,
39
+ US_STREET_SUFFIX_PREFERRED_ABBR,
40
+ } from "@mailwoman/codex/us"
41
+ import type { ComponentTag } from "@mailwoman/core/types"
42
+
43
+ import { stableSourceID } from "../adapter.ts"
44
+ import { alignRow } from "../align.ts"
45
+ import type { CanonicalRow } from "../types.ts"
46
+ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
47
+
48
+ // Same OA cache as the unit shard. Train = every NON-Vermont state; eval = Vermont (the holdout).
49
+ interface USSource {
50
+ zip: string
51
+ csv: string
52
+ region: string
53
+ }
54
+ const TRAIN_SOURCES: readonly USSource[] = [
55
+ { zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
56
+ { zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
57
+ { zip: "/tmp/oa-cache/us__dc__statewide.zip", csv: "us/dc/statewide.csv", region: "DC" },
58
+ { zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", region: "IA" },
59
+ { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", region: "IL" },
60
+ { zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
61
+ { zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
62
+ ]
63
+ const EVAL_SOURCE: USSource = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
64
+
65
+ // Multi-locale BALANCE sources (--multilocale-count > 0). These rows carry NO affix split — they exist
66
+ // only to keep the postcode-ORDER distribution multi-locale. Native-order rendering mirrors
67
+ // build-country-shard-balanced.mjs: FR = number-street, postcode-city; DE/IT/NL = street-number,
68
+ // postcode-city. `order` drives the body.
69
+ interface BalanceSource {
70
+ zip: string
71
+ csv: string
72
+ iso2: string
73
+ region: string
74
+ order: string
75
+ }
76
+ const MULTILOCALE_SOURCES: readonly BalanceSource[] = [
77
+ { zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", iso2: "DE", region: "", order: "eu" },
78
+ { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv", iso2: "FR", region: "", order: "fr" },
79
+ { zip: "/tmp/oa-cache/it__countrywide.zip", csv: "it/countrywide.csv", iso2: "IT", region: "", order: "eu" },
80
+ { zip: "/tmp/oa-cache/nl__countrywide.zip", csv: "nl/countrywide.csv", iso2: "NL", region: "", order: "eu" },
81
+ ]
82
+ const MULTILOCALE_EVAL_SOURCES: readonly BalanceSource[] = [
83
+ { zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
84
+ ]
85
+
86
+ const DIRECTIONAL_ABBRS = Object.values(DirectionalAbbreviation) // ["N","E","S","W","NE","NW","SE","SW"]
87
+ const INJECT_PREFIX_PROB = 0.3 // fraction of prefix-less streets that get a synthetic directional
88
+
89
+ /** A real US skeleton tuple read from a cached OA zip. */
90
+ interface USTuple {
91
+ house_number: string
92
+ street: string
93
+ locality: string
94
+ region: string
95
+ postcode: string
96
+ }
97
+
98
+ /** A non-US BALANCE tuple (carries a postcode + native order). */
99
+ interface BalanceTuple {
100
+ house_number: string
101
+ street: string
102
+ locality: string
103
+ region: string
104
+ postcode: string
105
+ iso2: string
106
+ order: string
107
+ }
108
+
109
+ /** Prefix carried through render — the (canonical, abbreviation) pair `renderDirectional` consumes. */
110
+ type Prefix = Pick<NonNullable<ReturnType<typeof matchLeadingDirectional>>, "canonical" | "abbreviation">
111
+
112
+ /** Minimal RFC-4180-ish splitter (handles quoted fields). */
113
+ function splitCSV(line: string): string[] {
114
+ const out: string[] = []
115
+ let cur = ""
116
+ let inQ = false
117
+
118
+ for (let i = 0; i < line.length; i++) {
119
+ const c = line[i]
120
+
121
+ if (inQ) {
122
+ if (c === '"') {
123
+ if (line[i + 1] === '"') {
124
+ cur += '"'
125
+ i++
126
+ } else {
127
+ inQ = false
128
+ }
129
+ } else {
130
+ cur += c
131
+ }
132
+ } else if (c === '"') {
133
+ inQ = true
134
+ } else if (c === ",") {
135
+ out.push(cur)
136
+ cur = ""
137
+ } else {
138
+ cur += c
139
+ }
140
+ }
141
+ out.push(cur)
142
+
143
+ return out
144
+ }
145
+
146
+ /** Stream real US tuples (number/street/city/postcode) out of a cached OA zip. */
147
+ function readTuples(source: USSource): USTuple[] {
148
+ const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
149
+
150
+ if (r.status !== 0) {
151
+ console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
152
+
153
+ return []
154
+ }
155
+ const lines = r.stdout.toString("utf8").split(/\r?\n/)
156
+
157
+ if (lines.length < 2) return []
158
+ const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
159
+ const idx = (name: string): number => header.indexOf(name)
160
+ const iNum = idx("number"),
161
+ iStreet = idx("street"),
162
+ iCity = idx("city"),
163
+ iPost = idx("postcode")
164
+ const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
165
+ const tuples: USTuple[] = []
166
+ const seen = new Set<string>()
167
+
168
+ for (let li = 1; li < lines.length; li++) {
169
+ if (!lines[li]) continue
170
+ const cells = splitCSV(lines[li]!)
171
+ const street = get(cells, iStreet)
172
+ const locality = get(cells, iCity)
173
+ const house_number = get(cells, iNum)
174
+
175
+ if (!street || !locality || !house_number) continue
176
+ const key = `${house_number}|${street}|${locality}`.toLowerCase()
177
+
178
+ if (seen.has(key)) continue
179
+ seen.add(key)
180
+ tuples.push({ house_number, street, locality, region: source.region, postcode: get(cells, iPost) })
181
+ }
182
+
183
+ return tuples
184
+ }
185
+
186
+ const title = (s: string): string =>
187
+ s
188
+ .toLowerCase()
189
+ .split(/\s+/)
190
+ .map((w) => (w ? w.charAt(0).toUpperCase() + w.slice(1) : w))
191
+ .join(" ")
192
+
193
+ const isSuffixOrDirectional = (word: string): boolean =>
194
+ matchTrailingSuffix(word) !== null || matchLeadingDirectional(word) !== null
195
+
196
+ /**
197
+ * Split an OA street into { prefix?, name, suffix } using the codex. Requires a trailing suffix and a non-empty name
198
+ * that isn't itself an affix token. Returns null when the street has no usable suffix.
199
+ */
200
+ function parseStreet(street: string): { prefix: Prefix | null; name: string; suffix: string } | null {
201
+ let words = street.trim().split(/\s+/)
202
+
203
+ if (words.length < 2) return null
204
+ let prefix: Prefix | null = null
205
+ // Leading directional — only if it leaves ≥2 words behind (room for a name + suffix).
206
+ const lead = matchLeadingDirectional(street)
207
+
208
+ if (lead && words.length > 2) {
209
+ prefix = { canonical: lead.canonical, abbreviation: lead.abbreviation }
210
+ words = words.slice(1)
211
+ }
212
+ // Trailing USPS suffix — only if it leaves ≥1 word for the name.
213
+ const trail = matchTrailingSuffix(words.join(" "))
214
+
215
+ if (!trail || words.length < 2) return null
216
+ const suffix = trail.canonical
217
+ const name = words.slice(0, -1).join(" ")
218
+
219
+ if (!name || isSuffixOrDirectional(name)) return null
220
+
221
+ return { prefix, name, suffix }
222
+ }
223
+
224
+ /**
225
+ * Render the affix-split street in random surface forms (abbrev vs expanded per affix), Title-cased.
226
+ */
227
+ function renderStreet(
228
+ random: () => number,
229
+ parsed: { prefix: Prefix | null; name: string; suffix: string }
230
+ ): { street: string; components: Partial<Record<ComponentTag, string>> } {
231
+ const name = title(parsed.name)
232
+ const parts: string[] = []
233
+ const components: Partial<Record<ComponentTag, string>> = { street: name }
234
+
235
+ // Prefix: natural (from parse) or injected onto a prefix-less street to boost street_prefix signal.
236
+ let prefix = parsed.prefix
237
+
238
+ if (!prefix && random() < INJECT_PREFIX_PROB) {
239
+ const m = lookupDirectional(DIRECTIONAL_ABBRS[Math.floor(random() * DIRECTIONAL_ABBRS.length)])!
240
+ prefix = { canonical: m.directional, abbreviation: m.abbreviation }
241
+ }
242
+
243
+ if (prefix) {
244
+ const rendered = renderDirectional(prefix, random() < 0.5 ? "abbr" : "full", "Aa") // "Aa" → Title-case
245
+ components.street_prefix = rendered
246
+ parts.push(rendered)
247
+ }
248
+
249
+ parts.push(name)
250
+
251
+ // Suffix: abbreviated ("St") vs expanded ("Street"), Title-cased to match the name.
252
+ const full = title(parsed.suffix) // canonical is uppercase word → "Street"
253
+ const abbr = matchCase(
254
+ US_STREET_SUFFIX_PREFERRED_ABBR[parsed.suffix as keyof typeof US_STREET_SUFFIX_PREFERRED_ABBR],
255
+ "Aa"
256
+ ) // "AVE" → "Ave"
257
+ const renderedSuffix = random() < 0.5 ? abbr : full
258
+ components.street_suffix = renderedSuffix
259
+ parts.push(renderedSuffix)
260
+
261
+ return { street: parts.join(" "), components }
262
+ }
263
+
264
+ /** Synthetic recipient/venue prefixes — the arena's "JOHN DOE, ACME INC, …" pattern. */
265
+ const VENUES = ["John Doe", "Jane Smith", "Acme Inc", "Wayne Enterprises", "Maria Garcia", "Riverside Clinic"]
266
+
267
+ const tail = (loc: string, reg: string, pc: string): string => (pc ? `${loc}, ${reg} ${pc}` : `${loc}, ${reg}`)
268
+
269
+ /**
270
+ * Embed the rendered street in a RANDOM layout so the model recognizes affixes wherever the street sits: full address,
271
+ * bare house+street, street-only (pure affix parse), or venue-prefixed.
272
+ */
273
+ function renderRow(
274
+ random: () => number,
275
+ base: USTuple,
276
+ street: string,
277
+ streetComponents: Partial<Record<ComponentTag, string>>
278
+ ): { fmt: string; raw: string; components: Partial<Record<ComponentTag, string>> } {
279
+ const hn = base.house_number,
280
+ loc = base.locality,
281
+ reg = base.region,
282
+ pc = base.postcode
283
+ const road = `${hn} ${street}`
284
+ const withRoad: Partial<Record<ComponentTag, string>> = { house_number: hn, ...streetComponents }
285
+ const r = random()
286
+
287
+ if (r < 0.4)
288
+ return {
289
+ fmt: "full",
290
+ raw: `${road}, ${tail(loc, reg, pc)}`,
291
+ components: { ...withRoad, locality: loc, region: reg, ...(pc ? { postcode: pc } : {}) },
292
+ }
293
+
294
+ if (r < 0.65) return { fmt: "bare", raw: road, components: withRoad }
295
+
296
+ if (r < 0.85) return { fmt: "street-only", raw: street, components: { ...streetComponents } }
297
+ const v = VENUES[Math.floor(random() * VENUES.length)]!
298
+
299
+ return {
300
+ fmt: "venue",
301
+ raw: `${v}, ${road}, ${tail(loc, reg, pc)}`,
302
+ components: { venue: v, ...withRoad, locality: loc, region: reg, ...(pc ? { postcode: pc } : {}) },
303
+ }
304
+ }
305
+
306
+ /**
307
+ * Capped reader for the multi-locale BALANCE sources. The FR/IT/NL countrywide extracts are GB-scale; reading the whole
308
+ * CSV blows V8's string limit, so cap the bytes with `head` (mirrors build-country-shard-balanced.mjs). Only keeps
309
+ * tuples that carry a POSTCODE.
310
+ */
311
+ function readBalanceTuples(source: BalanceSource, limit: number): BalanceTuple[] {
312
+ const maxLines = Math.max(limit * 8, 20000) + 1
313
+ const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
314
+ maxBuffer: 1024 * 1024 * 1024,
315
+ encoding: "buffer",
316
+ })
317
+
318
+ if (r.status !== 0) {
319
+ console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
320
+
321
+ return []
322
+ }
323
+ const lines = r.stdout.toString("utf8").split(/\r?\n/)
324
+
325
+ if (lines.length < 2) return []
326
+ const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
327
+ const idx = (n: string): number => header.indexOf(n)
328
+ const iNum = idx("number"),
329
+ iStreet = idx("street"),
330
+ iCity = idx("city"),
331
+ iRegion = idx("region"),
332
+ iPost = idx("postcode")
333
+ const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
334
+ const tuples: BalanceTuple[] = []
335
+ const seen = new Set<string>()
336
+
337
+ for (let li = 1; li < lines.length && tuples.length < limit; li++) {
338
+ if (!lines[li]) continue
339
+ const cells = splitCSV(lines[li]!)
340
+ const street = get(cells, iStreet),
341
+ locality = get(cells, iCity),
342
+ house_number = get(cells, iNum),
343
+ postcode = get(cells, iPost)
344
+
345
+ if (!street || !locality || !house_number || !postcode) continue // postcode is required for balance
346
+ const key = `${house_number}|${street}|${locality}`.toLowerCase()
347
+
348
+ if (seen.has(key)) continue
349
+ seen.add(key)
350
+ tuples.push({
351
+ house_number,
352
+ street,
353
+ locality,
354
+ region: get(cells, iRegion) || source.region,
355
+ postcode,
356
+ iso2: source.iso2,
357
+ order: source.order,
358
+ })
359
+ }
360
+
361
+ return tuples
362
+ }
363
+
364
+ /**
365
+ * Render a non-US BALANCE row in native order — NO affix split, NO country token. `street` is the OA value verbatim.
366
+ * The sole job is to put a postcode in its native position so the shard doesn't pull the model US-ward.
367
+ */
368
+ function renderBalanceRow(t: BalanceTuple): { raw: string; components: Partial<Record<ComponentTag, string>> } {
369
+ const { house_number: hn, street, locality: loc, postcode: pc, order } = t
370
+ // region is intentionally omitted — it isn't rendered in `raw`, so labeling it would fail alignment.
371
+ const components: Partial<Record<ComponentTag, string>> = { house_number: hn, street, locality: loc, postcode: pc }
372
+ const raw =
373
+ order === "fr"
374
+ ? `${hn} ${street}, ${pc} ${loc}` // French: number-street, postcode-city
375
+ : `${street} ${hn}, ${pc} ${loc}`
376
+
377
+ // DE/IT/NL: street-number, postcode-city
378
+ return { raw, components }
379
+ }
380
+
381
+ export const streetAffixRecipe: ShardRecipe = {
382
+ name: "street-affix",
383
+ description: "US street-affix rows: OA streets split into street_prefix/street/street_suffix (+ multilocale balance)",
384
+ mode: "generate",
385
+ options: [
386
+ {
387
+ flag: "--multilocale-count <N>",
388
+ description: "Append N no-affix native-order balance rows (FR/DE/IT/NL). Default 0",
389
+ },
390
+ ],
391
+ async run(opts, write) {
392
+ // Legacy build-street-affix-shard.mjs seeded `mulberry32(opts.seed)`.
393
+ const random = makeMulberry32(opts.seed)
394
+ const count = opts.count ?? 50000
395
+ const source = opts.sourceName ?? "synth-affix"
396
+ const multilocaleCount = opts.multilocaleCount ?? 0
397
+ const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES
398
+
399
+ const pool: USTuple[] = []
400
+
401
+ for (const s of sources) {
402
+ const t = readTuples(s)
403
+ console.error(` ${s.csv}: ${t.length} unique tuples`)
404
+
405
+ for (const x of t) {
406
+ pool.push(x)
407
+ }
408
+ }
409
+
410
+ if (pool.length === 0) {
411
+ throw new Error("No US tuples found — are the cached OA zips present in /tmp/oa-cache?")
412
+ }
413
+
414
+ let emitted = 0
415
+ let skipped = 0
416
+ let noAffix = 0
417
+ let guard = 0
418
+ const formatCounts: Record<string, number> = {}
419
+ const affixCounts = { prefix: 0, suffix: 0, both: 0 }
420
+ const N = pool.length
421
+
422
+ while (emitted < count && guard++ < count * 10) {
423
+ const base = pool[Math.floor(random() * N)]!
424
+ const parsed = parseStreet(base.street)
425
+
426
+ if (!parsed) {
427
+ noAffix++
428
+ continue
429
+ }
430
+ const { street, components: streetComponents } = renderStreet(random, parsed)
431
+ const { fmt, raw, components } = renderRow(random, base, street, streetComponents)
432
+ // Every affix surface form must survive verbatim in raw, else alignment can't label it.
433
+ const surfaces = [streetComponents.street_prefix, streetComponents.street, streetComponents.street_suffix].filter(
434
+ (s): s is string => Boolean(s)
435
+ )
436
+
437
+ if (!surfaces.every((s) => raw.includes(s))) {
438
+ skipped++
439
+ continue
440
+ }
441
+ formatCounts[fmt] = (formatCounts[fmt] ?? 0) + 1
442
+ const hasP = !!streetComponents.street_prefix
443
+
444
+ if (hasP && streetComponents.street_suffix) {
445
+ affixCounts.both++
446
+ } else if (hasP) {
447
+ affixCounts.prefix++
448
+ } else {
449
+ affixCounts.suffix++
450
+ }
451
+
452
+ if (opts.golden) {
453
+ write(JSON.stringify({ raw, components, country: "US" }) + "\n")
454
+ emitted++
455
+ continue
456
+ }
457
+ const canonical: CanonicalRow = {
458
+ raw,
459
+ components,
460
+ country: "US",
461
+ locale: "en-US",
462
+ source,
463
+ source_id: stableSourceID(source, components),
464
+ corpus_version: "0.4.0",
465
+ license: "OpenAddresses US (non-VT) skeletons, street split via USPS Pub-28 C1/C2 (codex)",
466
+ }
467
+ const aligned = alignRow(canonical)
468
+
469
+ if (aligned.kind !== "labeled" || !aligned.row) {
470
+ skipped++
471
+ continue
472
+ }
473
+ write(JSON.stringify({ ...aligned.row, synth_method: "affix", synth_base_id: null }) + "\n")
474
+ emitted++
475
+ }
476
+
477
+ // ── Multi-locale balance rows (--multilocale-count) ─────────────────────────────────────────────
478
+ // Appended AFTER the US affix rows so the US affix signal is unchanged (same `--count`), and the
479
+ // non-US rows ride the SAME source weight. Native-order postcodes, no affix labels.
480
+ let balanceEmitted = 0
481
+ let balanceSkipped = 0
482
+ const balanceISO: Record<string, number> = {}
483
+
484
+ if (multilocaleCount > 0) {
485
+ const mlSources = opts.golden ? MULTILOCALE_EVAL_SOURCES : MULTILOCALE_SOURCES
486
+ const perSource = Math.ceil((multilocaleCount * 3) / mlSources.length) // over-read; balance locales
487
+ const mlPool: BalanceTuple[] = []
488
+
489
+ for (const s of mlSources) {
490
+ const t = readBalanceTuples(s, perSource)
491
+ console.error(` balance ${s.csv} (${s.iso2}): ${t.length} tuples`)
492
+
493
+ for (const x of t) {
494
+ mlPool.push(x)
495
+ }
496
+ }
497
+ const M = mlPool.length
498
+ let mlGuard = 0
499
+
500
+ while (M > 0 && balanceEmitted < multilocaleCount && mlGuard++ < multilocaleCount * 10) {
501
+ const t = mlPool[Math.floor(random() * M)]!
502
+ const { raw, components } = renderBalanceRow(t)
503
+
504
+ // Every component surface must survive in raw, else alignment can't label it.
505
+ if (![components.street, components.locality, components.postcode].every((s) => !!s && raw.includes(s))) {
506
+ balanceSkipped++
507
+ continue
508
+ }
509
+ balanceISO[t.iso2] = (balanceISO[t.iso2] ?? 0) + 1
510
+ const locale = `${t.iso2.toLowerCase()}-${t.iso2}`
511
+
512
+ if (opts.golden) {
513
+ write(JSON.stringify({ raw, components, country: t.iso2 }) + "\n")
514
+ balanceEmitted++
515
+ continue
516
+ }
517
+ const canonical: CanonicalRow = {
518
+ raw,
519
+ components,
520
+ country: t.iso2,
521
+ locale,
522
+ source,
523
+ source_id: stableSourceID(source, components),
524
+ corpus_version: "0.4.0",
525
+ license: "OpenAddresses non-US skeletons (native-order postcode balance for the affix shard)",
526
+ }
527
+ const aligned = alignRow(canonical)
528
+
529
+ if (aligned.kind !== "labeled" || !aligned.row) {
530
+ balanceSkipped++
531
+ continue
532
+ }
533
+ write(JSON.stringify({ ...aligned.row, synth_method: "affix-balance", synth_base_id: null }) + "\n")
534
+ balanceEmitted++
535
+ }
536
+ }
537
+
538
+ console.error(
539
+ `Done: emitted ${emitted} affix rows, skipped ${skipped}, no-affix ${noAffix} (pool ${pool.length}).\n` +
540
+ ` formats: ${JSON.stringify(formatCounts)}\n` +
541
+ ` affix mix: ${JSON.stringify(affixCounts)}` +
542
+ (multilocaleCount > 0
543
+ ? `\n balance: emitted ${balanceEmitted}, skipped ${balanceSkipped}, iso ${JSON.stringify(balanceISO)}`
544
+ : "")
545
+ )
546
+
547
+ return { emitted: emitted + balanceEmitted, skipped: skipped + balanceSkipped }
548
+ },
549
+ }
@@ -0,0 +1,90 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `street-bare` shard recipe — BARE-street rows (the v0.8.0 harness lever, 2026-06-05). The
7
+ * `functional.test.ts` cluster (bare street names — "10th Ave", "Main St", "1 Main Pl") was
8
+ * mislabeled `locality` because {@link synthesizeStreetRow} only ever emitted streets WITH a ",
9
+ * City, ST ZIP" tail. This recipe emits streets BARE (`--bare-prob`, default 0.6) — no tail, only
10
+ * `street_prefix`/`street`/`street_suffix` (+ optional `house_number` at `--hn-prob`, default
11
+ * 0.85) — over the built-in {@link DEFAULT_US_BASES} pool (round-robin). Generate-mode, US-only,
12
+ * in-distribution (no German-collapse risk). Ported from scripts/build-street-bare-shard.mjs.
13
+ *
14
+ * Byte-fidelity: the legacy script seeded its own mulberry32 from `--seed`
15
+ * (`mulberry32(opts.seed)`); this recipe re-creates the SAME generator
16
+ * (`makeMulberry32(opts.seed)`) and preserves the synthesis call order exactly, so `--seed N`
17
+ * reproduces the legacy run byte-for-byte.
18
+ */
19
+
20
+ import { alignRow } from "../align.ts"
21
+ import { DEFAULT_US_BASES } from "../synthesize-intersection.ts"
22
+ import { synthesizeStreetRow, type StreetBaseTuple } from "../synthesize-street.ts"
23
+ import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe } from "./scaffold.ts"
24
+
25
+ export const streetBareRecipe: ShardRecipe = {
26
+ name: "street-bare",
27
+ description: "Bare-street rows (US): DEFAULT_US_BASES → synthesizeStreetRow (bare) → aligned LabeledRow",
28
+ mode: "generate",
29
+ options: [
30
+ { flag: "--bare-prob <p>", description: "P(emit the street BARE — no city/region/postcode tail). Default 0.6" },
31
+ { flag: "--hn-prob <p>", description: "P(emit a house number). Default 0.85" },
32
+ ],
33
+ async run(opts, write) {
34
+ const random = makeMulberry32(opts.seed)
35
+ const count = opts.count ?? 3000
36
+ const bareProb = opts.bareProb ?? 0.6
37
+ const hnProb = opts.hnProb ?? 0.85
38
+ const source = opts.sourceName ?? "synth-street-bare"
39
+
40
+ let emitted = 0
41
+ let skipped = 0
42
+ let guard = 0
43
+
44
+ while (emitted < count && guard++ < count * 5) {
45
+ const base = DEFAULT_US_BASES[emitted % DEFAULT_US_BASES.length]!
46
+ const synth = synthesizeStreetRow(base as StreetBaseTuple, {
47
+ random,
48
+ bareProb,
49
+ includeHouseNumberProb: hnProb,
50
+ })
51
+
52
+ if (!synth) {
53
+ skipped++
54
+ continue
55
+ }
56
+ const isBare = synth.components.region === undefined
57
+
58
+ const canonical: CanonicalShardRow = {
59
+ raw: synth.raw,
60
+ components: synth.components,
61
+ country: base.country,
62
+ locale: synth.locale,
63
+ source,
64
+ source_id: shardSourceID(source, {
65
+ street: synth.components.street,
66
+ street_suffix: synth.components.street_suffix,
67
+ house_number: synth.components.house_number,
68
+ bare: String(isBare),
69
+ n: String(emitted),
70
+ }),
71
+ corpus_version: "0.4.0",
72
+ license: "Synthetic — US street templates, public-domain street/city pools",
73
+ }
74
+
75
+ // Strict labeled-only check (matches the legacy builder): alignRow always returns a `row`
76
+ // (labeled OR quarantined), so the scaffold's `alignAndWrite` would write quarantined rows
77
+ // too — call alignRow directly and skip anything not "labeled".
78
+ const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
79
+
80
+ if (aligned.kind !== "labeled" || !aligned.row) {
81
+ skipped++
82
+ continue
83
+ }
84
+ write(JSON.stringify({ ...aligned.row, synth_method: "street-bare", synth_base_id: null }) + "\n")
85
+ emitted++
86
+ }
87
+
88
+ return { emitted, skipped }
89
+ },
90
+ }