@mailwoman/corpus 7.2.0 → 7.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +59 -27
  2. package/src/adapter.ts +153 -0
  3. package/src/adapters/ban/adapter.ts +189 -0
  4. package/src/adapters/ban/street-decompose.ts +98 -0
  5. package/src/adapters/fcc-bdc/adapter.ts +188 -0
  6. package/src/adapters/geonames/adapter.ts +185 -0
  7. package/src/adapters/geonames-postal/adapter.ts +111 -0
  8. package/src/adapters/gnaf/adapter.ts +139 -0
  9. package/src/adapters/gnaf/assemble.ts +200 -0
  10. package/src/adapters/index.ts +135 -0
  11. package/src/adapters/openaddresses/adapter.ts +237 -0
  12. package/src/adapters/overture/adapter.ts +149 -0
  13. package/src/adapters/state-hi-schools/adapter.ts +171 -0
  14. package/src/adapters/state-ia-contractors/adapter.ts +148 -0
  15. package/src/adapters/state-ny-notaries/adapter.ts +169 -0
  16. package/src/adapters/state-tx-notaries/adapter.ts +160 -0
  17. package/src/adapters/synth-po-box/adapter.ts +197 -0
  18. package/src/adapters/tiger/adapter.ts +232 -0
  19. package/src/adapters/tiger/street-decompose.ts +120 -0
  20. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
  21. package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
  22. package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
  23. package/src/adapters/usgov-nad/adapter.ts +338 -0
  24. package/src/adapters/usgov-nppes/adapter.ts +169 -0
  25. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
  26. package/src/adapters/wof-admin-jp/adapter.ts +176 -0
  27. package/src/adapters/wof-admin-json/adapter.ts +280 -0
  28. package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
  29. package/src/align.ts +317 -0
  30. package/src/build.ts +348 -0
  31. package/src/codex/us-fips-state.ts +109 -0
  32. package/src/format.ts +14 -0
  33. package/src/golden.ts +147 -0
  34. package/src/index.ts +28 -0
  35. package/src/license.ts +48 -0
  36. package/src/parquet-wrapper/index.ts +9 -0
  37. package/src/parquet-wrapper/reader.ts +67 -0
  38. package/src/parquet-wrapper/schema.ts +80 -0
  39. package/src/parquet-wrapper/writer.ts +102 -0
  40. package/src/parquet.ts +360 -0
  41. package/src/runner.ts +283 -0
  42. package/src/shard-recipes/anchor-absorption.ts +86 -0
  43. package/src/shard-recipes/boundary-stress.ts +87 -0
  44. package/src/shard-recipes/country-balanced.ts +471 -0
  45. package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
  46. package/src/shard-recipes/fr-admin-split.ts +212 -0
  47. package/src/shard-recipes/fr-bare-street.ts +85 -0
  48. package/src/shard-recipes/fr-fragment.ts +304 -0
  49. package/src/shard-recipes/fr-order.ts +283 -0
  50. package/src/shard-recipes/german.ts +206 -0
  51. package/src/shard-recipes/house-venue.ts +78 -0
  52. package/src/shard-recipes/index.ts +76 -0
  53. package/src/shard-recipes/intersection.ts +617 -0
  54. package/src/shard-recipes/locale.ts +415 -0
  55. package/src/shard-recipes/nl-postcode.ts +105 -0
  56. package/src/shard-recipes/no-fragment.ts +206 -0
  57. package/src/shard-recipes/no-street-led.ts +130 -0
  58. package/src/shard-recipes/no-street.ts +78 -0
  59. package/src/shard-recipes/po-box-cedex.ts +853 -0
  60. package/src/shard-recipes/po-box.ts +111 -0
  61. package/src/shard-recipes/scaffold.ts +187 -0
  62. package/src/shard-recipes/si-bare-village.ts +88 -0
  63. package/src/shard-recipes/street-affix.ts +549 -0
  64. package/src/shard-recipes/street-bare.ts +90 -0
  65. package/src/shard-recipes/street.ts +74 -0
  66. package/src/shard-recipes/unit.ts +315 -0
  67. package/src/split.ts +251 -0
  68. package/src/synthesize-anchor-absorption.ts +263 -0
  69. package/src/synthesize-boundary-stress.ts +504 -0
  70. package/src/synthesize-german.ts +227 -0
  71. package/src/synthesize-house-venue.ts +192 -0
  72. package/src/synthesize-intersection.ts +205 -0
  73. package/src/synthesize-no-street.ts +341 -0
  74. package/src/synthesize-po-box.ts +307 -0
  75. package/src/synthesize-street.ts +264 -0
  76. package/src/synthesize.ts +872 -0
  77. package/src/tokenize.ts +68 -0
  78. package/src/tools/align-shard.ts +67 -0
  79. package/src/tools/audit.ts +352 -0
  80. package/src/tools/corpus-stats.ts +183 -0
  81. package/src/tools/fetch/ban.ts +288 -0
  82. package/src/tools/fetch/download.ts +119 -0
  83. package/src/tools/fetch/hrsa.ts +65 -0
  84. package/src/tools/fetch/imls-pls.ts +162 -0
  85. package/src/tools/fetch/index.ts +121 -0
  86. package/src/tools/fetch/nad.ts +304 -0
  87. package/src/tools/fetch/nppes.ts +177 -0
  88. package/src/tools/fetch/openaddresses.ts +380 -0
  89. package/src/tools/fetch/state-hi-schools.ts +217 -0
  90. package/src/tools/fetch/state-sources.ts +142 -0
  91. package/src/tools/fetch/tiger-full.ts +318 -0
  92. package/src/tools/golden-expand.ts +524 -0
  93. package/src/tools/golden-promote.ts +304 -0
  94. package/src/tools/index.ts +24 -0
  95. package/src/tools/ingest-csv.ts +420 -0
  96. package/src/tools/jsonl-to-parquet.ts +200 -0
  97. package/src/tools/lint-shard-vocab.ts +357 -0
  98. package/src/tools/lint-shard.ts +502 -0
  99. package/src/tools/overlay-manifest.ts +141 -0
  100. package/src/tools/shard-kryptonite.ts +131 -0
  101. package/src/tools/shard-translit.ts +244 -0
  102. package/src/types.ts +213 -0
  103. package/src/wof-json.ts +236 -0
@@ -0,0 +1,283 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `fr-order` shard recipe — French reversed-order coverage (#560). Reads REAL OpenAddresses FR
7
+ * tuples (`fr/countrywide.csv` from the cached zip), then for each row picks — by
8
+ * `--reversed-fraction` (default 0.5) — whether to render in CANONICAL French order
9
+ * (number-street, postcode-city) or one of four REVERSED / postcode-first variants the
10
+ * v4.4.0→v0.5.0 regression exposed (the model misses house_number in every reversed one):
11
+ *
12
+ * - A: "47110 Sainte-Livrade-sur-Lot, 69 Allée du Bugatel" (postcode city, HN street)
13
+ * - B: "Sainte-Livrade-sur-Lot, 47110, 619 Impasse de la Rose" (city, postcode, HN street)
14
+ * - C: "Sainte-Livrade-sur-Lot 59 bis Rue des Ecuries 47110" (city HN street postcode — NO commas)
15
+ * - D: "47110, 6 rue de la république, Sainte-Livrade-sur-Lot" (postcode, HN street, city)
16
+ *
17
+ * Sub-modes ride alongside order: `bis`/`ter`/`quater` ordinal suffixes in house_number, and
18
+ * ALL-CAPS locality. `--golden` emits a held-out reversed-order eval slice with a different
19
+ * seed.
20
+ *
21
+ * The inline synthesis (the OA-CSV reader, the ordinal/all-caps tables, the canonical + reversed
22
+ * renderers) is ported faithfully from scripts/build-fr-order-shard.mjs. This is a
23
+ * `generate`-mode recipe that still reads REAL tuples off disk — `--count` bounds the OUTPUT, not
24
+ * the input. The passed `random` (the framework LCG) is consumed in the exact call order the
25
+ * legacy script used.
26
+ *
27
+ * NOT ported (diagnostic-only, no effect on emitted bytes): the post-run `runSpanCheck` self-check
28
+ * (it reads the finished file back with a separate PRNG and prints to stderr; the recipe's output
29
+ * stream is still open during `run`), and the dead `renderReversed` helper (the legacy `main`
30
+ * inlined the variant logic and never called it).
31
+ */
32
+
33
+ import { spawnSync } from "node:child_process"
34
+
35
+ import type { ComponentTag } from "@mailwoman/core/types"
36
+
37
+ import { stableSourceID } from "../adapter.ts"
38
+ import { alignRow } from "../align.ts"
39
+ import type { CanonicalRow } from "../types.ts"
40
+ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
41
+
42
+ const SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }
43
+
44
+ // Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
45
+ const ORDINAL_SUFFIXES: readonly string[] = ["bis", "ter", "quater"]
46
+ // Probability that a row gets an ordinal suffix injected (matches the golden's ~10-15% rate).
47
+ const ORDINAL_PROB = 0.12
48
+ // Probability that a locality renders ALL-CAPS (another sub-mode: "SAINTE-LIVRADE-SUR-LOT").
49
+ const ALLCAPS_PROB = 0.1
50
+
51
+ /** A real FR tuple read out of the cached OA zip. */
52
+ interface FrTuple {
53
+ house_number: string
54
+ street: string
55
+ locality: string
56
+ postcode: string
57
+ }
58
+
59
+ /** Minimal RFC-4180-ish splitter (handles quoted fields with doubled-quote escaping). */
60
+ function splitCSV(line: string): string[] {
61
+ const out: string[] = []
62
+ let cur = ""
63
+ let inQ = false
64
+
65
+ for (let i = 0; i < line.length; i++) {
66
+ const c = line[i]
67
+
68
+ if (inQ) {
69
+ if (c === '"') {
70
+ if (line[i + 1] === '"') {
71
+ cur += '"'
72
+ i++
73
+ } else {
74
+ inQ = false
75
+ }
76
+ } else {
77
+ cur += c
78
+ }
79
+ } else if (c === '"') {
80
+ inQ = true
81
+ } else if (c === ",") {
82
+ out.push(cur)
83
+ cur = ""
84
+ } else {
85
+ cur += c
86
+ }
87
+ }
88
+ out.push(cur)
89
+
90
+ return out
91
+ }
92
+
93
+ /**
94
+ * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` to stay under V8's
95
+ * string limit. Only keeps rows with a house_number (the shard's core signal) and a postcode (required for
96
+ * reversed-order rendering to be meaningful).
97
+ */
98
+ function readTuples(limit: number): FrTuple[] {
99
+ const maxLines = Math.max(limit * 8, 40000) + 1
100
+ const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
101
+ maxBuffer: 1024 * 1024 * 1024,
102
+ encoding: "buffer",
103
+ })
104
+
105
+ if (r.status !== 0) {
106
+ console.error(` WARN: unzip failed for ${SOURCE.zip} (status ${r.status})`)
107
+
108
+ return []
109
+ }
110
+ const lines = r.stdout.toString("utf8").split(/\r?\n/)
111
+
112
+ if (lines.length < 2) return []
113
+ const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
114
+ const idx = (name: string): number => header.indexOf(name)
115
+ const iNum = idx("number"),
116
+ iStreet = idx("street"),
117
+ iCity = idx("city"),
118
+ iPost = idx("postcode")
119
+ const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
120
+ const tuples: FrTuple[] = []
121
+ const seen = new Set<string>()
122
+
123
+ for (let li = 1; li < lines.length && tuples.length < limit; li++) {
124
+ if (!lines[li]) continue
125
+ const cells = splitCSV(lines[li]!)
126
+ const street = get(cells, iStreet)
127
+ const locality = get(cells, iCity)
128
+ const house_number = get(cells, iNum)
129
+ const postcode = get(cells, iPost)
130
+
131
+ // Require all four fields: HN is the signal; postcode drives reversed-order variants.
132
+ if (!street || !locality || !house_number || !postcode) continue
133
+ const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase()
134
+
135
+ if (seen.has(key)) continue
136
+ seen.add(key)
137
+ tuples.push({ house_number, street, locality, postcode })
138
+ }
139
+
140
+ return tuples
141
+ }
142
+
143
+ /**
144
+ * Optionally augment a house_number with a French ordinal suffix ("59 bis", "4 ter"). Appended with a space so it forms
145
+ * one multi-token house_number string that alignRow can still locate verbatim.
146
+ */
147
+ function maybeAddOrdinal(random: () => number, house_number: string): string {
148
+ if (random() >= ORDINAL_PROB) return house_number
149
+ const suffix = ORDINAL_SUFFIXES[Math.floor(random() * ORDINAL_SUFFIXES.length)]!
150
+
151
+ // Vary suffix case: "bis" (lower) vs "BIS" (upper) — a real-world split in the golden.
152
+ return `${house_number} ${random() < 0.5 ? suffix : suffix.toUpperCase()}`
153
+ }
154
+
155
+ /** Render a tuple in CANONICAL French order: "9 Rue de la Promenade, 01200 Villes". */
156
+ function renderCanonical(
157
+ hn: string,
158
+ street: string,
159
+ postcode: string,
160
+ locality: string
161
+ ): { raw: string; components: Partial<Record<ComponentTag, string>> } {
162
+ const raw = `${hn} ${street}, ${postcode} ${locality}`
163
+
164
+ return { raw, components: { house_number: hn, street, postcode, locality } }
165
+ }
166
+
167
+ export const frOrderRecipe: ShardRecipe = {
168
+ name: "fr-order",
169
+ description: "French reversed-order rows (#560): real OA FR tuples rendered canonical + 4 postcode-first variants",
170
+ mode: "generate",
171
+ options: [
172
+ { flag: "--reversed-fraction <p>", description: "Fraction rendered reversed-order. Default 0.5" },
173
+ { flag: "--golden", description: "Emit the held-out reversed-order eval slice" },
174
+ ],
175
+ async run(opts, write) {
176
+ if (opts.count == null) throw new Error("fr-order recipe requires --count <N>")
177
+ const count = opts.count
178
+ // Legacy build-fr-order-shard.mjs seeded mulberry32 with the raw seed: `const random = mulberry32(opts.seed)`.
179
+ // (The omitted diagnostic runSpanCheck used a separate mulberry32(opts.seed + 1) — not part of generation.)
180
+ const random = makeMulberry32(opts.seed)
181
+ const source = opts.sourceName ?? "synth-fr-order"
182
+ const reversedFraction = opts.reversedFraction ?? 0.5
183
+
184
+ // Over-read from the CSV so the dedup + filter pass can fill `count` rows.
185
+ const poolLimit = Math.max(count * 8, 40000)
186
+ const pool = readTuples(poolLimit)
187
+ console.error(` ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
188
+
189
+ if (pool.length === 0) {
190
+ throw new Error("No FR tuples found — is /tmp/oa-cache/fr__countrywide.zip present?")
191
+ }
192
+
193
+ let emitted = 0
194
+ let skipped = 0
195
+ let guard = 0
196
+ const N = pool.length
197
+
198
+ while (emitted < count && guard++ < count * 10) {
199
+ const base = pool[Math.floor(random() * N)]!
200
+ const { street, postcode } = base
201
+ const locality = random() < ALLCAPS_PROB ? base.locality.toUpperCase() : base.locality
202
+ const house_number = maybeAddOrdinal(random, base.house_number)
203
+
204
+ // Pick canonical vs reversed by --reversed-fraction.
205
+ const isReversed = random() < reversedFraction
206
+
207
+ let rendered: { raw: string; components: Partial<Record<ComponentTag, string>> }
208
+
209
+ if (isReversed) {
210
+ const variantRoll = random()
211
+ let raw: string
212
+
213
+ if (variantRoll < 0.25) {
214
+ // Variant A: postcode+city as a unit, then HN+street
215
+ raw = `${postcode} ${locality}, ${house_number} ${street}`
216
+ } else if (variantRoll < 0.5) {
217
+ // Variant B: city, then postcode, then HN+street (comma-separated, postcode isolated)
218
+ raw = `${locality}, ${postcode}, ${house_number} ${street}`
219
+ } else if (variantRoll < 0.75) {
220
+ // Variant C: no commas — locality HN street postcode (the "run-together" format)
221
+ raw = `${locality} ${house_number} ${street} ${postcode}`
222
+ } else {
223
+ // Variant D: postcode, HN+street, city (reversed top-to-bottom)
224
+ raw = `${postcode}, ${house_number} ${street}, ${locality}`
225
+ }
226
+ rendered = { raw, components: { house_number, street, postcode, locality } }
227
+ } else {
228
+ rendered = renderCanonical(house_number, street, postcode, locality)
229
+ }
230
+
231
+ const { raw, components } = rendered
232
+
233
+ // Safety check: every component must appear verbatim in raw (alignment precondition).
234
+ const componentValues = Object.values(components).filter(Boolean) as string[]
235
+
236
+ if (!componentValues.every((v) => raw.includes(v))) {
237
+ skipped++
238
+ continue
239
+ }
240
+
241
+ // --golden: emit per-locale-f1 eval rows ({raw, components, country:"FR"}).
242
+ if (opts.golden) {
243
+ write(JSON.stringify({ raw, components, country: "FR" }) + "\n")
244
+ emitted++
245
+ continue
246
+ }
247
+
248
+ const sourceID = stableSourceID(source, {
249
+ street: components.street,
250
+ house_number: components.house_number,
251
+ locality: components.locality,
252
+ postcode: components.postcode,
253
+ })
254
+ const canonical: CanonicalRow = {
255
+ raw,
256
+ components,
257
+ country: "FR",
258
+ locale: "fr-FR",
259
+ source,
260
+ source_id: sourceID,
261
+ corpus_version: "0.5.0",
262
+ license: "OpenAddresses FR countrywide tuples, rendered canonical + reversed-order — see ingest SOURCE",
263
+ }
264
+ const aligned = alignRow(canonical)
265
+
266
+ if (aligned.kind !== "labeled" || !aligned.row) {
267
+ skipped++
268
+ continue
269
+ }
270
+ write(
271
+ JSON.stringify({
272
+ ...aligned.row,
273
+ synth_method: "fr-order",
274
+ synth_order: isReversed ? "reversed" : "canonical",
275
+ synth_base_id: null,
276
+ }) + "\n"
277
+ )
278
+ emitted++
279
+ }
280
+
281
+ return { emitted, skipped }
282
+ },
283
+ }
@@ -0,0 +1,206 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `german` shard recipe — German coverage rows from REAL OpenAddresses tuples (Berlin + Saxony,
7
+ * cached zips). Each sampled tuple is rendered via {@link synthesizeGermanRow} in BOTH orders —
8
+ * `--intl-fraction` (default 0.4) in international order (house-first / postcode-after-city), the
9
+ * rest in idiomatic German order — then aligned to BIO. Generate-mode: it builds a tuple pool
10
+ * from the cached zips, then draws `--count` rows from it with the passed `random` (so the emit
11
+ * stream matches the legacy reservoir-sample loop). Ported from scripts/build-german-shard.mjs.
12
+ *
13
+ * ORDER ROBUSTNESS (2026-06-06): mixing the two renderings stops a native-only shard from teaching
14
+ * German order so well it reads the US/feed-order eval as a "collapse". See
15
+ * docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md (the order-artifact correction).
16
+ */
17
+
18
+ import { spawnSync } from "node:child_process"
19
+
20
+ import { stableSourceID } from "../adapter.ts"
21
+ import { alignRow } from "../align.ts"
22
+ import { synthesizeGermanRow, type LocaleBaseTuple } from "../synthesize-german.ts"
23
+ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
24
+
25
+ /**
26
+ * A German OA source (cached zip) + the Bundesland the file covers (OA's REGION column is empty for DE).
27
+ */
28
+ interface GermanSource {
29
+ zip: string
30
+ csv: string
31
+ region: string
32
+ }
33
+
34
+ // `region` is the Bundesland the source covers. OA's REGION column is empty for DE, but the region is
35
+ // implied by the per-state file — the international order needs it for the "City, Region Postcode" tail
36
+ // (v0.9.3 / #327). berlin.csv → Berlin (a city-state, region==locality); sn/statewide → Sachsen.
37
+ const SOURCES: GermanSource[] = [
38
+ { zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
39
+ { zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
40
+ ]
41
+
42
+ /** Minimal RFC-4180-ish splitter (handles quoted fields). */
43
+ function splitCSV(line: string): string[] {
44
+ const out: string[] = []
45
+ let cur = ""
46
+ let inQ = false
47
+
48
+ for (let i = 0; i < line.length; i++) {
49
+ const c = line[i]!
50
+
51
+ if (inQ) {
52
+ if (c === '"') {
53
+ if (line[i + 1] === '"') {
54
+ cur += '"'
55
+ i++
56
+ } else {
57
+ inQ = false
58
+ }
59
+ } else {
60
+ cur += c
61
+ }
62
+ } else if (c === '"') {
63
+ inQ = true
64
+ } else if (c === ",") {
65
+ out.push(cur)
66
+ cur = ""
67
+ } else {
68
+ cur += c
69
+ }
70
+ }
71
+ out.push(cur)
72
+
73
+ return out
74
+ }
75
+
76
+ /** Stream real German tuples out of a cached OA zip (buffered `unzip -p`). */
77
+ function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
78
+ const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
79
+
80
+ if (r.status !== 0) {
81
+ console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`)
82
+
83
+ return []
84
+ }
85
+ const lines = r.stdout.toString("utf8").split(/\r?\n/)
86
+
87
+ if (lines.length < 2) return []
88
+ const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
89
+ const idx = (name: string): number => header.indexOf(name)
90
+ const iNum = idx("number"),
91
+ iStreet = idx("street"),
92
+ iCity = idx("city"),
93
+ iRegion = idx("region"),
94
+ iPost = idx("postcode")
95
+ const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
96
+ const tuples: LocaleBaseTuple[] = []
97
+ const seen = new Set<string>()
98
+
99
+ for (let li = 1; li < lines.length; li++) {
100
+ const lineStr = lines[li]
101
+
102
+ if (!lineStr) continue
103
+ const cells = splitCSV(lineStr)
104
+ const street = get(cells, iStreet)
105
+ const locality = get(cells, iCity)
106
+
107
+ if (!street || !locality) continue
108
+ const house_number = get(cells, iNum)
109
+ const postcode = get(cells, iPost)
110
+ // OA's REGION column is empty for DE — fall back to the source's Bundesland (set per file).
111
+ const region = get(cells, iRegion) || source.region || ""
112
+ const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase()
113
+
114
+ if (seen.has(key)) continue
115
+ seen.add(key)
116
+ tuples.push({ house_number, street, locality, region, postcode })
117
+ }
118
+
119
+ return tuples
120
+ }
121
+
122
+ export const germanRecipe: ShardRecipe = {
123
+ name: "german",
124
+ description: "German coverage rows from real OA tuples (Berlin/Saxony), both orders → synthesizeGermanRow",
125
+ mode: "generate",
126
+ options: [{ flag: "--intl-fraction <f>", description: "Fraction rendered international order. Default 0.4" }],
127
+ async run(opts, write) {
128
+ // Emit PRNG: the legacy build-german-shard.mjs seeded mulberry32(opts.seed).
129
+ const random = makeMulberry32(opts.seed)
130
+ const source = opts.sourceName ?? "synth-german"
131
+ const intlFraction = opts.intlFraction ?? 0.4
132
+
133
+ if (!(intlFraction >= 0 && intlFraction <= 1)) {
134
+ throw new Error(`--intl-fraction must be in [0, 1], got ${intlFraction}`)
135
+ }
136
+ const count = opts.count ?? 4000
137
+
138
+ // Pool real tuples from every German source, then sample `count` rows from it.
139
+ const pool: LocaleBaseTuple[] = []
140
+
141
+ for (const s of SOURCES) {
142
+ const t = readGermanTuples(s)
143
+ console.error(` ${s.csv}: ${t.length} unique tuples`)
144
+
145
+ for (const x of t) {
146
+ pool.push(x)
147
+ } // NOT pool.push(...t) — spreading ~840K args overflows the stack
148
+ }
149
+
150
+ if (pool.length === 0) {
151
+ throw new Error("No German tuples found — are the cached zips present in /tmp/oa-cache?")
152
+ }
153
+
154
+ let emitted = 0
155
+ let skipped = 0
156
+ let guard = 0
157
+ const N = pool.length
158
+
159
+ while (emitted < count && guard++ < count * 6) {
160
+ const base = pool[Math.floor(random() * N)]!
161
+ // Per-row order: `--intl-fraction` of rows render house-first / postcode-after-city (the US/feed
162
+ // layout), the rest in idiomatic German order. Same components either way.
163
+ const order = random() < intlFraction ? "international" : "native"
164
+ const synth = synthesizeGermanRow(base, { random, order })
165
+
166
+ if (!synth) {
167
+ skipped++
168
+ continue
169
+ }
170
+
171
+ // --golden: emit per-locale-f1 eval rows ({raw, components}) instead of aligned BIO. `order`
172
+ // rides along so the eval can stratify native vs international.
173
+ if (opts.golden) {
174
+ write(JSON.stringify({ raw: synth.raw, components: synth.components, country: "DE", order }) + "\n")
175
+ emitted++
176
+ continue
177
+ }
178
+ const sourceID = stableSourceID(source, {
179
+ street: synth.components.street,
180
+ house_number: synth.components.house_number,
181
+ locality: synth.components.locality,
182
+ postcode: synth.components.postcode,
183
+ })
184
+ const canonical = {
185
+ raw: synth.raw,
186
+ components: synth.components,
187
+ country: "DE",
188
+ locale: synth.locale,
189
+ source,
190
+ source_id: sourceID,
191
+ corpus_version: "0.4.0",
192
+ license: `OpenAddresses DE (Berlin/Saxony) tuples, rendered ${order}-order — see ingest SOURCES`,
193
+ }
194
+ const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
195
+
196
+ if (aligned.kind !== "labeled" || !aligned.row) {
197
+ skipped++
198
+ continue
199
+ }
200
+ write(JSON.stringify({ ...aligned.row, synth_method: "german", synth_order: order, synth_base_id: null }) + "\n")
201
+ emitted++
202
+ }
203
+
204
+ return { emitted, skipped }
205
+ },
206
+ }
@@ -0,0 +1,78 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `house-venue` shard recipe — synthetic house_number+venue+street co-occurrence rows: tuples →
7
+ * {@link synthesizeHouseVenueRow} → aligned LabeledRow. The v0.6.3 corrective companion to the
8
+ * no-street shard: every row carries BOTH house_number AND venue, restoring the house_number
9
+ * signal that no-street's distributional shift cost the model. Ported from
10
+ * scripts/build-house-venue-shard.mjs.
11
+ */
12
+
13
+ import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "../synthesize-house-venue.ts"
14
+ import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
15
+
16
+ const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
17
+
18
+ export const houseVenueRecipe: ShardRecipe = {
19
+ name: "house-venue",
20
+ description: "House_number+venue co-occurrence rows: tuples → synthesizeHouseVenueRow → aligned LabeledRow",
21
+ mode: "tuples",
22
+ async run(opts, write) {
23
+ if (!opts.input) throw new Error("house-venue recipe requires --input <tuples.jsonl>")
24
+ // Legacy build-house-venue-shard.mjs seeded the LCG via makeRandom(opts.seed) (s = seed).
25
+ const random = makeLcg(opts.seed)
26
+ const source = opts.sourceName ?? "synth-house-venue"
27
+ let read = 0
28
+ let emitted = 0
29
+ let skipped = 0
30
+
31
+ for await (const tuple of readTuples(opts.input)) {
32
+ read++
33
+
34
+ if (!tuple.locality || !tuple.region || !tuple.postcode || !tuple.country) {
35
+ skipped++
36
+ continue
37
+ }
38
+
39
+ for (let v = 0; v < opts.variants; v++) {
40
+ const synth = synthesizeHouseVenueRow(tuple as HouseVenueBaseTuple, { random })
41
+
42
+ if (!synth) {
43
+ skipped++
44
+ continue
45
+ }
46
+ const ok = alignAndWrite(
47
+ write,
48
+ {
49
+ raw: synth.raw,
50
+ components: synth.components,
51
+ country: tuple.country,
52
+ locale: synth.locale,
53
+ source,
54
+ source_id: shardSourceID(source, {
55
+ locality: tuple.locality,
56
+ region: tuple.region,
57
+ postcode: tuple.postcode,
58
+ country: tuple.country,
59
+ template: synth.template,
60
+ v: String(v),
61
+ }),
62
+ corpus_version: "0.4.0",
63
+ license: LICENSE,
64
+ },
65
+ synth.template
66
+ )
67
+
68
+ if (ok) {
69
+ emitted++
70
+ } else {
71
+ skipped++
72
+ }
73
+ }
74
+ }
75
+
76
+ return { read, emitted, skipped }
77
+ },
78
+ }
@@ -0,0 +1,76 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * The synthetic-corpus SHARD RECIPE registry. Each recipe ({@link ShardRecipe}) is what one legacy
7
+ * `build-*-shard.mjs` script used to be; the `mailwoman corpus shard <recipe>` command dispatches
8
+ * to them. A new shard is a new entry here, not a new script.
9
+ */
10
+
11
+ import { anchorAbsorptionRecipe } from "./anchor-absorption.ts"
12
+ import { boundaryStressRecipe } from "./boundary-stress.ts"
13
+ import { countryBalancedRecipe } from "./country-balanced.ts"
14
+ import { czPcFirstPrepositionRecipe } from "./cz-pcfirst-preposition.ts"
15
+ import { frAdminSplitRecipe } from "./fr-admin-split.ts"
16
+ import { frBareStreetRecipe } from "./fr-bare-street.ts"
17
+ import { frFragmentRecipe } from "./fr-fragment.ts"
18
+ import { frOrderRecipe } from "./fr-order.ts"
19
+ import { germanRecipe } from "./german.ts"
20
+ import { houseVenueRecipe } from "./house-venue.ts"
21
+ import { intersectionRecipe } from "./intersection.ts"
22
+ import { localeRecipe } from "./locale.ts"
23
+ import { nlPostcodeRecipe } from "./nl-postcode.ts"
24
+ import { noFragmentRecipe } from "./no-fragment.ts"
25
+ import { noStreetLedRecipe } from "./no-street-led.ts"
26
+ import { noStreetRecipe } from "./no-street.ts"
27
+ import { poBoxCedexRecipe } from "./po-box-cedex.ts"
28
+ import { poBoxRecipe } from "./po-box.ts"
29
+ import type { ShardRecipe } from "./scaffold.ts"
30
+ import { siBareVillageRecipe } from "./si-bare-village.ts"
31
+ import { streetAffixRecipe } from "./street-affix.ts"
32
+ import { streetBareRecipe } from "./street-bare.ts"
33
+ import { streetRecipe } from "./street.ts"
34
+ import { unitRecipe } from "./unit.ts"
35
+
36
+ export * from "./scaffold.ts"
37
+
38
+ /** Every registered recipe, in display order. */
39
+ const RECIPES: readonly ShardRecipe[] = [
40
+ streetRecipe,
41
+ streetBareRecipe,
42
+ streetAffixRecipe,
43
+ noStreetRecipe,
44
+ houseVenueRecipe,
45
+ poBoxRecipe,
46
+ poBoxCedexRecipe,
47
+ unitRecipe,
48
+ intersectionRecipe,
49
+ germanRecipe,
50
+ localeRecipe,
51
+ frOrderRecipe,
52
+ frAdminSplitRecipe,
53
+ frBareStreetRecipe,
54
+ frFragmentRecipe,
55
+ czPcFirstPrepositionRecipe,
56
+ nlPostcodeRecipe,
57
+ noStreetLedRecipe,
58
+ noFragmentRecipe,
59
+ siBareVillageRecipe,
60
+ countryBalancedRecipe,
61
+ boundaryStressRecipe,
62
+ anchorAbsorptionRecipe,
63
+ ]
64
+
65
+ /** Recipe name → recipe. */
66
+ export const SHARD_RECIPES: ReadonlyMap<string, ShardRecipe> = new Map(RECIPES.map((r) => [r.name, r]))
67
+
68
+ /** Look up a recipe by its `<recipe>` name. */
69
+ export function getShardRecipe(name: string): ShardRecipe | undefined {
70
+ return SHARD_RECIPES.get(name)
71
+ }
72
+
73
+ /** All recipes (for `--list` / help). */
74
+ export function listShardRecipes(): readonly ShardRecipe[] {
75
+ return RECIPES
76
+ }