@mailwoman/corpus 8.6.0 → 9.1.0

package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts

@@ -28,14 +28,11 @@
  *   distribution terms.
  */
 
-import { createReadStream } from "node:fs"
-
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
-import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
-import { reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { splitStreetLine, stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { lookupStateAbbreviation } from "@mailwoman/corpus/codex/us-fips-state"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { reconcileComponents } from "@mailwoman/formatter"
+import { CSVSpliterator } from "spliterator"
 
 /**
  * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
@@ -66,19 +63,6 @@ interface SamhsaSiteRow {
 	frid?: string
 }
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 /**
  * Join the SAMHSA two-line street: primary street + optional secondary line (suite / unit / floor / "behind main
  * building") on `", "`. The combined value is the `street` component surface form. Phase 1 does not break this out into
@@ -148,76 +132,65 @@ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
 				throw new Error(`usgov-samhsa adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<SamhsaSiteRow>) {
-					if (opts.signal?.aborted) break
+			for await (const record of rows as AsyncIterable<SamhsaSiteRow>) {
+				if (opts.signal?.aborted) break
 
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					const venue = composeVenue(record.name1 ?? "", record.name2)
-					const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
-					const split = splitAddress(street)
-					const city = (record.city ?? "").trim()
-					const stateAbbr = (record.state ?? "").trim()
-					const postcode = (record.zip ?? "").trim()
+				const venue = composeVenue(record.name1 ?? "", record.name2)
+				const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
+				const split = splitStreetLine(street)
+				const city = (record.city ?? "").trim()
+				const stateAbbr = (record.state ?? "").trim()
+				const postcode = (record.zip ?? "").trim()
 
-					if (!venue || !split || !city || !postcode) continue
-					const state = lookupStateAbbreviation(stateAbbr)
+				if (!venue || !split || !city || !postcode) continue
+				const state = lookupStateAbbreviation(stateAbbr)
 
-					if (!state) continue
+				if (!state) continue
 
-					// venue first — same kryptonite-defending insertion order as HRSA.
-					const components: CanonicalRow["components"] = {
-						venue,
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode,
-					}
-
-					const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
+				// venue first — same kryptonite-defending insertion order as HRSA.
+				const components: CanonicalRow["components"] = {
+					venue,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
 
-					if (!raw) continue
+				const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
 
-					const aligned = reconcileComponents(components, raw)
+				if (!raw) continue
 
-					if (!Object.keys(aligned).length) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const frID = (record.frid ?? "").trim()
+				if (!Object.keys(aligned).length) continue
 
-					const sourceID = frID
-						? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}`
-						: stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
+				const frID = (record.frid ?? "").trim()
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: USGOV_SAMHSA_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: USGOV_SAMHSA_DEFAULT_LICENSE,
-					}
+				const sourceID = frID ? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}` : stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_SAMHSA_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_SAMHSA_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}

package/src/{adapter.ts → adapters/utils/index.ts}

@@ -12,6 +12,8 @@
  *   - `InMemoryAdapterRegistry`: the default implementation.
  *   - `stableSourceID(adapterID, components)`: deterministic content-addressed id for adapters whose
  *       source data has no native primary key (CSV, GeoJSON).
+ *   - `HOUSE_NUMBER_PREFIX` + `splitStreetLine(line)`: the one house-number/street split every
+ *       US CSV adapter uses.
  *   - `canonicalDedupKey(row)`: normalized signature used to drop near-identical rows during a run.
  *       Adapter-internal dedup; cross-adapter dedup is the runner's job.
  *   - `streamingSha256()`: thin wrapper around `node:crypto` so the runner can hash JSONL output as it
@@ -23,8 +25,8 @@
 import { createHash, type Hash } from "node:crypto"
 
 import type { ComponentTag } from "@mailwoman/core/types"
-
-import type { CanonicalRow, CorpusAdapter } from "./types.ts"
+import { sha256Hex } from "@mailwoman/core/utils"
+import type { CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
 
 /**
  * Lookup table for corpus adapters.
@@ -103,11 +105,60 @@ export const defaultAdapterRegistry = new InMemoryAdapterRegistry()
 export function stableSourceID(adapterID: string, components: Partial<Record<ComponentTag, string>>): string {
 	const sortedKeys = Object.keys(components).toSorted() as ComponentTag[]
 	const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\u001F")
-	const digest = createHash("sha256").update(adapterID).update("\u001E").update(payload).digest("hex")
+	// One `update` over the joined string, not three chained ones — identical byte stream either way,
+	// but the separator has to stay inline or every id in every shard changes.
+	const digest = sha256Hex(`${adapterID}\u001E${payload}`)
 
 	return `${adapterID}-${digest.slice(0, 12)}`
 }
 
+/**
+ * Leading house number on a US-style street line: digits, an optional hyphenated range (Queens `40-12`), an optional
+ * single alpha suffix (`101A`), then whitespace, then the rest.
+ *
+ * This decides where the house number ends and the street begins for every US CSV adapter, so an edge case fixed here
+ * is fixed for all of them.
+ *
+ * The remainder must stay `(\S.*)` and must NOT become `(.+)`: `\s+` and `.` both match a tab, so `\s+(.+)$` lets the
+ * engine split a run of tabs between the two groups every possible way before failing — quadratic backtracking on
+ * attacker-shaped input. Requiring a non-space start removes the overlap. Group 2 is trimmed by the caller either way,
+ * so the two forms are indistinguishable on real input; only the failure cost differs.
+ */
+export const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(\S.*)$/
+
+/**
+ * A street line split into its house number and the remainder.
+ */
+export interface SplitStreetLine {
+	house_number?: string
+	street: string
+}
+
+/**
+ * Split a US-style street line on {@link HOUSE_NUMBER_PREFIX}.
+ *
+ * The US CSV sources follow USPS Publication 28 conventions with hand-entry drift. The leading digit run is the house
+ * number (`"123 Main St"`, `"6450 W Indian School Rd"`), and the regex tolerates one trailing letter (`"123A Main St"`)
+ * plus an optional hyphenated half (`"40-12 Bell Blvd"`, common in NYC and suburban garden-apartment numbering; Hawaii
+ * uses it island-wide — `"47-470 Hui Aeko Place"`).
+ *
+ * Returns `null` for blank input. Anything that does not match the prefix shape (`"PO Box 1234"`, `"RR 2 Box 67"`, `"HC
+ * 1"`) becomes a single `street` value rather than being mangled — the model sees the original surface form and
+ * downstream classifiers pick it up. Callers that need those forms recognized as something other than a street (see
+ * `usgov-irs-bmf`) test for them BEFORE calling this.
+ */
+export function splitStreetLine(line: string): SplitStreetLine | null {
+	const trimmed = line.trim()
+
+	if (!trimmed) return null
+
+	const match = HOUSE_NUMBER_PREFIX.exec(trimmed)
+
+	if (match) return { house_number: match[1], street: match[2]!.trim() }
+
+	return { street: trimmed }
+}
+
 /**
  * Canonical dedup key for a row.
  *
@@ -142,6 +193,11 @@ export interface StreamingHasher {
 /**
  * Default `StreamingHasher` (SHA-256, hex).
  */
+/**
+ * @see {@link StreamingHasher} — this is the incremental counterpart to `sha256Hex`, not a copy of
+ *   it. `@mailwoman/core/utils` exposes one-shot digests (`sha256Hex`) and whole-file digests
+ *   (`sha256File`); the runner needs neither, because it hashes JSONL lines as they stream past.
+ */
 export function streamingSha256(): StreamingHasher {
 	const h: Hash = createHash("sha256")
 	let finalized = false

package/src/adapters/wof-admin-jp/adapter.ts

@@ -31,7 +31,7 @@
 
 import { DatabaseSync } from "node:sqlite"
 
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
 
 /**
  * Ancestry hops walked upward from a WOF record before giving up — deeper than any real JP admin chain.
@@ -59,15 +59,21 @@ interface NameRow {
 }
 
 /**
- * Walk parent chain up to 6 levels.
+ * Walk parent chain up to 6 levels, over a place table already resident in memory.
+ *
+ * `resolve` is the escape hatch for a parent outside the preloaded set (a chain that leaves the country); it is a point
+ * query, so keeping the common case out of it is the whole point.
  */
-function chainOf(db: DatabaseSync, startID: number, _jpnNames: Map<number, string>): PlaceRow[] {
-	const stmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
+function chainOf(
+	byID: Map<number, PlaceRow>,
+	resolve: (id: number) => PlaceRow | undefined,
+	startID: number
+): PlaceRow[] {
 	const out: PlaceRow[] = []
 	let id = startID
 
 	for (let i = 0; i < MAX_ANCESTRY_DEPTH && id > 0; i++) {
-		const row = stmt.get(id) as PlaceRow | undefined
+		const row = byID.get(id) ?? resolve(id)
 
 		if (!row) break
 		out.push(row)
@@ -151,9 +157,29 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
 					}
 				}
 
-				const seeds = db.prepare(`SELECT id FROM spr WHERE country='JP' AND placetype='neighbourhood'`).all() as {
-					id: number
-				}[]
+				// One read of the JP place table instead of a fresh `prepare` + up to six point queries per
+				// seed; there are tens of thousands of neighbourhood seeds.
+				const byID = new Map<number, PlaceRow>()
+
+				for (const row of db
+					.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE country='JP'`)
+					.all() as unknown as PlaceRow[]) {
+					byID.set(row.id, row)
+				}
+
+				const outsideStmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
+
+				const resolveOutside = (id: number): PlaceRow | undefined => {
+					const row = outsideStmt.get(id) as unknown as PlaceRow | undefined
+
+					if (row) {
+						byID.set(id, row)
+					}
+
+					return row
+				}
+
+				const seeds = [...byID.values()].filter((row) => row.placetype === "neighbourhood")
 
 				let emitted = 0
 
@@ -162,7 +188,7 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
 
 					if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					const chain = chainOf(db, seed.id, jpnNames)
+					const chain = chainOf(byID, resolveOutside, seed.id)
 					const synth = synthesizeJpAddress(chain, jpnNames)
 
 					if (!synth) continue

package/src/adapters/wof-admin-json/adapter.ts

@@ -41,10 +41,9 @@
 
 import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
 import type { ComponentTag } from "@mailwoman/core/types"
-
-import { formatAddress, reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
-import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "../../wof-json.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "@mailwoman/corpus/utils"
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
 
 /**
  * Display name for the country, keyed by ISO 3166-1 alpha-2.

package/src/adapters/wof-postalcode-json/adapter.ts

@@ -39,10 +39,9 @@
 
 import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
 import type { ComponentTag } from "@mailwoman/core/types"
-
-import { formatAddress, reconcileComponents } from "../../format.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
-import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "../../wof-json.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
+import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "@mailwoman/corpus/utils"
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
 
 const COUNTRY_DISPLAY_NAME: Record<string, string> = {
 	US: "United States of America",

package/src/build.ts

@@ -52,22 +52,23 @@ import { mkdir, writeFile } from "node:fs/promises"
 import { join } from "node:path"
 
 import { $public } from "@mailwoman/core/env"
-import { JSONSpliterator } from "spliterator"
-
-import { defaultAdapterRegistry } from "./adapter.ts"
-import { alignRow } from "./align.ts"
-import { licenseExcluded } from "./license.ts"
-import { writeShards, type ShardManifest } from "./parquet.ts"
-import { runAdapter, type AdapterRunManifest } from "./runner.ts"
+import { parseJSONStrict } from "@mailwoman/core/objects"
+import { defaultAdapterRegistry } from "@mailwoman/corpus/adapters/utils"
+import { runAdapter, type AdapterRunManifest } from "@mailwoman/corpus/runner"
+import { defaultAugmentationsForCountry, synthesizeRow } from "@mailwoman/corpus/synthesizers/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "@mailwoman/corpus/types"
 import {
+	alignRow,
+	licenseExcluded,
+	writeShards,
 	defaultHoldouts,
 	splitForRow,
 	writeSplitManifestsFromLabeledFiles,
+	type ShardManifest,
 	type SplitManifest,
 	type SplitName,
-} from "./split.ts"
-import { defaultAugmentationsForCountry, synthesizeRow } from "./synthesize.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "./types.ts"
+} from "@mailwoman/corpus/utils"
+import { JSONSpliterator } from "spliterator"
 
 /**
  * Stage tags surfaced to `onProgress`.
@@ -188,7 +189,7 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 			existsSync(cachedManifest) &&
 			existsSync(join(adapterDir, "canonical.jsonl"))
 		) {
-			const cached = JSON.parse(readFileSync(cachedManifest, "utf8")) as AdapterRunManifest
+			const cached = parseJSONStrict<AdapterRunManifest>(readFileSync(cachedManifest, "utf8"))
 			opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`)
 			adapterRuns.push(cached)
 

package/src/index.ts

@@ -4,25 +4,11 @@
  * @author Teffen Ellis, et al.
  */
 
-export * from "./adapter.ts"
+export * from "./adapters/utils/index.ts"
 export * from "./adapters/gnaf/assemble.ts"
 export * from "./adapters/index.ts"
-export * from "./align.ts"
 export * from "./build.ts"
-export * from "./format.ts"
-export * from "./golden.ts"
-export * from "./parquet.ts"
-export * from "./parquet-wrapper/index.ts"
 export * from "./runner.ts"
+export * from "./parquet-wrapper/index.ts"
 export * from "./shard-recipes/index.ts"
-export * from "./split.ts"
-export * from "./synthesize-anchor-absorption.ts"
-export * from "./synthesize-german.ts"
-export * from "./synthesize-house-venue.ts"
-export * from "./synthesize-intersection.ts"
-export * from "./synthesize-no-street.ts"
-export * from "./synthesize-po-box.ts"
-export * from "./synthesize-street.ts"
-export * from "./synthesize.ts"
-export * from "./tokenize.ts"
-export * from "./types.ts"
+export * from "@mailwoman/corpus/types"

package/src/name-prone-us-suffixes.ts

@@ -0,0 +1,12 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Re-export shim: the name-prone set moved to `@mailwoman/codex/us` (data in
+ * `codex/us/street-suffix.json`, 2026-08-10) so the golden relabel flags, the #1569 shard
+ * recipe, AND the Python relabel pass (via the `gazetteer affix-relabel` v2 lexicon) all read
+ * one record. Import from `@mailwoman/codex/us` in new code.
+ */
+
+export { NAME_PRONE_US_SUFFIXES } from "@mailwoman/codex/us"

package/src/parquet-wrapper/reader.ts

@@ -59,6 +59,25 @@ export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReade
 		return super[Symbol.asyncIterator]() as AsyncGenerator<T, void, unknown>
 	}
 
+	/**
+	 * Iterate a subset of the columns, narrowed to the keys asked for.
+	 *
+	 * Parquet is columnar, so a projection is read avoided rather than read-then-discarded: on a 14-column corpus shard,
+	 * two columns cost 93 ms against 253 ms for the whole row. The default iterator reads every column, which is what a
+	 * caller wanting the whole record should use.
+	 */
+	public async *project<K extends keyof T>(...columns: K[]): AsyncGenerator<Pick<T, K>, void, unknown> {
+		const cursor = this.getCursor(columns.map((column) => [column as string]))
+
+		for (;;) {
+			const row = (await cursor.next()) as Pick<T, K> | null
+
+			if (!row) return
+
+			yield row
+		}
+	}
+
 	public async [Symbol.asyncDispose]() {
 		return this.close()
 	}

package/src/parquet-wrapper/schema.ts

@@ -24,6 +24,10 @@ import { LRUCache } from "lru-cache"
 /**
  * A Parquet record-like object, i.e. a record with string keys and JSON-serializable values.
  */
+/**
+ * Shape a row type must satisfy. Declare yours as a `type` alias — TypeScript gives an alias an implicit index
+ * signature and an `interface` none, so an otherwise-identical interface does not satisfy this.
+ */
 export type ParquetRecordLike = Record<string, unknown | undefined>
 
 /**

package/src/runner.ts

@@ -31,8 +31,13 @@ import { createWriteStream, type WriteStream } from "node:fs"
 import { mkdir, writeFile } from "node:fs/promises"
 import { dirname, join } from "node:path"
 
-import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "./adapter.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "./types.ts"
+import {
+	canonicalDedupKey,
+	streamingSha256,
+	type AdapterRegistry,
+	type StreamingHasher,
+} from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
 
 /**
  * Snapshot of the runner's state, emitted on every progress tick.

package/src/shard-recipes/anchor-absorption.ts

@@ -14,11 +14,12 @@
 
 import { readFileSync } from "node:fs"
 
+import { parseJSONStrict } from "@mailwoman/core/objects"
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { synthesizeAnchorAbsorptionRow } from "@mailwoman/corpus/synthesizers/anchor-absorption"
+import { alignRow } from "@mailwoman/corpus/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.ts"
 import { makeLcg, type ShardRecipe } from "./scaffold.ts"
 
 /**
@@ -31,7 +32,7 @@ const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json")
  * The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list).
  */
 function loadRealUsZips(path: string): string[] {
-	const d = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>
+	const d = parseJSONStrict<Record<string, unknown>>(readFileSync(path, "utf8"))
 	const zips: string[] = []
 
 	for (const [pc, v] of Object.entries(d)) {

package/src/shard-recipes/boundary-stress.ts

@@ -12,8 +12,12 @@
  *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
  */
 
-import { alignRow } from "../align.ts"
-import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "../synthesize-boundary-stress.ts"
+import {
+	type BoundaryStressTemplate,
+	synthesizeBoundaryStressRow,
+} from "@mailwoman/corpus/synthesizers/boundary-stress"
+import { alignRow } from "@mailwoman/corpus/utils"
+
 import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
 /**