@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts

@@ -0,0 +1,182 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `usgov-samhsa-treatment-locator`: SAMHSA Behavioral Health Treatment Services Locator CSV
+ *   consumer.
+ *
+ *   SAMHSA's Treatment Locator (`findtreatment.gov`) is the federal directory of substance-use and
+ *   mental-health treatment facilities. The published CSV carries the facility name, an optional
+ *   secondary name (typically the organizational parent), and the postal address quad split into
+ *   primary + secondary street lines. Phase 1.6 §1.2 (#22) selects this source for the same reason
+ *   it selects HRSA: facility names are hand-typed venue strings and the addresses pass through
+ *   enough human + system hands to accumulate the suite-designator + sub-tenant chaos ("Suite C,
+ *   behind main building") that pure gazetteer data does not.
+ *
+ *   SAMHSA's two-line address shape is the key adapter-specific concern. `street1` typically carries
+ *   the canonical postal address (`"123 Main St"`); `street2` carries the suite / unit / "second
+ *   floor" surface form. The adapter joins them with `", "` into a single `street` component (Phase
+ *   1 keeps `unit` as a deferred slot since the OpenCage template doesn't have a clean rendering
+ *   for it). Operators wanting a different join policy can subclass the factory.
+ *
+ *   Column names below match the canonical SAMHSA Behavioral Health Treatment Services Locator CSV
+ *   export header. Operators substituting a closely-related extract should rename columns to match;
+ *   the README has the mapping cheatsheet.
+ *
+ *   License: stamped `"Public Domain"` per the SAMHSA Open Data Foundry's federal-government
+ *   distribution terms.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address-format"
+import { CSVSpliterator } from "spliterator"
+
+import { splitStreetLine, stableSourceID } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { lookupStateAbbreviation } from "#us/fips-state"
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const USGOV_SAMHSA_ADAPTER_ID = "usgov-samhsa-treatment-locator"
+/**
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
+ * than having to look them up.
+ */
+export const USGOV_SAMHSA_DEFAULT_LICENSE = "Public Domain"
+
+/**
+ * Subset of SAMHSA Treatment Locator CSV columns consulted by the adapter. Column names match the canonical SAMHSA Open
+ * Data Foundry export header. `name1` is the venue; `name2` is optional and folded into the venue when present.
+ */
+interface SamhsaSiteRow {
+	name1: string
+	name2?: string
+	street1: string
+	street2?: string
+	city: string
+	state: string
+	zip: string
+	/**
+	 * Optional. Falls back to `stableSourceID` derived from components when missing.
+	 */
+	frid?: string
+}
+
+/**
+ * Join the SAMHSA two-line street: primary street + optional secondary line (suite / unit / floor / "behind main
+ * building") on `", "`. The combined value is the `street` component surface form. Phase 1 does not break this out into
+ * the `unit` component — see the file-level comment.
+ */
+function joinTwoLineStreet(street1: string, street2: string | undefined): string {
+	const s1 = street1.trim()
+	const s2 = (street2 ?? "").trim()
+
+	if (!s1 && !s2) return ""
+
+	if (!s2) return s1
+
+	if (!s1) return s2
+
+	return `${s1}, ${s2}`
+}
+
+/**
+ * Combine `name1` + optional `name2` into a single venue surface form. SAMHSA conventions:
+ *
+ * - `name1` is the program / clinic name ("Mountain Plains Counseling Services").
+ * - `name2` is the parent organization ("Catholic Charities of Wyoming"), if any.
+ *
+ * Both render together as `"<name1> - <name2>"` when both are present — geocoder users typically type either form, so
+ * the model benefits from the joined surface.
+ */
+function composeVenue(name1: string, name2: string | undefined): string {
+	const n1 = name1.trim()
+	const n2 = (name2 ?? "").trim()
+
+	if (!n1 && !n2) return ""
+
+	if (!n2) return n1
+
+	if (!n1) return n2
+
+	return `${n1} - ${n2}`
+}
+
+export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
+	return {
+		id: USGOV_SAMHSA_ADAPTER_ID,
+		defaultLicense: USGOV_SAMHSA_DEFAULT_LICENSE,
+		description:
+			"SAMHSA Behavioral Health Treatment Services Locator (public-domain). Adversarial source: venue + two-line address co-occurrence, hand-entered.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "US") {
+				throw new Error(`usgov-samhsa adapter: only US supported, got country=${opts.country}`)
+			}
+
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				normalizeKeys: false,
+			})
+
+			let emitted = 0
+
+			for await (const record of rows as AsyncIterable<SamhsaSiteRow>) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const venue = composeVenue(record.name1 ?? "", record.name2)
+				const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
+				const split = splitStreetLine(street)
+				const city = (record.city ?? "").trim()
+				const stateAbbr = (record.state ?? "").trim()
+				const postcode = (record.zip ?? "").trim()
+
+				if (!venue || !split || !city || !postcode) continue
+				const state = lookupStateAbbreviation(stateAbbr)
+
+				if (!state) continue
+
+				// venue first — same kryptonite-defending insertion order as HRSA.
+				const components: CanonicalRow["components"] = {
+					venue,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
+
+				const rendered = formatAddressRow(components, "US", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				const frID = (record.frid ?? "").trim()
+
+				const sourceID = frID ? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}` : stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_SAMHSA_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_SAMHSA_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const usgovSamhsaTreatmentLocatorAdapter = createUsgovSamhsaTreatmentLocatorAdapter()

package/lib/us/tools/fetch/hrsa.ts

@@ -0,0 +1,59 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
+ *   adapter. US Public Domain.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
+ *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
+ *   downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
+ */
+
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download/index"
+import { downloadToFile, writeManifest } from "#tools/fetch/download/index"
+
+const SLUG = "usgov-hrsa-fqhc"
+const FILENAME = "Health_Center_Service_Delivery_and_LookAlike_Sites.csv"
+const SOURCE_URL = `https://data.hrsa.gov/DataDownload/DD_Files/${FILENAME}`
+
+export type FetchHRSAOptions = BaseFetchOptions
+
+export async function fetchHRSA(options: FetchHRSAOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+	const dest = join(destDir, FILENAME)
+
+	report?.(`=== ${SLUG} / ${FILENAME}`)
+
+	const { bytes } = await downloadToFile({
+		url: SOURCE_URL,
+		dest,
+		timeoutMs: 600_000,
+		headers: { "Accept-Encoding": "gzip, br" },
+		report,
+	})
+
+	const sha = await sha256File(dest)
+
+	const manifest: SourceManifest = {
+		source_url: SOURCE_URL,
+		downloaded_at: new Date().toISOString(),
+		filename: FILENAME,
+		sha256: sha,
+		bytes,
+	}
+
+	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+	report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}

package/lib/us/tools/fetch/imls-pls.ts

@@ -0,0 +1,149 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
+ *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
+ *   Public Domain (federal statistical survey).
+ *
+ *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
+ *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
+ *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
+ *   no per-branch address detail.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
+ *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
+ *   member listing + selective extraction).
+ *
+ *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
+ *   and sha matches MANIFEST, skips download.
+ */
+
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/hash"
+import { basename, join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download/index"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download/index"
+
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+
+/**
+ * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
+ */
+const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip"
+const SLUG = "usgov-imls-pls"
+
+export type FetchIMLSPLSOptions = BaseFetchOptions
+
+export async function fetchIMLSPLS(
+	options: FetchIMLSPLSOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+
+	const zipDest = join(destDir, basename(ZIP_URL))
+	const manifestPath = join(destDir, "MANIFEST.json")
+
+	report?.(`=== ${SLUG}`)
+
+	// MARK: Idempotency check
+
+	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
+
+	if (recorded?.sha256 && recorded.filename) {
+		const recordedPath = join(destDir, recorded.filename)
+
+		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
+			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
+
+			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+		}
+	}
+
+	// MARK: Download ZIP
+
+	report?.(`  Downloading ${ZIP_URL} ...`)
+
+	const { bytes: zipSize } = await downloadToFile({
+		url: ZIP_URL,
+		dest: zipDest,
+		timeoutMs: 600_000,
+		headers: { "Accept-Encoding": "gzip, br" },
+		report,
+	})
+
+	report?.(`  Downloaded: ${ByteFormatter.formatIEC(zipSize)}`)
+
+	if (zipSize < BYTES_PER_KIB) {
+		report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	// Discover the outlet-level CSV inside the ZIP.
+	// Outlet files match: pls_fy*outlet*.csv (case-insensitive)
+	// Administrative-entity files match: pls_fy*ae*.csv — we skip those.
+	report?.("  Inspecting ZIP contents ...")
+	const entries = (await listZipEntries(zipDest)).map((entry) => entry.name)
+
+	let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name))
+
+	// Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
+	if (!csvName) {
+		csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name))
+	}
+
+	if (!csvName) {
+		report?.("  Available files in ZIP:")
+
+		for (const name of entries) {
+			report?.(`    ${name}`)
+		}
+
+		report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module")
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	report?.(`  Extracting outlet CSV: ${csvName}`)
+
+	const csvDest = join(destDir, basename(csvName))
+
+	await extractZipEntry(zipDest, csvName, csvDest)
+	const csvSize = (await statPath(csvDest)).size
+	const csvSha = await sha256File(csvDest)
+
+	// MARK: Remove ZIP (small, but keep destDir clean)
+
+	await removePathIfPresent(zipDest)
+	report?.("  Removed ZIP (CSV kept)")
+
+	// MARK: Write MANIFEST
+
+	const manifest: SourceManifest = {
+		source_url: ZIP_URL,
+		downloaded_at: new Date().toISOString(),
+		filename: basename(csvName),
+		sha256: csvSha,
+		bytes: csvSize,
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}