@mailwoman/tiger 1.0.0 → 4.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/tiger",
-  "version": "1.0.0",
+  "version": "4.13.0",
   "description": "US Census TIGER/Line data processing and analysis.",
   "license": "AGPL-3.0",
   "contributors": [
@@ -9,6 +9,15 @@
       "email": "teffen@sister.software"
     }
   ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/sister-software/mailwoman.git",
+    "directory": "tiger"
+  },
+  "bugs": {
+    "url": "https://github.com/sister-software/mailwoman/issues"
+  },
+  "homepage": "https://github.com/sister-software/mailwoman",
   "scripts": {
     "test": "node ./out/sdk/TIGERService.test.js"
   },
@@ -21,9 +30,10 @@
     ".": "./out/index.js"
   },
   "dependencies": {
-    "@mailwoman/core": "4.11.0",
-    "@mailwoman/spatial": "4.11.0",
+    "@mailwoman/core": "4.13.0",
+    "@mailwoman/spatial": "4.13.0",
     "@turf/boolean-contains": "^7.3.5",
+    "kysely": "^0.29.2",
     "shapefile-parser": "^1.0.3",
     "type-fest": "^5.7.0"
   },

package/sdk/fetch.ts

@@ -0,0 +1,323 @@
+/**
+ * @copyright Sister Software.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pull TIGER geometry/attributes into a `node:sqlite` database via the Kysely
+ *   {@link DatabaseClient}.
+ *
+ *   Replaces the ad-hoc wget and the retired corpus TIGER build script. Pared down from isp-nexus's
+ *   `generate-tiger-tiles.ts` (its per-level column mapping is the spec) into the playpen
+ *   `repo-tools` idiom: an async generator of progress events, idempotent, no SpatiaLite.
+ *
+ *   Three levels:
+ *
+ *   - `tabblock20` (per state) — tabulation blocks → `tabblock20`, geometry as GeoJSON text.
+ *   - `place` (per state) — incorporated/census places → `tiger_places` (attribute-only).
+ *   - `addrfeat` (per county) — named street segments + ZIPs → `tiger_streets` (attribute-only).
+ *
+ *   `tiger_streets` + `tiger_places` match the schema the corpus `tiger` adapter reads, so this is a
+ *   drop-in replacement for the retired corpus TIGER build script.
+ *
+ *   Flow per source unit: download (skips a valid cached zip) → unzip → stream `ogr2ogr -f
+ *   GeoJSONSeq` (mapping shapefile columns to the schema, WGS84 for geometry levels) → batched
+ *   inserts. Re-running a state replaces its rows.
+ */
+
+import { DatabaseClient } from "@mailwoman/core/kysley/client"
+import { spawn } from "node:child_process"
+import { createWriteStream, existsSync } from "node:fs"
+import { mkdir, rename } from "node:fs/promises"
+import { dirname, join } from "node:path"
+import { createInterface } from "node:readline"
+import { DatabaseSync } from "node:sqlite"
+import { Readable } from "node:stream"
+import { pipeline } from "node:stream/promises"
+import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "./schema.js"
+import { initializeTIGERSchema, TIGER_PRAGMAS } from "./schema.js"
+
+const CENSUS_HOST = "https://www2.census.gov"
+const DEFAULT_DATA_ROOT = process.env.MAILWOMAN_DATA_ROOT ?? "/mnt/playpen/mailwoman-data"
+
+/**
+ * Supported TIGER levels. `tabblock20` is per state + carries geometry; `place`/`addrfeat` are
+ * attribute-only.
+ */
+export type TIGERFetchLevel = "tabblock20" | "place" | "addrfeat"
+
+const LEVEL_DIR: Record<TIGERFetchLevel, string> = {
+	tabblock20: "TABBLOCK20",
+	place: "PLACE",
+	addrfeat: "ADDRFEAT",
+}
+const LEVEL_TABLE: Record<TIGERFetchLevel, keyof TIGERDatabase> = {
+	tabblock20: "tabblock20",
+	place: "tiger_places",
+	addrfeat: "tiger_streets",
+}
+
+export interface FetchTIGEROptions {
+	/** Two-digit state FIPS, e.g. `"06"`. */
+	stateFIPS: string
+	/** TIGER level. Default `tabblock20`. */
+	level?: TIGERFetchLevel
+	/** Vintage. Default 2020 for blocks (matches the 2020 P.L.), 2024 for place/addrfeat (current). */
+	vintage?: number
+	/**
+	 * Output SQLite path. Default `<dataRoot>/tiger/tiger.db` (the name the corpus `tiger` adapter
+	 * reads).
+	 */
+	outPath?: string
+	/** Download cache + default output root. */
+	dataRoot?: string
+	/** Optional three-digit county FIPS filter (blocks only — addrfeat is already per-county). */
+	county?: string
+	/** Rows per insert. Default 1000. */
+	batchSize?: number
+}
+
+export type FetchTIGEREvent =
+	| { phase: "download"; file: string; cached: boolean }
+	| { phase: "extract"; file: string }
+	| { phase: "load"; inserted: number; total: number }
+
+export interface FetchTIGERResult {
+	outPath: string
+	table: string
+	inserted: number
+}
+
+/** The isp-nexus column map for `tabblock20`. Geometry rides along implicitly. */
+function blockSelectSQL(layer: string, county?: string): string {
+	const where = county ? ` WHERE COUNTYFP20 = '${county}'` : ""
+	return (
+		`SELECT GEOID20 AS GEOID, STATEFP20 AS state_code, COUNTYFP20 AS county_code, ` +
+		`SUBSTR(GEOID20, 6, 6) AS tract_code, ` +
+		`SUBSTR(GEOID20, 12, 1) AS block_group_code, BLOCKCE20 AS block_code, ` +
+		`UACE20 AS urbanized_area_code, UR20 AS urban_rural_code, ` +
+		`HOUSING20 AS housing_unit_count, ALAND20 AS land_area_sqm, AWATER20 AS water_area_sqm, ` +
+		`POP20 AS population FROM "${layer}"${where}`
+	)
+}
+
+function selectSQL(level: TIGERFetchLevel, layer: string, county?: string): string {
+	switch (level) {
+		case "tabblock20":
+			return blockSelectSQL(layer, county)
+		case "place":
+			return `SELECT GEOID AS geoid, NAME AS name, STATEFP AS statefp, LSAD AS lsad, NAMELSAD AS namelsad, CLASSFP AS classfp FROM "${layer}"`
+		case "addrfeat":
+			// ADDRFEAT has no STATEFP column — injected per-row from the state we're fetching.
+			return `SELECT LINEARID AS linearid, FULLNAME AS fullname, ZIPL AS zipl, ZIPR AS zipr FROM "${layer}" WHERE FULLNAME IS NOT NULL AND FULLNAME != ''`
+	}
+}
+
+type Row = TIGERBlockTable | TIGERPlaceTable | TIGERStreetTable
+
+function buildRow(level: TIGERFetchLevel, p: Record<string, unknown>, geometry: unknown, state: string): Row {
+	switch (level) {
+		case "place":
+			return {
+				geoid: String(p.geoid),
+				name: String(p.name ?? ""),
+				statefp: String(p.statefp ?? state),
+				lsad: (p.lsad as string) || null,
+				namelsad: (p.namelsad as string) || null,
+				classfp: (p.classfp as string) || null,
+			}
+		case "addrfeat":
+			return {
+				linearid: String(p.linearid),
+				fullname: String(p.fullname ?? ""),
+				zipl: (p.zipl as string) || null,
+				zipr: (p.zipr as string) || null,
+				statefp: state,
+			}
+		case "tabblock20":
+			return {
+				GEOID: String(p.GEOID),
+				state_code: String(p.state_code),
+				county_code: String(p.county_code),
+				tract_code: String(p.tract_code ?? ""),
+				block_group_code: String(p.block_group_code ?? ""),
+				block_code: String(p.block_code ?? ""),
+				urbanized_area_code: (p.urbanized_area_code as string) || null,
+				urban_rural_code: (p.urban_rural_code as string) || null,
+				housing_unit_count: Number(p.housing_unit_count ?? 0),
+				land_area_sqm: Number(p.land_area_sqm ?? 0),
+				water_area_sqm: Number(p.water_area_sqm ?? 0),
+				population: Number(p.population ?? 0),
+				geometry: JSON.stringify(geometry),
+			}
+	}
+}
+
+function runCapture(cmd: string, args: string[]): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const child = spawn(cmd, args)
+		let out = ""
+		let err = ""
+		child.stdout.on("data", (d) => (out += d))
+		child.stderr.on("data", (d) => (err += d))
+		child.on("error", reject)
+		child.on("close", (code) =>
+			code === 0 ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.slice(0, 500)}`))
+		)
+	})
+}
+
+async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
+	if (existsSync(dest)) {
+		try {
+			await runCapture("unzip", ["-tq", dest])
+			return true
+		} catch {
+			// corrupt cache — re-download
+		}
+	}
+	const tmp = dest + ".tmp"
+	const res = await fetch(url, { redirect: "follow" })
+	if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
+	await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), createWriteStream(tmp))
+	await rename(tmp, dest)
+	return false
+}
+
+/** Scrape the ADDRFEAT directory listing for a state's county FIPS codes. */
+async function discoverCounties(state: string, vintage: number): Promise<string[]> {
+	const res = await fetch(`${CENSUS_HOST}/geo/tiger/TIGER${vintage}/ADDRFEAT/`, { redirect: "follow" })
+	if (!res.ok) throw new Error(`HTTP ${res.status} listing ADDRFEAT for vintage ${vintage}`)
+	const html = await res.text()
+	const re = new RegExp(`tl_${vintage}_${state}(\\d{3})_addrfeat\\.zip`, "g")
+	const counties = new Set<string>()
+	for (let m = re.exec(html); m; m = re.exec(html)) counties.add(m[1]!)
+	return [...counties].sort()
+}
+
+/**
+ * Fetch one state's TIGER data at `level` into a SQLite DB. Yields progress; returns the final
+ * tally.
+ */
+export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<FetchTIGEREvent, FetchTIGERResult> {
+	const level = options.level ?? "tabblock20"
+	const vintage = options.vintage ?? (level === "tabblock20" ? 2020 : 2024)
+	const dataRoot = options.dataRoot ?? DEFAULT_DATA_ROOT
+	const batchSize = options.batchSize ?? 1000
+	const state = options.stateFIPS
+	const table = LEVEL_TABLE[level]
+
+	const cacheDir = join(dataRoot, "tiger", String(vintage), state)
+	// Default to a stable, vintage-agnostic `tiger.db` — the filename the corpus `tiger` adapter reads
+	// (run-corpus-build → `${ROOT}/tiger/tiger.db`). The vintage is a content detail, not a path one;
+	// the per-table idempotent delete keeps a re-fetch (newer vintage) clean. The download CACHE stays
+	// vintage-partitioned below so zips don't collide across vintages.
+	const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
+	await mkdir(cacheDir, { recursive: true })
+	await mkdir(dirname(outPath), { recursive: true })
+
+	// Source units: one (per-state) for block/place; one per county for addrfeat.
+	const geoCodes = level === "addrfeat" ? await discoverCounties(state, vintage) : [""]
+	if (level === "addrfeat" && geoCodes.length === 0) {
+		throw new Error(`No ADDRFEAT counties found for state ${state} vintage ${vintage}`)
+	}
+
+	const db = new DatabaseSync(outPath)
+	db.exec(TIGER_PRAGMAS)
+	const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
+	await initializeTIGERSchema(kdb)
+
+	const insertBatch = async (rows: Row[]): Promise<void> => {
+		if (level === "tabblock20")
+			await kdb
+				.insertInto("tabblock20")
+				.values(rows as TIGERBlockTable[])
+				.execute()
+		else if (level === "place")
+			await kdb
+				.insertInto("tiger_places")
+				.values(rows as TIGERPlaceTable[])
+				.execute()
+		else
+			await kdb
+				.insertInto("tiger_streets")
+				.values(rows as TIGERStreetTable[])
+				.execute()
+	}
+
+	try {
+		// Idempotent re-run: drop this state's rows first (state_code for blocks, statefp otherwise).
+		if (level === "tabblock20") await kdb.deleteFrom("tabblock20").where("state_code", "=", state).execute()
+		else if (level === "place") await kdb.deleteFrom("tiger_places").where("statefp", "=", state).execute()
+		else await kdb.deleteFrom("tiger_streets").where("statefp", "=", state).execute()
+
+		let inserted = 0
+		let batch: Row[] = []
+		const flush = async () => {
+			if (!batch.length) return
+			const rows = batch
+			batch = []
+			await insertBatch(rows)
+			inserted += rows.length
+		}
+
+		for (const geo of geoCodes) {
+			const unit = level === "addrfeat" ? state + geo : state
+			const zipName = `tl_${vintage}_${unit}_${level}.zip`
+			const zipPath = join(cacheDir, zipName)
+			const url = `${CENSUS_HOST}/geo/tiger/TIGER${vintage}/${LEVEL_DIR[level]}/${zipName}`
+
+			const cached = await downloadIfNeeded(url, zipPath)
+			yield { phase: "download", file: zipName, cached }
+
+			await runCapture("unzip", ["-o", "-q", zipPath, "-d", cacheDir])
+			const layer = `tl_${vintage}_${unit}_${level}`
+			const shpPath = join(cacheDir, layer + ".shp")
+			yield { phase: "extract", file: layer + ".shp" }
+
+			const child = spawn(
+				"ogr2ogr",
+				[
+					"-f",
+					"GeoJSONSeq",
+					"-t_srs",
+					"EPSG:4326",
+					"-sql",
+					selectSQL(level, layer, options.county),
+					"/vsistdout/",
+					shpPath,
+				],
+				{ stdio: ["ignore", "pipe", "pipe"] }
+			)
+			let stderr = ""
+			child.stderr.on("data", (d) => (stderr += d))
+			const exited = new Promise<number>((resolve) => child.on("close", (code) => resolve(code ?? 0)))
+
+			for await (const line of createInterface({ input: child.stdout, crlfDelay: Infinity })) {
+				if (!line) continue
+				let feat: { properties?: Record<string, unknown>; geometry?: unknown }
+				try {
+					feat = JSON.parse(line)
+				} catch {
+					continue
+				}
+				if (!feat.properties) continue
+				if (level === "tabblock20" && !feat.geometry) continue
+				batch.push(buildRow(level, feat.properties, feat.geometry, state))
+				if (batch.length >= batchSize) {
+					await flush()
+					yield { phase: "load", inserted, total: 0 }
+				}
+			}
+			await flush()
+
+			const code = await exited
+			if (code !== 0) throw new Error(`ogr2ogr exited ${code} on ${layer}: ${stderr.slice(0, 500)}`)
+			yield { phase: "load", inserted, total: 0 }
+		}
+
+		db.exec("PRAGMA wal_checkpoint(TRUNCATE);")
+		return { outPath, table, inserted }
+	} finally {
+		await kdb.destroy()
+	}
+}

package/sdk/index.ts

@@ -4,4 +4,7 @@
  * @author Teffen Ellis, et al.
  */
 
+export * from "./fetch.js"
+export * from "./redistricting.js"
+export * from "./schema.js"
 export * from "./state/index.js"

package/sdk/redistricting.ts

@@ -0,0 +1,223 @@
+/**
+ * @copyright Sister Software.
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pull a state's Census 2020 P.L. 94-171 redistricting counts (table P2 — Hispanic-or-Latino by
+ *   race, per block) into the `pl_block` table of a TIGER {@link DatabaseClient} DB, keyed on the
+ *   same 15-char block `GEOID` as {@link fetchTIGER}'s `tabblock20`. Join the two for block-level
+ *   race + geometry (e.g. a dot-density map).
+ *
+ *   Keyless public data. The per-state ZIP holds a pipe-delimited geographic header
+ *   (`<st>geo<yr>.pl`) and three data segments; segment 1 (`<st>00001<yr>.pl`) carries P1 + P2. We
+ *   join the header (filtered to SUMLEV 750 = block) to segment 1 by LOGRECNO. Field offsets are
+ *   fixed by the 2020 P.L. layout (verified against the real files).
+ *
+ *   https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
+ */
+
+import { DatabaseClient } from "@mailwoman/core/kysley/client"
+import { spawn } from "node:child_process"
+import { createReadStream, createWriteStream, existsSync } from "node:fs"
+import { mkdir, rename } from "node:fs/promises"
+import { dirname, join } from "node:path"
+import { createInterface } from "node:readline"
+import { DatabaseSync } from "node:sqlite"
+import { Readable } from "node:stream"
+import { pipeline } from "node:stream/promises"
+import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "../state.js"
+import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "./schema.js"
+
+const REDISTRICTING_BASE =
+	"https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171"
+const DEFAULT_DATA_ROOT = process.env.MAILWOMAN_DATA_ROOT ?? "/mnt/playpen/mailwoman-data"
+
+// P.L. 94-171 (2020) pipe-delimited field offsets (0-based).
+// Geographic header: …|SUMLEV(2)|…|LOGRECNO(7)|GEOID(8)|GEOCODE(9)|… — GEOCODE is the bare 15-char
+// block FIPS (matches TIGER GEOID20); SUMLEV 750 = tabulation block.
+const GEO_SUMLEV = 2
+const GEO_LOGRECNO = 7
+const GEO_GEOCODE = 9
+// Segment 1: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P1×71 | P2×73. P0020001 is at index 76.
+const SEG_LOGRECNO = 4
+const P2 = (fieldNo: number) => 76 + (fieldNo - 1)
+// The eight P2 categories that partition the total (P0020001), in `pl_block` column order.
+const CATEGORY_INDEX = {
+	pop_total: P2(1),
+	hispanic: P2(2), // Hispanic or Latino (any race)
+	white: P2(5), // Not Hispanic: White alone
+	black: P2(6),
+	aian: P2(7),
+	asian: P2(8),
+	nhpi: P2(9),
+	other: P2(10),
+	multi: P2(11), // Two or more races
+} as const
+
+export interface FetchRedistrictingOptions {
+	/** Two-digit state FIPS, e.g. `"06"`. */
+	stateFIPS: string
+	/** Decennial vintage. Default 2020 (the only P.L. 94-171 release this parses). */
+	vintage?: number
+	/** Output SQLite path. Default `<dataRoot>/tiger/tiger.db` (same DB as `fetchTIGER`). */
+	outPath?: string
+	/** Download cache + default output root. */
+	dataRoot?: string
+	/** Optional three-digit county FIPS filter, e.g. `"059"`. */
+	county?: string
+	/** Rows per insert. Default 2000. */
+	batchSize?: number
+}
+
+export type FetchRedistrictingEvent =
+	| { phase: "download"; file: string; cached: boolean }
+	| { phase: "extract"; file: string }
+	| { phase: "header"; blocks: number }
+	| { phase: "load"; inserted: number; total: number }
+
+export interface FetchRedistrictingResult {
+	outPath: string
+	table: string
+	inserted: number
+}
+
+function runCapture(cmd: string, args: string[]): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const child = spawn(cmd, args)
+		let out = ""
+		let err = ""
+		child.stdout.on("data", (d) => (out += d))
+		child.stderr.on("data", (d) => (err += d))
+		child.on("error", reject)
+		child.on("close", (code) =>
+			code === 0 ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.slice(0, 500)}`))
+		)
+	})
+}
+
+async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
+	if (existsSync(dest)) {
+		try {
+			await runCapture("unzip", ["-tq", dest])
+			return true
+		} catch {
+			// corrupt cache — re-download
+		}
+	}
+	const tmp = dest + ".tmp"
+	const res = await fetch(url, { redirect: "follow" })
+	if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
+	await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), createWriteStream(tmp))
+	await rename(tmp, dest)
+	return false
+}
+
+async function eachLine(path: string, fn: (line: string) => void): Promise<void> {
+	for await (const line of createInterface({ input: createReadStream(path), crlfDelay: Infinity })) {
+		if (line) fn(line)
+	}
+}
+
+/**
+ * Fetch one state's P.L. 94-171 block race counts into `pl_block`. Yields progress; returns the
+ * tally.
+ */
+export async function* fetchRedistricting(
+	options: FetchRedistrictingOptions
+): AsyncGenerator<FetchRedistrictingEvent, FetchRedistrictingResult> {
+	const vintage = options.vintage ?? 2020
+	const dataRoot = options.dataRoot ?? DEFAULT_DATA_ROOT
+	const batchSize = options.batchSize ?? 2000
+	const state = options.stateFIPS
+
+	const abbr = AdminLevel1CodeToAbbreviation[state as AdminLevel1Code]
+	if (!abbr) throw new Error(`Unknown state FIPS "${state}"`)
+	const stateName = StateName[abbr as keyof typeof StateName]
+	const dirName = stateName.replace(/ /g, "_")
+	const fileAbbr = abbr.toLowerCase()
+
+	const cacheDir = join(dataRoot, "census", "redistricting", String(vintage), state)
+	// Same stable `tiger.db` default as fetchTIGER — pl_block lives alongside tabblock20 in one DB.
+	const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
+	await mkdir(cacheDir, { recursive: true })
+	await mkdir(dirname(outPath), { recursive: true })
+
+	const zipName = `${fileAbbr}${vintage}.pl.zip`
+	const zipPath = join(cacheDir, zipName)
+	const url = `${REDISTRICTING_BASE}/${dirName}/${zipName}`
+
+	const cached = await downloadIfNeeded(url, zipPath)
+	yield { phase: "download", file: zipName, cached }
+
+	await runCapture("unzip", ["-o", "-q", zipPath, "-d", cacheDir])
+	const geoPath = join(cacheDir, `${fileAbbr}geo${vintage}.pl`)
+	const seg1Path = join(cacheDir, `${fileAbbr}00001${vintage}.pl`)
+	yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` }
+
+	// Pass 1: header → LOGRECNO → GEOID for the blocks we want.
+	const prefix = options.county ? state + options.county : state
+	const logToGeoid = new Map<string, string>()
+	await eachLine(geoPath, (line) => {
+		const f = line.split("|")
+		if (f[GEO_SUMLEV] !== "750") return
+		const geoid = f[GEO_GEOCODE] ?? ""
+		if (!geoid.startsWith(prefix)) return
+		logToGeoid.set(f[GEO_LOGRECNO] ?? "", geoid)
+	})
+	const total = logToGeoid.size
+	yield { phase: "header", blocks: total }
+
+	const db = new DatabaseSync(outPath)
+	db.exec(TIGER_PRAGMAS)
+	const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
+	await initializeTIGERSchema(kdb)
+
+	try {
+		// Idempotent re-run: drop the rows we're about to (re)load.
+		await kdb
+			.deleteFrom("pl_block")
+			.where("GEOID", "like", prefix + "%")
+			.execute()
+
+		let inserted = 0
+		let batch: PLBlockTable[] = []
+		const flush = async () => {
+			if (!batch.length) return
+			const rows = batch
+			batch = []
+			await kdb.insertInto("pl_block").values(rows).execute()
+			inserted += rows.length
+		}
+
+		// Pass 2: segment 1 → P2 counts for the mapped LOGRECNOs, flushing as we go.
+		for await (const line of createInterface({ input: createReadStream(seg1Path), crlfDelay: Infinity })) {
+			if (!line) continue
+			const f = line.split("|")
+			const geoid = logToGeoid.get(f[SEG_LOGRECNO] ?? "")
+			if (!geoid) continue
+			batch.push({
+				GEOID: geoid,
+				pop_total: Number(f[CATEGORY_INDEX.pop_total] ?? 0),
+				hispanic: Number(f[CATEGORY_INDEX.hispanic] ?? 0),
+				white: Number(f[CATEGORY_INDEX.white] ?? 0),
+				black: Number(f[CATEGORY_INDEX.black] ?? 0),
+				aian: Number(f[CATEGORY_INDEX.aian] ?? 0),
+				asian: Number(f[CATEGORY_INDEX.asian] ?? 0),
+				nhpi: Number(f[CATEGORY_INDEX.nhpi] ?? 0),
+				other: Number(f[CATEGORY_INDEX.other] ?? 0),
+				multi: Number(f[CATEGORY_INDEX.multi] ?? 0),
+			})
+			if (batch.length >= batchSize) {
+				await flush()
+				yield { phase: "load", inserted, total }
+			}
+		}
+		await flush()
+
+		yield { phase: "load", inserted, total }
+		db.exec("PRAGMA wal_checkpoint(TRUNCATE);")
+		return { outPath, table: "pl_block", inserted }
+	} finally {
+		await kdb.destroy()
+	}
+}