@mailwoman/tiger 5.6.0 → 5.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/tiger",
-  "version": "5.6.0",
+  "version": "5.7.0",
   "description": "US Census TIGER/Line data processing and analysis.",
   "keywords": [
     "census",
@@ -38,13 +38,17 @@
     "test": "node ./out/sdk/TIGERService.test.js"
   },
   "dependencies": {
-    "@mailwoman/core": "5.6.0",
-    "@mailwoman/spatial": "5.6.0",
+    "@mailwoman/core": "5.7.0",
+    "@mailwoman/spatial": "5.7.0",
     "@turf/boolean-contains": "^7.3.5",
     "kysely": "^0.29.2",
     "shapefile-parser": "^1.0.3",
+    "spliterator": "^3.2.0",
     "type-fest": "^5.7.0"
   },
+  "devDependencies": {
+    "@protomaps/basemaps": "^5.7.2"
+  },
   "engines": {
     "node": ">=24.18.0"
   }

package/sdk/fetch.ts

@@ -28,13 +28,13 @@ import { spawn } from "node:child_process"
 import { createWriteStream, existsSync } from "node:fs"
 import { mkdir, rename } from "node:fs/promises"
 import { dirname, join } from "node:path"
-import { createInterface } from "node:readline"
 import { DatabaseSync } from "node:sqlite"
 import { Readable } from "node:stream"
 import { pipeline } from "node:stream/promises"
 
 import { DatabaseClient } from "@mailwoman/core/kysley/client"
 import { mailwomanDataRoot } from "@mailwoman/core/utils"
+import { TextSpliterator } from "spliterator"
 
 import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "./schema.js"
 import { initializeTIGERSchema, TIGER_PRAGMAS } from "./schema.js"
@@ -197,7 +197,9 @@ async function discoverCounties(state: string, vintage: number): Promise<string[
 	const re = new RegExp(`tl_${vintage}_${state}(\\d{3})_addrfeat\\.zip`, "g")
 	const counties = new Set<string>()
 
-	for (let m = re.exec(html); m; m = re.exec(html)) counties.add(m[1]!)
+	for (let m = re.exec(html); m; m = re.exec(html)) {
+		counties.add(m[1]!)
+	}
 
 	return [...counties].sort()
 }
@@ -235,28 +237,33 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
 	await initializeTIGERSchema(kdb)
 
 	const insertBatch = async (rows: Row[]): Promise<void> => {
-		if (level === "tabblock20")
+		if (level === "tabblock20") {
 			await kdb
 				.insertInto("tabblock20")
 				.values(rows as TIGERBlockTable[])
 				.execute()
-		else if (level === "place")
+		} else if (level === "place") {
 			await kdb
 				.insertInto("tiger_places")
 				.values(rows as TIGERPlaceTable[])
 				.execute()
-		else
+		} else {
 			await kdb
 				.insertInto("tiger_streets")
 				.values(rows as TIGERStreetTable[])
 				.execute()
+		}
 	}
 
 	try {
 		// Idempotent re-run: drop this state's rows first (state_code for blocks, statefp otherwise).
-		if (level === "tabblock20") await kdb.deleteFrom("tabblock20").where("state_code", "=", state).execute()
-		else if (level === "place") await kdb.deleteFrom("tiger_places").where("statefp", "=", state).execute()
-		else await kdb.deleteFrom("tiger_streets").where("statefp", "=", state).execute()
+		if (level === "tabblock20") {
+			await kdb.deleteFrom("tabblock20").where("state_code", "=", state).execute()
+		} else if (level === "place") {
+			await kdb.deleteFrom("tiger_places").where("statefp", "=", state).execute()
+		} else {
+			await kdb.deleteFrom("tiger_streets").where("statefp", "=", state).execute()
+		}
 
 		let inserted = 0
 		let batch: Row[] = []
@@ -300,7 +307,9 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
 			child.stderr.on("data", (d) => (stderr += d))
 			const exited = new Promise<number>((resolve) => child.on("close", (code) => resolve(code ?? 0)))
 
-			for await (const line of createInterface({ input: child.stdout, crlfDelay: Infinity })) {
+			// GeoJSONSeq is line-delimited; keep the per-line `JSON.parse` in a try/catch so a malformed
+			// record is tolerated (skipped), not thrown.
+			for await (const line of TextSpliterator.fromAsync(child.stdout)) {
 				if (!line) continue
 				let feat: { properties?: Record<string, unknown>; geometry?: unknown }
 

package/sdk/redistricting.ts

@@ -17,16 +17,16 @@
  */
 
 import { spawn } from "node:child_process"
-import { createReadStream, createWriteStream, existsSync } from "node:fs"
+import { createWriteStream, existsSync } from "node:fs"
 import { mkdir, rename } from "node:fs/promises"
 import { dirname, join } from "node:path"
-import { createInterface } from "node:readline"
 import { DatabaseSync } from "node:sqlite"
 import { Readable } from "node:stream"
 import { pipeline } from "node:stream/promises"
 
 import { DatabaseClient } from "@mailwoman/core/kysley/client"
 import { mailwomanDataRoot } from "@mailwoman/core/utils"
+import { TextSpliterator } from "spliterator"
 
 import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "../state.js"
 import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "./schema.js"
@@ -118,9 +118,15 @@ async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
 	return false
 }
 
+// Pipe-delimited, fixed field-offset census rows — the manual `split("|")` at each call site stays (the
+// parse indexes by position, not by header). spliterator keeps CRLF's trailing CR where readline stripped
+// it, but every field this parser reads (geo GEOCODE/LOGRECNO ≤ 9, segment-1 LOGRECNO + P2 ≤ 86) sits well
+// before the final column, so the retained CR only ever lands on an unread trailing field.
 async function eachLine(path: string, fn: (line: string) => void): Promise<void> {
-	for await (const line of createInterface({ input: createReadStream(path), crlfDelay: Infinity })) {
-		if (line) fn(line)
+	for await (const line of TextSpliterator.fromAsync(path)) {
+		if (line) {
+			fn(line)
+		}
 	}
 }
 
@@ -198,7 +204,7 @@ export async function* fetchRedistricting(
 		}
 
 		// Pass 2: segment 1 → P2 counts for the mapped LOGRECNOs, flushing as we go.
-		for await (const line of createInterface({ input: createReadStream(seg1Path), crlfDelay: Infinity })) {
+		for await (const line of TextSpliterator.fromAsync(seg1Path)) {
 			if (!line) continue
 			const f = line.split("|")
 			const geoid = logToGeoid.get(f[SEG_LOGRECNO] ?? "")

package/tools/race-dots-map.ts

@@ -0,0 +1,166 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Render a race-by-dot-density PMTiles tileset (from `race-dots.ts` → tippecanoe) as a standalone
+ *   MapLibre page on the house stack: the Protomaps `basemap-v4` vector basemap (layers generated +
+ *   inlined here, à la `registry/map-html.ts`) under a circle layer of the dots, colored by race
+ *   category. The dots PMTiles is read client-side via the `pmtiles` protocol.
+ *
+ *   Each dot is one of `--per` people of a category, placed at random inside its Census block — a
+ *   representation, not a record about any address. Serve over localhost (the house tile server
+ *   CORS-restricts to localhost + the docs domains).
+ *
+ *   Run: node --experimental-strip-types scripts/census/race-dots-map.ts\
+ *   --pmtiles-url http://localhost:8899/race-dots-oc.pmtiles --out /tmp/race-dots-oc.html
+ */
+
+import { writeFileSync } from "node:fs"
+import { parseArgs } from "node:util"
+
+import { layers, namedFlavor } from "@protomaps/basemaps"
+
+// Loose scan parity with the retired local argv helpers: unknown flags tolerated.
+const { values: rawValues } = parseArgs({
+	options: {
+		lat: { type: "string" },
+		lng: { type: "string" },
+		out: { type: "string" },
+		per: { type: "string" },
+		"pmtiles-url": { type: "string" },
+		title: { type: "string" },
+		zoom: { type: "string" },
+	},
+	strict: false,
+	allowPositionals: true,
+})
+// Typed view: strict:false loosens TS inference, but declared options always parse to their schema type.
+const values = rawValues as {
+	lat?: string
+	lng?: string
+	out?: string
+	per?: string
+	"pmtiles-url"?: string
+	title?: string
+	zoom?: string
+}
+const PMTILES_URL = values["pmtiles-url"] || "http://localhost:8899/race-dots-oc.pmtiles"
+const OUT = values["out"] || "/tmp/race-dots-oc.html"
+const PER = Number(values["per"] || "5")
+const PER_PHRASE = PER === 1 ? "one dot is one person" : `one dot ≈ ${PER} people`
+const TITLE = values["title"] || `Race in Orange County, CA — ${PER_PHRASE} (2020 Census)`
+const CENTER_LNG = Number(values["lng"] || "-117.83")
+const CENTER_LAT = Number(values["lat"] || "33.68")
+const ZOOM = Number(values["zoom"] || "9.4")
+
+const MAPLIBRE_VERSION = "5.24.0"
+const MAPLIBRE_JS_SRI = "sha384-5+cfbwT0iiub6VsQAdn6yz16nr6sDiQoHx6tm4O8OVYXHYOxcffFmCJBL0dgdvGp"
+const MAPLIBRE_CSS_SRI = "sha384-uTttxo/aOKbdE5RlD/SPzSDoDmNvGlUYPjONi2MN/b7c9HPSvW07OIuyP7uL6jxK"
+const PMTILES_VERSION = "4.4.1"
+
+const BASEMAP_SOURCE_ID = "basemap-v4"
+const BASEMAP_TILEJSON_URL = "https://tiles.sister.software/basemap-v4.json"
+const GLYPHS_URL = "https://protomaps.github.io/basemaps-assets/fonts/{fontstack}/{range}.pbf"
+const SPRITE_URL = "https://protomaps.github.io/basemaps-assets/sprites/v4/light"
+
+// Cooper Center "Racial Dot Map" palette, by `pl_block`/P2 category.
+const CATEGORY_COLOR: Record<string, string> = {
+	white: "#1f78b4", // blue
+	black: "#33a02c", // green
+	hispanic: "#ff7f00", // orange
+	asian: "#e31a1c", // red
+	aian: "#6a3d9a", // purple
+	nhpi: "#00b3b3", // teal
+	other: "#b15928", // brown
+	multi: "#9e9e9e", // grey
+}
+const CATEGORY_LABEL: Record<string, string> = {
+	white: "White (non-Hispanic)",
+	black: "Black (non-Hispanic)",
+	hispanic: "Hispanic or Latino",
+	asian: "Asian (non-Hispanic)",
+	aian: "Native American",
+	nhpi: "Pacific Islander",
+	other: "Other (non-Hispanic)",
+	multi: "Two or more races",
+}
+
+const colorMatch: unknown[] = ["match", ["get", "cat"]]
+
+for (const [cat, color] of Object.entries(CATEGORY_COLOR)) {
+	colorMatch.push(cat, color)
+}
+colorMatch.push("#9e9e9e")
+
+const style = {
+	version: 8,
+	glyphs: GLYPHS_URL,
+	sprite: SPRITE_URL,
+	sources: {
+		[BASEMAP_SOURCE_ID]: { type: "vector", url: BASEMAP_TILEJSON_URL },
+		dots: { type: "vector", url: "pmtiles://" + PMTILES_URL },
+	},
+	layers: [
+		...(layers(BASEMAP_SOURCE_ID, namedFlavor("light"), { lang: "en" }) as unknown[]),
+		{
+			id: "race-dots",
+			type: "circle",
+			source: "dots",
+			"source-layer": "dots",
+			paint: {
+				"circle-radius": ["interpolate", ["linear"], ["zoom"], 4, 0.7, 9, 1.7, 12, 2.8, 16, 4.5],
+				"circle-color": colorMatch,
+				"circle-opacity": 0.85,
+			},
+		},
+	],
+}
+
+const legendRows = Object.keys(CATEGORY_COLOR)
+	.map((c) => `<div><i style="background:${CATEGORY_COLOR[c]}"></i>${CATEGORY_LABEL[c]}</div>`)
+	.join("")
+
+// The client script avoids template literals / `${` so it survives this outer template verbatim.
+const html = `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<title>${TITLE}</title>
+<link rel="stylesheet" href="https://unpkg.com/maplibre-gl@${MAPLIBRE_VERSION}/dist/maplibre-gl.css" integrity="${MAPLIBRE_CSS_SRI}" crossorigin="anonymous" />
+<style>
+	html, body { margin: 0; height: 100%; font-family: system-ui, -apple-system, "Segoe UI", sans-serif; }
+	#map { position: absolute; inset: 0; background: #f4f4f2; }
+	.mw-panel { position: absolute; z-index: 1; background: rgba(255,255,255,0.94); padding: 10px 12px; border-radius: 8px; box-shadow: 0 1px 6px rgba(0,0,0,0.3); font-size: 12px; line-height: 1.5; color: #1a1a1a; }
+	.mw-title { top: 10px; left: 10px; max-width: 64%; }
+	.mw-title h1 { font-size: 14px; margin: 0 0 4px; }
+	.mw-legend { bottom: 22px; right: 10px; }
+	.mw-legend i { display: inline-block; width: 11px; height: 11px; margin-right: 6px; border-radius: 50%; vertical-align: -1px; }
+	.muted { color: #666; }
+</style>
+</head>
+<body>
+<div id="map"></div>
+<div class="mw-panel mw-title"><h1>${TITLE}</h1><div class="muted">Each dot is ${PER === 1 ? "a person" : `~${PER} people`}, placed at random inside their Census block. Source: 2020 P.L. 94-171 (table P2) + TIGER blocks.</div></div>
+<div class="mw-panel mw-legend">${legendRows}</div>
+<script src="https://unpkg.com/maplibre-gl@${MAPLIBRE_VERSION}/dist/maplibre-gl.js" integrity="${MAPLIBRE_JS_SRI}" crossorigin="anonymous"></script>
+<script src="https://unpkg.com/pmtiles@${PMTILES_VERSION}/dist/pmtiles.js" crossorigin="anonymous"></script>
+<script>
+	var protocol = new pmtiles.Protocol();
+	maplibregl.addProtocol("pmtiles", protocol.tile);
+	var map = new maplibregl.Map({
+		container: "map",
+		style: ${JSON.stringify(style)},
+		center: [${CENTER_LNG}, ${CENTER_LAT}],
+		zoom: ${ZOOM},
+		attributionControl: false
+	});
+	map.addControl(new maplibregl.AttributionControl({ compact: true, customAttribution: "US Census 2020 · TIGER · Protomaps · OpenStreetMap" }));
+</script>
+</body>
+</html>
+`
+
+writeFileSync(OUT, html)
+console.error(`[written] ${OUT}  (pmtiles: ${PMTILES_URL})`)

package/tools/race-dots.ts

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Race-by-dot-density builder — the Cooper Center "Racial Dot Map" recipe, from the TIGER DB the
+ *   `mailwoman tiger` CLI produces.
+ *
+ *   Reads `tabblock20 ⋈ pl_block` (block geometry + Census 2020 P.L. 94-171 table P2 counts) and
+ *   scatters one dot per `--per` people uniformly at random inside each block, tagged with its
+ *   race/ethnicity category. Output is NDJSON (one GeoJSON Point Feature per line, with a
+ *   `tippecanoe` layer hint) ready for `tippecanoe -o race-dots.pmtiles`.
+ *
+ *   Point-in-polygon uses `@turf/boolean-contains` (ships with `@mailwoman/tiger`). The dot is a
+ *   _representation_, not a record: a random position inside the block it belongs to, standing in
+ *   for `--per` real people of that category. It says nothing about any individual address.
+ *
+ *   Build the input DB first: mailwoman tiger fetch --state 06 --county 059 --out tiger-oc.db
+ *   mailwoman tiger redistricting --state 06 --county 059 --out tiger-oc.db
+ *
+ *   Run: node --experimental-strip-types scripts/census/race-dots.ts\
+ *   --db tiger-oc.db --per 10 --out /tmp/race-dots.ndjson
+ */
+
+import { createWriteStream } from "node:fs"
+import { DatabaseSync } from "node:sqlite"
+import { parseArgs } from "node:util"
+
+import { dataRootPath } from "@mailwoman/core/utils"
+import booleanContains from "@turf/boolean-contains"
+
+// Loose scan parity with the retired local argv helpers: unknown flags tolerated.
+const { values: rawValues } = parseArgs({
+	options: { db: { type: "string" }, layer: { type: "string" }, out: { type: "string" }, per: { type: "string" } },
+	strict: false,
+	allowPositionals: true,
+})
+// Typed view: strict:false loosens TS inference, but declared options always parse to their schema type.
+const values = rawValues as { db?: string; layer?: string; out?: string; per?: string }
+const DB = values["db"] || dataRootPath("tiger", "tiger-oc.db")
+const OUT = values["out"] || "/tmp/race-dots.ndjson"
+const PER = Number(values["per"] || "10") // people represented by one dot
+const LAYER = values["layer"] || "dots"
+
+// The eight P2 categories (columns in pl_block) that partition each block's population.
+const CATEGORIES = ["hispanic", "white", "black", "asian", "aian", "nhpi", "other", "multi"] as const
+
+type Ring = number[][]
+type PolygonCoords = Ring[]
+
+function bbox(rings: PolygonCoords): [number, number, number, number] {
+	let minX = Infinity,
+		minY = Infinity,
+		maxX = -Infinity,
+		maxY = -Infinity
+
+	for (const [x, y] of rings[0]!) {
+		if (x! < minX) {
+			minX = x!
+		}
+
+		if (x! > maxX) {
+			maxX = x!
+		}
+
+		if (y! < minY) {
+			minY = y!
+		}
+
+		if (y! > maxY) {
+			maxY = y!
+		}
+	}
+
+	return [minX, minY, maxX, maxY]
+}
+
+// A block geometry is one or more polygons. Pick a sub-polygon weighted by bbox area, then
+// rejection-sample inside it with a turf containment test (handles holes + winding correctly).
+function randomPointIn(polys: PolygonCoords[], areas: number[], totalArea: number): [number, number] | null {
+	let r = Math.random() * totalArea
+	let pick = 0
+
+	while (pick < polys.length - 1 && (r -= areas[pick]!) > 0) {
+		pick++
+	}
+	const poly = polys[pick]!
+	const polyFeature = {
+		type: "Feature" as const,
+		geometry: { type: "Polygon" as const, coordinates: poly },
+		properties: {},
+	}
+	const [minX, minY, maxX, maxY] = bbox(poly)
+
+	for (let tries = 0; tries < 60; tries++) {
+		const x = minX + Math.random() * (maxX - minX)
+		const y = minY + Math.random() * (maxY - minY)
+		const pt = { type: "Feature" as const, geometry: { type: "Point" as const, coordinates: [x, y] }, properties: {} }
+
+		if (booleanContains(polyFeature, pt)) return [x, y]
+	}
+
+	return null
+}
+
+const db = new DatabaseSync(DB, { readOnly: true })
+const rows = db
+	.prepare(
+		`SELECT b.geometry AS geometry, ${CATEGORIES.map((c) => `p.${c} AS ${c}`).join(", ")}
+		 FROM tabblock20 b JOIN pl_block p ON b.GEOID = p.GEOID
+		 WHERE p.pop_total > 0`
+	)
+	.all() as Array<{ geometry: string } & Record<(typeof CATEGORIES)[number], number>>
+db.close()
+
+const out = createWriteStream(OUT)
+const totals = new Map<string, number>()
+let dots = 0,
+	skipped = 0
+
+for (const row of rows) {
+	let geom: { type: string; coordinates: unknown }
+
+	try {
+		geom = JSON.parse(row.geometry)
+	} catch {
+		continue
+	}
+	const polys: PolygonCoords[] =
+		geom.type === "Polygon" ? [geom.coordinates as PolygonCoords] : (geom.coordinates as PolygonCoords[])
+	const areas = polys.map((p) => {
+		const [a, b, c, d] = bbox(p)
+
+		return Math.max((c - a) * (d - b), 1e-12)
+	})
+	const totalArea = areas.reduce((s, a) => s + a, 0)
+
+	for (const cat of CATEGORIES) {
+		const people = row[cat]
+
+		if (people <= 0) continue
+		const exact = people / PER
+		const n = Math.floor(exact) + (Math.random() < exact - Math.floor(exact) ? 1 : 0)
+
+		for (let k = 0; k < n; k++) {
+			const pt = randomPointIn(polys, areas, totalArea)
+
+			if (!pt) {
+				skipped++
+				continue
+			}
+			out.write(
+				JSON.stringify({
+					type: "Feature",
+					tippecanoe: { layer: LAYER },
+					properties: { cat },
+					geometry: { type: "Point", coordinates: [Math.round(pt[0] * 1e5) / 1e5, Math.round(pt[1] * 1e5) / 1e5] },
+				}) + "\n"
+			)
+			dots++
+			totals.set(cat, (totals.get(cat) ?? 0) + 1)
+		}
+	}
+}
+
+out.end()
+console.error(`[done] ${dots} dots from ${rows.length} blocks (1 dot ≈ ${PER} people); ${skipped} skipped`)
+
+for (const [cat, n] of [...totals.entries()].sort((a, b) => b[1] - a[1])) {
+	console.error(`  ${n.toString().padStart(7)}  ${cat}`)
+}

package/tools/serve-range.ts

@@ -0,0 +1,61 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Minimal static file server that honours HTTP Range requests (206) — Python's `http.server` does
+ *   not, and PMTiles reads via Range, so it can't be served by it. Used to preview/render a local
+ *   `.pmtiles` tileset (e.g. the race-dot map). Not for production; the deployed tiles go through
+ *   the tile worker.
+ *
+ *   Run: node scripts/census/serve-range.ts [dir=/tmp] [port=8899]
+ */
+
+import { createReadStream, statSync, type Stats } from "node:fs"
+import { createServer, type IncomingMessage, type ServerResponse } from "node:http"
+import { extname, join, normalize } from "node:path"
+import { parseArgs } from "node:util"
+
+const { positionals } = parseArgs({ allowPositionals: true, strict: false })
+const DIR = positionals[0] || "/tmp"
+const PORT = Number(positionals[1] || 8899)
+
+const TYPES: Record<string, string> = {
+	".html": "text/html; charset=utf-8",
+	".json": "application/json",
+	".pmtiles": "application/octet-stream",
+	".pbf": "application/x-protobuf",
+	".png": "image/png",
+}
+
+createServer((req: IncomingMessage, res: ServerResponse) => {
+	const rel = normalize(decodeURIComponent((req.url || "/").split("?")[0]!)).replace(/^(\.\.[/\\])+/, "")
+	const path = join(DIR, rel)
+	let st: Stats
+
+	try {
+		st = statSync(path)
+	} catch {
+		res.writeHead(404)
+
+		return res.end("not found")
+	}
+	const type = TYPES[extname(path)] || "application/octet-stream"
+	const base = { "Content-Type": type, "Accept-Ranges": "bytes", "Access-Control-Allow-Origin": "*" }
+	const range = req.headers.range
+	const m = range && /bytes=(\d+)-(\d*)/.exec(range)
+
+	if (m) {
+		const start = Number(m[1])
+		const end = m[2] ? Number(m[2]) : st.size - 1
+		res.writeHead(206, {
+			...base,
+			"Content-Range": `bytes ${start}-${end}/${st.size}`,
+			"Content-Length": end - start + 1,
+		})
+		createReadStream(path, { start, end }).pipe(res)
+	} else {
+		res.writeHead(200, { ...base, "Content-Length": st.size })
+		createReadStream(path).pipe(res)
+	}
+}).listen(PORT, () => console.error(`range server: ${DIR} on http://localhost:${PORT}`))