@mailwoman/tiger 8.6.0 → 9.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/tiger",
-  "version": "8.6.0",
+  "version": "9.1.0",
   "description": "US Census TIGER/Line data processing and analysis.",
   "keywords": [
     "census",
@@ -73,12 +73,12 @@
     "test": "node ./out/sdk/TIGERService.test.js"
   },
   "dependencies": {
-    "@mailwoman/core": "8.6.0",
-    "@mailwoman/spatial": "8.6.0",
+    "@mailwoman/core": "9.1.0",
+    "@mailwoman/spatial": "9.1.0",
     "@turf/boolean-contains": "^7.3.5",
     "kysely": "^0.29.4",
     "shapefile-parser": "^1.0.3",
-    "spliterator": "^4.0.0",
+    "spliterator": "^6.1.0",
     "type-fest": "^5.8.0"
   },
   "devDependencies": {

package/sdk/download.ts

@@ -0,0 +1,73 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Cached archive download + subprocess capture, shared by `fetch.ts` and `redistricting.ts`.
+ *
+ *   RAW `fetch` HERE IS DELIBERATE. `AGENTS.md` requires HTTP clients to extend or instantiate
+ *   `APIClient`, and that rule is about API REQUESTS — small bodies, repeated calls, rate-limited
+ *   hosts, where its pacing, bounded retry, response caching and `ResourceError` mapping all earn
+ *   their keep. This is a multi-gigabyte file transfer streamed straight to disk. Response caching
+ *   would be nonsense at that size, pacing has nothing to pace (one request), and axios buffers a
+ *   non-stream response type in memory. The primitive that fits a body this large is the one that
+ *   never holds it: a web stream piped to a write stream.
+ */
+
+import { spawn } from "node:child_process"
+import { createWriteStream, existsSync } from "node:fs"
+import { rename } from "node:fs/promises"
+import { Readable } from "node:stream"
+import { pipeline } from "node:stream/promises"
+
+import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
+
+/**
+ * Run a command and resolve its stdout, rejecting with the captured stderr on a non-zero exit.
+ *
+ * Used for the `unzip -tq` integrity probe below — a shelled-out check rather than a library one because the archives
+ * are large and `unzip -t` streams them without materializing the contents.
+ */
+export function runCapture(cmd: string, args: string[]): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const child = spawn(cmd, args)
+		let out = ""
+		let err = ""
+		child.stdout.on("data", (d) => (out += d))
+		child.stderr.on("data", (d) => (err += d))
+		child.on("error", reject)
+
+		child.on("close", (code) =>
+			code === 0 ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.slice(0, 500)}`))
+		)
+	})
+}
+
+/**
+ * Download `url` to `dest` unless a VALID copy is already there. Returns `true` when the cache was reused, `false` when
+ * a download happened.
+ *
+ * "Valid" means every member's CRC-32 checks out — an existence check alone is not enough, because an interrupted
+ * download leaves a plausible-looking file that fails only much later, inside ogr2ogr. Writes to a `.tmp` sibling and
+ * renames, so `dest` is never a partial archive.
+ */
+export async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
+	if (existsSync(dest)) {
+		try {
+			await verifyZipIntegrity(dest)
+
+			return true
+		} catch {
+			// corrupt cache — re-download
+		}
+	}
+
+	const tmp = dest + ".tmp"
+	const res = await fetch(url, { redirect: "follow" })
+
+	if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
+	await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), createWriteStream(tmp))
+	await rename(tmp, dest)
+
+	return false
+}

package/sdk/fetch.ts

@@ -32,10 +32,13 @@ import { DatabaseSync } from "node:sqlite"
 import { Readable } from "node:stream"
 import { pipeline } from "node:stream/promises"
 
+import { extractZipEntries } from "@mailwoman/core/fs/zip"
 import { DatabaseClient } from "@mailwoman/core/kysley/client"
+import { tryParsingJSON } from "@mailwoman/core/objects"
 import { mailwomanDataRoot } from "@mailwoman/core/utils"
 import { TextSpliterator } from "spliterator"
 
+import { downloadIfNeeded } from "./download.ts"
 import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "./schema.ts"
 import { initializeTIGERSchema, TIGER_PRAGMAS } from "./schema.ts"
 
@@ -169,42 +172,6 @@ function buildRow(level: TIGERFetchLevel, p: Record<string, unknown>, geometry:
 	}
 }
 
-function runCapture(cmd: string, args: string[]): Promise<string> {
-	return new Promise((resolve, reject) => {
-		const child = spawn(cmd, args)
-		let out = ""
-		let err = ""
-		child.stdout.on("data", (d) => (out += d))
-		child.stderr.on("data", (d) => (err += d))
-		child.on("error", reject)
-
-		child.on("close", (code) =>
-			code === 0 ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.slice(0, 500)}`))
-		)
-	})
-}
-
-async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
-	if (existsSync(dest)) {
-		try {
-			await runCapture("unzip", ["-tq", dest])
-
-			return true
-		} catch {
-			// corrupt cache — re-download
-		}
-	}
-
-	const tmp = dest + ".tmp"
-	const res = await fetch(url, { redirect: "follow" })
-
-	if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
-	await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), createWriteStream(tmp))
-	await rename(tmp, dest)
-
-	return false
-}
-
 /**
  * Scrape the ADDRFEAT directory listing for a state's county FIPS codes.
  */
@@ -304,7 +271,7 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
 			const cached = await downloadIfNeeded(url, zipPath)
 			yield { phase: "download", file: zipName, cached }
 
-			await runCapture("unzip", ["-o", "-q", zipPath, "-d", cacheDir])
+			await extractZipEntries(zipPath, cacheDir)
 			const layer = `tl_${vintage}_${unit}_${level}`
 			const shpPath = join(cacheDir, layer + ".shp")
 			yield { phase: "extract", file: layer + ".shp" }
@@ -331,17 +298,14 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
 				child.on("close", (code) => resolve(code ?? 0))
 			})
 
-			// GeoJSONSeq is line-delimited; keep the per-line `JSON.parse` in a try/catch so a malformed
-			// record is tolerated (skipped), not thrown.
+			// GeoJSONSeq is line-delimited; per-line `tryParsingJSON` so a malformed record is
+			// tolerated (skipped), not thrown.
 			for await (const line of TextSpliterator.fromAsync(child.stdout)) {
 				if (!line) continue
-				let feat: { properties?: Record<string, unknown>; geometry?: unknown }
 
-				try {
-					feat = JSON.parse(line)
-				} catch {
-					continue
-				}
+				const feat = tryParsingJSON<{ properties?: Record<string, unknown>; geometry?: unknown }>(line)
+
+				if (!feat) continue
 
 				if (!feat.properties) continue
 

package/sdk/redistricting.ts

@@ -24,11 +24,13 @@ import { DatabaseSync } from "node:sqlite"
 import { Readable } from "node:stream"
 import { pipeline } from "node:stream/promises"
 
+import { extractZipEntries } from "@mailwoman/core/fs/zip"
 import { DatabaseClient } from "@mailwoman/core/kysley/client"
 import { mailwomanDataRoot } from "@mailwoman/core/utils"
 import { TextSpliterator } from "spliterator"
 
 import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "../state.ts"
+import { downloadIfNeeded } from "./download.ts"
 import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "./schema.ts"
 
 const REDISTRICTING_BASE =
@@ -104,42 +106,6 @@ export interface FetchRedistrictingResult {
 	inserted: number
 }
 
-function runCapture(cmd: string, args: string[]): Promise<string> {
-	return new Promise((resolve, reject) => {
-		const child = spawn(cmd, args)
-		let out = ""
-		let err = ""
-		child.stdout.on("data", (d) => (out += d))
-		child.stderr.on("data", (d) => (err += d))
-		child.on("error", reject)
-
-		child.on("close", (code) =>
-			code === 0 ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.slice(0, 500)}`))
-		)
-	})
-}
-
-async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
-	if (existsSync(dest)) {
-		try {
-			await runCapture("unzip", ["-tq", dest])
-
-			return true
-		} catch {
-			// corrupt cache — re-download
-		}
-	}
-
-	const tmp = dest + ".tmp"
-	const res = await fetch(url, { redirect: "follow" })
-
-	if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
-	await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), createWriteStream(tmp))
-	await rename(tmp, dest)
-
-	return false
-}
-
 // Pipe-delimited, fixed field-offset census rows — the manual `split("|")` at each call site stays (the
 // parse indexes by position, not by header). spliterator keeps CRLF's trailing CR where readline stripped
 // it, but every field this parser reads (geo GEOCODE/LOGRECNO ≤ 9, segment-1 LOGRECNO + P2 ≤ 86) sits well
@@ -183,7 +149,7 @@ export async function* fetchRedistricting(
 	const cached = await downloadIfNeeded(url, zipPath)
 	yield { phase: "download", file: zipName, cached }
 
-	await runCapture("unzip", ["-o", "-q", zipPath, "-d", cacheDir])
+	await extractZipEntries(zipPath, cacheDir)
 	const geoPath = join(cacheDir, `${fileAbbr}geo${vintage}.pl`)
 	const seg1Path = join(cacheDir, `${fileAbbr}00001${vintage}.pl`)
 	yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` }

package/tools/race-dots.ts

@@ -24,6 +24,7 @@
 import { createWriteStream } from "node:fs"
 import { DatabaseSync } from "node:sqlite"
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
 import { dataRootPath } from "@mailwoman/core/utils"
 
 /**
@@ -168,13 +169,9 @@ export async function raceDots(
 		skipped = 0
 
 	for (const row of rows) {
-		let geom: { type: string; coordinates: unknown }
+		const geom = tryParsingJSON<{ type: string; coordinates: unknown }>(row.geometry)
 
-		try {
-			geom = JSON.parse(row.geometry)
-		} catch {
-			continue
-		}
+		if (!geom) continue
 
 		const polys: PolygonCoords[] =
 			geom.type === "Polygon" ? [geom.coordinates as PolygonCoords] : (geom.coordinates as PolygonCoords[])