@mailwoman/tiger 9.1.0 → 9.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/{block-group.ts → lib/block-group.ts} +4 -4
  2. package/{class-code.ts → lib/class-code.ts} +1 -1
  3. package/{constants.ts → lib/constants.ts} +5 -5
  4. package/{county-subdivision.ts → lib/county-subdivision.ts} +4 -4
  5. package/{county.ts → lib/county.ts} +4 -4
  6. package/{files.ts → lib/files.ts} +1 -1
  7. package/{geoid.ts → lib/geoid.ts} +3 -4
  8. package/lib/index.ts +18 -0
  9. package/{sdk → lib/sdk}/download.ts +6 -30
  10. package/{sdk → lib/sdk}/fetch.ts +38 -59
  11. package/lib/sdk/index.ts +10 -0
  12. package/{sdk → lib/sdk}/redistricting.ts +91 -20
  13. package/{sdk → lib/sdk}/schema.ts +36 -0
  14. package/{sdk → lib/sdk}/state/TIGERTabulatedBlockEntity.ts +2 -1
  15. package/{sdk → lib/sdk}/state/TIGERTractEntity.ts +1 -1
  16. package/{sdk → lib/sdk}/state/block-operations.ts +3 -2
  17. package/lib/sdk/state/index.ts +9 -0
  18. package/{state.ts → lib/state.ts} +2 -2
  19. package/{tabulation-block.ts → lib/tabulation-block.ts} +2 -2
  20. package/lib/tools/index.ts +11 -0
  21. package/{tools → lib/tools}/race-dots-map.ts +5 -4
  22. package/{tools → lib/tools}/race-dots.ts +8 -9
  23. package/{tools → lib/tools}/serve-range.ts +20 -12
  24. package/{tract.ts → lib/tract.ts} +2 -2
  25. package/out/block-group.d.ts +4 -4
  26. package/out/block-group.d.ts.map +1 -1
  27. package/out/block-group.js.map +1 -1
  28. package/out/class-code.d.ts +1 -1
  29. package/out/class-code.d.ts.map +1 -1
  30. package/out/class-code.js +1 -1
  31. package/out/class-code.js.map +1 -1
  32. package/out/constants.d.ts +5 -5
  33. package/out/constants.d.ts.map +1 -1
  34. package/out/constants.js.map +1 -1
  35. package/out/county-subdivision.d.ts +4 -4
  36. package/out/county-subdivision.d.ts.map +1 -1
  37. package/out/county-subdivision.js.map +1 -1
  38. package/out/county.d.ts +4 -4
  39. package/out/county.d.ts.map +1 -1
  40. package/out/county.js.map +1 -1
  41. package/out/files.d.ts +1 -1
  42. package/out/files.d.ts.map +1 -1
  43. package/out/files.js.map +1 -1
  44. package/out/functional-status.d.ts.map +1 -1
  45. package/out/functional-status.js.map +1 -1
  46. package/out/geoid.d.ts +1 -2
  47. package/out/geoid.d.ts.map +1 -1
  48. package/out/geoid.js.map +1 -1
  49. package/out/index.d.ts +12 -12
  50. package/out/index.d.ts.map +1 -1
  51. package/out/index.js +12 -12
  52. package/out/index.js.map +1 -1
  53. package/out/legal-statistical-area.d.ts.map +1 -1
  54. package/out/legal-statistical-area.js.map +1 -1
  55. package/out/sdk/download.d.ts +0 -7
  56. package/out/sdk/download.d.ts.map +1 -1
  57. package/out/sdk/download.js +6 -25
  58. package/out/sdk/download.js.map +1 -1
  59. package/out/sdk/fetch.d.ts.map +1 -1
  60. package/out/sdk/fetch.js +25 -41
  61. package/out/sdk/fetch.js.map +1 -1
  62. package/out/sdk/index.d.ts +4 -4
  63. package/out/sdk/index.d.ts.map +1 -1
  64. package/out/sdk/index.js +4 -4
  65. package/out/sdk/index.js.map +1 -1
  66. package/out/sdk/redistricting.d.ts +29 -0
  67. package/out/sdk/redistricting.d.ts.map +1 -1
  68. package/out/sdk/redistricting.js +71 -19
  69. package/out/sdk/redistricting.js.map +1 -1
  70. package/out/sdk/schema.d.ts +12 -9
  71. package/out/sdk/schema.d.ts.map +1 -1
  72. package/out/sdk/schema.js +19 -0
  73. package/out/sdk/schema.js.map +1 -1
  74. package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts.map +1 -1
  75. package/out/sdk/state/TIGERTabulatedBlockEntity.js +1 -1
  76. package/out/sdk/state/TIGERTabulatedBlockEntity.js.map +1 -1
  77. package/out/sdk/state/TIGERTractEntity.d.ts +1 -1
  78. package/out/sdk/state/TIGERTractEntity.d.ts.map +1 -1
  79. package/out/sdk/state/TIGERTractEntity.js.map +1 -1
  80. package/out/sdk/state/block-operations.d.ts +1 -1
  81. package/out/sdk/state/block-operations.d.ts.map +1 -1
  82. package/out/sdk/state/block-operations.js +1 -1
  83. package/out/sdk/state/block-operations.js.map +1 -1
  84. package/out/sdk/state/index.d.ts +3 -3
  85. package/out/sdk/state/index.d.ts.map +1 -1
  86. package/out/sdk/state/index.js +3 -3
  87. package/out/sdk/state/index.js.map +1 -1
  88. package/out/state.d.ts +2 -2
  89. package/out/state.d.ts.map +1 -1
  90. package/out/state.js.map +1 -1
  91. package/out/tabulation-block.d.ts +2 -2
  92. package/out/tabulation-block.d.ts.map +1 -1
  93. package/out/tabulation-block.js.map +1 -1
  94. package/out/tools/index.d.ts +4 -8
  95. package/out/tools/index.d.ts.map +1 -1
  96. package/out/tools/index.js +4 -8
  97. package/out/tools/index.js.map +1 -1
  98. package/out/tools/race-dots-map.d.ts.map +1 -1
  99. package/out/tools/race-dots-map.js +5 -4
  100. package/out/tools/race-dots-map.js.map +1 -1
  101. package/out/tools/race-dots.d.ts.map +1 -1
  102. package/out/tools/race-dots.js +7 -8
  103. package/out/tools/race-dots.js.map +1 -1
  104. package/out/tools/serve-range.d.ts +2 -3
  105. package/out/tools/serve-range.d.ts.map +1 -1
  106. package/out/tools/serve-range.js +17 -10
  107. package/out/tools/serve-range.js.map +1 -1
  108. package/out/tract.d.ts +2 -2
  109. package/out/tract.d.ts.map +1 -1
  110. package/out/tract.js.map +1 -1
  111. package/out/tsconfig.test.tsbuildinfo +1 -0
  112. package/package.json +53 -10
  113. package/index.ts +0 -18
  114. package/out/tsconfig.tsbuildinfo +0 -1
  115. package/sdk/index.ts +0 -10
  116. package/sdk/state/index.ts +0 -9
  117. package/tools/index.ts +0 -15
  118. /package/{functional-status.ts → lib/functional-status.ts} +0 -0
  119. /package/{legal-statistical-area.ts → lib/legal-statistical-area.ts} +0 -0
@@ -4,10 +4,10 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
9
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
10
- import type { AdminLevel1Code } from "./state.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
10
+ import type { AdminLevel1Code } from "#state"
11
11
 
12
12
  /**
13
13
  * @title TIGER Block Group
@@ -59,7 +59,7 @@ export const TIGERClassCode = {
59
59
  /**
60
60
  * A movable barrier across a road.
61
61
  */
62
- Gate: "C3066",
62
+ Check: "C3066",
63
63
 
64
64
  /**
65
65
  * A structure or barrier where a fee is collected for using a road.
@@ -4,8 +4,8 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
9
  import type {
10
10
  GeoIDPart,
11
11
  FIPSBlockCode,
@@ -15,9 +15,9 @@ import type {
15
15
  FIPSCountySubDivisionCode,
16
16
  FIPSPlaceCode,
17
17
  FIPSTractCode,
18
- } from "./geoid.ts"
19
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
20
- import type { FIPSStateCode } from "./state.ts"
18
+ } from "#geoid"
19
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
20
+ import type { FIPSStateCode } from "#state"
21
21
 
22
22
  /**
23
23
  * A code indicating the classification of the block, i.e. urban or rural.
@@ -4,10 +4,10 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
9
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
10
- import type { AdminLevel1Code } from "./state.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
10
+ import type { AdminLevel1Code } from "#state"
11
11
 
12
12
  /**
13
13
  * Census Geographic subdivision of a county.
@@ -4,10 +4,10 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
9
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
10
- import type { AdminLevel1Code } from "./state.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
10
+ import type { AdminLevel1Code } from "#state"
11
11
 
12
12
  /**
13
13
  * @title TIGER County
@@ -6,7 +6,7 @@
6
6
  * TIGER Census data utilities.
7
7
  */
8
8
 
9
- import type { AdminLevel1Code } from "./state.ts"
9
+ import type { AdminLevel1Code } from "#state"
10
10
 
11
11
  //#region Constants
12
12
 
@@ -6,7 +6,7 @@
6
6
 
7
7
  import type { Tagged } from "type-fest"
8
8
 
9
- import type { AdminLevel1Code, FIPSStateCode } from "./state.ts"
9
+ import type { AdminLevel1Code, FIPSStateCode } from "#state"
10
10
 
11
11
  //#region Tagged FIPS types
12
12
 
@@ -301,7 +301,6 @@ export interface ParsedGeoIDBlockLevel {
301
301
  [GeoIDPart.Place]: FIPSPlaceCode
302
302
  [GeoIDPart.BlockGroup]: FIPSBlockGroupCode
303
303
  [GeoIDPart.Block]: FIPSBlockCode
304
- [GeoIDPart.BlockGroup]: FIPSBlockGroupCode
305
304
  [GeoIDPart.CongressionalDistrict]: FIPSCongressionalDistrictCode
306
305
  }
307
306
 
@@ -509,10 +508,10 @@ export const GeoIDInputMatchers = {
509
508
  * @see {@linkcode GeoIDInputMatchers} for the input-level matchers.
510
509
  */
511
510
  const GeoIDComponentMatchers = Object.fromEntries(
512
- Object.values(GeoIDPart).map((value) => {
511
+ Object.values(GeoIDPart).map((value): [GeoIDPart, GeoIDInputMatcher] => {
513
512
  return [value, new GeoIDInputMatcher(value)]
514
513
  })
515
- ) as unknown as Record<GeoIDPart, GeoIDInputMatcher>
514
+ ) as Record<GeoIDPart, GeoIDInputMatcher>
516
515
 
517
516
  /**
518
517
  * Type-predicate for checking if a value appears to be a valid GeoID component.
package/lib/index.ts ADDED
@@ -0,0 +1,18 @@
1
+ /**
2
+ * @copyright Sister Software.
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+
7
+ export * from "#block-group"
8
+ export * from "#class-code"
9
+ export * from "#constants"
10
+ export * from "#county-subdivision"
11
+ export * from "#county"
12
+ export * from "#files"
13
+ export * from "#functional-status"
14
+ export * from "#geoid"
15
+ export * from "#legal-statistical-area"
16
+ export * from "#state"
17
+ export * from "#tabulation-block"
18
+ export * from "#tract"
@@ -14,35 +14,11 @@
14
14
  * never holds it: a web stream piped to a write stream.
15
15
  */
16
16
 
17
- import { spawn } from "node:child_process"
18
- import { createWriteStream, existsSync } from "node:fs"
19
- import { rename } from "node:fs/promises"
20
- import { Readable } from "node:stream"
21
- import { pipeline } from "node:stream/promises"
22
-
17
+ import { pathExists } from "@mailwoman/core/fs/readers"
18
+ import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
19
+ import { movePath } from "@mailwoman/core/fs/writers"
23
20
  import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
24
21
 
25
- /**
26
- * Run a command and resolve its stdout, rejecting with the captured stderr on a non-zero exit.
27
- *
28
- * Used for the `unzip -tq` integrity probe below — a shelled-out check rather than a library one because the archives
29
- * are large and `unzip -t` streams them without materializing the contents.
30
- */
31
- export function runCapture(cmd: string, args: string[]): Promise<string> {
32
- return new Promise((resolve, reject) => {
33
- const child = spawn(cmd, args)
34
- let out = ""
35
- let err = ""
36
- child.stdout.on("data", (d) => (out += d))
37
- child.stderr.on("data", (d) => (err += d))
38
- child.on("error", reject)
39
-
40
- child.on("close", (code) =>
41
- code === 0 ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.slice(0, 500)}`))
42
- )
43
- })
44
- }
45
-
46
22
  /**
47
23
  * Download `url` to `dest` unless a VALID copy is already there. Returns `true` when the cache was reused, `false` when
48
24
  * a download happened.
@@ -52,7 +28,7 @@ export function runCapture(cmd: string, args: string[]): Promise<string> {
52
28
  * renames, so `dest` is never a partial archive.
53
29
  */
54
30
  export async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
55
- if (existsSync(dest)) {
31
+ if (await pathExists(dest)) {
56
32
  try {
57
33
  await verifyZipIntegrity(dest)
58
34
 
@@ -66,8 +42,8 @@ export async function downloadIfNeeded(url: string, dest: string): Promise<boole
66
42
  const res = await fetch(url, { redirect: "follow" })
67
43
 
68
44
  if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
69
- await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), createWriteStream(tmp))
70
- await rename(tmp, dest)
45
+ await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), openWriteStream(tmp))
46
+ await movePath(tmp, dest)
71
47
 
72
48
  return false
73
49
  }
@@ -24,23 +24,17 @@
24
24
  * inserts. Re-running a state replaces its rows.
25
25
  */
26
26
 
27
- import { spawn } from "node:child_process"
28
- import { createWriteStream, existsSync } from "node:fs"
29
- import { mkdir, rename } from "node:fs/promises"
30
- import { dirname, join } from "node:path"
31
- import { DatabaseSync } from "node:sqlite"
32
- import { Readable } from "node:stream"
33
- import { pipeline } from "node:stream/promises"
34
-
27
+ import { APIClient, pluckResponseData } from "@mailwoman/core/api"
28
+ import { mailwomanDataRoot } from "@mailwoman/core/data-root"
29
+ import { makeDirectories } from "@mailwoman/core/fs/writers"
35
30
  import { extractZipEntries } from "@mailwoman/core/fs/zip"
36
- import { DatabaseClient } from "@mailwoman/core/kysley/client"
37
- import { tryParsingJSON } from "@mailwoman/core/objects"
38
- import { mailwomanDataRoot } from "@mailwoman/core/utils"
39
- import { TextSpliterator } from "spliterator"
31
+ import { ogr2ogrGeoJSONSeq } from "@mailwoman/spatial/tools/ogr-stream"
32
+ import { DatabaseClient } from "@mailwoman/sqlite/client"
33
+ import { dirname, join } from "path-ts"
40
34
 
41
- import { downloadIfNeeded } from "./download.ts"
42
- import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "./schema.ts"
43
- import { initializeTIGERSchema, TIGER_PRAGMAS } from "./schema.ts"
35
+ import { downloadIfNeeded } from "#sdk/download"
36
+ import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "#sdk/schema"
37
+ import { initializeTIGERSchema, TIGER_PRAGMAS } from "#sdk/schema"
44
38
 
45
39
  const CENSUS_HOST = "https://www2.census.gov"
46
40
  const DEFAULT_DATA_ROOT = mailwomanDataRoot()
@@ -176,10 +170,11 @@ function buildRow(level: TIGERFetchLevel, p: Record<string, unknown>, geometry:
176
170
  * Scrape the ADDRFEAT directory listing for a state's county FIPS codes.
177
171
  */
178
172
  async function discoverCounties(state: string, vintage: number): Promise<string[]> {
179
- const res = await fetch(`${CENSUS_HOST}/geo/tiger/TIGER${vintage}/ADDRFEAT/`, { redirect: "follow" })
173
+ // The LISTING only — a small HTML index. The per-county archives below stay on raw `fetch`, streaming to disk.
174
+ const html = await new APIClient({ displayName: "tiger-listing", retry: true })
175
+ .fetch<string>({ url: `${CENSUS_HOST}/geo/tiger/TIGER${vintage}/ADDRFEAT/`, responseType: "text" })
176
+ .then(pluckResponseData)
180
177
 
181
- if (!res.ok) throw new Error(`HTTP ${res.status} listing ADDRFEAT for vintage ${vintage}`)
182
- const html = await res.text()
183
178
  const re = new RegExp(`tl_${vintage}_${state}(\\d{3})_addrfeat\\.zip`, "g")
184
179
  const counties = new Set<string>()
185
180
 
@@ -207,8 +202,8 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
207
202
  // the per-table idempotent delete keeps a re-fetch (newer vintage) clean. The download CACHE stays
208
203
  // vintage-partitioned below so zips don't collide across vintages.
209
204
  const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
210
- await mkdir(cacheDir, { recursive: true })
211
- await mkdir(dirname(outPath), { recursive: true })
205
+ await makeDirectories(cacheDir)
206
+ await makeDirectories(dirname(outPath))
212
207
 
213
208
  // Source units: one (per-state) for block/place; one per county for addrfeat.
214
209
  const geoCodes = level === "addrfeat" ? await discoverCounties(state, vintage) : [""]
@@ -217,11 +212,15 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
217
212
  throw new Error(`No ADDRFEAT counties found for state ${state} vintage ${vintage}`)
218
213
  }
219
214
 
220
- const db = new DatabaseSync(outPath)
221
- db.exec(TIGER_PRAGMAS)
222
- const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
215
+ const kdb = new DatabaseClient<TIGERDatabase>(outPath)
216
+ kdb.exec(TIGER_PRAGMAS)
223
217
  await initializeTIGERSchema(kdb)
224
218
 
219
+ /**
220
+ * `level` picks the table AND the row shape, but nothing ties them at the type level: {@link buildRow} returns the
221
+ * union and Kysely needs the arm's concrete row. The narrowings below are sound only because `buildRow` switches on
222
+ * the SAME `level` — keep the two switches in step, or a row shape reaches the wrong table.
223
+ */
225
224
  const insertBatch = async (rows: Row[]): Promise<void> => {
226
225
  if (level === "tabblock20") {
227
226
  await kdb
@@ -276,37 +275,21 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
276
275
  const shpPath = join(cacheDir, layer + ".shp")
277
276
  yield { phase: "extract", file: layer + ".shp" }
278
277
 
279
- const child = spawn(
280
- "ogr2ogr",
281
- [
282
- "-f",
283
- "GeoJSONSeq",
284
- "-t_srs",
285
- "EPSG:4326",
286
- "-sql",
287
- selectSQL(level, layer, options.county),
288
- "/vsistdout/",
289
- shpPath,
290
- ],
291
- { stdio: ["ignore", "pipe", "pipe"] }
292
- )
293
-
294
- let stderr = ""
295
- child.stderr.on("data", (d) => (stderr += d))
296
-
297
- const exited = new Promise<number>((resolve) => {
298
- child.on("close", (code) => resolve(code ?? 0))
299
- })
300
-
301
- // GeoJSONSeq is line-delimited; per-line `tryParsingJSON` so a malformed record is
302
- // tolerated (skipped), not thrown.
303
- for await (const line of TextSpliterator.fromAsync(child.stdout)) {
304
- if (!line) continue
305
-
306
- const feat = tryParsingJSON<{ properties?: Record<string, unknown>; geometry?: unknown }>(line)
307
-
308
- if (!feat) continue
309
-
278
+ const args = [
279
+ "-f",
280
+ "GeoJSONSeq",
281
+ "-t_srs",
282
+ "EPSG:4326",
283
+ "-sql",
284
+ selectSQL(level, layer, options.county),
285
+ "/vsistdout/",
286
+ shpPath,
287
+ ]
288
+
289
+ for await (const feat of ogr2ogrGeoJSONSeq<{ properties?: Record<string, unknown>; geometry?: unknown }>(
290
+ args,
291
+ layer
292
+ )) {
310
293
  if (!feat.properties) continue
311
294
 
312
295
  if (level === "tabblock20" && !feat.geometry) continue
@@ -319,14 +302,10 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
319
302
  }
320
303
 
321
304
  await flush()
322
-
323
- const code = await exited
324
-
325
- if (code !== 0) throw new Error(`ogr2ogr exited ${code} on ${layer}: ${stderr.slice(0, 500)}`)
326
305
  yield { phase: "load", inserted, total: 0 }
327
306
  }
328
307
 
329
- db.exec("PRAGMA wal_checkpoint(TRUNCATE);")
308
+ kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
330
309
 
331
310
  return { outPath, table, inserted }
332
311
  } finally {
@@ -0,0 +1,10 @@
1
+ /**
2
+ * @copyright Sister Software.
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+
7
+ export * from "#sdk/fetch"
8
+ export * from "#sdk/redistricting"
9
+ export * from "#sdk/schema"
10
+ export * from "#sdk/state/index"
@@ -16,22 +16,16 @@
16
16
  * https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
17
17
  */
18
18
 
19
- import { spawn } from "node:child_process"
20
- import { createWriteStream, existsSync } from "node:fs"
21
- import { mkdir, rename } from "node:fs/promises"
22
- import { dirname, join } from "node:path"
23
- import { DatabaseSync } from "node:sqlite"
24
- import { Readable } from "node:stream"
25
- import { pipeline } from "node:stream/promises"
26
-
19
+ import { mailwomanDataRoot } from "@mailwoman/core/data-root"
20
+ import { makeDirectories } from "@mailwoman/core/fs/writers"
27
21
  import { extractZipEntries } from "@mailwoman/core/fs/zip"
28
- import { DatabaseClient } from "@mailwoman/core/kysley/client"
29
- import { mailwomanDataRoot } from "@mailwoman/core/utils"
22
+ import { DatabaseClient } from "@mailwoman/sqlite/client"
23
+ import { dirname, join } from "path-ts"
30
24
  import { TextSpliterator } from "spliterator"
31
25
 
32
- import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "../state.ts"
33
- import { downloadIfNeeded } from "./download.ts"
34
- import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "./schema.ts"
26
+ import { downloadIfNeeded } from "#sdk/download"
27
+ import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "#sdk/schema"
28
+ import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "#state"
35
29
 
36
30
  const REDISTRICTING_BASE =
37
31
  "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171"
@@ -52,6 +46,60 @@ const GEO_GEOCODE = 9
52
46
  const SEG_LOGRECNO = 4
53
47
  const P2 = (fieldNo: number) => 76 + (fieldNo - 1)
54
48
 
49
+ /**
50
+ * Segment 2: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
51
+ * against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623 / 916,517, the published CA 2020
52
+ * figures. `occupied + vacant === housing_units` is the invariant that distinguishes a correct offset from a plausible
53
+ * one, and {@link parseH1} refuses a row where it does not hold.
54
+ */
55
+ export const SEG2_FIELD_COUNT = 152
56
+ /**
57
+ * H0010001 — total housing units, the third-from-last field of a segment-2 row.
58
+ */
59
+ export const H1_TOTAL = SEG2_FIELD_COUNT - 3
60
+ /**
61
+ * H0010002 — occupied housing units, the second-from-last field.
62
+ */
63
+ export const H1_OCCUPIED = SEG2_FIELD_COUNT - 2
64
+ /**
65
+ * H0010003 — vacant housing units, the last field (and so the one that carries a CRLF file's trailing CR).
66
+ */
67
+ export const H1_VACANT = SEG2_FIELD_COUNT - 1
68
+
69
+ export interface H1Counts {
70
+ housing_units: number
71
+ occupied: number
72
+ vacant: number
73
+ }
74
+
75
+ /**
76
+ * The three H1 counts from one segment-2 row. The last field carries CRLF's trailing CR when the file has one, so each
77
+ * field is trimmed before it is read as a number; a row whose counts do not add up is refused rather than stored.
78
+ */
79
+ export function parseH1(fields: readonly string[]): H1Counts {
80
+ if (fields.length !== SEG2_FIELD_COUNT) {
81
+ throw new Error(`segment 2 row has ${fields.length} fields, expected ${SEG2_FIELD_COUNT}`)
82
+ }
83
+
84
+ const counts = {
85
+ housing_units: Number(fields[H1_TOTAL]!.trim()),
86
+ occupied: Number(fields[H1_OCCUPIED]!.trim()),
87
+ vacant: Number(fields[H1_VACANT]!.trim()),
88
+ }
89
+
90
+ if (![counts.housing_units, counts.occupied, counts.vacant].every(Number.isInteger)) {
91
+ throw new Error(`segment 2 row carries a non-integer H1 count: ${fields.slice(H1_TOTAL).join("|")}`)
92
+ }
93
+
94
+ if (counts.occupied + counts.vacant !== counts.housing_units) {
95
+ throw new Error(
96
+ `segment 2 row breaks the H1 invariant: occupied ${counts.occupied} + vacant ${counts.vacant} != housing_units ${counts.housing_units} (LOGRECNO ${fields[SEG_LOGRECNO]})`
97
+ )
98
+ }
99
+
100
+ return counts
101
+ }
102
+
55
103
  /**
56
104
  * The eight P2 categories that partition the total (P0020001), in `pl_block` column order.
57
105
  */
@@ -139,8 +187,8 @@ export async function* fetchRedistricting(
139
187
  const cacheDir = join(dataRoot, "census", "redistricting", String(vintage), state)
140
188
  // Same stable `tiger.db` default as fetchTIGER — pl_block lives alongside tabblock20 in one DB.
141
189
  const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
142
- await mkdir(cacheDir, { recursive: true })
143
- await mkdir(dirname(outPath), { recursive: true })
190
+ await makeDirectories(cacheDir)
191
+ await makeDirectories(dirname(outPath))
144
192
 
145
193
  const zipName = `${fileAbbr}${vintage}.pl.zip`
146
194
  const zipPath = join(cacheDir, zipName)
@@ -152,6 +200,7 @@ export async function* fetchRedistricting(
152
200
  await extractZipEntries(zipPath, cacheDir)
153
201
  const geoPath = join(cacheDir, `${fileAbbr}geo${vintage}.pl`)
154
202
  const seg1Path = join(cacheDir, `${fileAbbr}00001${vintage}.pl`)
203
+ const seg2Path = join(cacheDir, `${fileAbbr}00002${vintage}.pl`)
155
204
  yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` }
156
205
 
157
206
  // Pass 1: header → LOGRECNO → GEOID for the blocks we want.
@@ -171,9 +220,23 @@ export async function* fetchRedistricting(
171
220
  const total = logToGeoid.size
172
221
  yield { phase: "header", blocks: total }
173
222
 
174
- const db = new DatabaseSync(outPath)
175
- db.exec(TIGER_PRAGMAS)
176
- const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
223
+ // Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs. Read before segment 1 so a block's P2 and H1
224
+ // land in one row; a mapped block with no segment-2 row is a data defect the load refuses, never a zero.
225
+ const h1ByLogrecno = new Map<string, H1Counts>()
226
+
227
+ await eachLine(seg2Path, (line) => {
228
+ const f = line.split("|")
229
+ const logrecno = f[SEG_LOGRECNO] ?? ""
230
+
231
+ if (!logToGeoid.has(logrecno)) return
232
+
233
+ h1ByLogrecno.set(logrecno, parseH1(f))
234
+ })
235
+
236
+ yield { phase: "extract", file: `${fileAbbr}00002${vintage}.pl` }
237
+
238
+ const kdb = new DatabaseClient<TIGERDatabase>(outPath)
239
+ kdb.exec(TIGER_PRAGMAS)
177
240
  await initializeTIGERSchema(kdb)
178
241
 
179
242
  try {
@@ -198,12 +261,20 @@ export async function* fetchRedistricting(
198
261
  for await (const line of TextSpliterator.fromAsync(seg1Path)) {
199
262
  if (!line) continue
200
263
  const f = line.split("|")
201
- const geoid = logToGeoid.get(f[SEG_LOGRECNO] ?? "")
264
+ const logrecno = f[SEG_LOGRECNO] ?? ""
265
+ const geoid = logToGeoid.get(logrecno)
202
266
 
203
267
  if (!geoid) continue
204
268
 
269
+ const h1 = h1ByLogrecno.get(logrecno)
270
+
271
+ if (!h1) {
272
+ throw new Error(`block ${geoid} (LOGRECNO ${logrecno}) has a segment-1 row and no segment-2 row`)
273
+ }
274
+
205
275
  batch.push({
206
276
  GEOID: geoid,
277
+ ...h1,
207
278
  pop_total: Number(f[CATEGORY_INDEX.pop_total] ?? 0),
208
279
  hispanic: Number(f[CATEGORY_INDEX.hispanic] ?? 0),
209
280
  white: Number(f[CATEGORY_INDEX.white] ?? 0),
@@ -224,7 +295,7 @@ export async function* fetchRedistricting(
224
295
  await flush()
225
296
 
226
297
  yield { phase: "load", inserted, total }
227
- db.exec("PRAGMA wal_checkpoint(TRUNCATE);")
298
+ kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
228
299
 
229
300
  return { outPath, table: "pl_block", inserted }
230
301
  } finally {
@@ -46,6 +46,18 @@ export interface PLBlockTable {
46
46
  nhpi: number
47
47
  other: number
48
48
  multi: number
49
+ /**
50
+ * P.L. 94-171 table H1 — total housing units in the block.
51
+ */
52
+ housing_units: number
53
+ /**
54
+ * H1 occupied. `occupied + vacant === housing_units` by construction; the reader refuses a row where it is not.
55
+ */
56
+ occupied: number
57
+ /**
58
+ * H1 vacant.
59
+ */
60
+ vacant: number
49
61
  }
50
62
 
51
63
  /**
@@ -106,6 +118,25 @@ PRAGMA journal_mode = WAL;
106
118
  * The `text(N)` length hints in the prior raw DDL were documentary only (SQLite uses TEXT affinity regardless); the
107
119
  * lengths live on the {@link TIGERBlockTable} interface instead.
108
120
  */
121
+ /**
122
+ * Refuse a `pl_block` built before the H1 columns existed. `createTable(...).ifNotExists()` leaves an older table as it
123
+ * is, and the next load would fail on the first insert with a message about a column, not about the cause. The table is
124
+ * derived (the redistricting command reloads it state by state), so the remedy is a rebuild: drop it and re-run
125
+ * `mailwoman tiger redistricting` for each state you hold. Databases are never patched in place.
126
+ */
127
+ async function assertPLBlockShape(db: Kysely<TIGERDatabase>): Promise<void> {
128
+ const columns = await sql<{ name: string }>`select name from pragma_table_info('pl_block')`.execute(db)
129
+ const names = new Set(columns.rows.map((row) => row.name))
130
+
131
+ for (const required of ["housing_units", "occupied", "vacant"]) {
132
+ if (!names.has(required)) {
133
+ throw new Error(
134
+ `pl_block predates its H1 columns (missing ${required}). It is a derived table: drop it and re-run \`mailwoman tiger redistricting\` for each state, which reloads P2 and H1 together.`
135
+ )
136
+ }
137
+ }
138
+ }
139
+
109
140
  export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<void> {
110
141
  await db.schema
111
142
  .createTable("us_state")
@@ -177,11 +208,16 @@ export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<
177
208
  .addColumn("nhpi", "integer", (c) => c.notNull())
178
209
  .addColumn("other", "integer", (c) => c.notNull())
179
210
  .addColumn("multi", "integer", (c) => c.notNull())
211
+ .addColumn("housing_units", "integer", (c) => c.notNull())
212
+ .addColumn("occupied", "integer", (c) => c.notNull())
213
+ .addColumn("vacant", "integer", (c) => c.notNull())
180
214
  // pl_block is small (no geometry) and always probed by its GEOID PK (1:1 join to tabblock20), so
181
215
  // cluster it WITHOUT ROWID — one B-tree probe per join, no separate rowid + PK-index pair.
182
216
  .modifyEnd(sql`without rowid`)
183
217
  .execute()
184
218
 
219
+ await assertPLBlockShape(db)
220
+
185
221
  await db.schema
186
222
  .createTable("tiger_streets")
187
223
  .ifNotExists()
@@ -5,7 +5,8 @@
5
5
  */
6
6
 
7
7
  import { smartSnakeCase } from "@mailwoman/core"
8
- import { TIGERProperty } from "@mailwoman/tiger"
8
+
9
+ import { TIGERProperty } from "#index"
9
10
 
10
11
  /**
11
12
  * Default column selections for TIGER tabulated block entities.
@@ -4,7 +4,7 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { AdminLevel1Code, FIPSCountyCode, FIPSTractCode } from "@mailwoman/tiger"
7
+ import type { AdminLevel1Code, FIPSCountyCode, FIPSTractCode } from "#index"
8
8
 
9
9
  /**
10
10
  * @internal
@@ -5,7 +5,8 @@
5
5
  */
6
6
 
7
7
  import type { GeometryLiteral, MultiPolygonLiteral } from "@mailwoman/spatial"
8
- import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/sdk"
8
+ import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/well-known-text"
9
+
9
10
  import type {
10
11
  FIPSStateCode,
11
12
  FIPSBlockGeoID,
@@ -13,7 +14,7 @@ import type {
13
14
  TIGERBlockFeature,
14
15
  TIGERBlockFeatureCollection,
15
16
  TIGERTabulatedBlockProperties,
16
- } from "@mailwoman/tiger"
17
+ } from "#index"
17
18
 
18
19
  export interface StateBlockIntersectionCriteria {
19
20
  stateCode: FIPSStateCode
@@ -0,0 +1,9 @@
1
+ /**
2
+ * @copyright Sister Software.
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+
7
+ export * from "#sdk/state/block-operations"
8
+ export * from "#sdk/state/TIGERTabulatedBlockEntity"
9
+ export * from "#sdk/state/TIGERTractEntity"
@@ -6,8 +6,8 @@
6
6
 
7
7
  import type { GeoFeature, MultiPolygonLiteral } from "@mailwoman/spatial"
8
8
 
9
- import type { GeoIDPart } from "./geoid.ts"
10
- import type { TIGERTabulatedBlockProperties } from "./tabulation-block.ts"
9
+ import type { GeoIDPart } from "#geoid"
10
+ import type { TIGERTabulatedBlockProperties } from "#tabulation-block"
11
11
 
12
12
  //#region FIPS Codes
13
13
 
@@ -6,8 +6,8 @@
6
6
 
7
7
  import type { MultiPolygonLiteral } from "@mailwoman/spatial"
8
8
 
9
- import type { TIGERProperty, TIGERPropertyRecord } from "./constants.ts"
10
- import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "./geoid.ts"
9
+ import type { TIGERProperty, TIGERPropertyRecord } from "#constants"
10
+ import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "#geoid"
11
11
 
12
12
  /**
13
13
  * The properties of a tabulated block from the TIGER/Line dataset.