@mailwoman/tiger 9.2.0 → 9.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/{block-group.ts → lib/block-group.ts} +4 -4
  2. package/{class-code.ts → lib/class-code.ts} +1 -1
  3. package/{constants.ts → lib/constants.ts} +5 -5
  4. package/{county-subdivision.ts → lib/county-subdivision.ts} +4 -4
  5. package/{county.ts → lib/county.ts} +4 -4
  6. package/{files.ts → lib/files.ts} +1 -1
  7. package/{geoid.ts → lib/geoid.ts} +3 -4
  8. package/lib/index.ts +18 -0
  9. package/{sdk → lib/sdk}/download.ts +6 -8
  10. package/{sdk → lib/sdk}/fetch.ts +28 -53
  11. package/lib/sdk/index.ts +10 -0
  12. package/{sdk → lib/sdk}/redistricting.ts +91 -16
  13. package/{sdk → lib/sdk}/schema.ts +36 -0
  14. package/{sdk → lib/sdk}/state/block-operations.ts +1 -1
  15. package/lib/sdk/state/index.ts +9 -0
  16. package/{state.ts → lib/state.ts} +2 -2
  17. package/{tabulation-block.ts → lib/tabulation-block.ts} +2 -2
  18. package/{tools → lib/tools}/index.ts +3 -3
  19. package/{tools → lib/tools}/race-dots-map.ts +4 -5
  20. package/{tools → lib/tools}/race-dots.ts +7 -8
  21. package/{tools → lib/tools}/serve-range.ts +18 -12
  22. package/{tract.ts → lib/tract.ts} +2 -2
  23. package/out/block-group.d.ts +4 -4
  24. package/out/block-group.d.ts.map +1 -1
  25. package/out/block-group.js.map +1 -1
  26. package/out/class-code.d.ts +1 -1
  27. package/out/class-code.d.ts.map +1 -1
  28. package/out/class-code.js +1 -1
  29. package/out/class-code.js.map +1 -1
  30. package/out/constants.d.ts +5 -5
  31. package/out/constants.d.ts.map +1 -1
  32. package/out/constants.js.map +1 -1
  33. package/out/county-subdivision.d.ts +4 -4
  34. package/out/county-subdivision.d.ts.map +1 -1
  35. package/out/county-subdivision.js.map +1 -1
  36. package/out/county.d.ts +4 -4
  37. package/out/county.d.ts.map +1 -1
  38. package/out/county.js.map +1 -1
  39. package/out/files.d.ts +1 -1
  40. package/out/files.d.ts.map +1 -1
  41. package/out/files.js.map +1 -1
  42. package/out/functional-status.d.ts.map +1 -1
  43. package/out/functional-status.js.map +1 -1
  44. package/out/geoid.d.ts +1 -2
  45. package/out/geoid.d.ts.map +1 -1
  46. package/out/geoid.js.map +1 -1
  47. package/out/index.d.ts +12 -12
  48. package/out/index.d.ts.map +1 -1
  49. package/out/index.js +12 -12
  50. package/out/index.js.map +1 -1
  51. package/out/legal-statistical-area.d.ts.map +1 -1
  52. package/out/legal-statistical-area.js.map +1 -1
  53. package/out/sdk/download.d.ts.map +1 -1
  54. package/out/sdk/download.js +6 -7
  55. package/out/sdk/download.js.map +1 -1
  56. package/out/sdk/fetch.d.ts.map +1 -1
  57. package/out/sdk/fetch.js +15 -34
  58. package/out/sdk/fetch.js.map +1 -1
  59. package/out/sdk/index.d.ts +4 -4
  60. package/out/sdk/index.d.ts.map +1 -1
  61. package/out/sdk/index.js +4 -4
  62. package/out/sdk/index.js.map +1 -1
  63. package/out/sdk/redistricting.d.ts +29 -0
  64. package/out/sdk/redistricting.d.ts.map +1 -1
  65. package/out/sdk/redistricting.js +71 -15
  66. package/out/sdk/redistricting.js.map +1 -1
  67. package/out/sdk/schema.d.ts +12 -9
  68. package/out/sdk/schema.d.ts.map +1 -1
  69. package/out/sdk/schema.js +19 -0
  70. package/out/sdk/schema.js.map +1 -1
  71. package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts.map +1 -1
  72. package/out/sdk/state/TIGERTabulatedBlockEntity.js.map +1 -1
  73. package/out/sdk/state/TIGERTractEntity.d.ts.map +1 -1
  74. package/out/sdk/state/TIGERTractEntity.js.map +1 -1
  75. package/out/sdk/state/block-operations.d.ts.map +1 -1
  76. package/out/sdk/state/block-operations.js +1 -1
  77. package/out/sdk/state/block-operations.js.map +1 -1
  78. package/out/sdk/state/index.d.ts +3 -3
  79. package/out/sdk/state/index.d.ts.map +1 -1
  80. package/out/sdk/state/index.js +3 -3
  81. package/out/sdk/state/index.js.map +1 -1
  82. package/out/state.d.ts +2 -2
  83. package/out/state.d.ts.map +1 -1
  84. package/out/state.js.map +1 -1
  85. package/out/tabulation-block.d.ts +2 -2
  86. package/out/tabulation-block.d.ts.map +1 -1
  87. package/out/tabulation-block.js.map +1 -1
  88. package/out/tools/index.d.ts +3 -3
  89. package/out/tools/index.d.ts.map +1 -1
  90. package/out/tools/index.js +3 -3
  91. package/out/tools/index.js.map +1 -1
  92. package/out/tools/race-dots-map.d.ts.map +1 -1
  93. package/out/tools/race-dots-map.js +4 -4
  94. package/out/tools/race-dots-map.js.map +1 -1
  95. package/out/tools/race-dots.d.ts.map +1 -1
  96. package/out/tools/race-dots.js +6 -7
  97. package/out/tools/race-dots.js.map +1 -1
  98. package/out/tools/serve-range.d.ts +2 -3
  99. package/out/tools/serve-range.d.ts.map +1 -1
  100. package/out/tools/serve-range.js +16 -10
  101. package/out/tools/serve-range.js.map +1 -1
  102. package/out/tract.d.ts +2 -2
  103. package/out/tract.d.ts.map +1 -1
  104. package/out/tract.js.map +1 -1
  105. package/out/tsconfig.test.tsbuildinfo +1 -0
  106. package/package.json +38 -7
  107. package/index.ts +0 -18
  108. package/out/tsconfig.tsbuildinfo +0 -1
  109. package/sdk/index.ts +0 -10
  110. package/sdk/state/index.ts +0 -9
  111. /package/{functional-status.ts → lib/functional-status.ts} +0 -0
  112. /package/{legal-statistical-area.ts → lib/legal-statistical-area.ts} +0 -0
  113. /package/{sdk → lib/sdk}/state/TIGERTabulatedBlockEntity.ts +0 -0
  114. /package/{sdk → lib/sdk}/state/TIGERTractEntity.ts +0 -0
@@ -4,10 +4,10 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
9
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
10
- import type { AdminLevel1Code } from "./state.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
10
+ import type { AdminLevel1Code } from "#state"
11
11
 
12
12
  /**
13
13
  * @title TIGER Block Group
@@ -59,7 +59,7 @@ export const TIGERClassCode = {
59
59
  /**
60
60
  * A movable barrier across a road.
61
61
  */
62
- Gate: "C3066",
62
+ Check: "C3066",
63
63
 
64
64
  /**
65
65
  * A structure or barrier where a fee is collected for using a road.
@@ -4,8 +4,8 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
9
  import type {
10
10
  GeoIDPart,
11
11
  FIPSBlockCode,
@@ -15,9 +15,9 @@ import type {
15
15
  FIPSCountySubDivisionCode,
16
16
  FIPSPlaceCode,
17
17
  FIPSTractCode,
18
- } from "./geoid.ts"
19
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
20
- import type { FIPSStateCode } from "./state.ts"
18
+ } from "#geoid"
19
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
20
+ import type { FIPSStateCode } from "#state"
21
21
 
22
22
  /**
23
23
  * A code indicating the classification of the block, i.e. urban or rural.
@@ -4,10 +4,10 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
9
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
10
- import type { AdminLevel1Code } from "./state.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
10
+ import type { AdminLevel1Code } from "#state"
11
11
 
12
12
  /**
13
13
  * Census Geographic subdivision of a county.
@@ -4,10 +4,10 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  */
6
6
 
7
- import type { TIGERClassCode } from "./class-code.ts"
8
- import type { TIGERFunctionalStatus } from "./functional-status.ts"
9
- import type { LegalStatisticalAreaDescription } from "./legal-statistical-area.ts"
10
- import type { AdminLevel1Code } from "./state.ts"
7
+ import type { TIGERClassCode } from "#class-code"
8
+ import type { TIGERFunctionalStatus } from "#functional-status"
9
+ import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
10
+ import type { AdminLevel1Code } from "#state"
11
11
 
12
12
  /**
13
13
  * @title TIGER County
@@ -6,7 +6,7 @@
6
6
  * TIGER Census data utilities.
7
7
  */
8
8
 
9
- import type { AdminLevel1Code } from "./state.ts"
9
+ import type { AdminLevel1Code } from "#state"
10
10
 
11
11
  //#region Constants
12
12
 
@@ -6,7 +6,7 @@
6
6
 
7
7
  import type { Tagged } from "type-fest"
8
8
 
9
- import type { AdminLevel1Code, FIPSStateCode } from "./state.ts"
9
+ import type { AdminLevel1Code, FIPSStateCode } from "#state"
10
10
 
11
11
  //#region Tagged FIPS types
12
12
 
@@ -301,7 +301,6 @@ export interface ParsedGeoIDBlockLevel {
301
301
  [GeoIDPart.Place]: FIPSPlaceCode
302
302
  [GeoIDPart.BlockGroup]: FIPSBlockGroupCode
303
303
  [GeoIDPart.Block]: FIPSBlockCode
304
- [GeoIDPart.BlockGroup]: FIPSBlockGroupCode
305
304
  [GeoIDPart.CongressionalDistrict]: FIPSCongressionalDistrictCode
306
305
  }
307
306
 
@@ -509,10 +508,10 @@ export const GeoIDInputMatchers = {
509
508
  * @see {@linkcode GeoIDInputMatchers} for the input-level matchers.
510
509
  */
511
510
  const GeoIDComponentMatchers = Object.fromEntries(
512
- Object.values(GeoIDPart).map((value) => {
511
+ Object.values(GeoIDPart).map((value): [GeoIDPart, GeoIDInputMatcher] => {
513
512
  return [value, new GeoIDInputMatcher(value)]
514
513
  })
515
- ) as unknown as Record<GeoIDPart, GeoIDInputMatcher>
514
+ ) as Record<GeoIDPart, GeoIDInputMatcher>
516
515
 
517
516
  /**
518
517
  * Type-predicate for checking if a value appears to be a valid GeoID component.
package/lib/index.ts ADDED
@@ -0,0 +1,18 @@
1
+ /**
2
+ * @copyright Sister Software.
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+
7
+ export * from "#block-group"
8
+ export * from "#class-code"
9
+ export * from "#constants"
10
+ export * from "#county-subdivision"
11
+ export * from "#county"
12
+ export * from "#files"
13
+ export * from "#functional-status"
14
+ export * from "#geoid"
15
+ export * from "#legal-statistical-area"
16
+ export * from "#state"
17
+ export * from "#tabulation-block"
18
+ export * from "#tract"
@@ -14,11 +14,9 @@
14
14
  * never holds it: a web stream piped to a write stream.
15
15
  */
16
16
 
17
- import { createWriteStream, existsSync } from "node:fs"
18
- import { rename } from "node:fs/promises"
19
- import { Readable } from "node:stream"
20
- import { pipeline } from "node:stream/promises"
21
-
17
+ import { pathExists } from "@mailwoman/core/fs/readers"
18
+ import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
19
+ import { movePath } from "@mailwoman/core/fs/writers"
22
20
  import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
23
21
 
24
22
  /**
@@ -30,7 +28,7 @@ import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
30
28
  * renames, so `dest` is never a partial archive.
31
29
  */
32
30
  export async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
33
- if (existsSync(dest)) {
31
+ if (await pathExists(dest)) {
34
32
  try {
35
33
  await verifyZipIntegrity(dest)
36
34
 
@@ -44,8 +42,8 @@ export async function downloadIfNeeded(url: string, dest: string): Promise<boole
44
42
  const res = await fetch(url, { redirect: "follow" })
45
43
 
46
44
  if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
47
- await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), createWriteStream(tmp))
48
- await rename(tmp, dest)
45
+ await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), openWriteStream(tmp))
46
+ await movePath(tmp, dest)
49
47
 
50
48
  return false
51
49
  }
@@ -24,21 +24,17 @@
24
24
  * inserts. Re-running a state replaces its rows.
25
25
  */
26
26
 
27
- import { spawn } from "node:child_process"
28
- import { mkdir } from "node:fs/promises"
29
- import { dirname, join } from "node:path"
30
- import { DatabaseSync } from "node:sqlite"
31
-
32
27
  import { APIClient, pluckResponseData } from "@mailwoman/core/api"
28
+ import { mailwomanDataRoot } from "@mailwoman/core/data-root"
29
+ import { makeDirectories } from "@mailwoman/core/fs/writers"
33
30
  import { extractZipEntries } from "@mailwoman/core/fs/zip"
34
- import { DatabaseClient } from "@mailwoman/core/kysley/client"
35
- import { tryParsingJSON } from "@mailwoman/core/objects"
36
- import { mailwomanDataRoot } from "@mailwoman/core/utils"
37
- import { TextSpliterator } from "spliterator"
31
+ import { ogr2ogrGeoJSONSeq } from "@mailwoman/spatial/tools/ogr-stream"
32
+ import { DatabaseClient } from "@mailwoman/sqlite/client"
33
+ import { dirname, join } from "path-ts"
38
34
 
39
- import { downloadIfNeeded } from "./download.ts"
40
- import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "./schema.ts"
41
- import { initializeTIGERSchema, TIGER_PRAGMAS } from "./schema.ts"
35
+ import { downloadIfNeeded } from "#sdk/download"
36
+ import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "#sdk/schema"
37
+ import { initializeTIGERSchema, TIGER_PRAGMAS } from "#sdk/schema"
42
38
 
43
39
  const CENSUS_HOST = "https://www2.census.gov"
44
40
  const DEFAULT_DATA_ROOT = mailwomanDataRoot()
@@ -206,8 +202,8 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
206
202
  // the per-table idempotent delete keeps a re-fetch (newer vintage) clean. The download CACHE stays
207
203
  // vintage-partitioned below so zips don't collide across vintages.
208
204
  const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
209
- await mkdir(cacheDir, { recursive: true })
210
- await mkdir(dirname(outPath), { recursive: true })
205
+ await makeDirectories(cacheDir)
206
+ await makeDirectories(dirname(outPath))
211
207
 
212
208
  // Source units: one (per-state) for block/place; one per county for addrfeat.
213
209
  const geoCodes = level === "addrfeat" ? await discoverCounties(state, vintage) : [""]
@@ -216,9 +212,8 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
216
212
  throw new Error(`No ADDRFEAT counties found for state ${state} vintage ${vintage}`)
217
213
  }
218
214
 
219
- const db = new DatabaseSync(outPath)
220
- db.exec(TIGER_PRAGMAS)
221
- const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
215
+ const kdb = new DatabaseClient<TIGERDatabase>(outPath)
216
+ kdb.exec(TIGER_PRAGMAS)
222
217
  await initializeTIGERSchema(kdb)
223
218
 
224
219
  /**
@@ -280,37 +275,21 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
280
275
  const shpPath = join(cacheDir, layer + ".shp")
281
276
  yield { phase: "extract", file: layer + ".shp" }
282
277
 
283
- const child = spawn(
284
- "ogr2ogr",
285
- [
286
- "-f",
287
- "GeoJSONSeq",
288
- "-t_srs",
289
- "EPSG:4326",
290
- "-sql",
291
- selectSQL(level, layer, options.county),
292
- "/vsistdout/",
293
- shpPath,
294
- ],
295
- { stdio: ["ignore", "pipe", "pipe"] }
296
- )
297
-
298
- let stderr = ""
299
- child.stderr.on("data", (d) => (stderr += d))
300
-
301
- const exited = new Promise<number>((resolve) => {
302
- child.on("close", (code) => resolve(code ?? 0))
303
- })
304
-
305
- // GeoJSONSeq is line-delimited; per-line `tryParsingJSON` so a malformed record is
306
- // tolerated (skipped), not thrown.
307
- for await (const line of TextSpliterator.fromAsync(child.stdout)) {
308
- if (!line) continue
309
-
310
- const feat = tryParsingJSON<{ properties?: Record<string, unknown>; geometry?: unknown }>(line)
311
-
312
- if (!feat) continue
313
-
278
+ const args = [
279
+ "-f",
280
+ "GeoJSONSeq",
281
+ "-t_srs",
282
+ "EPSG:4326",
283
+ "-sql",
284
+ selectSQL(level, layer, options.county),
285
+ "/vsistdout/",
286
+ shpPath,
287
+ ]
288
+
289
+ for await (const feat of ogr2ogrGeoJSONSeq<{ properties?: Record<string, unknown>; geometry?: unknown }>(
290
+ args,
291
+ layer
292
+ )) {
314
293
  if (!feat.properties) continue
315
294
 
316
295
  if (level === "tabblock20" && !feat.geometry) continue
@@ -323,14 +302,10 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
323
302
  }
324
303
 
325
304
  await flush()
326
-
327
- const code = await exited
328
-
329
- if (code !== 0) throw new Error(`ogr2ogr exited ${code} on ${layer}: ${stderr.slice(0, 500)}`)
330
305
  yield { phase: "load", inserted, total: 0 }
331
306
  }
332
307
 
333
- db.exec("PRAGMA wal_checkpoint(TRUNCATE);")
308
+ kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
334
309
 
335
310
  return { outPath, table, inserted }
336
311
  } finally {
@@ -0,0 +1,10 @@
1
+ /**
2
+ * @copyright Sister Software.
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+
7
+ export * from "#sdk/fetch"
8
+ export * from "#sdk/redistricting"
9
+ export * from "#sdk/schema"
10
+ export * from "#sdk/state/index"
@@ -16,18 +16,16 @@
16
16
  * https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
17
17
  */
18
18
 
19
- import { mkdir } from "node:fs/promises"
20
- import { dirname, join } from "node:path"
21
- import { DatabaseSync } from "node:sqlite"
22
-
19
+ import { mailwomanDataRoot } from "@mailwoman/core/data-root"
20
+ import { makeDirectories } from "@mailwoman/core/fs/writers"
23
21
  import { extractZipEntries } from "@mailwoman/core/fs/zip"
24
- import { DatabaseClient } from "@mailwoman/core/kysley/client"
25
- import { mailwomanDataRoot } from "@mailwoman/core/utils"
22
+ import { DatabaseClient } from "@mailwoman/sqlite/client"
23
+ import { dirname, join } from "path-ts"
26
24
  import { TextSpliterator } from "spliterator"
27
25
 
28
- import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "../state.ts"
29
- import { downloadIfNeeded } from "./download.ts"
30
- import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "./schema.ts"
26
+ import { downloadIfNeeded } from "#sdk/download"
27
+ import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "#sdk/schema"
28
+ import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "#state"
31
29
 
32
30
  const REDISTRICTING_BASE =
33
31
  "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171"
@@ -48,6 +46,60 @@ const GEO_GEOCODE = 9
48
46
  const SEG_LOGRECNO = 4
49
47
  const P2 = (fieldNo: number) => 76 + (fieldNo - 1)
50
48
 
49
+ /**
50
+ * Segment 2: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
51
+ * against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623 / 916,517, the published CA 2020
52
+ * figures. `occupied + vacant === housing_units` is the invariant that distinguishes a correct offset from a plausible
53
+ * one, and {@link parseH1} refuses a row where it does not hold.
54
+ */
55
+ export const SEG2_FIELD_COUNT = 152
56
+ /**
57
+ * H0010001 — total housing units, the third-from-last field of a segment-2 row.
58
+ */
59
+ export const H1_TOTAL = SEG2_FIELD_COUNT - 3
60
+ /**
61
+ * H0010002 — occupied housing units, the second-from-last field.
62
+ */
63
+ export const H1_OCCUPIED = SEG2_FIELD_COUNT - 2
64
+ /**
65
+ * H0010003 — vacant housing units, the last field (and so the one that carries a CRLF file's trailing CR).
66
+ */
67
+ export const H1_VACANT = SEG2_FIELD_COUNT - 1
68
+
69
+ export interface H1Counts {
70
+ housing_units: number
71
+ occupied: number
72
+ vacant: number
73
+ }
74
+
75
+ /**
76
+ * The three H1 counts from one segment-2 row. The last field carries CRLF's trailing CR when the file has one, so each
77
+ * field is trimmed before it is read as a number; a row whose counts do not add up is refused rather than stored.
78
+ */
79
+ export function parseH1(fields: readonly string[]): H1Counts {
80
+ if (fields.length !== SEG2_FIELD_COUNT) {
81
+ throw new Error(`segment 2 row has ${fields.length} fields, expected ${SEG2_FIELD_COUNT}`)
82
+ }
83
+
84
+ const counts = {
85
+ housing_units: Number(fields[H1_TOTAL]!.trim()),
86
+ occupied: Number(fields[H1_OCCUPIED]!.trim()),
87
+ vacant: Number(fields[H1_VACANT]!.trim()),
88
+ }
89
+
90
+ if (![counts.housing_units, counts.occupied, counts.vacant].every(Number.isInteger)) {
91
+ throw new Error(`segment 2 row carries a non-integer H1 count: ${fields.slice(H1_TOTAL).join("|")}`)
92
+ }
93
+
94
+ if (counts.occupied + counts.vacant !== counts.housing_units) {
95
+ throw new Error(
96
+ `segment 2 row breaks the H1 invariant: occupied ${counts.occupied} + vacant ${counts.vacant} != housing_units ${counts.housing_units} (LOGRECNO ${fields[SEG_LOGRECNO]})`
97
+ )
98
+ }
99
+
100
+ return counts
101
+ }
102
+
51
103
  /**
52
104
  * The eight P2 categories that partition the total (P0020001), in `pl_block` column order.
53
105
  */
@@ -135,8 +187,8 @@ export async function* fetchRedistricting(
135
187
  const cacheDir = join(dataRoot, "census", "redistricting", String(vintage), state)
136
188
  // Same stable `tiger.db` default as fetchTIGER — pl_block lives alongside tabblock20 in one DB.
137
189
  const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
138
- await mkdir(cacheDir, { recursive: true })
139
- await mkdir(dirname(outPath), { recursive: true })
190
+ await makeDirectories(cacheDir)
191
+ await makeDirectories(dirname(outPath))
140
192
 
141
193
  const zipName = `${fileAbbr}${vintage}.pl.zip`
142
194
  const zipPath = join(cacheDir, zipName)
@@ -148,6 +200,7 @@ export async function* fetchRedistricting(
148
200
  await extractZipEntries(zipPath, cacheDir)
149
201
  const geoPath = join(cacheDir, `${fileAbbr}geo${vintage}.pl`)
150
202
  const seg1Path = join(cacheDir, `${fileAbbr}00001${vintage}.pl`)
203
+ const seg2Path = join(cacheDir, `${fileAbbr}00002${vintage}.pl`)
151
204
  yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` }
152
205
 
153
206
  // Pass 1: header → LOGRECNO → GEOID for the blocks we want.
@@ -167,9 +220,23 @@ export async function* fetchRedistricting(
167
220
  const total = logToGeoid.size
168
221
  yield { phase: "header", blocks: total }
169
222
 
170
- const db = new DatabaseSync(outPath)
171
- db.exec(TIGER_PRAGMAS)
172
- const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
223
+ // Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs. Read before segment 1 so a block's P2 and H1
224
+ // land in one row; a mapped block with no segment-2 row is a data defect the load refuses, never a zero.
225
+ const h1ByLogrecno = new Map<string, H1Counts>()
226
+
227
+ await eachLine(seg2Path, (line) => {
228
+ const f = line.split("|")
229
+ const logrecno = f[SEG_LOGRECNO] ?? ""
230
+
231
+ if (!logToGeoid.has(logrecno)) return
232
+
233
+ h1ByLogrecno.set(logrecno, parseH1(f))
234
+ })
235
+
236
+ yield { phase: "extract", file: `${fileAbbr}00002${vintage}.pl` }
237
+
238
+ const kdb = new DatabaseClient<TIGERDatabase>(outPath)
239
+ kdb.exec(TIGER_PRAGMAS)
173
240
  await initializeTIGERSchema(kdb)
174
241
 
175
242
  try {
@@ -194,12 +261,20 @@ export async function* fetchRedistricting(
194
261
  for await (const line of TextSpliterator.fromAsync(seg1Path)) {
195
262
  if (!line) continue
196
263
  const f = line.split("|")
197
- const geoid = logToGeoid.get(f[SEG_LOGRECNO] ?? "")
264
+ const logrecno = f[SEG_LOGRECNO] ?? ""
265
+ const geoid = logToGeoid.get(logrecno)
198
266
 
199
267
  if (!geoid) continue
200
268
 
269
+ const h1 = h1ByLogrecno.get(logrecno)
270
+
271
+ if (!h1) {
272
+ throw new Error(`block ${geoid} (LOGRECNO ${logrecno}) has a segment-1 row and no segment-2 row`)
273
+ }
274
+
201
275
  batch.push({
202
276
  GEOID: geoid,
277
+ ...h1,
203
278
  pop_total: Number(f[CATEGORY_INDEX.pop_total] ?? 0),
204
279
  hispanic: Number(f[CATEGORY_INDEX.hispanic] ?? 0),
205
280
  white: Number(f[CATEGORY_INDEX.white] ?? 0),
@@ -220,7 +295,7 @@ export async function* fetchRedistricting(
220
295
  await flush()
221
296
 
222
297
  yield { phase: "load", inserted, total }
223
- db.exec("PRAGMA wal_checkpoint(TRUNCATE);")
298
+ kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
224
299
 
225
300
  return { outPath, table: "pl_block", inserted }
226
301
  } finally {
@@ -46,6 +46,18 @@ export interface PLBlockTable {
46
46
  nhpi: number
47
47
  other: number
48
48
  multi: number
49
+ /**
50
+ * P.L. 94-171 table H1 — total housing units in the block.
51
+ */
52
+ housing_units: number
53
+ /**
54
+ * H1 occupied. `occupied + vacant === housing_units` by construction; the reader refuses a row where it is not.
55
+ */
56
+ occupied: number
57
+ /**
58
+ * H1 vacant.
59
+ */
60
+ vacant: number
49
61
  }
50
62
 
51
63
  /**
@@ -106,6 +118,25 @@ PRAGMA journal_mode = WAL;
106
118
  * The `text(N)` length hints in the prior raw DDL were documentary only (SQLite uses TEXT affinity regardless); the
107
119
  * lengths live on the {@link TIGERBlockTable} interface instead.
108
120
  */
121
+ /**
122
+ * Refuse a `pl_block` built before the H1 columns existed. `createTable(...).ifNotExists()` leaves an older table as it
123
+ * is, and the next load would fail on the first insert with a message about a column, not about the cause. The table is
124
+ * derived (the redistricting command reloads it state by state), so the remedy is a rebuild: drop it and re-run
125
+ * `mailwoman tiger redistricting` for each state you hold. Databases are never patched in place.
126
+ */
127
+ async function assertPLBlockShape(db: Kysely<TIGERDatabase>): Promise<void> {
128
+ const columns = await sql<{ name: string }>`select name from pragma_table_info('pl_block')`.execute(db)
129
+ const names = new Set(columns.rows.map((row) => row.name))
130
+
131
+ for (const required of ["housing_units", "occupied", "vacant"]) {
132
+ if (!names.has(required)) {
133
+ throw new Error(
134
+ `pl_block predates its H1 columns (missing ${required}). It is a derived table: drop it and re-run \`mailwoman tiger redistricting\` for each state, which reloads P2 and H1 together.`
135
+ )
136
+ }
137
+ }
138
+ }
139
+
109
140
  export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<void> {
110
141
  await db.schema
111
142
  .createTable("us_state")
@@ -177,11 +208,16 @@ export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<
177
208
  .addColumn("nhpi", "integer", (c) => c.notNull())
178
209
  .addColumn("other", "integer", (c) => c.notNull())
179
210
  .addColumn("multi", "integer", (c) => c.notNull())
211
+ .addColumn("housing_units", "integer", (c) => c.notNull())
212
+ .addColumn("occupied", "integer", (c) => c.notNull())
213
+ .addColumn("vacant", "integer", (c) => c.notNull())
180
214
  // pl_block is small (no geometry) and always probed by its GEOID PK (1:1 join to tabblock20), so
181
215
  // cluster it WITHOUT ROWID — one B-tree probe per join, no separate rowid + PK-index pair.
182
216
  .modifyEnd(sql`without rowid`)
183
217
  .execute()
184
218
 
219
+ await assertPLBlockShape(db)
220
+
185
221
  await db.schema
186
222
  .createTable("tiger_streets")
187
223
  .ifNotExists()
@@ -5,7 +5,7 @@
5
5
  */
6
6
 
7
7
  import type { GeometryLiteral, MultiPolygonLiteral } from "@mailwoman/spatial"
8
- import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/sdk"
8
+ import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/well-known-text"
9
9
 
10
10
  import type {
11
11
  FIPSStateCode,
@@ -0,0 +1,9 @@
1
+ /**
2
+ * @copyright Sister Software.
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ */
6
+
7
+ export * from "#sdk/state/block-operations"
8
+ export * from "#sdk/state/TIGERTabulatedBlockEntity"
9
+ export * from "#sdk/state/TIGERTractEntity"
@@ -6,8 +6,8 @@
6
6
 
7
7
  import type { GeoFeature, MultiPolygonLiteral } from "@mailwoman/spatial"
8
8
 
9
- import type { GeoIDPart } from "./geoid.ts"
10
- import type { TIGERTabulatedBlockProperties } from "./tabulation-block.ts"
9
+ import type { GeoIDPart } from "#geoid"
10
+ import type { TIGERTabulatedBlockProperties } from "#tabulation-block"
11
11
 
12
12
  //#region FIPS Codes
13
13
 
@@ -6,8 +6,8 @@
6
6
 
7
7
  import type { MultiPolygonLiteral } from "@mailwoman/spatial"
8
8
 
9
- import type { TIGERProperty, TIGERPropertyRecord } from "./constants.ts"
10
- import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "./geoid.ts"
9
+ import type { TIGERProperty, TIGERPropertyRecord } from "#constants"
10
+ import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "#geoid"
11
11
 
12
12
  /**
13
13
  * The properties of a tabulated block from the TIGER/Line dataset.
@@ -6,6 +6,6 @@
6
6
  * TIGER operator tool exports.
7
7
  */
8
8
 
9
- export * from "./race-dots.ts"
10
- export * from "./race-dots-map.ts"
11
- export * from "./serve-range.ts"
9
+ export * from "#tools/race-dots"
10
+ export * from "#tools/race-dots-map"
11
+ export * from "#tools/serve-range"
@@ -17,9 +17,8 @@
17
17
  * http://localhost:8899/race-dots-oc.pmtiles --out /tmp/race-dots-oc.html`
18
18
  */
19
19
 
20
- import { writeFileSync } from "node:fs"
21
-
22
- import { tempRootPath } from "@mailwoman/core/utils"
20
+ import { tempRootPath } from "@mailwoman/core/data-root"
21
+ import { writeLocalFile } from "@mailwoman/core/fs/writers"
23
22
 
24
23
  /**
25
24
  * Options for {@linkcode raceDotsMap}.
@@ -69,7 +68,7 @@ const MAPLIBRE_CSS_SRI = "sha384-uTttxo/aOKbdE5RlD/SPzSDoDmNvGlUYPjONi2MN/b7c9HP
69
68
  const PMTILES_VERSION = "4.4.1"
70
69
 
71
70
  const BASEMAP_SOURCE_ID = "basemap-v4"
72
- const BASEMAP_TILEJSON_URL = "https://tiles.sister.software/basemap-v4.json"
71
+ const BASEMAP_TILEJSON_URL = "https://tiles.mailwoman.ai/basemap-v4.json"
73
72
  const GLYPHS_URL = "https://protomaps.github.io/basemaps-assets/fonts/{fontstack}/{range}.pbf"
74
73
  const SPRITE_URL = "https://protomaps.github.io/basemaps-assets/sprites/v4/light"
75
74
 
@@ -194,7 +193,7 @@ export async function raceDotsMap(
194
193
  </html>
195
194
  `
196
195
 
197
- writeFileSync(OUT, html)
196
+ await writeLocalFile(html, OUT)
198
197
  report?.(`[written] ${OUT} (pmtiles: ${PMTILES_URL})`)
199
198
 
200
199
  return { outPath: OUT, pmtilesURL: PMTILES_URL }
@@ -21,11 +21,12 @@
21
21
  * Run: `mailwoman tiger race-dots --db tiger-oc.db --per 10 --out /tmp/race-dots.ndjson`
22
22
  */
23
23
 
24
- import { createWriteStream } from "node:fs"
25
- import { DatabaseSync } from "node:sqlite"
24
+ import { dataRootPath, tempRootPath } from "@mailwoman/core/data-root"
25
+ import { openWriteStream } from "@mailwoman/core/fs/streams"
26
+ import { tryParsingJSON } from "@mailwoman/core/json"
27
+ import { DatabaseClient } from "@mailwoman/sqlite/client"
26
28
 
27
- import { tryParsingJSON } from "@mailwoman/core/objects"
28
- import { dataRootPath, tempRootPath } from "@mailwoman/core/utils"
29
+ import type { TIGERDatabase } from "#sdk/schema"
29
30
 
30
31
  /**
31
32
  * Attempts to place a dot inside its polygon by rejection sampling before giving up on it.
@@ -150,7 +151,7 @@ export async function raceDots(
150
151
  return null
151
152
  }
152
153
 
153
- const db = new DatabaseSync(DB, { readOnly: true })
154
+ using db = new DatabaseClient<TIGERDatabase>(DB, { readOnly: true })
154
155
 
155
156
  const rows = db
156
157
  .prepare(
@@ -160,9 +161,7 @@ export async function raceDots(
160
161
  )
161
162
  .all() as Array<{ geometry: string } & Record<(typeof CATEGORIES)[number], number>>
162
163
 
163
- db.close()
164
-
165
- const out = createWriteStream(OUT)
164
+ const out = openWriteStream(OUT)
166
165
  const totals = new Map<string, number>()
167
166
 
168
167
  let dots = 0,