@mailwoman/tiger 9.2.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{block-group.ts → lib/block-group.ts} +4 -4
- package/{class-code.ts → lib/class-code.ts} +1 -1
- package/{constants.ts → lib/constants.ts} +5 -5
- package/{county-subdivision.ts → lib/county-subdivision.ts} +4 -4
- package/{county.ts → lib/county.ts} +4 -4
- package/{files.ts → lib/files.ts} +1 -1
- package/{geoid.ts → lib/geoid.ts} +3 -4
- package/lib/index.ts +18 -0
- package/{sdk → lib/sdk}/download.ts +6 -8
- package/{sdk → lib/sdk}/fetch.ts +28 -53
- package/lib/sdk/index.ts +10 -0
- package/{sdk → lib/sdk}/redistricting.ts +91 -16
- package/{sdk → lib/sdk}/schema.ts +36 -0
- package/{sdk → lib/sdk}/state/block-operations.ts +1 -1
- package/lib/sdk/state/index.ts +9 -0
- package/{state.ts → lib/state.ts} +2 -2
- package/{tabulation-block.ts → lib/tabulation-block.ts} +2 -2
- package/{tools → lib/tools}/index.ts +3 -3
- package/{tools → lib/tools}/race-dots-map.ts +4 -5
- package/{tools → lib/tools}/race-dots.ts +7 -8
- package/{tools → lib/tools}/serve-range.ts +18 -12
- package/{tract.ts → lib/tract.ts} +2 -2
- package/out/block-group.d.ts +4 -4
- package/out/block-group.d.ts.map +1 -1
- package/out/block-group.js.map +1 -1
- package/out/class-code.d.ts +1 -1
- package/out/class-code.d.ts.map +1 -1
- package/out/class-code.js +1 -1
- package/out/class-code.js.map +1 -1
- package/out/constants.d.ts +5 -5
- package/out/constants.d.ts.map +1 -1
- package/out/constants.js.map +1 -1
- package/out/county-subdivision.d.ts +4 -4
- package/out/county-subdivision.d.ts.map +1 -1
- package/out/county-subdivision.js.map +1 -1
- package/out/county.d.ts +4 -4
- package/out/county.d.ts.map +1 -1
- package/out/county.js.map +1 -1
- package/out/files.d.ts +1 -1
- package/out/files.d.ts.map +1 -1
- package/out/files.js.map +1 -1
- package/out/functional-status.d.ts.map +1 -1
- package/out/functional-status.js.map +1 -1
- package/out/geoid.d.ts +1 -2
- package/out/geoid.d.ts.map +1 -1
- package/out/geoid.js.map +1 -1
- package/out/index.d.ts +12 -12
- package/out/index.d.ts.map +1 -1
- package/out/index.js +12 -12
- package/out/index.js.map +1 -1
- package/out/legal-statistical-area.d.ts.map +1 -1
- package/out/legal-statistical-area.js.map +1 -1
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +6 -7
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/fetch.d.ts.map +1 -1
- package/out/sdk/fetch.js +15 -34
- package/out/sdk/fetch.js.map +1 -1
- package/out/sdk/index.d.ts +4 -4
- package/out/sdk/index.d.ts.map +1 -1
- package/out/sdk/index.js +4 -4
- package/out/sdk/index.js.map +1 -1
- package/out/sdk/redistricting.d.ts +29 -0
- package/out/sdk/redistricting.d.ts.map +1 -1
- package/out/sdk/redistricting.js +71 -15
- package/out/sdk/redistricting.js.map +1 -1
- package/out/sdk/schema.d.ts +12 -9
- package/out/sdk/schema.d.ts.map +1 -1
- package/out/sdk/schema.js +19 -0
- package/out/sdk/schema.js.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.js.map +1 -1
- package/out/sdk/state/TIGERTractEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTractEntity.js.map +1 -1
- package/out/sdk/state/block-operations.d.ts.map +1 -1
- package/out/sdk/state/block-operations.js +1 -1
- package/out/sdk/state/block-operations.js.map +1 -1
- package/out/sdk/state/index.d.ts +3 -3
- package/out/sdk/state/index.d.ts.map +1 -1
- package/out/sdk/state/index.js +3 -3
- package/out/sdk/state/index.js.map +1 -1
- package/out/state.d.ts +2 -2
- package/out/state.d.ts.map +1 -1
- package/out/state.js.map +1 -1
- package/out/tabulation-block.d.ts +2 -2
- package/out/tabulation-block.d.ts.map +1 -1
- package/out/tabulation-block.js.map +1 -1
- package/out/tools/index.d.ts +3 -3
- package/out/tools/index.d.ts.map +1 -1
- package/out/tools/index.js +3 -3
- package/out/tools/index.js.map +1 -1
- package/out/tools/race-dots-map.d.ts.map +1 -1
- package/out/tools/race-dots-map.js +4 -4
- package/out/tools/race-dots-map.js.map +1 -1
- package/out/tools/race-dots.d.ts.map +1 -1
- package/out/tools/race-dots.js +6 -7
- package/out/tools/race-dots.js.map +1 -1
- package/out/tools/serve-range.d.ts +2 -3
- package/out/tools/serve-range.d.ts.map +1 -1
- package/out/tools/serve-range.js +16 -10
- package/out/tools/serve-range.js.map +1 -1
- package/out/tract.d.ts +2 -2
- package/out/tract.d.ts.map +1 -1
- package/out/tract.js.map +1 -1
- package/out/tsconfig.test.tsbuildinfo +1 -0
- package/package.json +38 -7
- package/index.ts +0 -18
- package/out/tsconfig.tsbuildinfo +0 -1
- package/sdk/index.ts +0 -10
- package/sdk/state/index.ts +0 -9
- /package/{functional-status.ts → lib/functional-status.ts} +0 -0
- /package/{legal-statistical-area.ts → lib/legal-statistical-area.ts} +0 -0
- /package/{sdk → lib/sdk}/state/TIGERTabulatedBlockEntity.ts +0 -0
- /package/{sdk → lib/sdk}/state/TIGERTractEntity.ts +0 -0
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
9
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
10
|
-
import type { AdminLevel1Code } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
10
|
+
import type { AdminLevel1Code } from "#state"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* @title TIGER Block Group
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
9
|
import type {
|
|
10
10
|
GeoIDPart,
|
|
11
11
|
FIPSBlockCode,
|
|
@@ -15,9 +15,9 @@ import type {
|
|
|
15
15
|
FIPSCountySubDivisionCode,
|
|
16
16
|
FIPSPlaceCode,
|
|
17
17
|
FIPSTractCode,
|
|
18
|
-
} from "
|
|
19
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
20
|
-
import type { FIPSStateCode } from "
|
|
18
|
+
} from "#geoid"
|
|
19
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
20
|
+
import type { FIPSStateCode } from "#state"
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* A code indicating the classification of the block, i.e. urban or rural.
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
9
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
10
|
-
import type { AdminLevel1Code } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
10
|
+
import type { AdminLevel1Code } from "#state"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Census Geographic subdivision of a county.
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
9
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
10
|
-
import type { AdminLevel1Code } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
10
|
+
import type { AdminLevel1Code } from "#state"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* @title TIGER County
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import type { Tagged } from "type-fest"
|
|
8
8
|
|
|
9
|
-
import type { AdminLevel1Code, FIPSStateCode } from "
|
|
9
|
+
import type { AdminLevel1Code, FIPSStateCode } from "#state"
|
|
10
10
|
|
|
11
11
|
//#region Tagged FIPS types
|
|
12
12
|
|
|
@@ -301,7 +301,6 @@ export interface ParsedGeoIDBlockLevel {
|
|
|
301
301
|
[GeoIDPart.Place]: FIPSPlaceCode
|
|
302
302
|
[GeoIDPart.BlockGroup]: FIPSBlockGroupCode
|
|
303
303
|
[GeoIDPart.Block]: FIPSBlockCode
|
|
304
|
-
[GeoIDPart.BlockGroup]: FIPSBlockGroupCode
|
|
305
304
|
[GeoIDPart.CongressionalDistrict]: FIPSCongressionalDistrictCode
|
|
306
305
|
}
|
|
307
306
|
|
|
@@ -509,10 +508,10 @@ export const GeoIDInputMatchers = {
|
|
|
509
508
|
* @see {@linkcode GeoIDInputMatchers} for the input-level matchers.
|
|
510
509
|
*/
|
|
511
510
|
const GeoIDComponentMatchers = Object.fromEntries(
|
|
512
|
-
Object.values(GeoIDPart).map((value) => {
|
|
511
|
+
Object.values(GeoIDPart).map((value): [GeoIDPart, GeoIDInputMatcher] => {
|
|
513
512
|
return [value, new GeoIDInputMatcher(value)]
|
|
514
513
|
})
|
|
515
|
-
) as
|
|
514
|
+
) as Record<GeoIDPart, GeoIDInputMatcher>
|
|
516
515
|
|
|
517
516
|
/**
|
|
518
517
|
* Type-predicate for checking if a value appears to be a valid GeoID component.
|
package/lib/index.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software.
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export * from "#block-group"
|
|
8
|
+
export * from "#class-code"
|
|
9
|
+
export * from "#constants"
|
|
10
|
+
export * from "#county-subdivision"
|
|
11
|
+
export * from "#county"
|
|
12
|
+
export * from "#files"
|
|
13
|
+
export * from "#functional-status"
|
|
14
|
+
export * from "#geoid"
|
|
15
|
+
export * from "#legal-statistical-area"
|
|
16
|
+
export * from "#state"
|
|
17
|
+
export * from "#tabulation-block"
|
|
18
|
+
export * from "#tract"
|
|
@@ -14,11 +14,9 @@
|
|
|
14
14
|
* never holds it: a web stream piped to a write stream.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import { pipeline } from "node:stream/promises"
|
|
21
|
-
|
|
17
|
+
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
18
|
+
import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
|
|
19
|
+
import { movePath } from "@mailwoman/core/fs/writers"
|
|
22
20
|
import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
|
|
23
21
|
|
|
24
22
|
/**
|
|
@@ -30,7 +28,7 @@ import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
|
|
|
30
28
|
* renames, so `dest` is never a partial archive.
|
|
31
29
|
*/
|
|
32
30
|
export async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
|
|
33
|
-
if (
|
|
31
|
+
if (await pathExists(dest)) {
|
|
34
32
|
try {
|
|
35
33
|
await verifyZipIntegrity(dest)
|
|
36
34
|
|
|
@@ -44,8 +42,8 @@ export async function downloadIfNeeded(url: string, dest: string): Promise<boole
|
|
|
44
42
|
const res = await fetch(url, { redirect: "follow" })
|
|
45
43
|
|
|
46
44
|
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
|
|
47
|
-
await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]),
|
|
48
|
-
await
|
|
45
|
+
await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), openWriteStream(tmp))
|
|
46
|
+
await movePath(tmp, dest)
|
|
49
47
|
|
|
50
48
|
return false
|
|
51
49
|
}
|
package/{sdk → lib/sdk}/fetch.ts
RENAMED
|
@@ -24,21 +24,17 @@
|
|
|
24
24
|
* inserts. Re-running a state replaces its rows.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import { spawn } from "node:child_process"
|
|
28
|
-
import { mkdir } from "node:fs/promises"
|
|
29
|
-
import { dirname, join } from "node:path"
|
|
30
|
-
import { DatabaseSync } from "node:sqlite"
|
|
31
|
-
|
|
32
27
|
import { APIClient, pluckResponseData } from "@mailwoman/core/api"
|
|
28
|
+
import { mailwomanDataRoot } from "@mailwoman/core/data-root"
|
|
29
|
+
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
33
30
|
import { extractZipEntries } from "@mailwoman/core/fs/zip"
|
|
34
|
-
import {
|
|
35
|
-
import {
|
|
36
|
-
import {
|
|
37
|
-
import { TextSpliterator } from "spliterator"
|
|
31
|
+
import { ogr2ogrGeoJSONSeq } from "@mailwoman/spatial/tools/ogr-stream"
|
|
32
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
33
|
+
import { dirname, join } from "path-ts"
|
|
38
34
|
|
|
39
|
-
import { downloadIfNeeded } from "
|
|
40
|
-
import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "
|
|
41
|
-
import { initializeTIGERSchema, TIGER_PRAGMAS } from "
|
|
35
|
+
import { downloadIfNeeded } from "#sdk/download"
|
|
36
|
+
import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "#sdk/schema"
|
|
37
|
+
import { initializeTIGERSchema, TIGER_PRAGMAS } from "#sdk/schema"
|
|
42
38
|
|
|
43
39
|
const CENSUS_HOST = "https://www2.census.gov"
|
|
44
40
|
const DEFAULT_DATA_ROOT = mailwomanDataRoot()
|
|
@@ -206,8 +202,8 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
206
202
|
// the per-table idempotent delete keeps a re-fetch (newer vintage) clean. The download CACHE stays
|
|
207
203
|
// vintage-partitioned below so zips don't collide across vintages.
|
|
208
204
|
const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
|
|
209
|
-
await
|
|
210
|
-
await
|
|
205
|
+
await makeDirectories(cacheDir)
|
|
206
|
+
await makeDirectories(dirname(outPath))
|
|
211
207
|
|
|
212
208
|
// Source units: one (per-state) for block/place; one per county for addrfeat.
|
|
213
209
|
const geoCodes = level === "addrfeat" ? await discoverCounties(state, vintage) : [""]
|
|
@@ -216,9 +212,8 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
216
212
|
throw new Error(`No ADDRFEAT counties found for state ${state} vintage ${vintage}`)
|
|
217
213
|
}
|
|
218
214
|
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
|
|
215
|
+
const kdb = new DatabaseClient<TIGERDatabase>(outPath)
|
|
216
|
+
kdb.exec(TIGER_PRAGMAS)
|
|
222
217
|
await initializeTIGERSchema(kdb)
|
|
223
218
|
|
|
224
219
|
/**
|
|
@@ -280,37 +275,21 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
280
275
|
const shpPath = join(cacheDir, layer + ".shp")
|
|
281
276
|
yield { phase: "extract", file: layer + ".shp" }
|
|
282
277
|
|
|
283
|
-
const
|
|
284
|
-
"
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
let stderr = ""
|
|
299
|
-
child.stderr.on("data", (d) => (stderr += d))
|
|
300
|
-
|
|
301
|
-
const exited = new Promise<number>((resolve) => {
|
|
302
|
-
child.on("close", (code) => resolve(code ?? 0))
|
|
303
|
-
})
|
|
304
|
-
|
|
305
|
-
// GeoJSONSeq is line-delimited; per-line `tryParsingJSON` so a malformed record is
|
|
306
|
-
// tolerated (skipped), not thrown.
|
|
307
|
-
for await (const line of TextSpliterator.fromAsync(child.stdout)) {
|
|
308
|
-
if (!line) continue
|
|
309
|
-
|
|
310
|
-
const feat = tryParsingJSON<{ properties?: Record<string, unknown>; geometry?: unknown }>(line)
|
|
311
|
-
|
|
312
|
-
if (!feat) continue
|
|
313
|
-
|
|
278
|
+
const args = [
|
|
279
|
+
"-f",
|
|
280
|
+
"GeoJSONSeq",
|
|
281
|
+
"-t_srs",
|
|
282
|
+
"EPSG:4326",
|
|
283
|
+
"-sql",
|
|
284
|
+
selectSQL(level, layer, options.county),
|
|
285
|
+
"/vsistdout/",
|
|
286
|
+
shpPath,
|
|
287
|
+
]
|
|
288
|
+
|
|
289
|
+
for await (const feat of ogr2ogrGeoJSONSeq<{ properties?: Record<string, unknown>; geometry?: unknown }>(
|
|
290
|
+
args,
|
|
291
|
+
layer
|
|
292
|
+
)) {
|
|
314
293
|
if (!feat.properties) continue
|
|
315
294
|
|
|
316
295
|
if (level === "tabblock20" && !feat.geometry) continue
|
|
@@ -323,14 +302,10 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
323
302
|
}
|
|
324
303
|
|
|
325
304
|
await flush()
|
|
326
|
-
|
|
327
|
-
const code = await exited
|
|
328
|
-
|
|
329
|
-
if (code !== 0) throw new Error(`ogr2ogr exited ${code} on ${layer}: ${stderr.slice(0, 500)}`)
|
|
330
305
|
yield { phase: "load", inserted, total: 0 }
|
|
331
306
|
}
|
|
332
307
|
|
|
333
|
-
|
|
308
|
+
kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
|
334
309
|
|
|
335
310
|
return { outPath, table, inserted }
|
|
336
311
|
} finally {
|
package/lib/sdk/index.ts
ADDED
|
@@ -16,18 +16,16 @@
|
|
|
16
16
|
* https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import { DatabaseSync } from "node:sqlite"
|
|
22
|
-
|
|
19
|
+
import { mailwomanDataRoot } from "@mailwoman/core/data-root"
|
|
20
|
+
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
23
21
|
import { extractZipEntries } from "@mailwoman/core/fs/zip"
|
|
24
|
-
import { DatabaseClient } from "@mailwoman/
|
|
25
|
-
import {
|
|
22
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
23
|
+
import { dirname, join } from "path-ts"
|
|
26
24
|
import { TextSpliterator } from "spliterator"
|
|
27
25
|
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import {
|
|
26
|
+
import { downloadIfNeeded } from "#sdk/download"
|
|
27
|
+
import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "#sdk/schema"
|
|
28
|
+
import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "#state"
|
|
31
29
|
|
|
32
30
|
const REDISTRICTING_BASE =
|
|
33
31
|
"https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171"
|
|
@@ -48,6 +46,60 @@ const GEO_GEOCODE = 9
|
|
|
48
46
|
const SEG_LOGRECNO = 4
|
|
49
47
|
const P2 = (fieldNo: number) => 76 + (fieldNo - 1)
|
|
50
48
|
|
|
49
|
+
/**
|
|
50
|
+
* Segment 2: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
|
|
51
|
+
* against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623 / 916,517, the published CA 2020
|
|
52
|
+
* figures. `occupied + vacant === housing_units` is the invariant that distinguishes a correct offset from a plausible
|
|
53
|
+
* one, and {@link parseH1} refuses a row where it does not hold.
|
|
54
|
+
*/
|
|
55
|
+
export const SEG2_FIELD_COUNT = 152
|
|
56
|
+
/**
|
|
57
|
+
* H0010001 — total housing units, the third-from-last field of a segment-2 row.
|
|
58
|
+
*/
|
|
59
|
+
export const H1_TOTAL = SEG2_FIELD_COUNT - 3
|
|
60
|
+
/**
|
|
61
|
+
* H0010002 — occupied housing units, the second-from-last field.
|
|
62
|
+
*/
|
|
63
|
+
export const H1_OCCUPIED = SEG2_FIELD_COUNT - 2
|
|
64
|
+
/**
|
|
65
|
+
* H0010003 — vacant housing units, the last field (and so the one that carries a CRLF file's trailing CR).
|
|
66
|
+
*/
|
|
67
|
+
export const H1_VACANT = SEG2_FIELD_COUNT - 1
|
|
68
|
+
|
|
69
|
+
export interface H1Counts {
|
|
70
|
+
housing_units: number
|
|
71
|
+
occupied: number
|
|
72
|
+
vacant: number
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The three H1 counts from one segment-2 row. The last field carries CRLF's trailing CR when the file has one, so each
|
|
77
|
+
* field is trimmed before it is read as a number; a row whose counts do not add up is refused rather than stored.
|
|
78
|
+
*/
|
|
79
|
+
export function parseH1(fields: readonly string[]): H1Counts {
|
|
80
|
+
if (fields.length !== SEG2_FIELD_COUNT) {
|
|
81
|
+
throw new Error(`segment 2 row has ${fields.length} fields, expected ${SEG2_FIELD_COUNT}`)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const counts = {
|
|
85
|
+
housing_units: Number(fields[H1_TOTAL]!.trim()),
|
|
86
|
+
occupied: Number(fields[H1_OCCUPIED]!.trim()),
|
|
87
|
+
vacant: Number(fields[H1_VACANT]!.trim()),
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (![counts.housing_units, counts.occupied, counts.vacant].every(Number.isInteger)) {
|
|
91
|
+
throw new Error(`segment 2 row carries a non-integer H1 count: ${fields.slice(H1_TOTAL).join("|")}`)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (counts.occupied + counts.vacant !== counts.housing_units) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
`segment 2 row breaks the H1 invariant: occupied ${counts.occupied} + vacant ${counts.vacant} != housing_units ${counts.housing_units} (LOGRECNO ${fields[SEG_LOGRECNO]})`
|
|
97
|
+
)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return counts
|
|
101
|
+
}
|
|
102
|
+
|
|
51
103
|
/**
|
|
52
104
|
* The eight P2 categories that partition the total (P0020001), in `pl_block` column order.
|
|
53
105
|
*/
|
|
@@ -135,8 +187,8 @@ export async function* fetchRedistricting(
|
|
|
135
187
|
const cacheDir = join(dataRoot, "census", "redistricting", String(vintage), state)
|
|
136
188
|
// Same stable `tiger.db` default as fetchTIGER — pl_block lives alongside tabblock20 in one DB.
|
|
137
189
|
const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
|
|
138
|
-
await
|
|
139
|
-
await
|
|
190
|
+
await makeDirectories(cacheDir)
|
|
191
|
+
await makeDirectories(dirname(outPath))
|
|
140
192
|
|
|
141
193
|
const zipName = `${fileAbbr}${vintage}.pl.zip`
|
|
142
194
|
const zipPath = join(cacheDir, zipName)
|
|
@@ -148,6 +200,7 @@ export async function* fetchRedistricting(
|
|
|
148
200
|
await extractZipEntries(zipPath, cacheDir)
|
|
149
201
|
const geoPath = join(cacheDir, `${fileAbbr}geo${vintage}.pl`)
|
|
150
202
|
const seg1Path = join(cacheDir, `${fileAbbr}00001${vintage}.pl`)
|
|
203
|
+
const seg2Path = join(cacheDir, `${fileAbbr}00002${vintage}.pl`)
|
|
151
204
|
yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` }
|
|
152
205
|
|
|
153
206
|
// Pass 1: header → LOGRECNO → GEOID for the blocks we want.
|
|
@@ -167,9 +220,23 @@ export async function* fetchRedistricting(
|
|
|
167
220
|
const total = logToGeoid.size
|
|
168
221
|
yield { phase: "header", blocks: total }
|
|
169
222
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
const
|
|
223
|
+
// Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs. Read before segment 1 so a block's P2 and H1
|
|
224
|
+
// land in one row; a mapped block with no segment-2 row is a data defect the load refuses, never a zero.
|
|
225
|
+
const h1ByLogrecno = new Map<string, H1Counts>()
|
|
226
|
+
|
|
227
|
+
await eachLine(seg2Path, (line) => {
|
|
228
|
+
const f = line.split("|")
|
|
229
|
+
const logrecno = f[SEG_LOGRECNO] ?? ""
|
|
230
|
+
|
|
231
|
+
if (!logToGeoid.has(logrecno)) return
|
|
232
|
+
|
|
233
|
+
h1ByLogrecno.set(logrecno, parseH1(f))
|
|
234
|
+
})
|
|
235
|
+
|
|
236
|
+
yield { phase: "extract", file: `${fileAbbr}00002${vintage}.pl` }
|
|
237
|
+
|
|
238
|
+
const kdb = new DatabaseClient<TIGERDatabase>(outPath)
|
|
239
|
+
kdb.exec(TIGER_PRAGMAS)
|
|
173
240
|
await initializeTIGERSchema(kdb)
|
|
174
241
|
|
|
175
242
|
try {
|
|
@@ -194,12 +261,20 @@ export async function* fetchRedistricting(
|
|
|
194
261
|
for await (const line of TextSpliterator.fromAsync(seg1Path)) {
|
|
195
262
|
if (!line) continue
|
|
196
263
|
const f = line.split("|")
|
|
197
|
-
const
|
|
264
|
+
const logrecno = f[SEG_LOGRECNO] ?? ""
|
|
265
|
+
const geoid = logToGeoid.get(logrecno)
|
|
198
266
|
|
|
199
267
|
if (!geoid) continue
|
|
200
268
|
|
|
269
|
+
const h1 = h1ByLogrecno.get(logrecno)
|
|
270
|
+
|
|
271
|
+
if (!h1) {
|
|
272
|
+
throw new Error(`block ${geoid} (LOGRECNO ${logrecno}) has a segment-1 row and no segment-2 row`)
|
|
273
|
+
}
|
|
274
|
+
|
|
201
275
|
batch.push({
|
|
202
276
|
GEOID: geoid,
|
|
277
|
+
...h1,
|
|
203
278
|
pop_total: Number(f[CATEGORY_INDEX.pop_total] ?? 0),
|
|
204
279
|
hispanic: Number(f[CATEGORY_INDEX.hispanic] ?? 0),
|
|
205
280
|
white: Number(f[CATEGORY_INDEX.white] ?? 0),
|
|
@@ -220,7 +295,7 @@ export async function* fetchRedistricting(
|
|
|
220
295
|
await flush()
|
|
221
296
|
|
|
222
297
|
yield { phase: "load", inserted, total }
|
|
223
|
-
|
|
298
|
+
kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
|
224
299
|
|
|
225
300
|
return { outPath, table: "pl_block", inserted }
|
|
226
301
|
} finally {
|
|
@@ -46,6 +46,18 @@ export interface PLBlockTable {
|
|
|
46
46
|
nhpi: number
|
|
47
47
|
other: number
|
|
48
48
|
multi: number
|
|
49
|
+
/**
|
|
50
|
+
* P.L. 94-171 table H1 — total housing units in the block.
|
|
51
|
+
*/
|
|
52
|
+
housing_units: number
|
|
53
|
+
/**
|
|
54
|
+
* H1 occupied. `occupied + vacant === housing_units` by construction; the reader refuses a row where it is not.
|
|
55
|
+
*/
|
|
56
|
+
occupied: number
|
|
57
|
+
/**
|
|
58
|
+
* H1 vacant.
|
|
59
|
+
*/
|
|
60
|
+
vacant: number
|
|
49
61
|
}
|
|
50
62
|
|
|
51
63
|
/**
|
|
@@ -106,6 +118,25 @@ PRAGMA journal_mode = WAL;
|
|
|
106
118
|
* The `text(N)` length hints in the prior raw DDL were documentary only (SQLite uses TEXT affinity regardless); the
|
|
107
119
|
* lengths live on the {@link TIGERBlockTable} interface instead.
|
|
108
120
|
*/
|
|
121
|
+
/**
|
|
122
|
+
* Refuse a `pl_block` built before the H1 columns existed. `createTable(...).ifNotExists()` leaves an older table as it
|
|
123
|
+
* is, and the next load would fail on the first insert with a message about a column, not about the cause. The table is
|
|
124
|
+
* derived (the redistricting command reloads it state by state), so the remedy is a rebuild: drop it and re-run
|
|
125
|
+
* `mailwoman tiger redistricting` for each state you hold. Databases are never patched in place.
|
|
126
|
+
*/
|
|
127
|
+
async function assertPLBlockShape(db: Kysely<TIGERDatabase>): Promise<void> {
|
|
128
|
+
const columns = await sql<{ name: string }>`select name from pragma_table_info('pl_block')`.execute(db)
|
|
129
|
+
const names = new Set(columns.rows.map((row) => row.name))
|
|
130
|
+
|
|
131
|
+
for (const required of ["housing_units", "occupied", "vacant"]) {
|
|
132
|
+
if (!names.has(required)) {
|
|
133
|
+
throw new Error(
|
|
134
|
+
`pl_block predates its H1 columns (missing ${required}). It is a derived table: drop it and re-run \`mailwoman tiger redistricting\` for each state, which reloads P2 and H1 together.`
|
|
135
|
+
)
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
109
140
|
export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<void> {
|
|
110
141
|
await db.schema
|
|
111
142
|
.createTable("us_state")
|
|
@@ -177,11 +208,16 @@ export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<
|
|
|
177
208
|
.addColumn("nhpi", "integer", (c) => c.notNull())
|
|
178
209
|
.addColumn("other", "integer", (c) => c.notNull())
|
|
179
210
|
.addColumn("multi", "integer", (c) => c.notNull())
|
|
211
|
+
.addColumn("housing_units", "integer", (c) => c.notNull())
|
|
212
|
+
.addColumn("occupied", "integer", (c) => c.notNull())
|
|
213
|
+
.addColumn("vacant", "integer", (c) => c.notNull())
|
|
180
214
|
// pl_block is small (no geometry) and always probed by its GEOID PK (1:1 join to tabblock20), so
|
|
181
215
|
// cluster it WITHOUT ROWID — one B-tree probe per join, no separate rowid + PK-index pair.
|
|
182
216
|
.modifyEnd(sql`without rowid`)
|
|
183
217
|
.execute()
|
|
184
218
|
|
|
219
|
+
await assertPLBlockShape(db)
|
|
220
|
+
|
|
185
221
|
await db.schema
|
|
186
222
|
.createTable("tiger_streets")
|
|
187
223
|
.ifNotExists()
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import type { GeometryLiteral, MultiPolygonLiteral } from "@mailwoman/spatial"
|
|
8
|
-
import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/
|
|
8
|
+
import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/well-known-text"
|
|
9
9
|
|
|
10
10
|
import type {
|
|
11
11
|
FIPSStateCode,
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
|
|
7
7
|
import type { GeoFeature, MultiPolygonLiteral } from "@mailwoman/spatial"
|
|
8
8
|
|
|
9
|
-
import type { GeoIDPart } from "
|
|
10
|
-
import type { TIGERTabulatedBlockProperties } from "
|
|
9
|
+
import type { GeoIDPart } from "#geoid"
|
|
10
|
+
import type { TIGERTabulatedBlockProperties } from "#tabulation-block"
|
|
11
11
|
|
|
12
12
|
//#region FIPS Codes
|
|
13
13
|
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
|
|
7
7
|
import type { MultiPolygonLiteral } from "@mailwoman/spatial"
|
|
8
8
|
|
|
9
|
-
import type { TIGERProperty, TIGERPropertyRecord } from "
|
|
10
|
-
import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "
|
|
9
|
+
import type { TIGERProperty, TIGERPropertyRecord } from "#constants"
|
|
10
|
+
import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "#geoid"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* The properties of a tabulated block from the TIGER/Line dataset.
|
|
@@ -17,9 +17,8 @@
|
|
|
17
17
|
* http://localhost:8899/race-dots-oc.pmtiles --out /tmp/race-dots-oc.html`
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
import { tempRootPath } from "@mailwoman/core/utils"
|
|
20
|
+
import { tempRootPath } from "@mailwoman/core/data-root"
|
|
21
|
+
import { writeLocalFile } from "@mailwoman/core/fs/writers"
|
|
23
22
|
|
|
24
23
|
/**
|
|
25
24
|
* Options for {@linkcode raceDotsMap}.
|
|
@@ -69,7 +68,7 @@ const MAPLIBRE_CSS_SRI = "sha384-uTttxo/aOKbdE5RlD/SPzSDoDmNvGlUYPjONi2MN/b7c9HP
|
|
|
69
68
|
const PMTILES_VERSION = "4.4.1"
|
|
70
69
|
|
|
71
70
|
const BASEMAP_SOURCE_ID = "basemap-v4"
|
|
72
|
-
const BASEMAP_TILEJSON_URL = "https://tiles.
|
|
71
|
+
const BASEMAP_TILEJSON_URL = "https://tiles.mailwoman.ai/basemap-v4.json"
|
|
73
72
|
const GLYPHS_URL = "https://protomaps.github.io/basemaps-assets/fonts/{fontstack}/{range}.pbf"
|
|
74
73
|
const SPRITE_URL = "https://protomaps.github.io/basemaps-assets/sprites/v4/light"
|
|
75
74
|
|
|
@@ -194,7 +193,7 @@ export async function raceDotsMap(
|
|
|
194
193
|
</html>
|
|
195
194
|
`
|
|
196
195
|
|
|
197
|
-
|
|
196
|
+
await writeLocalFile(html, OUT)
|
|
198
197
|
report?.(`[written] ${OUT} (pmtiles: ${PMTILES_URL})`)
|
|
199
198
|
|
|
200
199
|
return { outPath: OUT, pmtilesURL: PMTILES_URL }
|
|
@@ -21,11 +21,12 @@
|
|
|
21
21
|
* Run: `mailwoman tiger race-dots --db tiger-oc.db --per 10 --out /tmp/race-dots.ndjson`
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
24
|
+
import { dataRootPath, tempRootPath } from "@mailwoman/core/data-root"
|
|
25
|
+
import { openWriteStream } from "@mailwoman/core/fs/streams"
|
|
26
|
+
import { tryParsingJSON } from "@mailwoman/core/json"
|
|
27
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
26
28
|
|
|
27
|
-
import {
|
|
28
|
-
import { dataRootPath, tempRootPath } from "@mailwoman/core/utils"
|
|
29
|
+
import type { TIGERDatabase } from "#sdk/schema"
|
|
29
30
|
|
|
30
31
|
/**
|
|
31
32
|
* Attempts to place a dot inside its polygon by rejection sampling before giving up on it.
|
|
@@ -150,7 +151,7 @@ export async function raceDots(
|
|
|
150
151
|
return null
|
|
151
152
|
}
|
|
152
153
|
|
|
153
|
-
|
|
154
|
+
using db = new DatabaseClient<TIGERDatabase>(DB, { readOnly: true })
|
|
154
155
|
|
|
155
156
|
const rows = db
|
|
156
157
|
.prepare(
|
|
@@ -160,9 +161,7 @@ export async function raceDots(
|
|
|
160
161
|
)
|
|
161
162
|
.all() as Array<{ geometry: string } & Record<(typeof CATEGORIES)[number], number>>
|
|
162
163
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
const out = createWriteStream(OUT)
|
|
164
|
+
const out = openWriteStream(OUT)
|
|
166
165
|
const totals = new Map<string, number>()
|
|
167
166
|
|
|
168
167
|
let dots = 0,
|