@mailwoman/tiger 9.1.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{block-group.ts → lib/block-group.ts} +4 -4
- package/{class-code.ts → lib/class-code.ts} +1 -1
- package/{constants.ts → lib/constants.ts} +5 -5
- package/{county-subdivision.ts → lib/county-subdivision.ts} +4 -4
- package/{county.ts → lib/county.ts} +4 -4
- package/{files.ts → lib/files.ts} +1 -1
- package/{geoid.ts → lib/geoid.ts} +3 -4
- package/lib/index.ts +18 -0
- package/{sdk → lib/sdk}/download.ts +6 -30
- package/{sdk → lib/sdk}/fetch.ts +38 -59
- package/lib/sdk/index.ts +10 -0
- package/{sdk → lib/sdk}/redistricting.ts +91 -20
- package/{sdk → lib/sdk}/schema.ts +36 -0
- package/{sdk → lib/sdk}/state/TIGERTabulatedBlockEntity.ts +2 -1
- package/{sdk → lib/sdk}/state/TIGERTractEntity.ts +1 -1
- package/{sdk → lib/sdk}/state/block-operations.ts +3 -2
- package/lib/sdk/state/index.ts +9 -0
- package/{state.ts → lib/state.ts} +2 -2
- package/{tabulation-block.ts → lib/tabulation-block.ts} +2 -2
- package/lib/tools/index.ts +11 -0
- package/{tools → lib/tools}/race-dots-map.ts +5 -4
- package/{tools → lib/tools}/race-dots.ts +8 -9
- package/{tools → lib/tools}/serve-range.ts +20 -12
- package/{tract.ts → lib/tract.ts} +2 -2
- package/out/block-group.d.ts +4 -4
- package/out/block-group.d.ts.map +1 -1
- package/out/block-group.js.map +1 -1
- package/out/class-code.d.ts +1 -1
- package/out/class-code.d.ts.map +1 -1
- package/out/class-code.js +1 -1
- package/out/class-code.js.map +1 -1
- package/out/constants.d.ts +5 -5
- package/out/constants.d.ts.map +1 -1
- package/out/constants.js.map +1 -1
- package/out/county-subdivision.d.ts +4 -4
- package/out/county-subdivision.d.ts.map +1 -1
- package/out/county-subdivision.js.map +1 -1
- package/out/county.d.ts +4 -4
- package/out/county.d.ts.map +1 -1
- package/out/county.js.map +1 -1
- package/out/files.d.ts +1 -1
- package/out/files.d.ts.map +1 -1
- package/out/files.js.map +1 -1
- package/out/functional-status.d.ts.map +1 -1
- package/out/functional-status.js.map +1 -1
- package/out/geoid.d.ts +1 -2
- package/out/geoid.d.ts.map +1 -1
- package/out/geoid.js.map +1 -1
- package/out/index.d.ts +12 -12
- package/out/index.d.ts.map +1 -1
- package/out/index.js +12 -12
- package/out/index.js.map +1 -1
- package/out/legal-statistical-area.d.ts.map +1 -1
- package/out/legal-statistical-area.js.map +1 -1
- package/out/sdk/download.d.ts +0 -7
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +6 -25
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/fetch.d.ts.map +1 -1
- package/out/sdk/fetch.js +25 -41
- package/out/sdk/fetch.js.map +1 -1
- package/out/sdk/index.d.ts +4 -4
- package/out/sdk/index.d.ts.map +1 -1
- package/out/sdk/index.js +4 -4
- package/out/sdk/index.js.map +1 -1
- package/out/sdk/redistricting.d.ts +29 -0
- package/out/sdk/redistricting.d.ts.map +1 -1
- package/out/sdk/redistricting.js +71 -19
- package/out/sdk/redistricting.js.map +1 -1
- package/out/sdk/schema.d.ts +12 -9
- package/out/sdk/schema.d.ts.map +1 -1
- package/out/sdk/schema.js +19 -0
- package/out/sdk/schema.js.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.js +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.js.map +1 -1
- package/out/sdk/state/TIGERTractEntity.d.ts +1 -1
- package/out/sdk/state/TIGERTractEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTractEntity.js.map +1 -1
- package/out/sdk/state/block-operations.d.ts +1 -1
- package/out/sdk/state/block-operations.d.ts.map +1 -1
- package/out/sdk/state/block-operations.js +1 -1
- package/out/sdk/state/block-operations.js.map +1 -1
- package/out/sdk/state/index.d.ts +3 -3
- package/out/sdk/state/index.d.ts.map +1 -1
- package/out/sdk/state/index.js +3 -3
- package/out/sdk/state/index.js.map +1 -1
- package/out/state.d.ts +2 -2
- package/out/state.d.ts.map +1 -1
- package/out/state.js.map +1 -1
- package/out/tabulation-block.d.ts +2 -2
- package/out/tabulation-block.d.ts.map +1 -1
- package/out/tabulation-block.js.map +1 -1
- package/out/tools/index.d.ts +4 -8
- package/out/tools/index.d.ts.map +1 -1
- package/out/tools/index.js +4 -8
- package/out/tools/index.js.map +1 -1
- package/out/tools/race-dots-map.d.ts.map +1 -1
- package/out/tools/race-dots-map.js +5 -4
- package/out/tools/race-dots-map.js.map +1 -1
- package/out/tools/race-dots.d.ts.map +1 -1
- package/out/tools/race-dots.js +7 -8
- package/out/tools/race-dots.js.map +1 -1
- package/out/tools/serve-range.d.ts +2 -3
- package/out/tools/serve-range.d.ts.map +1 -1
- package/out/tools/serve-range.js +17 -10
- package/out/tools/serve-range.js.map +1 -1
- package/out/tract.d.ts +2 -2
- package/out/tract.d.ts.map +1 -1
- package/out/tract.js.map +1 -1
- package/out/tsconfig.test.tsbuildinfo +1 -0
- package/package.json +53 -10
- package/index.ts +0 -18
- package/out/tsconfig.tsbuildinfo +0 -1
- package/sdk/index.ts +0 -10
- package/sdk/state/index.ts +0 -9
- package/tools/index.ts +0 -15
- /package/{functional-status.ts → lib/functional-status.ts} +0 -0
- /package/{legal-statistical-area.ts → lib/legal-statistical-area.ts} +0 -0
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
9
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
10
|
-
import type { AdminLevel1Code } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
10
|
+
import type { AdminLevel1Code } from "#state"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* @title TIGER Block Group
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
9
|
import type {
|
|
10
10
|
GeoIDPart,
|
|
11
11
|
FIPSBlockCode,
|
|
@@ -15,9 +15,9 @@ import type {
|
|
|
15
15
|
FIPSCountySubDivisionCode,
|
|
16
16
|
FIPSPlaceCode,
|
|
17
17
|
FIPSTractCode,
|
|
18
|
-
} from "
|
|
19
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
20
|
-
import type { FIPSStateCode } from "
|
|
18
|
+
} from "#geoid"
|
|
19
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
20
|
+
import type { FIPSStateCode } from "#state"
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
23
|
* A code indicating the classification of the block, i.e. urban or rural.
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
9
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
10
|
-
import type { AdminLevel1Code } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
10
|
+
import type { AdminLevel1Code } from "#state"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* Census Geographic subdivision of a county.
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import type { TIGERClassCode } from "
|
|
8
|
-
import type { TIGERFunctionalStatus } from "
|
|
9
|
-
import type { LegalStatisticalAreaDescription } from "
|
|
10
|
-
import type { AdminLevel1Code } from "
|
|
7
|
+
import type { TIGERClassCode } from "#class-code"
|
|
8
|
+
import type { TIGERFunctionalStatus } from "#functional-status"
|
|
9
|
+
import type { LegalStatisticalAreaDescription } from "#legal-statistical-area"
|
|
10
|
+
import type { AdminLevel1Code } from "#state"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* @title TIGER County
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import type { Tagged } from "type-fest"
|
|
8
8
|
|
|
9
|
-
import type { AdminLevel1Code, FIPSStateCode } from "
|
|
9
|
+
import type { AdminLevel1Code, FIPSStateCode } from "#state"
|
|
10
10
|
|
|
11
11
|
//#region Tagged FIPS types
|
|
12
12
|
|
|
@@ -301,7 +301,6 @@ export interface ParsedGeoIDBlockLevel {
|
|
|
301
301
|
[GeoIDPart.Place]: FIPSPlaceCode
|
|
302
302
|
[GeoIDPart.BlockGroup]: FIPSBlockGroupCode
|
|
303
303
|
[GeoIDPart.Block]: FIPSBlockCode
|
|
304
|
-
[GeoIDPart.BlockGroup]: FIPSBlockGroupCode
|
|
305
304
|
[GeoIDPart.CongressionalDistrict]: FIPSCongressionalDistrictCode
|
|
306
305
|
}
|
|
307
306
|
|
|
@@ -509,10 +508,10 @@ export const GeoIDInputMatchers = {
|
|
|
509
508
|
* @see {@linkcode GeoIDInputMatchers} for the input-level matchers.
|
|
510
509
|
*/
|
|
511
510
|
const GeoIDComponentMatchers = Object.fromEntries(
|
|
512
|
-
Object.values(GeoIDPart).map((value) => {
|
|
511
|
+
Object.values(GeoIDPart).map((value): [GeoIDPart, GeoIDInputMatcher] => {
|
|
513
512
|
return [value, new GeoIDInputMatcher(value)]
|
|
514
513
|
})
|
|
515
|
-
) as
|
|
514
|
+
) as Record<GeoIDPart, GeoIDInputMatcher>
|
|
516
515
|
|
|
517
516
|
/**
|
|
518
517
|
* Type-predicate for checking if a value appears to be a valid GeoID component.
|
package/lib/index.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software.
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export * from "#block-group"
|
|
8
|
+
export * from "#class-code"
|
|
9
|
+
export * from "#constants"
|
|
10
|
+
export * from "#county-subdivision"
|
|
11
|
+
export * from "#county"
|
|
12
|
+
export * from "#files"
|
|
13
|
+
export * from "#functional-status"
|
|
14
|
+
export * from "#geoid"
|
|
15
|
+
export * from "#legal-statistical-area"
|
|
16
|
+
export * from "#state"
|
|
17
|
+
export * from "#tabulation-block"
|
|
18
|
+
export * from "#tract"
|
|
@@ -14,35 +14,11 @@
|
|
|
14
14
|
* never holds it: a web stream piped to a write stream.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import { Readable } from "node:stream"
|
|
21
|
-
import { pipeline } from "node:stream/promises"
|
|
22
|
-
|
|
17
|
+
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
18
|
+
import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
|
|
19
|
+
import { movePath } from "@mailwoman/core/fs/writers"
|
|
23
20
|
import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
|
|
24
21
|
|
|
25
|
-
/**
|
|
26
|
-
* Run a command and resolve its stdout, rejecting with the captured stderr on a non-zero exit.
|
|
27
|
-
*
|
|
28
|
-
* Used for the `unzip -tq` integrity probe below — a shelled-out check rather than a library one because the archives
|
|
29
|
-
* are large and `unzip -t` streams them without materializing the contents.
|
|
30
|
-
*/
|
|
31
|
-
export function runCapture(cmd: string, args: string[]): Promise<string> {
|
|
32
|
-
return new Promise((resolve, reject) => {
|
|
33
|
-
const child = spawn(cmd, args)
|
|
34
|
-
let out = ""
|
|
35
|
-
let err = ""
|
|
36
|
-
child.stdout.on("data", (d) => (out += d))
|
|
37
|
-
child.stderr.on("data", (d) => (err += d))
|
|
38
|
-
child.on("error", reject)
|
|
39
|
-
|
|
40
|
-
child.on("close", (code) =>
|
|
41
|
-
code === 0 ? resolve(out) : reject(new Error(`${cmd} exited ${code}: ${err.slice(0, 500)}`))
|
|
42
|
-
)
|
|
43
|
-
})
|
|
44
|
-
}
|
|
45
|
-
|
|
46
22
|
/**
|
|
47
23
|
* Download `url` to `dest` unless a VALID copy is already there. Returns `true` when the cache was reused, `false` when
|
|
48
24
|
* a download happened.
|
|
@@ -52,7 +28,7 @@ export function runCapture(cmd: string, args: string[]): Promise<string> {
|
|
|
52
28
|
* renames, so `dest` is never a partial archive.
|
|
53
29
|
*/
|
|
54
30
|
export async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
|
|
55
|
-
if (
|
|
31
|
+
if (await pathExists(dest)) {
|
|
56
32
|
try {
|
|
57
33
|
await verifyZipIntegrity(dest)
|
|
58
34
|
|
|
@@ -66,8 +42,8 @@ export async function downloadIfNeeded(url: string, dest: string): Promise<boole
|
|
|
66
42
|
const res = await fetch(url, { redirect: "follow" })
|
|
67
43
|
|
|
68
44
|
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
|
|
69
|
-
await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]),
|
|
70
|
-
await
|
|
45
|
+
await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), openWriteStream(tmp))
|
|
46
|
+
await movePath(tmp, dest)
|
|
71
47
|
|
|
72
48
|
return false
|
|
73
49
|
}
|
package/{sdk → lib/sdk}/fetch.ts
RENAMED
|
@@ -24,23 +24,17 @@
|
|
|
24
24
|
* inserts. Re-running a state replaces its rows.
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
import { dirname, join } from "node:path"
|
|
31
|
-
import { DatabaseSync } from "node:sqlite"
|
|
32
|
-
import { Readable } from "node:stream"
|
|
33
|
-
import { pipeline } from "node:stream/promises"
|
|
34
|
-
|
|
27
|
+
import { APIClient, pluckResponseData } from "@mailwoman/core/api"
|
|
28
|
+
import { mailwomanDataRoot } from "@mailwoman/core/data-root"
|
|
29
|
+
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
35
30
|
import { extractZipEntries } from "@mailwoman/core/fs/zip"
|
|
36
|
-
import {
|
|
37
|
-
import {
|
|
38
|
-
import {
|
|
39
|
-
import { TextSpliterator } from "spliterator"
|
|
31
|
+
import { ogr2ogrGeoJSONSeq } from "@mailwoman/spatial/tools/ogr-stream"
|
|
32
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
33
|
+
import { dirname, join } from "path-ts"
|
|
40
34
|
|
|
41
|
-
import { downloadIfNeeded } from "
|
|
42
|
-
import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "
|
|
43
|
-
import { initializeTIGERSchema, TIGER_PRAGMAS } from "
|
|
35
|
+
import { downloadIfNeeded } from "#sdk/download"
|
|
36
|
+
import type { TIGERBlockTable, TIGERDatabase, TIGERPlaceTable, TIGERStreetTable } from "#sdk/schema"
|
|
37
|
+
import { initializeTIGERSchema, TIGER_PRAGMAS } from "#sdk/schema"
|
|
44
38
|
|
|
45
39
|
const CENSUS_HOST = "https://www2.census.gov"
|
|
46
40
|
const DEFAULT_DATA_ROOT = mailwomanDataRoot()
|
|
@@ -176,10 +170,11 @@ function buildRow(level: TIGERFetchLevel, p: Record<string, unknown>, geometry:
|
|
|
176
170
|
* Scrape the ADDRFEAT directory listing for a state's county FIPS codes.
|
|
177
171
|
*/
|
|
178
172
|
async function discoverCounties(state: string, vintage: number): Promise<string[]> {
|
|
179
|
-
|
|
173
|
+
// The LISTING only — a small HTML index. The per-county archives below stay on raw `fetch`, streaming to disk.
|
|
174
|
+
const html = await new APIClient({ displayName: "tiger-listing", retry: true })
|
|
175
|
+
.fetch<string>({ url: `${CENSUS_HOST}/geo/tiger/TIGER${vintage}/ADDRFEAT/`, responseType: "text" })
|
|
176
|
+
.then(pluckResponseData)
|
|
180
177
|
|
|
181
|
-
if (!res.ok) throw new Error(`HTTP ${res.status} listing ADDRFEAT for vintage ${vintage}`)
|
|
182
|
-
const html = await res.text()
|
|
183
178
|
const re = new RegExp(`tl_${vintage}_${state}(\\d{3})_addrfeat\\.zip`, "g")
|
|
184
179
|
const counties = new Set<string>()
|
|
185
180
|
|
|
@@ -207,8 +202,8 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
207
202
|
// the per-table idempotent delete keeps a re-fetch (newer vintage) clean. The download CACHE stays
|
|
208
203
|
// vintage-partitioned below so zips don't collide across vintages.
|
|
209
204
|
const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
|
|
210
|
-
await
|
|
211
|
-
await
|
|
205
|
+
await makeDirectories(cacheDir)
|
|
206
|
+
await makeDirectories(dirname(outPath))
|
|
212
207
|
|
|
213
208
|
// Source units: one (per-state) for block/place; one per county for addrfeat.
|
|
214
209
|
const geoCodes = level === "addrfeat" ? await discoverCounties(state, vintage) : [""]
|
|
@@ -217,11 +212,15 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
217
212
|
throw new Error(`No ADDRFEAT counties found for state ${state} vintage ${vintage}`)
|
|
218
213
|
}
|
|
219
214
|
|
|
220
|
-
const
|
|
221
|
-
|
|
222
|
-
const kdb = new DatabaseClient<TIGERDatabase>({ database: db })
|
|
215
|
+
const kdb = new DatabaseClient<TIGERDatabase>(outPath)
|
|
216
|
+
kdb.exec(TIGER_PRAGMAS)
|
|
223
217
|
await initializeTIGERSchema(kdb)
|
|
224
218
|
|
|
219
|
+
/**
|
|
220
|
+
* `level` picks the table AND the row shape, but nothing ties them at the type level: {@link buildRow} returns the
|
|
221
|
+
* union and Kysely needs the arm's concrete row. The narrowings below are sound only because `buildRow` switches on
|
|
222
|
+
* the SAME `level` — keep the two switches in step, or a row shape reaches the wrong table.
|
|
223
|
+
*/
|
|
225
224
|
const insertBatch = async (rows: Row[]): Promise<void> => {
|
|
226
225
|
if (level === "tabblock20") {
|
|
227
226
|
await kdb
|
|
@@ -276,37 +275,21 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
276
275
|
const shpPath = join(cacheDir, layer + ".shp")
|
|
277
276
|
yield { phase: "extract", file: layer + ".shp" }
|
|
278
277
|
|
|
279
|
-
const
|
|
280
|
-
"
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
let stderr = ""
|
|
295
|
-
child.stderr.on("data", (d) => (stderr += d))
|
|
296
|
-
|
|
297
|
-
const exited = new Promise<number>((resolve) => {
|
|
298
|
-
child.on("close", (code) => resolve(code ?? 0))
|
|
299
|
-
})
|
|
300
|
-
|
|
301
|
-
// GeoJSONSeq is line-delimited; per-line `tryParsingJSON` so a malformed record is
|
|
302
|
-
// tolerated (skipped), not thrown.
|
|
303
|
-
for await (const line of TextSpliterator.fromAsync(child.stdout)) {
|
|
304
|
-
if (!line) continue
|
|
305
|
-
|
|
306
|
-
const feat = tryParsingJSON<{ properties?: Record<string, unknown>; geometry?: unknown }>(line)
|
|
307
|
-
|
|
308
|
-
if (!feat) continue
|
|
309
|
-
|
|
278
|
+
const args = [
|
|
279
|
+
"-f",
|
|
280
|
+
"GeoJSONSeq",
|
|
281
|
+
"-t_srs",
|
|
282
|
+
"EPSG:4326",
|
|
283
|
+
"-sql",
|
|
284
|
+
selectSQL(level, layer, options.county),
|
|
285
|
+
"/vsistdout/",
|
|
286
|
+
shpPath,
|
|
287
|
+
]
|
|
288
|
+
|
|
289
|
+
for await (const feat of ogr2ogrGeoJSONSeq<{ properties?: Record<string, unknown>; geometry?: unknown }>(
|
|
290
|
+
args,
|
|
291
|
+
layer
|
|
292
|
+
)) {
|
|
310
293
|
if (!feat.properties) continue
|
|
311
294
|
|
|
312
295
|
if (level === "tabblock20" && !feat.geometry) continue
|
|
@@ -319,14 +302,10 @@ export async function* fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<Fe
|
|
|
319
302
|
}
|
|
320
303
|
|
|
321
304
|
await flush()
|
|
322
|
-
|
|
323
|
-
const code = await exited
|
|
324
|
-
|
|
325
|
-
if (code !== 0) throw new Error(`ogr2ogr exited ${code} on ${layer}: ${stderr.slice(0, 500)}`)
|
|
326
305
|
yield { phase: "load", inserted, total: 0 }
|
|
327
306
|
}
|
|
328
307
|
|
|
329
|
-
|
|
308
|
+
kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
|
330
309
|
|
|
331
310
|
return { outPath, table, inserted }
|
|
332
311
|
} finally {
|
package/lib/sdk/index.ts
ADDED
|
@@ -16,22 +16,16 @@
|
|
|
16
16
|
* https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import { mkdir, rename } from "node:fs/promises"
|
|
22
|
-
import { dirname, join } from "node:path"
|
|
23
|
-
import { DatabaseSync } from "node:sqlite"
|
|
24
|
-
import { Readable } from "node:stream"
|
|
25
|
-
import { pipeline } from "node:stream/promises"
|
|
26
|
-
|
|
19
|
+
import { mailwomanDataRoot } from "@mailwoman/core/data-root"
|
|
20
|
+
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
27
21
|
import { extractZipEntries } from "@mailwoman/core/fs/zip"
|
|
28
|
-
import { DatabaseClient } from "@mailwoman/
|
|
29
|
-
import {
|
|
22
|
+
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
23
|
+
import { dirname, join } from "path-ts"
|
|
30
24
|
import { TextSpliterator } from "spliterator"
|
|
31
25
|
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
26
|
+
import { downloadIfNeeded } from "#sdk/download"
|
|
27
|
+
import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "#sdk/schema"
|
|
28
|
+
import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "#state"
|
|
35
29
|
|
|
36
30
|
const REDISTRICTING_BASE =
|
|
37
31
|
"https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171"
|
|
@@ -52,6 +46,60 @@ const GEO_GEOCODE = 9
|
|
|
52
46
|
const SEG_LOGRECNO = 4
|
|
53
47
|
const P2 = (fieldNo: number) => 76 + (fieldNo - 1)
|
|
54
48
|
|
|
49
|
+
/**
|
|
50
|
+
* Segment 2: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
|
|
51
|
+
* against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623 / 916,517, the published CA 2020
|
|
52
|
+
* figures. `occupied + vacant === housing_units` is the invariant that distinguishes a correct offset from a plausible
|
|
53
|
+
* one, and {@link parseH1} refuses a row where it does not hold.
|
|
54
|
+
*/
|
|
55
|
+
export const SEG2_FIELD_COUNT = 152
|
|
56
|
+
/**
|
|
57
|
+
* H0010001 — total housing units, the third-from-last field of a segment-2 row.
|
|
58
|
+
*/
|
|
59
|
+
export const H1_TOTAL = SEG2_FIELD_COUNT - 3
|
|
60
|
+
/**
|
|
61
|
+
* H0010002 — occupied housing units, the second-from-last field.
|
|
62
|
+
*/
|
|
63
|
+
export const H1_OCCUPIED = SEG2_FIELD_COUNT - 2
|
|
64
|
+
/**
|
|
65
|
+
* H0010003 — vacant housing units, the last field (and so the one that carries a CRLF file's trailing CR).
|
|
66
|
+
*/
|
|
67
|
+
export const H1_VACANT = SEG2_FIELD_COUNT - 1
|
|
68
|
+
|
|
69
|
+
export interface H1Counts {
|
|
70
|
+
housing_units: number
|
|
71
|
+
occupied: number
|
|
72
|
+
vacant: number
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* The three H1 counts from one segment-2 row. The last field carries CRLF's trailing CR when the file has one, so each
|
|
77
|
+
* field is trimmed before it is read as a number; a row whose counts do not add up is refused rather than stored.
|
|
78
|
+
*/
|
|
79
|
+
export function parseH1(fields: readonly string[]): H1Counts {
|
|
80
|
+
if (fields.length !== SEG2_FIELD_COUNT) {
|
|
81
|
+
throw new Error(`segment 2 row has ${fields.length} fields, expected ${SEG2_FIELD_COUNT}`)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const counts = {
|
|
85
|
+
housing_units: Number(fields[H1_TOTAL]!.trim()),
|
|
86
|
+
occupied: Number(fields[H1_OCCUPIED]!.trim()),
|
|
87
|
+
vacant: Number(fields[H1_VACANT]!.trim()),
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
if (![counts.housing_units, counts.occupied, counts.vacant].every(Number.isInteger)) {
|
|
91
|
+
throw new Error(`segment 2 row carries a non-integer H1 count: ${fields.slice(H1_TOTAL).join("|")}`)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (counts.occupied + counts.vacant !== counts.housing_units) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
`segment 2 row breaks the H1 invariant: occupied ${counts.occupied} + vacant ${counts.vacant} != housing_units ${counts.housing_units} (LOGRECNO ${fields[SEG_LOGRECNO]})`
|
|
97
|
+
)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return counts
|
|
101
|
+
}
|
|
102
|
+
|
|
55
103
|
/**
|
|
56
104
|
* The eight P2 categories that partition the total (P0020001), in `pl_block` column order.
|
|
57
105
|
*/
|
|
@@ -139,8 +187,8 @@ export async function* fetchRedistricting(
|
|
|
139
187
|
const cacheDir = join(dataRoot, "census", "redistricting", String(vintage), state)
|
|
140
188
|
// Same stable `tiger.db` default as fetchTIGER — pl_block lives alongside tabblock20 in one DB.
|
|
141
189
|
const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db")
|
|
142
|
-
await
|
|
143
|
-
await
|
|
190
|
+
await makeDirectories(cacheDir)
|
|
191
|
+
await makeDirectories(dirname(outPath))
|
|
144
192
|
|
|
145
193
|
const zipName = `${fileAbbr}${vintage}.pl.zip`
|
|
146
194
|
const zipPath = join(cacheDir, zipName)
|
|
@@ -152,6 +200,7 @@ export async function* fetchRedistricting(
|
|
|
152
200
|
await extractZipEntries(zipPath, cacheDir)
|
|
153
201
|
const geoPath = join(cacheDir, `${fileAbbr}geo${vintage}.pl`)
|
|
154
202
|
const seg1Path = join(cacheDir, `${fileAbbr}00001${vintage}.pl`)
|
|
203
|
+
const seg2Path = join(cacheDir, `${fileAbbr}00002${vintage}.pl`)
|
|
155
204
|
yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` }
|
|
156
205
|
|
|
157
206
|
// Pass 1: header → LOGRECNO → GEOID for the blocks we want.
|
|
@@ -171,9 +220,23 @@ export async function* fetchRedistricting(
|
|
|
171
220
|
const total = logToGeoid.size
|
|
172
221
|
yield { phase: "header", blocks: total }
|
|
173
222
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
const
|
|
223
|
+
// Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs. Read before segment 1 so a block's P2 and H1
|
|
224
|
+
// land in one row; a mapped block with no segment-2 row is a data defect the load refuses, never a zero.
|
|
225
|
+
const h1ByLogrecno = new Map<string, H1Counts>()
|
|
226
|
+
|
|
227
|
+
await eachLine(seg2Path, (line) => {
|
|
228
|
+
const f = line.split("|")
|
|
229
|
+
const logrecno = f[SEG_LOGRECNO] ?? ""
|
|
230
|
+
|
|
231
|
+
if (!logToGeoid.has(logrecno)) return
|
|
232
|
+
|
|
233
|
+
h1ByLogrecno.set(logrecno, parseH1(f))
|
|
234
|
+
})
|
|
235
|
+
|
|
236
|
+
yield { phase: "extract", file: `${fileAbbr}00002${vintage}.pl` }
|
|
237
|
+
|
|
238
|
+
const kdb = new DatabaseClient<TIGERDatabase>(outPath)
|
|
239
|
+
kdb.exec(TIGER_PRAGMAS)
|
|
177
240
|
await initializeTIGERSchema(kdb)
|
|
178
241
|
|
|
179
242
|
try {
|
|
@@ -198,12 +261,20 @@ export async function* fetchRedistricting(
|
|
|
198
261
|
for await (const line of TextSpliterator.fromAsync(seg1Path)) {
|
|
199
262
|
if (!line) continue
|
|
200
263
|
const f = line.split("|")
|
|
201
|
-
const
|
|
264
|
+
const logrecno = f[SEG_LOGRECNO] ?? ""
|
|
265
|
+
const geoid = logToGeoid.get(logrecno)
|
|
202
266
|
|
|
203
267
|
if (!geoid) continue
|
|
204
268
|
|
|
269
|
+
const h1 = h1ByLogrecno.get(logrecno)
|
|
270
|
+
|
|
271
|
+
if (!h1) {
|
|
272
|
+
throw new Error(`block ${geoid} (LOGRECNO ${logrecno}) has a segment-1 row and no segment-2 row`)
|
|
273
|
+
}
|
|
274
|
+
|
|
205
275
|
batch.push({
|
|
206
276
|
GEOID: geoid,
|
|
277
|
+
...h1,
|
|
207
278
|
pop_total: Number(f[CATEGORY_INDEX.pop_total] ?? 0),
|
|
208
279
|
hispanic: Number(f[CATEGORY_INDEX.hispanic] ?? 0),
|
|
209
280
|
white: Number(f[CATEGORY_INDEX.white] ?? 0),
|
|
@@ -224,7 +295,7 @@ export async function* fetchRedistricting(
|
|
|
224
295
|
await flush()
|
|
225
296
|
|
|
226
297
|
yield { phase: "load", inserted, total }
|
|
227
|
-
|
|
298
|
+
kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
|
228
299
|
|
|
229
300
|
return { outPath, table: "pl_block", inserted }
|
|
230
301
|
} finally {
|
|
@@ -46,6 +46,18 @@ export interface PLBlockTable {
|
|
|
46
46
|
nhpi: number
|
|
47
47
|
other: number
|
|
48
48
|
multi: number
|
|
49
|
+
/**
|
|
50
|
+
* P.L. 94-171 table H1 — total housing units in the block.
|
|
51
|
+
*/
|
|
52
|
+
housing_units: number
|
|
53
|
+
/**
|
|
54
|
+
* H1 occupied. `occupied + vacant === housing_units` by construction; the reader refuses a row where it is not.
|
|
55
|
+
*/
|
|
56
|
+
occupied: number
|
|
57
|
+
/**
|
|
58
|
+
* H1 vacant.
|
|
59
|
+
*/
|
|
60
|
+
vacant: number
|
|
49
61
|
}
|
|
50
62
|
|
|
51
63
|
/**
|
|
@@ -106,6 +118,25 @@ PRAGMA journal_mode = WAL;
|
|
|
106
118
|
* The `text(N)` length hints in the prior raw DDL were documentary only (SQLite uses TEXT affinity regardless); the
|
|
107
119
|
* lengths live on the {@link TIGERBlockTable} interface instead.
|
|
108
120
|
*/
|
|
121
|
+
/**
|
|
122
|
+
* Refuse a `pl_block` built before the H1 columns existed. `createTable(...).ifNotExists()` leaves an older table as it
|
|
123
|
+
* is, and the next load would fail on the first insert with a message about a column, not about the cause. The table is
|
|
124
|
+
* derived (the redistricting command reloads it state by state), so the remedy is a rebuild: drop it and re-run
|
|
125
|
+
* `mailwoman tiger redistricting` for each state you hold. Databases are never patched in place.
|
|
126
|
+
*/
|
|
127
|
+
async function assertPLBlockShape(db: Kysely<TIGERDatabase>): Promise<void> {
|
|
128
|
+
const columns = await sql<{ name: string }>`select name from pragma_table_info('pl_block')`.execute(db)
|
|
129
|
+
const names = new Set(columns.rows.map((row) => row.name))
|
|
130
|
+
|
|
131
|
+
for (const required of ["housing_units", "occupied", "vacant"]) {
|
|
132
|
+
if (!names.has(required)) {
|
|
133
|
+
throw new Error(
|
|
134
|
+
`pl_block predates its H1 columns (missing ${required}). It is a derived table: drop it and re-run \`mailwoman tiger redistricting\` for each state, which reloads P2 and H1 together.`
|
|
135
|
+
)
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
109
140
|
export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<void> {
|
|
110
141
|
await db.schema
|
|
111
142
|
.createTable("us_state")
|
|
@@ -177,11 +208,16 @@ export async function initializeTIGERSchema(db: Kysely<TIGERDatabase>): Promise<
|
|
|
177
208
|
.addColumn("nhpi", "integer", (c) => c.notNull())
|
|
178
209
|
.addColumn("other", "integer", (c) => c.notNull())
|
|
179
210
|
.addColumn("multi", "integer", (c) => c.notNull())
|
|
211
|
+
.addColumn("housing_units", "integer", (c) => c.notNull())
|
|
212
|
+
.addColumn("occupied", "integer", (c) => c.notNull())
|
|
213
|
+
.addColumn("vacant", "integer", (c) => c.notNull())
|
|
180
214
|
// pl_block is small (no geometry) and always probed by its GEOID PK (1:1 join to tabblock20), so
|
|
181
215
|
// cluster it WITHOUT ROWID — one B-tree probe per join, no separate rowid + PK-index pair.
|
|
182
216
|
.modifyEnd(sql`without rowid`)
|
|
183
217
|
.execute()
|
|
184
218
|
|
|
219
|
+
await assertPLBlockShape(db)
|
|
220
|
+
|
|
185
221
|
await db.schema
|
|
186
222
|
.createTable("tiger_streets")
|
|
187
223
|
.ifNotExists()
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import type { GeometryLiteral, MultiPolygonLiteral } from "@mailwoman/spatial"
|
|
8
|
-
import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/
|
|
8
|
+
import { wellKnownGeometryToGeoJSON } from "@mailwoman/spatial/well-known-text"
|
|
9
|
+
|
|
9
10
|
import type {
|
|
10
11
|
FIPSStateCode,
|
|
11
12
|
FIPSBlockGeoID,
|
|
@@ -13,7 +14,7 @@ import type {
|
|
|
13
14
|
TIGERBlockFeature,
|
|
14
15
|
TIGERBlockFeatureCollection,
|
|
15
16
|
TIGERTabulatedBlockProperties,
|
|
16
|
-
} from "
|
|
17
|
+
} from "#index"
|
|
17
18
|
|
|
18
19
|
export interface StateBlockIntersectionCriteria {
|
|
19
20
|
stateCode: FIPSStateCode
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
|
|
7
7
|
import type { GeoFeature, MultiPolygonLiteral } from "@mailwoman/spatial"
|
|
8
8
|
|
|
9
|
-
import type { GeoIDPart } from "
|
|
10
|
-
import type { TIGERTabulatedBlockProperties } from "
|
|
9
|
+
import type { GeoIDPart } from "#geoid"
|
|
10
|
+
import type { TIGERTabulatedBlockProperties } from "#tabulation-block"
|
|
11
11
|
|
|
12
12
|
//#region FIPS Codes
|
|
13
13
|
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
|
|
7
7
|
import type { MultiPolygonLiteral } from "@mailwoman/spatial"
|
|
8
8
|
|
|
9
|
-
import type { TIGERProperty, TIGERPropertyRecord } from "
|
|
10
|
-
import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "
|
|
9
|
+
import type { TIGERProperty, TIGERPropertyRecord } from "#constants"
|
|
10
|
+
import type { GeoIDPart, FIPSBlockGeoID, ParsedGeoIDBlockLevel } from "#geoid"
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
13
|
* The properties of a tabulated block from the TIGER/Line dataset.
|