@mailwoman/tiger 9.4.0 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/block-group.ts +7 -7
- package/lib/class-code.ts +387 -266
- package/lib/constants.ts +7 -6
- package/lib/{county-subdivision.ts → county/subdivision.ts} +7 -7
- package/lib/county.ts +7 -7
- package/lib/files.ts +37 -20
- package/lib/functional-status.ts +6 -4
- package/lib/geoid.ts +44 -26
- package/lib/index.ts +1 -1
- package/lib/legal-statistical-area.ts +7 -5
- package/lib/{sdk/schema.ts → schema.ts} +32 -42
- package/lib/state.ts +23 -20
- package/lib/tabulation-block.ts +4 -4
- package/lib/tract.ts +3 -3
- package/out/block-group.d.ts +7 -7
- package/out/class-code.d.ts +387 -266
- package/out/class-code.d.ts.map +1 -1
- package/out/class-code.js +387 -266
- package/out/class-code.js.map +1 -1
- package/out/constants.d.ts +7 -6
- package/out/constants.d.ts.map +1 -1
- package/out/constants.js +3 -3
- package/out/{county-subdivision.d.ts → county/subdivision.d.ts} +8 -8
- package/out/county/subdivision.d.ts.map +1 -0
- package/out/{county-subdivision.js → county/subdivision.js} +1 -1
- package/out/county/subdivision.js.map +1 -0
- package/out/county.d.ts +7 -7
- package/out/county.js +1 -1
- package/out/files.d.ts +21 -13
- package/out/files.d.ts.map +1 -1
- package/out/files.js +27 -13
- package/out/files.js.map +1 -1
- package/out/functional-status.d.ts +6 -4
- package/out/functional-status.d.ts.map +1 -1
- package/out/functional-status.js +6 -4
- package/out/functional-status.js.map +1 -1
- package/out/geoid.d.ts +34 -16
- package/out/geoid.d.ts.map +1 -1
- package/out/geoid.js +16 -10
- package/out/geoid.js.map +1 -1
- package/out/index.d.ts +1 -1
- package/out/index.js +1 -1
- package/out/legal-statistical-area.d.ts +7 -5
- package/out/legal-statistical-area.d.ts.map +1 -1
- package/out/legal-statistical-area.js +7 -5
- package/out/legal-statistical-area.js.map +1 -1
- package/out/{sdk/schema.d.ts → schema.d.ts} +26 -20
- package/out/schema.d.ts.map +1 -0
- package/out/{sdk/schema.js → schema.js} +12 -31
- package/out/schema.js.map +1 -0
- package/out/sdk/download.d.ts +7 -15
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +5 -14
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/fetch.d.ts +32 -29
- package/out/sdk/fetch.d.ts.map +1 -1
- package/out/sdk/fetch.js +18 -53
- package/out/sdk/fetch.js.map +1 -1
- package/out/sdk/redistricting.d.ts +29 -15
- package/out/sdk/redistricting.d.ts.map +1 -1
- package/out/sdk/redistricting.js +47 -36
- package/out/sdk/redistricting.js.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts +2 -2
- package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.js +3 -3
- package/out/sdk/state/TIGERTabulatedBlockEntity.js.map +1 -1
- package/out/sdk/state/TIGERTractEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTractEntity.js.map +1 -1
- package/out/sdk/state/block-operations.d.ts +3 -2
- package/out/sdk/state/block-operations.d.ts.map +1 -1
- package/out/sdk/state/block-operations.js +4 -3
- package/out/sdk/state/block-operations.js.map +1 -1
- package/out/sdk/{state/index.d.ts → state.d.ts} +1 -1
- package/out/sdk/state.d.ts.map +1 -0
- package/out/sdk/{state/index.js → state.js} +1 -1
- package/out/sdk/state.js.map +1 -0
- package/out/sdk/tsconfig.tsbuildinfo +1 -0
- package/out/state.d.ts +45 -42
- package/out/state.d.ts.map +1 -1
- package/out/state.js +17 -14
- package/out/state.js.map +1 -1
- package/out/tabulation-block.d.ts +4 -4
- package/out/tools/{race-dots-map.d.ts → race/dots/map.d.ts} +24 -10
- package/out/tools/race/dots/map.d.ts.map +1 -0
- package/out/tools/{race-dots-map.js → race/dots/map.js} +10 -6
- package/out/tools/race/dots/map.js.map +1 -0
- package/out/tools/race/dots.d.ts +47 -0
- package/out/tools/race/dots.d.ts.map +1 -0
- package/out/tools/{race-dots.js → race/dots.js} +10 -27
- package/out/tools/race/dots.js.map +1 -0
- package/out/tools/serve-range.d.ts +13 -7
- package/out/tools/serve-range.d.ts.map +1 -1
- package/out/tools/serve-range.js +9 -7
- package/out/tools/serve-range.js.map +1 -1
- package/out/tools/tsconfig.tsbuildinfo +1 -0
- package/out/tract.d.ts +3 -3
- package/out/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +59 -42
- package/sdk/download.ts +41 -0
- package/{lib/sdk → sdk}/fetch.ts +54 -62
- package/{lib/sdk → sdk}/redistricting.ts +57 -40
- package/{lib/sdk → sdk}/state/TIGERTabulatedBlockEntity.ts +3 -3
- package/{lib/sdk → sdk}/state/block-operations.ts +5 -4
- package/{lib/tools/race-dots-map.ts → tools/race/dots/map.ts} +30 -12
- package/{lib/tools/race-dots.ts → tools/race/dots.ts} +14 -31
- package/{lib/tools → tools}/serve-range.ts +15 -9
- package/lib/sdk/download.ts +0 -49
- package/lib/sdk/index.ts +0 -10
- package/lib/tools/index.ts +0 -11
- package/out/county-subdivision.d.ts.map +0 -1
- package/out/county-subdivision.js.map +0 -1
- package/out/sdk/index.d.ts +0 -10
- package/out/sdk/index.d.ts.map +0 -1
- package/out/sdk/index.js +0 -10
- package/out/sdk/index.js.map +0 -1
- package/out/sdk/schema.d.ts.map +0 -1
- package/out/sdk/schema.js.map +0 -1
- package/out/sdk/state/index.d.ts.map +0 -1
- package/out/sdk/state/index.js.map +0 -1
- package/out/tools/index.d.ts +0 -11
- package/out/tools/index.d.ts.map +0 -1
- package/out/tools/index.js +0 -11
- package/out/tools/index.js.map +0 -1
- package/out/tools/race-dots-map.d.ts.map +0 -1
- package/out/tools/race-dots-map.js.map +0 -1
- package/out/tools/race-dots.d.ts +0 -61
- package/out/tools/race-dots.d.ts.map +0 -1
- package/out/tools/race-dots.js.map +0 -1
- /package/{lib/sdk → sdk}/state/TIGERTractEntity.ts +0 -0
- /package/{lib/sdk/state/index.ts → sdk/state.ts} +0 -0
|
@@ -4,53 +4,56 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
6
|
* Pull a state's Census 2020 P.L. 94-171 redistricting counts (table P2 — Hispanic-or-Latino by
|
|
7
|
-
* race, per block) into the `pl_block` table of a
|
|
8
|
-
* same 15-char block `
|
|
7
|
+
* race, per block) into the `pl_block` table of a tiger {@link DatabaseClient} DB, keyed on the
|
|
8
|
+
* same 15-char block `geoid` as {@link fetchTIGER}'s `tabblock20`. Join the two for block-level
|
|
9
9
|
* race + geometry (e.g. a dot-density map).
|
|
10
10
|
*
|
|
11
11
|
* Keyless public data. The per-state ZIP holds a pipe-delimited geographic header
|
|
12
|
-
* (`<st>geo<yr>.pl`) and three data segments
|
|
13
|
-
* join the header (filtered to
|
|
12
|
+
* (`<st>geo<yr>.pl`) and three data segments. segment 1 (`<st>00001<yr>.pl`) contains P1 + P2. We
|
|
13
|
+
* join the header (filtered to sumlev 750 = block) to segment 1 by logrecno. Field offsets are
|
|
14
14
|
* fixed by the 2020 P.L. layout (verified against the real files).
|
|
15
15
|
*
|
|
16
16
|
* https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import {
|
|
19
|
+
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
20
20
|
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
21
21
|
import { extractZipEntries } from "@mailwoman/core/fs/zip"
|
|
22
22
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
23
|
-
import {
|
|
23
|
+
import { PathBuilder } from "path-ts"
|
|
24
24
|
import { TextSpliterator } from "spliterator"
|
|
25
25
|
|
|
26
|
+
import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "#schema"
|
|
26
27
|
import { downloadIfNeeded } from "#sdk/download"
|
|
27
|
-
import { initializeTIGERSchema, TIGER_PRAGMAS, type PLBlockTable, type TIGERDatabase } from "#sdk/schema"
|
|
28
28
|
import { AdminLevel1CodeToAbbreviation, StateName, type AdminLevel1Code } from "#state"
|
|
29
29
|
|
|
30
30
|
const REDISTRICTING_BASE =
|
|
31
31
|
"https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171"
|
|
32
32
|
|
|
33
|
-
const DEFAULT_DATA_ROOT =
|
|
33
|
+
const DEFAULT_DATA_ROOT = dataRootPath()
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
|
-
* P.L. 94-171 (2020) pipe-delimited field offsets (0-based).
|
|
37
|
-
*
|
|
38
|
-
*
|
|
36
|
+
* P.L. 94-171 (2020) pipe-delimited field offsets (0-based).
|
|
37
|
+
*
|
|
38
|
+
* Geographic header: …|sumlev(2)|…|logrecno(7)|geoid(8)|geocode(9)|… — geocode is the
|
|
39
|
+
* bare 15-char block FIPS (matches tiger GEOID20); sumlev 750 = tabulation block.
|
|
39
40
|
*/
|
|
40
41
|
const GEO_SUMLEV = 2
|
|
41
42
|
const GEO_LOGRECNO = 7
|
|
42
43
|
const GEO_GEOCODE = 9
|
|
43
44
|
/**
|
|
44
|
-
* Segment 1:
|
|
45
|
+
* Segment 1: fileid|stusab|chariter|cifsn|logrecno(4)| P1×71 | P2×73. P0020001 is at index 76.
|
|
45
46
|
*/
|
|
46
47
|
const SEG_LOGRECNO = 4
|
|
47
48
|
const P2 = (fieldNo: number) => 76 + (fieldNo - 1)
|
|
48
49
|
|
|
49
50
|
/**
|
|
50
|
-
* Segment 2:
|
|
51
|
-
* against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623
|
|
52
|
-
*
|
|
53
|
-
*
|
|
51
|
+
* Segment 2: fileid|stusab|chariter|cifsn|logrecno(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
|
|
52
|
+
* against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623
|
|
53
|
+
* / 916,517, the published CA 2020 figures.
|
|
54
|
+
*
|
|
55
|
+
* `occupied + vacant === housing_units` is the invariant that distinguishes a correct
|
|
56
|
+
* offset from a plausible one. {@link parseH1} refuses a row when it does not hold.
|
|
54
57
|
*/
|
|
55
58
|
export const SEG2_FIELD_COUNT = 152
|
|
56
59
|
/**
|
|
@@ -62,7 +65,7 @@ export const H1_TOTAL = SEG2_FIELD_COUNT - 3
|
|
|
62
65
|
*/
|
|
63
66
|
export const H1_OCCUPIED = SEG2_FIELD_COUNT - 2
|
|
64
67
|
/**
|
|
65
|
-
* H0010003 — vacant housing units, the last field (and so the one that
|
|
68
|
+
* H0010003 — vacant housing units, the last field (and so the one that contains a crlf file's trailing CR).
|
|
66
69
|
*/
|
|
67
70
|
export const H1_VACANT = SEG2_FIELD_COUNT - 1
|
|
68
71
|
|
|
@@ -73,8 +76,11 @@ export interface H1Counts {
|
|
|
73
76
|
}
|
|
74
77
|
|
|
75
78
|
/**
|
|
76
|
-
* The three H1 counts from one segment-2 row.
|
|
77
|
-
*
|
|
79
|
+
* The three H1 counts from one segment-2 row.
|
|
80
|
+
*
|
|
81
|
+
* The last field contains crlf's trailing CR when the file has one, so each field
|
|
82
|
+
* is trimmed before it is read as a number.
|
|
83
|
+
* A row whose counts do not add up is refused rather than stored.
|
|
78
84
|
*/
|
|
79
85
|
export function parseH1(fields: readonly string[]): H1Counts {
|
|
80
86
|
if (fields.length !== SEG2_FIELD_COUNT) {
|
|
@@ -106,7 +112,7 @@ export function parseH1(fields: readonly string[]): H1Counts {
|
|
|
106
112
|
const CATEGORY_INDEX = {
|
|
107
113
|
pop_total: P2(1),
|
|
108
114
|
hispanic: P2(2), // Hispanic or Latino (any race)
|
|
109
|
-
white: P2(5), // Not Hispanic: White
|
|
115
|
+
white: P2(5), // Not Hispanic: White without another race
|
|
110
116
|
black: P2(6),
|
|
111
117
|
aian: P2(7),
|
|
112
118
|
asian: P2(8),
|
|
@@ -121,11 +127,15 @@ export interface FetchRedistrictingOptions {
|
|
|
121
127
|
*/
|
|
122
128
|
stateFIPS: string
|
|
123
129
|
/**
|
|
124
|
-
* Decennial vintage.
|
|
130
|
+
* Decennial vintage.
|
|
131
|
+
*
|
|
132
|
+
* Default 2020 (the only P.L. 94-171 release this parses).
|
|
125
133
|
*/
|
|
126
134
|
vintage?: number
|
|
127
135
|
/**
|
|
128
|
-
* Output SQLite path.
|
|
136
|
+
* Output SQLite path.
|
|
137
|
+
*
|
|
138
|
+
* Default `<dataRoot>/tiger/tiger.db` (same DB as `fetchTIGER`).
|
|
129
139
|
*/
|
|
130
140
|
outPath?: string
|
|
131
141
|
/**
|
|
@@ -137,7 +147,9 @@ export interface FetchRedistrictingOptions {
|
|
|
137
147
|
*/
|
|
138
148
|
county?: string
|
|
139
149
|
/**
|
|
140
|
-
* Rows per insert.
|
|
150
|
+
* Rows per insert.
|
|
151
|
+
*
|
|
152
|
+
* Default 2000.
|
|
141
153
|
*/
|
|
142
154
|
batchSize?: number
|
|
143
155
|
}
|
|
@@ -154,11 +166,12 @@ export interface FetchRedistrictingResult {
|
|
|
154
166
|
inserted: number
|
|
155
167
|
}
|
|
156
168
|
|
|
157
|
-
// Pipe-delimited, fixed field-offset census rows
|
|
158
|
-
// parse indexes by position
|
|
159
|
-
//
|
|
169
|
+
// Pipe-delimited, fixed field-offset census rows.
|
|
170
|
+
// The manual `split("|")` at each call site stays (the parse indexes by position rather than by header).
|
|
171
|
+
// Spliterator keeps crlf's trailing CR where readline stripped it, but every field
|
|
172
|
+
// this parser reads (geo geocode/logrecno ≤ 9, segment-1 logrecno + P2 ≤ 86) sits well
|
|
160
173
|
// before the final column, so the retained CR only ever lands on an unread trailing field.
|
|
161
|
-
async function eachLine(path:
|
|
174
|
+
async function eachLine(path: PathBuilder, fn: (line: string) => void): Promise<void> {
|
|
162
175
|
for await (const line of TextSpliterator.fromAsync(path)) {
|
|
163
176
|
if (line) {
|
|
164
177
|
fn(line)
|
|
@@ -167,13 +180,16 @@ async function eachLine(path: string, fn: (line: string) => void): Promise<void>
|
|
|
167
180
|
}
|
|
168
181
|
|
|
169
182
|
/**
|
|
170
|
-
* Fetch one state's P.L. 94-171 block race counts into `pl_block`.
|
|
183
|
+
* Fetch one state's P.L. 94-171 block race counts into `pl_block`.
|
|
184
|
+
*
|
|
185
|
+
* Yields progress.
|
|
186
|
+
* Returns the tally.
|
|
171
187
|
*/
|
|
172
188
|
export async function* fetchRedistricting(
|
|
173
189
|
options: FetchRedistrictingOptions
|
|
174
190
|
): AsyncGenerator<FetchRedistrictingEvent, FetchRedistrictingResult> {
|
|
175
191
|
const vintage = options.vintage ?? 2020
|
|
176
|
-
const dataRoot = options.dataRoot ?? DEFAULT_DATA_ROOT
|
|
192
|
+
const dataRoot = PathBuilder.from(options.dataRoot ?? DEFAULT_DATA_ROOT)
|
|
177
193
|
const batchSize = options.batchSize ?? 2000
|
|
178
194
|
const state = options.stateFIPS
|
|
179
195
|
|
|
@@ -184,26 +200,26 @@ export async function* fetchRedistricting(
|
|
|
184
200
|
const dirName = stateName.replaceAll(" ", "_")
|
|
185
201
|
const fileAbbr = abbr.toLowerCase()
|
|
186
202
|
|
|
187
|
-
const cacheDir =
|
|
203
|
+
const cacheDir = dataRoot("census", "redistricting", String(vintage), state)
|
|
188
204
|
// Same stable `tiger.db` default as fetchTIGER — pl_block lives alongside tabblock20 in one DB.
|
|
189
|
-
const outPath = options.outPath ??
|
|
205
|
+
const outPath = PathBuilder.from(options.outPath ?? dataRoot("tiger", "tiger.db"))
|
|
190
206
|
await makeDirectories(cacheDir)
|
|
191
|
-
await makeDirectories(dirname(
|
|
207
|
+
await makeDirectories(outPath.dirname())
|
|
192
208
|
|
|
193
209
|
const zipName = `${fileAbbr}${vintage}.pl.zip`
|
|
194
|
-
const zipPath =
|
|
210
|
+
const zipPath = cacheDir(zipName)
|
|
195
211
|
const url = `${REDISTRICTING_BASE}/${dirName}/${zipName}`
|
|
196
212
|
|
|
197
213
|
const cached = await downloadIfNeeded(url, zipPath)
|
|
198
214
|
yield { phase: "download", file: zipName, cached }
|
|
199
215
|
|
|
200
216
|
await extractZipEntries(zipPath, cacheDir)
|
|
201
|
-
const geoPath =
|
|
202
|
-
const seg1Path =
|
|
203
|
-
const seg2Path =
|
|
217
|
+
const geoPath = cacheDir(`${fileAbbr}geo${vintage}.pl`)
|
|
218
|
+
const seg1Path = cacheDir(`${fileAbbr}00001${vintage}.pl`)
|
|
219
|
+
const seg2Path = cacheDir(`${fileAbbr}00002${vintage}.pl`)
|
|
204
220
|
yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` }
|
|
205
221
|
|
|
206
|
-
// Pass 1: header →
|
|
222
|
+
// Pass 1: header → logrecno → geoid for the blocks we want.
|
|
207
223
|
const prefix = options.county ? state + options.county : state
|
|
208
224
|
const logToGeoid = new Map<string, string>()
|
|
209
225
|
|
|
@@ -220,8 +236,9 @@ export async function* fetchRedistricting(
|
|
|
220
236
|
const total = logToGeoid.size
|
|
221
237
|
yield { phase: "header", blocks: total }
|
|
222
238
|
|
|
223
|
-
// Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs.
|
|
224
|
-
//
|
|
239
|
+
// Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs.
|
|
240
|
+
// Read before segment 1 so a block's P2 and H1 land in one row.
|
|
241
|
+
// A mapped block with no segment-2 row is a data defect the load refuses, never a zero.
|
|
225
242
|
const h1ByLogrecno = new Map<string, H1Counts>()
|
|
226
243
|
|
|
227
244
|
await eachLine(seg2Path, (line) => {
|
|
@@ -297,7 +314,7 @@ export async function* fetchRedistricting(
|
|
|
297
314
|
yield { phase: "load", inserted, total }
|
|
298
315
|
kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
|
299
316
|
|
|
300
|
-
return { outPath, table: "pl_block", inserted }
|
|
317
|
+
return { outPath: outPath.toString(), table: "pl_block", inserted }
|
|
301
318
|
} finally {
|
|
302
319
|
await kdb.destroy()
|
|
303
320
|
}
|
|
@@ -4,12 +4,12 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { smartSnakeCase } from "
|
|
7
|
+
import { smartSnakeCase } from "spliterator"
|
|
8
8
|
|
|
9
9
|
import { TIGERProperty } from "#index"
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
|
-
* Default column selections for
|
|
12
|
+
* Default column selections for tiger tabulated block entities.
|
|
13
13
|
*
|
|
14
14
|
* @internal
|
|
15
15
|
*/
|
|
@@ -23,4 +23,4 @@ export const TIGERTabulatedBlockEntitySelections = (
|
|
|
23
23
|
TIGERProperty.WaterAreaSqm,
|
|
24
24
|
TIGERProperty.Population,
|
|
25
25
|
] as const
|
|
26
|
-
).map(
|
|
26
|
+
).map(smartSnakeCase)
|
|
@@ -22,7 +22,7 @@ export interface StateBlockIntersectionCriteria {
|
|
|
22
22
|
tractCodes?: FIPSTractCode[]
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
// #region Row parsing
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
28
|
* @internal
|
|
@@ -36,7 +36,7 @@ export interface TIGERBlockRow extends TIGERTabulatedBlockProperties {
|
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
/**
|
|
39
|
-
* Given a row from the
|
|
39
|
+
* Given a row from the tiger tabulated block table, parses it into a GeoJSON feature.
|
|
40
40
|
*
|
|
41
41
|
* @internal
|
|
42
42
|
*/
|
|
@@ -52,7 +52,8 @@ export function parseTIGERBlockFromRow({ serializedGeometry, ...properties }: TI
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
/**
|
|
55
|
-
* Given a collection of rows from the
|
|
55
|
+
* Given a collection of rows from the tiger tabulated block table,
|
|
56
|
+
* parses them into a GeoJSON feature collection.
|
|
56
57
|
*
|
|
57
58
|
* @internal
|
|
58
59
|
*/
|
|
@@ -67,4 +68,4 @@ export function parseTIGERBlockFeatureCollectionFromRows(rows: TIGERBlockRow[]):
|
|
|
67
68
|
return featureCollection
|
|
68
69
|
}
|
|
69
70
|
|
|
70
|
-
|
|
71
|
+
// #endregion
|
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
* category. The dots PMTiles is read client-side via the `pmtiles` protocol.
|
|
10
10
|
*
|
|
11
11
|
* Each dot is one of `per` people of a category, placed at random inside its Census block — a
|
|
12
|
-
* representation
|
|
13
|
-
*
|
|
12
|
+
* representation rather than a record about any address. Serve over localhost (the house tile server
|
|
13
|
+
* cors-restricts to localhost + the docs domains) — `mailwoman tiger race-dots-map --serve` wires
|
|
14
14
|
* {@linkcode serveWithRangeSupport} for exactly this.
|
|
15
15
|
*
|
|
16
16
|
* Run: `mailwoman tiger race-dots-map --pmtiles-url
|
|
@@ -19,37 +19,52 @@
|
|
|
19
19
|
|
|
20
20
|
import { tempRootPath } from "@mailwoman/core/data-root"
|
|
21
21
|
import { writeLocalFile } from "@mailwoman/core/fs/writers"
|
|
22
|
+
import { stringifyJSON } from "@mailwoman/core/json"
|
|
22
23
|
|
|
23
24
|
/**
|
|
24
25
|
* Options for {@linkcode raceDotsMap}.
|
|
25
26
|
*/
|
|
26
27
|
export interface RaceDotsMapOptions {
|
|
27
28
|
/**
|
|
28
|
-
* Dots tileset URL the page reads client-side.
|
|
29
|
+
* Dots tileset URL the page reads client-side.
|
|
30
|
+
*
|
|
31
|
+
* Default `http://localhost:8899/race-dots-oc.pmtiles`.
|
|
29
32
|
*/
|
|
30
33
|
pmtilesURL?: string
|
|
31
34
|
/**
|
|
32
|
-
* Output
|
|
35
|
+
* Output html path.
|
|
36
|
+
*
|
|
37
|
+
* Default `/tmp/race-dots-oc.html`.
|
|
33
38
|
*/
|
|
34
39
|
out?: string
|
|
35
40
|
/**
|
|
36
|
-
* People represented by one dot (title/legend copy only).
|
|
41
|
+
* People represented by one dot (title/legend copy only).
|
|
42
|
+
*
|
|
43
|
+
* Default 5.
|
|
37
44
|
*/
|
|
38
45
|
per?: number
|
|
39
46
|
/**
|
|
40
|
-
* Page title.
|
|
47
|
+
* Page title.
|
|
48
|
+
*
|
|
49
|
+
* Default derives from `per`.
|
|
41
50
|
*/
|
|
42
51
|
title?: string
|
|
43
52
|
/**
|
|
44
|
-
* Initial map center longitude.
|
|
53
|
+
* Initial map center longitude.
|
|
54
|
+
*
|
|
55
|
+
* Default -117.83 (Orange County, CA).
|
|
45
56
|
*/
|
|
46
57
|
lng?: number
|
|
47
58
|
/**
|
|
48
|
-
* Initial map center latitude.
|
|
59
|
+
* Initial map center latitude.
|
|
60
|
+
*
|
|
61
|
+
* Default 33.68.
|
|
49
62
|
*/
|
|
50
63
|
lat?: number
|
|
51
64
|
/**
|
|
52
|
-
* Initial map zoom.
|
|
65
|
+
* Initial map zoom.
|
|
66
|
+
*
|
|
67
|
+
* Default 9.4.
|
|
53
68
|
*/
|
|
54
69
|
zoom?: number
|
|
55
70
|
}
|
|
@@ -69,7 +84,10 @@ const PMTILES_VERSION = "4.4.1"
|
|
|
69
84
|
|
|
70
85
|
const BASEMAP_SOURCE_ID = "basemap-v4"
|
|
71
86
|
const BASEMAP_TILEJSON_URL = "https://tiles.mailwoman.ai/basemap-v4.json"
|
|
72
|
-
|
|
87
|
+
// The glyph mirror and its Bliss Pro stacks, as `@mailwoman/cartographer/styles/fonts` documents them.
|
|
88
|
+
// Cartographer depends on this package, so these values are restated here rather than imported.
|
|
89
|
+
const GLYPHS_URL = "https://public.mailwoman.ai/protomaps/fonts/{fontstack}/{range}.pbf"
|
|
90
|
+
const MAP_FONTS = { regular: "Bliss Pro Regular", bold: "Bliss Pro Medium", italic: "Bliss Pro Italic" }
|
|
73
91
|
const SPRITE_URL = "https://protomaps.github.io/basemaps-assets/sprites/v4/light"
|
|
74
92
|
|
|
75
93
|
/**
|
|
@@ -133,7 +151,7 @@ export async function raceDotsMap(
|
|
|
133
151
|
dots: { type: "vector", url: "pmtiles://" + PMTILES_URL },
|
|
134
152
|
},
|
|
135
153
|
layers: [
|
|
136
|
-
...(layers(BASEMAP_SOURCE_ID, namedFlavor("light"), { lang: "en" }) as unknown[]),
|
|
154
|
+
...(layers(BASEMAP_SOURCE_ID, { ...namedFlavor("light"), ...MAP_FONTS }, { lang: "en" }) as unknown[]),
|
|
137
155
|
{
|
|
138
156
|
id: "race-dots",
|
|
139
157
|
type: "circle",
|
|
@@ -182,7 +200,7 @@ export async function raceDotsMap(
|
|
|
182
200
|
maplibregl.addProtocol("pmtiles", protocol.tile);
|
|
183
201
|
var map = new maplibregl.Map({
|
|
184
202
|
container: "map",
|
|
185
|
-
style: ${
|
|
203
|
+
style: ${stringifyJSON(style)},
|
|
186
204
|
center: [${CENTER_LNG}, ${CENTER_LAT}],
|
|
187
205
|
zoom: ${ZOOM},
|
|
188
206
|
attributionControl: false
|
|
@@ -3,34 +3,17 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* Reads `tabblock20 ⋈ pl_block` (block geometry + Census 2020 P.L. 94-171 table P2 counts) and
|
|
10
|
-
* scatters one dot per `per` people uniformly at random inside each block, tagged with its
|
|
11
|
-
* race/ethnicity category. Output is NDJSON (one GeoJSON Point Feature per line, with a
|
|
12
|
-
* `tippecanoe` layer hint) ready for `tippecanoe -o race-dots.pmtiles`.
|
|
13
|
-
*
|
|
14
|
-
* Point-in-polygon uses `@turf/boolean-contains` (ships with `@mailwoman/tiger`). The dot is a
|
|
15
|
-
* _representation_, not a record: a random position inside the block it belongs to, standing in
|
|
16
|
-
* for `per` real people of that category. It says nothing about any individual address.
|
|
17
|
-
*
|
|
18
|
-
* Build the input DB first: `mailwoman tiger fetch --state 06 --county 059 --out tiger-oc.db` then
|
|
19
|
-
* `mailwoman tiger redistricting --state 06 --county 059 --out tiger-oc.db`.
|
|
20
|
-
*
|
|
21
|
-
* Run: `mailwoman tiger race-dots --db tiger-oc.db --per 10 --out /tmp/race-dots.ndjson`
|
|
6
|
+
* A dot is a representation rather than a record: a random position inside the block it belongs to,
|
|
7
|
+
* making no statement about any individual address.
|
|
22
8
|
*/
|
|
23
9
|
|
|
24
10
|
import { dataRootPath, tempRootPath } from "@mailwoman/core/data-root"
|
|
25
11
|
import { openWriteStream } from "@mailwoman/core/fs/streams"
|
|
26
|
-
import { tryParsingJSON } from "@mailwoman/core/json"
|
|
12
|
+
import { tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
|
|
27
13
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
28
14
|
|
|
29
|
-
import type { TIGERDatabase } from "#
|
|
15
|
+
import type { TIGERDatabase } from "#schema"
|
|
30
16
|
|
|
31
|
-
/**
|
|
32
|
-
* Attempts to place a dot inside its polygon by rejection sampling before giving up on it.
|
|
33
|
-
*/
|
|
34
17
|
const MAX_PLACEMENT_TRIES = 60
|
|
35
18
|
|
|
36
19
|
/**
|
|
@@ -38,19 +21,19 @@ const MAX_PLACEMENT_TRIES = 60
|
|
|
38
21
|
*/
|
|
39
22
|
export interface RaceDotsOptions {
|
|
40
23
|
/**
|
|
41
|
-
*
|
|
24
|
+
* Tiger SQLite DB (`tabblock20` ⋈ `pl_block`).
|
|
42
25
|
*/
|
|
43
26
|
db?: string
|
|
44
27
|
/**
|
|
45
|
-
* Output
|
|
28
|
+
* Output ndjson path.
|
|
46
29
|
*/
|
|
47
30
|
out?: string
|
|
48
31
|
/**
|
|
49
|
-
* People represented by one dot.
|
|
32
|
+
* People represented by one dot.
|
|
50
33
|
*/
|
|
51
34
|
per?: number
|
|
52
35
|
/**
|
|
53
|
-
* Tippecanoe layer name.
|
|
36
|
+
* Tippecanoe layer name.
|
|
54
37
|
*/
|
|
55
38
|
layer?: string
|
|
56
39
|
}
|
|
@@ -70,7 +53,7 @@ export interface RaceDotsResult {
|
|
|
70
53
|
}
|
|
71
54
|
|
|
72
55
|
/**
|
|
73
|
-
* The
|
|
56
|
+
* The P2 categories (columns in `pl_block`) that partition each block's population.
|
|
74
57
|
*/
|
|
75
58
|
const CATEGORIES = ["hispanic", "white", "black", "asian", "aian", "nhpi", "other", "multi"] as const
|
|
76
59
|
|
|
@@ -106,7 +89,7 @@ function bbox(rings: PolygonCoords): [number, number, number, number] {
|
|
|
106
89
|
}
|
|
107
90
|
|
|
108
91
|
/**
|
|
109
|
-
* Race-by-dot-density
|
|
92
|
+
* Race-by-dot-density ndjson builder.
|
|
110
93
|
*/
|
|
111
94
|
export async function raceDots(
|
|
112
95
|
options: RaceDotsOptions = {},
|
|
@@ -114,14 +97,14 @@ export async function raceDots(
|
|
|
114
97
|
): Promise<RaceDotsResult> {
|
|
115
98
|
const DB = options.db || dataRootPath("tiger", "tiger-oc.db")
|
|
116
99
|
const OUT = options.out || tempRootPath("race-dots.ndjson")
|
|
117
|
-
const PER = options.per ?? 10
|
|
100
|
+
const PER = options.per ?? 10
|
|
118
101
|
const LAYER = options.layer || "dots"
|
|
119
102
|
|
|
120
103
|
// Heavy dep, lazy-imported so loading the tools barrel stays cheap.
|
|
121
104
|
const { default: booleanContains } = await import("@turf/boolean-contains")
|
|
122
105
|
|
|
123
|
-
//
|
|
124
|
-
//
|
|
106
|
+
// Pick a bbox-area-weighted sub-polygon, then rejection-sample inside it.
|
|
107
|
+
// Turf handles holes and winding.
|
|
125
108
|
function randomPointIn(polys: PolygonCoords[], areas: number[], totalArea: number): [number, number] | null {
|
|
126
109
|
let r = Math.random() * totalArea
|
|
127
110
|
let pick = 0
|
|
@@ -200,7 +183,7 @@ export async function raceDots(
|
|
|
200
183
|
}
|
|
201
184
|
|
|
202
185
|
out.write(
|
|
203
|
-
|
|
186
|
+
stringifyJSON({
|
|
204
187
|
type: "Feature",
|
|
205
188
|
tippecanoe: { layer: LAYER },
|
|
206
189
|
properties: { cat },
|
|
@@ -3,19 +3,19 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
* Minimal static file server that
|
|
7
|
-
* not
|
|
6
|
+
* Minimal static file server that honors http Range requests (206) — Python's `http.server` does
|
|
7
|
+
* not. PMTiles reads via Range, so this server cannot serve it. Used to preview/render a local
|
|
8
8
|
* `.pmtiles` tileset (e.g. the race-dot map, via `mailwoman tiger race-dots-map --serve`). Not for
|
|
9
|
-
* production
|
|
9
|
+
* production. the deployed tiles go through the tile worker.
|
|
10
10
|
*
|
|
11
11
|
* Internal helper module — no standalone command.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"
|
|
15
15
|
|
|
16
|
+
import { tempRootPath } from "@mailwoman/core/data-root"
|
|
16
17
|
import { statPath, type Stats } from "@mailwoman/core/fs/readers"
|
|
17
18
|
import { openReadStream } from "@mailwoman/core/fs/streams"
|
|
18
|
-
import { mailwomanTempRoot } from "@mailwoman/core/utils"
|
|
19
19
|
import { extname, join, normalize } from "path-ts"
|
|
20
20
|
|
|
21
21
|
/**
|
|
@@ -23,11 +23,15 @@ import { extname, join, normalize } from "path-ts"
|
|
|
23
23
|
*/
|
|
24
24
|
export interface ServeRangeOptions {
|
|
25
25
|
/**
|
|
26
|
-
* Directory to serve.
|
|
26
|
+
* Directory to serve.
|
|
27
|
+
*
|
|
28
|
+
* Default `/tmp`.
|
|
27
29
|
*/
|
|
28
30
|
dir?: string
|
|
29
31
|
/**
|
|
30
|
-
* Port to listen on.
|
|
32
|
+
* Port to listen on.
|
|
33
|
+
*
|
|
34
|
+
* Default 8899.
|
|
31
35
|
*/
|
|
32
36
|
port?: number
|
|
33
37
|
}
|
|
@@ -50,14 +54,16 @@ const TYPES: Record<string, string> = {
|
|
|
50
54
|
}
|
|
51
55
|
|
|
52
56
|
/**
|
|
53
|
-
* Serve `dir` over localhost with
|
|
54
|
-
*
|
|
57
|
+
* Serve `dir` over localhost with http Range support.
|
|
58
|
+
*
|
|
59
|
+
* Resolves once the server is listening.
|
|
60
|
+
* The caller owns the lifetime (dispose it to stop. Commands typically hold the process open instead).
|
|
55
61
|
*/
|
|
56
62
|
export async function serveWithRangeSupport(
|
|
57
63
|
options: ServeRangeOptions = {},
|
|
58
64
|
report?: (line: string) => void
|
|
59
65
|
): Promise<RangeServer> {
|
|
60
|
-
const dir = options.dir ||
|
|
66
|
+
const dir = options.dir || tempRootPath()
|
|
61
67
|
const port = options.port ?? 8899
|
|
62
68
|
|
|
63
69
|
const server = createServer(async (req: IncomingMessage, res: ServerResponse) => {
|
package/lib/sdk/download.ts
DELETED
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Cached archive download + subprocess capture, shared by `fetch.ts` and `redistricting.ts`.
|
|
7
|
-
*
|
|
8
|
-
* RAW `fetch` HERE IS DELIBERATE. `AGENTS.md` requires HTTP clients to extend or instantiate
|
|
9
|
-
* `APIClient`, and that rule is about API REQUESTS — small bodies, repeated calls, rate-limited
|
|
10
|
-
* hosts, where its pacing, bounded retry, response caching and `ResourceError` mapping all earn
|
|
11
|
-
* their keep. This is a multi-gigabyte file transfer streamed straight to disk. Response caching
|
|
12
|
-
* would be nonsense at that size, pacing has nothing to pace (one request), and axios buffers a
|
|
13
|
-
* non-stream response type in memory. The primitive that fits a body this large is the one that
|
|
14
|
-
* never holds it: a web stream piped to a write stream.
|
|
15
|
-
*/
|
|
16
|
-
|
|
17
|
-
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
18
|
-
import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
|
|
19
|
-
import { movePath } from "@mailwoman/core/fs/writers"
|
|
20
|
-
import { verifyZipIntegrity } from "@mailwoman/core/fs/zip"
|
|
21
|
-
|
|
22
|
-
/**
|
|
23
|
-
* Download `url` to `dest` unless a VALID copy is already there. Returns `true` when the cache was reused, `false` when
|
|
24
|
-
* a download happened.
|
|
25
|
-
*
|
|
26
|
-
* "Valid" means every member's CRC-32 checks out — an existence check alone is not enough, because an interrupted
|
|
27
|
-
* download leaves a plausible-looking file that fails only much later, inside ogr2ogr. Writes to a `.tmp` sibling and
|
|
28
|
-
* renames, so `dest` is never a partial archive.
|
|
29
|
-
*/
|
|
30
|
-
export async function downloadIfNeeded(url: string, dest: string): Promise<boolean> {
|
|
31
|
-
if (await pathExists(dest)) {
|
|
32
|
-
try {
|
|
33
|
-
await verifyZipIntegrity(dest)
|
|
34
|
-
|
|
35
|
-
return true
|
|
36
|
-
} catch {
|
|
37
|
-
// corrupt cache — re-download
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const tmp = dest + ".tmp"
|
|
42
|
-
const res = await fetch(url, { redirect: "follow" })
|
|
43
|
-
|
|
44
|
-
if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} fetching ${url}`)
|
|
45
|
-
await pipeline(Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]), openWriteStream(tmp))
|
|
46
|
-
await movePath(tmp, dest)
|
|
47
|
-
|
|
48
|
-
return false
|
|
49
|
-
}
|
package/lib/sdk/index.ts
DELETED
package/lib/tools/index.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"county-subdivision.d.ts","sourceRoot":"","sources":["../lib/county-subdivision.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AACjD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,KAAK,EAAE,+BAA+B,EAAE,MAAM,yBAAyB,CAAA;AAC9E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AAE7C;;;;;GAKG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,OAAO,EAAE,eAAe,CAAA;IAExB;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAA;IAEhB;;;;;;OAMG;IACH,QAAQ,EAAE,MAAM,CAAA;IAEhB;;;;;;;OAOG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;;;;;;OAOG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;;;;;OAMG;IACH,QAAQ,EAAE,MAAM,CAAA;IAEhB;;;;OAIG;IACH,IAAI,EAAE,+BAA+B,CAAA;IAErC;;;;OAIG;IACH,OAAO,EAAE,cAAc,CAAA;IAEvB;;;;;;;;OAQG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;;;;;;OAOG;IACH,QAAQ,EAAE,qBAAqB,CAAA;IAE/B;;;;;OAKG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;;;;OAKG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAA;IAEhB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAA;IAEhB;;;;;;OAMG;IACH,IAAI,EAAE,MAAM,CAAA;CACZ"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"county-subdivision.js","sourceRoot":"","sources":["../lib/county-subdivision.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
package/out/sdk/index.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software.
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*/
|
|
6
|
-
export * from "#sdk/fetch";
|
|
7
|
-
export * from "#sdk/redistricting";
|
|
8
|
-
export * from "#sdk/schema";
|
|
9
|
-
export * from "#sdk/state/index";
|
|
10
|
-
//# sourceMappingURL=index.d.ts.map
|
package/out/sdk/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../lib/sdk/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,aAAa,CAAA;AAC3B,cAAc,kBAAkB,CAAA"}
|
package/out/sdk/index.js
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software.
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*/
|
|
6
|
-
export * from "#sdk/fetch";
|
|
7
|
-
export * from "#sdk/redistricting";
|
|
8
|
-
export * from "#sdk/schema";
|
|
9
|
-
export * from "#sdk/state/index";
|
|
10
|
-
//# sourceMappingURL=index.js.map
|
package/out/sdk/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/sdk/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,aAAa,CAAA;AAC3B,cAAc,kBAAkB,CAAA"}
|