@mailwoman/tiger 10.0.0 → 10.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/block-group.ts +7 -7
- package/lib/class-code.ts +387 -266
- package/lib/constants.ts +7 -6
- package/lib/county/subdivision.ts +7 -7
- package/lib/{county/index.ts → county.ts} +7 -7
- package/lib/files.ts +37 -20
- package/lib/functional-status.ts +6 -4
- package/lib/geoid.ts +44 -26
- package/lib/index.ts +1 -1
- package/lib/legal-statistical-area.ts +7 -5
- package/lib/{sdk/schema.ts → schema.ts} +32 -42
- package/lib/state.ts +26 -23
- package/lib/tabulation-block.ts +4 -4
- package/lib/tract.ts +3 -3
- package/out/block-group.d.ts +7 -7
- package/out/class-code.d.ts +387 -266
- package/out/class-code.d.ts.map +1 -1
- package/out/class-code.js +387 -266
- package/out/class-code.js.map +1 -1
- package/out/constants.d.ts +7 -6
- package/out/constants.d.ts.map +1 -1
- package/out/constants.js +3 -3
- package/out/county/subdivision.d.ts +7 -7
- package/out/{county/index.d.ts → county.d.ts} +8 -8
- package/out/county.d.ts.map +1 -0
- package/out/{county/index.js → county.js} +2 -2
- package/out/county.js.map +1 -0
- package/out/files.d.ts +21 -13
- package/out/files.d.ts.map +1 -1
- package/out/files.js +27 -13
- package/out/files.js.map +1 -1
- package/out/functional-status.d.ts +6 -4
- package/out/functional-status.d.ts.map +1 -1
- package/out/functional-status.js +6 -4
- package/out/functional-status.js.map +1 -1
- package/out/geoid.d.ts +34 -16
- package/out/geoid.d.ts.map +1 -1
- package/out/geoid.js +16 -10
- package/out/geoid.js.map +1 -1
- package/out/index.d.ts +1 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +1 -1
- package/out/index.js.map +1 -1
- package/out/legal-statistical-area.d.ts +7 -5
- package/out/legal-statistical-area.d.ts.map +1 -1
- package/out/legal-statistical-area.js +7 -5
- package/out/legal-statistical-area.js.map +1 -1
- package/out/{sdk/schema.d.ts → schema.d.ts} +26 -20
- package/out/schema.d.ts.map +1 -0
- package/out/{sdk/schema.js → schema.js} +12 -31
- package/out/schema.js.map +1 -0
- package/out/sdk/download.d.ts +7 -15
- package/out/sdk/download.d.ts.map +1 -1
- package/out/sdk/download.js +6 -15
- package/out/sdk/download.js.map +1 -1
- package/out/sdk/fetch.d.ts +36 -29
- package/out/sdk/fetch.d.ts.map +1 -1
- package/out/sdk/fetch.js +18 -53
- package/out/sdk/fetch.js.map +1 -1
- package/out/sdk/redistricting.d.ts +29 -15
- package/out/sdk/redistricting.d.ts.map +1 -1
- package/out/sdk/redistricting.js +47 -36
- package/out/sdk/redistricting.js.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts +2 -2
- package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTabulatedBlockEntity.js +3 -3
- package/out/sdk/state/TIGERTabulatedBlockEntity.js.map +1 -1
- package/out/sdk/state/TIGERTractEntity.d.ts.map +1 -1
- package/out/sdk/state/TIGERTractEntity.js.map +1 -1
- package/out/sdk/state/block-operations.d.ts +3 -2
- package/out/sdk/state/block-operations.d.ts.map +1 -1
- package/out/sdk/state/block-operations.js +4 -3
- package/out/sdk/state/block-operations.js.map +1 -1
- package/out/sdk/{state/index.d.ts → state.d.ts} +1 -1
- package/out/sdk/state.d.ts.map +1 -0
- package/out/sdk/{state/index.js → state.js} +1 -1
- package/out/sdk/state.js.map +1 -0
- package/out/sdk/tsconfig.tsbuildinfo +1 -0
- package/out/state.d.ts +18 -15
- package/out/state.d.ts.map +1 -1
- package/out/state.js +17 -14
- package/out/state.js.map +1 -1
- package/out/tabulation-block.d.ts +4 -4
- package/out/tools/race/dots/map.d.ts +23 -9
- package/out/tools/race/dots/map.d.ts.map +1 -1
- package/out/tools/race/dots/map.js +9 -5
- package/out/tools/race/dots/map.js.map +1 -1
- package/out/tools/race/dots.d.ts +47 -0
- package/out/tools/race/dots.d.ts.map +1 -0
- package/out/tools/race/{dots/index.js → dots.js} +10 -27
- package/out/tools/race/dots.js.map +1 -0
- package/out/tools/serve-range.d.ts +13 -7
- package/out/tools/serve-range.d.ts.map +1 -1
- package/out/tools/serve-range.js +10 -8
- package/out/tools/serve-range.js.map +1 -1
- package/out/tools/tsconfig.tsbuildinfo +1 -0
- package/out/tract.d.ts +3 -3
- package/out/tsconfig.test.tsbuildinfo +1 -1
- package/package.json +61 -44
- package/sdk/download.ts +41 -0
- package/{lib/sdk → sdk}/fetch.ts +58 -62
- package/{lib/sdk → sdk}/redistricting.ts +57 -40
- package/{lib/sdk → sdk}/state/TIGERTabulatedBlockEntity.ts +3 -3
- package/{lib/sdk → sdk}/state/block-operations.ts +5 -4
- package/{lib/tools → tools}/race/dots/map.ts +30 -12
- package/{lib/tools/race/dots/index.ts → tools/race/dots.ts} +14 -31
- package/{lib/tools → tools}/serve-range.ts +17 -10
- package/lib/sdk/download.ts +0 -49
- package/lib/sdk/index.ts +0 -10
- package/lib/tools/index.ts +0 -11
- package/out/county/index.d.ts.map +0 -1
- package/out/county/index.js.map +0 -1
- package/out/sdk/index.d.ts +0 -10
- package/out/sdk/index.d.ts.map +0 -1
- package/out/sdk/index.js +0 -10
- package/out/sdk/index.js.map +0 -1
- package/out/sdk/schema.d.ts.map +0 -1
- package/out/sdk/schema.js.map +0 -1
- package/out/sdk/state/index.d.ts.map +0 -1
- package/out/sdk/state/index.js.map +0 -1
- package/out/tools/index.d.ts +0 -11
- package/out/tools/index.d.ts.map +0 -1
- package/out/tools/index.js +0 -11
- package/out/tools/index.js.map +0 -1
- package/out/tools/race/dots/index.d.ts +0 -61
- package/out/tools/race/dots/index.d.ts.map +0 -1
- package/out/tools/race/dots/index.js.map +0 -1
- /package/{lib/sdk → sdk}/state/TIGERTractEntity.ts +0 -0
- /package/{lib/sdk/state/index.ts → sdk/state.ts} +0 -0
package/out/sdk/download.d.ts
CHANGED
|
@@ -3,23 +3,15 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* RAW `fetch` HERE IS DELIBERATE. `AGENTS.md` requires HTTP clients to extend or instantiate
|
|
9
|
-
* `APIClient`, and that rule is about API REQUESTS — small bodies, repeated calls, rate-limited
|
|
10
|
-
* hosts, where its pacing, bounded retry, response caching and `ResourceError` mapping all earn
|
|
11
|
-
* their keep. This is a multi-gigabyte file transfer streamed straight to disk. Response caching
|
|
12
|
-
* would be nonsense at that size, pacing has nothing to pace (one request), and axios buffers a
|
|
13
|
-
* non-stream response type in memory. The primitive that fits a body this large is the one that
|
|
14
|
-
* never holds it: a web stream piped to a write stream.
|
|
6
|
+
* Raw `fetch` is deliberate for a multi-gigabyte streamed transfer: `APIClient`'s response caching
|
|
7
|
+
* and pacing do not apply.
|
|
15
8
|
*/
|
|
9
|
+
import type { PathBuilderLike } from "path-ts";
|
|
16
10
|
/**
|
|
17
|
-
* Download `url` to `dest` unless a
|
|
18
|
-
* a download happened.
|
|
11
|
+
* Download `url` to `dest` unless a valid copy is already there.
|
|
19
12
|
*
|
|
20
|
-
* "Valid" means every member's CRC-32 checks out
|
|
21
|
-
* download leaves a plausible
|
|
22
|
-
* renames, so `dest` is never a partial archive.
|
|
13
|
+
* "Valid" means every member's CRC-32 checks out.
|
|
14
|
+
* An interrupted download otherwise leaves a plausible file that fails later inside ogr2ogr.
|
|
23
15
|
*/
|
|
24
|
-
export declare function downloadIfNeeded(url: string, dest:
|
|
16
|
+
export declare function downloadIfNeeded(url: string, dest: PathBuilderLike): Promise<boolean>;
|
|
25
17
|
//# sourceMappingURL=download.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,CAmB3F"}
|
package/out/sdk/download.js
CHANGED
|
@@ -3,27 +3,18 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* RAW `fetch` HERE IS DELIBERATE. `AGENTS.md` requires HTTP clients to extend or instantiate
|
|
9
|
-
* `APIClient`, and that rule is about API REQUESTS — small bodies, repeated calls, rate-limited
|
|
10
|
-
* hosts, where its pacing, bounded retry, response caching and `ResourceError` mapping all earn
|
|
11
|
-
* their keep. This is a multi-gigabyte file transfer streamed straight to disk. Response caching
|
|
12
|
-
* would be nonsense at that size, pacing has nothing to pace (one request), and axios buffers a
|
|
13
|
-
* non-stream response type in memory. The primitive that fits a body this large is the one that
|
|
14
|
-
* never holds it: a web stream piped to a write stream.
|
|
6
|
+
* Raw `fetch` is deliberate for a multi-gigabyte streamed transfer: `APIClient`'s response caching
|
|
7
|
+
* and pacing do not apply.
|
|
15
8
|
*/
|
|
16
|
-
import { pathExists } from "@mailwoman/core/fs/readers";
|
|
9
|
+
import { pathExists } from "@mailwoman/core/fs/readers/stat";
|
|
17
10
|
import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
|
|
18
11
|
import { movePath } from "@mailwoman/core/fs/writers";
|
|
19
12
|
import { verifyZipIntegrity } from "@mailwoman/core/fs/zip";
|
|
20
13
|
/**
|
|
21
|
-
* Download `url` to `dest` unless a
|
|
22
|
-
* a download happened.
|
|
14
|
+
* Download `url` to `dest` unless a valid copy is already there.
|
|
23
15
|
*
|
|
24
|
-
* "Valid" means every member's CRC-32 checks out
|
|
25
|
-
* download leaves a plausible
|
|
26
|
-
* renames, so `dest` is never a partial archive.
|
|
16
|
+
* "Valid" means every member's CRC-32 checks out.
|
|
17
|
+
* An interrupted download otherwise leaves a plausible file that fails later inside ogr2ogr.
|
|
27
18
|
*/
|
|
28
19
|
export async function downloadIfNeeded(url, dest) {
|
|
29
20
|
if (await pathExists(dest)) {
|
package/out/sdk/download.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAG3D;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,IAAqB;IACxE,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,IAAI,CAAC;YACJ,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAE9B,OAAO,IAAI,CAAA;QACZ,CAAC;QAAC,MAAM,CAAC;YACR,8BAA8B;QAC/B,CAAC;IACF,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,MAAM,CAAA;IACzB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC,CAAA;IAC/E,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAA8C,CAAC,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1G,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAEzB,OAAO,KAAK,CAAA;AACb,CAAC"}
|
package/out/sdk/fetch.d.ts
CHANGED
|
@@ -2,61 +2,62 @@
|
|
|
2
2
|
* @copyright Sister Software.
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Pull TIGER geometry/attributes into a `node:sqlite` database via the Kysely
|
|
7
|
-
* {@link DatabaseClient}.
|
|
8
|
-
*
|
|
9
|
-
* Replaces the ad-hoc wget and the retired corpus TIGER build script. Pared down from isp-nexus's
|
|
10
|
-
* `generate-tiger-tiles.ts` (its per-level column mapping is the spec) into the playpen
|
|
11
|
-
* `repo-tools` idiom: an async generator of progress events, idempotent, no SpatiaLite.
|
|
12
|
-
*
|
|
13
|
-
* Three levels:
|
|
14
|
-
*
|
|
15
|
-
* - `tabblock20` (per state) — tabulation blocks → `tabblock20`, geometry as GeoJSON text.
|
|
16
|
-
* - `place` (per state) — incorporated/census places → `tiger_places` (attribute-only).
|
|
17
|
-
* - `addrfeat` (per county) — named street segments + ZIPs → `tiger_streets` (attribute-only).
|
|
18
|
-
*
|
|
19
|
-
* `tiger_streets` + `tiger_places` match the schema the corpus `tiger` adapter reads, so this is a
|
|
20
|
-
* drop-in replacement for the retired corpus TIGER build script.
|
|
21
|
-
*
|
|
22
|
-
* Flow per source unit: download (skips a valid cached zip) → unzip → stream `ogr2ogr -f
|
|
23
|
-
* GeoJSONSeq` (mapping shapefile columns to the schema, WGS84 for geometry levels) → batched
|
|
24
|
-
* inserts. Re-running a state replaces its rows.
|
|
25
5
|
*/
|
|
26
6
|
/**
|
|
27
|
-
*
|
|
7
|
+
* A TIGER/Line product that {@link fetchTIGER} can load.
|
|
8
|
+
* Only the `tabblock20` level keeps geometry.
|
|
28
9
|
*/
|
|
29
10
|
export type TIGERFetchLevel = "tabblock20" | "place" | "addrfeat";
|
|
11
|
+
/**
|
|
12
|
+
* Options for {@link fetchTIGER}.
|
|
13
|
+
*/
|
|
30
14
|
export interface FetchTIGEROptions {
|
|
31
15
|
/**
|
|
32
|
-
*
|
|
16
|
+
* The two-digit state FIPS code, such as `"06"`.
|
|
33
17
|
*/
|
|
34
18
|
stateFIPS: string;
|
|
35
19
|
/**
|
|
36
|
-
* TIGER level
|
|
20
|
+
* The TIGER level to fetch.
|
|
21
|
+
*
|
|
22
|
+
* @defaultValue `tabblock20`
|
|
37
23
|
*/
|
|
38
24
|
level?: TIGERFetchLevel;
|
|
39
25
|
/**
|
|
40
|
-
*
|
|
26
|
+
* The TIGER vintage year.
|
|
27
|
+
*
|
|
28
|
+
* Other levels default to 2024.
|
|
29
|
+
*
|
|
30
|
+
* @defaultValue `2020` for blocks to match redistricting data.
|
|
41
31
|
*/
|
|
42
32
|
vintage?: number;
|
|
43
33
|
/**
|
|
44
|
-
*
|
|
34
|
+
* The output SQLite path.
|
|
35
|
+
*
|
|
36
|
+
* @defaultValue `<dataRoot>/tiger/tiger.db`, which the corpus `tiger` adapter reads.
|
|
45
37
|
*/
|
|
46
38
|
outPath?: string;
|
|
47
39
|
/**
|
|
48
|
-
*
|
|
40
|
+
* The root of the download cache and the default output.
|
|
41
|
+
*
|
|
42
|
+
* @defaultValue the Mailwoman data root.
|
|
49
43
|
*/
|
|
50
44
|
dataRoot?: string;
|
|
51
45
|
/**
|
|
52
|
-
*
|
|
46
|
+
* A three-digit county FIPS filter that applies only to the `tabblock20` level.
|
|
53
47
|
*/
|
|
54
48
|
county?: string;
|
|
55
49
|
/**
|
|
56
|
-
*
|
|
50
|
+
* The number of rows per insert batch.
|
|
51
|
+
*
|
|
52
|
+
* @defaultValue `1000`
|
|
57
53
|
*/
|
|
58
54
|
batchSize?: number;
|
|
59
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* A progress event from {@link fetchTIGER}.
|
|
58
|
+
*
|
|
59
|
+
* Each event reports a file downloaded or found in the cache, a shapefile extracted, or rows inserted.
|
|
60
|
+
*/
|
|
60
61
|
export type FetchTIGEREvent = {
|
|
61
62
|
phase: "download";
|
|
62
63
|
file: string;
|
|
@@ -69,13 +70,19 @@ export type FetchTIGEREvent = {
|
|
|
69
70
|
inserted: number;
|
|
70
71
|
total: number;
|
|
71
72
|
};
|
|
73
|
+
/**
|
|
74
|
+
* The database path, table name and inserted row count that {@link fetchTIGER} returns.
|
|
75
|
+
*/
|
|
72
76
|
export interface FetchTIGERResult {
|
|
73
77
|
outPath: string;
|
|
74
78
|
table: string;
|
|
75
79
|
inserted: number;
|
|
76
80
|
}
|
|
77
81
|
/**
|
|
78
|
-
*
|
|
82
|
+
* Downloads one state's TIGER/Line files at `level`, loads them into a SQLite database
|
|
83
|
+
* and yields progress events.
|
|
84
|
+
*
|
|
85
|
+
* It first deletes the state's existing rows from the target table so that a rerun replaces them.
|
|
79
86
|
*/
|
|
80
87
|
export declare function fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<FetchTIGEREvent, FetchTIGERResult>;
|
|
81
88
|
//# sourceMappingURL=fetch.d.ts.map
|
package/out/sdk/fetch.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../
|
|
1
|
+
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../sdk/fetch.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,OAAO,GAAG,UAAU,CAAA;AAcjE;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;;;OAIG;IACH,KAAK,CAAC,EAAE,eAAe,CAAA;IAEvB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACxB;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAA;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAiFD;;;;;GAKG;AACH,wBAAuB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,cAAc,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAiH/G"}
|
package/out/sdk/fetch.js
CHANGED
|
@@ -2,38 +2,19 @@
|
|
|
2
2
|
* @copyright Sister Software.
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Pull TIGER geometry/attributes into a `node:sqlite` database via the Kysely
|
|
7
|
-
* {@link DatabaseClient}.
|
|
8
|
-
*
|
|
9
|
-
* Replaces the ad-hoc wget and the retired corpus TIGER build script. Pared down from isp-nexus's
|
|
10
|
-
* `generate-tiger-tiles.ts` (its per-level column mapping is the spec) into the playpen
|
|
11
|
-
* `repo-tools` idiom: an async generator of progress events, idempotent, no SpatiaLite.
|
|
12
|
-
*
|
|
13
|
-
* Three levels:
|
|
14
|
-
*
|
|
15
|
-
* - `tabblock20` (per state) — tabulation blocks → `tabblock20`, geometry as GeoJSON text.
|
|
16
|
-
* - `place` (per state) — incorporated/census places → `tiger_places` (attribute-only).
|
|
17
|
-
* - `addrfeat` (per county) — named street segments + ZIPs → `tiger_streets` (attribute-only).
|
|
18
|
-
*
|
|
19
|
-
* `tiger_streets` + `tiger_places` match the schema the corpus `tiger` adapter reads, so this is a
|
|
20
|
-
* drop-in replacement for the retired corpus TIGER build script.
|
|
21
|
-
*
|
|
22
|
-
* Flow per source unit: download (skips a valid cached zip) → unzip → stream `ogr2ogr -f
|
|
23
|
-
* GeoJSONSeq` (mapping shapefile columns to the schema, WGS84 for geometry levels) → batched
|
|
24
|
-
* inserts. Re-running a state replaces its rows.
|
|
25
5
|
*/
|
|
26
6
|
import { APIClient, pluckResponseData } from "@mailwoman/core/api";
|
|
27
|
-
import {
|
|
7
|
+
import { dataRootPath } from "@mailwoman/core/data-root";
|
|
28
8
|
import { makeDirectories } from "@mailwoman/core/fs/writers";
|
|
29
9
|
import { extractZipEntries } from "@mailwoman/core/fs/zip";
|
|
30
|
-
import {
|
|
10
|
+
import { stringifyJSON } from "@mailwoman/core/json";
|
|
11
|
+
import { ogr2ogrGeoJSONSeq } from "@mailwoman/spatial/tools/ogr/stream";
|
|
31
12
|
import { DatabaseClient } from "@mailwoman/sqlite/client";
|
|
32
|
-
import {
|
|
13
|
+
import { PathBuilder } from "path-ts";
|
|
14
|
+
import { initializeTIGERSchema, TIGER_PRAGMAS } from "#schema";
|
|
33
15
|
import { downloadIfNeeded } from "#sdk/download";
|
|
34
|
-
import { initializeTIGERSchema, TIGER_PRAGMAS } from "#sdk/schema";
|
|
35
16
|
const CENSUS_HOST = "https://www2.census.gov";
|
|
36
|
-
const DEFAULT_DATA_ROOT =
|
|
17
|
+
const DEFAULT_DATA_ROOT = dataRootPath();
|
|
37
18
|
const LEVEL_DIR = {
|
|
38
19
|
tabblock20: "TABBLOCK20",
|
|
39
20
|
place: "PLACE",
|
|
@@ -44,9 +25,6 @@ const LEVEL_TABLE = {
|
|
|
44
25
|
place: "tiger_places",
|
|
45
26
|
addrfeat: "tiger_streets",
|
|
46
27
|
};
|
|
47
|
-
/**
|
|
48
|
-
* The isp-nexus column map for `tabblock20`. Geometry rides along implicitly.
|
|
49
|
-
*/
|
|
50
28
|
function blockSelectSQL(layer, county) {
|
|
51
29
|
const where = county ? ` WHERE COUNTYFP20 = '${county}'` : "";
|
|
52
30
|
return (`SELECT GEOID20 AS GEOID, STATEFP20 AS state_code, COUNTYFP20 AS county_code, ` +
|
|
@@ -63,7 +41,6 @@ function selectSQL(level, layer, county) {
|
|
|
63
41
|
case "place":
|
|
64
42
|
return `SELECT GEOID AS geoid, NAME AS name, STATEFP AS statefp, LSAD AS lsad, NAMELSAD AS namelsad, CLASSFP AS classfp FROM "${layer}"`;
|
|
65
43
|
case "addrfeat":
|
|
66
|
-
// ADDRFEAT has no STATEFP column — injected per-row from the state we're fetching.
|
|
67
44
|
return `SELECT LINEARID AS linearid, FULLNAME AS fullname, ZIPL AS zipl, ZIPR AS zipr FROM "${layer}" WHERE FULLNAME IS NOT NULL AND FULLNAME != ''`;
|
|
68
45
|
}
|
|
69
46
|
}
|
|
@@ -100,15 +77,11 @@ function buildRow(level, p, geometry, state) {
|
|
|
100
77
|
land_area_sqm: Number(p.land_area_sqm ?? 0),
|
|
101
78
|
water_area_sqm: Number(p.water_area_sqm ?? 0),
|
|
102
79
|
population: Number(p.population ?? 0),
|
|
103
|
-
geometry:
|
|
80
|
+
geometry: stringifyJSON(geometry),
|
|
104
81
|
};
|
|
105
82
|
}
|
|
106
83
|
}
|
|
107
|
-
/**
|
|
108
|
-
* Scrape the ADDRFEAT directory listing for a state's county FIPS codes.
|
|
109
|
-
*/
|
|
110
84
|
async function discoverCounties(state, vintage) {
|
|
111
|
-
// The LISTING only — a small HTML index. The per-county archives below stay on raw `fetch`, streaming to disk.
|
|
112
85
|
const html = await new APIClient({ displayName: "tiger-listing", retry: true })
|
|
113
86
|
.fetch({ url: `${CENSUS_HOST}/geo/tiger/TIGER${vintage}/ADDRFEAT/`, responseType: "text" })
|
|
114
87
|
.then(pluckResponseData);
|
|
@@ -120,24 +93,22 @@ async function discoverCounties(state, vintage) {
|
|
|
120
93
|
return [...counties].toSorted();
|
|
121
94
|
}
|
|
122
95
|
/**
|
|
123
|
-
*
|
|
96
|
+
* Downloads one state's TIGER/Line files at `level`, loads them into a SQLite database
|
|
97
|
+
* and yields progress events.
|
|
98
|
+
*
|
|
99
|
+
* It first deletes the state's existing rows from the target table so that a rerun replaces them.
|
|
124
100
|
*/
|
|
125
101
|
export async function* fetchTIGER(options) {
|
|
126
102
|
const level = options.level ?? "tabblock20";
|
|
127
103
|
const vintage = options.vintage ?? (level === "tabblock20" ? 2020 : 2024);
|
|
128
|
-
const dataRoot = options.dataRoot ?? DEFAULT_DATA_ROOT;
|
|
104
|
+
const dataRoot = PathBuilder.from(options.dataRoot ?? DEFAULT_DATA_ROOT);
|
|
129
105
|
const batchSize = options.batchSize ?? 1000;
|
|
130
106
|
const state = options.stateFIPS;
|
|
131
107
|
const table = LEVEL_TABLE[level];
|
|
132
|
-
const cacheDir =
|
|
133
|
-
|
|
134
|
-
// (run-corpus-build → `${ROOT}/tiger/tiger.db`). The vintage is a content detail, not a path one;
|
|
135
|
-
// the per-table idempotent delete keeps a re-fetch (newer vintage) clean. The download CACHE stays
|
|
136
|
-
// vintage-partitioned below so zips don't collide across vintages.
|
|
137
|
-
const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db");
|
|
108
|
+
const cacheDir = dataRoot("tiger", String(vintage), state);
|
|
109
|
+
const outPath = PathBuilder.from(options.outPath ?? dataRoot("tiger", "tiger.db"));
|
|
138
110
|
await makeDirectories(cacheDir);
|
|
139
|
-
await makeDirectories(dirname(
|
|
140
|
-
// Source units: one (per-state) for block/place; one per county for addrfeat.
|
|
111
|
+
await makeDirectories(outPath.dirname());
|
|
141
112
|
const geoCodes = level === "addrfeat" ? await discoverCounties(state, vintage) : [""];
|
|
142
113
|
if (level === "addrfeat" && !geoCodes.length) {
|
|
143
114
|
throw new Error(`No ADDRFEAT counties found for state ${state} vintage ${vintage}`);
|
|
@@ -145,11 +116,6 @@ export async function* fetchTIGER(options) {
|
|
|
145
116
|
const kdb = new DatabaseClient(outPath);
|
|
146
117
|
kdb.exec(TIGER_PRAGMAS);
|
|
147
118
|
await initializeTIGERSchema(kdb);
|
|
148
|
-
/**
|
|
149
|
-
* `level` picks the table AND the row shape, but nothing ties them at the type level: {@link buildRow} returns the
|
|
150
|
-
* union and Kysely needs the arm's concrete row. The narrowings below are sound only because `buildRow` switches on
|
|
151
|
-
* the SAME `level` — keep the two switches in step, or a row shape reaches the wrong table.
|
|
152
|
-
*/
|
|
153
119
|
const insertBatch = async (rows) => {
|
|
154
120
|
if (level === "tabblock20") {
|
|
155
121
|
await kdb
|
|
@@ -171,7 +137,6 @@ export async function* fetchTIGER(options) {
|
|
|
171
137
|
}
|
|
172
138
|
};
|
|
173
139
|
try {
|
|
174
|
-
// Idempotent re-run: drop this state's rows first (state_code for blocks, statefp otherwise).
|
|
175
140
|
if (level === "tabblock20") {
|
|
176
141
|
await kdb.deleteFrom("tabblock20").where("state_code", "=", state).execute();
|
|
177
142
|
}
|
|
@@ -194,13 +159,13 @@ export async function* fetchTIGER(options) {
|
|
|
194
159
|
for (const geo of geoCodes) {
|
|
195
160
|
const unit = level === "addrfeat" ? state + geo : state;
|
|
196
161
|
const zipName = `tl_${vintage}_${unit}_${level}.zip`;
|
|
197
|
-
const zipPath =
|
|
162
|
+
const zipPath = cacheDir(zipName);
|
|
198
163
|
const url = `${CENSUS_HOST}/geo/tiger/TIGER${vintage}/${LEVEL_DIR[level]}/${zipName}`;
|
|
199
164
|
const cached = await downloadIfNeeded(url, zipPath);
|
|
200
165
|
yield { phase: "download", file: zipName, cached };
|
|
201
166
|
await extractZipEntries(zipPath, cacheDir);
|
|
202
167
|
const layer = `tl_${vintage}_${unit}_${level}`;
|
|
203
|
-
const shpPath =
|
|
168
|
+
const shpPath = cacheDir(layer + ".shp");
|
|
204
169
|
yield { phase: "extract", file: layer + ".shp" };
|
|
205
170
|
const args = [
|
|
206
171
|
"-f",
|
|
@@ -227,7 +192,7 @@ export async function* fetchTIGER(options) {
|
|
|
227
192
|
yield { phase: "load", inserted, total: 0 };
|
|
228
193
|
}
|
|
229
194
|
kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);");
|
|
230
|
-
return { outPath, table, inserted };
|
|
195
|
+
return { outPath: outPath.toString(), table, inserted };
|
|
231
196
|
}
|
|
232
197
|
finally {
|
|
233
198
|
await kdb.destroy();
|
package/out/sdk/fetch.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../sdk/fetch.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAGrC,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAEhD,MAAM,WAAW,GAAG,yBAAyB,CAAA;AAC7C,MAAM,iBAAiB,GAAG,YAAY,EAAE,CAAA;AAQxC,MAAM,SAAS,GAAoC;IAClD,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,OAAO;IACd,QAAQ,EAAE,UAAU;CACpB,CAAA;AAED,MAAM,WAAW,GAAiD;IACjE,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,cAAc;IACrB,QAAQ,EAAE,eAAe;CACzB,CAAA;AAyED,SAAS,cAAc,CAAC,KAAa,EAAE,MAAe;IACrD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,wBAAwB,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IAE7D,OAAO,CACN,+EAA+E;QAC/E,uCAAuC;QACvC,uEAAuE;QACvE,2DAA2D;QAC3D,yFAAyF;QACzF,6BAA6B,KAAK,IAAI,KAAK,EAAE,CAC7C,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAsB,EAAE,KAAa,EAAE,MAAe;IACxE,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,YAAY;YAChB,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACrC,KAAK,OAAO;YACX,OAAO,yHAAyH,KAAK,GAAG,CAAA;QACzI,KAAK,UAAU;YACd,OAAO,uFAAuF,KAAK,iDAAiD,CAAA;IACtJ,CAAC;AACF,CAAC;AAID,SAAS,QAAQ,CAAC,KAAsB,EAAE,CAA0B,EAAE,QAAiB,EAAE,KAAa;IACrG,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,OAAO;YACX,OAAO;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtB,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC1B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;gBACnC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,QAAQ,EAAG,CAAC,CAAC,QAAmB,IAAI,IAAI;gBACxC,OAAO,EAAG,CAAC,CAAC,OAAkB,IAAI,IAAI;aACtC,CAAA;QACF,KAAK,UAAU;YACd,OAAO;gBACN,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAClC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,OAAO,EAAE,KAAK;aACd,CAAA;QACF,KAAK,YAAY;YAChB,OAAO;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtB,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;gBAChC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;gBAClC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;gBAClD,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,mBAAmB,EAAG,CAAC,CAAC,mBAA8B,IAAI,IAAI;gBAC9D,gBAAgB,EAAG,CAAC,CAAC,gBAA2B,IAAI,IAAI;gBACxD,kBAAkB,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC;gBACrD,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC;gBAC3C,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;gBAC7C,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;gBACrC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC;aACjC,CAAA;IACH,CAAC;AACF,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,OAAe;IAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SAC7E,KAAK,CAAS,EAAE,GAAG,EAAE,GAAG,WAAW,mBAAmB,OAAO,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;SAClG,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,OAAO,IAAI,KAAK,yBAAyB,EAAE,GAAG,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAElC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;IACpB,CAAC;IAED,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAA;AAChC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,OAA0B;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAA;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAA;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAA;IAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAA;IAEhC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAA;IAE1D,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAA;IAClF,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;IAC/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IAExC,MAAM,QAAQ,GAAG,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAErF,IAAI,KAAK,KAAK,UAAU,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,YAAY,OAAO,EAAE,CAAC,CAAA;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,cAAc,CAAgB,OAAO,CAAC,CAAA;IACtD,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IACvB,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAA;IAEhC,MAAM,WAAW,GAAG,KAAK,EAAE,IAAW,EAAiB,EAAE;QACxD,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG;iBACP,UAAU,CAAC,YAAY,CAAC;iBACxB,MAAM,CAAC,IAAyB,CAAC;iBACjC,OAAO,EAAE,CAAA;QACZ,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,GAAG;iBACP,UAAU,CAAC,cAAc,CAAC;iBAC1B,MAAM,CAAC,IAAyB,CAAC;iBACjC,OAAO,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,MAAM,GAAG;iBACP,UAAU,CAAC,eAAe,CAAC;iBAC3B,MAAM,CAAC,IAA0B,CAAC;iBAClC,OAAO,EAAE,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC7E,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC5E,CAAC;aAAM,CAAC;YACP,MAAM,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC7E,CAAC;QAED,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,IAAI,KAAK,GAAU,EAAE,CAAA;QAErB,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACxB,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,OAAM;YACzB,MAAM,IAAI,GAAG,KAAK,CAAA;YAClB,KAAK,GAAG,EAAE,CAAA;YACV,MAAM,WAAW,CAAC,IAAI,CAAC,CAAA;YACvB,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAA;QACxB,CAAC,CAAA;QAED,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;YACvD,MAAM,OAAO,GAAG,MAAM,OAAO,IAAI,IAAI,IAAI,KAAK,MAAM,CAAA;YACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAA;YACjC,MAAM,GAAG,GAAG,GAAG,WAAW,mBAAmB,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,OAAO,EAAE,CAAA;YAErF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YACnD,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;YAElD,MAAM,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;YAC1C,MAAM,KAAK,GAAG,MAAM,OAAO,IAAI,IAAI,IAAI,KAAK,EAAE,CAAA;YAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;YACxC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,GAAG,MAAM,EAAE,CAAA;YAEhD,MAAM,IAAI,GAAG;gBACZ,IAAI;gBACJ,YAAY;gBACZ,QAAQ;gBACR,WAAW;gBACX,MAAM;gBACN,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC;gBACvC,aAAa;gBACb,OAAO;aACP,CAAA;YAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,iBAAiB,CACzC,IAAI,EACJ,KAAK,CACL,EAAE,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,UAAU;oBAAE,SAAQ;gBAE9B,IAAI,KAAK,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAAE,SAAQ;gBACtD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAElE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,EAAE,CAAA;oBACb,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;gBAC5C,CAAC;YACF,CAAC;YAED,MAAM,KAAK,EAAE,CAAA;YACb,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;QAC5C,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAA;QAE5C,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;IACxD,CAAC;YAAS,CAAC;QACV,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;IACpB,CAAC;AACF,CAAC"}
|
|
@@ -4,22 +4,24 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
6
|
* Pull a state's Census 2020 P.L. 94-171 redistricting counts (table P2 — Hispanic-or-Latino by
|
|
7
|
-
* race, per block) into the `pl_block` table of a
|
|
8
|
-
* same 15-char block `
|
|
7
|
+
* race, per block) into the `pl_block` table of a tiger {@link DatabaseClient} DB, keyed on the
|
|
8
|
+
* same 15-char block `geoid` as {@link fetchTIGER}'s `tabblock20`. Join the two for block-level
|
|
9
9
|
* race + geometry (e.g. a dot-density map).
|
|
10
10
|
*
|
|
11
11
|
* Keyless public data. The per-state ZIP holds a pipe-delimited geographic header
|
|
12
|
-
* (`<st>geo<yr>.pl`) and three data segments
|
|
13
|
-
* join the header (filtered to
|
|
12
|
+
* (`<st>geo<yr>.pl`) and three data segments. segment 1 (`<st>00001<yr>.pl`) contains P1 + P2. We
|
|
13
|
+
* join the header (filtered to sumlev 750 = block) to segment 1 by logrecno. Field offsets are
|
|
14
14
|
* fixed by the 2020 P.L. layout (verified against the real files).
|
|
15
15
|
*
|
|
16
16
|
* https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
|
|
17
17
|
*/
|
|
18
18
|
/**
|
|
19
|
-
* Segment 2:
|
|
20
|
-
* against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623
|
|
21
|
-
*
|
|
22
|
-
*
|
|
19
|
+
* Segment 2: fileid|stusab|chariter|cifsn|logrecno(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
|
|
20
|
+
* against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623
|
|
21
|
+
* / 916,517, the published CA 2020 figures.
|
|
22
|
+
*
|
|
23
|
+
* `occupied + vacant === housing_units` is the invariant that distinguishes a correct
|
|
24
|
+
* offset from a plausible one. {@link parseH1} refuses a row when it does not hold.
|
|
23
25
|
*/
|
|
24
26
|
export declare const SEG2_FIELD_COUNT = 152;
|
|
25
27
|
/**
|
|
@@ -31,7 +33,7 @@ export declare const H1_TOTAL: number;
|
|
|
31
33
|
*/
|
|
32
34
|
export declare const H1_OCCUPIED: number;
|
|
33
35
|
/**
|
|
34
|
-
* H0010003 — vacant housing units, the last field (and so the one that
|
|
36
|
+
* H0010003 — vacant housing units, the last field (and so the one that contains a crlf file's trailing CR).
|
|
35
37
|
*/
|
|
36
38
|
export declare const H1_VACANT: number;
|
|
37
39
|
export interface H1Counts {
|
|
@@ -40,8 +42,11 @@ export interface H1Counts {
|
|
|
40
42
|
vacant: number;
|
|
41
43
|
}
|
|
42
44
|
/**
|
|
43
|
-
* The three H1 counts from one segment-2 row.
|
|
44
|
-
*
|
|
45
|
+
* The three H1 counts from one segment-2 row.
|
|
46
|
+
*
|
|
47
|
+
* The last field contains crlf's trailing CR when the file has one, so each field
|
|
48
|
+
* is trimmed before it is read as a number.
|
|
49
|
+
* A row whose counts do not add up is refused rather than stored.
|
|
45
50
|
*/
|
|
46
51
|
export declare function parseH1(fields: readonly string[]): H1Counts;
|
|
47
52
|
export interface FetchRedistrictingOptions {
|
|
@@ -50,11 +55,15 @@ export interface FetchRedistrictingOptions {
|
|
|
50
55
|
*/
|
|
51
56
|
stateFIPS: string;
|
|
52
57
|
/**
|
|
53
|
-
* Decennial vintage.
|
|
58
|
+
* Decennial vintage.
|
|
59
|
+
*
|
|
60
|
+
* @defaultValue `2020` (the only P.L. 94-171 release this parses)
|
|
54
61
|
*/
|
|
55
62
|
vintage?: number;
|
|
56
63
|
/**
|
|
57
|
-
* Output SQLite path.
|
|
64
|
+
* Output SQLite path.
|
|
65
|
+
*
|
|
66
|
+
* @defaultValue `<dataRoot>/tiger/tiger.db` (same DB as `fetchTIGER`).
|
|
58
67
|
*/
|
|
59
68
|
outPath?: string;
|
|
60
69
|
/**
|
|
@@ -66,7 +75,9 @@ export interface FetchRedistrictingOptions {
|
|
|
66
75
|
*/
|
|
67
76
|
county?: string;
|
|
68
77
|
/**
|
|
69
|
-
* Rows per insert.
|
|
78
|
+
* Rows per insert.
|
|
79
|
+
*
|
|
80
|
+
* @defaultValue `2000`
|
|
70
81
|
*/
|
|
71
82
|
batchSize?: number;
|
|
72
83
|
}
|
|
@@ -91,7 +102,10 @@ export interface FetchRedistrictingResult {
|
|
|
91
102
|
inserted: number;
|
|
92
103
|
}
|
|
93
104
|
/**
|
|
94
|
-
* Fetch one state's P.L. 94-171 block race counts into `pl_block`.
|
|
105
|
+
* Fetch one state's P.L. 94-171 block race counts into `pl_block`.
|
|
106
|
+
*
|
|
107
|
+
* Yields progress.
|
|
108
|
+
* Returns the tally.
|
|
95
109
|
*/
|
|
96
110
|
export declare function fetchRedistricting(options: FetchRedistrictingOptions): AsyncGenerator<FetchRedistrictingEvent, FetchRedistrictingResult>;
|
|
97
111
|
//# sourceMappingURL=redistricting.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"redistricting.d.ts","sourceRoot":"","sources":["../../
|
|
1
|
+
{"version":3,"file":"redistricting.d.ts","sourceRoot":"","sources":["../../sdk/redistricting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAiCH;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAA;AACnC;;GAEG;AACH,eAAO,MAAM,QAAQ,QAAuB,CAAA;AAC5C;;GAEG;AACH,eAAO,MAAM,WAAW,QAAuB,CAAA;AAC/C;;GAEG;AACH,eAAO,MAAM,SAAS,QAAuB,CAAA;AAE7C,MAAM,WAAW,QAAQ;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,QAAQ,CAsB3D;AAiBD,MAAM,WAAW,yBAAyB;IACzC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,MAAM,uBAAuB,GAChC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClC;IAAE,KAAK,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAA;AAErD,MAAM,WAAW,wBAAwB;IACxC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAeD;;;;;GAKG;AACH,wBAAuB,kBAAkB,CACxC,OAAO,EAAE,yBAAyB,GAChC,cAAc,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAmInE"}
|