@mailwoman/tiger 9.4.0 → 10.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/lib/block-group.ts +7 -7
  2. package/lib/class-code.ts +387 -266
  3. package/lib/constants.ts +7 -6
  4. package/lib/{county-subdivision.ts → county/subdivision.ts} +7 -7
  5. package/lib/county.ts +7 -7
  6. package/lib/files.ts +37 -20
  7. package/lib/functional-status.ts +6 -4
  8. package/lib/geoid.ts +44 -26
  9. package/lib/index.ts +1 -1
  10. package/lib/legal-statistical-area.ts +7 -5
  11. package/lib/{sdk/schema.ts → schema.ts} +32 -42
  12. package/lib/state.ts +23 -20
  13. package/lib/tabulation-block.ts +4 -4
  14. package/lib/tract.ts +3 -3
  15. package/out/block-group.d.ts +7 -7
  16. package/out/class-code.d.ts +387 -266
  17. package/out/class-code.d.ts.map +1 -1
  18. package/out/class-code.js +387 -266
  19. package/out/class-code.js.map +1 -1
  20. package/out/constants.d.ts +7 -6
  21. package/out/constants.d.ts.map +1 -1
  22. package/out/constants.js +3 -3
  23. package/out/{county-subdivision.d.ts → county/subdivision.d.ts} +8 -8
  24. package/out/county/subdivision.d.ts.map +1 -0
  25. package/out/{county-subdivision.js → county/subdivision.js} +1 -1
  26. package/out/county/subdivision.js.map +1 -0
  27. package/out/county.d.ts +7 -7
  28. package/out/county.js +1 -1
  29. package/out/files.d.ts +21 -13
  30. package/out/files.d.ts.map +1 -1
  31. package/out/files.js +27 -13
  32. package/out/files.js.map +1 -1
  33. package/out/functional-status.d.ts +6 -4
  34. package/out/functional-status.d.ts.map +1 -1
  35. package/out/functional-status.js +6 -4
  36. package/out/functional-status.js.map +1 -1
  37. package/out/geoid.d.ts +34 -16
  38. package/out/geoid.d.ts.map +1 -1
  39. package/out/geoid.js +16 -10
  40. package/out/geoid.js.map +1 -1
  41. package/out/index.d.ts +1 -1
  42. package/out/index.js +1 -1
  43. package/out/legal-statistical-area.d.ts +7 -5
  44. package/out/legal-statistical-area.d.ts.map +1 -1
  45. package/out/legal-statistical-area.js +7 -5
  46. package/out/legal-statistical-area.js.map +1 -1
  47. package/out/{sdk/schema.d.ts → schema.d.ts} +26 -20
  48. package/out/schema.d.ts.map +1 -0
  49. package/out/{sdk/schema.js → schema.js} +12 -31
  50. package/out/schema.js.map +1 -0
  51. package/out/sdk/download.d.ts +7 -15
  52. package/out/sdk/download.d.ts.map +1 -1
  53. package/out/sdk/download.js +5 -14
  54. package/out/sdk/download.js.map +1 -1
  55. package/out/sdk/fetch.d.ts +32 -29
  56. package/out/sdk/fetch.d.ts.map +1 -1
  57. package/out/sdk/fetch.js +18 -53
  58. package/out/sdk/fetch.js.map +1 -1
  59. package/out/sdk/redistricting.d.ts +29 -15
  60. package/out/sdk/redistricting.d.ts.map +1 -1
  61. package/out/sdk/redistricting.js +47 -36
  62. package/out/sdk/redistricting.js.map +1 -1
  63. package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts +2 -2
  64. package/out/sdk/state/TIGERTabulatedBlockEntity.d.ts.map +1 -1
  65. package/out/sdk/state/TIGERTabulatedBlockEntity.js +3 -3
  66. package/out/sdk/state/TIGERTabulatedBlockEntity.js.map +1 -1
  67. package/out/sdk/state/TIGERTractEntity.d.ts.map +1 -1
  68. package/out/sdk/state/TIGERTractEntity.js.map +1 -1
  69. package/out/sdk/state/block-operations.d.ts +3 -2
  70. package/out/sdk/state/block-operations.d.ts.map +1 -1
  71. package/out/sdk/state/block-operations.js +4 -3
  72. package/out/sdk/state/block-operations.js.map +1 -1
  73. package/out/sdk/{state/index.d.ts → state.d.ts} +1 -1
  74. package/out/sdk/state.d.ts.map +1 -0
  75. package/out/sdk/{state/index.js → state.js} +1 -1
  76. package/out/sdk/state.js.map +1 -0
  77. package/out/sdk/tsconfig.tsbuildinfo +1 -0
  78. package/out/state.d.ts +45 -42
  79. package/out/state.d.ts.map +1 -1
  80. package/out/state.js +17 -14
  81. package/out/state.js.map +1 -1
  82. package/out/tabulation-block.d.ts +4 -4
  83. package/out/tools/{race-dots-map.d.ts → race/dots/map.d.ts} +24 -10
  84. package/out/tools/race/dots/map.d.ts.map +1 -0
  85. package/out/tools/{race-dots-map.js → race/dots/map.js} +10 -6
  86. package/out/tools/race/dots/map.js.map +1 -0
  87. package/out/tools/race/dots.d.ts +47 -0
  88. package/out/tools/race/dots.d.ts.map +1 -0
  89. package/out/tools/{race-dots.js → race/dots.js} +10 -27
  90. package/out/tools/race/dots.js.map +1 -0
  91. package/out/tools/serve-range.d.ts +13 -7
  92. package/out/tools/serve-range.d.ts.map +1 -1
  93. package/out/tools/serve-range.js +9 -7
  94. package/out/tools/serve-range.js.map +1 -1
  95. package/out/tools/tsconfig.tsbuildinfo +1 -0
  96. package/out/tract.d.ts +3 -3
  97. package/out/tsconfig.test.tsbuildinfo +1 -1
  98. package/package.json +59 -42
  99. package/sdk/download.ts +41 -0
  100. package/{lib/sdk → sdk}/fetch.ts +54 -62
  101. package/{lib/sdk → sdk}/redistricting.ts +57 -40
  102. package/{lib/sdk → sdk}/state/TIGERTabulatedBlockEntity.ts +3 -3
  103. package/{lib/sdk → sdk}/state/block-operations.ts +5 -4
  104. package/{lib/tools/race-dots-map.ts → tools/race/dots/map.ts} +30 -12
  105. package/{lib/tools/race-dots.ts → tools/race/dots.ts} +14 -31
  106. package/{lib/tools → tools}/serve-range.ts +15 -9
  107. package/lib/sdk/download.ts +0 -49
  108. package/lib/sdk/index.ts +0 -10
  109. package/lib/tools/index.ts +0 -11
  110. package/out/county-subdivision.d.ts.map +0 -1
  111. package/out/county-subdivision.js.map +0 -1
  112. package/out/sdk/index.d.ts +0 -10
  113. package/out/sdk/index.d.ts.map +0 -1
  114. package/out/sdk/index.js +0 -10
  115. package/out/sdk/index.js.map +0 -1
  116. package/out/sdk/schema.d.ts.map +0 -1
  117. package/out/sdk/schema.js.map +0 -1
  118. package/out/sdk/state/index.d.ts.map +0 -1
  119. package/out/sdk/state/index.js.map +0 -1
  120. package/out/tools/index.d.ts +0 -11
  121. package/out/tools/index.d.ts.map +0 -1
  122. package/out/tools/index.js +0 -11
  123. package/out/tools/index.js.map +0 -1
  124. package/out/tools/race-dots-map.d.ts.map +0 -1
  125. package/out/tools/race-dots-map.js.map +0 -1
  126. package/out/tools/race-dots.d.ts +0 -61
  127. package/out/tools/race-dots.d.ts.map +0 -1
  128. package/out/tools/race-dots.js.map +0 -1
  129. /package/{lib/sdk → sdk}/state/TIGERTractEntity.ts +0 -0
  130. /package/{lib/sdk/state/index.ts → sdk/state.ts} +0 -0
@@ -3,27 +3,18 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Cached archive download + subprocess capture, shared by `fetch.ts` and `redistricting.ts`.
7
- *
8
- * RAW `fetch` HERE IS DELIBERATE. `AGENTS.md` requires HTTP clients to extend or instantiate
9
- * `APIClient`, and that rule is about API REQUESTS — small bodies, repeated calls, rate-limited
10
- * hosts, where its pacing, bounded retry, response caching and `ResourceError` mapping all earn
11
- * their keep. This is a multi-gigabyte file transfer streamed straight to disk. Response caching
12
- * would be nonsense at that size, pacing has nothing to pace (one request), and axios buffers a
13
- * non-stream response type in memory. The primitive that fits a body this large is the one that
14
- * never holds it: a web stream piped to a write stream.
6
+ * Raw `fetch` is deliberate for a multi-gigabyte streamed transfer: `APIClient`'s response caching
7
+ * and pacing do not apply.
15
8
  */
16
9
  import { pathExists } from "@mailwoman/core/fs/readers";
17
10
  import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
18
11
  import { movePath } from "@mailwoman/core/fs/writers";
19
12
  import { verifyZipIntegrity } from "@mailwoman/core/fs/zip";
20
13
  /**
21
- * Download `url` to `dest` unless a VALID copy is already there. Returns `true` when the cache was reused, `false` when
22
- * a download happened.
14
+ * Download `url` to `dest` unless a valid copy is already there.
23
15
  *
24
- * "Valid" means every member's CRC-32 checks out — an existence check alone is not enough, because an interrupted
25
- * download leaves a plausible-looking file that fails only much later, inside ogr2ogr. Writes to a `.tmp` sibling and
26
- * renames, so `dest` is never a partial archive.
16
+ * "Valid" means every member's CRC-32 checks out.
17
+ * An interrupted download otherwise leaves a plausible file that fails later inside ogr2ogr.
27
18
  */
28
19
  export async function downloadIfNeeded(url, dest) {
29
20
  if (await pathExists(dest)) {
@@ -1 +1 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../lib/sdk/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAE3D;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,IAAY;IAC/D,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,IAAI,CAAC;YACJ,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAE9B,OAAO,IAAI,CAAA;QACZ,CAAC;QAAC,MAAM,CAAC;YACR,8BAA8B;QAC/B,CAAC;IACF,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,MAAM,CAAA;IACzB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC,CAAA;IAC/E,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAA8C,CAAC,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1G,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAEzB,OAAO,KAAK,CAAA;AACb,CAAC"}
1
+ {"version":3,"file":"download.js","sourceRoot":"","sources":["../../sdk/download.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAG3D;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,IAAqB;IACxE,IAAI,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,IAAI,CAAC;YACJ,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAE9B,OAAO,IAAI,CAAA;QACZ,CAAC;QAAC,MAAM,CAAC;YACR,8BAA8B;QAC/B,CAAC;IACF,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,MAAM,CAAA;IACzB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC,CAAA;IAC/E,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAA8C,CAAC,EAAE,eAAe,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1G,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAEzB,OAAO,KAAK,CAAA;AACb,CAAC"}
@@ -2,61 +2,58 @@
2
2
  * @copyright Sister Software.
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
- *
6
- * Pull TIGER geometry/attributes into a `node:sqlite` database via the Kysely
7
- * {@link DatabaseClient}.
8
- *
9
- * Replaces the ad-hoc wget and the retired corpus TIGER build script. Pared down from isp-nexus's
10
- * `generate-tiger-tiles.ts` (its per-level column mapping is the spec) into the playpen
11
- * `repo-tools` idiom: an async generator of progress events, idempotent, no SpatiaLite.
12
- *
13
- * Three levels:
14
- *
15
- * - `tabblock20` (per state) — tabulation blocks → `tabblock20`, geometry as GeoJSON text.
16
- * - `place` (per state) — incorporated/census places → `tiger_places` (attribute-only).
17
- * - `addrfeat` (per county) — named street segments + ZIPs → `tiger_streets` (attribute-only).
18
- *
19
- * `tiger_streets` + `tiger_places` match the schema the corpus `tiger` adapter reads, so this is a
20
- * drop-in replacement for the retired corpus TIGER build script.
21
- *
22
- * Flow per source unit: download (skips a valid cached zip) → unzip → stream `ogr2ogr -f
23
- * GeoJSONSeq` (mapping shapefile columns to the schema, WGS84 for geometry levels) → batched
24
- * inserts. Re-running a state replaces its rows.
25
5
  */
26
6
  /**
27
- * Supported TIGER levels. `tabblock20` is per state + carries geometry; `place`/`addrfeat` are attribute-only.
7
+ * A TIGER/Line product that {@link fetchTIGER} can load.
8
+ * Only the `tabblock20` level keeps geometry.
28
9
  */
29
10
  export type TIGERFetchLevel = "tabblock20" | "place" | "addrfeat";
11
+ /**
12
+ * Options for {@link fetchTIGER}.
13
+ */
30
14
  export interface FetchTIGEROptions {
31
15
  /**
32
- * Two-digit state FIPS, e.g. `"06"`.
16
+ * The two-digit state FIPS code, such as `"06"`.
33
17
  */
34
18
  stateFIPS: string;
35
19
  /**
36
- * TIGER level. Default `tabblock20`.
20
+ * The TIGER level to fetch.
21
+ * The default is `tabblock20`.
37
22
  */
38
23
  level?: TIGERFetchLevel;
39
24
  /**
40
- * Vintage. Default 2020 for blocks (matches the 2020 P.L.), 2024 for place/addrfeat (current).
25
+ * The TIGER vintage year.
26
+ *
27
+ * The default is 2020 for blocks to match redistricting data.
28
+ * Other levels default to 2024.
41
29
  */
42
30
  vintage?: number;
43
31
  /**
44
- * Output SQLite path. Default `<dataRoot>/tiger/tiger.db` (the name the corpus `tiger` adapter reads).
32
+ * The output SQLite path.
33
+ *
34
+ * The default is `<dataRoot>/tiger/tiger.db`, which the corpus `tiger` adapter reads.
45
35
  */
46
36
  outPath?: string;
47
37
  /**
48
- * Download cache + default output root.
38
+ * The root of the download cache and the default output.
39
+ * The default is the Mailwoman data root.
49
40
  */
50
41
  dataRoot?: string;
51
42
  /**
52
- * Optional three-digit county FIPS filter (blocks only — addrfeat is already per-county).
43
+ * A three-digit county FIPS filter that applies only to the `tabblock20` level.
53
44
  */
54
45
  county?: string;
55
46
  /**
56
- * Rows per insert. Default 1000.
47
+ * The number of rows per insert batch.
48
+ * The default is 1000.
57
49
  */
58
50
  batchSize?: number;
59
51
  }
52
+ /**
53
+ * A progress event from {@link fetchTIGER}.
54
+ *
55
+ * Each event reports a file downloaded or found in the cache, a shapefile extracted, or rows inserted.
56
+ */
60
57
  export type FetchTIGEREvent = {
61
58
  phase: "download";
62
59
  file: string;
@@ -69,13 +66,19 @@ export type FetchTIGEREvent = {
69
66
  inserted: number;
70
67
  total: number;
71
68
  };
69
+ /**
70
+ * The database path, table name and inserted row count that {@link fetchTIGER} returns.
71
+ */
72
72
  export interface FetchTIGERResult {
73
73
  outPath: string;
74
74
  table: string;
75
75
  inserted: number;
76
76
  }
77
77
  /**
78
- * Fetch one state's TIGER data at `level` into a SQLite DB. Yields progress; returns the final tally.
78
+ * Downloads one state's TIGER/Line files at `level`, loads them into a SQLite database
79
+ * and yields progress events.
80
+ *
81
+ * It first deletes the state's existing rows from the target table so that a rerun replaces them.
79
82
  */
80
83
  export declare function fetchTIGER(options: FetchTIGEROptions): AsyncGenerator<FetchTIGEREvent, FetchTIGERResult>;
81
84
  //# sourceMappingURL=fetch.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../lib/sdk/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAiBH;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,OAAO,GAAG,UAAU,CAAA;AAcjE,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,KAAK,CAAC,EAAE,eAAe,CAAA;IACvB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,MAAM,eAAe,GACxB;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAA;AAErD,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAyFD;;GAEG;AACH,wBAAuB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,cAAc,CAAC,eAAe,EAAE,gBAAgB,CAAC,CA2H/G"}
1
+ {"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../sdk/fetch.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAkBH;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,OAAO,GAAG,UAAU,CAAA;AAcjE;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,KAAK,CAAC,EAAE,eAAe,CAAA;IAEvB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACxB;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAA;AAErD;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAiFD;;;;;GAKG;AACH,wBAAuB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,cAAc,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAiH/G"}
package/out/sdk/fetch.js CHANGED
@@ -2,38 +2,19 @@
2
2
  * @copyright Sister Software.
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
- *
6
- * Pull TIGER geometry/attributes into a `node:sqlite` database via the Kysely
7
- * {@link DatabaseClient}.
8
- *
9
- * Replaces the ad-hoc wget and the retired corpus TIGER build script. Pared down from isp-nexus's
10
- * `generate-tiger-tiles.ts` (its per-level column mapping is the spec) into the playpen
11
- * `repo-tools` idiom: an async generator of progress events, idempotent, no SpatiaLite.
12
- *
13
- * Three levels:
14
- *
15
- * - `tabblock20` (per state) — tabulation blocks → `tabblock20`, geometry as GeoJSON text.
16
- * - `place` (per state) — incorporated/census places → `tiger_places` (attribute-only).
17
- * - `addrfeat` (per county) — named street segments + ZIPs → `tiger_streets` (attribute-only).
18
- *
19
- * `tiger_streets` + `tiger_places` match the schema the corpus `tiger` adapter reads, so this is a
20
- * drop-in replacement for the retired corpus TIGER build script.
21
- *
22
- * Flow per source unit: download (skips a valid cached zip) → unzip → stream `ogr2ogr -f
23
- * GeoJSONSeq` (mapping shapefile columns to the schema, WGS84 for geometry levels) → batched
24
- * inserts. Re-running a state replaces its rows.
25
5
  */
26
6
  import { APIClient, pluckResponseData } from "@mailwoman/core/api";
27
- import { mailwomanDataRoot } from "@mailwoman/core/data-root";
7
+ import { dataRootPath } from "@mailwoman/core/data-root";
28
8
  import { makeDirectories } from "@mailwoman/core/fs/writers";
29
9
  import { extractZipEntries } from "@mailwoman/core/fs/zip";
30
- import { ogr2ogrGeoJSONSeq } from "@mailwoman/spatial/tools/ogr-stream";
10
+ import { stringifyJSON } from "@mailwoman/core/json";
11
+ import { ogr2ogrGeoJSONSeq } from "@mailwoman/spatial/tools/ogr/stream";
31
12
  import { DatabaseClient } from "@mailwoman/sqlite/client";
32
- import { dirname, join } from "path-ts";
13
+ import { PathBuilder } from "path-ts";
14
+ import { initializeTIGERSchema, TIGER_PRAGMAS } from "#schema";
33
15
  import { downloadIfNeeded } from "#sdk/download";
34
- import { initializeTIGERSchema, TIGER_PRAGMAS } from "#sdk/schema";
35
16
  const CENSUS_HOST = "https://www2.census.gov";
36
- const DEFAULT_DATA_ROOT = mailwomanDataRoot();
17
+ const DEFAULT_DATA_ROOT = dataRootPath();
37
18
  const LEVEL_DIR = {
38
19
  tabblock20: "TABBLOCK20",
39
20
  place: "PLACE",
@@ -44,9 +25,6 @@ const LEVEL_TABLE = {
44
25
  place: "tiger_places",
45
26
  addrfeat: "tiger_streets",
46
27
  };
47
- /**
48
- * The isp-nexus column map for `tabblock20`. Geometry rides along implicitly.
49
- */
50
28
  function blockSelectSQL(layer, county) {
51
29
  const where = county ? ` WHERE COUNTYFP20 = '${county}'` : "";
52
30
  return (`SELECT GEOID20 AS GEOID, STATEFP20 AS state_code, COUNTYFP20 AS county_code, ` +
@@ -63,7 +41,6 @@ function selectSQL(level, layer, county) {
63
41
  case "place":
64
42
  return `SELECT GEOID AS geoid, NAME AS name, STATEFP AS statefp, LSAD AS lsad, NAMELSAD AS namelsad, CLASSFP AS classfp FROM "${layer}"`;
65
43
  case "addrfeat":
66
- // ADDRFEAT has no STATEFP column — injected per-row from the state we're fetching.
67
44
  return `SELECT LINEARID AS linearid, FULLNAME AS fullname, ZIPL AS zipl, ZIPR AS zipr FROM "${layer}" WHERE FULLNAME IS NOT NULL AND FULLNAME != ''`;
68
45
  }
69
46
  }
@@ -100,15 +77,11 @@ function buildRow(level, p, geometry, state) {
100
77
  land_area_sqm: Number(p.land_area_sqm ?? 0),
101
78
  water_area_sqm: Number(p.water_area_sqm ?? 0),
102
79
  population: Number(p.population ?? 0),
103
- geometry: JSON.stringify(geometry),
80
+ geometry: stringifyJSON(geometry),
104
81
  };
105
82
  }
106
83
  }
107
- /**
108
- * Scrape the ADDRFEAT directory listing for a state's county FIPS codes.
109
- */
110
84
  async function discoverCounties(state, vintage) {
111
- // The LISTING only — a small HTML index. The per-county archives below stay on raw `fetch`, streaming to disk.
112
85
  const html = await new APIClient({ displayName: "tiger-listing", retry: true })
113
86
  .fetch({ url: `${CENSUS_HOST}/geo/tiger/TIGER${vintage}/ADDRFEAT/`, responseType: "text" })
114
87
  .then(pluckResponseData);
@@ -120,24 +93,22 @@ async function discoverCounties(state, vintage) {
120
93
  return [...counties].toSorted();
121
94
  }
122
95
  /**
123
- * Fetch one state's TIGER data at `level` into a SQLite DB. Yields progress; returns the final tally.
96
+ * Downloads one state's TIGER/Line files at `level`, loads them into a SQLite database
97
+ * and yields progress events.
98
+ *
99
+ * It first deletes the state's existing rows from the target table so that a rerun replaces them.
124
100
  */
125
101
  export async function* fetchTIGER(options) {
126
102
  const level = options.level ?? "tabblock20";
127
103
  const vintage = options.vintage ?? (level === "tabblock20" ? 2020 : 2024);
128
- const dataRoot = options.dataRoot ?? DEFAULT_DATA_ROOT;
104
+ const dataRoot = PathBuilder.from(options.dataRoot ?? DEFAULT_DATA_ROOT);
129
105
  const batchSize = options.batchSize ?? 1000;
130
106
  const state = options.stateFIPS;
131
107
  const table = LEVEL_TABLE[level];
132
- const cacheDir = join(dataRoot, "tiger", String(vintage), state);
133
- // Default to a stable, vintage-agnostic `tiger.db` — the filename the corpus `tiger` adapter reads
134
- // (run-corpus-build → `${ROOT}/tiger/tiger.db`). The vintage is a content detail, not a path one;
135
- // the per-table idempotent delete keeps a re-fetch (newer vintage) clean. The download CACHE stays
136
- // vintage-partitioned below so zips don't collide across vintages.
137
- const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db");
108
+ const cacheDir = dataRoot("tiger", String(vintage), state);
109
+ const outPath = PathBuilder.from(options.outPath ?? dataRoot("tiger", "tiger.db"));
138
110
  await makeDirectories(cacheDir);
139
- await makeDirectories(dirname(outPath));
140
- // Source units: one (per-state) for block/place; one per county for addrfeat.
111
+ await makeDirectories(outPath.dirname());
141
112
  const geoCodes = level === "addrfeat" ? await discoverCounties(state, vintage) : [""];
142
113
  if (level === "addrfeat" && !geoCodes.length) {
143
114
  throw new Error(`No ADDRFEAT counties found for state ${state} vintage ${vintage}`);
@@ -145,11 +116,6 @@ export async function* fetchTIGER(options) {
145
116
  const kdb = new DatabaseClient(outPath);
146
117
  kdb.exec(TIGER_PRAGMAS);
147
118
  await initializeTIGERSchema(kdb);
148
- /**
149
- * `level` picks the table AND the row shape, but nothing ties them at the type level: {@link buildRow} returns the
150
- * union and Kysely needs the arm's concrete row. The narrowings below are sound only because `buildRow` switches on
151
- * the SAME `level` — keep the two switches in step, or a row shape reaches the wrong table.
152
- */
153
119
  const insertBatch = async (rows) => {
154
120
  if (level === "tabblock20") {
155
121
  await kdb
@@ -171,7 +137,6 @@ export async function* fetchTIGER(options) {
171
137
  }
172
138
  };
173
139
  try {
174
- // Idempotent re-run: drop this state's rows first (state_code for blocks, statefp otherwise).
175
140
  if (level === "tabblock20") {
176
141
  await kdb.deleteFrom("tabblock20").where("state_code", "=", state).execute();
177
142
  }
@@ -194,13 +159,13 @@ export async function* fetchTIGER(options) {
194
159
  for (const geo of geoCodes) {
195
160
  const unit = level === "addrfeat" ? state + geo : state;
196
161
  const zipName = `tl_${vintage}_${unit}_${level}.zip`;
197
- const zipPath = join(cacheDir, zipName);
162
+ const zipPath = cacheDir(zipName);
198
163
  const url = `${CENSUS_HOST}/geo/tiger/TIGER${vintage}/${LEVEL_DIR[level]}/${zipName}`;
199
164
  const cached = await downloadIfNeeded(url, zipPath);
200
165
  yield { phase: "download", file: zipName, cached };
201
166
  await extractZipEntries(zipPath, cacheDir);
202
167
  const layer = `tl_${vintage}_${unit}_${level}`;
203
- const shpPath = join(cacheDir, layer + ".shp");
168
+ const shpPath = cacheDir(layer + ".shp");
204
169
  yield { phase: "extract", file: layer + ".shp" };
205
170
  const args = [
206
171
  "-f",
@@ -227,7 +192,7 @@ export async function* fetchTIGER(options) {
227
192
  yield { phase: "load", inserted, total: 0 };
228
193
  }
229
194
  kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);");
230
- return { outPath, table, inserted };
195
+ return { outPath: outPath.toString(), table, inserted };
231
196
  }
232
197
  finally {
233
198
  await kdb.destroy();
@@ -1 +1 @@
1
- {"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../lib/sdk/fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAEhD,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAElE,MAAM,WAAW,GAAG,yBAAyB,CAAA;AAC7C,MAAM,iBAAiB,GAAG,iBAAiB,EAAE,CAAA;AAO7C,MAAM,SAAS,GAAoC;IAClD,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,OAAO;IACd,QAAQ,EAAE,UAAU;CACpB,CAAA;AAED,MAAM,WAAW,GAAiD;IACjE,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,cAAc;IACrB,QAAQ,EAAE,eAAe;CACzB,CAAA;AA4CD;;GAEG;AACH,SAAS,cAAc,CAAC,KAAa,EAAE,MAAe;IACrD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,wBAAwB,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IAE7D,OAAO,CACN,+EAA+E;QAC/E,uCAAuC;QACvC,uEAAuE;QACvE,2DAA2D;QAC3D,yFAAyF;QACzF,6BAA6B,KAAK,IAAI,KAAK,EAAE,CAC7C,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAsB,EAAE,KAAa,EAAE,MAAe;IACxE,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,YAAY;YAChB,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACrC,KAAK,OAAO;YACX,OAAO,yHAAyH,KAAK,GAAG,CAAA;QACzI,KAAK,UAAU;YACd,mFAAmF;YACnF,OAAO,uFAAuF,KAAK,iDAAiD,CAAA;IACtJ,CAAC;AACF,CAAC;AAID,SAAS,QAAQ,CAAC,KAAsB,EAAE,CAA0B,EAAE,QAAiB,EAAE,KAAa;IACrG,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,OAAO;YACX,OAAO;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtB,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC1B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;gBACnC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,QAAQ,EAAG,CAAC,CAAC,QAAmB,IAAI,IAAI;gBACxC,OAAO,EAAG,CAAC,CAAC,OAAkB,IAAI,IAAI;aACtC,CAAA;QACF,KAAK,UAAU;YACd,OAAO;gBACN,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAClC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,OAAO,EAAE,KAAK;aACd,CAAA;QACF,KAAK,YAAY;YAChB,OAAO;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtB,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;gBAChC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;gBAClC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;gBAClD,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,mBAAmB,EAAG,CAAC,CAAC,mBAA8B,IAAI,IAAI;gBAC9D,gBAAgB,EAAG,CAAC,CAAC,gBAA2B,IAAI,IAAI;gBACxD,kBAAkB,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC;gBACrD,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC;gBAC3C,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;gBAC7C,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;gBACrC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;aAClC,CAAA;IACH,CAAC;AACF,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,OAAe;IAC7D,+GAA+G;IAC/G,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SAC7E,KAAK,CAAS,EAAE,GAAG,EAAE,GAAG,WAAW,mBAAmB,OAAO,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;SAClG,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,OAAO,IAAI,KAAK,yBAAyB,EAAE,GAAG,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAElC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;IACpB,CAAC;IAED,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,OAA0B;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAA;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAA;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAA;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAA;IAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAA;IAEhC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAA;IAChE,mGAAmG;IACnG,kGAAkG;IAClG,mGAAmG;IACnG,mEAAmE;IACnE,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,CAAA;IACtE,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;IAC/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;IAEvC,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAErF,IAAI,KAAK,KAAK,UAAU,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,YAAY,OAAO,EAAE,CAAC,CAAA;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,cAAc,CAAgB,OAAO,CAAC,CAAA;IACtD,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IACvB,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAA;IAEhC;;;;OAIG;IACH,MAAM,WAAW,GAAG,KAAK,EAAE,IAAW,EAAiB,EAAE;QACxD,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG;iBACP,UAAU,CAAC,YAAY,CAAC;iBACxB,MAAM,CAAC,IAAyB,CAAC;iBACjC,OAAO,EAAE,CAAA;QACZ,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,GAAG;iBACP,UAAU,CAAC,cAAc,CAAC;iBAC1B,MAAM,CAAC,IAAyB,CAAC;iBACjC,OAAO,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,MAAM,GAAG;iBACP,UAAU,CAAC,eAAe,CAAC;iBAC3B,MAAM,CAAC,IAA0B,CAAC;iBAClC,OAAO,EAAE,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,8FAA8F;QAC9F,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC7E,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC5E,CAAC;aAAM,CAAC;YACP,MAAM,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC7E,CAAC;QAED,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,IAAI,KAAK,GAAU,EAAE,CAAA;QAErB,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACxB,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,OAAM;YACzB,MAAM,IAAI,GAAG,KAAK,CAAA;YAClB,KAAK,GAAG,EAAE,CAAA;YACV,MAAM,WAAW,CAAC,IAAI,CAAC,CAAA;YACvB,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAA;QACxB,CAAC,CAAA;QAED,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;YACvD,MAAM,OAAO,GAAG,MAAM,OAAO,IAAI,IAAI,IAAI,KAAK,MAAM,CAAA;YACpD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;YACvC,MAAM,GAAG,GAAG,GAAG,WAAW,mBAAmB,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,OAAO,EAAE,CAAA;YAErF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YACnD,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;YAElD,MAAM,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;YAC1C,MAAM,KAAK,GAAG,MAAM,OAAO,IAAI,IAAI,IAAI,KAAK,EAAE,CAAA;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,GAAG,MAAM,CAAC,CAAA;YAC9C,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,GAAG,MAAM,EAAE,CAAA;YAEhD,MAAM,IAAI,GAAG;gBACZ,IAAI;gBACJ,YAAY;gBACZ,QAAQ;gBACR,WAAW;gBACX,MAAM;gBACN,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC;gBACvC,aAAa;gBACb,OAAO;aACP,CAAA;YAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,iBAAiB,CACzC,IAAI,EACJ,KAAK,CACL,EAAE,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,UAAU;oBAAE,SAAQ;gBAE9B,IAAI,KAAK,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAAE,SAAQ;gBACtD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAElE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,EAAE,CAAA;oBACb,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;gBAC5C,CAAC;YACF,CAAC;YAED,MAAM,KAAK,EAAE,CAAA;YACb,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;QAC5C,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAA;QAE5C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;IACpC,CAAC;YAAS,CAAC;QACV,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;IACpB,CAAC;AACF,CAAC"}
1
+ {"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../sdk/fetch.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAGrC,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAEhD,MAAM,WAAW,GAAG,yBAAyB,CAAA;AAC7C,MAAM,iBAAiB,GAAG,YAAY,EAAE,CAAA;AAQxC,MAAM,SAAS,GAAoC;IAClD,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,OAAO;IACd,QAAQ,EAAE,UAAU;CACpB,CAAA;AAED,MAAM,WAAW,GAAiD;IACjE,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,cAAc;IACrB,QAAQ,EAAE,eAAe;CACzB,CAAA;AAqED,SAAS,cAAc,CAAC,KAAa,EAAE,MAAe;IACrD,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,wBAAwB,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;IAE7D,OAAO,CACN,+EAA+E;QAC/E,uCAAuC;QACvC,uEAAuE;QACvE,2DAA2D;QAC3D,yFAAyF;QACzF,6BAA6B,KAAK,IAAI,KAAK,EAAE,CAC7C,CAAA;AACF,CAAC;AAED,SAAS,SAAS,CAAC,KAAsB,EAAE,KAAa,EAAE,MAAe;IACxE,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,YAAY;YAChB,OAAO,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACrC,KAAK,OAAO;YACX,OAAO,yHAAyH,KAAK,GAAG,CAAA;QACzI,KAAK,UAAU;YACd,OAAO,uFAAuF,KAAK,iDAAiD,CAAA;IACtJ,CAAC;AACF,CAAC;AAID,SAAS,QAAQ,CAAC,KAAsB,EAAE,CAA0B,EAAE,QAAiB,EAAE,KAAa;IACrG,QAAQ,KAAK,EAAE,CAAC;QACf,KAAK,OAAO;YACX,OAAO;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtB,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC1B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC;gBACnC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,QAAQ,EAAG,CAAC,CAAC,QAAmB,IAAI,IAAI;gBACxC,OAAO,EAAG,CAAC,CAAC,OAAkB,IAAI,IAAI;aACtC,CAAA;QACF,KAAK,UAAU;YACd,OAAO;gBACN,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAClC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,IAAI,EAAG,CAAC,CAAC,IAAe,IAAI,IAAI;gBAChC,OAAO,EAAE,KAAK;aACd,CAAA;QACF,KAAK,YAAY;YAChB,OAAO;gBACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtB,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;gBAChC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;gBAClC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,gBAAgB,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;gBAClD,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC;gBACtC,mBAAmB,EAAG,CAAC,CAAC,mBAA8B,IAAI,IAAI;gBAC9D,gBAAgB,EAAG,CAAC,CAAC,gBAA2B,IAAI,IAAI;gBACxD,kBAAkB,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC;gBACrD,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC;gBAC3C,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;gBAC7C,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;gBACrC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC;aACjC,CAAA;IACH,CAAC;AACF,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,KAAa,EAAE,OAAe;IAC7D,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SAC7E,KAAK,CAAS,EAAE,GAAG,EAAE,GAAG,WAAW,mBAAmB,OAAO,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;SAClG,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,OAAO,IAAI,KAAK,yBAAyB,EAAE,GAAG,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAElC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;IACpB,CAAC;IAED,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAA;AAChC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,OAA0B;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAA;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,KAAK,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAA;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAA;IAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAA;IAEhC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAA;IAE1D,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAA;IAClF,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;IAC/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IAExC,MAAM,QAAQ,GAAG,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAErF,IAAI,KAAK,KAAK,UAAU,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,wCAAwC,KAAK,YAAY,OAAO,EAAE,CAAC,CAAA;IACpF,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,cAAc,CAAgB,OAAO,CAAC,CAAA;IACtD,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IACvB,MAAM,qBAAqB,CAAC,GAAG,CAAC,CAAA;IAEhC,MAAM,WAAW,GAAG,KAAK,EAAE,IAAW,EAAiB,EAAE;QACxD,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG;iBACP,UAAU,CAAC,YAAY,CAAC;iBACxB,MAAM,CAAC,IAAyB,CAAC;iBACjC,OAAO,EAAE,CAAA;QACZ,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,GAAG;iBACP,UAAU,CAAC,cAAc,CAAC;iBAC1B,MAAM,CAAC,IAAyB,CAAC;iBACjC,OAAO,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,MAAM,GAAG;iBACP,UAAU,CAAC,eAAe,CAAC;iBAC3B,MAAM,CAAC,IAA0B,CAAC;iBAClC,OAAO,EAAE,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC7E,CAAC;aAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC9B,MAAM,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC5E,CAAC;aAAM,CAAC;YACP,MAAM,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAA;QAC7E,CAAC;QAED,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,IAAI,KAAK,GAAU,EAAE,CAAA;QAErB,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACxB,IAAI,CAAC,KAAK,CAAC,MAAM;gBAAE,OAAM;YACzB,MAAM,IAAI,GAAG,KAAK,CAAA;YAClB,KAAK,GAAG,EAAE,CAAA;YACV,MAAM,WAAW,CAAC,IAAI,CAAC,CAAA;YACvB,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAA;QACxB,CAAC,CAAA;QAED,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;YACvD,MAAM,OAAO,GAAG,MAAM,OAAO,IAAI,IAAI,IAAI,KAAK,MAAM,CAAA;YACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAA;YACjC,MAAM,GAAG,GAAG,GAAG,WAAW,mBAAmB,OAAO,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,OAAO,EAAE,CAAA;YAErF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YACnD,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;YAElD,MAAM,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;YAC1C,MAAM,KAAK,GAAG,MAAM,OAAO,IAAI,IAAI,IAAI,KAAK,EAAE,CAAA;YAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;YACxC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,GAAG,MAAM,EAAE,CAAA;YAEhD,MAAM,IAAI,GAAG;gBACZ,IAAI;gBACJ,YAAY;gBACZ,QAAQ;gBACR,WAAW;gBACX,MAAM;gBACN,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC;gBACvC,aAAa;gBACb,OAAO;aACP,CAAA;YAED,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,iBAAiB,CACzC,IAAI,EACJ,KAAK,CACL,EAAE,CAAC;gBACH,IAAI,CAAC,IAAI,CAAC,UAAU;oBAAE,SAAQ;gBAE9B,IAAI,KAAK,KAAK,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ;oBAAE,SAAQ;gBACtD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAElE,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,EAAE,CAAA;oBACb,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;gBAC5C,CAAC;YACF,CAAC;YAED,MAAM,KAAK,EAAE,CAAA;YACb,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;QAC5C,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAA;QAE5C,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;IACxD,CAAC;YAAS,CAAC;QACV,MAAM,GAAG,CAAC,OAAO,EAAE,CAAA;IACpB,CAAC;AACF,CAAC"}
@@ -4,22 +4,24 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
6
  * Pull a state's Census 2020 P.L. 94-171 redistricting counts (table P2 — Hispanic-or-Latino by
7
- * race, per block) into the `pl_block` table of a TIGER {@link DatabaseClient} DB, keyed on the
8
- * same 15-char block `GEOID` as {@link fetchTIGER}'s `tabblock20`. Join the two for block-level
7
+ * race, per block) into the `pl_block` table of a tiger {@link DatabaseClient} DB, keyed on the
8
+ * same 15-char block `geoid` as {@link fetchTIGER}'s `tabblock20`. Join the two for block-level
9
9
  * race + geometry (e.g. a dot-density map).
10
10
  *
11
11
  * Keyless public data. The per-state ZIP holds a pipe-delimited geographic header
12
- * (`<st>geo<yr>.pl`) and three data segments; segment 1 (`<st>00001<yr>.pl`) carries P1 + P2. We
13
- * join the header (filtered to SUMLEV 750 = block) to segment 1 by LOGRECNO. Field offsets are
12
+ * (`<st>geo<yr>.pl`) and three data segments. segment 1 (`<st>00001<yr>.pl`) contains P1 + P2. We
13
+ * join the header (filtered to sumlev 750 = block) to segment 1 by logrecno. Field offsets are
14
14
  * fixed by the 2020 P.L. layout (verified against the real files).
15
15
  *
16
16
  * https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
17
17
  */
18
18
  /**
19
- * Segment 2: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
20
- * against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623 / 916,517, the published CA 2020
21
- * figures. `occupied + vacant === housing_units` is the invariant that distinguishes a correct offset from a plausible
22
- * one, and {@link parseH1} refuses a row where it does not hold.
19
+ * Segment 2: fileid|stusab|chariter|cifsn|logrecno(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
20
+ * against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623
21
+ * / 916,517, the published CA 2020 figures.
22
+ *
23
+ * `occupied + vacant === housing_units` is the invariant that distinguishes a correct
24
+ * offset from a plausible one. {@link parseH1} refuses a row when it does not hold.
23
25
  */
24
26
  export declare const SEG2_FIELD_COUNT = 152;
25
27
  /**
@@ -31,7 +33,7 @@ export declare const H1_TOTAL: number;
31
33
  */
32
34
  export declare const H1_OCCUPIED: number;
33
35
  /**
34
- * H0010003 — vacant housing units, the last field (and so the one that carries a CRLF file's trailing CR).
36
+ * H0010003 — vacant housing units, the last field (and so the one that contains a crlf file's trailing CR).
35
37
  */
36
38
  export declare const H1_VACANT: number;
37
39
  export interface H1Counts {
@@ -40,8 +42,11 @@ export interface H1Counts {
40
42
  vacant: number;
41
43
  }
42
44
  /**
43
- * The three H1 counts from one segment-2 row. The last field carries CRLF's trailing CR when the file has one, so each
44
- * field is trimmed before it is read as a number; a row whose counts do not add up is refused rather than stored.
45
+ * The three H1 counts from one segment-2 row.
46
+ *
47
+ * The last field contains crlf's trailing CR when the file has one, so each field
48
+ * is trimmed before it is read as a number.
49
+ * A row whose counts do not add up is refused rather than stored.
45
50
  */
46
51
  export declare function parseH1(fields: readonly string[]): H1Counts;
47
52
  export interface FetchRedistrictingOptions {
@@ -50,11 +55,15 @@ export interface FetchRedistrictingOptions {
50
55
  */
51
56
  stateFIPS: string;
52
57
  /**
53
- * Decennial vintage. Default 2020 (the only P.L. 94-171 release this parses).
58
+ * Decennial vintage.
59
+ *
60
+ * Default 2020 (the only P.L. 94-171 release this parses).
54
61
  */
55
62
  vintage?: number;
56
63
  /**
57
- * Output SQLite path. Default `<dataRoot>/tiger/tiger.db` (same DB as `fetchTIGER`).
64
+ * Output SQLite path.
65
+ *
66
+ * Default `<dataRoot>/tiger/tiger.db` (same DB as `fetchTIGER`).
58
67
  */
59
68
  outPath?: string;
60
69
  /**
@@ -66,7 +75,9 @@ export interface FetchRedistrictingOptions {
66
75
  */
67
76
  county?: string;
68
77
  /**
69
- * Rows per insert. Default 2000.
78
+ * Rows per insert.
79
+ *
80
+ * Default 2000.
70
81
  */
71
82
  batchSize?: number;
72
83
  }
@@ -91,7 +102,10 @@ export interface FetchRedistrictingResult {
91
102
  inserted: number;
92
103
  }
93
104
  /**
94
- * Fetch one state's P.L. 94-171 block race counts into `pl_block`. Yields progress; returns the tally.
105
+ * Fetch one state's P.L. 94-171 block race counts into `pl_block`.
106
+ *
107
+ * Yields progress.
108
+ * Returns the tally.
95
109
  */
96
110
  export declare function fetchRedistricting(options: FetchRedistrictingOptions): AsyncGenerator<FetchRedistrictingEvent, FetchRedistrictingResult>;
97
111
  //# sourceMappingURL=redistricting.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"redistricting.d.ts","sourceRoot":"","sources":["../../lib/sdk/redistricting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAgCH;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAA;AACnC;;GAEG;AACH,eAAO,MAAM,QAAQ,QAAuB,CAAA;AAC5C;;GAEG;AACH,eAAO,MAAM,WAAW,QAAuB,CAAA;AAC/C;;GAEG;AACH,eAAO,MAAM,SAAS,QAAuB,CAAA;AAE7C,MAAM,WAAW,QAAQ;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,QAAQ,CAsB3D;AAiBD,MAAM,WAAW,yBAAyB;IACzC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,MAAM,uBAAuB,GAChC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClC;IAAE,KAAK,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAA;AAErD,MAAM,WAAW,wBAAwB;IACxC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAcD;;GAEG;AACH,wBAAuB,kBAAkB,CACxC,OAAO,EAAE,yBAAyB,GAChC,cAAc,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAkInE"}
1
+ {"version":3,"file":"redistricting.d.ts","sourceRoot":"","sources":["../../sdk/redistricting.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAiCH;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAA;AACnC;;GAEG;AACH,eAAO,MAAM,QAAQ,QAAuB,CAAA;AAC5C;;GAEG;AACH,eAAO,MAAM,WAAW,QAAuB,CAAA;AAC/C;;GAEG;AACH,eAAO,MAAM,SAAS,QAAuB,CAAA;AAE7C,MAAM,WAAW,QAAQ;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,QAAQ,CAsB3D;AAiBD,MAAM,WAAW,yBAAyB;IACzC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,MAAM,uBAAuB,GAChC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACpD;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClC;IAAE,KAAK,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAA;AAErD,MAAM,WAAW,wBAAwB;IACxC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;CAChB;AAeD;;;;;GAKG;AACH,wBAAuB,kBAAkB,CACxC,OAAO,EAAE,yBAAyB,GAChC,cAAc,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAmInE"}
@@ -4,46 +4,49 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
6
  * Pull a state's Census 2020 P.L. 94-171 redistricting counts (table P2 — Hispanic-or-Latino by
7
- * race, per block) into the `pl_block` table of a TIGER {@link DatabaseClient} DB, keyed on the
8
- * same 15-char block `GEOID` as {@link fetchTIGER}'s `tabblock20`. Join the two for block-level
7
+ * race, per block) into the `pl_block` table of a tiger {@link DatabaseClient} DB, keyed on the
8
+ * same 15-char block `geoid` as {@link fetchTIGER}'s `tabblock20`. Join the two for block-level
9
9
  * race + geometry (e.g. a dot-density map).
10
10
  *
11
11
  * Keyless public data. The per-state ZIP holds a pipe-delimited geographic header
12
- * (`<st>geo<yr>.pl`) and three data segments; segment 1 (`<st>00001<yr>.pl`) carries P1 + P2. We
13
- * join the header (filtered to SUMLEV 750 = block) to segment 1 by LOGRECNO. Field offsets are
12
+ * (`<st>geo<yr>.pl`) and three data segments. segment 1 (`<st>00001<yr>.pl`) contains P1 + P2. We
13
+ * join the header (filtered to sumlev 750 = block) to segment 1 by logrecno. Field offsets are
14
14
  * fixed by the 2020 P.L. layout (verified against the real files).
15
15
  *
16
16
  * https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171/
17
17
  */
18
- import { mailwomanDataRoot } from "@mailwoman/core/data-root";
18
+ import { dataRootPath } from "@mailwoman/core/data-root";
19
19
  import { makeDirectories } from "@mailwoman/core/fs/writers";
20
20
  import { extractZipEntries } from "@mailwoman/core/fs/zip";
21
21
  import { DatabaseClient } from "@mailwoman/sqlite/client";
22
- import { dirname, join } from "path-ts";
22
+ import { PathBuilder } from "path-ts";
23
23
  import { TextSpliterator } from "spliterator";
24
+ import { initializeTIGERSchema, TIGER_PRAGMAS } from "#schema";
24
25
  import { downloadIfNeeded } from "#sdk/download";
25
- import { initializeTIGERSchema, TIGER_PRAGMAS } from "#sdk/schema";
26
26
  import { AdminLevel1CodeToAbbreviation, StateName } from "#state";
27
27
  const REDISTRICTING_BASE = "https://www2.census.gov/programs-surveys/decennial/2020/data/01-Redistricting_File--PL_94-171";
28
- const DEFAULT_DATA_ROOT = mailwomanDataRoot();
28
+ const DEFAULT_DATA_ROOT = dataRootPath();
29
29
  /**
30
- * P.L. 94-171 (2020) pipe-delimited field offsets (0-based). Geographic header:
31
- * …|SUMLEV(2)|…|LOGRECNO(7)|GEOID(8)|GEOCODE(9)|… — GEOCODE is the bare 15-char block FIPS (matches TIGER GEOID20);
32
- * SUMLEV 750 = tabulation block.
30
+ * P.L. 94-171 (2020) pipe-delimited field offsets (0-based).
31
+ *
32
+ * Geographic header: …|sumlev(2)|…|logrecno(7)|geoid(8)|geocode(9)|… — geocode is the
33
+ * bare 15-char block FIPS (matches tiger GEOID20); sumlev 750 = tabulation block.
33
34
  */
34
35
  const GEO_SUMLEV = 2;
35
36
  const GEO_LOGRECNO = 7;
36
37
  const GEO_GEOCODE = 9;
37
38
  /**
38
- * Segment 1: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P1×71 | P2×73. P0020001 is at index 76.
39
+ * Segment 1: fileid|stusab|chariter|cifsn|logrecno(4)| P1×71 | P2×73. P0020001 is at index 76.
39
40
  */
40
41
  const SEG_LOGRECNO = 4;
41
42
  const P2 = (fieldNo) => 76 + (fieldNo - 1);
42
43
  /**
43
- * Segment 2: FILEID|STUSAB|CHARITER|CIFSN|LOGRECNO(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
44
- * against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623 / 916,517, the published CA 2020
45
- * figures. `occupied + vacant === housing_units` is the invariant that distinguishes a correct offset from a plausible
46
- * one, and {@link parseH1} refuses a row where it does not hold.
44
+ * Segment 2: fileid|stusab|chariter|cifsn|logrecno(4)| P3×71 | P4×73 | H1×3 — 152 fields, H1 at the tail. Verified
45
+ * against the real file: `ca000022020.pl`'s state row reads 14,392,140 / 13,475,623
46
+ * / 916,517, the published CA 2020 figures.
47
+ *
48
+ * `occupied + vacant === housing_units` is the invariant that distinguishes a correct
49
+ * offset from a plausible one. {@link parseH1} refuses a row when it does not hold.
47
50
  */
48
51
  export const SEG2_FIELD_COUNT = 152;
49
52
  /**
@@ -55,12 +58,15 @@ export const H1_TOTAL = SEG2_FIELD_COUNT - 3;
55
58
  */
56
59
  export const H1_OCCUPIED = SEG2_FIELD_COUNT - 2;
57
60
  /**
58
- * H0010003 — vacant housing units, the last field (and so the one that carries a CRLF file's trailing CR).
61
+ * H0010003 — vacant housing units, the last field (and so the one that contains a crlf file's trailing CR).
59
62
  */
60
63
  export const H1_VACANT = SEG2_FIELD_COUNT - 1;
61
64
  /**
62
- * The three H1 counts from one segment-2 row. The last field carries CRLF's trailing CR when the file has one, so each
63
- * field is trimmed before it is read as a number; a row whose counts do not add up is refused rather than stored.
65
+ * The three H1 counts from one segment-2 row.
66
+ *
67
+ * The last field contains crlf's trailing CR when the file has one, so each field
68
+ * is trimmed before it is read as a number.
69
+ * A row whose counts do not add up is refused rather than stored.
64
70
  */
65
71
  export function parseH1(fields) {
66
72
  if (fields.length !== SEG2_FIELD_COUNT) {
@@ -85,7 +91,7 @@ export function parseH1(fields) {
85
91
  const CATEGORY_INDEX = {
86
92
  pop_total: P2(1),
87
93
  hispanic: P2(2), // Hispanic or Latino (any race)
88
- white: P2(5), // Not Hispanic: White alone
94
+ white: P2(5), // Not Hispanic: White without another race
89
95
  black: P2(6),
90
96
  aian: P2(7),
91
97
  asian: P2(8),
@@ -93,9 +99,10 @@ const CATEGORY_INDEX = {
93
99
  other: P2(10),
94
100
  multi: P2(11), // Two or more races
95
101
  };
96
- // Pipe-delimited, fixed field-offset census rows — the manual `split("|")` at each call site stays (the
97
- // parse indexes by position, not by header). spliterator keeps CRLF's trailing CR where readline stripped
98
- // it, but every field this parser reads (geo GEOCODE/LOGRECNO ≤ 9, segment-1 LOGRECNO + P2 ≤ 86) sits well
102
+ // Pipe-delimited, fixed field-offset census rows.
103
+ // The manual `split("|")` at each call site stays (the parse indexes by position rather than by header).
104
+ // Spliterator keeps crlf's trailing CR where readline stripped it, but every field
105
+ // this parser reads (geo geocode/logrecno ≤ 9, segment-1 logrecno + P2 ≤ 86) sits well
99
106
  // before the final column, so the retained CR only ever lands on an unread trailing field.
100
107
  async function eachLine(path, fn) {
101
108
  for await (const line of TextSpliterator.fromAsync(path)) {
@@ -105,11 +112,14 @@ async function eachLine(path, fn) {
105
112
  }
106
113
  }
107
114
  /**
108
- * Fetch one state's P.L. 94-171 block race counts into `pl_block`. Yields progress; returns the tally.
115
+ * Fetch one state's P.L. 94-171 block race counts into `pl_block`.
116
+ *
117
+ * Yields progress.
118
+ * Returns the tally.
109
119
  */
110
120
  export async function* fetchRedistricting(options) {
111
121
  const vintage = options.vintage ?? 2020;
112
- const dataRoot = options.dataRoot ?? DEFAULT_DATA_ROOT;
122
+ const dataRoot = PathBuilder.from(options.dataRoot ?? DEFAULT_DATA_ROOT);
113
123
  const batchSize = options.batchSize ?? 2000;
114
124
  const state = options.stateFIPS;
115
125
  const abbr = AdminLevel1CodeToAbbreviation[state];
@@ -118,22 +128,22 @@ export async function* fetchRedistricting(options) {
118
128
  const stateName = StateName[abbr];
119
129
  const dirName = stateName.replaceAll(" ", "_");
120
130
  const fileAbbr = abbr.toLowerCase();
121
- const cacheDir = join(dataRoot, "census", "redistricting", String(vintage), state);
131
+ const cacheDir = dataRoot("census", "redistricting", String(vintage), state);
122
132
  // Same stable `tiger.db` default as fetchTIGER — pl_block lives alongside tabblock20 in one DB.
123
- const outPath = options.outPath ?? join(dataRoot, "tiger", "tiger.db");
133
+ const outPath = PathBuilder.from(options.outPath ?? dataRoot("tiger", "tiger.db"));
124
134
  await makeDirectories(cacheDir);
125
- await makeDirectories(dirname(outPath));
135
+ await makeDirectories(outPath.dirname());
126
136
  const zipName = `${fileAbbr}${vintage}.pl.zip`;
127
- const zipPath = join(cacheDir, zipName);
137
+ const zipPath = cacheDir(zipName);
128
138
  const url = `${REDISTRICTING_BASE}/${dirName}/${zipName}`;
129
139
  const cached = await downloadIfNeeded(url, zipPath);
130
140
  yield { phase: "download", file: zipName, cached };
131
141
  await extractZipEntries(zipPath, cacheDir);
132
- const geoPath = join(cacheDir, `${fileAbbr}geo${vintage}.pl`);
133
- const seg1Path = join(cacheDir, `${fileAbbr}00001${vintage}.pl`);
134
- const seg2Path = join(cacheDir, `${fileAbbr}00002${vintage}.pl`);
142
+ const geoPath = cacheDir(`${fileAbbr}geo${vintage}.pl`);
143
+ const seg1Path = cacheDir(`${fileAbbr}00001${vintage}.pl`);
144
+ const seg2Path = cacheDir(`${fileAbbr}00002${vintage}.pl`);
135
145
  yield { phase: "extract", file: `${fileAbbr}geo${vintage}.pl` };
136
- // Pass 1: header → LOGRECNO → GEOID for the blocks we want.
146
+ // Pass 1: header → logrecno → geoid for the blocks we want.
137
147
  const prefix = options.county ? state + options.county : state;
138
148
  const logToGeoid = new Map();
139
149
  await eachLine(geoPath, (line) => {
@@ -147,8 +157,9 @@ export async function* fetchRedistricting(options) {
147
157
  });
148
158
  const total = logToGeoid.size;
149
159
  yield { phase: "header", blocks: total };
150
- // Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs. Read before segment 1 so a block's P2 and H1
151
- // land in one row; a mapped block with no segment-2 row is a data defect the load refuses, never a zero.
160
+ // Pass 1b: segment 2 → H1 housing counts for the mapped LOGRECNOs.
161
+ // Read before segment 1 so a block's P2 and H1 land in one row.
162
+ // A mapped block with no segment-2 row is a data defect the load refuses, never a zero.
152
163
  const h1ByLogrecno = new Map();
153
164
  await eachLine(seg2Path, (line) => {
154
165
  const f = line.split("|");
@@ -211,7 +222,7 @@ export async function* fetchRedistricting(options) {
211
222
  await flush();
212
223
  yield { phase: "load", inserted, total };
213
224
  kdb.exec("PRAGMA wal_checkpoint(TRUNCATE);");
214
- return { outPath, table: "pl_block", inserted };
225
+ return { outPath: outPath.toString(), table: "pl_block", inserted };
215
226
  }
216
227
  finally {
217
228
  await kdb.destroy();