@mailwoman/corpus 5.9.0 → 5.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/out/src/adapter.d.ts +1 -1
  2. package/out/src/adapters/ban/adapter.d.ts +1 -1
  3. package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
  4. package/out/src/adapters/geonames/adapter.d.ts +1 -1
  5. package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
  6. package/out/src/adapters/gnaf/adapter.d.ts +1 -1
  7. package/out/src/adapters/index.d.ts +19 -19
  8. package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
  9. package/out/src/adapters/overture/adapter.d.ts +1 -1
  10. package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
  11. package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
  12. package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
  13. package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
  14. package/out/src/adapters/synth-po-box/adapter.d.ts +2 -2
  15. package/out/src/adapters/tiger/adapter.d.ts +1 -1
  16. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
  17. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
  18. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
  19. package/out/src/adapters/usgov-nad/adapter.d.ts +2 -2
  20. package/out/src/adapters/usgov-nad/adapter.js +1 -1
  21. package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
  22. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
  23. package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
  24. package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
  25. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
  26. package/out/src/align.d.ts +2 -2
  27. package/out/src/build.d.ts +4 -4
  28. package/out/src/index.d.ts +22 -21
  29. package/out/src/index.d.ts.map +1 -1
  30. package/out/src/index.js +1 -0
  31. package/out/src/index.js.map +1 -1
  32. package/out/src/parquet-wrapper/index.d.ts +3 -3
  33. package/out/src/parquet-wrapper/reader.d.ts +1 -1
  34. package/out/src/parquet-wrapper/writer.d.ts +4 -2
  35. package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
  36. package/out/src/parquet-wrapper/writer.js +3 -1
  37. package/out/src/parquet-wrapper/writer.js.map +1 -1
  38. package/out/src/parquet.d.ts +3 -3
  39. package/out/src/runner.d.ts +2 -2
  40. package/out/src/shard-recipes/anchor-absorption.d.ts +1 -1
  41. package/out/src/shard-recipes/boundary-stress.d.ts +1 -1
  42. package/out/src/shard-recipes/country-balanced.d.ts +1 -1
  43. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +1 -1
  44. package/out/src/shard-recipes/fr-admin-split.d.ts +1 -1
  45. package/out/src/shard-recipes/fr-bare-street.d.ts +1 -1
  46. package/out/src/shard-recipes/fr-order.d.ts +1 -1
  47. package/out/src/shard-recipes/german.d.ts +1 -1
  48. package/out/src/shard-recipes/house-venue.d.ts +1 -1
  49. package/out/src/shard-recipes/index.d.ts +2 -2
  50. package/out/src/shard-recipes/intersection.d.ts +1 -1
  51. package/out/src/shard-recipes/locale.d.ts +2 -2
  52. package/out/src/shard-recipes/nl-postcode.d.ts +1 -1
  53. package/out/src/shard-recipes/no-street-led.d.ts +1 -1
  54. package/out/src/shard-recipes/no-street.d.ts +1 -1
  55. package/out/src/shard-recipes/po-box-cedex.d.ts +1 -1
  56. package/out/src/shard-recipes/po-box.d.ts +1 -1
  57. package/out/src/shard-recipes/si-bare-village.d.ts +1 -1
  58. package/out/src/shard-recipes/street-affix.d.ts +1 -1
  59. package/out/src/shard-recipes/street-bare.d.ts +1 -1
  60. package/out/src/shard-recipes/street.d.ts +1 -1
  61. package/out/src/shard-recipes/unit.d.ts +1 -1
  62. package/out/src/split.d.ts +1 -1
  63. package/out/src/split.js +1 -1
  64. package/out/src/split.js.map +1 -1
  65. package/out/src/synthesize-boundary-stress.d.ts +1 -1
  66. package/out/src/synthesize-german.d.ts +1 -1
  67. package/out/src/synthesize-house-venue.d.ts +1 -1
  68. package/out/src/synthesize-intersection.d.ts +1 -1
  69. package/out/src/synthesize-no-street.d.ts +1 -1
  70. package/out/src/synthesize-po-box.d.ts +1 -1
  71. package/out/src/synthesize-street.d.ts +1 -1
  72. package/out/src/synthesize.d.ts +2 -2
  73. package/out/src/tools/align-shard.d.ts +8 -0
  74. package/out/src/tools/align-shard.d.ts.map +1 -0
  75. package/out/src/tools/align-shard.js +58 -0
  76. package/out/src/tools/align-shard.js.map +1 -0
  77. package/out/src/tools/audit.d.ts +28 -0
  78. package/out/src/tools/audit.d.ts.map +1 -0
  79. package/out/src/tools/audit.js +273 -0
  80. package/out/src/tools/audit.js.map +1 -0
  81. package/out/src/tools/corpus-stats.d.ts +44 -0
  82. package/out/src/tools/corpus-stats.d.ts.map +1 -0
  83. package/out/src/tools/corpus-stats.js +151 -0
  84. package/out/src/tools/corpus-stats.js.map +1 -0
  85. package/out/src/tools/fetch/ban.d.ts +31 -0
  86. package/out/src/tools/fetch/ban.d.ts.map +1 -0
  87. package/out/src/tools/fetch/ban.js +248 -0
  88. package/out/src/tools/fetch/ban.js.map +1 -0
  89. package/out/src/tools/fetch/download.d.ts +49 -0
  90. package/out/src/tools/fetch/download.d.ts.map +1 -0
  91. package/out/src/tools/fetch/download.js +81 -0
  92. package/out/src/tools/fetch/download.js.map +1 -0
  93. package/out/src/tools/fetch/hrsa.d.ts +18 -0
  94. package/out/src/tools/fetch/hrsa.d.ts.map +1 -0
  95. package/out/src/tools/fetch/hrsa.js +46 -0
  96. package/out/src/tools/fetch/hrsa.js.map +1 -0
  97. package/out/src/tools/fetch/imls-pls.d.ts +25 -0
  98. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -0
  99. package/out/src/tools/fetch/imls-pls.js +120 -0
  100. package/out/src/tools/fetch/imls-pls.js.map +1 -0
  101. package/out/src/tools/fetch/index.d.ts +118 -0
  102. package/out/src/tools/fetch/index.d.ts.map +1 -0
  103. package/out/src/tools/fetch/index.js +117 -0
  104. package/out/src/tools/fetch/index.js.map +1 -0
  105. package/out/src/tools/fetch/nad.d.ts +56 -0
  106. package/out/src/tools/fetch/nad.d.ts.map +1 -0
  107. package/out/src/tools/fetch/nad.js +225 -0
  108. package/out/src/tools/fetch/nad.js.map +1 -0
  109. package/out/src/tools/fetch/nppes.d.ts +26 -0
  110. package/out/src/tools/fetch/nppes.d.ts.map +1 -0
  111. package/out/src/tools/fetch/nppes.js +136 -0
  112. package/out/src/tools/fetch/nppes.js.map +1 -0
  113. package/out/src/tools/fetch/openaddresses.d.ts +59 -0
  114. package/out/src/tools/fetch/openaddresses.d.ts.map +1 -0
  115. package/out/src/tools/fetch/openaddresses.js +307 -0
  116. package/out/src/tools/fetch/openaddresses.js.map +1 -0
  117. package/out/src/tools/fetch/state-hi-schools.d.ts +25 -0
  118. package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -0
  119. package/out/src/tools/fetch/state-hi-schools.js +171 -0
  120. package/out/src/tools/fetch/state-hi-schools.js.map +1 -0
  121. package/out/src/tools/fetch/state-sources.d.ts +22 -0
  122. package/out/src/tools/fetch/state-sources.d.ts.map +1 -0
  123. package/out/src/tools/fetch/state-sources.js +110 -0
  124. package/out/src/tools/fetch/state-sources.js.map +1 -0
  125. package/out/src/tools/fetch/tiger-full.d.ts +36 -0
  126. package/out/src/tools/fetch/tiger-full.d.ts.map +1 -0
  127. package/out/src/tools/fetch/tiger-full.js +239 -0
  128. package/out/src/tools/fetch/tiger-full.js.map +1 -0
  129. package/out/src/tools/golden-expand.d.ts +74 -0
  130. package/out/src/tools/golden-expand.d.ts.map +1 -0
  131. package/out/src/tools/golden-expand.js +394 -0
  132. package/out/src/tools/golden-expand.js.map +1 -0
  133. package/out/src/tools/golden-promote.d.ts +60 -0
  134. package/out/src/tools/golden-promote.d.ts.map +1 -0
  135. package/out/src/tools/golden-promote.js +223 -0
  136. package/out/src/tools/golden-promote.js.map +1 -0
  137. package/out/src/tools/index.d.ts +24 -0
  138. package/out/src/tools/index.d.ts.map +1 -0
  139. package/out/src/tools/index.js +24 -0
  140. package/out/src/tools/index.js.map +1 -0
  141. package/out/src/tools/ingest-csv.d.ts +45 -0
  142. package/out/src/tools/ingest-csv.d.ts.map +1 -0
  143. package/out/src/tools/ingest-csv.js +309 -0
  144. package/out/src/tools/ingest-csv.js.map +1 -0
  145. package/out/src/tools/jsonl-to-parquet.d.ts +49 -0
  146. package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -0
  147. package/out/src/tools/jsonl-to-parquet.js +157 -0
  148. package/out/src/tools/jsonl-to-parquet.js.map +1 -0
  149. package/out/src/tools/lint-shard-vocab.d.ts +66 -0
  150. package/out/src/tools/lint-shard-vocab.d.ts.map +1 -0
  151. package/out/src/tools/lint-shard-vocab.js +266 -0
  152. package/out/src/tools/lint-shard-vocab.js.map +1 -0
  153. package/out/src/tools/lint-shard.d.ts +73 -0
  154. package/out/src/tools/lint-shard.d.ts.map +1 -0
  155. package/out/src/tools/lint-shard.js +350 -0
  156. package/out/src/tools/lint-shard.js.map +1 -0
  157. package/out/src/tools/overlay-manifest.d.ts +35 -0
  158. package/out/src/tools/overlay-manifest.d.ts.map +1 -0
  159. package/out/src/tools/overlay-manifest.js +85 -0
  160. package/out/src/tools/overlay-manifest.js.map +1 -0
  161. package/out/src/tools/shard-kryptonite.d.ts +28 -0
  162. package/out/src/tools/shard-kryptonite.d.ts.map +1 -0
  163. package/out/src/tools/shard-kryptonite.js +101 -0
  164. package/out/src/tools/shard-kryptonite.js.map +1 -0
  165. package/out/src/tools/shard-translit.d.ts +42 -0
  166. package/out/src/tools/shard-translit.d.ts.map +1 -0
  167. package/out/src/tools/shard-translit.js +196 -0
  168. package/out/src/tools/shard-translit.js.map +1 -0
  169. package/package.json +63 -7
@@ -0,0 +1,110 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
7
+ * providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
8
+ *
9
+ * HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
10
+ * upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
11
+ * before the adapter can consume it.
12
+ *
13
+ * Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
14
+ * recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
15
+ *
16
+ * Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
17
+ * (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
18
+ */
19
+ import { mkdirSync } from "node:fs";
20
+ import { join } from "node:path";
21
+ import { sha256File } from "@mailwoman/core/utils";
22
+ import { downloadToFile, writeManifest } from "./download.js";
23
+ const SOURCES = [
24
+ {
25
+ slug: "state-ny-notaries",
26
+ filename: "NY_Commissioned_Notaries.csv",
27
+ url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
28
+ },
29
+ {
30
+ slug: "state-tx-notaries",
31
+ filename: "TX_Notary_Public_Commissions.csv",
32
+ url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
33
+ },
34
+ {
35
+ slug: "state-de-notaries",
36
+ filename: "DE_Notaries_Commissioned.csv",
37
+ url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
38
+ },
39
+ {
40
+ slug: "state-or-notaries",
41
+ filename: "OR_Active_Notaries.csv",
42
+ url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
43
+ },
44
+ {
45
+ slug: "state-ia-contractors",
46
+ filename: "IA_Active_Construction_Contractor_Registrations.csv",
47
+ url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
48
+ },
49
+ {
50
+ slug: "state-wa-health-providers",
51
+ filename: "WA_Health_Care_Provider_Credential_Data.csv",
52
+ url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
53
+ },
54
+ {
55
+ slug: "state-hi-lobbyists",
56
+ filename: "HI_Lobbyist_Registration_Statements.csv",
57
+ url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
58
+ },
59
+ ];
60
+ export async function fetchStateSources(options, report) {
61
+ mkdirSync(options.outRoot, { recursive: true });
62
+ let fetched = 0;
63
+ let failed = 0;
64
+ const failedCodes = [];
65
+ for (const { slug, filename, url } of SOURCES) {
66
+ const destDir = join(options.outRoot, slug);
67
+ mkdirSync(destDir, { recursive: true });
68
+ const dest = join(destDir, filename);
69
+ report?.(`=== ${slug} / ${filename}`);
70
+ let bytes;
71
+ try {
72
+ ;
73
+ ({ bytes } = await downloadToFile({
74
+ url,
75
+ dest,
76
+ timeoutMs: 600_000,
77
+ headers: { "Accept-Encoding": "gzip, br" },
78
+ report,
79
+ }));
80
+ }
81
+ catch (err) {
82
+ report?.(` ✗ download failed for ${url}: ${err.message}`);
83
+ failed++;
84
+ failedCodes.push(slug);
85
+ continue;
86
+ }
87
+ if (bytes < 1024) {
88
+ report?.(` ✗ response too small (${bytes} bytes) — probable 404 / error page`);
89
+ failed++;
90
+ failedCodes.push(slug);
91
+ continue;
92
+ }
93
+ const sha = await sha256File(dest);
94
+ const manifest = {
95
+ source_url: url,
96
+ downloaded_at: new Date().toISOString(),
97
+ filename,
98
+ sha256: sha,
99
+ bytes,
100
+ };
101
+ await writeManifest(join(destDir, "MANIFEST.json"), manifest);
102
+ report?.(` ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB sha256=${sha}`);
103
+ fetched++;
104
+ }
105
+ report?.(`=== summary ===`);
106
+ report?.(`fetched: ${fetched}`);
107
+ report?.(`failed: ${failed}`);
108
+ return { fetched, skipped: 0, failed, failedCodes };
109
+ }
110
+ //# sourceMappingURL=state-sources.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"state-sources.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAU7D,MAAM,OAAO,GAAsB;IAClC;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,kCAAkC;QAC5C,GAAG,EAAE,yEAAyE;KAC9E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,4EAA4E;KACjF;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,wBAAwB;QAClC,GAAG,EAAE,0EAA0E;KAC/E;IACD;QACC,IAAI,EAAE,sBAAsB;QAC5B,QAAQ,EAAE,qDAAqD;QAC/D,GAAG,EAAE,wEAAwE;KAC7E;IACD;QACC,IAAI,EAAE,2BAA2B;QACjC,QAAQ,EAAE,6CAA6C;QACvD,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,yCAAyC;QACnD,GAAG,EAAE,0EAA0E;KAC/E;CACD,CAAA;AAUD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAiC,EACjC,MAA+B;IAE/B,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE/C,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,CAAC;YAAA,CAAC,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;gBAClC,GAAG;gBACH,IAAI;gBACJ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,EAAE,CAAC,2BAA2B,GAAG,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAA;YACrE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,EAAE,CAAC,2BAA2B,KAAK,qCAAqC,CAAC,CAAA;YAC/E,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtB,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAClC,MAAM,QAAQ,GAAmB;YAChC,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAA;QACD,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;QAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QACrE,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,YAAY,OAAO,EAAE,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}
@@ -0,0 +1,36 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
7
+ *
8
+ * TIGER ADDRFEAT 2024 source:
9
+ *
10
+ * - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
11
+ * - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
12
+ *
13
+ * Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
14
+ * `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
15
+ * already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
16
+ * this module is download + provenance only.)
17
+ *
18
+ * Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
19
+ * county ZIP to disk (no curl subprocess).
20
+ */
21
+ import type { BaseFetchOptions, FetchSummary } from "./download.ts";
22
+ export interface FetchTigerFullOptions extends BaseFetchOptions {
23
+ /**
24
+ * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
25
+ * v0.1.1.
26
+ */
27
+ skipStateFips?: string;
28
+ /** Seconds to sleep between downloads. Default `0.2`. */
29
+ rateSleep?: number;
30
+ /** Max concurrent download workers per state. Default `4`. */
31
+ maxParallel?: number;
32
+ /** Print planned downloads without fetching. Default `false`. */
33
+ dryRun?: boolean;
34
+ }
35
+ export declare function fetchTigerFull(options: FetchTigerFullOptions, report?: (line: string) => void): Promise<FetchSummary>;
36
+ //# sourceMappingURL=tiger-full.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tiger-full.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAUH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAKnE,MAAM,WAAW,qBAAsB,SAAQ,gBAAgB;IAC9D;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,iEAAiE;IACjE,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AA+GD,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA6JvB"}
@@ -0,0 +1,239 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
7
+ *
8
+ * TIGER ADDRFEAT 2024 source:
9
+ *
10
+ * - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
11
+ * - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
12
+ *
13
+ * Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
14
+ * `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
15
+ * already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
16
+ * this module is download + provenance only.)
17
+ *
18
+ * Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
19
+ * county ZIP to disk (no curl subprocess).
20
+ */
21
+ import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
22
+ import { basename, join } from "node:path";
23
+ import { Readable } from "node:stream";
24
+ import { pipeline } from "node:stream/promises";
25
+ import { setTimeout as sleep } from "node:timers/promises";
26
+ import { sha256File } from "@mailwoman/core/utils";
27
+ import { isTransientStatus, readManifest, writeManifest } from "./download.js";
28
+ const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT";
29
+ function humanBytes(bytes) {
30
+ const units = ["B", "KiB", "MiB", "GiB", "TiB"];
31
+ let value = bytes;
32
+ let unit = 0;
33
+ while (value >= 1024 && unit < units.length - 1) {
34
+ value /= 1024;
35
+ unit++;
36
+ }
37
+ return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`;
38
+ }
39
+ /**
40
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries).
41
+ *
42
+ * NOTE(phase1): kept local instead of the shared `downloadToFile` — this one streams each county ZIP to disk (the
43
+ * shared util buffers via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-county result
44
+ * collector consumes.
45
+ */
46
+ async function streamDownload(url, dest, opts) {
47
+ for (let attempt = 0; attempt <= opts.retries; attempt++) {
48
+ try {
49
+ const res = await fetch(url, { redirect: "follow", signal: AbortSignal.timeout(opts.timeoutMs) });
50
+ if (res.ok && res.body) {
51
+ await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
52
+ return res.status;
53
+ }
54
+ if (attempt < opts.retries && isTransientStatus(res.status)) {
55
+ await sleep(opts.retryDelayMs);
56
+ continue;
57
+ }
58
+ return res.status;
59
+ }
60
+ catch {
61
+ if (attempt < opts.retries) {
62
+ await sleep(opts.retryDelayMs);
63
+ continue;
64
+ }
65
+ return 0;
66
+ }
67
+ }
68
+ return 0;
69
+ }
70
+ /** Read a per-state MANIFEST.json into a filename → entry map. */
71
+ async function readCountyManifest(manifestPath) {
72
+ const map = new Map();
73
+ const parsed = await readManifest(manifestPath);
74
+ for (const c of parsed?.counties ?? []) {
75
+ if (c.filename) {
76
+ map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes });
77
+ }
78
+ }
79
+ return map;
80
+ }
81
+ /** Check whether a file already matches a recorded sha256 and byte count. */
82
+ async function fileMatchesSha(path, expectedSha, expectedBytes) {
83
+ if (!existsSync(path))
84
+ return false;
85
+ if (statSync(path).size !== expectedBytes)
86
+ return false;
87
+ return (await sha256File(path)) === expectedSha;
88
+ }
89
+ /** Download one county ZIP (size sanity check + sha256). */
90
+ async function downloadCounty(url, dest) {
91
+ const filename = basename(dest);
92
+ const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5_000 });
93
+ if (status < 200 || status >= 300) {
94
+ return { ok: false, filename, reason: `HTTP ${status}` };
95
+ }
96
+ const bytes = statSync(dest).size;
97
+ if (bytes < 1024) {
98
+ rmSync(dest, { force: true });
99
+ return { ok: false, filename, reason: `too small (${bytes} bytes)` };
100
+ }
101
+ const sha256 = await sha256File(dest);
102
+ return { ok: true, filename, sha256, bytes };
103
+ }
104
+ export async function fetchTigerFull(options, report) {
105
+ const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter(Boolean);
106
+ const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000);
107
+ const maxParallel = options.maxParallel ?? 4;
108
+ const dryRun = options.dryRun ?? false;
109
+ const addrfeatDir = join(options.outRoot, "tiger", "addrfeat");
110
+ mkdirSync(addrfeatDir, { recursive: true });
111
+ // -------------------------------------------------------------------------
112
+ // Step 1: Discover the full county file list from the TIGER directory listing.
113
+ // -------------------------------------------------------------------------
114
+ report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`);
115
+ const listingRes = await fetch(`${TIGER_BASE_URL}/`, {
116
+ headers: { "Accept-Encoding": "gzip, br" },
117
+ signal: AbortSignal.timeout(60_000),
118
+ });
119
+ if (!listingRes.ok)
120
+ throw new Error(`Failed to fetch TIGER directory listing: HTTP ${listingRes.status}`);
121
+ const html = await listingRes.text();
122
+ const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g) ?? [])].sort();
123
+ const totalCounties = allZips.length;
124
+ report?.(` Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`);
125
+ // Build a map: state_fips -> list of filenames.
126
+ // tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
127
+ const stateFiles = new Map();
128
+ for (const fname of allZips) {
129
+ const stateFips = fname.slice(8, 10);
130
+ const list = stateFiles.get(stateFips) ?? [];
131
+ list.push(fname);
132
+ stateFiles.set(stateFips, list);
133
+ }
134
+ report?.(` Spans ${stateFiles.size} state/territory FIPS codes.`);
135
+ // -------------------------------------------------------------------------
136
+ // Step 2: For each state, download missing/unverified county ZIPs.
137
+ // -------------------------------------------------------------------------
138
+ let totalFetched = 0;
139
+ let totalSkipped = 0;
140
+ let totalSkippedState = 0;
141
+ let totalFailed = 0;
142
+ let totalBytesFetched = 0;
143
+ const failedCodes = [];
144
+ // Process states in sorted FIPS order for predictable output.
145
+ const sortedStates = [...stateFiles.keys()].sort();
146
+ for (const stateFips of sortedStates) {
147
+ const countyFiles = stateFiles.get(stateFips) ?? [];
148
+ // --- Skip entire state if requested ------------------------------------------
149
+ if (skipStateFips.includes(stateFips)) {
150
+ report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`);
151
+ totalSkippedState += countyFiles.length;
152
+ continue;
153
+ }
154
+ const stateDir = join(addrfeatDir, `state-${stateFips}`);
155
+ mkdirSync(stateDir, { recursive: true });
156
+ const manifestPath = join(stateDir, "MANIFEST.json");
157
+ // Load existing manifest for O(1) verified-skip lookup.
158
+ const manifest = await readCountyManifest(manifestPath);
159
+ report?.(`--- State ${stateFips} — ${countyFiles.length} counties`);
160
+ // Build a list of URLs+dests that need fetching.
161
+ const pending = [];
162
+ for (const fname of countyFiles) {
163
+ const dest = join(stateDir, fname);
164
+ const url = `${TIGER_BASE_URL}/${fname}`;
165
+ const known = manifest.get(fname);
166
+ // Skip if already verified via MANIFEST.
167
+ if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
168
+ report?.(` skip (verified) ${fname}`);
169
+ totalSkipped++;
170
+ continue;
171
+ }
172
+ if (dryRun) {
173
+ report?.(` would fetch: ${url}`);
174
+ totalFetched++;
175
+ continue;
176
+ }
177
+ pending.push({ url, dest });
178
+ }
179
+ if (dryRun)
180
+ continue;
181
+ if (pending.length === 0)
182
+ continue;
183
+ // --- Download pending files with bounded parallelism + rate-limit spacing ---
184
+ const results = new Array(pending.length);
185
+ let cursor = 0;
186
+ const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
187
+ while (true) {
188
+ const i = cursor++;
189
+ if (i >= pending.length)
190
+ return;
191
+ const item = pending[i];
192
+ // Rate-limit: polite spacing before each fetch.
193
+ await sleep(rateSleepMs);
194
+ results[i] = await downloadCounty(item.url, item.dest);
195
+ }
196
+ });
197
+ await Promise.all(workers);
198
+ // Collect results from this state.
199
+ for (const result of results) {
200
+ if (result.ok) {
201
+ report?.(` ok ${result.filename} ${humanBytes(result.bytes)} sha256=${result.sha256.slice(0, 12)}...`);
202
+ manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes });
203
+ totalFetched++;
204
+ totalBytesFetched += result.bytes;
205
+ }
206
+ else {
207
+ report?.(` FAIL ${result.filename} -- ${result.reason}`);
208
+ totalFailed++;
209
+ failedCodes.push(result.filename);
210
+ }
211
+ }
212
+ // Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
213
+ const counties = [...manifest.values()].sort((a, b) => a.filename.localeCompare(b.filename));
214
+ const manifestDoc = {
215
+ state_fips: stateFips,
216
+ updated_at: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
217
+ tiger_base_url: TIGER_BASE_URL,
218
+ counties,
219
+ };
220
+ await writeManifest(manifestPath, manifestDoc);
221
+ }
222
+ // -------------------------------------------------------------------------
223
+ // Summary
224
+ // -------------------------------------------------------------------------
225
+ report?.(`=== Summary ===`);
226
+ report?.(` Total counties in index : ${totalCounties}`);
227
+ report?.(` State(s) fully skipped : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`);
228
+ report?.(` Counties already present : ${totalSkipped}`);
229
+ report?.(` Counties fetched this run : ${totalFetched}`);
230
+ report?.(` Counties failed : ${totalFailed}`);
231
+ if (totalBytesFetched > 0) {
232
+ report?.(` Bytes fetched this run : ${humanBytes(totalBytesFetched)} (${totalBytesFetched})`);
233
+ }
234
+ if (totalFailed > 0) {
235
+ report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`);
236
+ }
237
+ return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes };
238
+ }
239
+ //# sourceMappingURL=tiger-full.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tiger-full.js","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACpF,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE9E,MAAM,cAAc,GAAG,sDAAsD,CAAA;AAsB7E,SAAS,UAAU,CAAC,KAAa;IAChC,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAC/C,IAAI,KAAK,GAAG,KAAK,CAAA;IACjB,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,KAAK,IAAI,IAAI,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjD,KAAK,IAAI,IAAI,CAAA;QACb,IAAI,EAAE,CAAA;IACP,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5D,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,cAAc,CAC5B,GAAW,EACX,IAAY,EACZ,IAAkE;IAElE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAEjG,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEnE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAC9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAC9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,kBAAkB,CAAC,YAAoB;IACrD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAA+B,YAAY,CAAC,CAAA;IAE7E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QAChF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,WAAmB,EAAE,aAAqB;IACrF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IAEnC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,KAAK,CAAA;IAEvD,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,WAAW,CAAA;AAChD,CAAC;AAMD,4DAA4D;AAC5D,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAA;IAEvG,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACnC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,MAAM,EAAE,EAAE,CAAA;IACzD,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAA;IAEjC,IAAI,KAAK,GAAG,IAAI,EAAE,CAAC;QAClB,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAE7B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,KAAK,SAAS,EAAE,CAAA;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAErC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAClF,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAA;IAC9D,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,4EAA4E;IAC5E,+EAA+E;IAC/E,4EAA4E;IAC5E,MAAM,EAAE,CAAC,uDAAuD,CAAC,CAAA;IACjE,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,GAAG,cAAc,GAAG,EAAE;QACpD,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;KACnC,CAAC,CAAA;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;IACzG,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAA;IACpC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACxF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAA;IACpC,MAAM,EAAE,CAAC,WAAW,aAAa,gDAAgD,CAAC,CAAA;IAElF,gDAAgD;IAChD,uFAAuF;IACvF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChB,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,UAAU,CAAC,IAAI,8BAA8B,CAAC,CAAA;IAElE,4EAA4E;IAC5E,mEAAmE;IACnE,4EAA4E;IAC5E,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,8DAA8D;IAC9D,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAElD,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAEnD,gFAAgF;QAChF,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,MAAM,EAAE,CAAC,aAAa,SAAS,qCAAqC,WAAW,CAAC,MAAM,YAAY,CAAC,CAAA;YACnG,iBAAiB,IAAI,WAAW,CAAC,MAAM,CAAA;YACvC,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,SAAS,EAAE,CAAC,CAAA;QACxD,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACxC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAA;QAEpD,wDAAwD;QACxD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAA;QAEvD,MAAM,EAAE,CAAC,aAAa,SAAS,MAAM,WAAW,CAAC,MAAM,WAAW,CAAC,CAAA;QAEnE,iDAAiD;QACjD,MAAM,OAAO,GAAyC,EAAE,CAAA;QAExD,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAClC,MAAM,GAAG,GAAG,GAAG,cAAc,IAAI,KAAK,EAAE,CAAA;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAEjC,yCAAyC;YACzC,IAAI,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtE,MAAM,EAAE,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;gBACtC,YAAY,EAAE,CAAA;gBACd,SAAQ;YACT,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;gBACjC,YAAY,EAAE,CAAA;gBACd,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QAED,IAAI,MAAM;YAAE,SAAQ;QAEpB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAElC,+EAA+E;QAC/E,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACzD,IAAI,MAAM,GAAG,CAAC,CAAA;QACd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YACxF,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;gBAElB,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM;oBAAE,OAAM;gBAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBACxB,gDAAgD;gBAChD,MAAM,KAAK,CAAC,WAAW,CAAC,CAAA;gBACxB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACvD,CAAC;QACF,CAAC,CAAC,CAAA;QACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAE1B,mCAAmC;QACnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,EAAE,CAAC,QAAQ,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAA;gBACzG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;gBACxG,YAAY,EAAE,CAAA;gBACd,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,UAAU,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBACzD,WAAW,EAAE,CAAA;gBACb,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAClC,CAAC;QACF,CAAC;QAED,yFAAyF;QACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAC5F,MAAM,WAAW,GAAG;YACnB,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;YAC9D,cAAc,EAAE,cAAc;YAC9B,QAAQ;SACR,CAAA;QACD,MAAM,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAC/C,CAAC;IAED,4EAA4E;IAC5E,UAAU;IACV,4EAA4E;IAC5E,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,EAAE,CAAC,CAAA;IAC1D,MAAM,EAAE,CAAC,iCAAiC,iBAAiB,wBAAwB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,WAAW,EAAE,CAAC,CAAA;IAExD,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,CAAC,iCAAiC,UAAU,CAAC,iBAAiB,CAAC,KAAK,iBAAiB,GAAG,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,YAAY,WAAW,uCAAuC,CAAC,CAAA;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,GAAG,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9G,CAAC"}
@@ -0,0 +1,74 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Generate golden-set candidate entries by LLM-driven surface-form synthesis from a verified-label
7
+ * seed pulled out of the corpus's labeled test split.
8
+ *
9
+ * ## Why this approach
10
+ *
11
+ * The Phase 2 golden set has 74 entries; session-notes.md called for ≥500/locale. Manual curation
12
+ * doesn't scale. Pure-LLM generation (invent raw + labels from scratch) is too noisy — labels
13
+ * would be unverified.
14
+ *
15
+ * This module takes the middle path:
16
+ *
17
+ * 1. **Seeds come from corpus-v0.2.0 test shard** — already through the alignment pipeline, so labels
18
+ * are pipeline-verified.
19
+ * 2. **LLM only varies the surface form** — case, abbreviations, reordering, dropped components. The
20
+ * component VALUES (locality string, postcode digits, etc.) are preserved verbatim.
21
+ * 3. **Programmatic validator drops hallucinations** — every component value must appear as a
22
+ * substring (case-insensitive, whitespace-normalized) of the variant's raw. Failures dropped
23
+ * silently; cost is wasted tokens, never bad-labeled golden entries.
24
+ *
25
+ * ## Usage
26
+ *
27
+ * ```sh
28
+ * DEEPSEEK_API_KEY=sk-... \
29
+ * mailwoman corpus golden expand \
30
+ * --count 1000 \
31
+ * --variants 5 \
32
+ * --output data/eval/golden/candidates/expand-$(date +%Y%m%d-%H%M%S).jsonl
33
+ * ```
34
+ *
35
+ * ## Env
36
+ *
37
+ * - `DEEPSEEK_API_KEY` — required for provider `deepseek`
38
+ * - `ANTHROPIC_API_KEY` — required for provider `anthropic`
39
+ *
40
+ * ## What this module does NOT do
41
+ *
42
+ * - Does not commit anything or modify the versioned golden dir. Candidates land in
43
+ * `data/eval/golden/candidates/` for operator review (skim, prune, then run
44
+ * `mailwoman corpus golden promote`).
45
+ * - Does not score the LLM's quality — that's an eyeball job after pilot lands.
46
+ * - Does not retry hallucinated candidates. Cost of wasted tokens is trivial (~$0.0006/each).
47
+ */
48
+ export interface ExpandGoldenOptions {
49
+ /** Corpus test shard path(s), comma-separated. Default: the v0.2.0 test shard under the data root. */
50
+ corpus?: string;
51
+ /** Total seeds to process. Default `100` (pilot). */
52
+ count?: number;
53
+ /** Variants requested per seed. Default `5`. */
54
+ variants?: number;
55
+ /** JSONL output path. Default `data/eval/golden/candidates/expand-<ts>.jsonl`. */
56
+ output?: string;
57
+ /** LLM provider. Default `deepseek`. */
58
+ provider?: "deepseek" | "anthropic";
59
+ /** Model id. Default depends on provider. */
60
+ model?: string;
61
+ /** Parallel LLM calls. Default `4`. */
62
+ concurrency?: number;
63
+ /** Comma-separated source allow-list. */
64
+ includeSources?: string;
65
+ }
66
+ export interface ExpandGoldenSummary {
67
+ seedsProcessed: number;
68
+ kept: number;
69
+ dropped: number;
70
+ errored: number;
71
+ outputPath: string;
72
+ }
73
+ export declare function expandGolden(options?: ExpandGoldenOptions, report?: (line: string) => void): Promise<ExpandGoldenSummary>;
74
+ //# sourceMappingURL=golden-expand.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"golden-expand.d.ts","sourceRoot":"","sources":["../../../src/tools/golden-expand.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AA6CH,MAAM,WAAW,mBAAmB;IACnC,sGAAsG;IACtG,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,wCAAwC;IACxC,QAAQ,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;IACnC,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yCAAyC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;CAClB;AAgUD,wBAAsB,YAAY,CACjC,OAAO,GAAE,mBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,mBAAmB,CAAC,CAoF9B"}