@mailwoman/corpus 9.0.0 → 9.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (524) hide show
  1. package/README.md +3 -3
  2. package/data/PROVENANCE.md +227 -0
  3. package/data/reviewed-ve-postcode-tuples.json +74 -0
  4. package/data/sub-venue-lexicon.json +18269 -0
  5. package/out/src/adapters/ban/adapter.d.ts +1 -1
  6. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  7. package/out/src/adapters/ban/adapter.js +5 -4
  8. package/out/src/adapters/ban/adapter.js.map +1 -1
  9. package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
  10. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  11. package/out/src/adapters/fcc-bdc/adapter.js +3 -3
  12. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  13. package/out/src/adapters/geonames/adapter.d.ts +1 -1
  14. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  15. package/out/src/adapters/geonames/adapter.js +2 -2
  16. package/out/src/adapters/geonames/adapter.js.map +1 -1
  17. package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
  18. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  19. package/out/src/adapters/geonames-postal/adapter.js +3 -3
  20. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  21. package/out/src/adapters/gnaf/adapter.d.ts +1 -1
  22. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  23. package/out/src/adapters/gnaf/adapter.js +2 -2
  24. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  25. package/out/src/adapters/gnaf/assemble.js +1 -1
  26. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  27. package/out/src/adapters/index.d.ts +1 -1
  28. package/out/src/adapters/index.d.ts.map +1 -1
  29. package/out/src/adapters/index.js +1 -1
  30. package/out/src/adapters/index.js.map +1 -1
  31. package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
  32. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  33. package/out/src/adapters/openaddresses/adapter.js +3 -3
  34. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  35. package/out/src/adapters/overture/adapter.d.ts +1 -1
  36. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  37. package/out/src/adapters/overture/adapter.js +2 -2
  38. package/out/src/adapters/overture/adapter.js.map +1 -1
  39. package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
  40. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  41. package/out/src/adapters/state-hi-schools/adapter.js +7 -6
  42. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  43. package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
  44. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  45. package/out/src/adapters/state-ia-contractors/adapter.js +12 -7
  46. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  47. package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
  48. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  49. package/out/src/adapters/state-ny-notaries/adapter.js +12 -18
  50. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  51. package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
  52. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  53. package/out/src/adapters/state-tx-notaries/adapter.js +12 -7
  54. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  55. package/out/src/adapters/synth-po-box/adapter.d.ts +3 -3
  56. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  57. package/out/src/adapters/synth-po-box/adapter.js +3 -3
  58. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  59. package/out/src/adapters/tiger/adapter.d.ts +1 -1
  60. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  61. package/out/src/adapters/tiger/adapter.js +2 -2
  62. package/out/src/adapters/tiger/adapter.js.map +1 -1
  63. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
  64. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  65. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +7 -6
  66. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  67. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
  68. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  69. package/out/src/adapters/usgov-imls-pls/adapter.js +11 -7
  70. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  71. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
  72. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  73. package/out/src/adapters/usgov-irs-bmf/adapter.js +6 -5
  74. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  75. package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
  76. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  77. package/out/src/adapters/usgov-nad/adapter.js +8 -8
  78. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  79. package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
  80. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  81. package/out/src/adapters/usgov-nppes/adapter.js +10 -10
  82. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  83. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
  84. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  85. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +7 -6
  86. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  87. package/out/src/{adapter.d.ts → adapters/utils/index.d.ts} +8 -2
  88. package/out/src/adapters/utils/index.d.ts.map +1 -0
  89. package/out/src/{adapter.js → adapters/utils/index.js} +11 -3
  90. package/out/src/adapters/utils/index.js.map +1 -0
  91. package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
  92. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  93. package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
  94. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  95. package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
  96. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  97. package/out/src/adapters/wof-admin-json/adapter.js +11 -35
  98. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  99. package/out/src/adapters/wof-json-rows.d.ts +32 -0
  100. package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
  101. package/out/src/adapters/wof-json-rows.js +45 -0
  102. package/out/src/adapters/wof-json-rows.js.map +1 -0
  103. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
  104. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  105. package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
  106. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  107. package/out/src/build.d.ts +4 -4
  108. package/out/src/build.d.ts.map +1 -1
  109. package/out/src/build.js +7 -7
  110. package/out/src/build.js.map +1 -1
  111. package/out/src/index.d.ts +3 -17
  112. package/out/src/index.d.ts.map +1 -1
  113. package/out/src/index.js +3 -17
  114. package/out/src/index.js.map +1 -1
  115. package/out/src/name-prone-us-suffixes.d.ts +12 -0
  116. package/out/src/name-prone-us-suffixes.d.ts.map +1 -0
  117. package/out/src/name-prone-us-suffixes.js +12 -0
  118. package/out/src/name-prone-us-suffixes.js.map +1 -0
  119. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  120. package/out/src/parquet-wrapper/reader.js +5 -0
  121. package/out/src/parquet-wrapper/reader.js.map +1 -1
  122. package/out/src/runner.d.ts +2 -2
  123. package/out/src/runner.d.ts.map +1 -1
  124. package/out/src/runner.js +1 -1
  125. package/out/src/runner.js.map +1 -1
  126. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  127. package/out/src/shard-recipes/anchor-absorption.js +4 -5
  128. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  129. package/out/src/shard-recipes/bare-country.d.ts +25 -0
  130. package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
  131. package/out/src/shard-recipes/bare-country.js +87 -0
  132. package/out/src/shard-recipes/bare-country.js.map +1 -0
  133. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  134. package/out/src/shard-recipes/boundary-stress.js +2 -2
  135. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  136. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  137. package/out/src/shard-recipes/country-balanced.js +14 -21
  138. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  139. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
  140. package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
  141. package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
  142. package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
  143. package/out/src/shard-recipes/fr-admin-split.js +4 -3
  144. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  145. package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
  146. package/out/src/shard-recipes/fr-bare-street.js +72 -11
  147. package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
  148. package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
  149. package/out/src/shard-recipes/fr-lieudit.js +4 -3
  150. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  151. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  152. package/out/src/shard-recipes/fr-order.js +11 -20
  153. package/out/src/shard-recipes/fr-order.js.map +1 -1
  154. package/out/src/shard-recipes/german.d.ts.map +1 -1
  155. package/out/src/shard-recipes/german.js +8 -14
  156. package/out/src/shard-recipes/german.js.map +1 -1
  157. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  158. package/out/src/shard-recipes/house-venue.js +1 -1
  159. package/out/src/shard-recipes/house-venue.js.map +1 -1
  160. package/out/src/shard-recipes/index.d.ts.map +1 -1
  161. package/out/src/shard-recipes/index.js +8 -1
  162. package/out/src/shard-recipes/index.js.map +1 -1
  163. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  164. package/out/src/shard-recipes/intersection.js +7 -13
  165. package/out/src/shard-recipes/intersection.js.map +1 -1
  166. package/out/src/shard-recipes/locale.d.ts +5 -5
  167. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  168. package/out/src/shard-recipes/locale.js +13 -20
  169. package/out/src/shard-recipes/locale.js.map +1 -1
  170. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  171. package/out/src/shard-recipes/no-street.js +1 -1
  172. package/out/src/shard-recipes/no-street.js.map +1 -1
  173. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  174. package/out/src/shard-recipes/po-box-cedex.js +60 -43
  175. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  176. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  177. package/out/src/shard-recipes/po-box.js +1 -1
  178. package/out/src/shard-recipes/po-box.js.map +1 -1
  179. package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
  180. package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
  181. package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
  182. package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
  183. package/out/src/shard-recipes/scaffold.d.ts +51 -23
  184. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  185. package/out/src/shard-recipes/scaffold.js +59 -23
  186. package/out/src/shard-recipes/scaffold.js.map +1 -1
  187. package/out/src/shard-recipes/street-affix.d.ts +54 -0
  188. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  189. package/out/src/shard-recipes/street-affix.js +203 -35
  190. package/out/src/shard-recipes/street-affix.js.map +1 -1
  191. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  192. package/out/src/shard-recipes/street-bare.js +3 -3
  193. package/out/src/shard-recipes/street-bare.js.map +1 -1
  194. package/out/src/shard-recipes/street.d.ts.map +1 -1
  195. package/out/src/shard-recipes/street.js +2 -2
  196. package/out/src/shard-recipes/street.js.map +1 -1
  197. package/out/src/shard-recipes/sub-venue-sources.d.ts +3 -6
  198. package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
  199. package/out/src/shard-recipes/sub-venue-sources.js +4 -9
  200. package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
  201. package/out/src/shard-recipes/sub-venue.d.ts +2 -2
  202. package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
  203. package/out/src/shard-recipes/sub-venue.js +7 -6
  204. package/out/src/shard-recipes/sub-venue.js.map +1 -1
  205. package/out/src/shard-recipes/trailing-region.d.ts +74 -0
  206. package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
  207. package/out/src/shard-recipes/trailing-region.js +157 -0
  208. package/out/src/shard-recipes/trailing-region.js.map +1 -0
  209. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  210. package/out/src/shard-recipes/unit.js +6 -12
  211. package/out/src/shard-recipes/unit.js.map +1 -1
  212. package/out/src/{synthesize-anchor-absorption.d.ts → synthesizers/anchor-absorption.d.ts} +1 -1
  213. package/out/src/synthesizers/anchor-absorption.d.ts.map +1 -0
  214. package/out/src/{synthesize-anchor-absorption.js → synthesizers/anchor-absorption.js} +1 -1
  215. package/out/src/synthesizers/anchor-absorption.js.map +1 -0
  216. package/out/src/{synthesize-boundary-stress.d.ts → synthesizers/boundary-stress.d.ts} +3 -3
  217. package/out/src/synthesizers/boundary-stress.d.ts.map +1 -0
  218. package/out/src/{synthesize-boundary-stress.js → synthesizers/boundary-stress.js} +2 -2
  219. package/out/src/synthesizers/boundary-stress.js.map +1 -0
  220. package/out/src/{synthesize-german.d.ts → synthesizers/german.d.ts} +2 -2
  221. package/out/src/synthesizers/german.d.ts.map +1 -0
  222. package/out/src/{synthesize-german.js → synthesizers/german.js} +2 -2
  223. package/out/src/synthesizers/german.js.map +1 -0
  224. package/out/src/{synthesize-house-venue.d.ts → synthesizers/house-venue.d.ts} +4 -4
  225. package/out/src/synthesizers/house-venue.d.ts.map +1 -0
  226. package/out/src/{synthesize-house-venue.js → synthesizers/house-venue.js} +20 -6
  227. package/out/src/synthesizers/house-venue.js.map +1 -0
  228. package/out/src/{synthesize-intersection.d.ts → synthesizers/intersection.d.ts} +2 -2
  229. package/out/src/synthesizers/intersection.d.ts.map +1 -0
  230. package/out/src/{synthesize-intersection.js → synthesizers/intersection.js} +1 -1
  231. package/out/src/synthesizers/intersection.js.map +1 -0
  232. package/out/src/{synthesize-no-street.d.ts → synthesizers/no-street.d.ts} +3 -3
  233. package/out/src/synthesizers/no-street.d.ts.map +1 -0
  234. package/out/src/{synthesize-no-street.js → synthesizers/no-street.js} +2 -2
  235. package/out/src/synthesizers/no-street.js.map +1 -0
  236. package/out/src/{synthesize-po-box.d.ts → synthesizers/po-box.d.ts} +2 -2
  237. package/out/src/synthesizers/po-box.d.ts.map +1 -0
  238. package/out/src/{synthesize-po-box.js → synthesizers/po-box.js} +1 -1
  239. package/out/src/synthesizers/po-box.js.map +1 -0
  240. package/out/src/{synthesize-street.d.ts → synthesizers/street.d.ts} +2 -2
  241. package/out/src/synthesizers/street.d.ts.map +1 -0
  242. package/out/src/{synthesize-street.js → synthesizers/street.js} +4 -3
  243. package/out/src/synthesizers/street.js.map +1 -0
  244. package/out/src/{synthesize.d.ts → synthesizers/utils.d.ts} +3 -3
  245. package/out/src/synthesizers/utils.d.ts.map +1 -0
  246. package/out/src/{synthesize.js → synthesizers/utils.js} +5 -4
  247. package/out/src/synthesizers/utils.js.map +1 -0
  248. package/out/src/tools/align-shard.d.ts.map +1 -1
  249. package/out/src/tools/align-shard.js +2 -2
  250. package/out/src/tools/align-shard.js.map +1 -1
  251. package/out/src/tools/fetch/ban.d.ts.map +1 -1
  252. package/out/src/tools/fetch/ban.js +2 -18
  253. package/out/src/tools/fetch/ban.js.map +1 -1
  254. package/out/src/tools/fetch/download.d.ts +12 -3
  255. package/out/src/tools/fetch/download.d.ts.map +1 -1
  256. package/out/src/tools/fetch/download.js +20 -4
  257. package/out/src/tools/fetch/download.js.map +1 -1
  258. package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
  259. package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
  260. package/out/src/tools/fetch/geonames-dump.js +178 -0
  261. package/out/src/tools/fetch/geonames-dump.js.map +1 -0
  262. package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
  263. package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
  264. package/out/src/tools/fetch/geonames-postal.js +125 -0
  265. package/out/src/tools/fetch/geonames-postal.js.map +1 -0
  266. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  267. package/out/src/tools/fetch/imls-pls.js +5 -17
  268. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  269. package/out/src/tools/fetch/index.d.ts +11 -0
  270. package/out/src/tools/fetch/index.d.ts.map +1 -1
  271. package/out/src/tools/fetch/index.js +11 -0
  272. package/out/src/tools/fetch/index.js.map +1 -1
  273. package/out/src/tools/fetch/nad.d.ts.map +1 -1
  274. package/out/src/tools/fetch/nad.js +20 -11
  275. package/out/src/tools/fetch/nad.js.map +1 -1
  276. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  277. package/out/src/tools/fetch/nppes.js +17 -22
  278. package/out/src/tools/fetch/nppes.js.map +1 -1
  279. package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
  280. package/out/src/tools/fetch/openaddresses.js +18 -17
  281. package/out/src/tools/fetch/openaddresses.js.map +1 -1
  282. package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
  283. package/out/src/tools/fetch/ourairports.js +7 -2
  284. package/out/src/tools/fetch/ourairports.js.map +1 -1
  285. package/out/src/tools/fetch/ppd.d.ts +0 -4
  286. package/out/src/tools/fetch/ppd.d.ts.map +1 -1
  287. package/out/src/tools/fetch/ppd.js +1 -1
  288. package/out/src/tools/fetch/ppd.js.map +1 -1
  289. package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
  290. package/out/src/tools/fetch/state-hi-schools.js +3 -19
  291. package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
  292. package/out/src/tools/fetch/state-sources.d.ts +4 -0
  293. package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
  294. package/out/src/tools/fetch/state-sources.js +1 -5
  295. package/out/src/tools/fetch/state-sources.js.map +1 -1
  296. package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
  297. package/out/src/tools/fetch/tiger-full.js +15 -21
  298. package/out/src/tools/fetch/tiger-full.js.map +1 -1
  299. package/out/src/tools/golden-expand.d.ts.map +1 -1
  300. package/out/src/tools/golden-expand.js +34 -25
  301. package/out/src/tools/golden-expand.js.map +1 -1
  302. package/out/src/tools/golden-relabel-street.d.ts.map +1 -1
  303. package/out/src/tools/golden-relabel-street.js +4 -77
  304. package/out/src/tools/golden-relabel-street.js.map +1 -1
  305. package/out/src/tools/index.d.ts +2 -2
  306. package/out/src/tools/index.d.ts.map +1 -1
  307. package/out/src/tools/index.js +2 -2
  308. package/out/src/tools/index.js.map +1 -1
  309. package/out/src/tools/ingest-csv.d.ts.map +1 -1
  310. package/out/src/tools/ingest-csv.js +0 -11
  311. package/out/src/tools/ingest-csv.js.map +1 -1
  312. package/out/src/tools/overlay-manifest.d.ts +4 -0
  313. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  314. package/out/src/tools/overlay-manifest.js +16 -6
  315. package/out/src/tools/overlay-manifest.js.map +1 -1
  316. package/out/src/tools/postcode-triples.d.ts +175 -0
  317. package/out/src/tools/postcode-triples.d.ts.map +1 -0
  318. package/out/src/tools/postcode-triples.js +304 -0
  319. package/out/src/tools/postcode-triples.js.map +1 -0
  320. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  321. package/out/src/tools/shard-kryptonite.js +1 -3
  322. package/out/src/tools/shard-kryptonite.js.map +1 -1
  323. package/out/src/tools/shard-translit.d.ts.map +1 -1
  324. package/out/src/tools/shard-translit.js +4 -27
  325. package/out/src/tools/shard-translit.js.map +1 -1
  326. package/out/src/tools/sub-venue/harvest.d.ts +100 -0
  327. package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
  328. package/out/src/tools/sub-venue/harvest.js +168 -0
  329. package/out/src/tools/sub-venue/harvest.js.map +1 -0
  330. package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
  331. package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
  332. package/out/src/tools/sub-venue/head-nouns.js +210 -0
  333. package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
  334. package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
  335. package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
  336. package/out/src/tools/sub-venue/surfaces.js +77 -0
  337. package/out/src/tools/sub-venue/surfaces.js.map +1 -0
  338. package/out/src/tools/sub-venue/table.d.ts +229 -0
  339. package/out/src/tools/sub-venue/table.d.ts.map +1 -0
  340. package/out/src/tools/sub-venue/table.js +108 -0
  341. package/out/src/tools/sub-venue/table.js.map +1 -0
  342. package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
  343. package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
  344. package/out/src/tools/sub-venue/wikidata.js +62 -0
  345. package/out/src/tools/sub-venue/wikidata.js.map +1 -0
  346. package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
  347. package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
  348. package/out/src/tools/sub-venue-lexicon.js +30 -565
  349. package/out/src/tools/sub-venue-lexicon.js.map +1 -1
  350. package/out/src/{align.d.ts → utils/align.d.ts} +1 -1
  351. package/out/src/utils/align.d.ts.map +1 -0
  352. package/out/src/utils/align.js.map +1 -0
  353. package/out/src/utils/golden.d.ts.map +1 -0
  354. package/out/src/{golden.js → utils/golden.js} +2 -2
  355. package/out/src/utils/golden.js.map +1 -0
  356. package/out/src/utils/index.d.ts +14 -0
  357. package/out/src/utils/index.d.ts.map +1 -0
  358. package/out/src/utils/index.js +14 -0
  359. package/out/src/utils/index.js.map +1 -0
  360. package/out/src/utils/license.d.ts.map +1 -0
  361. package/out/src/{license.js → utils/license.js} +1 -1
  362. package/out/src/utils/license.js.map +1 -0
  363. package/out/src/{parquet.d.ts → utils/parquet.d.ts} +24 -5
  364. package/out/src/utils/parquet.d.ts.map +1 -0
  365. package/out/src/{parquet.js → utils/parquet.js} +6 -3
  366. package/out/src/utils/parquet.js.map +1 -0
  367. package/out/src/{split.d.ts → utils/split.d.ts} +1 -1
  368. package/out/src/utils/split.d.ts.map +1 -0
  369. package/out/src/{split.js → utils/split.js} +2 -2
  370. package/out/src/utils/split.js.map +1 -0
  371. package/out/src/utils/tokenize.d.ts.map +1 -0
  372. package/out/src/utils/tokenize.js.map +1 -0
  373. package/out/src/utils/wof-json.d.ts.map +1 -0
  374. package/out/src/utils/wof-json.js.map +1 -0
  375. package/package.json +287 -21
  376. package/src/adapters/ban/adapter.ts +7 -5
  377. package/src/adapters/fcc-bdc/adapter.ts +5 -4
  378. package/src/adapters/geonames/adapter.ts +3 -3
  379. package/src/adapters/geonames-postal/adapter.ts +4 -4
  380. package/src/adapters/gnaf/adapter.ts +3 -3
  381. package/src/adapters/gnaf/assemble.ts +1 -1
  382. package/src/adapters/index.ts +3 -2
  383. package/src/adapters/openaddresses/adapter.ts +4 -4
  384. package/src/adapters/overture/adapter.ts +3 -3
  385. package/src/adapters/state-hi-schools/adapter.ts +8 -7
  386. package/src/adapters/state-ia-contractors/adapter.ts +13 -8
  387. package/src/adapters/state-ny-notaries/adapter.ts +13 -32
  388. package/src/adapters/state-tx-notaries/adapter.ts +13 -8
  389. package/src/adapters/synth-po-box/adapter.ts +4 -4
  390. package/src/adapters/tiger/adapter.ts +5 -3
  391. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -7
  392. package/src/adapters/usgov-imls-pls/adapter.ts +12 -8
  393. package/src/adapters/usgov-irs-bmf/adapter.ts +7 -6
  394. package/src/adapters/usgov-nad/adapter.ts +9 -9
  395. package/src/adapters/usgov-nppes/adapter.ts +13 -12
  396. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -7
  397. package/src/{adapter.ts → adapters/utils/index.ts} +15 -3
  398. package/src/adapters/wof-admin-jp/adapter.ts +9 -12
  399. package/src/adapters/wof-admin-json/adapter.ts +15 -54
  400. package/src/adapters/wof-json-rows.ts +79 -0
  401. package/src/adapters/wof-postalcode-json/adapter.ts +16 -50
  402. package/src/build.ts +8 -8
  403. package/src/index.ts +3 -17
  404. package/src/name-prone-us-suffixes.ts +12 -0
  405. package/src/parquet-wrapper/reader.ts +9 -0
  406. package/src/runner.ts +2 -2
  407. package/src/shard-recipes/anchor-absorption.ts +5 -5
  408. package/src/shard-recipes/bare-country.ts +95 -0
  409. package/src/shard-recipes/boundary-stress.ts +3 -2
  410. package/src/shard-recipes/country-balanced.ts +16 -27
  411. package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
  412. package/src/shard-recipes/fr-admin-split.ts +6 -4
  413. package/src/shard-recipes/fr-bare-street.ts +85 -10
  414. package/src/shard-recipes/fr-lieudit.ts +6 -4
  415. package/src/shard-recipes/fr-order.ts +13 -26
  416. package/src/shard-recipes/german.ts +9 -18
  417. package/src/shard-recipes/house-venue.ts +2 -1
  418. package/src/shard-recipes/index.ts +8 -1
  419. package/src/shard-recipes/intersection.ts +9 -17
  420. package/src/shard-recipes/locale.ts +16 -24
  421. package/src/shard-recipes/no-street.ts +2 -1
  422. package/src/shard-recipes/po-box-cedex.ts +76 -58
  423. package/src/shard-recipes/po-box.ts +2 -1
  424. package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
  425. package/src/shard-recipes/scaffold.ts +99 -24
  426. package/src/shard-recipes/street-affix.ts +284 -45
  427. package/src/shard-recipes/street-bare.ts +4 -3
  428. package/src/shard-recipes/street.ts +3 -2
  429. package/src/shard-recipes/sub-venue-sources.ts +13 -23
  430. package/src/shard-recipes/sub-venue.ts +11 -9
  431. package/src/shard-recipes/trailing-region.ts +183 -0
  432. package/src/shard-recipes/unit.ts +8 -17
  433. package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts} +2 -2
  434. package/src/{synthesize-german.ts → synthesizers/german.ts} +3 -2
  435. package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts} +20 -6
  436. package/src/{synthesize-intersection.ts → synthesizers/intersection.ts} +1 -1
  437. package/src/{synthesize-no-street.ts → synthesizers/no-street.ts} +2 -2
  438. package/src/{synthesize-po-box.ts → synthesizers/po-box.ts} +1 -1
  439. package/src/{synthesize-street.ts → synthesizers/street.ts} +5 -3
  440. package/src/{synthesize.ts → synthesizers/utils.ts} +6 -4
  441. package/src/tools/align-shard.ts +3 -2
  442. package/src/tools/fetch/ban.ts +2 -22
  443. package/src/tools/fetch/download.ts +22 -4
  444. package/src/tools/fetch/geonames-dump.ts +267 -0
  445. package/src/tools/fetch/geonames-postal.ts +180 -0
  446. package/src/tools/fetch/imls-pls.ts +6 -20
  447. package/src/tools/fetch/index.ts +11 -0
  448. package/src/tools/fetch/nad.ts +21 -10
  449. package/src/tools/fetch/nppes.ts +17 -23
  450. package/src/tools/fetch/openaddresses.ts +18 -22
  451. package/src/tools/fetch/ourairports.ts +7 -2
  452. package/src/tools/fetch/ppd.ts +1 -1
  453. package/src/tools/fetch/state-hi-schools.ts +3 -23
  454. package/src/tools/fetch/state-sources.ts +1 -1
  455. package/src/tools/fetch/tiger-full.ts +17 -24
  456. package/src/tools/golden-expand.ts +44 -33
  457. package/src/tools/golden-relabel-street.ts +5 -68
  458. package/src/tools/index.ts +2 -2
  459. package/src/tools/ingest-csv.ts +0 -13
  460. package/src/tools/overlay-manifest.ts +20 -8
  461. package/src/tools/postcode-triples.ts +375 -0
  462. package/src/tools/shard-kryptonite.ts +4 -5
  463. package/src/tools/shard-translit.ts +16 -35
  464. package/src/tools/sub-venue/harvest.ts +235 -0
  465. package/src/tools/sub-venue/head-nouns.ts +242 -0
  466. package/src/tools/sub-venue/surfaces.ts +95 -0
  467. package/src/tools/sub-venue/table.ts +281 -0
  468. package/src/tools/sub-venue/wikidata.ts +85 -0
  469. package/src/tools/sub-venue-lexicon.ts +67 -886
  470. package/src/{align.ts → utils/align.ts} +2 -1
  471. package/src/{golden.ts → utils/golden.ts} +2 -3
  472. package/src/utils/index.ts +14 -0
  473. package/src/{license.ts → utils/license.ts} +1 -1
  474. package/src/{parquet.ts → utils/parquet.ts} +22 -9
  475. package/src/{split.ts → utils/split.ts} +3 -3
  476. package/out/src/adapter.d.ts.map +0 -1
  477. package/out/src/adapter.js.map +0 -1
  478. package/out/src/align.d.ts.map +0 -1
  479. package/out/src/align.js.map +0 -1
  480. package/out/src/format.d.ts +0 -14
  481. package/out/src/format.d.ts.map +0 -1
  482. package/out/src/format.js +0 -14
  483. package/out/src/format.js.map +0 -1
  484. package/out/src/golden.d.ts.map +0 -1
  485. package/out/src/golden.js.map +0 -1
  486. package/out/src/license.d.ts.map +0 -1
  487. package/out/src/license.js.map +0 -1
  488. package/out/src/parquet.d.ts.map +0 -1
  489. package/out/src/parquet.js.map +0 -1
  490. package/out/src/split.d.ts.map +0 -1
  491. package/out/src/split.js.map +0 -1
  492. package/out/src/synthesize-anchor-absorption.d.ts.map +0 -1
  493. package/out/src/synthesize-anchor-absorption.js.map +0 -1
  494. package/out/src/synthesize-boundary-stress.d.ts.map +0 -1
  495. package/out/src/synthesize-boundary-stress.js.map +0 -1
  496. package/out/src/synthesize-german.d.ts.map +0 -1
  497. package/out/src/synthesize-german.js.map +0 -1
  498. package/out/src/synthesize-house-venue.d.ts.map +0 -1
  499. package/out/src/synthesize-house-venue.js.map +0 -1
  500. package/out/src/synthesize-intersection.d.ts.map +0 -1
  501. package/out/src/synthesize-intersection.js.map +0 -1
  502. package/out/src/synthesize-no-street.d.ts.map +0 -1
  503. package/out/src/synthesize-no-street.js.map +0 -1
  504. package/out/src/synthesize-po-box.d.ts.map +0 -1
  505. package/out/src/synthesize-po-box.js.map +0 -1
  506. package/out/src/synthesize-street.d.ts.map +0 -1
  507. package/out/src/synthesize-street.js.map +0 -1
  508. package/out/src/synthesize.d.ts.map +0 -1
  509. package/out/src/synthesize.js.map +0 -1
  510. package/out/src/tokenize.d.ts.map +0 -1
  511. package/out/src/tokenize.js.map +0 -1
  512. package/out/src/wof-json.d.ts.map +0 -1
  513. package/out/src/wof-json.js.map +0 -1
  514. package/src/format.ts +0 -14
  515. /package/out/src/{align.js → utils/align.js} +0 -0
  516. /package/out/src/{golden.d.ts → utils/golden.d.ts} +0 -0
  517. /package/out/src/{license.d.ts → utils/license.d.ts} +0 -0
  518. /package/out/src/{tokenize.d.ts → utils/tokenize.d.ts} +0 -0
  519. /package/out/src/{tokenize.js → utils/tokenize.js} +0 -0
  520. /package/out/src/{wof-json.d.ts → utils/wof-json.d.ts} +0 -0
  521. /package/out/src/{wof-json.js → utils/wof-json.js} +0 -0
  522. /package/src/{synthesize-anchor-absorption.ts → synthesizers/anchor-absorption.ts} +0 -0
  523. /package/src/{tokenize.ts → utils/tokenize.ts} +0 -0
  524. /package/src/{wof-json.ts → utils/wof-json.ts} +0 -0
@@ -1 +1 @@
1
- {"version":3,"file":"align-shard.js","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAE3C;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAQ7C,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAuB;IAChE,+DAA+D;IAC/D,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACtE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,eAAe,CAAC,SAAS,CAAiC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrG,4FAA4F;QAC5F,SAAS,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAA;QAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAElC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;YAElD,OAAO,EAAE,CAAA;QACV,CAAC;aAAM,CAAC;YACP,WAAW,EAAE,CAAA;YACb,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAA;YACxC,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,KAAK,CACZ,0BAA0B,OAAO,aAAa,WAAW,kBAAkB,IAAI,CAAC,MAAM,IAAI;QACzF,yBAAyB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAC7D,CAAA;AACF,CAAC"}
1
+ {"version":3,"file":"align-shard.js","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAQjC,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAuB;IAChE,+DAA+D;IAC/D,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACtE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,eAAe,CAAC,SAAS,CAAiC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrG,4FAA4F;QAC5F,SAAS,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAA;QAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAElC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;YAElD,OAAO,EAAE,CAAA;QACV,CAAC;aAAM,CAAC;YACP,WAAW,EAAE,CAAA;YACb,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAA;YACxC,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,KAAK,CACZ,0BAA0B,OAAO,aAAa,WAAW,kBAAkB,IAAI,CAAC,MAAM,IAAI;QACzF,yBAAyB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAC7D,CAAA;AACF,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"ban.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAUH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAuHnE,MAAM,WAAW,gBAAgB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,MAAM,eAAe,GAAG,gBAAgB,CAAA;AAsB9C,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAmI/G"}
1
+ {"version":3,"file":"ban.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAWH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAsHnE,MAAM,WAAW,gBAAgB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,MAAM,eAAe,GAAG,gBAAgB,CAAA;AAE9C,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAmI/G"}
@@ -22,13 +22,13 @@ import { readFile, unlink, writeFile } from "node:fs/promises";
22
22
  import { join } from "node:path";
23
23
  import { setTimeout as sleep } from "node:timers/promises";
24
24
  import { gunzipSync } from "node:zlib";
25
+ import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils";
25
26
  import { sha256File } from "@mailwoman/core/utils";
26
27
  import { downloadToFile, loadManifestEntries, writeManifest } from "./download.js";
27
28
  /**
28
29
  * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
29
30
  * than data.
30
31
  */
31
- const BYTES_PER_KIB = 1024;
32
32
  const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv";
33
33
  /**
34
34
  * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
@@ -137,22 +137,6 @@ const DEPT_CODES = [
137
137
  "974",
138
138
  "976",
139
139
  ];
140
- /**
141
- * Mimic `numfmt --to=iec` for a friendly byte-size log line.
142
- */
143
- function iec(bytes) {
144
- if (bytes < BYTES_PER_KIB)
145
- return String(bytes);
146
- const units = ["K", "M", "G", "T", "P"];
147
- let value = bytes / 1024;
148
- let i = 0;
149
- while (value >= BYTES_PER_KIB && i < units.length - 1) {
150
- value /= 1024;
151
- i++;
152
- }
153
- const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString();
154
- return `${rounded}${units[i] ?? ""}`;
155
- }
156
140
  export async function fetchBan(options, report) {
157
141
  const banDir = join(options.outRoot, "ban");
158
142
  const manifestPath = join(banDir, "MANIFEST.json");
@@ -234,7 +218,7 @@ export async function fetchBan(options, report) {
234
218
  sha256: sha,
235
219
  bytes,
236
220
  });
237
- report?.(` ✓ ${iec(bytes)} sha256=${sha}`);
221
+ report?.(` ✓ ${ByteFormatter.formatIEC(bytes)} sha256=${sha}`);
238
222
  fetched++;
239
223
  // Be a polite citizen — short pause between requests.
240
224
  await sleep(200);
@@ -1 +1 @@
1
- {"version":3,"file":"ban.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAElF;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B,MAAM,QAAQ,GAAG,2DAA2D,CAAA;AAE5E;;;GAGG;AACH,MAAM,UAAU,GAAG;IAClB,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;CACL,CAAA;AAaD;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa;IACzB,IAAI,KAAK,GAAG,aAAa;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAA;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IACvC,IAAI,KAAK,GAAG,KAAK,GAAG,IAAI,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,KAAK,IAAI,aAAa,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvD,KAAK,IAAI,IAAI,CAAA;QAEb,CAAC,EAAE,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAA;IAE5E,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAA;AACrC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAClD,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtC,2FAA2F;IAC3F,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAmB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAErG,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,YAAY,IAAI,MAAM,CAAA;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QACtC,MAAM,GAAG,GAAG,GAAG,QAAQ,aAAa,IAAI,SAAS,CAAA;QAEjD,MAAM,EAAE,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAE5B,sEAAsE;QACtE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;YAE7C,IAAI,WAAW,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;gBAChD,MAAM,EAAE,CAAC,8CAA8C,CAAC,CAAA;gBAExD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,EAAE,CAAC,iEAAiE,CAAC,CAAA;YAC3E,MAAM,MAAM,CAAC,OAAO,CAAC,CAAA;QACtB,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC;gBACpB,GAAG;gBACH,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,wBAAwB,GAAG,KAAM,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;YAErE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,2CAA2C;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAA;QAEpC,IAAI,MAAM,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,2BAA2B,MAAM,qCAAqC,CAAC,CAAA;YAChF,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;YAEpB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACJ,MAAM,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QAC7D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,0BAA2B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC9D,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;YAEpB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;QAEpB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,MAAM,EAAE,CAAC,sCAAsC,OAAO,EAAE,CAAC,CAAA;YAEzD,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;QACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;QAErC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;YACjB,SAAS,EAAE,IAAI;YACf,QAAQ;YACR,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAA;QAE5C,OAAO,EAAE,CAAA;QAET,sDAAsD;QACtD,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;IACjB,CAAC;IAED,uFAAuF;IACvF,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtD,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAClE,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,SAAS,YAAY,SAAS,MAAM,CAAC,MAAM,WAAW,CAAC,CAAA;IAEhE,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,aAAa,OAAO,kCAAkC,CAAC,CAAA;IAChE,MAAM,EAAE,CAAC,aAAa,MAAM,EAAE,CAAC,CAAA;IAE/B,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,iBAAiB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACnD,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACjD,CAAC"}
1
+ {"version":3,"file":"ban.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAElF;;;GAGG;AAEH,MAAM,QAAQ,GAAG,2DAA2D,CAAA;AAE5E;;;GAGG;AACH,MAAM,UAAU,GAAG;IAClB,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;CACL,CAAA;AAaD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAClD,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtC,2FAA2F;IAC3F,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAmB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAErG,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,YAAY,IAAI,MAAM,CAAA;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QACtC,MAAM,GAAG,GAAG,GAAG,QAAQ,aAAa,IAAI,SAAS,CAAA;QAEjD,MAAM,EAAE,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAE5B,sEAAsE;QACtE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;YAE7C,IAAI,WAAW,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;gBAChD,MAAM,EAAE,CAAC,8CAA8C,CAAC,CAAA;gBAExD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,EAAE,CAAC,iEAAiE,CAAC,CAAA;YAC3E,MAAM,MAAM,CAAC,OAAO,CAAC,CAAA;QACtB,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC;gBACpB,GAAG;gBACH,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,wBAAwB,GAAG,KAAM,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;YAErE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,2CAA2C;QAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAA;QAEpC,IAAI,MAAM,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,2BAA2B,MAAM,qCAAqC,CAAC,CAAA;YAChF,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;YAEpB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACJ,MAAM,SAAS,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QAC7D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,0BAA2B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC9D,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;YAEpB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,CAAC,MAAM,CAAC,CAAA;QAEpB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,MAAM,EAAE,CAAC,sCAAsC,OAAO,EAAE,CAAC,CAAA;YAEzD,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;QACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;QAErC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;YACjB,SAAS,EAAE,IAAI;YACf,QAAQ;YACR,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAA;QAEhE,OAAO,EAAE,CAAA;QAET,sDAAsD;QACtD,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;IACjB,CAAC;IAED,uFAAuF;IACvF,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtD,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAClE,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,SAAS,YAAY,SAAS,MAAM,CAAC,MAAM,WAAW,CAAC,CAAA;IAEhE,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,aAAa,OAAO,kCAAkC,CAAC,CAAA;IAChE,MAAM,EAAE,CAAC,aAAa,MAAM,EAAE,CAAC,CAAA;IAE/B,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,iBAAiB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACnD,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACjD,CAAC"}
@@ -3,9 +3,7 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Download + manifest plumbing for the `mailwoman corpus fetch <source>` family — one download-with-retry and
7
- * one MANIFEST.json idiom instead of the per-script clones the 2026-07-09 dedupe survey counted
8
- * (6× `downloadToFile`, 2× `isTransientStatus`, 9× manifest writes).
6
+ * Download and manifest utilities for `mailwoman corpus fetch <source>`.
9
7
  */
10
8
  /**
11
9
  * The option base every `mailwoman corpus fetch <source>` module extends.
@@ -28,6 +26,17 @@ export interface FetchSummary {
28
26
  /**
29
27
  * A status worth retrying: rate limiting or a server-side failure.
30
28
  */
29
+ /**
30
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
31
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
32
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is
33
+ * 500. Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns
34
+ * a CI run and vanishes locally.
35
+ */
36
+ export declare class HTTPStatusError extends Error {
37
+ readonly status: number;
38
+ constructor(status: number, message: string);
39
+ }
31
40
  export declare function isTransientStatus(status: number): boolean;
32
41
  export interface DownloadOptions {
33
42
  url: string;
@@ -1 +1 @@
1
- {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAuBH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA0CzF;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAS7G;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF"}
1
+ {"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAuBH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH;;;;;;GAMG;AACH,qBAAa,eAAgB,SAAQ,KAAK;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;gBAEX,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK3C;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CA6CzF;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAS7G;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF"}
@@ -3,9 +3,7 @@
3
3
  * @license AGPL-3.0
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
- * Download + manifest plumbing for the `mailwoman corpus fetch <source>` family — one download-with-retry and
7
- * one MANIFEST.json idiom instead of the per-script clones the 2026-07-09 dedupe survey counted
8
- * (6× `downloadToFile`, 2× `isTransientStatus`, 9× manifest writes).
6
+ * Download and manifest utilities for `mailwoman corpus fetch <source>`.
9
7
  */
10
8
  import { existsSync } from "node:fs";
11
9
  import { readFile, writeFile } from "node:fs/promises";
@@ -26,6 +24,21 @@ const HTTP_SERVER_ERROR_MAX = 599;
26
24
  /**
27
25
  * A status worth retrying: rate limiting or a server-side failure.
28
26
  */
27
+ /**
28
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
29
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
30
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is
31
+ * 500. Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns
32
+ * a CI run and vanishes locally.
33
+ */
34
+ export class HTTPStatusError extends Error {
35
+ status;
36
+ constructor(status, message) {
37
+ super(message);
38
+ this.name = "HTTPStatusError";
39
+ this.status = status;
40
+ }
41
+ }
29
42
  export function isTransientStatus(status) {
30
43
  return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX);
31
44
  }
@@ -43,6 +56,9 @@ export async function downloadToFile(options) {
43
56
  }
44
57
  let res;
45
58
  try {
59
+ // Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
60
+ // `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
61
+ // non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
46
62
  res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) });
47
63
  }
48
64
  catch (error) {
@@ -51,7 +67,7 @@ export async function downloadToFile(options) {
51
67
  continue;
52
68
  }
53
69
  if (!res.ok) {
54
- const error = new Error(`HTTP ${res.status} ${res.statusText} — ${url}`);
70
+ const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`);
55
71
  if (!isTransientStatus(res.status))
56
72
  throw error;
57
73
  lastError = error;
@@ -1 +1 @@
1
- {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAExD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAsBjC;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACrG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAExE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,+FAA+F;IAC/F,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAE3D,OAAO,cAAc,CAAI,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AAChE,CAAC"}
1
+ {"version":3,"file":"download.js","sourceRoot":"","sources":["../../../../src/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAExD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAsBjC;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAChC,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,OAAe;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACrB,CAAC;CACD;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,GAAG,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IACrG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,qGAAqG;YACrG,kGAAkG;YAClG,+FAA+F;YAC/F,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAE9F,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAElC,+FAA+F;IAC/F,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAE3D,OAAO,cAAc,CAAI,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AAChE,CAAC"}
@@ -0,0 +1,87 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
7
+ * `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
8
+ * `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
9
+ * `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
10
+ *
11
+ * The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
12
+ * enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
13
+ * capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
14
+ * the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
15
+ * rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
16
+ * overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
17
+ * the basename) is reported as `wrong_format_present`, never counted as coverage.
18
+ */
19
+ import type { BaseFetchOptions, FetchSummary } from "./download.ts";
20
+ export interface FetchGeonamesDumpOptions extends BaseFetchOptions {
21
+ /**
22
+ * ISO alpha-2 codes, any casing. Absent → every country `countryInfo.txt` enumerates.
23
+ */
24
+ countries?: readonly string[];
25
+ /**
26
+ * Dump directory to read from. Defaults to GeoNames' own; exists so the 404 and coverage behaviour can be exercised
27
+ * against a local server.
28
+ */
29
+ baseURL?: string;
30
+ /**
31
+ * Refetch a dump whose `<CC>.txt` already exists. Default false — the tool fills gaps.
32
+ */
33
+ force?: boolean;
34
+ }
35
+ interface GeonamesDumpFileEntry {
36
+ country: string;
37
+ filename: string;
38
+ source_url: string;
39
+ sha256: string;
40
+ bytes: number;
41
+ }
42
+ export interface GeonamesDumpManifest {
43
+ source: string;
44
+ base_url: string;
45
+ license: string;
46
+ attribution: string;
47
+ downloaded_at: string;
48
+ files: GeonamesDumpFileEntry[];
49
+ /**
50
+ * `<CC>.txt` files already on disk and left alone — the hand-fetched population this tool extends.
51
+ */
52
+ skipped_present: string[];
53
+ /**
54
+ * Countries in the source catalog that the source's dump directory nonetheless 404s — a fact about the source,
55
+ * recorded so a later reader does not spend the fetch to rediscover it.
56
+ */
57
+ unavailable: string[];
58
+ /**
59
+ * Present `<CC>.txt` files that are NOT 19-column gazetteer dumps — GeoNames' postal exports share the same basename,
60
+ * and seven tier-1 postal files sat at these paths reading as "present" until the capitals build found them
61
+ * capital-less. Left in place (this tool never overwrites data it did not fetch); the fix is to move the file to its
62
+ * own home and rerun.
63
+ */
64
+ wrong_format_present: string[];
65
+ }
66
+ /**
67
+ * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
68
+ * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
69
+ */
70
+ export declare function looksLikeGazetteerDump(text: string): boolean;
71
+ /**
72
+ * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
73
+ * file's own commentary.
74
+ */
75
+ export declare function parseCountryInfo(text: string): Array<{
76
+ country: string;
77
+ capital: string;
78
+ }>;
79
+ /**
80
+ * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
81
+ * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
82
+ */
83
+ export declare function fetchGeonamesDumps(options: FetchGeonamesDumpOptions, report?: (line: string) => void): Promise<FetchSummary & {
84
+ skippedPresent: string[];
85
+ }>;
86
+ export {};
87
+ //# sourceMappingURL=geonames-dump.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"geonames-dump.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/geonames-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAgBnE,MAAM,WAAW,wBAAyB,SAAQ,gBAAgB;IACjE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,oBAAoB;IACpC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,qBAAqB,EAAE,CAAA;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB;;;OAGG;IACH,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB;;;;;OAKG;IACH,oBAAoB,EAAE,MAAM,EAAE,CAAA;CAC9B;AAQD;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAuB5D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB1F;AAqBD;;;GAGG;AACH,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,GAAG;IAAE,cAAc,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA2FtD"}
@@ -0,0 +1,178 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
7
+ * `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
8
+ * `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
9
+ * `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
10
+ *
11
+ * The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
12
+ * enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
13
+ * capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
14
+ * the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
15
+ * rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
16
+ * overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
17
+ * the basename) is reported as `wrong_format_present`, never counted as coverage.
18
+ */
19
+ import { existsSync, mkdirSync } from "node:fs";
20
+ import { open, readFile, rm } from "node:fs/promises";
21
+ import { join } from "node:path";
22
+ import { extractZipEntry } from "@mailwoman/core/fs/zip";
23
+ import { sha256File } from "@mailwoman/core/utils";
24
+ import { downloadToFile, HTTPStatusError, writeManifest } from "./download.js";
25
+ /**
26
+ * The one status that means "the source does not publish this country" rather than "the transfer failed".
27
+ */
28
+ const HTTP_NOT_FOUND = 404;
29
+ const SLUG = "geonames-dump";
30
+ /**
31
+ * GeoNames' gazetteer-dump directory — one zip per ISO alpha-2 code holding `<CC>.txt`, plus `countryInfo.txt` as a
32
+ * bare text file.
33
+ */
34
+ const BASE_URL = "https://download.geonames.org/export/dump";
35
+ /**
36
+ * Column count of a gazetteer dump row — the discriminator against GeoNames' 12-column postal exports, which share the
37
+ * `<CC>.txt` basename.
38
+ */
39
+ const GAZETTEER_DUMP_COLUMNS = 19;
40
+ /**
41
+ * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
42
+ * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
43
+ */
44
+ export function looksLikeGazetteerDump(text) {
45
+ let start = 0;
46
+ while (start < text.length) {
47
+ const end = text.indexOf("\n", start);
48
+ const line = end === -1 ? text.slice(start) : text.slice(start, end);
49
+ if (line.trim()) {
50
+ let tabs = 0;
51
+ for (let i = line.indexOf("\t"); i !== -1; i = line.indexOf("\t", i + 1)) {
52
+ tabs++;
53
+ }
54
+ return tabs === GAZETTEER_DUMP_COLUMNS - 1;
55
+ }
56
+ if (end === -1)
57
+ break;
58
+ start = end + 1;
59
+ }
60
+ return false;
61
+ }
62
+ /**
63
+ * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
64
+ * file's own commentary.
65
+ */
66
+ export function parseCountryInfo(text) {
67
+ const rows = [];
68
+ // oxlint-disable-next-line mailwoman/prefer-spliterator -- countryInfo.txt is ~40 KB, one row per country on Earth; it cannot grow past that
69
+ for (const line of text.split("\n")) {
70
+ if (line.startsWith("#") || !line.trim())
71
+ continue;
72
+ // oxlint-disable-next-line mailwoman/prefer-spliterator -- one bounded 19-column catalog row
73
+ const cols = line.split("\t");
74
+ const country = cols[0]?.trim().toUpperCase();
75
+ if (country?.length === 2) {
76
+ rows.push({ country, capital: cols[5]?.trim() ?? "" });
77
+ }
78
+ }
79
+ return rows;
80
+ }
81
+ /**
82
+ * Bytes read to classify a present file: enough to cover a first dump row whose alternate-names column runs long (they
83
+ * reach several KB), a fraction of the largest dumps (US.txt is ~350 MB).
84
+ */
85
+ const FORMAT_SNIFF_BYTES = 65_536;
86
+ async function readFileHead(path) {
87
+ const handle = await open(path, "r");
88
+ try {
89
+ const buffer = Buffer.alloc(FORMAT_SNIFF_BYTES);
90
+ const { bytesRead } = await handle.read(buffer, 0, FORMAT_SNIFF_BYTES, 0);
91
+ return buffer.subarray(0, bytesRead).toString("utf8");
92
+ }
93
+ finally {
94
+ await handle.close();
95
+ }
96
+ }
97
+ /**
98
+ * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
99
+ * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
100
+ */
101
+ export async function fetchGeonamesDumps(options, report) {
102
+ mkdirSync(options.outRoot, { recursive: true });
103
+ const baseURL = options.baseURL ?? BASE_URL;
104
+ const countryInfoDest = join(options.outRoot, "countryInfo.txt");
105
+ await downloadToFile({
106
+ url: `${baseURL}/countryInfo.txt`,
107
+ dest: countryInfoDest,
108
+ timeoutMs: 120_000,
109
+ retries: 2,
110
+ report,
111
+ });
112
+ const catalog = parseCountryInfo(await readFile(countryInfoDest, "utf8"));
113
+ const countries = options.countries?.map((code) => code.trim().toUpperCase()) ?? catalog.map((row) => row.country);
114
+ const entries = [];
115
+ const failedCodes = [];
116
+ const unavailable = [];
117
+ const skippedPresent = [];
118
+ const wrongFormatPresent = [];
119
+ let fetched = 0;
120
+ for (const country of countries) {
121
+ const txtDest = join(options.outRoot, `${country}.txt`);
122
+ if (!options.force && existsSync(txtDest)) {
123
+ if (looksLikeGazetteerDump(await readFileHead(txtDest))) {
124
+ skippedPresent.push(country);
125
+ }
126
+ else {
127
+ report?.(`✗ ${country}.txt is present but is not a 19-column gazetteer dump — move it aside and rerun`);
128
+ wrongFormatPresent.push(country);
129
+ }
130
+ continue;
131
+ }
132
+ const filename = `${country}.zip`;
133
+ const url = `${baseURL}/${filename}`;
134
+ const zipDest = join(options.outRoot, filename);
135
+ report?.(`=== ${SLUG} / ${country}`);
136
+ try {
137
+ await downloadToFile({ url, dest: zipDest, timeoutMs: 300_000, retries: 2, report });
138
+ await extractZipEntry(zipDest, `${country}.txt`, txtDest);
139
+ await rm(zipDest, { force: true });
140
+ entries.push({
141
+ country,
142
+ filename: `${country}.txt`,
143
+ source_url: url,
144
+ sha256: await sha256File(txtDest),
145
+ bytes: (await readFile(txtDest)).byteLength,
146
+ });
147
+ fetched++;
148
+ }
149
+ catch (error) {
150
+ await rm(zipDest, { force: true });
151
+ const message = error instanceof Error ? error.message : String(error);
152
+ // Branch on the TYPED status (the geonames-postal lesson): message prose contains the URL, and a URL
153
+ // can contain any substring.
154
+ if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
155
+ report?.(`✗ ${country}: GeoNames publishes no gazetteer dump for this country`);
156
+ unavailable.push(country);
157
+ }
158
+ else {
159
+ report?.(`✗ ${country}: ${message}`);
160
+ }
161
+ failedCodes.push(country);
162
+ }
163
+ }
164
+ const manifest = {
165
+ source: SLUG,
166
+ base_url: baseURL,
167
+ license: "CC-BY-4.0",
168
+ attribution: "GeoNames",
169
+ downloaded_at: new Date().toISOString(),
170
+ files: entries,
171
+ skipped_present: skippedPresent.toSorted(),
172
+ unavailable,
173
+ wrong_format_present: wrongFormatPresent.toSorted(),
174
+ };
175
+ await writeManifest(join(options.outRoot, "MANIFEST.json"), manifest);
176
+ return { fetched, skipped: skippedPresent.length, failed: failedCodes.length, failedCodes, skippedPresent };
177
+ }
178
+ //# sourceMappingURL=geonames-dump.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"geonames-dump.js","sourceRoot":"","sources":["../../../../src/tools/fetch/geonames-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AAC/C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE9E;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,eAAe,CAAA;AAE5B;;;GAGG;AACH,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAmD5D;;;GAGG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAEjC;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACrC,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEpE,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAA;YAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC1E,IAAI,EAAE,CAAA;YACP,CAAC;YAED,OAAO,IAAI,KAAK,sBAAsB,GAAG,CAAC,CAAA;QAC3C,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QAErB,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,IAAI,GAAgD,EAAE,CAAA;IAE5D,6IAA6I;IAC7I,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAElD,6FAA6F;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE7C,IAAI,OAAO,EAAE,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,kBAAkB,GAAG,MAAM,CAAA;AAEjC,KAAK,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAEpC,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAA;QAC/C,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAA;QAEzE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACtD,CAAC;YAAS,CAAC;QACV,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAiC,EACjC,MAA+B;IAE/B,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE/C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAA;IAEhE,MAAM,cAAc,CAAC;QACpB,GAAG,EAAE,GAAG,OAAO,kBAAkB;QACjC,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,CAAC;QACV,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAA;IACzE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAElH,MAAM,OAAO,GAA4B,EAAE,CAAA;IAC3C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,MAAM,kBAAkB,GAAa,EAAE,CAAA;IACvC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,CAAC,CAAA;QAEvD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,IAAI,sBAAsB,CAAC,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;gBACzD,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,iFAAiF,CAAC,CAAA;gBACvG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACjC,CAAC;YAED,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAE/C,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YACpF,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,EAAE,OAAO,CAAC,CAAA;YACzD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YAElC,OAAO,CAAC,IAAI,CAAC;gBACZ,OAAO;gBACP,QAAQ,EAAE,GAAG,OAAO,MAAM;gBAC1B,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC;gBACjC,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU;aAC3C,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YAElC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,qGAAqG;YACrG,6BAA6B;YAC7B,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,yDAAyD,CAAC,CAAA;gBAC/E,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAyB;QACtC,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,eAAe,EAAE,cAAc,CAAC,QAAQ,EAAE;QAC1C,WAAW;QACX,oBAAoB,EAAE,kBAAkB,CAAC,QAAQ,EAAE;KACnD,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAErE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;AAC5G,CAAC"}
@@ -0,0 +1,76 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
7
+ * `(postcode, locality, region)` triple with the NAMES inline.
8
+ *
9
+ * Source : https://download.geonames.org/export/zip/<CC>.zip
10
+ * License: CC-BY-4.0, attribute "GeoNames". Tier B.
11
+ *
12
+ * ## Why this source and not a join
13
+ *
14
+ * A postcode shard is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
15
+ * only one of them works everywhere:
16
+ *
17
+ * - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
18
+ * 97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
19
+ * `locality`/`localadmin`). It is also the ONLY shard that does: `postalcode-geonames-intl.db` and every
20
+ * `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
21
+ * have none.
22
+ * - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
23
+ * on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
24
+ * sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
25
+ * own. Under 50% for four of the five, which is not a join.
26
+ *
27
+ * The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
28
+ * nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
29
+ *
30
+ * ## Coverage is not universal, and the gap is the point
31
+ *
32
+ * GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode shard cannot be built
33
+ * from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
34
+ * before assuming it is there; an absent country fails as one entry, never as the whole run.
35
+ *
36
+ * ## Row counts from this source overstate, for some countries by exactly 2×
37
+ *
38
+ * Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
39
+ * on the shard built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
40
+ * exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a shard from the row
41
+ * count doubles its estimate for those countries.
42
+ *
43
+ * ## Why `downloadToFile` and not `APIClient`
44
+ *
45
+ * Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
46
+ * file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
47
+ *
48
+ * Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
49
+ */
50
+ import type { BaseFetchOptions, FetchSummary } from "./download.ts";
51
+ /**
52
+ * Countries fetched when the caller names none.
53
+ *
54
+ * These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
55
+ * why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
56
+ */
57
+ export declare const GEONAMES_POSTAL_DEFAULT_COUNTRIES: readonly ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
58
+ export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
59
+ /**
60
+ * ISO alpha-2 codes, in any casing. Defaults to {@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}.
61
+ */
62
+ countries?: readonly string[];
63
+ /**
64
+ * Export directory to read from. Defaults to GeoNames' own. Exists so the 404-is-a-coverage-finding behaviour can be
65
+ * exercised against a local server rather than by asking GeoNames for a country it does not have.
66
+ */
67
+ baseURL?: string;
68
+ }
69
+ /**
70
+ * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
71
+ * carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
72
+ *
73
+ * A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
74
+ */
75
+ export declare function fetchGeonamesPostal(options: FetchGeonamesPostalOptions, report?: (line: string) => void): Promise<FetchSummary>;
76
+ //# sourceMappingURL=geonames-postal.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"geonames-postal.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/geonames-postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAenE;;;;;GAKG;AACH,eAAO,MAAM,iCAAiC,qDAAsD,CAAA;AAEpG,MAAM,WAAW,0BAA2B,SAAQ,gBAAgB;IACnE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAwBD;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA0DvB"}