@mailwoman/corpus 8.1.0 → 8.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (483) hide show
  1. package/out/src/adapter.d.ts +15 -5
  2. package/out/src/adapter.d.ts.map +1 -1
  3. package/out/src/adapter.js +11 -9
  4. package/out/src/adapter.js.map +1 -1
  5. package/out/src/adapters/ban/adapter.d.ts +7 -0
  6. package/out/src/adapters/ban/adapter.d.ts.map +1 -1
  7. package/out/src/adapters/ban/adapter.js +9 -2
  8. package/out/src/adapters/ban/adapter.js.map +1 -1
  9. package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
  10. package/out/src/adapters/ban/street-decompose.js +5 -1
  11. package/out/src/adapters/ban/street-decompose.js.map +1 -1
  12. package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
  13. package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
  14. package/out/src/adapters/fcc-bdc/adapter.js +15 -2
  15. package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
  16. package/out/src/adapters/geonames/adapter.d.ts +11 -0
  17. package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
  18. package/out/src/adapters/geonames/adapter.js +24 -5
  19. package/out/src/adapters/geonames/adapter.js.map +1 -1
  20. package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
  21. package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
  22. package/out/src/adapters/geonames-postal/adapter.js +14 -1
  23. package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
  24. package/out/src/adapters/gnaf/adapter.d.ts +10 -1
  25. package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
  26. package/out/src/adapters/gnaf/adapter.js +11 -2
  27. package/out/src/adapters/gnaf/adapter.js.map +1 -1
  28. package/out/src/adapters/gnaf/assemble.d.ts +18 -6
  29. package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
  30. package/out/src/adapters/gnaf/assemble.js +16 -6
  31. package/out/src/adapters/gnaf/assemble.js.map +1 -1
  32. package/out/src/adapters/index.d.ts.map +1 -1
  33. package/out/src/adapters/index.js.map +1 -1
  34. package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
  35. package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
  36. package/out/src/adapters/openaddresses/adapter.js +18 -3
  37. package/out/src/adapters/openaddresses/adapter.js.map +1 -1
  38. package/out/src/adapters/overture/adapter.d.ts +11 -0
  39. package/out/src/adapters/overture/adapter.d.ts.map +1 -1
  40. package/out/src/adapters/overture/adapter.js +12 -1
  41. package/out/src/adapters/overture/adapter.js.map +1 -1
  42. package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
  43. package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
  44. package/out/src/adapters/state-hi-schools/adapter.js +11 -0
  45. package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
  46. package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
  47. package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
  48. package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
  49. package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
  50. package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
  51. package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
  52. package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
  53. package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
  54. package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
  55. package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
  56. package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
  57. package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
  58. package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
  59. package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
  60. package/out/src/adapters/synth-po-box/adapter.js +13 -2
  61. package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
  62. package/out/src/adapters/tiger/adapter.d.ts +14 -1
  63. package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
  64. package/out/src/adapters/tiger/adapter.js +19 -4
  65. package/out/src/adapters/tiger/adapter.js.map +1 -1
  66. package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
  67. package/out/src/adapters/tiger/street-decompose.js.map +1 -1
  68. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
  69. package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
  70. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
  71. package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
  72. package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
  73. package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
  74. package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
  75. package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
  76. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
  77. package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
  78. package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
  79. package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
  80. package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
  81. package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
  82. package/out/src/adapters/usgov-nad/adapter.js +12 -1
  83. package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
  84. package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
  85. package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
  86. package/out/src/adapters/usgov-nppes/adapter.js +11 -0
  87. package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
  88. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
  89. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
  90. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
  91. package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
  92. package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
  93. package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
  94. package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
  95. package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
  96. package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
  97. package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
  98. package/out/src/adapters/wof-admin-json/adapter.js +15 -5
  99. package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
  100. package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
  101. package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
  102. package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
  103. package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
  104. package/out/src/align.d.ts +12 -4
  105. package/out/src/align.d.ts.map +1 -1
  106. package/out/src/align.js +9 -5
  107. package/out/src/align.js.map +1 -1
  108. package/out/src/build.d.ts +24 -8
  109. package/out/src/build.d.ts.map +1 -1
  110. package/out/src/build.js +3 -3
  111. package/out/src/build.js.map +1 -1
  112. package/out/src/codex/us-fips-state.d.ts +6 -2
  113. package/out/src/codex/us-fips-state.d.ts.map +1 -1
  114. package/out/src/codex/us-fips-state.js +3 -1
  115. package/out/src/codex/us-fips-state.js.map +1 -1
  116. package/out/src/golden.d.ts +21 -7
  117. package/out/src/golden.d.ts.map +1 -1
  118. package/out/src/golden.js +18 -10
  119. package/out/src/golden.js.map +1 -1
  120. package/out/src/license.d.ts +3 -1
  121. package/out/src/license.d.ts.map +1 -1
  122. package/out/src/license.js +4 -2
  123. package/out/src/license.js.map +1 -1
  124. package/out/src/parquet-wrapper/reader.d.ts +4 -2
  125. package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
  126. package/out/src/parquet-wrapper/reader.js +5 -4
  127. package/out/src/parquet-wrapper/reader.js.map +1 -1
  128. package/out/src/parquet-wrapper/schema.d.ts +14 -10
  129. package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
  130. package/out/src/parquet-wrapper/schema.js +6 -2
  131. package/out/src/parquet-wrapper/schema.js.map +1 -1
  132. package/out/src/parquet-wrapper/writer.d.ts +12 -4
  133. package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
  134. package/out/src/parquet-wrapper/writer.js +12 -4
  135. package/out/src/parquet-wrapper/writer.js.map +1 -1
  136. package/out/src/parquet.d.ts +21 -7
  137. package/out/src/parquet.d.ts.map +1 -1
  138. package/out/src/parquet.js +15 -4
  139. package/out/src/parquet.js.map +1 -1
  140. package/out/src/runner.d.ts +39 -13
  141. package/out/src/runner.d.ts.map +1 -1
  142. package/out/src/runner.js +7 -3
  143. package/out/src/runner.js.map +1 -1
  144. package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
  145. package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
  146. package/out/src/shard-recipes/anchor-absorption.js +14 -5
  147. package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
  148. package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
  149. package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
  150. package/out/src/shard-recipes/boundary-stress.js +13 -7
  151. package/out/src/shard-recipes/boundary-stress.js.map +1 -1
  152. package/out/src/shard-recipes/country-balanced.d.ts +4 -0
  153. package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
  154. package/out/src/shard-recipes/country-balanced.js +39 -11
  155. package/out/src/shard-recipes/country-balanced.js.map +1 -1
  156. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
  157. package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
  158. package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
  159. package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
  160. package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
  161. package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
  162. package/out/src/shard-recipes/fr-admin-split.js +14 -3
  163. package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
  164. package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
  165. package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
  166. package/out/src/shard-recipes/fr-bare-street.js +4 -0
  167. package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
  168. package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
  169. package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
  170. package/out/src/shard-recipes/fr-fragment.js +25 -8
  171. package/out/src/shard-recipes/fr-fragment.js.map +1 -1
  172. package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
  173. package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
  174. package/out/src/shard-recipes/fr-lieudit.js +21 -8
  175. package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
  176. package/out/src/shard-recipes/fr-order.d.ts +4 -0
  177. package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
  178. package/out/src/shard-recipes/fr-order.js +27 -8
  179. package/out/src/shard-recipes/fr-order.js.map +1 -1
  180. package/out/src/shard-recipes/german.d.ts +4 -0
  181. package/out/src/shard-recipes/german.d.ts.map +1 -1
  182. package/out/src/shard-recipes/german.js +16 -6
  183. package/out/src/shard-recipes/german.js.map +1 -1
  184. package/out/src/shard-recipes/house-venue.d.ts +4 -0
  185. package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
  186. package/out/src/shard-recipes/house-venue.js +7 -1
  187. package/out/src/shard-recipes/house-venue.js.map +1 -1
  188. package/out/src/shard-recipes/index.d.ts +9 -3
  189. package/out/src/shard-recipes/index.d.ts.map +1 -1
  190. package/out/src/shard-recipes/index.js +12 -4
  191. package/out/src/shard-recipes/index.js.map +1 -1
  192. package/out/src/shard-recipes/intersection.d.ts +4 -0
  193. package/out/src/shard-recipes/intersection.d.ts.map +1 -1
  194. package/out/src/shard-recipes/intersection.js +27 -13
  195. package/out/src/shard-recipes/intersection.js.map +1 -1
  196. package/out/src/shard-recipes/locale.d.ts +4 -0
  197. package/out/src/shard-recipes/locale.d.ts.map +1 -1
  198. package/out/src/shard-recipes/locale.js +9 -4
  199. package/out/src/shard-recipes/locale.js.map +1 -1
  200. package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
  201. package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
  202. package/out/src/shard-recipes/nl-postcode.js +8 -2
  203. package/out/src/shard-recipes/nl-postcode.js.map +1 -1
  204. package/out/src/shard-recipes/no-fragment.d.ts +4 -0
  205. package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
  206. package/out/src/shard-recipes/no-fragment.js +8 -2
  207. package/out/src/shard-recipes/no-fragment.js.map +1 -1
  208. package/out/src/shard-recipes/no-street-led.d.ts +4 -0
  209. package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
  210. package/out/src/shard-recipes/no-street-led.js +5 -1
  211. package/out/src/shard-recipes/no-street-led.js.map +1 -1
  212. package/out/src/shard-recipes/no-street.d.ts +4 -0
  213. package/out/src/shard-recipes/no-street.d.ts.map +1 -1
  214. package/out/src/shard-recipes/no-street.js +4 -0
  215. package/out/src/shard-recipes/no-street.js.map +1 -1
  216. package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
  217. package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
  218. package/out/src/shard-recipes/po-box-cedex.js +99 -43
  219. package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
  220. package/out/src/shard-recipes/po-box.d.ts +4 -0
  221. package/out/src/shard-recipes/po-box.d.ts.map +1 -1
  222. package/out/src/shard-recipes/po-box.js +4 -0
  223. package/out/src/shard-recipes/po-box.js.map +1 -1
  224. package/out/src/shard-recipes/scaffold.d.ts +48 -16
  225. package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
  226. package/out/src/shard-recipes/scaffold.js +10 -6
  227. package/out/src/shard-recipes/scaffold.js.map +1 -1
  228. package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
  229. package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
  230. package/out/src/shard-recipes/si-bare-village.js +4 -0
  231. package/out/src/shard-recipes/si-bare-village.js.map +1 -1
  232. package/out/src/shard-recipes/street-affix.d.ts +4 -0
  233. package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
  234. package/out/src/shard-recipes/street-affix.js +29 -10
  235. package/out/src/shard-recipes/street-affix.js.map +1 -1
  236. package/out/src/shard-recipes/street-bare.d.ts +4 -0
  237. package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
  238. package/out/src/shard-recipes/street-bare.js +4 -0
  239. package/out/src/shard-recipes/street-bare.js.map +1 -1
  240. package/out/src/shard-recipes/street.d.ts +4 -0
  241. package/out/src/shard-recipes/street.d.ts.map +1 -1
  242. package/out/src/shard-recipes/street.js +4 -0
  243. package/out/src/shard-recipes/street.js.map +1 -1
  244. package/out/src/shard-recipes/unit.d.ts +4 -0
  245. package/out/src/shard-recipes/unit.d.ts.map +1 -1
  246. package/out/src/shard-recipes/unit.js +45 -14
  247. package/out/src/shard-recipes/unit.js.map +1 -1
  248. package/out/src/split.d.ts +18 -6
  249. package/out/src/split.d.ts.map +1 -1
  250. package/out/src/split.js +4 -2
  251. package/out/src/split.js.map +1 -1
  252. package/out/src/synthesize-anchor-absorption.d.ts +7 -1
  253. package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
  254. package/out/src/synthesize-anchor-absorption.js +28 -14
  255. package/out/src/synthesize-anchor-absorption.js.map +1 -1
  256. package/out/src/synthesize-boundary-stress.d.ts +3 -1
  257. package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
  258. package/out/src/synthesize-boundary-stress.js +35 -28
  259. package/out/src/synthesize-boundary-stress.js.map +1 -1
  260. package/out/src/synthesize-german.d.ts +15 -5
  261. package/out/src/synthesize-german.d.ts.map +1 -1
  262. package/out/src/synthesize-german.js +9 -3
  263. package/out/src/synthesize-german.js.map +1 -1
  264. package/out/src/synthesize-house-venue.d.ts.map +1 -1
  265. package/out/src/synthesize-house-venue.js +34 -18
  266. package/out/src/synthesize-house-venue.js.map +1 -1
  267. package/out/src/synthesize-intersection.d.ts +9 -3
  268. package/out/src/synthesize-intersection.d.ts.map +1 -1
  269. package/out/src/synthesize-intersection.js +25 -6
  270. package/out/src/synthesize-intersection.js.map +1 -1
  271. package/out/src/synthesize-no-street.d.ts +3 -1
  272. package/out/src/synthesize-no-street.d.ts.map +1 -1
  273. package/out/src/synthesize-no-street.js +5 -6
  274. package/out/src/synthesize-no-street.js.map +1 -1
  275. package/out/src/synthesize-po-box.d.ts +15 -5
  276. package/out/src/synthesize-po-box.d.ts.map +1 -1
  277. package/out/src/synthesize-po-box.js +21 -7
  278. package/out/src/synthesize-po-box.js.map +1 -1
  279. package/out/src/synthesize-street.d.ts +3 -1
  280. package/out/src/synthesize-street.d.ts.map +1 -1
  281. package/out/src/synthesize-street.js +6 -1
  282. package/out/src/synthesize-street.js.map +1 -1
  283. package/out/src/synthesize.d.ts +39 -13
  284. package/out/src/synthesize.d.ts.map +1 -1
  285. package/out/src/synthesize.js +66 -36
  286. package/out/src/synthesize.js.map +1 -1
  287. package/out/src/tokenize.d.ts +15 -5
  288. package/out/src/tokenize.d.ts.map +1 -1
  289. package/out/src/tokenize.js.map +1 -1
  290. package/out/src/tools/align-shard.d.ts.map +1 -1
  291. package/out/src/tools/align-shard.js +3 -1
  292. package/out/src/tools/align-shard.js.map +1 -1
  293. package/out/src/tools/audit.d.ts.map +1 -1
  294. package/out/src/tools/audit.js +20 -8
  295. package/out/src/tools/audit.js.map +1 -1
  296. package/out/src/tools/corpus-stats.d.ts.map +1 -1
  297. package/out/src/tools/corpus-stats.js.map +1 -1
  298. package/out/src/tools/fetch/ban.d.ts.map +1 -1
  299. package/out/src/tools/fetch/ban.js +17 -10
  300. package/out/src/tools/fetch/ban.js.map +1 -1
  301. package/out/src/tools/fetch/download.d.ts +30 -10
  302. package/out/src/tools/fetch/download.d.ts.map +1 -1
  303. package/out/src/tools/fetch/download.js +26 -6
  304. package/out/src/tools/fetch/download.js.map +1 -1
  305. package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
  306. package/out/src/tools/fetch/hrsa.js.map +1 -1
  307. package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
  308. package/out/src/tools/fetch/imls-pls.js +18 -18
  309. package/out/src/tools/fetch/imls-pls.js.map +1 -1
  310. package/out/src/tools/fetch/index.d.ts +3 -1
  311. package/out/src/tools/fetch/index.d.ts.map +1 -1
  312. package/out/src/tools/fetch/index.js +3 -1
  313. package/out/src/tools/fetch/index.js.map +1 -1
  314. package/out/src/tools/fetch/nad.d.ts +21 -7
  315. package/out/src/tools/fetch/nad.d.ts.map +1 -1
  316. package/out/src/tools/fetch/nad.js +5 -5
  317. package/out/src/tools/fetch/nad.js.map +1 -1
  318. package/out/src/tools/fetch/nppes.d.ts.map +1 -1
  319. package/out/src/tools/fetch/nppes.js +9 -15
  320. package/out/src/tools/fetch/nppes.js.map +1 -1
  321. package/out/src/tools/fetch/openaddresses.d.ts +3 -1
  322. package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
  323. package/out/src/tools/fetch/openaddresses.js +32 -23
  324. package/out/src/tools/fetch/openaddresses.js.map +1 -1
  325. package/out/src/tools/fetch/ppd.d.ts +3 -1
  326. package/out/src/tools/fetch/ppd.d.ts.map +1 -1
  327. package/out/src/tools/fetch/ppd.js +14 -3
  328. package/out/src/tools/fetch/ppd.js.map +1 -1
  329. package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
  330. package/out/src/tools/fetch/state-hi-schools.js +13 -6
  331. package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
  332. package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
  333. package/out/src/tools/fetch/state-sources.js +8 -3
  334. package/out/src/tools/fetch/state-sources.js.map +1 -1
  335. package/out/src/tools/fetch/tiger-full.d.ts +9 -3
  336. package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
  337. package/out/src/tools/fetch/tiger-full.js +35 -20
  338. package/out/src/tools/fetch/tiger-full.js.map +1 -1
  339. package/out/src/tools/gb-title-case.d.ts.map +1 -1
  340. package/out/src/tools/gb-title-case.js.map +1 -1
  341. package/out/src/tools/golden-expand.d.ts +24 -8
  342. package/out/src/tools/golden-expand.d.ts.map +1 -1
  343. package/out/src/tools/golden-expand.js +20 -12
  344. package/out/src/tools/golden-expand.js.map +1 -1
  345. package/out/src/tools/golden-promote.d.ts +18 -6
  346. package/out/src/tools/golden-promote.d.ts.map +1 -1
  347. package/out/src/tools/golden-promote.js +22 -7
  348. package/out/src/tools/golden-promote.js.map +1 -1
  349. package/out/src/tools/ingest-csv.d.ts +3 -1
  350. package/out/src/tools/ingest-csv.d.ts.map +1 -1
  351. package/out/src/tools/ingest-csv.js +17 -19
  352. package/out/src/tools/ingest-csv.js.map +1 -1
  353. package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
  354. package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
  355. package/out/src/tools/jsonl-to-parquet.js +11 -5
  356. package/out/src/tools/jsonl-to-parquet.js.map +1 -1
  357. package/out/src/tools/lint-shard-vocab.d.ts +36 -12
  358. package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
  359. package/out/src/tools/lint-shard-vocab.js +25 -11
  360. package/out/src/tools/lint-shard-vocab.js.map +1 -1
  361. package/out/src/tools/lint-shard.d.ts +30 -10
  362. package/out/src/tools/lint-shard.d.ts.map +1 -1
  363. package/out/src/tools/lint-shard.js +33 -20
  364. package/out/src/tools/lint-shard.js.map +1 -1
  365. package/out/src/tools/overlay-manifest.d.ts.map +1 -1
  366. package/out/src/tools/overlay-manifest.js +6 -4
  367. package/out/src/tools/overlay-manifest.js.map +1 -1
  368. package/out/src/tools/shard-kryptonite.d.ts +6 -2
  369. package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
  370. package/out/src/tools/shard-kryptonite.js +1 -1
  371. package/out/src/tools/shard-kryptonite.js.map +1 -1
  372. package/out/src/tools/shard-translit.d.ts +9 -3
  373. package/out/src/tools/shard-translit.d.ts.map +1 -1
  374. package/out/src/tools/shard-translit.js +2 -2
  375. package/out/src/tools/shard-translit.js.map +1 -1
  376. package/out/src/types.d.ts +51 -17
  377. package/out/src/types.d.ts.map +1 -1
  378. package/out/src/wof-json.d.ts +9 -3
  379. package/out/src/wof-json.d.ts.map +1 -1
  380. package/out/src/wof-json.js +6 -4
  381. package/out/src/wof-json.js.map +1 -1
  382. package/package.json +7 -6
  383. package/src/adapter.ts +24 -13
  384. package/src/adapters/ban/adapter.ts +12 -2
  385. package/src/adapters/ban/street-decompose.ts +7 -1
  386. package/src/adapters/fcc-bdc/adapter.ts +19 -3
  387. package/src/adapters/geonames/adapter.ts +28 -5
  388. package/src/adapters/geonames-postal/adapter.ts +17 -1
  389. package/src/adapters/gnaf/adapter.ts +15 -2
  390. package/src/adapters/gnaf/assemble.ts +40 -10
  391. package/src/adapters/index.ts +13 -0
  392. package/src/adapters/openaddresses/adapter.ts +22 -3
  393. package/src/adapters/overture/adapter.ts +18 -2
  394. package/src/adapters/state-hi-schools/adapter.ts +15 -0
  395. package/src/adapters/state-ia-contractors/adapter.ts +15 -0
  396. package/src/adapters/state-ny-notaries/adapter.ts +15 -0
  397. package/src/adapters/state-tx-notaries/adapter.ts +18 -2
  398. package/src/adapters/synth-po-box/adapter.ts +19 -2
  399. package/src/adapters/tiger/adapter.ts +23 -4
  400. package/src/adapters/tiger/street-decompose.ts +1 -0
  401. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
  402. package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
  403. package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
  404. package/src/adapters/usgov-nad/adapter.ts +14 -1
  405. package/src/adapters/usgov-nppes/adapter.ts +13 -0
  406. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
  407. package/src/adapters/wof-admin-jp/adapter.ts +20 -3
  408. package/src/adapters/wof-admin-json/adapter.ts +22 -7
  409. package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
  410. package/src/align.ts +21 -8
  411. package/src/build.ts +48 -11
  412. package/src/codex/us-fips-state.ts +6 -2
  413. package/src/golden.ts +28 -13
  414. package/src/license.ts +4 -2
  415. package/src/parquet-wrapper/reader.ts +6 -4
  416. package/src/parquet-wrapper/schema.ts +14 -10
  417. package/src/parquet-wrapper/writer.ts +12 -4
  418. package/src/parquet.ts +39 -8
  419. package/src/runner.ts +49 -15
  420. package/src/shard-recipes/anchor-absorption.ts +21 -5
  421. package/src/shard-recipes/boundary-stress.ts +20 -7
  422. package/src/shard-recipes/country-balanced.ts +78 -13
  423. package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
  424. package/src/shard-recipes/fr-admin-split.ts +30 -5
  425. package/src/shard-recipes/fr-bare-street.ts +10 -0
  426. package/src/shard-recipes/fr-fragment.ts +33 -8
  427. package/src/shard-recipes/fr-lieudit.ts +35 -9
  428. package/src/shard-recipes/fr-order.ts +47 -9
  429. package/src/shard-recipes/german.ts +32 -6
  430. package/src/shard-recipes/house-venue.ts +10 -1
  431. package/src/shard-recipes/index.ts +12 -4
  432. package/src/shard-recipes/intersection.ts +63 -14
  433. package/src/shard-recipes/locale.ts +37 -4
  434. package/src/shard-recipes/nl-postcode.ts +12 -2
  435. package/src/shard-recipes/no-fragment.ts +12 -2
  436. package/src/shard-recipes/no-street-led.ts +11 -1
  437. package/src/shard-recipes/no-street.ts +7 -0
  438. package/src/shard-recipes/po-box-cedex.ts +145 -45
  439. package/src/shard-recipes/po-box.ts +7 -0
  440. package/src/shard-recipes/scaffold.ts +52 -20
  441. package/src/shard-recipes/si-bare-village.ts +9 -0
  442. package/src/shard-recipes/street-affix.ts +88 -13
  443. package/src/shard-recipes/street-bare.ts +10 -0
  444. package/src/shard-recipes/street.ts +7 -0
  445. package/src/shard-recipes/unit.ts +79 -16
  446. package/src/split.ts +28 -7
  447. package/src/synthesize-anchor-absorption.ts +41 -14
  448. package/src/synthesize-boundary-stress.ts +57 -30
  449. package/src/synthesize-german.ts +30 -7
  450. package/src/synthesize-house-venue.ts +43 -19
  451. package/src/synthesize-intersection.ts +34 -7
  452. package/src/synthesize-no-street.ts +18 -10
  453. package/src/synthesize-po-box.ts +42 -10
  454. package/src/synthesize-street.ts +10 -2
  455. package/src/synthesize.ts +81 -38
  456. package/src/tokenize.ts +15 -5
  457. package/src/tools/align-shard.ts +6 -1
  458. package/src/tools/audit.ts +57 -11
  459. package/src/tools/corpus-stats.ts +9 -0
  460. package/src/tools/fetch/ban.ts +32 -10
  461. package/src/tools/fetch/download.ts +49 -12
  462. package/src/tools/fetch/hrsa.ts +3 -0
  463. package/src/tools/fetch/imls-pls.ts +28 -18
  464. package/src/tools/fetch/index.ts +3 -1
  465. package/src/tools/fetch/nad.ts +39 -12
  466. package/src/tools/fetch/nppes.ts +17 -15
  467. package/src/tools/fetch/openaddresses.ts +52 -24
  468. package/src/tools/fetch/ppd.ts +22 -3
  469. package/src/tools/fetch/state-hi-schools.ts +17 -6
  470. package/src/tools/fetch/state-sources.ts +16 -3
  471. package/src/tools/fetch/tiger-full.ts +66 -23
  472. package/src/tools/gb-title-case.ts +1 -0
  473. package/src/tools/golden-expand.ts +64 -20
  474. package/src/tools/golden-promote.ts +53 -13
  475. package/src/tools/ingest-csv.ts +41 -26
  476. package/src/tools/jsonl-to-parquet.ts +30 -10
  477. package/src/tools/lint-shard-vocab.ts +75 -24
  478. package/src/tools/lint-shard.ts +75 -31
  479. package/src/tools/overlay-manifest.ts +7 -4
  480. package/src/tools/shard-kryptonite.ts +11 -3
  481. package/src/tools/shard-translit.ts +20 -5
  482. package/src/types.ts +51 -17
  483. package/src/wof-json.ts +18 -7
package/src/adapter.ts CHANGED
@@ -34,16 +34,24 @@ import type { CanonicalRow, CorpusAdapter } from "./types.ts"
34
34
  * the dependency graph stays traceable.
35
35
  */
36
36
  export interface AdapterRegistry {
37
- /** Add an adapter. Throws if `adapter.id` is already registered. */
37
+ /**
38
+ * Add an adapter. Throws if `adapter.id` is already registered.
39
+ */
38
40
  register(adapter: CorpusAdapter): void
39
41
 
40
- /** Return the adapter for `id`, or `undefined`. */
42
+ /**
43
+ * Return the adapter for `id`, or `undefined`.
44
+ */
41
45
  get(id: string): CorpusAdapter | undefined
42
46
 
43
- /** All registered adapters, in insertion order. */
47
+ /**
48
+ * All registered adapters, in insertion order.
49
+ */
44
50
  list(): readonly CorpusAdapter[]
45
51
 
46
- /** Convenience: ids only, in insertion order. */
52
+ /**
53
+ * Convenience: ids only, in insertion order.
54
+ */
47
55
  ids(): readonly string[]
48
56
  }
49
57
 
@@ -58,6 +66,7 @@ export class InMemoryAdapterRegistry implements AdapterRegistry {
58
66
  if (this.#byID.has(adapter.id)) {
59
67
  throw new Error(`AdapterRegistry: id ${JSON.stringify(adapter.id)} already registered`)
60
68
  }
69
+
61
70
  this.#byID.set(adapter.id, adapter)
62
71
  }
63
72
 
@@ -92,9 +101,9 @@ export const defaultAdapterRegistry = new InMemoryAdapterRegistry()
92
101
  * length.
93
102
  */
94
103
  export function stableSourceID(adapterID: string, components: Partial<Record<ComponentTag, string>>): string {
95
- const sortedKeys = Object.keys(components).sort() as ComponentTag[]
96
- const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\x1f")
97
- const digest = createHash("sha256").update(adapterID).update("\x1e").update(payload).digest("hex")
104
+ const sortedKeys = Object.keys(components).toSorted() as ComponentTag[]
105
+ const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\u001F")
106
+ const digest = createHash("sha256").update(adapterID).update("\u001E").update(payload).digest("hex")
98
107
 
99
108
  return `${adapterID}-${digest.slice(0, 12)}`
100
109
  }
@@ -110,12 +119,12 @@ export function stableSourceID(adapterID: string, components: Partial<Record<Com
110
119
  * ensuring each augmentation variant survives.
111
120
  */
112
121
  export function canonicalDedupKey(row: CanonicalRow): string {
113
- const sortedKeys = Object.keys(row.components).sort() as ComponentTag[]
114
- const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\x1f")
115
- const rawNorm = row.raw.toLowerCase().replace(/\s+/g, " ").trim()
116
- const synthPart = row.synth ? `\x1e${row.synth.method}` : ""
122
+ const sortedKeys = Object.keys(row.components).toSorted() as ComponentTag[]
123
+ const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\u001F")
124
+ const rawNorm = row.raw.toLowerCase().replaceAll(/\s+/g, " ").trim()
125
+ const synthPart = row.synth ? `\u001E${row.synth.method}` : ""
117
126
 
118
- return `${row.country}\x1e${rawNorm}\x1e${compPart}${synthPart}`
127
+ return `${row.country}\u001E${rawNorm}\u001E${compPart}${synthPart}`
119
128
  }
120
129
 
121
130
  /**
@@ -130,7 +139,9 @@ export interface StreamingHasher {
130
139
  digest(): string
131
140
  }
132
141
 
133
- /** Default `StreamingHasher` (SHA-256, hex). */
142
+ /**
143
+ * Default `StreamingHasher` (SHA-256, hex).
144
+ */
134
145
  export function streamingSha256(): StreamingHasher {
135
146
  const h: Hash = createHash("sha256")
136
147
  let finalized = false
@@ -39,6 +39,10 @@ import { reconcileComponents } from "../../format.ts"
39
39
  import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
40
40
  import { decomposeFrStreet } from "./street-decompose.ts"
41
41
 
42
+ /**
43
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
44
+ * it came from.
45
+ */
42
46
  export const BAN_ADAPTER_ID = "ban"
43
47
 
44
48
  /**
@@ -83,13 +87,14 @@ function composeRaw(house: string, street: string, postcode: string, locality: s
83
87
  if (streetPart) {
84
88
  parts.push(streetPart)
85
89
  }
90
+
86
91
  const cityPart = [postcode, locality].filter(Boolean).join(" ").trim()
87
92
 
88
93
  if (cityPart) {
89
94
  parts.push(cityPart)
90
95
  }
91
96
 
92
- return parts.join(", ").replace(/\s+/g, " ").trim()
97
+ return parts.join(", ").replaceAll(/\s+/g, " ").trim()
93
98
  }
94
99
 
95
100
  export function createBanAdapter(): CorpusAdapter {
@@ -104,6 +109,7 @@ export function createBanAdapter(): CorpusAdapter {
104
109
  }
105
110
 
106
111
  const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
112
+
107
113
  const parser = stream.pipe(
108
114
  csvParse({
109
115
  delimiter: ";",
@@ -161,7 +167,7 @@ export function createBanAdapter(): CorpusAdapter {
161
167
 
162
168
  const aligned = reconcileComponents(components, raw)
163
169
 
164
- if (Object.keys(aligned).length === 0) continue
170
+ if (!Object.keys(aligned).length) continue
165
171
 
166
172
  const sourceID = record.id?.trim()
167
173
  ? `${BAN_ADAPTER_ID}-${record.id.trim()}`
@@ -177,6 +183,7 @@ export function createBanAdapter(): CorpusAdapter {
177
183
  corpus_version: "",
178
184
  license: "Licence Ouverte 2.0",
179
185
  }
186
+
180
187
  emitted++
181
188
  }
182
189
  } finally {
@@ -186,4 +193,7 @@ export function createBanAdapter(): CorpusAdapter {
186
193
  }
187
194
  }
188
195
 
196
+ /**
197
+ * The configured adapter instance registered with the corpus builder.
198
+ */
189
199
  export const banAdapter = createBanAdapter()
@@ -20,6 +20,11 @@ import { resolve } from "node:path"
20
20
 
21
21
  import { resourceDictionaryPath } from "@mailwoman/core/utils"
22
22
 
23
+ /**
24
+ * Tokens a BAN street needs before a type/article/name decomposition is attempted.
25
+ */
26
+ const MIN_TOKENS_FOR_DECOMPOSE = 3
27
+
23
28
  function loadDictionary(filename: string): Set<string> {
24
29
  const candidates = [
25
30
  resourceDictionaryPath("libpostal", "fr", filename),
@@ -51,6 +56,7 @@ function loadDictionary(filename: string): Set<string> {
51
56
  // try next
52
57
  }
53
58
  }
59
+
54
60
  throw new Error(`Could not load FR libpostal dictionary: ${filename}`)
55
61
  }
56
62
 
@@ -79,7 +85,7 @@ export function decomposeFrStreet(fullname: string): DecomposedFrStreet {
79
85
  const norm = (s: string) => s.toLowerCase().replace(/[.,;]$/, "")
80
86
 
81
87
  // Try 2-word prefix first (e.g. "ancien chemin")
82
- if (tokens.length >= 3) {
88
+ if (tokens.length >= MIN_TOKENS_FOR_DECOMPOSE) {
83
89
  const twoWord = norm(tokens[0]!) + " " + norm(tokens[1]!)
84
90
 
85
91
  if (STREET_TYPES_FR.has(twoWord)) {
@@ -40,10 +40,20 @@ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
40
40
  import { formatAddress, reconcileComponents } from "../../format.ts"
41
41
  import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
42
42
 
43
+ /**
44
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
45
+ * it came from.
46
+ */
43
47
  export const FCC_BDC_ADAPTER_ID = "fcc-bdc"
48
+ /**
49
+ * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
50
+ * than having to look them up.
51
+ */
44
52
  export const FCC_BDC_DEFAULT_LICENSE = "Public Domain"
45
53
 
46
- /** SQLite row shape — one row per BSL `location_id`. Columns mirror NTIARecord. */
54
+ /**
55
+ * SQLite row shape — one row per BSL `location_id`. Columns mirror NTIARecord.
56
+ */
47
57
  interface BdcLocationRow {
48
58
  location_id: number
49
59
  address_primary: string
@@ -110,7 +120,9 @@ export function buildPostcode(zip: string, suffix: string | null): string {
110
120
  return `${z}-${s}`
111
121
  }
112
122
 
113
- /** Build a BDC adapter. Pure factory so multiple instances can be created in tests. */
123
+ /**
124
+ * Build a BDC adapter. Pure factory so multiple instances can be created in tests.
125
+ */
114
126
  export function createFccBdcAdapter(): CorpusAdapter {
115
127
  return {
116
128
  id: FCC_BDC_ADAPTER_ID,
@@ -164,7 +176,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
164
176
  if (!raw) continue
165
177
  const aligned = reconcileComponents(components, raw)
166
178
 
167
- if (Object.keys(aligned).length === 0) continue
179
+ if (!Object.keys(aligned).length) continue
168
180
 
169
181
  yield {
170
182
  raw,
@@ -176,6 +188,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
176
188
  corpus_version: "",
177
189
  license: FCC_BDC_DEFAULT_LICENSE,
178
190
  }
191
+
179
192
  emitted++
180
193
  }
181
194
  } finally {
@@ -185,4 +198,7 @@ export function createFccBdcAdapter(): CorpusAdapter {
185
198
  }
186
199
  }
187
200
 
201
+ /**
202
+ * The configured adapter instance registered with the corpus builder.
203
+ */
188
204
  export const fccBdcAdapter = createFccBdcAdapter()
@@ -37,10 +37,20 @@ import { stableSourceID } from "../../adapter.ts"
37
37
  import { reconcileComponents } from "../../format.ts"
38
38
  import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
39
39
 
40
+ /**
41
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
42
+ * it came from.
43
+ */
40
44
  export const GEONAMES_ADAPTER_ID = "geonames"
45
+ /**
46
+ * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
47
+ * than having to look them up.
48
+ */
41
49
  export const GEONAMES_DEFAULT_LICENSE = "CC-BY-4.0"
42
50
 
43
- // GeoNames main-table column indices (0-based; see the export README).
51
+ /**
52
+ * GeoNames main-table column indices (0-based; see the export README).
53
+ */
44
54
  const COL = {
45
55
  geonameid: 0,
46
56
  name: 1,
@@ -51,10 +61,14 @@ const COL = {
51
61
  admin1: 10,
52
62
  } as const
53
63
 
54
- // Populated-place feature codes that are NOT current real places — skip them.
64
+ /**
65
+ * Populated-place feature codes that are NOT current real places — skip them.
66
+ */
55
67
  const NON_CURRENT_PPL = new Set(["PPLH", "PPLQ", "PPLW", "PPLCH"])
56
68
 
57
- /** Load `admin1CodesASCII.txt` → Map("<CC>.<admin1>" → region name). Empty map if absent. */
69
+ /**
70
+ * Load `admin1CodesASCII.txt` → Map("<CC>.<admin1>" → region name). Empty map if absent.
71
+ */
58
72
  function loadAdmin1(dir: string): Map<string, string> {
59
73
  const map = new Map<string, string>()
60
74
  const fp = join(dir, "admin1CodesASCII.txt")
@@ -73,7 +87,9 @@ function loadAdmin1(dir: string): Map<string, string> {
73
87
  return map
74
88
  }
75
89
 
76
- /** Load `countryInfo.txt` → Map(ISO → country name). Empty map if absent. The file is `#`-commented. */
90
+ /**
91
+ * Load `countryInfo.txt` → Map(ISO → country name). Empty map if absent. The file is `#`-commented.
92
+ */
77
93
  function loadCountries(dir: string): Map<string, string> {
78
94
  const map = new Map<string, string>()
79
95
  const fp = join(dir, "countryInfo.txt")
@@ -106,6 +122,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
106
122
  const countries = loadCountries(dir)
107
123
 
108
124
  const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
125
+
109
126
  const parser = stream.pipe(
110
127
  csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
111
128
  )
@@ -159,10 +176,12 @@ export function createGeonamesAdapter(): CorpusAdapter {
159
176
  if (opts.limit !== undefined && emitted >= opts.limit) break
160
177
  const aligned = reconcileComponents(v.comp, v.raw)
161
178
 
162
- if (Object.keys(aligned).length === 0) continue
179
+ if (!Object.keys(aligned).length) continue
180
+
163
181
  const sourceID = geonameid
164
182
  ? `${GEONAMES_ADAPTER_ID}-${geonameid}-${v.slot}`
165
183
  : stableSourceID(GEONAMES_ADAPTER_ID, aligned)
184
+
166
185
  yield {
167
186
  raw: v.raw,
168
187
  components: aligned,
@@ -172,6 +191,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
172
191
  corpus_version: "",
173
192
  license: GEONAMES_DEFAULT_LICENSE,
174
193
  }
194
+
175
195
  emitted++
176
196
  }
177
197
  }
@@ -182,4 +202,7 @@ export function createGeonamesAdapter(): CorpusAdapter {
182
202
  }
183
203
  }
184
204
 
205
+ /**
206
+ * The configured adapter instance registered with the corpus builder.
207
+ */
185
208
  export const geonamesAdapter = createGeonamesAdapter()
@@ -31,10 +31,20 @@ import { stableSourceID } from "../../adapter.ts"
31
31
  import { reconcileComponents } from "../../format.ts"
32
32
  import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
33
33
 
34
+ /**
35
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
36
+ * it came from.
37
+ */
34
38
  export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal"
39
+ /**
40
+ * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
41
+ * than having to look them up.
42
+ */
35
43
  export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0"
36
44
 
37
- // GeoNames postal-dump columns (0-based).
45
+ /**
46
+ * GeoNames postal-dump columns (0-based).
47
+ */
38
48
  const COL = { country: 0, postcode: 1, place: 2, admin1Name: 3 } as const
39
49
 
40
50
  export function createGeonamesPostalAdapter(): CorpusAdapter {
@@ -46,6 +56,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
46
56
 
47
57
  async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
48
58
  const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
59
+
49
60
  const parser = stream.pipe(
50
61
  csvParse({ delimiter: "\t", quote: false, relax_column_count: true, skip_empty_lines: true })
51
62
  )
@@ -89,6 +100,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
89
100
  const aligned = reconcileComponents(v.comp, v.raw)
90
101
 
91
102
  if (Object.keys(aligned).length < 2) continue
103
+
92
104
  yield {
93
105
  raw: v.raw,
94
106
  components: aligned,
@@ -98,6 +110,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
98
110
  corpus_version: "",
99
111
  license: GEONAMES_POSTAL_DEFAULT_LICENSE,
100
112
  }
113
+
101
114
  emitted++
102
115
  }
103
116
  }
@@ -108,4 +121,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
108
121
  }
109
122
  }
110
123
 
124
+ /**
125
+ * The configured adapter instance registered with the corpus builder.
126
+ */
111
127
  export const geonamesPostalAdapter = createGeonamesPostalAdapter()
@@ -34,8 +34,14 @@ import { stableSourceID } from "../../adapter.ts"
34
34
  import { reconcileComponents } from "../../format.ts"
35
35
  import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
36
36
 
37
+ /**
38
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
39
+ * it came from.
40
+ */
37
41
  export const GNAF_ADAPTER_ID = "gnaf"
38
- /** Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style). */
42
+ /**
43
+ * Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
44
+ */
39
45
  export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0"
40
46
 
41
47
  interface GNAFTuple {
@@ -102,8 +108,10 @@ export function createGNAFAdapter(): CorpusAdapter {
102
108
 
103
109
  const orders = renderOrders(t)
104
110
  const order = idx % orders.length
111
+
105
112
  idx++
106
113
  const raw = orders[order]!
114
+
107
115
  const components: CanonicalRow["components"] = {
108
116
  house_number: t.house_number,
109
117
  street: t.street,
@@ -119,7 +127,8 @@ export function createGNAFAdapter(): CorpusAdapter {
119
127
 
120
128
  const aligned = reconcileComponents(components, raw)
121
129
 
122
- if (Object.keys(aligned).length === 0) continue
130
+ if (!Object.keys(aligned).length) continue
131
+
123
132
  yield {
124
133
  raw,
125
134
  components: aligned,
@@ -130,10 +139,14 @@ export function createGNAFAdapter(): CorpusAdapter {
130
139
  corpus_version: "",
131
140
  license: GNAF_DEFAULT_LICENSE,
132
141
  }
142
+
133
143
  emitted++
134
144
  }
135
145
  },
136
146
  }
137
147
  }
138
148
 
149
+ /**
150
+ * The configured adapter instance registered with the corpus builder.
151
+ */
139
152
  export const gnafAdapter = createGNAFAdapter()
@@ -33,17 +33,25 @@ import { join } from "node:path"
33
33
  import { PSVSpliterator } from "spliterator"
34
34
 
35
35
  export interface GNAFAssembleOptions {
36
- /** G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables). */
36
+ /**
37
+ * G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
38
+ */
37
39
  standardDir: string
38
- /** Target sample size (uniform reservoir → population-proportional across states). */
40
+ /**
41
+ * Target sample size (uniform reservoir → population-proportional across states).
42
+ */
39
43
  sampleSize: number
40
- /** Output JSONL path. */
44
+ /**
45
+ * Output JSONL path.
46
+ */
41
47
  out: string
42
48
  /**
43
49
  * Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
44
50
  */
45
51
  holdoutPath?: string
46
- /** Progress sink (the CLI passes a setter). */
52
+ /**
53
+ * Progress sink (the CLI passes a setter).
54
+ */
47
55
  onProgress?: (message: string) => void
48
56
  }
49
57
 
@@ -54,25 +62,32 @@ export interface GNAFAssembleResult {
54
62
  byState: Record<string, number>
55
63
  }
56
64
 
57
- /** UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour). */
65
+ /**
66
+ * UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
67
+ */
58
68
  export function titlecase(s: string): string {
59
69
  return s
60
70
  .toLowerCase()
61
- .replace(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
71
+ .replaceAll(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
62
72
  .trim()
63
73
  }
64
74
 
65
- /** Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative). */
75
+ /**
76
+ * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
77
+ */
66
78
  export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
67
79
  return `${street}|${locality}|${postcode}`.toLowerCase()
68
80
  }
69
81
 
70
82
  type Row = Record<string, string | number | undefined>
83
+
71
84
  async function* psvObjects(path: string): AsyncIterable<Row> {
72
85
  yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true }) as AsyncIterable<Row>
73
86
  }
74
87
 
75
- /** Load a small lookup table fully into a Map keyed by `keyCol`. */
88
+ /**
89
+ * Load a small lookup table fully into a Map keyed by `keyCol`.
90
+ */
76
91
  async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
77
92
  const m = new Map<string, V>()
78
93
 
@@ -89,7 +104,9 @@ async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V):
89
104
  return m
90
105
  }
91
106
 
92
- /** Build the held-out key set from an eval JSONL whose rows carry a `components` object. */
107
+ /**
108
+ * Build the held-out key set from an eval JSONL whose rows carry a `components` object.
109
+ */
93
110
  async function loadHoldout(path: string): Promise<Set<string>> {
94
111
  const keys = new Set<string>()
95
112
  const text = await readFile(path, "utf8")
@@ -114,6 +131,7 @@ async function loadHoldout(path: string): Promise<Set<string>> {
114
131
  export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
115
132
  const progress = opts.onProgress ?? (() => {})
116
133
  const files = await readdir(opts.standardDir)
134
+
117
135
  const pick = (re: RegExp, exclude?: RegExp) =>
118
136
  files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
119
137
 
@@ -129,16 +147,19 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
129
147
  }
130
148
 
131
149
  progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
150
+
132
151
  const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
133
152
  name: String(r.STREET_NAME ?? ""),
134
153
  type: String(r.STREET_TYPE_CODE ?? ""),
135
154
  suffix: String(r.STREET_SUFFIX_CODE ?? ""),
136
155
  }))
156
+
137
157
  const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""))
138
158
  progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
139
159
 
140
160
  const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
141
161
  []
162
+
142
163
  let seen = 0
143
164
  let heldOut = 0
144
165
 
@@ -159,8 +180,10 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
159
180
 
160
181
  if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
161
182
  heldOut++
183
+
162
184
  continue
163
185
  }
186
+
164
187
  let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "")
165
188
 
166
189
  if (r.NUMBER_LAST) {
@@ -170,7 +193,9 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
170
193
  if (r.FLAT_NUMBER) {
171
194
  house = `${String(r.FLAT_NUMBER)}/${house}`
172
195
  }
196
+
173
197
  const tuple = { house_number: house, street, locality, region: state, postcode }
198
+
174
199
  seen++
175
200
 
176
201
  if (reservoir.length < opts.sampleSize) {
@@ -183,6 +208,7 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
183
208
  }
184
209
  }
185
210
  }
211
+
186
212
  progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
187
213
  }
188
214
 
@@ -193,7 +219,11 @@ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssem
193
219
  out.write(JSON.stringify(t) + "\n")
194
220
  byState[t.region] = (byState[t.region] ?? 0) + 1
195
221
  }
196
- await new Promise<void>((res) => out.end(res))
222
+
223
+ await new Promise<void>((res) => {
224
+ out.end(res)
225
+ })
226
+
197
227
  progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
198
228
 
199
229
  return { written: reservoir.length, seen, heldOut, byState }
@@ -81,55 +81,68 @@ for (const adapter of BUILTIN_ADAPTERS) {
81
81
 
82
82
  export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts"
83
83
  export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts"
84
+
84
85
  export {
85
86
  GEONAMES_POSTAL_ADAPTER_ID,
86
87
  GEONAMES_POSTAL_DEFAULT_LICENSE,
87
88
  geonamesPostalAdapter,
88
89
  } from "./geonames-postal/adapter.ts"
90
+
89
91
  export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts"
92
+
90
93
  export {
91
94
  OPENADDRESSES_ADAPTER_ID,
92
95
  OPENADDRESSES_DEFAULT_LICENSE,
93
96
  openaddressesAdapter,
94
97
  } from "./openaddresses/adapter.ts"
98
+
95
99
  export {
96
100
  STATE_HI_SCHOOLS_ADAPTER_ID,
97
101
  STATE_HI_SCHOOLS_DEFAULT_LICENSE,
98
102
  stateHiSchoolsAdapter,
99
103
  } from "./state-hi-schools/adapter.ts"
104
+
100
105
  export {
101
106
  STATE_IA_CONTRACTORS_ADAPTER_ID,
102
107
  STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
103
108
  stateIaContractorsAdapter,
104
109
  } from "./state-ia-contractors/adapter.ts"
110
+
105
111
  export {
106
112
  STATE_NY_NOTARIES_ADAPTER_ID,
107
113
  STATE_NY_NOTARIES_DEFAULT_LICENSE,
108
114
  stateNyNotariesAdapter,
109
115
  } from "./state-ny-notaries/adapter.ts"
116
+
110
117
  export {
111
118
  STATE_TX_NOTARIES_ADAPTER_ID,
112
119
  STATE_TX_NOTARIES_DEFAULT_LICENSE,
113
120
  stateTxNotariesAdapter,
114
121
  } from "./state-tx-notaries/adapter.ts"
122
+
115
123
  export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts"
124
+
116
125
  export {
117
126
  USGOV_HRSA_FQHC_ADAPTER_ID,
118
127
  USGOV_HRSA_FQHC_DEFAULT_LICENSE,
119
128
  usgovHrsaFqhcAdapter,
120
129
  } from "./usgov-hrsa-fqhc/adapter.ts"
130
+
121
131
  export {
122
132
  USGOV_IMLS_PLS_ADAPTER_ID,
123
133
  USGOV_IMLS_PLS_DEFAULT_LICENSE,
124
134
  usgovImlsPlsAdapter,
125
135
  } from "./usgov-imls-pls/adapter.ts"
136
+
126
137
  export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
127
138
  export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts"
128
139
  export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
140
+
129
141
  export {
130
142
  USGOV_SAMHSA_ADAPTER_ID,
131
143
  USGOV_SAMHSA_DEFAULT_LICENSE,
132
144
  usgovSamhsaTreatmentLocatorAdapter,
133
145
  } from "./usgov-samhsa-treatment-locator/adapter.ts"
146
+
134
147
  export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts"
135
148
  export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"