@mailwoman/corpus 9.3.0 → 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (991) hide show
  1. package/README.md +16 -0
  2. package/lib/adapters/geonames/adapter.ts +3 -3
  3. package/lib/adapters/geonames/postal/adapter.ts +122 -0
  4. package/lib/adapters/index.ts +39 -32
  5. package/lib/adapters/openaddresses/adapter.ts +4 -7
  6. package/lib/adapters/osm/adapter.ts +278 -0
  7. package/lib/adapters/overture/adapter.ts +22 -7
  8. package/lib/adapters/utils/index.ts +1 -1
  9. package/lib/adapters/wof/admin/json/adapter.ts +239 -0
  10. package/lib/adapters/wof/json-rows.ts +181 -0
  11. package/lib/adapters/wof/postalcode-json/adapter.ts +196 -0
  12. package/lib/au/adapters/gnaf/adapter.ts +152 -0
  13. package/lib/au/adapters/gnaf/assemble.ts +225 -0
  14. package/lib/cz/recipes/pcfirst-preposition.ts +112 -0
  15. package/lib/de/recipes/locale.ts +360 -0
  16. package/lib/fr/adapters/ban/adapter.ts +159 -0
  17. package/lib/fr/recipes/admin-split.ts +243 -0
  18. package/lib/fr/recipes/bare-street.ts +171 -0
  19. package/lib/fr/recipes/fragment.ts +334 -0
  20. package/lib/fr/recipes/lieudit.ts +274 -0
  21. package/lib/fr/recipes/order.ts +246 -0
  22. package/lib/fr/tools/fetch/ban.ts +289 -0
  23. package/lib/index.ts +1 -1
  24. package/lib/international/recipes/locale.ts +598 -0
  25. package/lib/jp/tools/fetch/houjin.ts +141 -0
  26. package/lib/kr/adapters/juso/label-rows.ts +259 -0
  27. package/lib/kr/adapters/localdata/align.ts +299 -0
  28. package/lib/kr/adapters/localdata/key-index.ts +185 -0
  29. package/lib/kr/adapters/localdata/permit-rows.ts +186 -0
  30. package/lib/kr/tools/fetch/juso.ts +172 -0
  31. package/lib/kr/tools/fetch/localdata.ts +206 -0
  32. package/lib/nl/recipes/postcode.ts +117 -0
  33. package/lib/no/recipes/fragment.ts +213 -0
  34. package/lib/no/recipes/street/index.ts +91 -0
  35. package/lib/no/recipes/street/led.ts +135 -0
  36. package/lib/recipes/anchor-absorption.ts +2 -2
  37. package/lib/recipes/bare/country.ts +95 -0
  38. package/lib/recipes/bare/postcode/eval.ts +99 -0
  39. package/lib/recipes/bare/postcode/index.ts +327 -0
  40. package/lib/recipes/boundary-stress.ts +3 -1
  41. package/lib/recipes/country-balanced.ts +3 -2
  42. package/lib/recipes/house-venue.ts +2 -1
  43. package/lib/recipes/index.ts +28 -21
  44. package/lib/recipes/intersection.ts +3 -2
  45. package/lib/recipes/po/box/cedex/au-nz.ts +82 -0
  46. package/lib/recipes/po/box/cedex/ca.ts +58 -0
  47. package/lib/recipes/po/box/cedex/fr.ts +76 -0
  48. package/lib/recipes/po/box/cedex/phrases.ts +108 -0
  49. package/lib/recipes/po/box/cedex/recipe.ts +203 -0
  50. package/lib/recipes/po/box/cedex/sources.ts +189 -0
  51. package/lib/recipes/po/box/cedex/types.ts +35 -0
  52. package/lib/recipes/po/box/cedex/us.ts +69 -0
  53. package/lib/recipes/po/box/cedex/vocabulary.ts +106 -0
  54. package/lib/recipes/po/box/cedex.ts +9 -0
  55. package/lib/recipes/po/box/index.ts +128 -0
  56. package/lib/recipes/scaffold.ts +18 -42
  57. package/lib/recipes/street/affix.ts +807 -0
  58. package/lib/recipes/street/bare.ts +102 -0
  59. package/lib/recipes/street.ts +3 -1
  60. package/lib/recipes/sub/venue/context.ts +149 -0
  61. package/lib/recipes/sub/venue/index.ts +997 -0
  62. package/lib/recipes/sub/venue/sources.ts +739 -0
  63. package/lib/recipes/trailing-region.ts +9 -9
  64. package/lib/recipes/unit.ts +3 -3
  65. package/lib/sg/recipes/register.ts +270 -0
  66. package/lib/sg/tools/fetch/acra.ts +179 -0
  67. package/lib/si/recipes/bare-village.ts +99 -0
  68. package/lib/south-asia/recipes/register.ts +214 -0
  69. package/lib/synthesizers/anchor-absorption.ts +1 -1
  70. package/lib/synthesizers/locale.ts +261 -0
  71. package/lib/synthesizers/street.ts +1 -1
  72. package/lib/synthesizers/utils.ts +2 -2
  73. package/lib/tools/audit.ts +3 -2
  74. package/lib/tools/corpus-stats.ts +3 -3
  75. package/lib/tools/fetch/download/index.ts +10 -0
  76. package/lib/tools/fetch/download/manifest.ts +102 -0
  77. package/lib/tools/fetch/download/network.ts +226 -0
  78. package/lib/tools/fetch/download/retry.ts +173 -0
  79. package/lib/tools/fetch/geonames/dump.ts +257 -0
  80. package/lib/tools/fetch/geonames/postal.ts +180 -0
  81. package/lib/tools/fetch/index.ts +47 -20
  82. package/lib/tools/fetch/openaddresses.ts +2 -2
  83. package/lib/tools/fetch/ourairports.ts +2 -2
  84. package/lib/tools/fetch/ppd.ts +1 -3
  85. package/lib/tools/fetch/wikidata-subvenue.ts +2 -2
  86. package/lib/tools/golden/promote.ts +340 -0
  87. package/lib/tools/golden/relabel-street/directory.ts +340 -0
  88. package/lib/tools/golden/relabel-street/index.ts +9 -0
  89. package/lib/tools/golden/relabel-street/row.ts +361 -0
  90. package/lib/tools/index.ts +10 -10
  91. package/lib/tools/ingest-csv.ts +5 -7
  92. package/lib/tools/lint/slice/vocab.ts +376 -0
  93. package/lib/tools/overlay-manifest.ts +92 -25
  94. package/lib/tools/overture-subvenue.ts +1 -1
  95. package/lib/tools/postcode-triples.ts +138 -14
  96. package/lib/tools/sub/venue/harvest.ts +234 -0
  97. package/lib/tools/sub/venue/head-nouns.ts +245 -0
  98. package/lib/tools/sub/venue/lexicon.ts +429 -0
  99. package/lib/tools/sub/venue/surfaces.ts +95 -0
  100. package/lib/tools/sub/venue/table.ts +281 -0
  101. package/lib/tools/sub/venue/wikidata.ts +85 -0
  102. package/lib/tw/tools/fetch/gcis.ts +180 -0
  103. package/lib/types.ts +1 -1
  104. package/lib/us/adapters/fcc-bdc/adapter.ts +173 -0
  105. package/lib/us/adapters/state/hi-schools/adapter.ts +160 -0
  106. package/lib/us/adapters/state/ia-contractors/adapter.ts +136 -0
  107. package/lib/us/adapters/state/ny-notaries/adapter.ts +133 -0
  108. package/lib/us/adapters/state/tx-notaries/adapter.ts +149 -0
  109. package/lib/us/adapters/tiger/adapter.ts +250 -0
  110. package/lib/us/adapters/usgov/hrsa-fqhc/adapter.ts +156 -0
  111. package/lib/us/adapters/usgov/imls-pls/adapter.ts +134 -0
  112. package/lib/us/adapters/usgov/irs-bmf/adapter.ts +145 -0
  113. package/lib/us/adapters/usgov/nad/adapter.ts +324 -0
  114. package/lib/us/adapters/usgov/nppes/adapter.ts +143 -0
  115. package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +182 -0
  116. package/lib/us/tools/fetch/hrsa.ts +59 -0
  117. package/lib/us/tools/fetch/imls-pls.ts +149 -0
  118. package/lib/us/tools/fetch/nad.ts +341 -0
  119. package/lib/us/tools/fetch/nppes.ts +165 -0
  120. package/lib/us/tools/fetch/state/hi-schools.ts +161 -0
  121. package/lib/us/tools/fetch/state/sources.ts +146 -0
  122. package/lib/us/tools/fetch/tiger-full.ts +308 -0
  123. package/lib/utils/align.ts +1 -1
  124. package/lib/utils/golden.ts +12 -8
  125. package/lib/utils/wof-json.ts +7 -13
  126. package/out/adapters/geonames/adapter.d.ts +1 -1
  127. package/out/adapters/geonames/adapter.js +3 -3
  128. package/out/adapters/geonames/adapter.js.map +1 -1
  129. package/out/adapters/geonames/postal/adapter.d.ts.map +1 -0
  130. package/out/adapters/geonames/postal/adapter.js +107 -0
  131. package/out/adapters/geonames/postal/adapter.js.map +1 -0
  132. package/out/adapters/index.d.ts +16 -16
  133. package/out/adapters/index.d.ts.map +1 -1
  134. package/out/adapters/index.js +34 -32
  135. package/out/adapters/index.js.map +1 -1
  136. package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
  137. package/out/adapters/openaddresses/adapter.js +4 -6
  138. package/out/adapters/openaddresses/adapter.js.map +1 -1
  139. package/out/adapters/osm/adapter.d.ts +95 -0
  140. package/out/adapters/osm/adapter.d.ts.map +1 -0
  141. package/out/adapters/osm/adapter.js +211 -0
  142. package/out/adapters/osm/adapter.js.map +1 -0
  143. package/out/adapters/overture/adapter.d.ts +10 -0
  144. package/out/adapters/overture/adapter.d.ts.map +1 -1
  145. package/out/adapters/overture/adapter.js +20 -7
  146. package/out/adapters/overture/adapter.js.map +1 -1
  147. package/out/adapters/synth-po-box/adapter.js.map +1 -1
  148. package/out/adapters/utils/index.d.ts +1 -1
  149. package/out/adapters/utils/index.d.ts.map +1 -1
  150. package/out/adapters/utils/index.js.map +1 -1
  151. package/out/adapters/wof/admin/json/adapter.d.ts +79 -0
  152. package/out/adapters/wof/admin/json/adapter.d.ts.map +1 -0
  153. package/out/adapters/wof/admin/json/adapter.js +204 -0
  154. package/out/adapters/wof/admin/json/adapter.js.map +1 -0
  155. package/out/adapters/wof/json-rows.d.ts +73 -0
  156. package/out/adapters/wof/json-rows.d.ts.map +1 -0
  157. package/out/adapters/wof/json-rows.js +115 -0
  158. package/out/adapters/wof/json-rows.js.map +1 -0
  159. package/out/adapters/wof/postalcode-json/adapter.d.ts +65 -0
  160. package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -0
  161. package/out/adapters/wof/postalcode-json/adapter.js +167 -0
  162. package/out/adapters/wof/postalcode-json/adapter.js.map +1 -0
  163. package/out/au/adapters/gnaf/adapter.d.ts.map +1 -0
  164. package/out/au/adapters/gnaf/adapter.js +127 -0
  165. package/out/au/adapters/gnaf/adapter.js.map +1 -0
  166. package/out/au/adapters/gnaf/assemble.d.ts +65 -0
  167. package/out/au/adapters/gnaf/assemble.d.ts.map +1 -0
  168. package/out/au/adapters/gnaf/assemble.js +154 -0
  169. package/out/au/adapters/gnaf/assemble.js.map +1 -0
  170. package/out/cn/recipes/organizational-units.d.ts +36 -0
  171. package/out/cn/recipes/organizational-units.d.ts.map +1 -0
  172. package/out/cn/recipes/organizational-units.js +159 -0
  173. package/out/cn/recipes/organizational-units.js.map +1 -0
  174. package/out/cz/recipes/pcfirst-preposition.d.ts +22 -0
  175. package/out/cz/recipes/pcfirst-preposition.d.ts.map +1 -0
  176. package/out/cz/recipes/pcfirst-preposition.js +101 -0
  177. package/out/cz/recipes/pcfirst-preposition.js.map +1 -0
  178. package/out/de/recipes/locale.d.ts +44 -0
  179. package/out/de/recipes/locale.d.ts.map +1 -0
  180. package/out/de/recipes/locale.js +289 -0
  181. package/out/de/recipes/locale.js.map +1 -0
  182. package/out/env.d.ts +1 -1
  183. package/out/fr/adapters/ban/adapter.d.ts.map +1 -0
  184. package/out/fr/adapters/ban/adapter.js +123 -0
  185. package/out/fr/adapters/ban/adapter.js.map +1 -0
  186. package/out/fr/adapters/ban/street-decompose.d.ts.map +1 -0
  187. package/out/fr/adapters/ban/street-decompose.js.map +1 -0
  188. package/out/fr/recipes/admin-split.d.ts +38 -0
  189. package/out/fr/recipes/admin-split.d.ts.map +1 -0
  190. package/out/fr/recipes/admin-split.js +192 -0
  191. package/out/fr/recipes/admin-split.js.map +1 -0
  192. package/out/fr/recipes/bare-street.d.ts +28 -0
  193. package/out/fr/recipes/bare-street.d.ts.map +1 -0
  194. package/out/fr/recipes/bare-street.js +144 -0
  195. package/out/fr/recipes/bare-street.js.map +1 -0
  196. package/out/fr/recipes/fragment.d.ts +77 -0
  197. package/out/fr/recipes/fragment.d.ts.map +1 -0
  198. package/out/fr/recipes/fragment.js +277 -0
  199. package/out/fr/recipes/fragment.js.map +1 -0
  200. package/out/fr/recipes/lieudit.d.ts +32 -0
  201. package/out/fr/recipes/lieudit.d.ts.map +1 -0
  202. package/out/fr/recipes/lieudit.js +211 -0
  203. package/out/fr/recipes/lieudit.js.map +1 -0
  204. package/out/fr/recipes/order.d.ts +38 -0
  205. package/out/fr/recipes/order.d.ts.map +1 -0
  206. package/out/fr/recipes/order.js +198 -0
  207. package/out/fr/recipes/order.js.map +1 -0
  208. package/out/fr/tools/fetch/ban.d.ts +31 -0
  209. package/out/fr/tools/fetch/ban.d.ts.map +1 -0
  210. package/out/fr/tools/fetch/ban.js +239 -0
  211. package/out/fr/tools/fetch/ban.js.map +1 -0
  212. package/out/index.d.ts +1 -1
  213. package/out/index.d.ts.map +1 -1
  214. package/out/index.js +1 -1
  215. package/out/index.js.map +1 -1
  216. package/out/international/recipes/locale.d.ts +136 -0
  217. package/out/international/recipes/locale.d.ts.map +1 -0
  218. package/out/international/recipes/locale.js +458 -0
  219. package/out/international/recipes/locale.js.map +1 -0
  220. package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -0
  221. package/out/jp/adapters/wof-admin/adapter.js.map +1 -0
  222. package/out/jp/tools/fetch/houjin.d.ts +24 -0
  223. package/out/jp/tools/fetch/houjin.d.ts.map +1 -0
  224. package/out/jp/tools/fetch/houjin.js +102 -0
  225. package/out/jp/tools/fetch/houjin.js.map +1 -0
  226. package/out/kr/adapters/juso/label-rows.d.ts +76 -0
  227. package/out/kr/adapters/juso/label-rows.d.ts.map +1 -0
  228. package/out/kr/adapters/juso/label-rows.js +194 -0
  229. package/out/kr/adapters/juso/label-rows.js.map +1 -0
  230. package/out/kr/adapters/localdata/align.d.ts +43 -0
  231. package/out/kr/adapters/localdata/align.d.ts.map +1 -0
  232. package/out/kr/adapters/localdata/align.js +223 -0
  233. package/out/kr/adapters/localdata/align.js.map +1 -0
  234. package/out/kr/adapters/localdata/key-index.d.ts +71 -0
  235. package/out/kr/adapters/localdata/key-index.d.ts.map +1 -0
  236. package/out/kr/adapters/localdata/key-index.js +133 -0
  237. package/out/kr/adapters/localdata/key-index.js.map +1 -0
  238. package/out/kr/adapters/localdata/permit-rows.d.ts +66 -0
  239. package/out/kr/adapters/localdata/permit-rows.d.ts.map +1 -0
  240. package/out/kr/adapters/localdata/permit-rows.js +131 -0
  241. package/out/kr/adapters/localdata/permit-rows.js.map +1 -0
  242. package/out/kr/tools/fetch/juso.d.ts +30 -0
  243. package/out/kr/tools/fetch/juso.d.ts.map +1 -0
  244. package/out/kr/tools/fetch/juso.js +120 -0
  245. package/out/kr/tools/fetch/juso.js.map +1 -0
  246. package/out/kr/tools/fetch/localdata.d.ts +30 -0
  247. package/out/kr/tools/fetch/localdata.d.ts.map +1 -0
  248. package/out/kr/tools/fetch/localdata.js +139 -0
  249. package/out/kr/tools/fetch/localdata.js.map +1 -0
  250. package/out/nl/recipes/postcode.d.ts +26 -0
  251. package/out/nl/recipes/postcode.d.ts.map +1 -0
  252. package/out/nl/recipes/postcode.js +103 -0
  253. package/out/nl/recipes/postcode.js.map +1 -0
  254. package/out/no/recipes/fragment.d.ts +43 -0
  255. package/out/no/recipes/fragment.d.ts.map +1 -0
  256. package/out/no/recipes/fragment.js +183 -0
  257. package/out/no/recipes/fragment.js.map +1 -0
  258. package/out/no/recipes/street/index.d.ts +18 -0
  259. package/out/no/recipes/street/index.d.ts.map +1 -0
  260. package/out/no/recipes/street/index.js +72 -0
  261. package/out/no/recipes/street/index.js.map +1 -0
  262. package/out/no/recipes/street/led.d.ts +24 -0
  263. package/out/no/recipes/street/led.d.ts.map +1 -0
  264. package/out/no/recipes/street/led.js +114 -0
  265. package/out/no/recipes/street/led.js.map +1 -0
  266. package/out/recipes/anchor-absorption.d.ts.map +1 -1
  267. package/out/recipes/anchor-absorption.js +2 -1
  268. package/out/recipes/anchor-absorption.js.map +1 -1
  269. package/out/recipes/bare/country.d.ts +25 -0
  270. package/out/recipes/bare/country.d.ts.map +1 -0
  271. package/out/recipes/bare/country.js +88 -0
  272. package/out/recipes/bare/country.js.map +1 -0
  273. package/out/recipes/bare/postcode/eval.d.ts +29 -0
  274. package/out/recipes/bare/postcode/eval.d.ts.map +1 -0
  275. package/out/recipes/bare/postcode/eval.js +85 -0
  276. package/out/recipes/bare/postcode/eval.js.map +1 -0
  277. package/out/recipes/bare/postcode/index.d.ts +62 -0
  278. package/out/recipes/bare/postcode/index.d.ts.map +1 -0
  279. package/out/recipes/bare/postcode/index.js +267 -0
  280. package/out/recipes/bare/postcode/index.js.map +1 -0
  281. package/out/recipes/boundary-stress.d.ts.map +1 -1
  282. package/out/recipes/boundary-stress.js +2 -1
  283. package/out/recipes/boundary-stress.js.map +1 -1
  284. package/out/recipes/country-balanced.d.ts.map +1 -1
  285. package/out/recipes/country-balanced.js +2 -1
  286. package/out/recipes/country-balanced.js.map +1 -1
  287. package/out/recipes/house-venue.d.ts.map +1 -1
  288. package/out/recipes/house-venue.js +2 -1
  289. package/out/recipes/house-venue.js.map +1 -1
  290. package/out/recipes/index.d.ts.map +1 -1
  291. package/out/recipes/index.js +28 -21
  292. package/out/recipes/index.js.map +1 -1
  293. package/out/recipes/intersection.d.ts.map +1 -1
  294. package/out/recipes/intersection.js +2 -1
  295. package/out/recipes/intersection.js.map +1 -1
  296. package/out/recipes/po/box/cedex/au-nz.d.ts +7 -0
  297. package/out/recipes/po/box/cedex/au-nz.d.ts.map +1 -0
  298. package/out/recipes/po/box/cedex/au-nz.js +56 -0
  299. package/out/recipes/po/box/cedex/au-nz.js.map +1 -0
  300. package/out/recipes/po/box/cedex/ca.d.ts +7 -0
  301. package/out/recipes/po/box/cedex/ca.d.ts.map +1 -0
  302. package/out/recipes/po/box/cedex/ca.js +38 -0
  303. package/out/recipes/po/box/cedex/ca.js.map +1 -0
  304. package/out/recipes/po/box/cedex/fr.d.ts +7 -0
  305. package/out/recipes/po/box/cedex/fr.d.ts.map +1 -0
  306. package/out/recipes/po/box/cedex/fr.js +57 -0
  307. package/out/recipes/po/box/cedex/fr.js.map +1 -0
  308. package/out/recipes/po/box/cedex/phrases.d.ts +8 -0
  309. package/out/recipes/po/box/cedex/phrases.d.ts.map +1 -0
  310. package/out/recipes/po/box/cedex/phrases.js +76 -0
  311. package/out/recipes/po/box/cedex/phrases.js.map +1 -0
  312. package/out/recipes/po/box/cedex/recipe.d.ts +9 -0
  313. package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -0
  314. package/out/recipes/po/box/cedex/recipe.js +153 -0
  315. package/out/recipes/po/box/cedex/recipe.js.map +1 -0
  316. package/out/recipes/po/box/cedex/sources.d.ts +86 -0
  317. package/out/recipes/po/box/cedex/sources.d.ts.map +1 -0
  318. package/out/recipes/po/box/cedex/sources.js +143 -0
  319. package/out/recipes/po/box/cedex/sources.js.map +1 -0
  320. package/out/recipes/po/box/cedex/types.d.ts +31 -0
  321. package/out/recipes/po/box/cedex/types.d.ts.map +1 -0
  322. package/out/recipes/po/box/cedex/types.js +5 -0
  323. package/out/recipes/po/box/cedex/types.js.map +1 -0
  324. package/out/recipes/po/box/cedex/us.d.ts +7 -0
  325. package/out/recipes/po/box/cedex/us.d.ts.map +1 -0
  326. package/out/recipes/po/box/cedex/us.js +51 -0
  327. package/out/recipes/po/box/cedex/us.js.map +1 -0
  328. package/out/recipes/po/box/cedex/vocabulary.d.ts +68 -0
  329. package/out/recipes/po/box/cedex/vocabulary.d.ts.map +1 -0
  330. package/out/recipes/po/box/cedex/vocabulary.js +90 -0
  331. package/out/recipes/po/box/cedex/vocabulary.js.map +1 -0
  332. package/out/recipes/po/box/cedex.d.ts +9 -0
  333. package/out/recipes/po/box/cedex.d.ts.map +1 -0
  334. package/out/recipes/po/box/cedex.js +9 -0
  335. package/out/recipes/po/box/cedex.js.map +1 -0
  336. package/out/recipes/po/box/index.d.ts +17 -0
  337. package/out/recipes/po/box/index.d.ts.map +1 -0
  338. package/out/recipes/po/box/index.js +105 -0
  339. package/out/recipes/po/box/index.js.map +1 -0
  340. package/out/recipes/scaffold.d.ts +20 -1
  341. package/out/recipes/scaffold.d.ts.map +1 -1
  342. package/out/recipes/scaffold.js +18 -49
  343. package/out/recipes/scaffold.js.map +1 -1
  344. package/out/recipes/street/affix.d.ts +89 -0
  345. package/out/recipes/street/affix.d.ts.map +1 -0
  346. package/out/recipes/street/affix.js +565 -0
  347. package/out/recipes/street/affix.js.map +1 -0
  348. package/out/recipes/street/bare.d.ts +25 -0
  349. package/out/recipes/street/bare.d.ts.map +1 -0
  350. package/out/recipes/street/bare.js +87 -0
  351. package/out/recipes/street/bare.js.map +1 -0
  352. package/out/recipes/street.d.ts.map +1 -1
  353. package/out/recipes/street.js +2 -1
  354. package/out/recipes/street.js.map +1 -1
  355. package/out/recipes/sub/venue/context.d.ts +29 -0
  356. package/out/recipes/sub/venue/context.d.ts.map +1 -0
  357. package/out/recipes/sub/venue/context.js +106 -0
  358. package/out/recipes/sub/venue/context.js.map +1 -0
  359. package/out/recipes/sub/venue/index.d.ts +270 -0
  360. package/out/recipes/sub/venue/index.d.ts.map +1 -0
  361. package/out/recipes/sub/venue/index.js +728 -0
  362. package/out/recipes/sub/venue/index.js.map +1 -0
  363. package/out/recipes/sub/venue/sources.d.ts +209 -0
  364. package/out/recipes/sub/venue/sources.d.ts.map +1 -0
  365. package/out/recipes/sub/venue/sources.js +555 -0
  366. package/out/recipes/sub/venue/sources.js.map +1 -0
  367. package/out/recipes/trailing-region.d.ts.map +1 -1
  368. package/out/recipes/trailing-region.js +8 -2
  369. package/out/recipes/trailing-region.js.map +1 -1
  370. package/out/recipes/unit.d.ts.map +1 -1
  371. package/out/recipes/unit.js +2 -1
  372. package/out/recipes/unit.js.map +1 -1
  373. package/out/sg/recipes/register.d.ts +69 -0
  374. package/out/sg/recipes/register.d.ts.map +1 -0
  375. package/out/sg/recipes/register.js +211 -0
  376. package/out/sg/recipes/register.js.map +1 -0
  377. package/out/sg/tools/fetch/acra.d.ts +23 -0
  378. package/out/sg/tools/fetch/acra.d.ts.map +1 -0
  379. package/out/sg/tools/fetch/acra.js +126 -0
  380. package/out/sg/tools/fetch/acra.js.map +1 -0
  381. package/out/si/recipes/bare-village.d.ts +32 -0
  382. package/out/si/recipes/bare-village.d.ts.map +1 -0
  383. package/out/si/recipes/bare-village.js +89 -0
  384. package/out/si/recipes/bare-village.js.map +1 -0
  385. package/out/south-asia/recipes/register.d.ts +51 -0
  386. package/out/south-asia/recipes/register.d.ts.map +1 -0
  387. package/out/south-asia/recipes/register.js +154 -0
  388. package/out/south-asia/recipes/register.js.map +1 -0
  389. package/out/synthesizers/anchor-absorption.d.ts +1 -1
  390. package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
  391. package/out/synthesizers/anchor-absorption.js.map +1 -1
  392. package/out/synthesizers/boundary-stress.js.map +1 -1
  393. package/out/synthesizers/house-venue.js.map +1 -1
  394. package/out/synthesizers/intersection.js.map +1 -1
  395. package/out/synthesizers/locale.d.ts +116 -0
  396. package/out/synthesizers/locale.d.ts.map +1 -0
  397. package/out/synthesizers/locale.js +150 -0
  398. package/out/synthesizers/locale.js.map +1 -0
  399. package/out/synthesizers/no-street.d.ts.map +1 -1
  400. package/out/synthesizers/no-street.js.map +1 -1
  401. package/out/synthesizers/po-box.js.map +1 -1
  402. package/out/synthesizers/street.d.ts.map +1 -1
  403. package/out/synthesizers/street.js +1 -1
  404. package/out/synthesizers/street.js.map +1 -1
  405. package/out/synthesizers/utils.js +1 -1
  406. package/out/synthesizers/utils.js.map +1 -1
  407. package/out/test-kit/corpus-recipe.d.ts.map +1 -1
  408. package/out/test-kit/corpus-recipe.js.map +1 -1
  409. package/out/tools/audit.d.ts.map +1 -1
  410. package/out/tools/audit.js +3 -2
  411. package/out/tools/audit.js.map +1 -1
  412. package/out/tools/corpus-stats.js +3 -3
  413. package/out/tools/corpus-stats.js.map +1 -1
  414. package/out/tools/fetch/download/index.d.ts +10 -0
  415. package/out/tools/fetch/download/index.d.ts.map +1 -0
  416. package/out/tools/fetch/download/index.js +10 -0
  417. package/out/tools/fetch/download/index.js.map +1 -0
  418. package/out/tools/fetch/download/manifest.d.ts +47 -0
  419. package/out/tools/fetch/download/manifest.d.ts.map +1 -0
  420. package/out/tools/fetch/download/manifest.js +74 -0
  421. package/out/tools/fetch/download/manifest.js.map +1 -0
  422. package/out/tools/fetch/download/network.d.ts +105 -0
  423. package/out/tools/fetch/download/network.d.ts.map +1 -0
  424. package/out/tools/fetch/download/network.js +130 -0
  425. package/out/tools/fetch/download/network.js.map +1 -0
  426. package/out/tools/fetch/download/retry.d.ts +45 -0
  427. package/out/tools/fetch/download/retry.d.ts.map +1 -0
  428. package/out/tools/fetch/download/retry.js +136 -0
  429. package/out/tools/fetch/download/retry.js.map +1 -0
  430. package/out/tools/fetch/geonames/dump.d.ts +91 -0
  431. package/out/tools/fetch/geonames/dump.d.ts.map +1 -0
  432. package/out/tools/fetch/geonames/dump.js +171 -0
  433. package/out/tools/fetch/geonames/dump.js.map +1 -0
  434. package/out/tools/fetch/geonames/postal.d.ts +76 -0
  435. package/out/tools/fetch/geonames/postal.d.ts.map +1 -0
  436. package/out/tools/fetch/geonames/postal.js +126 -0
  437. package/out/tools/fetch/geonames/postal.js.map +1 -0
  438. package/out/tools/fetch/index.d.ts +47 -20
  439. package/out/tools/fetch/index.d.ts.map +1 -1
  440. package/out/tools/fetch/index.js +47 -20
  441. package/out/tools/fetch/index.js.map +1 -1
  442. package/out/tools/fetch/openaddresses.d.ts +1 -1
  443. package/out/tools/fetch/openaddresses.d.ts.map +1 -1
  444. package/out/tools/fetch/openaddresses.js +1 -1
  445. package/out/tools/fetch/openaddresses.js.map +1 -1
  446. package/out/tools/fetch/ourairports.d.ts +1 -1
  447. package/out/tools/fetch/ourairports.d.ts.map +1 -1
  448. package/out/tools/fetch/ourairports.js +1 -1
  449. package/out/tools/fetch/ourairports.js.map +1 -1
  450. package/out/tools/fetch/ppd.js +1 -3
  451. package/out/tools/fetch/ppd.js.map +1 -1
  452. package/out/tools/fetch/wikidata-subvenue.d.ts +1 -1
  453. package/out/tools/fetch/wikidata-subvenue.d.ts.map +1 -1
  454. package/out/tools/fetch/wikidata-subvenue.js +1 -1
  455. package/out/tools/fetch/wikidata-subvenue.js.map +1 -1
  456. package/out/tools/golden/expand.d.ts +90 -0
  457. package/out/tools/golden/expand.d.ts.map +1 -0
  458. package/out/tools/golden/expand.js +406 -0
  459. package/out/tools/golden/expand.js.map +1 -0
  460. package/out/tools/golden/promote.d.ts +72 -0
  461. package/out/tools/golden/promote.d.ts.map +1 -0
  462. package/out/tools/golden/promote.js +243 -0
  463. package/out/tools/golden/promote.js.map +1 -0
  464. package/out/tools/golden/relabel-street/directory.d.ts +83 -0
  465. package/out/tools/golden/relabel-street/directory.d.ts.map +1 -0
  466. package/out/tools/golden/relabel-street/directory.js +216 -0
  467. package/out/tools/golden/relabel-street/directory.js.map +1 -0
  468. package/out/tools/golden/relabel-street/index.d.ts +9 -0
  469. package/out/tools/golden/relabel-street/index.d.ts.map +1 -0
  470. package/out/tools/golden/relabel-street/index.js +9 -0
  471. package/out/tools/golden/relabel-street/index.js.map +1 -0
  472. package/out/tools/golden/relabel-street/row.d.ts +121 -0
  473. package/out/tools/golden/relabel-street/row.d.ts.map +1 -0
  474. package/out/tools/golden/relabel-street/row.js +232 -0
  475. package/out/tools/golden/relabel-street/row.js.map +1 -0
  476. package/out/tools/index.d.ts +10 -10
  477. package/out/tools/index.d.ts.map +1 -1
  478. package/out/tools/index.js +10 -10
  479. package/out/tools/index.js.map +1 -1
  480. package/out/tools/ingest-csv.d.ts.map +1 -1
  481. package/out/tools/ingest-csv.js +5 -7
  482. package/out/tools/ingest-csv.js.map +1 -1
  483. package/out/tools/lint/slice/index.d.ts +93 -0
  484. package/out/tools/lint/slice/index.d.ts.map +1 -0
  485. package/out/tools/lint/slice/index.js +322 -0
  486. package/out/tools/lint/slice/index.js.map +1 -0
  487. package/out/tools/lint/slice/vocab.d.ts +90 -0
  488. package/out/tools/lint/slice/vocab.d.ts.map +1 -0
  489. package/out/tools/lint/slice/vocab.js +254 -0
  490. package/out/tools/lint/slice/vocab.js.map +1 -0
  491. package/out/tools/overlay-manifest.d.ts +56 -6
  492. package/out/tools/overlay-manifest.d.ts.map +1 -1
  493. package/out/tools/overlay-manifest.js +56 -15
  494. package/out/tools/overlay-manifest.js.map +1 -1
  495. package/out/tools/overture-subvenue.d.ts +1 -1
  496. package/out/tools/postcode-triples.d.ts +36 -0
  497. package/out/tools/postcode-triples.d.ts.map +1 -1
  498. package/out/tools/postcode-triples.js +91 -12
  499. package/out/tools/postcode-triples.js.map +1 -1
  500. package/out/tools/slice/kryptonite.d.ts +32 -0
  501. package/out/tools/slice/kryptonite.d.ts.map +1 -0
  502. package/out/tools/slice/kryptonite.js +99 -0
  503. package/out/tools/slice/kryptonite.js.map +1 -0
  504. package/out/tools/slice/translit.d.ts +51 -0
  505. package/out/tools/slice/translit.d.ts.map +1 -0
  506. package/out/tools/slice/translit.js +166 -0
  507. package/out/tools/slice/translit.js.map +1 -0
  508. package/out/tools/sub/venue/harvest.d.ts +100 -0
  509. package/out/tools/sub/venue/harvest.d.ts.map +1 -0
  510. package/out/tools/sub/venue/harvest.js +168 -0
  511. package/out/tools/sub/venue/harvest.js.map +1 -0
  512. package/out/tools/sub/venue/head-nouns.d.ts +50 -0
  513. package/out/tools/sub/venue/head-nouns.d.ts.map +1 -0
  514. package/out/tools/sub/venue/head-nouns.js.map +1 -0
  515. package/out/tools/sub/venue/lexicon.d.ts +156 -0
  516. package/out/tools/sub/venue/lexicon.d.ts.map +1 -0
  517. package/out/tools/sub/venue/lexicon.js +283 -0
  518. package/out/tools/sub/venue/lexicon.js.map +1 -0
  519. package/out/tools/sub/venue/promotions.d.ts +94 -0
  520. package/out/tools/sub/venue/promotions.d.ts.map +1 -0
  521. package/out/tools/sub/venue/promotions.js +266 -0
  522. package/out/tools/sub/venue/promotions.js.map +1 -0
  523. package/out/tools/sub/venue/surfaces.d.ts +45 -0
  524. package/out/tools/sub/venue/surfaces.d.ts.map +1 -0
  525. package/out/tools/sub/venue/surfaces.js.map +1 -0
  526. package/out/tools/sub/venue/table.d.ts +229 -0
  527. package/out/tools/sub/venue/table.d.ts.map +1 -0
  528. package/out/tools/sub/venue/table.js.map +1 -0
  529. package/out/tools/sub/venue/wikidata.d.ts +26 -0
  530. package/out/tools/sub/venue/wikidata.d.ts.map +1 -0
  531. package/out/tools/sub/venue/wikidata.js +62 -0
  532. package/out/tools/sub/venue/wikidata.js.map +1 -0
  533. package/out/tw/tools/fetch/gcis.d.ts +23 -0
  534. package/out/tw/tools/fetch/gcis.d.ts.map +1 -0
  535. package/out/tw/tools/fetch/gcis.js +135 -0
  536. package/out/tw/tools/fetch/gcis.js.map +1 -0
  537. package/out/types.d.ts +1 -1
  538. package/out/types.d.ts.map +1 -1
  539. package/out/us/adapters/fcc-bdc/adapter.d.ts.map +1 -0
  540. package/out/us/adapters/fcc-bdc/adapter.js +142 -0
  541. package/out/us/adapters/fcc-bdc/adapter.js.map +1 -0
  542. package/out/us/adapters/state/hi-schools/adapter.d.ts.map +1 -0
  543. package/out/us/adapters/state/hi-schools/adapter.js +136 -0
  544. package/out/us/adapters/state/hi-schools/adapter.js.map +1 -0
  545. package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -0
  546. package/out/us/adapters/state/hi-schools/workbook.js.map +1 -0
  547. package/out/us/adapters/state/ia-contractors/adapter.d.ts.map +1 -0
  548. package/out/us/adapters/state/ia-contractors/adapter.js +104 -0
  549. package/out/us/adapters/state/ia-contractors/adapter.js.map +1 -0
  550. package/out/us/adapters/state/ny-notaries/adapter.d.ts.map +1 -0
  551. package/out/us/adapters/state/ny-notaries/adapter.js +112 -0
  552. package/out/us/adapters/state/ny-notaries/adapter.js.map +1 -0
  553. package/out/us/adapters/state/tx-notaries/adapter.d.ts.map +1 -0
  554. package/out/us/adapters/state/tx-notaries/adapter.js +118 -0
  555. package/out/us/adapters/state/tx-notaries/adapter.js.map +1 -0
  556. package/out/us/adapters/tiger/adapter.d.ts +55 -0
  557. package/out/us/adapters/tiger/adapter.d.ts.map +1 -0
  558. package/out/us/adapters/tiger/adapter.js +197 -0
  559. package/out/us/adapters/tiger/adapter.js.map +1 -0
  560. package/out/us/adapters/tiger/street-decompose.d.ts.map +1 -0
  561. package/out/us/adapters/tiger/street-decompose.js.map +1 -0
  562. package/out/us/adapters/usgov/hrsa-fqhc/adapter.d.ts.map +1 -0
  563. package/out/us/adapters/usgov/hrsa-fqhc/adapter.js +121 -0
  564. package/out/us/adapters/usgov/hrsa-fqhc/adapter.js.map +1 -0
  565. package/out/us/adapters/usgov/imls-pls/adapter.d.ts.map +1 -0
  566. package/out/us/adapters/usgov/imls-pls/adapter.js +106 -0
  567. package/out/us/adapters/usgov/imls-pls/adapter.js.map +1 -0
  568. package/out/us/adapters/usgov/irs-bmf/adapter.d.ts.map +1 -0
  569. package/out/us/adapters/usgov/irs-bmf/adapter.js +116 -0
  570. package/out/us/adapters/usgov/irs-bmf/adapter.js.map +1 -0
  571. package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -0
  572. package/out/us/adapters/usgov/nad/adapter.js +239 -0
  573. package/out/us/adapters/usgov/nad/adapter.js.map +1 -0
  574. package/out/us/adapters/usgov/nppes/adapter.d.ts.map +1 -0
  575. package/out/us/adapters/usgov/nppes/adapter.js +110 -0
  576. package/out/us/adapters/usgov/nppes/adapter.js.map +1 -0
  577. package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.d.ts.map +1 -0
  578. package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +143 -0
  579. package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -0
  580. package/out/us/fips-state.d.ts +41 -0
  581. package/out/us/fips-state.d.ts.map +1 -0
  582. package/out/us/fips-state.js +100 -0
  583. package/out/us/fips-state.js.map +1 -0
  584. package/out/us/tools/fetch/hrsa.d.ts +18 -0
  585. package/out/us/tools/fetch/hrsa.d.ts.map +1 -0
  586. package/out/us/tools/fetch/hrsa.js +46 -0
  587. package/out/us/tools/fetch/hrsa.js.map +1 -0
  588. package/out/us/tools/fetch/imls-pls.d.ts +25 -0
  589. package/out/us/tools/fetch/imls-pls.d.ts.map +1 -0
  590. package/out/us/tools/fetch/imls-pls.js +108 -0
  591. package/out/us/tools/fetch/imls-pls.js.map +1 -0
  592. package/out/us/tools/fetch/nad.d.ts +70 -0
  593. package/out/us/tools/fetch/nad.d.ts.map +1 -0
  594. package/out/us/tools/fetch/nad.js +234 -0
  595. package/out/us/tools/fetch/nad.js.map +1 -0
  596. package/out/us/tools/fetch/nppes.d.ts +26 -0
  597. package/out/us/tools/fetch/nppes.d.ts.map +1 -0
  598. package/out/us/tools/fetch/nppes.js +126 -0
  599. package/out/us/tools/fetch/nppes.js.map +1 -0
  600. package/out/us/tools/fetch/state/hi-schools.d.ts +26 -0
  601. package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -0
  602. package/out/us/tools/fetch/state/hi-schools.js +112 -0
  603. package/out/us/tools/fetch/state/hi-schools.js.map +1 -0
  604. package/out/us/tools/fetch/state/sources.d.ts +26 -0
  605. package/out/us/tools/fetch/state/sources.d.ts.map +1 -0
  606. package/out/us/tools/fetch/state/sources.js +111 -0
  607. package/out/us/tools/fetch/state/sources.js.map +1 -0
  608. package/out/us/tools/fetch/tiger-full.d.ts +22 -0
  609. package/out/us/tools/fetch/tiger-full.d.ts.map +1 -0
  610. package/out/us/tools/fetch/tiger-full.js +217 -0
  611. package/out/us/tools/fetch/tiger-full.js.map +1 -0
  612. package/out/utils/align.d.ts +1 -1
  613. package/out/utils/align.d.ts.map +1 -1
  614. package/out/utils/align.js.map +1 -1
  615. package/out/utils/golden.d.ts +7 -3
  616. package/out/utils/golden.d.ts.map +1 -1
  617. package/out/utils/golden.js +12 -8
  618. package/out/utils/golden.js.map +1 -1
  619. package/out/utils/parquet.d.ts.map +1 -1
  620. package/out/utils/split.js.map +1 -1
  621. package/out/utils/wof-json.d.ts.map +1 -1
  622. package/out/utils/wof-json.js +7 -12
  623. package/out/utils/wof-json.js.map +1 -1
  624. package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -0
  625. package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -0
  626. package/package.json +196 -46
  627. package/lib/adapters/ban/adapter.ts +0 -190
  628. package/lib/adapters/fcc-bdc/adapter.ts +0 -174
  629. package/lib/adapters/geonames-postal/adapter.ts +0 -122
  630. package/lib/adapters/gnaf/adapter.ts +0 -149
  631. package/lib/adapters/gnaf/assemble.ts +0 -225
  632. package/lib/adapters/state-hi-schools/adapter.ts +0 -170
  633. package/lib/adapters/state-ia-contractors/adapter.ts +0 -144
  634. package/lib/adapters/state-ny-notaries/adapter.ts +0 -141
  635. package/lib/adapters/state-tx-notaries/adapter.ts +0 -157
  636. package/lib/adapters/tiger/adapter.ts +0 -252
  637. package/lib/adapters/usgov-hrsa-fqhc/adapter.ts +0 -167
  638. package/lib/adapters/usgov-imls-pls/adapter.ts +0 -143
  639. package/lib/adapters/usgov-irs-bmf/adapter.ts +0 -151
  640. package/lib/adapters/usgov-nad/adapter.ts +0 -334
  641. package/lib/adapters/usgov-nppes/adapter.ts +0 -151
  642. package/lib/adapters/usgov-samhsa-treatment-locator/adapter.ts +0 -193
  643. package/lib/adapters/wof-admin-json/adapter.ts +0 -231
  644. package/lib/adapters/wof-json-rows.ts +0 -129
  645. package/lib/adapters/wof-postalcode-json/adapter.ts +0 -176
  646. package/lib/name-prone-us-suffixes.ts +0 -12
  647. package/lib/recipes/bare-country.ts +0 -95
  648. package/lib/recipes/cz-pcfirst-preposition.ts +0 -111
  649. package/lib/recipes/fr-admin-split.ts +0 -242
  650. package/lib/recipes/fr-bare-street.ts +0 -170
  651. package/lib/recipes/fr-fragment.ts +0 -333
  652. package/lib/recipes/fr-lieudit.ts +0 -301
  653. package/lib/recipes/fr-order.ts +0 -245
  654. package/lib/recipes/german.ts +0 -359
  655. package/lib/recipes/locale.ts +0 -599
  656. package/lib/recipes/nl-postcode.ts +0 -115
  657. package/lib/recipes/no-fragment.ts +0 -219
  658. package/lib/recipes/no-street-led.ts +0 -141
  659. package/lib/recipes/no-street.ts +0 -90
  660. package/lib/recipes/po-box-cedex.ts +0 -976
  661. package/lib/recipes/po-box.ts +0 -123
  662. package/lib/recipes/si-bare-village.ts +0 -97
  663. package/lib/recipes/street-affix.ts +0 -804
  664. package/lib/recipes/street-bare.ts +0 -100
  665. package/lib/recipes/sub-venue-sources.ts +0 -877
  666. package/lib/recipes/sub-venue.ts +0 -997
  667. package/lib/synthesizers/german.ts +0 -260
  668. package/lib/tools/fetch/ban.ts +0 -289
  669. package/lib/tools/fetch/download.ts +0 -262
  670. package/lib/tools/fetch/geonames-dump.ts +0 -257
  671. package/lib/tools/fetch/geonames-postal.ts +0 -180
  672. package/lib/tools/fetch/hrsa.ts +0 -59
  673. package/lib/tools/fetch/imls-pls.ts +0 -149
  674. package/lib/tools/fetch/nad.ts +0 -341
  675. package/lib/tools/fetch/nppes.ts +0 -165
  676. package/lib/tools/fetch/state-hi-schools.ts +0 -161
  677. package/lib/tools/fetch/state-sources.ts +0 -146
  678. package/lib/tools/fetch/tiger-full.ts +0 -308
  679. package/lib/tools/golden-promote.ts +0 -337
  680. package/lib/tools/golden-relabel-street.ts +0 -683
  681. package/lib/tools/lint-slice-vocab.ts +0 -388
  682. package/lib/tools/sub-venue/harvest.ts +0 -234
  683. package/lib/tools/sub-venue/head-nouns.ts +0 -245
  684. package/lib/tools/sub-venue/surfaces.ts +0 -95
  685. package/lib/tools/sub-venue/table.ts +0 -281
  686. package/lib/tools/sub-venue/wikidata.ts +0 -85
  687. package/lib/tools/sub-venue-lexicon.ts +0 -429
  688. package/out/adapters/ban/adapter.d.ts.map +0 -1
  689. package/out/adapters/ban/adapter.js +0 -149
  690. package/out/adapters/ban/adapter.js.map +0 -1
  691. package/out/adapters/ban/street-decompose.d.ts.map +0 -1
  692. package/out/adapters/ban/street-decompose.js.map +0 -1
  693. package/out/adapters/fcc-bdc/adapter.d.ts.map +0 -1
  694. package/out/adapters/fcc-bdc/adapter.js +0 -144
  695. package/out/adapters/fcc-bdc/adapter.js.map +0 -1
  696. package/out/adapters/geonames-postal/adapter.d.ts.map +0 -1
  697. package/out/adapters/geonames-postal/adapter.js +0 -107
  698. package/out/adapters/geonames-postal/adapter.js.map +0 -1
  699. package/out/adapters/gnaf/adapter.d.ts.map +0 -1
  700. package/out/adapters/gnaf/adapter.js +0 -124
  701. package/out/adapters/gnaf/adapter.js.map +0 -1
  702. package/out/adapters/gnaf/assemble.d.ts +0 -65
  703. package/out/adapters/gnaf/assemble.d.ts.map +0 -1
  704. package/out/adapters/gnaf/assemble.js +0 -154
  705. package/out/adapters/gnaf/assemble.js.map +0 -1
  706. package/out/adapters/state-hi-schools/adapter.d.ts.map +0 -1
  707. package/out/adapters/state-hi-schools/adapter.js +0 -145
  708. package/out/adapters/state-hi-schools/adapter.js.map +0 -1
  709. package/out/adapters/state-hi-schools/workbook.d.ts.map +0 -1
  710. package/out/adapters/state-hi-schools/workbook.js.map +0 -1
  711. package/out/adapters/state-ia-contractors/adapter.d.ts.map +0 -1
  712. package/out/adapters/state-ia-contractors/adapter.js +0 -111
  713. package/out/adapters/state-ia-contractors/adapter.js.map +0 -1
  714. package/out/adapters/state-ny-notaries/adapter.d.ts.map +0 -1
  715. package/out/adapters/state-ny-notaries/adapter.js +0 -119
  716. package/out/adapters/state-ny-notaries/adapter.js.map +0 -1
  717. package/out/adapters/state-tx-notaries/adapter.d.ts.map +0 -1
  718. package/out/adapters/state-tx-notaries/adapter.js +0 -125
  719. package/out/adapters/state-tx-notaries/adapter.js.map +0 -1
  720. package/out/adapters/tiger/adapter.d.ts +0 -55
  721. package/out/adapters/tiger/adapter.d.ts.map +0 -1
  722. package/out/adapters/tiger/adapter.js +0 -201
  723. package/out/adapters/tiger/adapter.js.map +0 -1
  724. package/out/adapters/tiger/street-decompose.d.ts.map +0 -1
  725. package/out/adapters/tiger/street-decompose.js.map +0 -1
  726. package/out/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +0 -1
  727. package/out/adapters/usgov-hrsa-fqhc/adapter.js +0 -132
  728. package/out/adapters/usgov-hrsa-fqhc/adapter.js.map +0 -1
  729. package/out/adapters/usgov-imls-pls/adapter.d.ts.map +0 -1
  730. package/out/adapters/usgov-imls-pls/adapter.js +0 -114
  731. package/out/adapters/usgov-imls-pls/adapter.js.map +0 -1
  732. package/out/adapters/usgov-irs-bmf/adapter.d.ts.map +0 -1
  733. package/out/adapters/usgov-irs-bmf/adapter.js +0 -120
  734. package/out/adapters/usgov-irs-bmf/adapter.js.map +0 -1
  735. package/out/adapters/usgov-nad/adapter.d.ts.map +0 -1
  736. package/out/adapters/usgov-nad/adapter.js +0 -249
  737. package/out/adapters/usgov-nad/adapter.js.map +0 -1
  738. package/out/adapters/usgov-nppes/adapter.d.ts.map +0 -1
  739. package/out/adapters/usgov-nppes/adapter.js +0 -118
  740. package/out/adapters/usgov-nppes/adapter.js.map +0 -1
  741. package/out/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +0 -1
  742. package/out/adapters/usgov-samhsa-treatment-locator/adapter.js +0 -154
  743. package/out/adapters/usgov-samhsa-treatment-locator/adapter.js.map +0 -1
  744. package/out/adapters/wof-admin-jp/adapter.d.ts.map +0 -1
  745. package/out/adapters/wof-admin-jp/adapter.js.map +0 -1
  746. package/out/adapters/wof-admin-json/adapter.d.ts +0 -79
  747. package/out/adapters/wof-admin-json/adapter.d.ts.map +0 -1
  748. package/out/adapters/wof-admin-json/adapter.js +0 -197
  749. package/out/adapters/wof-admin-json/adapter.js.map +0 -1
  750. package/out/adapters/wof-json-rows.d.ts +0 -62
  751. package/out/adapters/wof-json-rows.d.ts.map +0 -1
  752. package/out/adapters/wof-json-rows.js +0 -83
  753. package/out/adapters/wof-json-rows.js.map +0 -1
  754. package/out/adapters/wof-postalcode-json/adapter.d.ts +0 -65
  755. package/out/adapters/wof-postalcode-json/adapter.d.ts.map +0 -1
  756. package/out/adapters/wof-postalcode-json/adapter.js +0 -151
  757. package/out/adapters/wof-postalcode-json/adapter.js.map +0 -1
  758. package/out/codex/us-fips-state.d.ts +0 -41
  759. package/out/codex/us-fips-state.d.ts.map +0 -1
  760. package/out/codex/us-fips-state.js +0 -100
  761. package/out/codex/us-fips-state.js.map +0 -1
  762. package/out/name-prone-us-suffixes.d.ts +0 -12
  763. package/out/name-prone-us-suffixes.d.ts.map +0 -1
  764. package/out/name-prone-us-suffixes.js +0 -12
  765. package/out/name-prone-us-suffixes.js.map +0 -1
  766. package/out/recipes/bare-country.d.ts +0 -25
  767. package/out/recipes/bare-country.d.ts.map +0 -1
  768. package/out/recipes/bare-country.js +0 -87
  769. package/out/recipes/bare-country.js.map +0 -1
  770. package/out/recipes/cn-organizational-units.d.ts +0 -36
  771. package/out/recipes/cn-organizational-units.d.ts.map +0 -1
  772. package/out/recipes/cn-organizational-units.js +0 -159
  773. package/out/recipes/cn-organizational-units.js.map +0 -1
  774. package/out/recipes/cz-pcfirst-preposition.d.ts +0 -22
  775. package/out/recipes/cz-pcfirst-preposition.d.ts.map +0 -1
  776. package/out/recipes/cz-pcfirst-preposition.js +0 -100
  777. package/out/recipes/cz-pcfirst-preposition.js.map +0 -1
  778. package/out/recipes/fr-admin-split.d.ts +0 -38
  779. package/out/recipes/fr-admin-split.d.ts.map +0 -1
  780. package/out/recipes/fr-admin-split.js +0 -192
  781. package/out/recipes/fr-admin-split.js.map +0 -1
  782. package/out/recipes/fr-bare-street.d.ts +0 -28
  783. package/out/recipes/fr-bare-street.d.ts.map +0 -1
  784. package/out/recipes/fr-bare-street.js +0 -143
  785. package/out/recipes/fr-bare-street.js.map +0 -1
  786. package/out/recipes/fr-fragment.d.ts +0 -77
  787. package/out/recipes/fr-fragment.d.ts.map +0 -1
  788. package/out/recipes/fr-fragment.js +0 -276
  789. package/out/recipes/fr-fragment.js.map +0 -1
  790. package/out/recipes/fr-lieudit.d.ts +0 -32
  791. package/out/recipes/fr-lieudit.d.ts.map +0 -1
  792. package/out/recipes/fr-lieudit.js +0 -227
  793. package/out/recipes/fr-lieudit.js.map +0 -1
  794. package/out/recipes/fr-order.d.ts +0 -38
  795. package/out/recipes/fr-order.d.ts.map +0 -1
  796. package/out/recipes/fr-order.js +0 -197
  797. package/out/recipes/fr-order.js.map +0 -1
  798. package/out/recipes/german.d.ts +0 -44
  799. package/out/recipes/german.d.ts.map +0 -1
  800. package/out/recipes/german.js +0 -288
  801. package/out/recipes/german.js.map +0 -1
  802. package/out/recipes/locale.d.ts +0 -136
  803. package/out/recipes/locale.d.ts.map +0 -1
  804. package/out/recipes/locale.js +0 -460
  805. package/out/recipes/locale.js.map +0 -1
  806. package/out/recipes/nl-postcode.d.ts +0 -26
  807. package/out/recipes/nl-postcode.d.ts.map +0 -1
  808. package/out/recipes/nl-postcode.js +0 -102
  809. package/out/recipes/nl-postcode.js.map +0 -1
  810. package/out/recipes/no-fragment.d.ts +0 -43
  811. package/out/recipes/no-fragment.d.ts.map +0 -1
  812. package/out/recipes/no-fragment.js +0 -182
  813. package/out/recipes/no-fragment.js.map +0 -1
  814. package/out/recipes/no-street-led.d.ts +0 -24
  815. package/out/recipes/no-street-led.d.ts.map +0 -1
  816. package/out/recipes/no-street-led.js +0 -113
  817. package/out/recipes/no-street-led.js.map +0 -1
  818. package/out/recipes/no-street.d.ts +0 -18
  819. package/out/recipes/no-street.d.ts.map +0 -1
  820. package/out/recipes/no-street.js +0 -71
  821. package/out/recipes/no-street.js.map +0 -1
  822. package/out/recipes/po-box-cedex.d.ts +0 -38
  823. package/out/recipes/po-box-cedex.d.ts.map +0 -1
  824. package/out/recipes/po-box-cedex.js +0 -767
  825. package/out/recipes/po-box-cedex.js.map +0 -1
  826. package/out/recipes/po-box.d.ts +0 -17
  827. package/out/recipes/po-box.d.ts.map +0 -1
  828. package/out/recipes/po-box.js +0 -100
  829. package/out/recipes/po-box.js.map +0 -1
  830. package/out/recipes/reviewed-postcode-tail.d.ts.map +0 -1
  831. package/out/recipes/reviewed-postcode-tail.js.map +0 -1
  832. package/out/recipes/si-bare-village.d.ts +0 -32
  833. package/out/recipes/si-bare-village.d.ts.map +0 -1
  834. package/out/recipes/si-bare-village.js +0 -88
  835. package/out/recipes/si-bare-village.js.map +0 -1
  836. package/out/recipes/street-affix.d.ts +0 -89
  837. package/out/recipes/street-affix.d.ts.map +0 -1
  838. package/out/recipes/street-affix.js +0 -563
  839. package/out/recipes/street-affix.js.map +0 -1
  840. package/out/recipes/street-bare.d.ts +0 -25
  841. package/out/recipes/street-bare.d.ts.map +0 -1
  842. package/out/recipes/street-bare.js +0 -86
  843. package/out/recipes/street-bare.js.map +0 -1
  844. package/out/recipes/sub-venue-sources.d.ts +0 -229
  845. package/out/recipes/sub-venue-sources.d.ts.map +0 -1
  846. package/out/recipes/sub-venue-sources.js +0 -653
  847. package/out/recipes/sub-venue-sources.js.map +0 -1
  848. package/out/recipes/sub-venue.d.ts +0 -269
  849. package/out/recipes/sub-venue.d.ts.map +0 -1
  850. package/out/recipes/sub-venue.js +0 -725
  851. package/out/recipes/sub-venue.js.map +0 -1
  852. package/out/synthesizers/german.d.ts +0 -115
  853. package/out/synthesizers/german.d.ts.map +0 -1
  854. package/out/synthesizers/german.js +0 -149
  855. package/out/synthesizers/german.js.map +0 -1
  856. package/out/tools/fetch/ban.d.ts +0 -31
  857. package/out/tools/fetch/ban.d.ts.map +0 -1
  858. package/out/tools/fetch/ban.js +0 -239
  859. package/out/tools/fetch/ban.js.map +0 -1
  860. package/out/tools/fetch/download.d.ts +0 -117
  861. package/out/tools/fetch/download.d.ts.map +0 -1
  862. package/out/tools/fetch/download.js +0 -160
  863. package/out/tools/fetch/download.js.map +0 -1
  864. package/out/tools/fetch/geonames-dump.d.ts +0 -91
  865. package/out/tools/fetch/geonames-dump.d.ts.map +0 -1
  866. package/out/tools/fetch/geonames-dump.js +0 -171
  867. package/out/tools/fetch/geonames-dump.js.map +0 -1
  868. package/out/tools/fetch/geonames-postal.d.ts +0 -76
  869. package/out/tools/fetch/geonames-postal.d.ts.map +0 -1
  870. package/out/tools/fetch/geonames-postal.js +0 -126
  871. package/out/tools/fetch/geonames-postal.js.map +0 -1
  872. package/out/tools/fetch/hrsa.d.ts +0 -18
  873. package/out/tools/fetch/hrsa.d.ts.map +0 -1
  874. package/out/tools/fetch/hrsa.js +0 -46
  875. package/out/tools/fetch/hrsa.js.map +0 -1
  876. package/out/tools/fetch/imls-pls.d.ts +0 -25
  877. package/out/tools/fetch/imls-pls.d.ts.map +0 -1
  878. package/out/tools/fetch/imls-pls.js +0 -108
  879. package/out/tools/fetch/imls-pls.js.map +0 -1
  880. package/out/tools/fetch/nad.d.ts +0 -70
  881. package/out/tools/fetch/nad.d.ts.map +0 -1
  882. package/out/tools/fetch/nad.js +0 -234
  883. package/out/tools/fetch/nad.js.map +0 -1
  884. package/out/tools/fetch/nppes.d.ts +0 -26
  885. package/out/tools/fetch/nppes.d.ts.map +0 -1
  886. package/out/tools/fetch/nppes.js +0 -126
  887. package/out/tools/fetch/nppes.js.map +0 -1
  888. package/out/tools/fetch/state-hi-schools.d.ts +0 -26
  889. package/out/tools/fetch/state-hi-schools.d.ts.map +0 -1
  890. package/out/tools/fetch/state-hi-schools.js +0 -112
  891. package/out/tools/fetch/state-hi-schools.js.map +0 -1
  892. package/out/tools/fetch/state-sources.d.ts +0 -26
  893. package/out/tools/fetch/state-sources.d.ts.map +0 -1
  894. package/out/tools/fetch/state-sources.js +0 -111
  895. package/out/tools/fetch/state-sources.js.map +0 -1
  896. package/out/tools/fetch/tiger-full.d.ts +0 -22
  897. package/out/tools/fetch/tiger-full.d.ts.map +0 -1
  898. package/out/tools/fetch/tiger-full.js +0 -217
  899. package/out/tools/fetch/tiger-full.js.map +0 -1
  900. package/out/tools/golden-expand.d.ts +0 -90
  901. package/out/tools/golden-expand.d.ts.map +0 -1
  902. package/out/tools/golden-expand.js +0 -406
  903. package/out/tools/golden-expand.js.map +0 -1
  904. package/out/tools/golden-promote.d.ts +0 -72
  905. package/out/tools/golden-promote.d.ts.map +0 -1
  906. package/out/tools/golden-promote.js +0 -240
  907. package/out/tools/golden-promote.js.map +0 -1
  908. package/out/tools/golden-relabel-street.d.ts +0 -196
  909. package/out/tools/golden-relabel-street.d.ts.map +0 -1
  910. package/out/tools/golden-relabel-street.js +0 -440
  911. package/out/tools/golden-relabel-street.js.map +0 -1
  912. package/out/tools/lint-slice-vocab.d.ts +0 -90
  913. package/out/tools/lint-slice-vocab.d.ts.map +0 -1
  914. package/out/tools/lint-slice-vocab.js +0 -267
  915. package/out/tools/lint-slice-vocab.js.map +0 -1
  916. package/out/tools/lint-slice.d.ts +0 -93
  917. package/out/tools/lint-slice.d.ts.map +0 -1
  918. package/out/tools/lint-slice.js +0 -322
  919. package/out/tools/lint-slice.js.map +0 -1
  920. package/out/tools/slice-kryptonite.d.ts +0 -32
  921. package/out/tools/slice-kryptonite.d.ts.map +0 -1
  922. package/out/tools/slice-kryptonite.js +0 -99
  923. package/out/tools/slice-kryptonite.js.map +0 -1
  924. package/out/tools/slice-translit.d.ts +0 -51
  925. package/out/tools/slice-translit.d.ts.map +0 -1
  926. package/out/tools/slice-translit.js +0 -166
  927. package/out/tools/slice-translit.js.map +0 -1
  928. package/out/tools/sub-venue/harvest.d.ts +0 -100
  929. package/out/tools/sub-venue/harvest.d.ts.map +0 -1
  930. package/out/tools/sub-venue/harvest.js +0 -168
  931. package/out/tools/sub-venue/harvest.js.map +0 -1
  932. package/out/tools/sub-venue/head-nouns.d.ts +0 -50
  933. package/out/tools/sub-venue/head-nouns.d.ts.map +0 -1
  934. package/out/tools/sub-venue/head-nouns.js.map +0 -1
  935. package/out/tools/sub-venue/surfaces.d.ts +0 -45
  936. package/out/tools/sub-venue/surfaces.d.ts.map +0 -1
  937. package/out/tools/sub-venue/surfaces.js.map +0 -1
  938. package/out/tools/sub-venue/table.d.ts +0 -229
  939. package/out/tools/sub-venue/table.d.ts.map +0 -1
  940. package/out/tools/sub-venue/table.js.map +0 -1
  941. package/out/tools/sub-venue/wikidata.d.ts +0 -26
  942. package/out/tools/sub-venue/wikidata.d.ts.map +0 -1
  943. package/out/tools/sub-venue/wikidata.js +0 -62
  944. package/out/tools/sub-venue/wikidata.js.map +0 -1
  945. package/out/tools/sub-venue-lexicon.d.ts +0 -156
  946. package/out/tools/sub-venue-lexicon.d.ts.map +0 -1
  947. package/out/tools/sub-venue-lexicon.js +0 -283
  948. package/out/tools/sub-venue-lexicon.js.map +0 -1
  949. package/out/tools/sub-venue-promotions.d.ts +0 -94
  950. package/out/tools/sub-venue-promotions.d.ts.map +0 -1
  951. package/out/tools/sub-venue-promotions.js +0 -266
  952. package/out/tools/sub-venue-promotions.js.map +0 -1
  953. /package/lib/{recipes/cn-organizational-units.ts → cn/recipes/organizational-units.ts} +0 -0
  954. /package/lib/{adapters → fr/adapters}/ban/street-decompose.ts +0 -0
  955. /package/lib/{adapters/wof-admin-jp → jp/adapters/wof-admin}/adapter.ts +0 -0
  956. /package/lib/tools/{golden-expand.ts → golden/expand.ts} +0 -0
  957. /package/lib/tools/{lint-slice.ts → lint/slice/index.ts} +0 -0
  958. /package/lib/tools/{slice-kryptonite.ts → slice/kryptonite.ts} +0 -0
  959. /package/lib/tools/{slice-translit.ts → slice/translit.ts} +0 -0
  960. /package/lib/tools/{sub-venue-promotions.ts → sub/venue/promotions.ts} +0 -0
  961. /package/lib/{adapters/state-hi-schools → us/adapters/state/hi-schools}/workbook.ts +0 -0
  962. /package/lib/{adapters → us/adapters}/tiger/street-decompose.ts +0 -0
  963. /package/lib/{codex/us-fips-state.ts → us/fips-state.ts} +0 -0
  964. /package/lib/{recipes → ve/recipes}/reviewed-postcode-tail.ts +0 -0
  965. /package/out/adapters/{geonames-postal → geonames/postal}/adapter.d.ts +0 -0
  966. /package/out/{adapters → au/adapters}/gnaf/adapter.d.ts +0 -0
  967. /package/out/{adapters → fr/adapters}/ban/adapter.d.ts +0 -0
  968. /package/out/{adapters → fr/adapters}/ban/street-decompose.d.ts +0 -0
  969. /package/out/{adapters → fr/adapters}/ban/street-decompose.js +0 -0
  970. /package/out/{adapters/wof-admin-jp → jp/adapters/wof-admin}/adapter.d.ts +0 -0
  971. /package/out/{adapters/wof-admin-jp → jp/adapters/wof-admin}/adapter.js +0 -0
  972. /package/out/tools/{sub-venue → sub/venue}/head-nouns.js +0 -0
  973. /package/out/tools/{sub-venue → sub/venue}/surfaces.js +0 -0
  974. /package/out/tools/{sub-venue → sub/venue}/table.js +0 -0
  975. /package/out/{adapters → us/adapters}/fcc-bdc/adapter.d.ts +0 -0
  976. /package/out/{adapters/state-hi-schools → us/adapters/state/hi-schools}/adapter.d.ts +0 -0
  977. /package/out/{adapters/state-hi-schools → us/adapters/state/hi-schools}/workbook.d.ts +0 -0
  978. /package/out/{adapters/state-hi-schools → us/adapters/state/hi-schools}/workbook.js +0 -0
  979. /package/out/{adapters/state-ia-contractors → us/adapters/state/ia-contractors}/adapter.d.ts +0 -0
  980. /package/out/{adapters/state-ny-notaries → us/adapters/state/ny-notaries}/adapter.d.ts +0 -0
  981. /package/out/{adapters/state-tx-notaries → us/adapters/state/tx-notaries}/adapter.d.ts +0 -0
  982. /package/out/{adapters → us/adapters}/tiger/street-decompose.d.ts +0 -0
  983. /package/out/{adapters → us/adapters}/tiger/street-decompose.js +0 -0
  984. /package/out/{adapters/usgov-hrsa-fqhc → us/adapters/usgov/hrsa-fqhc}/adapter.d.ts +0 -0
  985. /package/out/{adapters/usgov-imls-pls → us/adapters/usgov/imls-pls}/adapter.d.ts +0 -0
  986. /package/out/{adapters/usgov-irs-bmf → us/adapters/usgov/irs-bmf}/adapter.d.ts +0 -0
  987. /package/out/{adapters/usgov-nad → us/adapters/usgov/nad}/adapter.d.ts +0 -0
  988. /package/out/{adapters/usgov-nppes → us/adapters/usgov/nppes}/adapter.d.ts +0 -0
  989. /package/out/{adapters/usgov-samhsa-treatment-locator → us/adapters/usgov/samhsa-treatment-locator}/adapter.d.ts +0 -0
  990. /package/out/{recipes → ve/recipes}/reviewed-postcode-tail.d.ts +0 -0
  991. /package/out/{recipes → ve/recipes}/reviewed-postcode-tail.js +0 -0
@@ -0,0 +1,254 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
7
+ *
8
+ * Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
9
+ * byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
10
+ * `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
11
+ * (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
12
+ * the Python default.
13
+ *
14
+ * The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
15
+ * something else, or training gets conflicting gradients on the same token and the minority (the
16
+ * slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
17
+ *
18
+ * WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
19
+ * three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
20
+ * street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
21
+ *
22
+ * 1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
23
+ * retry both false-flagged FR cities as "street" from US street-contexts).
24
+ * 2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
25
+ * dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
26
+ * its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
27
+ * country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
28
+ * (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
29
+ * context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
30
+ * EXPECTED — the loader's affix-relabel handles it; weigh those separately.
31
+ *
32
+ * Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
33
+ * [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
34
+ * 50]
35
+ */
36
+ import { dataRootPath } from "@mailwoman/core/data-root";
37
+ import { tryParsingJSON } from "@mailwoman/core/json";
38
+ import { pyRound } from "@mailwoman/core/numeric";
39
+ import { join } from "path-ts";
40
+ import { Globerator } from "spliterator/node/fs";
41
+ import { connectDuckDB } from "#utils/parquet";
42
+ /**
43
+ * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
44
+ */
45
+ function stripBIO(label) {
46
+ const head = label.slice(0, 2);
47
+ return head === "B-" || head === "I-" ? label.slice(2) : label;
48
+ }
49
+ /**
50
+ * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
51
+ * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
52
+ * corpora actually contain.
53
+ */
54
+ function isDigit(token) {
55
+ return token.length > 0 && /^\p{Nd}+$/u.test(token);
56
+ }
57
+ /**
58
+ * Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
59
+ */
60
+ function pct(frac) {
61
+ return `${pyRound(frac * 100)}%`;
62
+ }
63
+ /**
64
+ * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
65
+ * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
66
+ * print.
67
+ */
68
+ function formatPyFloat(n) {
69
+ return Number.isInteger(n) ? n.toFixed(1) : String(n);
70
+ }
71
+ /**
72
+ * Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
73
+ */
74
+ function pad(value, width) {
75
+ return value.padEnd(width);
76
+ }
77
+ /**
78
+ * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
79
+ * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
80
+ */
81
+ function dominant(counter) {
82
+ let total = 0;
83
+ let bestTag = "";
84
+ let bestCount = -1;
85
+ for (const [tag, count] of counter) {
86
+ total += count;
87
+ if (count > bestCount) {
88
+ bestCount = count;
89
+ bestTag = tag;
90
+ }
91
+ }
92
+ if (total === 0)
93
+ return ["", 0, 0];
94
+ return [bestTag, total, bestCount / total];
95
+ }
96
+ /**
97
+ * Bump a (key -> count) tally, creating the inner counter on first sight.
98
+ */
99
+ function bump(table, key, sub) {
100
+ let counter = table.get(key);
101
+ if (!counter) {
102
+ counter = new Map();
103
+ table.set(key, counter);
104
+ }
105
+ counter.set(sub, (counter.get(sub) ?? 0) + 1);
106
+ }
107
+ /**
108
+ * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
109
+ * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
110
+ * here.
111
+ */
112
+ async function readRows(con, path) {
113
+ const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
114
+ const raw = result.getRowObjects();
115
+ const rows = [];
116
+ for (const r of raw) {
117
+ const tokens = tryParsingJSON(String(r.tokens));
118
+ const labels = tryParsingJSON(String(r.labels));
119
+ if (!Array.isArray(tokens) || !Array.isArray(labels))
120
+ continue;
121
+ rows.push({
122
+ tokens: tokens,
123
+ labels: labels,
124
+ country: r.country == null ? null : String(r.country),
125
+ });
126
+ }
127
+ return rows;
128
+ }
129
+ /**
130
+ * Read just the first row's `source` value — used to group base parts for a proportional slice.
131
+ */
132
+ async function readSource(con, path) {
133
+ const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
134
+ const rows = result.getRowObjects();
135
+ return rows.length ? String(rows[0].source) : "";
136
+ }
137
+ /**
138
+ * Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
139
+ */
140
+ export async function lintSliceVocab(options) {
141
+ const baseVersion = options.baseVersion ?? "v0.5.0";
142
+ const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
143
+ const threshold = options.threshold ?? 0.7;
144
+ const minCount = options.minCount ?? 50;
145
+ const fraction = options.fraction ?? 1;
146
+ const con = await connectDuckDB();
147
+ // 1. the slice's own (token -> dominant tag) + the COUNTRIES it uses each token in
148
+ const sliceRows = await readRows(con, options.slice);
149
+ const sliceTags = new Map();
150
+ const sliceCountries = new Map();
151
+ for (const { tokens, labels, country } of sliceRows) {
152
+ const n = Math.min(tokens.length, labels.length);
153
+ for (let i = 0; i < n; i++) {
154
+ const w = tokens[i];
155
+ const l = labels[i];
156
+ if (isDigit(w))
157
+ continue; // numbers are context-determined (house_number/postcode), not lexical vocab
158
+ bump(sliceTags, w, stripBIO(l));
159
+ let set = sliceCountries.get(w);
160
+ if (!set) {
161
+ set = new Set();
162
+ sliceCountries.set(w, set);
163
+ }
164
+ set.add(country);
165
+ }
166
+ }
167
+ const sliceVocab = new Set(sliceTags.keys());
168
+ console.log(`slice: ${sliceRows.length} rows, ${sliceVocab.size} unique tokens`);
169
+ // 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
170
+ const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train");
171
+ let parts = (await Globerator.from("*.parquet", {
172
+ cwd: trainDir,
173
+ absolute: true,
174
+ }).toArray()).toSorted();
175
+ if (!parts.length) {
176
+ throw new Error("no base parts found");
177
+ }
178
+ if (fraction < 1) {
179
+ const bysrc = new Map();
180
+ for (const p of parts) {
181
+ const src = await readSource(con, p);
182
+ let list = bysrc.get(src);
183
+ if (!list) {
184
+ list = [];
185
+ bysrc.set(src, list);
186
+ }
187
+ list.push(p);
188
+ }
189
+ const sliced = [];
190
+ for (const ps of bysrc.values()) {
191
+ const take = Math.max(2, pyRound(ps.length * fraction));
192
+ for (const p of ps.slice(0, take)) {
193
+ sliced.push(p);
194
+ }
195
+ }
196
+ parts = sliced;
197
+ }
198
+ console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${formatPyFloat(fraction)}), COUNTRY-scoped`);
199
+ // 3. tally each slice token's base tag, SCOPED to the country the slice uses it in
200
+ const baseTags = new Map();
201
+ for (let i = 0; i < parts.length; i++) {
202
+ const rows = await readRows(con, parts[i]);
203
+ for (const { tokens, labels, country } of rows) {
204
+ const n = Math.min(tokens.length, labels.length);
205
+ for (let j = 0; j < n; j++) {
206
+ const w = tokens[j];
207
+ if (sliceVocab.has(w) && sliceCountries.get(w).has(country)) {
208
+ bump(baseTags, w, stripBIO(labels[j]));
209
+ }
210
+ }
211
+ }
212
+ if ((i + 1) % 100 === 0) {
213
+ console.log(` ...${i + 1}/${parts.length} parts`);
214
+ }
215
+ }
216
+ // 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
217
+ const flagged = [];
218
+ const affix = [];
219
+ for (const w of sliceVocab) {
220
+ const [sTag] = dominant(sliceTags.get(w));
221
+ const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
222
+ if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
223
+ continue;
224
+ const row = [w, sTag, bTag, bFrac, bTotal];
225
+ if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
226
+ affix.push(row);
227
+ }
228
+ else {
229
+ flagged.push(row);
230
+ }
231
+ }
232
+ const sections = [
233
+ ["CONTRADICTION", flagged],
234
+ ["affix-split (EXPECTED — affix-relabel handles)", affix],
235
+ ];
236
+ for (const [label, rows] of sections) {
237
+ if (!rows.length)
238
+ continue;
239
+ rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
240
+ console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
241
+ for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
242
+ console.log(` ${pad(w, 18)} slice=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
243
+ }
244
+ }
245
+ if (!flagged.length) {
246
+ console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — slice base-consistent`);
247
+ }
248
+ return {
249
+ errors: flagged.length,
250
+ warnings: affix.length,
251
+ findings: { contradictions: flagged, affixSplits: affix },
252
+ };
253
+ }
254
+ //# sourceMappingURL=vocab.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"vocab.js","sourceRoot":"","sources":["../../../../lib/tools/lint/slice/vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAWrE;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,CAAS;IAC/B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAoDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,GAAG,GAAG,MAAM,aAAa,EAAE,CAAA;IAEjC,mFAAmF;IACnF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE5D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ,CAAC,4EAA4E;YACrG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC3B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAE5C,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,UAAU,UAAU,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAEhF,gGAAgG;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAE9E,IAAI,KAAK,GAAG,CACX,MAAM,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE;QAClC,GAAG,EAAE,QAAQ;QACb,QAAQ,EAAE,IAAI;KACd,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC,QAAQ,EAAE,CAAA;IAEZ,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QAED,KAAK,GAAG,MAAM,CAAA;IACf,CAAC;IAED,OAAO,CAAC,GAAG,CACV,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAC3G,CAAA;IAED,mFAAmF;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAkB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqC;QAClD,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,2BAA2B,CACvH,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}
@@ -4,10 +4,12 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
6
  * Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
7
- * ADDS one slice parquet to a base corpus, keeping every base slice VERBATIM (pure overlay ADD),
8
- * and re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet +
9
- * --source so it works for any overlay slice (the fr-admin-split one is the original; #148's
10
- * overture-multilocale is the second user).
7
+ * ADDS slice parquets to a base corpus, keeping every base slice VERBATIM (pure overlay ADD), and
8
+ * re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet + --source, one
9
+ * label per parquet, so it works for any overlay (the fr-admin-split one is the original; #148's
10
+ * overture-multilocale is the second user; v0.29.0's eight target-family slices are why it takes a
11
+ * set rather than one — chaining eight single-slice overlays would leave seven dead directories and
12
+ * an eight-deep base chain for what is one version).
11
13
  *
12
14
  * Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
13
15
  * read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
@@ -22,13 +24,60 @@
22
24
  *
23
25
  * # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
24
26
  */
27
+ type ManifestSlice = Record<string, unknown> & {
28
+ path: string;
29
+ source?: string;
30
+ };
31
+ /**
32
+ * The two keys a manifest written before the 2026-09-01 vocabulary rename uses for its slice list and its size. Every
33
+ * corpus built before that date carries them, here and on the Modal volume, and a built corpus is an immutable artifact
34
+ * — so a reader accepts either spelling and a writer emits only the current one. Spelled by concatenation because the
35
+ * word is banned in this tree and the ratchet's baseline is zero.
36
+ */
37
+ declare const LEGACY_SLICES_KEY: "shards";
38
+ declare const LEGACY_ROWS_PER_SLICE_KEY: "rows_per_shard";
39
+ interface BaseManifest {
40
+ corpus_version?: string;
41
+ schema: unknown;
42
+ rows_per_slice?: unknown;
43
+ row_group_size: unknown;
44
+ slices?: ManifestSlice[];
45
+ counts: {
46
+ train: number;
47
+ val: number;
48
+ test: number;
49
+ };
50
+ total_rows: number;
51
+ /**
52
+ * The pre-rename spellings, present on every corpus built before 2026-09-01.
53
+ */
54
+ [LEGACY_SLICES_KEY]?: ManifestSlice[];
55
+ [LEGACY_ROWS_PER_SLICE_KEY]?: unknown;
56
+ }
57
+ /**
58
+ * A base manifest's slice list under either key.
59
+ *
60
+ * The rename moved this reader and the trainer's to the new key without migrating the manifests, and the trainer
61
+ * measured the cost on 2026-09-09: `v0.28.0-reviewed-postcode-tail` declares 706 train slices under the old key and the
62
+ * loader resolved ONE. An overlay assembled from a base read as empty would carry no base slices at all.
63
+ */
64
+ export declare function baseManifestSlices(base: BaseManifest): ManifestSlice[];
65
+ /**
66
+ * One parquet to add, with the source label its rows carry.
67
+ */
68
+ export interface OverlaySlice {
69
+ parquet: string;
70
+ source: string;
71
+ }
25
72
  export interface OverlayManifestOptions {
26
73
  base: string;
27
74
  newDir: string;
28
75
  modalRoot: string;
29
76
  version: string;
30
- sliceParquet: string;
31
- source: string;
77
+ /**
78
+ * The parquets this overlay adds, in the order they should appear after the base's own.
79
+ */
80
+ slices: readonly OverlaySlice[];
32
81
  note: string;
33
82
  }
34
83
  /**
@@ -36,4 +85,5 @@ export interface OverlayManifestOptions {
36
85
  */
37
86
  export declare function rerootBaseSlicePath(path: string, baseManifestPath: string): string;
38
87
  export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void>;
88
+ export {};
39
89
  //# sourceMappingURL=overlay-manifest.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"overlay-manifest.d.ts","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAwDH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAalF;AAED,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwCzF"}
1
+ {"version":3,"file":"overlay-manifest.d.ts","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAsBH,KAAK,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAEhF;;;;;GAKG;AACH,QAAA,MAAM,iBAAiB,UAAyB,CAAA;AAChD,QAAA,MAAM,yBAAyB,kBAAiC,CAAA;AAEhE,UAAU,YAAY;IACrB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,cAAc,EAAE,OAAO,CAAA;IACvB,MAAM,CAAC,EAAE,aAAa,EAAE,CAAA;IACxB,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IACpD,UAAU,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,CAAC,iBAAiB,CAAC,CAAC,EAAE,aAAa,EAAE,CAAA;IACrC,CAAC,yBAAyB,CAAC,CAAC,EAAE,OAAO,CAAA;CACrC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,EAAE,CAQtE;AASD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACd;AA0BD,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,SAAS,YAAY,EAAE,CAAA;IAC/B,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAalF;AAED,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuDzF"}
@@ -4,10 +4,12 @@
4
4
  * @author Teffen Ellis, et al.
5
5
  *
6
6
  * Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
7
- * ADDS one slice parquet to a base corpus, keeping every base slice VERBATIM (pure overlay ADD),
8
- * and re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet +
9
- * --source so it works for any overlay slice (the fr-admin-split one is the original; #148's
10
- * overture-multilocale is the second user).
7
+ * ADDS slice parquets to a base corpus, keeping every base slice VERBATIM (pure overlay ADD), and
8
+ * re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet + --source, one
9
+ * label per parquet, so it works for any overlay (the fr-admin-split one is the original; #148's
10
+ * overture-multilocale is the second user; v0.29.0's eight target-family slices are why it takes a
11
+ * set rather than one — chaining eight single-slice overlays would leave seven dead directories and
12
+ * an eight-deep base chain for what is one version).
11
13
  *
12
14
  * Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
13
15
  * read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
@@ -27,6 +29,33 @@ import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
27
29
  import { sha256Hex } from "@mailwoman/core/hash";
28
30
  import { basename, dirname, join } from "path-ts";
29
31
  import { connectDuckDB, escapeSQLString } from "#utils/parquet";
32
+ /**
33
+ * The two keys a manifest written before the 2026-09-01 vocabulary rename uses for its slice list and its size. Every
34
+ * corpus built before that date carries them, here and on the Modal volume, and a built corpus is an immutable artifact
35
+ * — so a reader accepts either spelling and a writer emits only the current one. Spelled by concatenation because the
36
+ * word is banned in this tree and the ratchet's baseline is zero.
37
+ */
38
+ const LEGACY_SLICES_KEY = `sh${"ards"}`;
39
+ const LEGACY_ROWS_PER_SLICE_KEY = `rows_per_sh${"ard"}`;
40
+ /**
41
+ * A base manifest's slice list under either key.
42
+ *
43
+ * The rename moved this reader and the trainer's to the new key without migrating the manifests, and the trainer
44
+ * measured the cost on 2026-09-09: `v0.28.0-reviewed-postcode-tail` declares 706 train slices under the old key and the
45
+ * loader resolved ONE. An overlay assembled from a base read as empty would carry no base slices at all.
46
+ */
47
+ export function baseManifestSlices(base) {
48
+ const slices = base.slices?.length ? base.slices : base[LEGACY_SLICES_KEY];
49
+ if (slices?.length)
50
+ return slices;
51
+ throw new Error(`base manifest lists no slices under "slices" or the pre-rename key — refusing to assemble an overlay whose base would be empty`);
52
+ }
53
+ /**
54
+ * The base's rows-per-slice under either key, carried through to the new manifest verbatim.
55
+ */
56
+ function baseRowsPerSlice(base) {
57
+ return base.rows_per_slice ?? base[LEGACY_ROWS_PER_SLICE_KEY];
58
+ }
30
59
  async function descriptor(localPath, modalPath, split, source) {
31
60
  const db = await connectDuckDB();
32
61
  const result = await db.runAndReadAll(`SELECT source_id FROM read_parquet('${escapeSQLString(localPath)}')`);
@@ -59,32 +88,44 @@ export function rerootBaseSlicePath(path, baseManifestPath) {
59
88
  return path;
60
89
  }
61
90
  export async function assembleOverlayManifest(args) {
91
+ if (!args.slices.length)
92
+ throw new Error("an overlay must add at least one slice");
62
93
  const base = await readLocalJSONFile(args.base);
63
- if (base.slices.some((s) => s.source === args.source)) {
64
- console.log(`WARN: base already contains source '${args.source}' — is this the right base?`);
94
+ const baseSlices = baseManifestSlices(base);
95
+ for (const slice of args.slices) {
96
+ if (baseSlices.some((s) => s.source === slice.source)) {
97
+ console.log(`WARN: base already contains source '${slice.source}' — is this the right base?`);
98
+ }
65
99
  }
66
- const kept = base.slices.map((s) => ({ ...s, path: rerootBaseSlicePath(s.path, args.base) }));
67
- const newTrain = await descriptor(join(args.newDir, "train", args.sliceParquet), `${args.modalRoot}/train/${args.sliceParquet}`, "train", args.source);
100
+ const kept = baseSlices.map((s) => ({ ...s, path: rerootBaseSlicePath(s.path, args.base) }));
101
+ const added = [];
102
+ for (const slice of args.slices) {
103
+ added.push(await descriptor(join(args.newDir, "train", slice.parquet), `${args.modalRoot}/train/${slice.parquet}`, "train", slice.source));
104
+ }
105
+ const addedRows = added.reduce((total, slice) => total + slice.rows, 0);
106
+ const sources = args.slices.map((slice) => slice.source).join(", ");
68
107
  const manifest = {
69
108
  corpus_version: args.version,
70
109
  overlay_base: base.corpus_version ?? null,
71
- note: args.note || `${base.corpus_version} slices (all kept verbatim) + the ${args.source} slice. Pure overlay add.`,
110
+ note: args.note || `${base.corpus_version} slices (all kept verbatim) + ${sources}. Pure overlay add.`,
72
111
  schema: base.schema,
73
- rows_per_slice: base.rows_per_slice,
112
+ rows_per_slice: baseRowsPerSlice(base),
74
113
  row_group_size: base.row_group_size,
75
- slices: [...kept, newTrain],
114
+ slices: [...kept, ...added],
76
115
  counts: {
77
- train: base.counts.train + newTrain.rows,
116
+ train: base.counts.train + addedRows,
78
117
  val: base.counts.val,
79
118
  test: base.counts.test,
80
119
  },
81
- total_rows: base.total_rows + newTrain.rows,
120
+ total_rows: base.total_rows + addedRows,
82
121
  };
83
122
  const out = join(args.newDir, "MANIFEST.json");
84
123
  await writeLocalJSONFile(manifest, out);
85
124
  console.log(`wrote ${out}`);
86
- console.log(` slices: ${manifest.slices.length} (${kept.length} base kept, +1 ${args.source})`);
125
+ console.log(` slices: ${manifest.slices.length} (${kept.length} base kept, +${added.length} added)`);
87
126
  console.log(` counts: ${JSON.stringify(manifest.counts)} total: ${manifest.total_rows}`);
88
- console.log(` ${args.source} train: ${newTrain.rows} rows (${newTrain.bytes} bytes)`);
127
+ for (const slice of added) {
128
+ console.log(` ${slice.source} train: ${slice.rows} rows (${slice.bytes} bytes)`);
129
+ }
89
130
  }
90
131
  //# sourceMappingURL=overlay-manifest.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"overlay-manifest.js","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEjD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAyB/D,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,SAAiB,EACjB,KAAa,EACb,MAAc;IAEd,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,uCAAuC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC5G,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAA;IAErE,OAAO;QACN,KAAK;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,QAAQ;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACnD,eAAe,EAAE,IAAI,CAAC,CAAC,CAAE;QACzB,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QAC5B,MAAM;KACN,CAAA;AACF,CAAC;AAYD;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,gBAAwB;IACzE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAEzD,IAAI,cAAc,KAAK,CAAC,CAAC;QAAE,OAAO,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAEtE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,0BAA0B,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAA;QAE3G,OAAO,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAA4B;IACzE,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAe,IAAI,CAAC,IAAI,CAAC,CAAA;IAE7D,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,uCAAuC,IAAI,CAAC,MAAM,6BAA6B,CAAC,CAAA;IAC7F,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7F,MAAM,QAAQ,GAAG,MAAM,UAAU,CAChC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,EAC7C,GAAG,IAAI,CAAC,SAAS,UAAU,IAAI,CAAC,YAAY,EAAE,EAC9C,OAAO,EACP,IAAI,CAAC,MAAM,CACX,CAAA;IAED,MAAM,QAAQ,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,OAAO;QAC5B,YAAY,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACzC,IAAI,EACH,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,qCAAqC,IAAI,CAAC,MAAM,2BAA2B;QAC/G,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC;QAC3B,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI;YACxC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,IAAI;KAC3C,CAAA;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAC9C,MAAM,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAEvC,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,CAAA;IAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,kBAAkB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IAChG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAC1F,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,WAAW,QAAQ,CAAC,IAAI,UAAU,QAAQ,CAAC,KAAK,SAAS,CAAC,CAAA;AACvF,CAAC"}
1
+ {"version":3,"file":"overlay-manifest.js","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEjD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAiB/D;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG,KAAK,MAAM,EAAW,CAAA;AAChD,MAAM,yBAAyB,GAAG,cAAc,KAAK,EAAW,CAAA;AAiBhE;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAkB;IACpD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAE1E,IAAI,MAAM,EAAE,MAAM;QAAE,OAAO,MAAM,CAAA;IAEjC,MAAM,IAAI,KAAK,CACd,gIAAgI,CAChI,CAAA;AACF,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAkB;IAC3C,OAAO,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,yBAAyB,CAAC,CAAA;AAC9D,CAAC;AAUD,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,SAAiB,EACjB,KAAa,EACb,MAAc;IAEd,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,uCAAuC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC5G,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAA;IAErE,OAAO;QACN,KAAK;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,QAAQ;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACnD,eAAe,EAAE,IAAI,CAAC,CAAC,CAAE;QACzB,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QAC5B,MAAM;KACN,CAAA;AACF,CAAC;AAcD;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,gBAAwB;IACzE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAEzD,IAAI,cAAc,KAAK,CAAC,CAAC;QAAE,OAAO,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAEtE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,0BAA0B,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAA;QAE3G,OAAO,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAA4B;IACzE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAElF,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAe,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7D,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IAE3C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,uCAAuC,KAAK,CAAC,MAAM,6BAA6B,CAAC,CAAA;QAC9F,CAAC;IACF,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAC5F,MAAM,KAAK,GAAsB,EAAE,CAAA;IAEnC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CACT,MAAM,UAAU,CACf,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EACzC,GAAG,IAAI,CAAC,SAAS,UAAU,KAAK,CAAC,OAAO,EAAE,EAC1C,OAAO,EACP,KAAK,CAAC,MAAM,CACZ,CACD,CAAA;IACF,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IACvE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEnE,MAAM,QAAQ,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,OAAO;QAC5B,YAAY,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACzC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,iCAAiC,OAAO,qBAAqB;QACtG,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC;QACtC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;QAC3B,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,SAAS;YACpC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,SAAS;KACvC,CAAA;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAC9C,MAAM,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAEvC,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,CAAA;IAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAA;IACrG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAE1F,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,WAAW,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,KAAK,SAAS,CAAC,CAAA;IAClF,CAAC;AACF,CAAC"}
@@ -37,7 +37,7 @@
37
37
  * provenance lie, and under ODbL a consequential one. Hence the `source`/`region` options on that
38
38
  * function and the source-neutral {@link SubVenueHarvestRow} name.
39
39
  */
40
- import type { SubVenueHarvestRow } from "#tools/sub-venue-lexicon";
40
+ import type { SubVenueHarvestRow } from "#tools/sub/venue/lexicon";
41
41
  /**
42
42
  * Overture Places category → the designator its rows attest.
43
43
  *
@@ -103,6 +103,35 @@ export declare function applyLocalityQuota(triples: readonly PostcodeTriple[], q
103
103
  * one city's postcode list.
104
104
  */
105
105
  export declare function applyCountryBudget(triples: readonly PostcodeTriple[], budget: number | ReadonlyMap<string, number>): PostcodeTriple[];
106
+ /**
107
+ * A place's preferred names in the languages its addresses are written in, read from the gazetteer's `names` table.
108
+ */
109
+ interface PreferredNames {
110
+ /**
111
+ * Preferred names in the country's official languages, in the table's order.
112
+ */
113
+ official: readonly string[];
114
+ /**
115
+ * Preferred names in the region's co-official languages.
116
+ */
117
+ coOfficial: readonly string[];
118
+ }
119
+ /**
120
+ * The surfaces a REGION is written as: its preferred names in the official language(s), then in the region's
121
+ * co-official languages, then the gazetteer's own `spr.name` (the English exonym). Deduplicated, order kept.
122
+ *
123
+ * `spr.name` is the English-preferred, diacritic-stripped label, and reading it alone is the defect #1673 measured:
124
+ * 53,078 ES rows teaching `Balearic Islands` (2,872 `Andalusia`, 5,680 `Castile and Leon`) against 4 rows of `Illes
125
+ * Balears`, and every province with its accent gone (`Cordoba`, `Leon`). A Spanish user writes `Islas Baleares` or
126
+ * `Illes Balears`; the exonym stays as ONE surface among them because a user may write it too.
127
+ */
128
+ export declare function regionWrittenForms(sprName: string, names: PreferredNames): string[];
129
+ /**
130
+ * The surface a LOCALITY is written as: the official-language preferred name that is `spr.name` with its diacritics
131
+ * restored (`Cordoba` → `Córdoba`), else the first official-language preferred name, else `spr.name`. One surface, not
132
+ * a fan-out: the region carries the multiplicity and the locality would multiply it.
133
+ */
134
+ export declare function localityWrittenForm(sprName: string, names: PreferredNames): string;
106
135
  /**
107
136
  * Read triples out of `postalcode-intl.db` by following each postcode's `parent_id` into the admin gazetteer and that
108
137
  * place's ancestry to a region.
@@ -110,6 +139,12 @@ export declare function applyCountryBudget(triples: readonly PostcodeTriple[], b
110
139
  * A postcode whose parent does not resolve, or whose parent has no region ancestor, is DROPPED rather than emitted with
111
140
  * a blank — the recipe already skips a tuple with no region, and a blank here would hide how much of the source is
112
141
  * actually reachable.
142
+ *
143
+ * One triple per REGION SURFACE (see {@link regionWrittenForms}): a Balearic postcode yields `Islas Baleares`, `Illes
144
+ * Balears` and `Balearic Islands` rows, a Zamora one `Zamora` alone. The languages come from the codex — the country's
145
+ * official languages plus the province's co-official ones — and never from the names table's own language list, whose
146
+ * "preferred" name in a language not spoken in the province is often the parent community's (`Zamora` → `Castella i
147
+ * Lleó`).
113
148
  */
114
149
  export declare function readTriplesFromParentJoin(countries: readonly string[], options?: {
115
150
  postcodeDB?: string;
@@ -171,4 +206,5 @@ export declare function readTriplesFromGeonames(country: string, path: string, c
171
206
  * Resolve a GeoNames export path under the standard fetch out-root.
172
207
  */
173
208
  export declare function geonamesPostalPath(country: string, sourcesRoot?: string): string;
209
+ export {};
174
210
  //# sourceMappingURL=postcode-triples.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"postcode-triples.d.ts","sourceRoot":"","sources":["../../lib/tools/postcode-triples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAUH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAG1D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB,EAAE,iBAAiB,CAAA;CACpC;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAW,CAAC,MAAM,EAAE;IAAE,SAAS,EAAE,iBAAiB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAiBrG,CAAA;AAEF;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAA;AAExC;;;;;GAKG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,KAAK,GAAE,MAA+B,GACpC,cAAc,EAAE,CAelB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAC1C,cAAc,EAAE,CAkBlB;AAED;;;;;;;GAOG;AACH,wBAAsB,yBAAyB,CAC9C,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GACrD,OAAO,CAAC,cAAc,EAAE,CAAC,CA6C3B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,wBAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAsBpH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,uBAAuB,CAC5C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IAAE,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,cAAc,EAAE,CAAC,CA4C3B;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF"}
1
+ {"version":3,"file":"postcode-triples.d.ts","sourceRoot":"","sources":["../../lib/tools/postcode-triples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAYH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAG1D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB,EAAE,iBAAiB,CAAA;CACpC;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAW,CAAC,MAAM,EAAE;IAAE,SAAS,EAAE,iBAAiB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAiBrG,CAAA;AAEF;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAA;AAExC;;;;;GAKG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,KAAK,GAAE,MAA+B,GACpC,cAAc,EAAE,CAelB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAC1C,cAAc,EAAE,CAkBlB;AAED;;GAEG;AACH,UAAU,cAAc;IACvB;;OAEG;IACH,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B;;OAEG;IACH,UAAU,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7B;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,EAAE,CAYnF;AASD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,CAIlF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,yBAAyB,CAC9C,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GACrD,OAAO,CAAC,cAAc,EAAE,CAAC,CA0G3B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,wBAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAsBpH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,uBAAuB,CAC5C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IAAE,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,cAAc,EAAE,CAAC,CA4C3B;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF"}