@mailwoman/corpus 9.2.0 → 9.3.0

package/out/src/tools/sub-venue-promotions.js

@@ -1,266 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   The sub-venue curation ledger (#35 wave 2) — every decision taken about whether a designator
- *   surface may be used for parsing IN A GIVEN LOCALE, with the census that backs it.
- *
- *   This file is HAND-AUTHORED. `sub-venue-lexicon.ts` is a pure function of its sources; nothing in it
- *   can decide that Spanish `terminal` is safe and British `hall` is not, because that judgement is
- *   about the confounds a token has in a language, and a confound is a fact about the world rather than
- *   about the data. What the builder does is APPLY these decisions — {@link SUBVENUE_PROMOTIONS} is the
- *   only thing that ever sets `curated: true` on a machine-derived surface.
- *
- *   ── Why every decision is per-LOCALE ─────────────────────────────────────────────────────────────
- *   Wave 1 measured `hall` at 3,274 hits in the Great Britain extract and stopped there, which reads as
- *   a verdict on the word. It is not. Re-censused per region 2026-08-05, the same token is a British
- *   disaster and a German designator, and no single global flag can express that. The rule this file
- *   enforces: a promotion names a designator, a phrase, AND a locale, and it is earned by a count taken
- *   in that locale's own data.
- *
- *   ── What "real" and "confound" mean in the numbers below ─────────────────────────────────────────
- *   Each census counts the surface's occurrences in one region's extract and splits them two ways:
- *
- *   - **real** — the phrase sits in a name that is genuinely venue-interior structure, judged by the
- *     rule designator of the feature carrying it (`terminal`, `gate`, `campus`) and by the name taking a
- *     `<designator> <identifier>` or `<modifier> <designator>` shape.
- *   - **confound** — everything else, and the `confoundNote` says WHAT it is. A count with no note is
- *     not a census; "3,274 hits" told nobody that 3,205 of them were bus stops named after a village
- *     hall.
- *
- *   The instrument is `context` on {@link SubVenueSurface} plus the name-shape split; both are
- *   reproducible from the committed extracts, and `PROVENANCE.md` records the commands.
- *
- *   ── A rejection is a deliverable ─────────────────────────────────────────────────────────────────
- *   Rejections stay in this table forever. They are what stops the next contributor re-proposing
- *   `hall` for en-GB from the same Wikidata concept that proposed it the first time.
- */
-/**
- * The ledger. Sorted by designator, then locale, then phrase in the emitted artifact; source order here is grouped by
- * designator for reading.
- *
- * ── A rejection of a SHIPPED designator is ADVISORY ──────────────────────────────────────────────
- * `neural/venue-structure.ts` ships `wing`, `terminal`, `concourse` and six more as a FLAT English vocabulary with no
- * locale gate. Nothing in this table can un-ship them: the `wing` / en-US rejection below tells a shard author which
- * locale to exclude from a generated line, and does not stop the span proposer from firing on "Red Wing". Giving the
- * shipped vocabulary a per-locale gate is the largest thing the shard will want that does not exist yet.
- */
-export const SUBVENUE_PROMOTIONS = [
-    // ── wing ────────────────────────────────────────────────────────────────────────────────────────
-    // The designator the corpus task's board rests on (`West Wing` is the one modifier case that
-    // already parses), and the cleanest per-locale split in the whole exercise.
-    {
-        designatorID: "wing",
-        phrase: "wing",
-        locale: "en-GB",
-        decision: "promote",
-        real: 23,
-        confound: 6,
-        confoundNote: "Wing Close, Wing Road and Wing Tollgate House — the Buckinghamshire village of Wing — plus Cambian Wing " +
-            "College. All six are the <wing> <word> shape; every real hit is a hospital wing (Bexley Wing, Sorby Wing, " +
-            "Maternity Wing, West Wing, Bedford Hospital North Wing). CAVEAT, and it limits how far this generalizes: " +
-            "24 of the 29 hits sit on a public_transport=platform, because British bus stops are named after the " +
-            "hospital wing they serve. The extractor maps no building wings at all — `aile` in France is 0 hits, `ala` " +
-            "in Spain 0, `flügel` in Germany 0 — so this evidence is a property of British stop naming, not of a source " +
-            "that has wings in it.",
-        census: "great-britain.jsonl 2026-08-05, 29 name hits across name + name:<lang>",
-    },
-    {
-        designatorID: "wing",
-        phrase: "wing",
-        locale: "en-US",
-        decision: "reject",
-        real: 4,
-        confound: 3354,
-        confoundNote: "Red Wing, the boot brand, is 676 shoe_store rows; chicken wings are another 759 " +
-            "(chicken_wings_restaurant 394, chicken_restaurant 365) plus 166 general restaurants. The four real hits are " +
-            "campus_building rows. This is the same word as en-GB's clean 23 and it is unusable here.",
-        census: "poi.db 2026-05-20.0, full 13,681,698-row scan 2026-08-05",
-    },
-    {
-        designatorID: "wing",
-        phrase: "wing",
-        locale: "fr-FR",
-        decision: "reject",
-        real: 0,
-        confound: 26,
-        confoundNote: "Wing Chun and Wing Tsun martial-arts clubs, 15 of 26. Nothing in the French slice of the layer is a " +
-            "building wing.",
-        census: "poi.db 2026-05-20.0, full scan 2026-08-05 (FR partition, 721,352 rows)",
-    },
-    // ── hall ────────────────────────────────────────────────────────────────────────────────────────
-    // Wave 1 flagged `hall` as per-locale poison on a GB number alone. Two locales measured; both
-    // reject, and for DIFFERENT reasons, which is the part worth keeping.
-    {
-        designatorID: "hall",
-        phrase: "hall",
-        locale: "en-GB",
-        decision: "reject",
-        real: 0,
-        confound: 3273,
-        confoundNote: "3,204 of 3,273 hits sit on a public_transport=platform: a British bus stop named after the village hall it " +
-            "stands outside. The distribution is Village Hall, Town Hall, British Legion Hall, and then Hall Lane / Hall " +
-            "Road / SPEKE HALL ROAD — streets. The 52 that take a <hall> <identifier> shape are bus-stop position codes " +
-            "(Village Hall (o/s), Town Hall (3)), and all 16 <modifier> <hall> hits are streets (East Hall Lane, North " +
-            "Hall Road).",
-        census: "great-britain.jsonl 2026-08-05, 3,273 name hits",
-    },
-    {
-        designatorID: "hall",
-        phrase: "hall",
-        locale: "en-US",
-        decision: "reject",
-        real: 2095,
-        confound: 27_081,
-        confoundNote: "City Hall (5,066 town_hall rows), Kingdom Hall of Jehovah's Witnesses (2,312), event halls (2,622) and " +
-            "dormitory halls (1,989). Every one names a whole VENUE, not interior structure, so promoting the surface " +
-            "would tag them `unit`. The 2,095 real hits are US campus buildings (UAA Cuddy Hall) and they are outnumbered " +
-            "13 to 1.",
-        census: "poi.db 2026-05-20.0, full 13,681,698-row scan 2026-08-05",
-    },
-    {
-        designatorID: "hall",
-        phrase: "halle",
-        locale: "de-DE",
-        decision: "promote",
-        shape: "identifier-required",
-        real: 32,
-        confound: 168,
-        confoundNote: "The confound is 97 hits of the CITY Halle — Halle (Saale), population 240,000, plus Halle (Westf) — and 68 " +
-            "village and event halls (PHOENIX Halle, Urexweiler Halle), which are whole venues. That is a worse " +
-            "confound than wave 1 assumed when it called `Halle 2` the German counter-example to en-GB: the word is " +
-            "also a locality. It is promoted anyway because the confound is SEPARABLE BY SHAPE, and that was checked " +
-            "rather than assumed — dumping all 32 <halle> <identifier> hits returns numbered factory, trade-fair and " +
-            "airport halls (VW Halle 42, Audi GVZ Halle G, Messe West Halle 8, Flughafen Halle 7, Speyer Halle 101) and " +
-            "not one instance of the city. Never modifier-eligible and never proposed bare.",
-        census: "germany.jsonl 2026-08-05 (403,863 rows), 200 hits; all 32 shape hits enumerated",
-    },
-    {
-        designatorID: "hall",
-        phrase: "hall",
-        locale: "fr-FR",
-        decision: "promote",
-        real: 35,
-        confound: 5,
-        confoundNote: "Five: an English `Town Hall` and `Multipurpose hall` on French name:en tags, plus `Manning Hall`. French " +
-            "uses `hall` as the airport designator directly — 19 hits are Charles de Gaulle (Terminal 2E - Hall K/L/M, " +
-            "Terminal 1 - Hall A/B, Hall D) and 16 more are building halls (Hall des Sources, Hall du Marché, Hôpital " +
-            "Hall Sud). The word that is poison in en-GB is the ordinary designator one country away.",
-        census: "france.jsonl 2026-08-05 (251,260 rows), 40 hits; all 19 shape hits enumerated",
-    },
-    // ── gate ────────────────────────────────────────────────────────────────────────────────────────
-    // The English surface ships and is deliberately NOT modifier-eligible. These decide its localized
-    // forms, and the two land on opposite sides for a reason worth keeping.
-    {
-        designatorID: "gate",
-        phrase: "flugsteig",
-        locale: "de-DE",
-        decision: "promote",
-        real: 19,
-        confound: 0,
-        confoundNote: "None, at full-country scale. 18 of 19 hits are <flugsteig> <identifier> (Flugsteig 10, Flugsteig 9/9A, " +
-            "Terminal 1 Flugsteig B) and the last is bare; context is 12 terminal features and 7 gates. The word is a " +
-            "pure aviation term with no street, locality or business collision in German. Frankfurt's own name:en " +
-            "renders Flugsteig J as `Pier J`, which is a second attestation of the same feature under a second " +
-            "designator. A 11-of-11 read on the Hessen extract was the hypothesis; this is the count that earned it.",
-        census: "germany.jsonl 2026-08-05 (403,863 rows), 19 hits across name + name:<lang>",
-    },
-    {
-        designatorID: "gate",
-        phrase: "porte",
-        locale: "fr-FR",
-        decision: "reject",
-        real: 19,
-        confound: 927,
-        confoundNote: "946 hits, of which 894 are Paris city gates and the Métro stations named after them — Porte d'Orléans, " +
-            "Porte Dauphine, Porte de Champerret, Porte de Saint-Cloud. Unlike `halle`, the confound REACHES the " +
-            "designator+identifier shape: 17 of the 36 hits in that bucket are Porte Saint-Martin, Porte Saint-Denis, " +
-            "Porte Notre-Dame. Shape cannot separate it, which is the same test that killed `pier` for en-US. The 19 " +
-            "real hits are ferry and cruise gates (Porte 4 : Ferrys, Berliet Porte C).",
-        census: "france.jsonl 2026-08-05 (251,260 rows), 946 hits; all 36 shape hits enumerated",
-    },
-    // ── pier ────────────────────────────────────────────────────────────────────────────────────────
-    // New in wave 2, and the second clean per-locale split: promotable in en-GB, killed in en-US by a
-    // furniture chain whose name is exactly the designator+identifier shape.
-    {
-        designatorID: "pier",
-        phrase: "pier",
-        locale: "en-GB",
-        decision: "promote",
-        real: 120,
-        confound: 44,
-        confoundNote: "Pier Road, Pier Street, Pier Avenue, Pier Terrace — 44 street names, all of them the <pier> <street-type> " +
-            "shape, which the designator+identifier rule cannot reach. The real side is 98 named piers (Eastbourne Pier, " +
-            "Cadogan Pier), 10 in <pier> <identifier> shape (Pier 1, Pier 2, `Terminal 3, Pier 6`), 10 bare and 2 with a " +
-            "modifier. Promoted as a SURFACE only; not modifier-eligible, on the same reasoning that excludes `gate`.",
-        census: "great-britain.jsonl 2026-08-05, 164 name hits",
-    },
-    {
-        designatorID: "pier",
-        phrase: "pier",
-        locale: "en-US",
-        decision: "reject",
-        real: 278,
-        confound: 2330,
-        confoundNote: "Pier 1 Imports and its franchises are 341 home_decor_store plus 217 furniture_store rows, and they carry the " +
-            "<pier> <identifier> shape EXACTLY — so unlike en-GB the confound is not separable by shape. Another 159 are " +
-            "seafood restaurants and 144 are parks.",
-        census: "poi.db 2026-05-20.0, full 13,681,698-row scan 2026-08-05",
-    },
-    // ── terminal ────────────────────────────────────────────────────────────────────────────────────
-    // The English surface already ships. These promote the LOCALIZED forms, which is what a non-English
-    // shard line needs and what wave 1 had none of.
-    {
-        designatorID: "terminal",
-        phrase: "terminal",
-        locale: "es-ES",
-        decision: "promote",
-        real: 190,
-        confound: 0,
-        confoundNote: "None. Spanish borrows the English word unchanged and uses it in every shape the task targets: Terminal A, " +
-            "Terminal T1, Terminal 4S, plus named terminals (Terminal Marítima de Santander). No Spanish street type " +
-            "collides with it. The Wikidata label `terminal aeroportuaria` is the encyclopaedic form and stays uncurated; " +
-            "this promotes the derived head noun.",
-        census: "spain.jsonl 2026-08-05, 190 hits across name + name:<lang>",
-    },
-    {
-        designatorID: "terminal",
-        phrase: "terminal",
-        locale: "ca-ES",
-        decision: "promote",
-        real: 15,
-        confound: 0,
-        confoundNote: "None. Catalan is the same borrowing (Terminal 1, Terminal Sortides, Terminal Arribades) and needs its own " +
-            "entry because a surface's language subtag is `ca`, which an es-ES decision does not reach.",
-        census: "spain.jsonl 2026-08-05, 15 name:ca hits",
-    },
-    {
-        designatorID: "terminal",
-        phrase: "terminal",
-        locale: "fr-FR",
-        decision: "promote",
-        real: 169,
-        confound: 0,
-        confoundNote: "None. 99 hits are <terminal> <identifier> (Aéroport CDG Terminal 2F, Terminal 2G, Lyon Saint-Exupéry " +
-            "Terminal 1), 51 are named terminals (Terminal de Grande-Bretagne, Terminal Croisières) and 13 trail a " +
-            "venue name. French borrows the word unchanged and no French street type collides with it.",
-        census: "france.jsonl 2026-08-05 (251,260 rows), 169 hits across name + name:<lang>",
-    },
-    {
-        designatorID: "terminal",
-        phrase: "ターミナル",
-        locale: "ja-JP",
-        decision: "promote",
-        real: 1213,
-        confound: 2,
-        confoundNote: "Two hits are `ターミナル前` — a bus stop named 'in front of the terminal', which is arguably still real. " +
-            "Japanese has no street-name class containing the word. 246 hits take the Japanese target shape " +
-            "<identifier><designator> (第2ターミナル, 羽田空港第1ターミナル); 957 are compounds naming a terminal " +
-            "(バスターミナル, フェリーターミナル, 成田空港第一ターミナル); 10 are <designator> <identifier>. This is the " +
-            "surface the corpus task named as missing, and it exists only because the head-noun derivation produced it " +
-            "from five compound Wikidata labels first.",
-        census: "japan.jsonl 2026-08-05, 1,215 hits across name + name:<lang>",
-    },
-];
-//# sourceMappingURL=sub-venue-promotions.js.map

package/out/src/tools/sub-venue-promotions.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"sub-venue-promotions.js","sourceRoot":"","sources":["../../../src/tools/sub-venue-promotions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AA+CH;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAiC;IAChE,mGAAmG;IACnG,6FAA6F;IAC7F,4EAA4E;IAC5E;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC;QACX,YAAY,EACX,0GAA0G;YAC1G,4GAA4G;YAC5G,2GAA2G;YAC3G,sGAAsG;YACtG,4GAA4G;YAC5G,6GAA6G;YAC7G,uBAAuB;QACxB,MAAM,EAAE,wEAAwE;KAChF;IACD;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,CAAC;QACP,QAAQ,EAAE,IAAI;QACd,YAAY,EACX,kFAAkF;YAClF,8GAA8G;YAC9G,0FAA0F;QAC3F,MAAM,EAAE,0DAA0D;KAClE;IACD;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,CAAC;QACP,QAAQ,EAAE,EAAE;QACZ,YAAY,EACX,sGAAsG;YACtG,gBAAgB;QACjB,MAAM,EAAE,wEAAwE;KAChF;IAED,mGAAmG;IACnG,8FAA8F;IAC9F,sEAAsE;IACtE;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,CAAC;QACP,QAAQ,EAAE,IAAI;QACd,YAAY,EACX,6GAA6G;YAC7G,8GAA8G;YAC9G,6GAA6G;YAC7G,4GAA4G;YAC5G,aAAa;QACd,MAAM,EAAE,iDAAiD;KACzD;IACD;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,IAAI;QACV,QAAQ,EAAE,MAAM;QAChB,YAAY,EACX,yGAAyG;YACzG,2GAA2G;YAC3G,+GAA+G;YAC/G,UAAU;QACX,MAAM,EAAE,0DAA0D;KAClE;IACD;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,KAAK,EAAE,qBAAqB;QAC5B,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,GAAG;QACb,YAAY,EACX,6GAA6G;YAC7G,qGAAqG;YACrG,yGAAyG;YACzG,0GAA0G;YAC1G,0GAA0G;YAC1G,6GAA6G;YAC7G,gFAAgF;QACjF,MAAM,EAAE,iFAAiF;KACzF;IACD;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC;QACX,YAAY,EACX,2GAA2G;YAC3G,4GAA4G;YAC5G,2GAA2G;YAC3G,0FAA0F;QAC3F,MAAM,EAAE,+EAA+E;KACvF;IAED,mGAAmG;IACnG,kGAAkG;IAClG,wEAAwE;IACxE;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,WAAW;QACnB,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC;QACX,YAAY,EACX,yGAAyG;YACzG,2GAA2G;YAC3G,uGAAuG;YACvG,oGAAoG;YACpG,yGAAyG;QAC1G,MAAM,EAAE,4EAA4E;KACpF;IACD;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,GAAG;QACb,YAAY,EACX,yGAAyG;YACzG,sGAAsG;YACtG,2GAA2G;YAC3G,0GAA0G;YAC1G,2EAA2E;QAC5E,MAAM,EAAE,gFAAgF;KACxF;IAED,mGAAmG;IACnG,kGAAkG;IAClG,yEAAyE;IACzE;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,GAAG;QACT,QAAQ,EAAE,EAAE;QACZ,YAAY,EACX,4GAA4G;YAC5G,8GAA8G;YAC9G,8GAA8G;YAC9G,0GAA0G;QAC3G,MAAM,EAAE,+CAA+C;KACvD;IACD;QACC,YAAY,EAAE,MAAM;QACpB,MAAM,EAAE,MAAM;QACd,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,GAAG;QACT,QAAQ,EAAE,IAAI;QACd,YAAY,EACX,+GAA+G;YAC/G,8GAA8G;YAC9G,wCAAwC;QACzC,MAAM,EAAE,0DAA0D;KAClE;IAED,mGAAmG;IACnG,oGAAoG;IACpG,gDAAgD;IAChD;QACC,YAAY,EAAE,UAAU;QACxB,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,GAAG;QACT,QAAQ,EAAE,CAAC;QACX,YAAY,EACX,4GAA4G;YAC5G,0GAA0G;YAC1G,+GAA+G;YAC/G,sCAAsC;QACvC,MAAM,EAAE,4DAA4D;KACpE;IACD;QACC,YAAY,EAAE,UAAU;QACxB,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,CAAC;QACX,YAAY,EACX,4GAA4G;YAC5G,4FAA4F;QAC7F,MAAM,EAAE,yCAAyC;KACjD;IACD;QACC,YAAY,EAAE,UAAU;QACxB,MAAM,EAAE,UAAU;QAClB,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,GAAG;QACT,QAAQ,EAAE,CAAC;QACX,YAAY,EACX,uGAAuG;YACvG,wGAAwG;YACxG,2FAA2F;QAC5F,MAAM,EAAE,4EAA4E;KACpF;IACD;QACC,YAAY,EAAE,UAAU;QACxB,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,SAAS;QACnB,IAAI,EAAE,IAAI;QACV,QAAQ,EAAE,CAAC;QACX,YAAY,EACX,qGAAqG;YACrG,iGAAiG;YACjG,uFAAuF;YACvF,mFAAmF;YACnF,4GAA4G;YAC5G,2CAA2C;QAC5C,MAAM,EAAE,8DAA8D;KACtE;CACD,CAAA"}

package/out/src/types.d.ts

@@ -1,221 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Canonical row schemas for the corpus pipeline (per #6 / Phase 1 plan).
- *
- *   The corpus pipeline produces two row shapes:
- *
- *   1. `CanonicalRow`: an adapter's raw output. Carries a free-form `raw` string, a per-component
- *        ground-truth dict, provenance, and an optional augmentation marker. Adapters emit these.
- *   2. `LabeledRow`: alignment's output. Adds a SentencePiece token list and a parallel BIO label list,
- *        suitable for direct ingestion by the neural training loop.
- *
- *   `CorpusAdapter` is the contract every data source implements; `AdapterOptions` is the
- *   per-invocation knob set (input path, optional country filter, row cap, abort signal).
- */
-import type { BIOLabel, ComponentTag } from "@mailwoman/core/types";
-/**
- * Provenance + augmentation metadata that travels with every corpus row.
- *
- * `synth` is `undefined` for natural (un-augmented) rows; present only when a row was produced by the synthesis
- * pipeline (see `synthesize.ts`).
- */
-export interface SourceProvenance {
-    /**
-     * Adapter id that emitted this row, e.g. `"wof-admin"`, `"ban"`, `"openaddresses"`.
-     */
-    source: string;
-    /**
-     * Stable id within the adapter's source. For SQLite-backed adapters this is the row's primary key; for CSV/GeoJSON, a
-     * hash of the canonical components. Must be stable across reruns so that dedup and holdout manifests are
-     * reproducible.
-     */
-    source_id: string;
-    /**
-     * Corpus version string. Stamped by the runner, not the adapter. Locked together with the tokenizer version:
-     * `corpus-v0.1.0` ships with `tokenizer-v0.1.0`.
-     */
-    corpus_version: string;
-    /**
-     * Short license label or SPDX id for _this_ row. Defaults to the adapter's `defaultLicense`, but per-row sources
-     * (OpenAddresses) override.
-     */
-    license: string;
-}
-/**
- * Marker placed on rows produced by `synthesize.ts`. Allows downstream code to weight, stratify, or exclude
- * augmentations.
- */
-export interface SynthMarker {
-    /**
-     * Pipeline id describing what augmentation produced this row. Free-form but stable — e.g. `"case-perturb"`,
-     * `"accent-strip"`, `"abbrev-swap"`, `"compose:case-perturb+typo"`.
-     */
-    method: string;
-    /**
-     * `source_id` of the un-augmented row this was derived from. Allows tracing every synthetic row back to its natural
-     * ancestor.
-     */
-    base_source_id: string;
-}
-/**
- * One address row, before tokenization + BIO labeling.
- *
- * `raw` is what a parser would see in the wild — possibly multi-line, with arbitrary whitespace. `components` is the
- * ground-truth tagging: every `ComponentTag` present in the source data, mapped to its surface form _as it appears in
- * `raw`_. Alignment uses this to assign BIO labels.
- *
- * Country is ISO 3166-1 alpha-2 (`"US"`, `"FR"`). Locale is BCP-47 (`"en-US"`, `"fr-FR"`) and is optional; adapters
- * that can't be sure leave it empty and let the runner default by country.
- */
-export interface CanonicalRow extends SourceProvenance {
-    /**
-     * Address string as it might appear in source data.
-     */
-    raw: string;
-    /**
-     * Component-by-tag ground truth. Surface forms must occur in `raw` (within the alignment edit distance threshold) or
-     * the row will land in the quarantine pile.
-     */
-    components: Partial<Record<ComponentTag, string>>;
-    /**
-     * ISO 3166-1 alpha-2 country code.
-     */
-    country: string;
-    /**
-     * Optional BCP-47 locale. Defaulted by country if absent.
-     */
-    locale?: string;
-    /**
-     * Present only on synthetic rows.
-     */
-    synth?: SynthMarker;
-}
-/**
- * Output of `align.ts`. Carries everything `CanonicalRow` does, plus parallel `tokens` and `labels` arrays of identical
- * length (`labels[i]` is the BIO tag for `tokens[i]`) and — as of the v0.5.0 char-offset migration (#519) — parallel
- * char-span arrays addressing `raw` directly.
- *
- * The span triple is the v0.5.0 source of truth; `tokens`/`labels` remain emitted during the transition (and stay
- * derivable afterwards: whitespace split + span lookup). The reverse derivation — today's token labels — is the lossy
- * direction (punctuation-mute).
- */
-export interface LabeledRow extends CanonicalRow {
-    /**
-     * SentencePiece subword tokens for `raw`.
-     */
-    tokens: readonly string[];
-    /**
-     * BIO labels, one per token. Same length as `tokens`.
-     */
-    labels: readonly BIOLabel[];
-    /**
-     * Char-offset label spans over `raw` (parallel arrays, per the #519 ruling): `span_starts[i]` is the inclusive start
-     * offset (UTF-16 code units) of span `i`, `span_ends[i]` its exclusive end, `span_tags[i]` its component tag.
-     * Invariants — enforced loudly by `alignRow`, documented for every other producer: sorted ascending by start,
-     * non-overlapping. `raw` must be NFC-normalized or the offsets are ambiguous (also enforced by `alignRow`).
-     *
-     * Optional during the v0.4.x → v0.5.0 transition only: alignment always emits the triple; frozen historical corpora
-     * and not-yet-migrated synthesis paths may lack it. Required once v0.5.0 lands and the token path is deleted.
-     */
-    span_starts?: readonly number[];
-    /**
-     * Exclusive end offsets, parallel to `span_starts`.
-     */
-    span_ends?: readonly number[];
-    /**
-     * Component tags, parallel to `span_starts`.
-     */
-    span_tags?: readonly ComponentTag[];
-}
-/**
- * A row that alignment refused to label. Lands in `/data/corpus/quarantine/` for human review.
- *
- * The `reason` is human-readable; common values are `"component-not-found:<tag>"`, `"edit-distance-exceeded:<tag>"`,
- * `"raw-empty"`. Re-running alignment after a fix should re-emit the quarantined rows; the runner keys them by
- * `source_id`.
- */
-export interface QuarantinedRow {
-    row: CanonicalRow;
-    reason: string;
-}
-/**
- * Per-invocation knobs handed to an adapter by the runner.
- *
- * `inputPath` is interpreted by the adapter — it might be a single file path, a directory of shards, or even an HTTPS
- * URL. Each adapter documents its own expected shape in its README.
- *
- * `country` filters to a single ISO 3166-1 alpha-2 country _at the adapter level_. Adapters that hold multi-country
- * data (OSM PBF, OpenAddresses) MUST honor this; single-country adapters (BAN) may ignore it but should reject
- * mismatches.
- *
- * `limit` is a soft cap on rows emitted; useful for fixture-driven tests and smoke runs.
- *
- * `signal` allows the runner to cancel a long-running scan cleanly.
- */
-export interface AdapterOptions {
-    /**
-     * Path to the adapter's input data (file, directory, or URL — adapter-specific).
-     */
-    inputPath: string;
-    /**
-     * Optional output directory, available to adapters that maintain side state (rare).
-     */
-    outputDir?: string;
-    /**
-     * ISO 3166-1 alpha-2 country filter.
-     */
-    country?: string;
-    /**
-     * Soft row cap. Adapters should stop iterating once this is reached.
-     */
-    limit?: number;
-    /**
-     * Cancellation hook. Adapters should respect this on every iteration boundary.
-     */
-    signal?: AbortSignal;
-}
-/**
- * The contract every data source implements.
- *
- * Adapters are async generators: they yield `CanonicalRow`s one at a time, the runner consumes them (writing JSONL +
- * maintaining checksums + driving alignment). Streaming is mandatory — many sources are tens of millions of rows and
- * cannot be buffered.
- *
- * `defaultLicense` is stamped onto every emitted row's `license` field unless the adapter sets `license` explicitly
- * (e.g. OpenAddresses, which carries per-source licenses).
- */
-export interface CorpusAdapter {
-    /**
-     * Stable, machine-friendly id used in paths and CLI args. E.g. `"wof-admin"`.
-     */
-    readonly id: string;
-    /**
-     * Default SPDX-ish license label for rows from this adapter. Per-row overrides allowed.
-     */
-    readonly defaultLicense: string;
-    /**
-     * One-sentence description shown by `npx mailwoman corpus list`.
-     */
-    readonly description: string;
-    /**
-     * Async iterable of canonical rows.
-     *
-     * Implementations MUST:
-     *
-     * - Honor `opts.country` (filter or reject mismatches).
-     * - Honor `opts.limit` (stop after N rows).
-     * - Respect `opts.signal` on every iteration.
-     * - Set `source` to `this.id` on every emitted row.
-     * - Set `license` to `this.defaultLicense` unless overriding per-row.
-     *
-     * Implementations MUST NOT:
-     *
-     * - Set `corpus_version` (the runner stamps it).
-     * - Mutate previously-yielded rows.
-     */
-    rows(opts: AdapterOptions): AsyncIterable<CanonicalRow>;
-}
-//# sourceMappingURL=types.d.ts.map

package/out/src/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;IAEtB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;CACtB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACrD;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IAEX;;;OAGG;IACH,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEjD;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,KAAK,CAAC,EAAE,WAAW,CAAA;CACnB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC/C;;OAEG;IACH,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IAEzB;;OAEG;IACH,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAA;IAE3B;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE/B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAE7B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAA;CACnC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,YAAY,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC7B;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAE/B;;OAEG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAE5B;;;;;;;;;;;;;;;OAeG;IACH,IAAI,CAAC,IAAI,EAAE,cAAc,GAAG,aAAa,CAAC,YAAY,CAAC,CAAA;CACvD"}

package/out/src/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG"}

package/out/src/utils/align.d.ts

@@ -1,93 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Alignment: turn a `CanonicalRow` (raw + components) into a `LabeledRow` (raw + tokens + BIO
- *   labels) or a `QuarantinedRow` (raw + reason) per the Phase 1 plan.
- *
- *   Pipeline:
- *
- *   1. For each `(tag, value)` in `components`, find the value's character span in `raw`. First try a
- *        verbatim substring match (case-insensitive, whitespace-collapsed). If that fails, fall
- *        back to fuzzy match via `fastest-levenshtein`, with a tunable edit distance threshold.
- *   2. If any component cannot be located, reject the row with a human-readable reason and send it to
- *        the quarantine pile (`reason: "component-not-found:<tag>"` or
- *        `"edit-distance-exceeded:<tag>:<dist>"`).
- *   3. Tokenize `raw` with the supplied `Tokenizer` (defaults to the whitespace tokenizer).
- *   4. For each token: walk the list of component spans, pick the one whose span contains the token's
- *        character range. First token in a component span → `B-<tag>`; subsequent tokens →
- *        `I-<tag>`; no overlap → `O`.
- *   5. Emit the located char spans verbatim as `span_starts[]` / `span_ends[]` / `span_tags[]` (the
- *        v0.5.0 char-offset format, #519). The token quantization in step 4 is the part the v0.5.0
- *        rebuild deletes; during the transition both representations ride on every labeled row.
- *
- *   Structural invariants the function preserves (the span ones loudly — a violation throws rather
- *   than quarantines, because it indicates a bug here, not bad source data):
- *
- *   - `tokens.length === labels.length` always.
- *   - Each component contributes at most one contiguous BIO run (no `B-tag … O … I-tag` gaps). This is
- *       enforced by greedy first-match span assignment + ordered token iteration.
- *   - The span triple is sorted ascending by start and non-overlapping.
- *   - `raw` is NFC-normalized (asserted per row; a non-NFC raw makes char offsets ambiguous downstream
- *       — NFD `é` occupies two code units where NFC `é` occupies one — and silently so).
- */
-import type { ComponentTag } from "@mailwoman/core/types";
-import type { CanonicalRow, LabeledRow, QuarantinedRow } from "#types";
-import { type Tokenizer } from "./tokenize.ts";
-/**
- * Options for `alignRow`.
- */
-export interface AlignOptions {
-    /**
-     * Tokenizer to use. Defaults to `whitespaceTokenizer()`.
-     */
-    tokenizer?: Tokenizer;
-    /**
-     * Max Levenshtein edit distance to accept when a verbatim substring match fails. Set `0` to require verbatim matches
-     * only. Default `2`.
-     *
-     * Distance is computed against same-length windows in `raw`, so the threshold scales naturally with the component
-     * value length.
-     */
-    maxEditDistance?: number;
-    /**
-     * Case-insensitive comparison for substring search. Default `true`. The retained span in `raw` is the original case;
-     * only matching is case-insensitive.
-     */
-    caseInsensitive?: boolean;
-}
-/**
- * Either a successful labeled row or a quarantined one.
- */
-export type AlignmentResult = {
-    kind: "labeled";
-    row: LabeledRow;
-} | {
-    kind: "quarantined";
-    row: QuarantinedRow;
-};
-/**
- * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind
- * the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow` (#519).
- */
-export interface ComponentSpan {
-    tag: ComponentTag;
-    start: number;
-    end: number;
-}
-/**
- * Align a single row.
- */
-export declare function alignRow(row: CanonicalRow, opts?: AlignOptions): AlignmentResult;
-/**
- * Enforce the #519 span-triple invariants — in-bounds, sorted ascending by start, non-overlapping — loudly.
- *
- * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible and the caller sorts, so
- * a violation here is a bug in this file, not bad source data: throw (naming the row) rather than quarantine, so the
- * corruption can't ride into a corpus. Exported for every OTHER span producer (`composeAdversarialRow`'s offset
- * arithmetic, future synthesis paths) — any code that emits the triple without going through `alignRow` must pass its
- * output through this.
- */
-export declare function assertSpanInvariants(spans: readonly ComponentSpan[], row: Pick<CanonicalRow, "raw" | "source" | "source_id">): void;
-//# sourceMappingURL=align.d.ts.map

package/out/src/utils/align.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"align.d.ts","sourceRoot":"","sources":["../../../src/utils/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAY,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAGnE,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAA;AAEtE,OAAO,EAAuC,KAAK,SAAS,EAAE,MAAM,eAAe,CAAA;AAEnF;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,CAAA;IAErB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,cAAc,CAAA;CAAE,CAAA;AAEjH;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC7B,GAAG,EAAE,YAAY,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,IAAI,GAAE,YAAiB,GAAG,eAAe,CAsFpF;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CACnC,KAAK,EAAE,SAAS,aAAa,EAAE,EAC/B,GAAG,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,GAAG,QAAQ,GAAG,WAAW,CAAC,GACrD,IAAI,CA4BN"}