@mailwoman/codex 6.1.0 → 6.2.1

package/out/tools/build-country-surface-lexicon.d.ts

@@ -0,0 +1,43 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build the COUNTRY-SURFACE LEXICON for the country-lexicon soft-feed channel (#1104). This is the
+ *   third atlas channel, a sibling of the postcode anchor (#239/#240) and the gazetteer anchor
+ *   (#464): a per-token multi-hot clue the neural GRAMMAR conditions on but never obeys. Country is a
+ *   CLOSED, ENUMERABLE class (~250 surfaces) — atlas, not grammar — so a dictionary phrase-lookup
+ *   recovers the WOF-admin / resolver hierarchy case ("United States of America, Wyoming, <locality>")
+ *   the learned tagger reads as a leading STREET. Pelias handled the same class the same way
+ *   (`WhosOnFirstClassifier extends PhraseClassifier`); this is the model-first analogue.
+ *
+ *   WHY A DEDICATED LEXICON (not just the gazetteer's `country` slot): the gazetteer already carries
+ *   these surfaces in slot 0, and the shipped model already consumes them — yet the WOF-admin case
+ *   still fails (model-card #1104: golden country recall 82.0% vs 88.6%). The country bit is one of a
+ *   5-hot vector sharing ONE learned projection with region/po_box/cedex/homograph, and it is ZEROED
+ *   adjacent to a postcode by `suppress_gazetteer_near_postcode` (exactly where a trailing "…12345
+ *   USA" sits). A dedicated channel de-entangles the country signal (its own projection + confidence
+ *   weight) and is immune to that suppression. See
+ *   docs/superpowers/plans/2026-07-14-country-lexicon-channel.md.
+ *
+ *   The matcher REUSES the gazetteer's phrase-scan (longest-first n-gram over whitespace words,
+ *   case-insensitive `entries` + uppercase-exact `code_entries`, char→piece projection) — one tested
+ *   algorithm, two vocabularies. Only the vocabulary + the emitted feature differ. The emitted
+ *   feature is 2-dim per piece: `[country_surface, country_ambiguous]`.
+ *
+ *   - `country_surface` (bit 1): the piece is part of a recognized country surface phrase.
+ *   - `country_ambiguous` (bit 2): the SURFACE is a homograph (also a US region) or a common-word
+ *     name ("Georgia", "America", "England", "IN") — a SOFT version of Pelias's hard blacklist. The
+ *     model learns to trust `surface & !ambiguous` (unambiguous long/code forms) strongly and
+ *     `surface & ambiguous` weakly, using context — model-first, never a hard drop, so recall on
+ *     "Republic of Georgia" is preserved.
+ *
+ *   Source of truth: `@mailwoman/codex` (COUNTRY_SURFACE_FORMS + ISO2_TO_NAME) — the SAME data the
+ *   corpus-python bridge `country-surfaces.json` is generated from (export-country-surfaces.ts), so
+ *   the channel and the corpus shard synthesizer cannot diverge on what a country surface IS.
+ *
+ *   Output: data/gazetteer/country-surface-lexicon-v1.json (small, committed, provenance-tracked).
+ *   Regenerate: `node codex/tools/build-country-surface-lexicon.ts`
+ */
+export {};
+//# sourceMappingURL=build-country-surface-lexicon.d.ts.map

package/out/tools/build-country-surface-lexicon.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-country-surface-lexicon.d.ts","sourceRoot":"","sources":["../../tools/build-country-surface-lexicon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG"}

package/out/tools/build-country-surface-lexicon.js

@@ -0,0 +1,139 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build the COUNTRY-SURFACE LEXICON for the country-lexicon soft-feed channel (#1104). This is the
+ *   third atlas channel, a sibling of the postcode anchor (#239/#240) and the gazetteer anchor
+ *   (#464): a per-token multi-hot clue the neural GRAMMAR conditions on but never obeys. Country is a
+ *   CLOSED, ENUMERABLE class (~250 surfaces) — atlas, not grammar — so a dictionary phrase-lookup
+ *   recovers the WOF-admin / resolver hierarchy case ("United States of America, Wyoming, <locality>")
+ *   the learned tagger reads as a leading STREET. Pelias handled the same class the same way
+ *   (`WhosOnFirstClassifier extends PhraseClassifier`); this is the model-first analogue.
+ *
+ *   WHY A DEDICATED LEXICON (not just the gazetteer's `country` slot): the gazetteer already carries
+ *   these surfaces in slot 0, and the shipped model already consumes them — yet the WOF-admin case
+ *   still fails (model-card #1104: golden country recall 82.0% vs 88.6%). The country bit is one of a
+ *   5-hot vector sharing ONE learned projection with region/po_box/cedex/homograph, and it is ZEROED
+ *   adjacent to a postcode by `suppress_gazetteer_near_postcode` (exactly where a trailing "…12345
+ *   USA" sits). A dedicated channel de-entangles the country signal (its own projection + confidence
+ *   weight) and is immune to that suppression. See
+ *   docs/superpowers/plans/2026-07-14-country-lexicon-channel.md.
+ *
+ *   The matcher REUSES the gazetteer's phrase-scan (longest-first n-gram over whitespace words,
+ *   case-insensitive `entries` + uppercase-exact `code_entries`, char→piece projection) — one tested
+ *   algorithm, two vocabularies. Only the vocabulary + the emitted feature differ. The emitted
+ *   feature is 2-dim per piece: `[country_surface, country_ambiguous]`.
+ *
+ *   - `country_surface` (bit 1): the piece is part of a recognized country surface phrase.
+ *   - `country_ambiguous` (bit 2): the SURFACE is a homograph (also a US region) or a common-word
+ *     name ("Georgia", "America", "England", "IN") — a SOFT version of Pelias's hard blacklist. The
+ *     model learns to trust `surface & !ambiguous` (unambiguous long/code forms) strongly and
+ *     `surface & ambiguous` weakly, using context — model-first, never a hard drop, so recall on
+ *     "Republic of Georgia" is preserved.
+ *
+ *   Source of truth: `@mailwoman/codex` (COUNTRY_SURFACE_FORMS + ISO2_TO_NAME) — the SAME data the
+ *   corpus-python bridge `country-surfaces.json` is generated from (export-country-surfaces.ts), so
+ *   the channel and the corpus shard synthesizer cannot diverge on what a country surface IS.
+ *
+ *   Output: data/gazetteer/country-surface-lexicon-v1.json (small, committed, provenance-tracked).
+ *   Regenerate: `node codex/tools/build-country-surface-lexicon.ts`
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { COUNTRY_SURFACE_FORMS, ISO2_TO_NAME } from "../country/country.js";
+import { US_STATE_ABBREVIATIONS, US_STATE_NAMES } from "../us/state.js";
+const BIT = { country_surface: 1, country_ambiguous: 2 };
+const SLOTS = ["country_surface", "country_ambiguous"];
+// Committed output path (a codex-derived artifact, like export-country-surfaces.ts — no argv, so the
+// no-process-globals lint policy holds; codex stays zero-runtime-dep).
+const OUTPUT = resolve(import.meta.dirname, "../../data/gazetteer/country-surface-lexicon-v1.json");
+/**
+ * THE shared word-normalization rule (identical to build-gazetteer-anchor-lexicon.mjs and mirrored in
+ * gazetteer_char_paint on both sides): per whitespace-word, strip LEADING/TRAILING characters that are not Unicode
+ * letters or digits (keep internal ones: "u.s.a", "timor-leste"), rejoin single-spaced. Entry keys and scanned tokens
+ * both pass through it, so "U.S.A." ≡ "u.s.a".
+ */
+const wordNorm = (s) => s
+    .split(/\s+/)
+    .map((w) => w.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""))
+    .filter(Boolean)
+    .join(" ");
+const norm = (s) => wordNorm(s).toLowerCase();
+/** Short alphabetic code (≤3 letters once punctuation is dropped) → exact-uppercase matching. */
+const isShortCode = (s) => {
+    const letters = s.replace(/[^\p{L}]/gu, "");
+    return letters.length > 0 && letters.length <= 3 && /^[\p{L}.\s]+$/u.test(s);
+};
+// Homograph set: a single-word country surface that is ALSO a US region (name or abbreviation) reads
+// ambiguously (Georgia the country vs the state, IN = India vs Indiana). Computed from codex so it
+// tracks the US region table, never hand-maintained.
+const usStateNames = new Set(US_STATE_NAMES.map((n) => n.toLowerCase()));
+const usStateAbbrevs = new Set(US_STATE_ABBREVIATIONS);
+// Curated common-word country surfaces — single tokens that appear far more often as ordinary
+// street/venue/locality words than as a trailing country. A SOFT flag (the model still decides), the
+// model-first analogue of Pelias's blacklist (north/south/east/west/street/city/king). Tunable.
+const COMMON_WORD_AMBIGUOUS = new Set(["america", "england", "britain", "turkey", "chad", "jordan", "jersey", "guinea"]);
+const isAmbiguousName = (lowerKey) => usStateNames.has(lowerKey) || COMMON_WORD_AMBIGUOUS.has(lowerKey);
+// surface → bits, split across the two match-rule maps (mirrors the gazetteer builder).
+const entries = new Map(); // lowercase key
+const codeEntries = new Map(); // exact-uppercase key
+let maxNgram = 1;
+function add(surface) {
+    const s = surface.trim();
+    if (!s)
+        return;
+    if (isShortCode(s)) {
+        const key = wordNorm(s).toUpperCase();
+        if (!key)
+            return;
+        // A code that collides with a US-state abbreviation (CA/IN/AL/CO/…) is a homograph → ambiguous.
+        const bits = BIT.country_surface | (usStateAbbrevs.has(key) ? BIT.country_ambiguous : 0);
+        codeEntries.set(key, (codeEntries.get(key) ?? 0) | bits);
+        return;
+    }
+    const key = norm(s);
+    if (!key)
+        return;
+    const words = key.split(" ");
+    maxNgram = Math.max(maxNgram, words.length);
+    // Multi-word phrases are unambiguous by construction; single tokens consult the homograph +
+    // common-word rule.
+    const ambiguous = words.length === 1 && isAmbiguousName(key);
+    const bits = BIT.country_surface | (ambiguous ? BIT.country_ambiguous : 0);
+    entries.set(key, (entries.get(key) ?? 0) | bits);
+}
+// Curated rich surface forms first (US/GB/DE/… endonyms + abbreviations), then the canonical English
+// name for every remaining ISO 3166-1 alpha-2 — exactly the merge country-surfaces.json performs.
+for (const forms of Object.values(COUNTRY_SURFACE_FORMS)) {
+    for (const f of forms) {
+        add(f);
+    }
+}
+for (const [, name] of ISO2_TO_NAME) {
+    add(name);
+}
+const ambiguousEntries = [...entries, ...codeEntries].filter(([, b]) => b & BIT.country_ambiguous).map(([k]) => k);
+const lexicon = {
+    version: 1,
+    generated_by: "codex/tools/build-country-surface-lexicon.ts (source: @mailwoman/codex COUNTRY_SURFACE_FORMS + ISO2_TO_NAME)",
+    feature_dim: SLOTS.length,
+    slots: SLOTS,
+    bits: BIT,
+    max_ngram: maxNgram,
+    rules: {
+        word_norm: "per whitespace-word: strip leading/trailing chars that are not Unicode letters/digits " +
+            "(keep internal: 'timor-leste', 'u.s.a'); rejoin single-spaced. Applied to BOTH entry keys and scanned tokens.",
+        entries: "case-insensitive; key = word_norm lowercased. country_surface always set; country_ambiguous set for single-token homographs (US region) or curated common-word names.",
+        code_entries: "case-SENSITIVE exact: word_norm(token) == key (keys uppercase; 'in' the word ≠ 'IN' India). n-gram length 1. country_ambiguous set when the code is also a US-state abbreviation.",
+        scan: "longest-first n-gram over whitespace words, left to right, non-overlapping (shared with the gazetteer matcher)",
+        feature: "emitted per-piece row = [country_surface, country_ambiguous] (the raw bits); confidence = 1.0 where country_surface fires.",
+    },
+    entries: Object.fromEntries([...entries].sort(([a], [b]) => a.localeCompare(b))),
+    code_entries: Object.fromEntries([...codeEntries].sort(([a], [b]) => a.localeCompare(b))),
+};
+mkdirSync(dirname(OUTPUT), { recursive: true });
+writeFileSync(OUTPUT, JSON.stringify(lexicon, null, 1) + "\n");
+process.stderr.write(`wrote ${OUTPUT}: ${entries.size} entries + ${codeEntries.size} code_entries, ` +
+    `max_ngram=${maxNgram}, ${ambiguousEntries.length} ambiguous: ${ambiguousEntries.slice(0, 12).join(", ")}${ambiguousEntries.length > 12 ? ", …" : ""}\n`);
+//# sourceMappingURL=build-country-surface-lexicon.js.map

package/out/tools/build-country-surface-lexicon.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-country-surface-lexicon.js","sourceRoot":"","sources":["../../tools/build-country-surface-lexicon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwCG;AAEH,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAClD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAE5C,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAC3E,OAAO,EAAE,sBAAsB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAEvE,MAAM,GAAG,GAAG,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;AACxD,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAA;AAEtD,qGAAqG;AACrG,uEAAuE;AACvE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,sDAAsD,CAAC,CAAA;AAEnG;;;;;GAKG;AACH,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE,CACtC,CAAC;KACC,KAAK,CAAC,KAAK,CAAC;KACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAC;KAC9D,MAAM,CAAC,OAAO,CAAC;KACf,IAAI,CAAC,GAAG,CAAC,CAAA;AAEZ,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;AAE7D,iGAAiG;AACjG,MAAM,WAAW,GAAG,CAAC,CAAS,EAAW,EAAE;IAC1C,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAA;IAE3C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAC7E,CAAC,CAAA;AAED,qGAAqG;AACrG,mGAAmG;AACnG,qDAAqD;AACrD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;AACxE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAS,sBAA2C,CAAC,CAAA;AAEnF,8FAA8F;AAC9F,qGAAqG;AACrG,gGAAgG;AAChG,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAA;AAExH,MAAM,eAAe,GAAG,CAAC,QAAgB,EAAW,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,qBAAqB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;AAExH,wFAAwF;AACxF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA,CAAC,gBAAgB;AAC1D,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA,CAAC,sBAAsB;AACpE,IAAI,QAAQ,GAAG,CAAC,CAAA;AAEhB,SAAS,GAAG,CAAC,OAAe;IAC3B,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;IAExB,IAAI,CAAC,CAAC;QAAE,OAAM;IAEd,IAAI,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;QAErC,IAAI,CAAC,GAAG;YAAE,OAAM;QAChB,gGAAgG;QAChG,MAAM,IAAI,GAAG,GAAG,CAAC,eAAe,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QACxF,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;QAExD,OAAM;IACP,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IAEnB,IAAI,CAAC,GAAG;QAAE,OAAM;IAChB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC5B,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;IAC3C,4FAA4F;IAC5F,oBAAoB;IACpB,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,GAAG,CAAC,CAAA;IAC5D,MAAM,IAAI,GAAG,GAAG,CAAC,eAAe,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1E,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AACjD,CAAC;AAED,qGAAqG;AACrG,kGAAkG;AAClG,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,qBAAqB,CAAC,EAAE,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,GAAG,CAAC,CAAC,CAAC,CAAA;IACP,CAAC;AACF,CAAC;AAED,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,YAAY,EAAE,CAAC;IACrC,GAAG,CAAC,IAAI,CAAC,CAAA;AACV,CAAC;AAED,MAAM,gBAAgB,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAA;AAElH,MAAM,OAAO,GAAG;IACf,OAAO,EAAE,CAAC;IACV,YAAY,EACX,8GAA8G;IAC/G,WAAW,EAAE,KAAK,CAAC,MAAM;IACzB,KAAK,EAAE,KAAK;IACZ,IAAI,EAAE,GAAG;IACT,SAAS,EAAE,QAAQ;IACnB,KAAK,EAAE;QACN,SAAS,EACR,wFAAwF;YACxF,+GAA+G;QAChH,OAAO,EACN,uKAAuK;QACxK,YAAY,EACX,mLAAmL;QACpL,IAAI,EAAE,gHAAgH;QACtH,OAAO,EACN,4HAA4H;KAC7H;IACD,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAChF,YAAY,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;CACzF,CAAA;AAED,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AAC/C,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;AAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,MAAM,KAAK,OAAO,CAAC,IAAI,cAAc,WAAW,CAAC,IAAI,iBAAiB;IAC9E,aAAa,QAAQ,KAAK,gBAAgB,CAAC,MAAM,eAAe,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CACzJ,CAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/codex",
-  "version": "6.1.0",
+  "version": "6.2.1",
   "description": "Per-address-system postal reference data + branded types (USPS street suffixes, US ZIP codes). Pure, zero-runtime-dep — the shared canonical home for postal-system primitives the parser, resolver, and synthesis layers all reach for.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "repository": {
@@ -116,6 +116,6 @@
     "type-fest": "^5.8.0"
   },
   "devDependencies": {
-    "@mailwoman/annotations": "6.1.0"
+    "@mailwoman/annotations": "6.2.1"
   }
 }