@mailwoman/corpus 9.1.0 → 9.3.0

package/out/src/adapters/wof-postalcode-json/adapter.d.ts

@@ -1,70 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `wof-postalcode`: Who's On First postalcode GeoJSON-bundle adapter.
- *
- *   **Phase 1.5.1 pivot.** Replaces the previous SpatiaLite-backed implementation (formerly at
- *   `packages/corpus/src/adapters/wof-postalcode/`, removed in this same change). The rationale is
- *   in `wof-admin-json/adapter.ts` and in `DECISIONS.md` — short version: the SQLite distribution
- *   mirror is dead, the live distro tags every postcode row `mz:is_current = -1` which the old
- *   `is_current = 1` predicate excluded, and localized `name:*` variants don't ship in the SQLite
- *   export at all.
- *
- *   Input: a directory containing one or more cloned `whosonfirst-data-postalcode-<cc>` repos plus
- *   the relevant `whosonfirst-data-admin-<cc>` repos (postcode records reference admin ancestry by
- *   `wof:parent_id`, so the locality / region / country records must be in the same walk for the
- *   ancestry chain to resolve). The corpus pipeline clones all four repos under
- *   `/data/corpus/sources/wof/repos/` and points the adapter at that root.
- *
- *   Per live postalcode record, the adapter emits one row per `(name-variant, hierarchy-variant)`
- *   pair:
- *
- *   - **Name variants**: canonical `wof:name` (slot key `default`, typically the postcode digits
- *       themselves) plus any `name:*` variants on the postcode feature. In practice WOF postcode
- *       records rarely carry localized name variants, so this expansion is usually a no-op — but
- *       the code path stays symmetric with the admin adapter for consistency.
- *   - **Hierarchy variants** (unchanged from the SQLite adapter): self, +locality, +locality+region,
- *       +locality+region+country.
- *
- *   `source_id` is `wof-postalcode-<wof_id>-<name-slot>-<hierarchy-variant>`. Ancestor names always
- *   come from the ancestor's canonical `wof:name`; this adapter does NOT iterate ancestor name
- *   variants (e.g. it does not emit `"75008 Париж"` even when Paris has a `name:rus_x_preferred`).
- *   That cross-product belongs to a future synthesis pass; emitting it here would multiply row
- *   counts ~10× without a clear training-value story.
- *
- *   License: CC0.
- */
-import type { ComponentTag } from "@mailwoman/core/types";
-import type { CorpusAdapter } from "@mailwoman/corpus/types";
-import { type WOFRecord } from "@mailwoman/corpus/utils";
-interface VariantSpec {
-    suffix: string;
-    components: Partial<Record<ComponentTag, string>>;
-}
-/**
- * Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
- * the `default` slot, a `name:*` localized variant otherwise).
- */
-export declare function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): VariantSpec[];
-/**
- * Build the per-record name-slot list. The `default` slot uses `wof:name` verbatim (postcode digits); subsequent slots
- * come from `name:*` variants dedup'd against the default.
- */
-export declare function nameSlotsFor(rec: WOFRecord): Array<{
-    key: string;
-    value: string;
-}>;
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export declare const WOF_POSTALCODE_ADAPTER_ID = "wof-postalcode";
-export declare function createWOFPostalcodeAdapter(): CorpusAdapter;
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export declare const wofPostalcodeAdapter: CorpusAdapter;
-export {};
-//# sourceMappingURL=adapter.d.ts.map

package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACzD,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,yBAAyB,CAAA;AAC1F,OAAO,EAAsD,KAAK,SAAS,EAAE,MAAM,yBAAyB,CAAA;AA8B5G,UAAU,WAAW;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;CACjD;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,EAAE,CAqC1G;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAWlF;AAED;;;GAGG;AACH,eAAO,MAAM,yBAAyB,mBAAmB,CAAA;AAEzD,wBAAgB,0BAA0B,IAAI,aAAa,CAmE1D;AAED;;GAEG;AACH,eAAO,MAAM,oBAAoB,eAA+B,CAAA"}

package/out/src/adapters/wof-postalcode-json/adapter.js

@@ -1,185 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `wof-postalcode`: Who's On First postalcode GeoJSON-bundle adapter.
- *
- *   **Phase 1.5.1 pivot.** Replaces the previous SpatiaLite-backed implementation (formerly at
- *   `packages/corpus/src/adapters/wof-postalcode/`, removed in this same change). The rationale is
- *   in `wof-admin-json/adapter.ts` and in `DECISIONS.md` — short version: the SQLite distribution
- *   mirror is dead, the live distro tags every postcode row `mz:is_current = -1` which the old
- *   `is_current = 1` predicate excluded, and localized `name:*` variants don't ship in the SQLite
- *   export at all.
- *
- *   Input: a directory containing one or more cloned `whosonfirst-data-postalcode-<cc>` repos plus
- *   the relevant `whosonfirst-data-admin-<cc>` repos (postcode records reference admin ancestry by
- *   `wof:parent_id`, so the locality / region / country records must be in the same walk for the
- *   ancestry chain to resolve). The corpus pipeline clones all four repos under
- *   `/data/corpus/sources/wof/repos/` and points the adapter at that root.
- *
- *   Per live postalcode record, the adapter emits one row per `(name-variant, hierarchy-variant)`
- *   pair:
- *
- *   - **Name variants**: canonical `wof:name` (slot key `default`, typically the postcode digits
- *       themselves) plus any `name:*` variants on the postcode feature. In practice WOF postcode
- *       records rarely carry localized name variants, so this expansion is usually a no-op — but
- *       the code path stays symmetric with the admin adapter for consistency.
- *   - **Hierarchy variants** (unchanged from the SQLite adapter): self, +locality, +locality+region,
- *       +locality+region+country.
- *
- *   `source_id` is `wof-postalcode-<wof_id>-<name-slot>-<hierarchy-variant>`. Ancestor names always
- *   come from the ancestor's canonical `wof:name`; this adapter does NOT iterate ancestor name
- *   variants (e.g. it does not emit `"75008 Париж"` even when Paris has a `name:rus_x_preferred`).
- *   That cross-product belongs to a future synthesis pass; emitting it here would multiply row
- *   counts ~10× without a clear training-value story.
- *
- *   License: CC0.
- */
-import { buildAncestryIndex, normalizeNameKey, walkFeatures } from "@mailwoman/corpus/utils";
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter";
-const COUNTRY_DISPLAY_NAME = {
-    US: "United States of America",
-    FR: "France",
-};
-const LOCALE_BY_COUNTRY = {
-    US: "en-US",
-    FR: "fr-FR",
-};
-function placetypeToTag(placetype) {
-    switch (placetype) {
-        case "country":
-        case "nation":
-            return "country";
-        case "macroregion":
-        case "region":
-            return "region";
-        case "locality":
-            return "locality";
-        case "postalcode":
-            return "postcode";
-        default:
-            return undefined;
-    }
-}
-/**
- * Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
- * the `default` slot, a `name:*` localized variant otherwise).
- */
-export function postcodeVariantsFor(row, ancestry, selfName) {
-    if (placetypeToTag(row.placetype) !== "postcode")
-        return [];
-    const locality = ancestry.find((a) => placetypeToTag(a.placetype) === "locality");
-    const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region");
-    const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country");
-    const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country;
-    const variants = [{ suffix: "self", components: { postcode: selfName } }];
-    if (locality) {
-        variants.push({
-            suffix: "with-locality",
-            components: { postcode: selfName, locality: locality.name },
-        });
-    }
-    if (locality && region) {
-        variants.push({
-            suffix: "with-locality-region",
-            components: { postcode: selfName, locality: locality.name, region: region.name },
-        });
-    }
-    if (locality && region && country) {
-        variants.push({
-            suffix: "with-locality-region-country",
-            components: {
-                postcode: selfName,
-                locality: locality.name,
-                region: region.name,
-                country: countryDisplay,
-            },
-        });
-    }
-    return variants;
-}
-/**
- * Build the per-record name-slot list. The `default` slot uses `wof:name` verbatim (postcode digits); subsequent slots
- * come from `name:*` variants dedup'd against the default.
- */
-export function nameSlotsFor(rec) {
-    const seen = new Set([rec.name]);
-    const slots = [{ key: "default", value: rec.name }];
-    for (const [rawKey, value] of rec.nameVariants) {
-        if (seen.has(value))
-            continue;
-        seen.add(value);
-        slots.push({ key: normalizeNameKey(rawKey), value });
-    }
-    return slots;
-}
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const WOF_POSTALCODE_ADAPTER_ID = "wof-postalcode";
-export function createWOFPostalcodeAdapter() {
-    return {
-        id: WOF_POSTALCODE_ADAPTER_ID,
-        defaultLicense: "CC0-1.0",
-        description: "Who's On First postalcode GeoJSON bundles (postcode → locality/region pairs). Ancestor names from sibling admin repos.",
-        async *rows(opts) {
-            // Pass 1: full walk. We keep every record whose placetype maps to a ComponentTag — the
-            // postcode adapter needs locality / region / country admin records in the index so it
-            // can resolve postcode ancestry, even though it only emits rows for postcode records.
-            const byID = new Map();
-            for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
-                if (opts.signal?.aborted)
-                    return;
-                if (opts.country && rec.country !== opts.country)
-                    continue;
-                if (!placetypeToTag(rec.placetype))
-                    continue;
-                byID.set(rec.id, rec);
-            }
-            const ancestry = buildAncestryIndex(byID);
-            // Pass 2: emit postcode rows only, sorted by id for determinism.
-            const ids = [...byID.keys()].toSorted((a, b) => a - b);
-            let emitted = 0;
-            for (const id of ids) {
-                if (opts.signal?.aborted)
-                    return;
-                const rec = byID.get(id);
-                if (placetypeToTag(rec.placetype) !== "postcode")
-                    continue;
-                const chain = ancestry.get(id) ?? [];
-                const slots = nameSlotsFor(rec);
-                for (const slot of slots) {
-                    const variants = postcodeVariantsFor(rec, chain, slot.value);
-                    for (const variant of variants) {
-                        if (opts.limit !== undefined && emitted >= opts.limit)
-                            return;
-                        const raw = formatAddress(variant.components, rec.country, { separator: ", " });
-                        if (!raw)
-                            continue;
-                        const aligned = reconcileComponents(variant.components, raw);
-                        if (!Object.keys(aligned).length)
-                            continue;
-                        yield {
-                            raw,
-                            components: aligned,
-                            country: rec.country,
-                            locale: LOCALE_BY_COUNTRY[rec.country],
-                            source: WOF_POSTALCODE_ADAPTER_ID,
-                            source_id: `${WOF_POSTALCODE_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
-                            corpus_version: "",
-                            license: "CC0-1.0",
-                        };
-                        emitted++;
-                    }
-                }
-            }
-        },
-    };
-}
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const wofPostalcodeAdapter = createWOFPostalcodeAdapter();
-//# sourceMappingURL=adapter.js.map

package/out/src/adapters/wof-postalcode-json/adapter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAKH,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,yBAAyB,CAAA;AAC5G,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAEzE,MAAM,oBAAoB,GAA2B;IACpD,EAAE,EAAE,0BAA0B;IAC9B,EAAE,EAAE,QAAQ;CACZ,CAAA;AAED,MAAM,iBAAiB,GAA2B;IACjD,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;CACX,CAAA;AAED,SAAS,cAAc,CAAC,SAAwC;IAC/D,QAAQ,SAAS,EAAE,CAAC;QACnB,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ;YACZ,OAAO,SAAS,CAAA;QACjB,KAAK,aAAa,CAAC;QACnB,KAAK,QAAQ;YACZ,OAAO,QAAQ,CAAA;QAChB,KAAK,UAAU;YACd,OAAO,UAAU,CAAA;QAClB,KAAK,YAAY;YAChB,OAAO,UAAU,CAAA;QAClB;YACC,OAAO,SAAS,CAAA;IAClB,CAAC;AACF,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAc,EAAE,QAAqB,EAAE,QAAgB;IAC1F,IAAI,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,UAAU;QAAE,OAAO,EAAE,CAAA;IAE3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,UAAU,CAAC,CAAA;IACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAA;IAC7E,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC,CAAA;IAC/E,MAAM,cAAc,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,IAAI,IAAI,GAAG,CAAC,OAAO,CAAA;IAExF,MAAM,QAAQ,GAAkB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAA;IAExF,IAAI,QAAQ,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,eAAe;YACvB,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE;SAC3D,CAAC,CAAA;IACH,CAAC;IAED,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,sBAAsB;YAC9B,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;SAChF,CAAC,CAAA;IACH,CAAC;IAED,IAAI,QAAQ,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,8BAA8B;YACtC,UAAU,EAAE;gBACX,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,OAAO,EAAE,cAAc;aACvB;SACD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAc;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;IACxC,MAAM,KAAK,GAA0C,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;IAE1F,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAQ;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACf,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,CAAA;AAEzD,MAAM,UAAU,0BAA0B;IACzC,OAAO;QACN,EAAE,EAAE,yBAAyB;QAC7B,cAAc,EAAE,SAAS;QACzB,WAAW,EACV,wHAAwH;QAEzH,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,uFAAuF;YACvF,sFAAsF;YACtF,sFAAsF;YACtF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAA;YAEzC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;gBAC/E,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,OAAM;gBAEhC,IAAI,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;oBAAE,SAAQ;gBAE1D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC;oBAAE,SAAQ;gBAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;YAED,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAEzC,iEAAiE;YACjE,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACtD,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACtB,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,OAAM;gBAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAE,CAAA;gBAEzB,IAAI,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,UAAU;oBAAE,SAAQ;gBAE1D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAA;gBACpC,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;gBAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBAC1B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAA;oBAE5D,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;wBAChC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;4BAAE,OAAM;wBAE7D,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;wBAE/E,IAAI,CAAC,GAAG;4BAAE,SAAQ;wBAClB,MAAM,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;wBAE5D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;4BAAE,SAAQ;wBAE1C,MAAM;4BACL,GAAG;4BACH,UAAU,EAAE,OAAO;4BACnB,OAAO,EAAE,GAAG,CAAC,OAAO;4BACpB,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC;4BACtC,MAAM,EAAE,yBAAyB;4BACjC,SAAS,EAAE,GAAG,yBAAyB,IAAI,GAAG,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE;4BACjF,cAAc,EAAE,EAAE;4BAClB,OAAO,EAAE,SAAS;yBAClB,CAAA;wBAED,OAAO,EAAE,CAAA;oBACV,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,0BAA0B,EAAE,CAAA"}

package/out/src/build.d.ts

@@ -1,131 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   End-to-end corpus build (Phase 1 task #10 in the plan).
- *
- *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
- *
- *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
- *        `<intermediate>/<adapter.id>/canonical.jsonl` shards.
- *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
- *        country-default policy emits an augmented row alongside the original.
- *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
- *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
- *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
- *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
- *        `test.txt`.
- *   5. **Parquet shards** — `writeShards` streams labeled rows into 1M-row `.parquet` shards per split
- *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
- *        row groups), with per-shard checksums + per-stage manifest in
- *        `corpus-v<version>/MANIFEST.json`.
- *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
- *        a top-level corpus_version, built_at, and aggregate counts.
- *
- *   Output layout:
- *
- *   ```
- *   <outputDir>/
- *   MANIFEST.json
- *   intermediate/
- *     <adapter.id>/canonical.jsonl   # one per adapter
- *     labeled.jsonl                  # post-alignment, pre-shard
- *     quarantine.jsonl               # rows that failed alignment
- *   splits/
- *     SPLIT_MANIFEST.json
- *     train.txt / val.txt / test.txt
- *   corpus-v<version>/
- *     MANIFEST.json
- *     train/part-NNNN.parquet
- *     val/part-NNNN.parquet
- *     test/part-NNNN.parquet
- * ```
- *
- *   The intermediate files live alongside the final shards for reproducibility + debugging. Operators
- *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
- *   self-contained.
- */
-import { type AdapterRunManifest } from "@mailwoman/corpus/runner";
-import type { AdapterOptions, CorpusAdapter } from "@mailwoman/corpus/types";
-import { type ShardManifest, type SplitManifest } from "@mailwoman/corpus/utils";
-/**
- * Stage tags surfaced to `onProgress`.
- */
-export type BuildStage = "adapter-run" | "align" | "split" | "shard" | "manifest";
-/**
- * Per-invocation options for `buildCorpus`.
- */
-export interface BuildCorpusOptions {
-    /**
-     * Root output directory. All build artifacts land beneath it.
-     */
-    outputDir: string;
-    /**
-     * Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name.
-     */
-    corpusVersion: string;
-    /**
-     * Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list to filter (e.g.
-     * `[wofAdminAdapter]` for a smoke run).
-     */
-    adapters?: readonly CorpusAdapter[];
-    /**
-     * Per-adapter `AdapterOptions` — looked up by adapter id. Adapters whose id is missing from this map are skipped (and
-     * noted in the manifest).
-     */
-    adapterInputs: Record<string, AdapterOptions>;
-    /**
-     * Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests.
-     */
-    synthesize?: boolean;
-    /**
-     * Forwarded to `writeShards`. Default 1_000_000.
-     */
-    rowsPerShard?: number;
-    /**
-     * Progress hook. Errors thrown abort the build.
-     */
-    onProgress?: (stage: BuildStage, message: string) => void;
-    /**
-     * License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see `compileLicenseExcludes` /
-     * `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license` matches any is dropped at ingest. Default (omitted)
-     * includes EVERYTHING — exclusion is a deliberate act, not a silent default. A proprietary-weights build passes the
-     * share-alike set (`--exclude-share-alike`).
-     */
-    excludeLicenses?: readonly RegExp[];
-}
-/**
- * Top-level manifest tying every stage together.
- */
-export interface BuildCorpusManifest {
-    corpus_version: string;
-    built_at: string;
-    adapters: AdapterRunManifest[];
-    skipped_adapters: string[];
-    splits: {
-        counts: SplitManifest["counts"];
-        holdouts: SplitManifest["holdouts"];
-    };
-    shards: {
-        counts: ShardManifest["counts"];
-        total_rows: number;
-    };
-    quarantine_count: number;
-    total_aligned_rows: number;
-    /**
-     * Resolved license set across all INCLUDED rows (license string → row count), + the count dropped by
-     * `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
-     */
-    licenses: Record<string, number>;
-    excluded_by_license: number;
-}
-/**
- * Drive the full corpus build to completion.
- *
- * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → shard hand-off.
- * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
- * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
- */
-export declare function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpusManifest>;
-//# sourceMappingURL=build.d.ts.map

package/out/src/build.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AASH,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAE9E,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,yBAAyB,CAAA;AACtG,OAAO,EAON,KAAK,aAAa,EAClB,KAAK,aAAa,EAElB,MAAM,yBAAyB,CAAA;AAGhC;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAyNxF"}

package/out/src/build.js

@@ -1,248 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   End-to-end corpus build (Phase 1 task #10 in the plan).
- *
- *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
- *
- *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
- *        `<intermediate>/<adapter.id>/canonical.jsonl` shards.
- *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
- *        country-default policy emits an augmented row alongside the original.
- *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
- *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
- *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
- *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
- *        `test.txt`.
- *   5. **Parquet shards** — `writeShards` streams labeled rows into 1M-row `.parquet` shards per split
- *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
- *        row groups), with per-shard checksums + per-stage manifest in
- *        `corpus-v<version>/MANIFEST.json`.
- *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
- *        a top-level corpus_version, built_at, and aggregate counts.
- *
- *   Output layout:
- *
- *   ```
- *   <outputDir>/
- *   MANIFEST.json
- *   intermediate/
- *     <adapter.id>/canonical.jsonl   # one per adapter
- *     labeled.jsonl                  # post-alignment, pre-shard
- *     quarantine.jsonl               # rows that failed alignment
- *   splits/
- *     SPLIT_MANIFEST.json
- *     train.txt / val.txt / test.txt
- *   corpus-v<version>/
- *     MANIFEST.json
- *     train/part-NNNN.parquet
- *     val/part-NNNN.parquet
- *     test/part-NNNN.parquet
- * ```
- *
- *   The intermediate files live alongside the final shards for reproducibility + debugging. Operators
- *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
- *   self-contained.
- */
-import { createWriteStream, existsSync, readFileSync } from "node:fs";
-import { mkdir, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { $public } from "@mailwoman/core/env";
-import { parseJSONStrict } from "@mailwoman/core/objects";
-import { defaultAdapterRegistry } from "@mailwoman/corpus/adapters/utils";
-import { runAdapter } from "@mailwoman/corpus/runner";
-import { defaultAugmentationsForCountry, synthesizeRow } from "@mailwoman/corpus/synthesizers/utils";
-import { alignRow, licenseExcluded, writeShards, defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "@mailwoman/corpus/utils";
-import { JSONSpliterator } from "spliterator";
-/**
- * Drive the full corpus build to completion.
- *
- * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → shard hand-off.
- * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
- * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
- */
-export async function buildCorpus(opts) {
-    const adapters = opts.adapters ?? defaultAdapterRegistry.list();
-    const synthesize = opts.synthesize ?? true;
-    const rowsPerShard = opts.rowsPerShard ?? 1_000_000;
-    const built_at = new Date().toISOString();
-    await mkdir(opts.outputDir, { recursive: true });
-    const intermediateDir = join(opts.outputDir, "intermediate");
-    await mkdir(intermediateDir, { recursive: true });
-    // 1. Adapter runs.
-    const adapterRuns = [];
-    const skipped = [];
-    for (const adapter of adapters) {
-        const adapterOptions = opts.adapterInputs[adapter.id];
-        if (!adapterOptions) {
-            skipped.push(adapter.id);
-            opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`);
-            continue;
-        }
-        // Opt-in resume (MAILWOMAN_RESUME=1): if a complete per-adapter canonical.jsonl + MANIFEST.json
-        // already exist, reuse them instead of re-emitting. The MANIFEST is written only after the
-        // canonical is fully flushed, so its presence guarantees completeness; row order is identical,
-        // so downstream holdout-split determinism is preserved. Recovers an align-phase crash without
-        // redoing the (expensive) emit phase. Default (unset) re-emits, preserving correctness. (2026-06-12.)
-        const adapterDir = join(intermediateDir, adapter.id);
-        const cachedManifest = join(adapterDir, "MANIFEST.json");
-        if ($public.MAILWOMAN_RESUME === "1" &&
-            existsSync(cachedManifest) &&
-            existsSync(join(adapterDir, "canonical.jsonl"))) {
-            const cached = parseJSONStrict(readFileSync(cachedManifest, "utf8"));
-            opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`);
-            adapterRuns.push(cached);
-            continue;
-        }
-        opts.onProgress?.("adapter-run", `running ${adapter.id}`);
-        const m = await runAdapter({
-            adapter,
-            adapterOptions,
-            outputDir: intermediateDir,
-            corpusVersion: opts.corpusVersion,
-        });
-        adapterRuns.push(m);
-    }
-    // 2 + 3. Synthesis + alignment: stream every canonical.jsonl, optionally augment, align,
-    // and route each labeled row directly to its split-specific JSONL (`labeled-{train,val,test}.
-    // jsonl`). Memory cost is O(1) — the prior in-memory `splitInputs` array + `splitByIDMap`
-    // + `SplitManifest.{train,val,test}` arrays are gone; per-row split is decided inline via
-    // `splitForRow` (a pure function of source_id + region + holdout policy).
-    const labeledPaths = {
-        train: join(intermediateDir, "labeled-train.jsonl"),
-        val: join(intermediateDir, "labeled-val.jsonl"),
-        test: join(intermediateDir, "labeled-test.jsonl"),
-    };
-    const labeledStreams = {
-        train: createWriteStream(labeledPaths.train, { encoding: "utf8" }),
-        val: createWriteStream(labeledPaths.val, { encoding: "utf8" }),
-        test: createWriteStream(labeledPaths.test, { encoding: "utf8" }),
-    };
-    const quarantinePath = join(intermediateDir, "quarantine.jsonl");
-    const quarantineStream = createWriteStream(quarantinePath, { encoding: "utf8" });
-    let aligned = 0;
-    let quarantined = 0;
-    const counts = { train: 0, val: 0, test: 0 };
-    const holdouts = defaultHoldouts();
-    // License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
-    // license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
-    // everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
-    const excludeLicenses = opts.excludeLicenses ?? [];
-    const licenseCounts = new Map();
-    let excludedByLicense = 0;
-    const writeQuarantine = (row, reason) => {
-        quarantineStream.write(`${JSON.stringify({ row, reason })}\n`);
-    };
-    for (const adapterRun of adapterRuns) {
-        opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
-        for await (const row of streamJSONL(adapterRun.jsonl_path)) {
-            licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
-            // Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
-            // kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
-            // purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
-            // set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
-            if (licenseExcluded(row.license, excludeLicenses)) {
-                excludedByLicense++;
-                continue;
-            }
-            const fanned = [row];
-            if (synthesize) {
-                for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
-                    fanned.push(aug);
-                }
-            }
-            for (const r of fanned) {
-                let result;
-                try {
-                    result = alignRow(r);
-                }
-                catch (error) {
-                    // Last-resort robustness (2026-06-12): no single row may crash a multi-hour build.
-                    // alignRow's targeted paths normalize/quarantine known issues with specific reasons;
-                    // this catches any UNKNOWN throw (e.g. assertSpanInvariants on an unforeseen span
-                    // shape) → quarantine + continue. A spike in `align-threw` reasons is a finding.
-                    writeQuarantine(r, `align-threw:${error.message.slice(0, 160)}`);
-                    quarantined++;
-                    continue;
-                }
-                if (result.kind === "labeled") {
-                    const split = splitForRow(result.row, holdouts);
-                    labeledStreams[split].write(`${JSON.stringify(result.row)}\n`);
-                    counts[split]++;
-                    aligned++;
-                }
-                else {
-                    writeQuarantine(r, result.row.reason);
-                    quarantined++;
-                }
-            }
-        }
-    }
-    for (const s of Object.values(labeledStreams)) {
-        s.end();
-    }
-    quarantineStream.end();
-    await Promise.all([...Object.values(labeledStreams).map(streamEnd), streamEnd(quarantineStream)]);
-    // 4. Splits — manifest derived by streaming the per-split labeled files; no in-memory
-    // source-id arrays. `sort(1)` from coreutils produces the deterministic per-split .txt
-    // manifests with disk spill for splits that exceed in-memory thresholds.
-    opts.onProgress?.("split", `splitting ${aligned} aligned rows`);
-    const splitsDir = join(opts.outputDir, "splits");
-    const splitCounts = await writeSplitManifestsFromLabeledFiles({
-        labeledPaths,
-        outputDir: splitsDir,
-        corpusVersion: opts.corpusVersion,
-        counts,
-        holdouts,
-    });
-    // 5. Parquet shards — per-split labeled JSONL streams in, sharded `.parquet` out. The prior
-    // `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
-    opts.onProgress?.("shard", "writing parquet shards");
-    const shardManifest = await writeShards({
-        train: streamJSONL(labeledPaths.train),
-        val: streamJSONL(labeledPaths.val),
-        test: streamJSONL(labeledPaths.test),
-    }, {
-        outputDir: opts.outputDir,
-        corpusVersion: opts.corpusVersion,
-        rowsPerShard,
-    });
-    // License-set visibility (#26): loudly report the resolved license set so a build is an obvious
-    // deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
-    const licenseSummary = [...licenseCounts.entries()].toSorted((a, b) => b[1] - a[1]);
-    opts.onProgress?.("manifest", `license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
-        (excludedByLicense > 0
-            ? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
-            : " | NO license exclusion applied (all rows kept)"));
-    // 6. Top-level manifest.
-    opts.onProgress?.("manifest", "writing top-level MANIFEST.json");
-    const manifest = {
-        corpus_version: opts.corpusVersion,
-        built_at,
-        adapters: adapterRuns,
-        skipped_adapters: skipped,
-        splits: { counts: splitCounts, holdouts },
-        shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
-        quarantine_count: quarantined,
-        total_aligned_rows: aligned,
-        licenses: Object.fromEntries(licenseSummary),
-        excluded_by_license: excludedByLicense,
-    };
-    await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
-    return manifest;
-}
-async function* streamJSONL(path) {
-    // JSONSpliterator yields already-parsed rows (skipEmpty is on by default, so blank
-    // lines are dropped at the row level) and throws SyntaxError on a malformed row —
-    // same fail-loud behavior as the prior readline + bare `JSON.parse`.
-    yield* JSONSpliterator.fromAsync(path);
-}
-function streamEnd(s) {
-    return new Promise((resolve, reject) => {
-        s.once("close", resolve);
-        s.once("error", reject);
-    });
-}
-//# sourceMappingURL=build.js.map