@mailwoman/corpus 9.1.0 → 9.3.0

package/out/adapters/wof-json-rows.js

@@ -0,0 +1,83 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Deterministic canonical-row emission shared by WOF GeoJSON adapters.
+ */
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter";
+import { normalizeNameKey } from "#utils";
+/**
+ * Display name for the country, keyed by ISO 3166-1 alpha-2.
+ *
+ * Must be the **OpenCage-canonical** surface form: the `address-formatter` library expands some country names en route
+ * to its output (e.g. `"United States"` → `"United States of America"`). If `components.country` and the formatted
+ * `raw` disagree, alignment will fail downstream. Keying off the canonical form keeps the two in lockstep.
+ *
+ * Phase 1 US + FR only; extend as new locales come online. Missing countries fall back to the country row's `wof:name`,
+ * accepting the alignment risk for non-canonicalized names.
+ */
+export const COUNTRY_DISPLAY_NAME = {
+    US: "United States of America",
+    FR: "France",
+};
+/**
+ * BCP-47 locale defaulting for the corpus row's `locale` field. Defaulted by country.
+ */
+export const LOCALE_BY_COUNTRY = {
+    US: "en-US",
+    FR: "fr-FR",
+};
+/**
+ * Build the per-record name-slot list: the canonical `"default"` slot, then every `name:*` variant deduplicated against
+ * it so a redundant `"default"`-equivalent row is not emitted under a localized key.
+ */
+export function nameSlotsFor(rec, options = {}) {
+    const canonicalSelfName = options.canonicalName?.(rec) ?? rec.name;
+    const seen = new Set([canonicalSelfName]);
+    const slots = [{ key: "default", value: canonicalSelfName }];
+    for (const [rawKey, value] of rec.nameVariants) {
+        if (seen.has(value))
+            continue;
+        seen.add(value);
+        slots.push({ key: normalizeNameKey(rawKey), value });
+    }
+    return slots;
+}
+/**
+ * Emit aligned rows in WOF-id order, enforcing the adapter limit across all name and hierarchy variants.
+ */
+export function* emitWOFJSONRows(options) {
+    const { records, ancestry, adapterOptions, adapterID, localeByCountry, shouldEmit = () => true, nameSlotsFor: slotsForRecord, variantsFor, } = options;
+    let emitted = 0;
+    for (const id of [...records.keys()].toSorted((a, b) => a - b)) {
+        if (adapterOptions.signal?.aborted)
+            return;
+        const record = records.get(id);
+        if (!shouldEmit(record))
+            continue;
+        for (const slot of slotsForRecord(record)) {
+            for (const variant of variantsFor(record, ancestry.get(id) ?? [], slot.value)) {
+                if (adapterOptions.limit !== undefined && emitted >= adapterOptions.limit)
+                    return;
+                const raw = formatAddress(variant.components, record.country, { separator: ", " });
+                if (!raw)
+                    continue;
+                const components = reconcileComponents(variant.components, raw);
+                if (!Object.keys(components).length)
+                    continue;
+                yield {
+                    raw,
+                    components,
+                    country: record.country,
+                    locale: localeByCountry[record.country],
+                    source: adapterID,
+                    source_id: `${adapterID}-${record.id}-${slot.key}-${variant.suffix}`,
+                    corpus_version: "",
+                    license: "CC0-1.0",
+                };
+                emitted++;
+            }
+        }
+    }
+}
+//# sourceMappingURL=wof-json-rows.js.map

package/out/adapters/wof-json-rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wof-json-rows.js","sourceRoot":"","sources":["../../lib/adapters/wof-json-rows.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAGzE,OAAO,EAAE,gBAAgB,EAAkB,MAAM,QAAQ,CAAA;AAEzD;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAA2B;IAC3D,EAAE,EAAE,0BAA0B;IAC9B,EAAE,EAAE,QAAQ;CACZ,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA2B;IACxD,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;CACX,CAAA;AAeD;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAc,EAAE,UAA2B,EAAE;IACzE,MAAM,iBAAiB,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAA;IAElE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,iBAAiB,CAAC,CAAC,CAAA;IACjD,MAAM,KAAK,GAA0C,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAA;IAEnG,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAQ;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACf,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAaD;;GAEG;AACH,MAAM,SAAS,CAAC,CAAC,eAAe,CAAC,OAA+B;IAC/D,MAAM,EACL,OAAO,EACP,QAAQ,EACR,cAAc,EACd,SAAS,EACT,eAAe,EACf,UAAU,GAAG,GAAG,EAAE,CAAC,IAAI,EACvB,YAAY,EAAE,cAAc,EAC5B,WAAW,GACX,GAAG,OAAO,CAAA;IAEX,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAChE,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAA;QAE/B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAQ;QAEjC,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,KAAK,MAAM,OAAO,IAAI,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/E,IAAI,cAAc,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,cAAc,CAAC,KAAK;oBAAE,OAAM;gBAEjF,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;gBAElF,IAAI,CAAC,GAAG;oBAAE,SAAQ;gBAClB,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;gBAE/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM;oBAAE,SAAQ;gBAE7C,MAAM;oBACL,GAAG;oBACH,UAAU;oBACV,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;oBACvC,MAAM,EAAE,SAAS;oBACjB,SAAS,EAAE,GAAG,SAAS,IAAI,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE;oBACpE,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,SAAS;iBAClB,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC"}

package/out/adapters/wof-postalcode-json/adapter.d.ts

@@ -0,0 +1,65 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `wof-postalcode`: Who's On First postalcode GeoJSON-bundle adapter.
+ *
+ *   **Phase 1.5.1 pivot.** Replaces the previous SpatiaLite-backed implementation (formerly at
+ *   `packages/corpus/lib/adapters/wof-postalcode/`, removed in this same change). The rationale is
+ *   in `wof-admin-json/adapter.ts` and in `DECISIONS.md` — short version: the SQLite distribution
+ *   mirror is dead, the live distro tags every postcode row `mz:is_current = -1` which the old
+ *   `is_current = 1` predicate excluded, and localized `name:*` variants don't ship in the SQLite
+ *   export at all.
+ *
+ *   Input: a directory containing one or more cloned `whosonfirst-data-postalcode-<cc>` repos plus
+ *   the relevant `whosonfirst-data-admin-<cc>` repos (postcode records reference admin ancestry by
+ *   `wof:parent_id`, so the locality / region / country records must be in the same walk for the
+ *   ancestry chain to resolve). The corpus pipeline clones all four repos under
+ *   `/data/corpus/sources/wof/repos/` and points the adapter at that root.
+ *
+ *   Per live postalcode record, the adapter emits one row per `(name-variant, hierarchy-variant)`
+ *   pair:
+ *
+ *   - **Name variants**: canonical `wof:name` (slot key `default`, typically the postcode digits
+ *       themselves) plus any `name:*` variants on the postcode feature. In practice WOF postcode
+ *       records rarely carry localized name variants, so this expansion is usually a no-op — but
+ *       the code path stays symmetric with the admin adapter for consistency.
+ *   - **Hierarchy variants** (unchanged from the SQLite adapter): self, +locality, +locality+region,
+ *       +locality+region+country.
+ *
+ *   `source_id` is `wof-postalcode-<wof_id>-<name-slot>-<hierarchy-variant>`. Ancestor names always
+ *   come from the ancestor's canonical `wof:name`; this adapter does NOT iterate ancestor name
+ *   variants (e.g. it does not emit `"75008 Париж"` even when Paris has a `name:rus_x_preferred`).
+ *   That cross-product belongs to a future synthesis pass; emitting it here would multiply row
+ *   counts ~10× without a clear training-value story.
+ *
+ *   License: CC0.
+ */
+import { type WOFVariantSpec } from "#adapters/wof-json-rows";
+import type { CorpusAdapter } from "#types";
+import { type WOFRecord } from "#utils";
+/**
+ * Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
+ * the `default` slot, a `name:*` localized variant otherwise).
+ */
+export declare function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[];
+/**
+ * Build the per-record name-slot list. The `default` slot uses `wof:name` verbatim (postcode digits); subsequent slots
+ * come from `name:*` variants dedup'd against the default.
+ */
+export declare function nameSlotsFor(rec: WOFRecord): Array<{
+    key: string;
+    value: string;
+}>;
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export declare const WOF_POSTALCODE_ADAPTER_ID = "wof-postalcode";
+export declare function createWOFPostalcodeAdapter(): CorpusAdapter;
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export declare const wofPostalcodeAdapter: CorpusAdapter;
+//# sourceMappingURL=adapter.d.ts.map

package/out/adapters/wof-postalcode-json/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAKH,OAAO,EAKN,KAAK,cAAc,EACnB,MAAM,yBAAyB,CAAA;AAChC,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AACzE,OAAO,EAAoC,KAAK,SAAS,EAAE,MAAM,QAAQ,CAAA;AA0BzE;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAqC7G;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAElF;AAED;;;GAGG;AACH,eAAO,MAAM,yBAAyB,mBAAmB,CAAA;AAEzD,wBAAgB,0BAA0B,IAAI,aAAa,CAqC1D;AAED;;GAEG;AACH,eAAO,MAAM,oBAAoB,eAA+B,CAAA"}

package/out/adapters/wof-postalcode-json/adapter.js

@@ -0,0 +1,151 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `wof-postalcode`: Who's On First postalcode GeoJSON-bundle adapter.
+ *
+ *   **Phase 1.5.1 pivot.** Replaces the previous SpatiaLite-backed implementation (formerly at
+ *   `packages/corpus/lib/adapters/wof-postalcode/`, removed in this same change). The rationale is
+ *   in `wof-admin-json/adapter.ts` and in `DECISIONS.md` — short version: the SQLite distribution
+ *   mirror is dead, the live distro tags every postcode row `mz:is_current = -1` which the old
+ *   `is_current = 1` predicate excluded, and localized `name:*` variants don't ship in the SQLite
+ *   export at all.
+ *
+ *   Input: a directory containing one or more cloned `whosonfirst-data-postalcode-<cc>` repos plus
+ *   the relevant `whosonfirst-data-admin-<cc>` repos (postcode records reference admin ancestry by
+ *   `wof:parent_id`, so the locality / region / country records must be in the same walk for the
+ *   ancestry chain to resolve). The corpus pipeline clones all four repos under
+ *   `/data/corpus/sources/wof/repos/` and points the adapter at that root.
+ *
+ *   Per live postalcode record, the adapter emits one row per `(name-variant, hierarchy-variant)`
+ *   pair:
+ *
+ *   - **Name variants**: canonical `wof:name` (slot key `default`, typically the postcode digits
+ *       themselves) plus any `name:*` variants on the postcode feature. In practice WOF postcode
+ *       records rarely carry localized name variants, so this expansion is usually a no-op — but
+ *       the code path stays symmetric with the admin adapter for consistency.
+ *   - **Hierarchy variants** (unchanged from the SQLite adapter): self, +locality, +locality+region,
+ *       +locality+region+country.
+ *
+ *   `source_id` is `wof-postalcode-<wof_id>-<name-slot>-<hierarchy-variant>`. Ancestor names always
+ *   come from the ancestor's canonical `wof:name`; this adapter does NOT iterate ancestor name
+ *   variants (e.g. it does not emit `"75008 Париж"` even when Paris has a `name:rus_x_preferred`).
+ *   That cross-product belongs to a future synthesis pass; emitting it here would multiply row
+ *   counts ~10× without a clear training-value story.
+ *
+ *   License: CC0.
+ */
+import { COUNTRY_DISPLAY_NAME, emitWOFJSONRows, LOCALE_BY_COUNTRY, nameSlotsFor as wofNameSlotsFor, } from "#adapters/wof-json-rows";
+import { buildAncestryIndex, walkFeatures } from "#utils";
+/**
+ * Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip.
+ *
+ * Per-adapter deliberately (the admin adapter carries its own): each table is a record FILTER for its adapter's
+ * emission set — this one keeps `postalcode` plus the ancestry placetypes its variants render — not a shared
+ * vocabulary.
+ */
+function placetypeToTag(placetype) {
+    switch (placetype) {
+        case "country":
+        case "nation":
+            return "country";
+        case "macroregion":
+        case "region":
+            return "region";
+        case "locality":
+            return "locality";
+        case "postalcode":
+            return "postcode";
+        default:
+            return undefined;
+    }
+}
+/**
+ * Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
+ * the `default` slot, a `name:*` localized variant otherwise).
+ */
+export function postcodeVariantsFor(row, ancestry, selfName) {
+    if (placetypeToTag(row.placetype) !== "postcode")
+        return [];
+    const locality = ancestry.find((a) => placetypeToTag(a.placetype) === "locality");
+    const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region");
+    const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country");
+    const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country;
+    const variants = [{ suffix: "self", components: { postcode: selfName } }];
+    if (locality) {
+        variants.push({
+            suffix: "with-locality",
+            components: { postcode: selfName, locality: locality.name },
+        });
+    }
+    if (locality && region) {
+        variants.push({
+            suffix: "with-locality-region",
+            components: { postcode: selfName, locality: locality.name, region: region.name },
+        });
+    }
+    if (locality && region && country) {
+        variants.push({
+            suffix: "with-locality-region-country",
+            components: {
+                postcode: selfName,
+                locality: locality.name,
+                region: region.name,
+                country: countryDisplay,
+            },
+        });
+    }
+    return variants;
+}
+/**
+ * Build the per-record name-slot list. The `default` slot uses `wof:name` verbatim (postcode digits); subsequent slots
+ * come from `name:*` variants dedup'd against the default.
+ */
+export function nameSlotsFor(rec) {
+    return wofNameSlotsFor(rec);
+}
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const WOF_POSTALCODE_ADAPTER_ID = "wof-postalcode";
+export function createWOFPostalcodeAdapter() {
+    return {
+        id: WOF_POSTALCODE_ADAPTER_ID,
+        defaultLicense: "CC0-1.0",
+        description: "Who's On First postalcode GeoJSON bundles (postcode → locality/region pairs). Ancestor names from sibling admin repos.",
+        async *rows(opts) {
+            // Pass 1: full walk. We keep every record whose placetype maps to a ComponentTag — the
+            // postcode adapter needs locality / region / country admin records in the index so it
+            // can resolve postcode ancestry, even though it only emits rows for postcode records.
+            const byID = new Map();
+            for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
+                if (opts.signal?.aborted)
+                    return;
+                if (opts.country && rec.country !== opts.country)
+                    continue;
+                if (!placetypeToTag(rec.placetype))
+                    continue;
+                byID.set(rec.id, rec);
+            }
+            const ancestry = buildAncestryIndex(byID);
+            // Pass 2: emit postcode rows only, sorted by id for determinism.
+            yield* emitWOFJSONRows({
+                records: byID,
+                ancestry,
+                adapterOptions: opts,
+                adapterID: WOF_POSTALCODE_ADAPTER_ID,
+                localeByCountry: LOCALE_BY_COUNTRY,
+                shouldEmit: (record) => placetypeToTag(record.placetype) === "postcode",
+                nameSlotsFor,
+                variantsFor: postcodeVariantsFor,
+            });
+        },
+    };
+}
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const wofPostalcodeAdapter = createWOFPostalcodeAdapter();
+//# sourceMappingURL=adapter.js.map

package/out/adapters/wof-postalcode-json/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../lib/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAKH,OAAO,EACN,oBAAoB,EACpB,eAAe,EACf,iBAAiB,EACjB,YAAY,IAAI,eAAe,GAE/B,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAkB,MAAM,QAAQ,CAAA;AAEzE;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,SAAwC;IAC/D,QAAQ,SAAS,EAAE,CAAC;QACnB,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ;YACZ,OAAO,SAAS,CAAA;QACjB,KAAK,aAAa,CAAC;QACnB,KAAK,QAAQ;YACZ,OAAO,QAAQ,CAAA;QAChB,KAAK,UAAU;YACd,OAAO,UAAU,CAAA;QAClB,KAAK,YAAY;YAChB,OAAO,UAAU,CAAA;QAClB;YACC,OAAO,SAAS,CAAA;IAClB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAc,EAAE,QAAqB,EAAE,QAAgB;IAC1F,IAAI,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,UAAU;QAAE,OAAO,EAAE,CAAA;IAE3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,UAAU,CAAC,CAAA;IACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAA;IAC7E,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC,CAAA;IAC/E,MAAM,cAAc,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,IAAI,IAAI,GAAG,CAAC,OAAO,CAAA;IAExF,MAAM,QAAQ,GAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE3F,IAAI,QAAQ,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,eAAe;YACvB,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE;SAC3D,CAAC,CAAA;IACH,CAAC;IAED,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,sBAAsB;YAC9B,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;SAChF,CAAC,CAAA;IACH,CAAC;IAED,IAAI,QAAQ,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,8BAA8B;YACtC,UAAU,EAAE;gBACX,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,OAAO,EAAE,cAAc;aACvB;SACD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAc;IAC1C,OAAO,eAAe,CAAC,GAAG,CAAC,CAAA;AAC5B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,CAAA;AAEzD,MAAM,UAAU,0BAA0B;IACzC,OAAO;QACN,EAAE,EAAE,yBAAyB;QAC7B,cAAc,EAAE,SAAS;QACzB,WAAW,EACV,wHAAwH;QAEzH,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,uFAAuF;YACvF,sFAAsF;YACtF,sFAAsF;YACtF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAA;YAEzC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;gBAC/E,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,OAAM;gBAEhC,IAAI,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;oBAAE,SAAQ;gBAE1D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC;oBAAE,SAAQ;gBAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;YAED,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAEzC,iEAAiE;YACjE,KAAK,CAAC,CAAC,eAAe,CAAC;gBACtB,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,cAAc,EAAE,IAAI;gBACpB,SAAS,EAAE,yBAAyB;gBACpC,eAAe,EAAE,iBAAiB;gBAClC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,UAAU;gBACvE,YAAY;gBACZ,WAAW,EAAE,mBAAmB;aAChC,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,0BAA0B,EAAE,CAAA"}

package/out/build.d.ts

@@ -0,0 +1,132 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   End-to-end corpus build (Phase 1 task #10 in the plan).
+ *
+ *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
+ *
+ *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
+ *        `<intermediate>/<adapter.id>/canonical.jsonl` slices.
+ *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
+ *        country-default policy emits an augmented row alongside the original.
+ *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
+ *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
+ *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
+ *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
+ *        `test.txt`.
+ *   5. **Parquet slices** — `writeSlices` streams labeled rows into 1M-row `.parquet` slices per split
+ *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
+ *        row groups), with per-slice checksums + per-stage manifest in
+ *        `corpus-v<version>/MANIFEST.json`.
+ *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
+ *        a top-level corpus_version, built_at, and aggregate counts.
+ *
+ *   Output layout:
+ *
+ *   ```
+ *   <outputDir>/
+ *   MANIFEST.json
+ *   intermediate/
+ *     <adapter.id>/canonical.jsonl   # one per adapter
+ *     labeled.jsonl                  # post-alignment, pre-slice
+ *     quarantine.jsonl               # rows that failed alignment
+ *   splits/
+ *     SPLIT_MANIFEST.json
+ *     train.txt / val.txt / test.txt
+ *   corpus-v<version>/
+ *     MANIFEST.json
+ *     train/part-NNNN.parquet
+ *     val/part-NNNN.parquet
+ *     test/part-NNNN.parquet
+ * ```
+ *
+ *   The intermediate files live alongside the final slices for reproducibility + debugging. Operators
+ *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
+ *   self-contained.
+ */
+import { type AdapterRunManifest } from "#runner";
+import type { AdapterOptions, CorpusAdapter } from "#types";
+import { type SliceManifest } from "#utils/parquet";
+import { type SplitManifest } from "#utils/split";
+/**
+ * Stage tags surfaced to `onProgress`.
+ */
+export type BuildStage = "adapter-run" | "align" | "split" | "slice" | "manifest";
+/**
+ * Per-invocation options for `buildCorpus`.
+ */
+export interface BuildCorpusOptions {
+    /**
+     * Root output directory. All build artifacts land beneath it.
+     */
+    outputDir: string;
+    /**
+     * Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name.
+     */
+    corpusVersion: string;
+    /**
+     * Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list to filter (e.g.
+     * `[wofAdminAdapter]` for a smoke run).
+     */
+    adapters?: readonly CorpusAdapter[];
+    /**
+     * Per-adapter `AdapterOptions` — looked up by adapter id. Adapters whose id is missing from this map are skipped (and
+     * noted in the manifest).
+     */
+    adapterInputs: Record<string, AdapterOptions>;
+    /**
+     * Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests.
+     */
+    synthesize?: boolean;
+    /**
+     * Forwarded to `writeSlices`. Default 1_000_000.
+     */
+    rowsPerSlice?: number;
+    /**
+     * Progress hook. Errors thrown abort the build.
+     */
+    onProgress?: (stage: BuildStage, message: string) => void;
+    /**
+     * License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see `compileLicenseExcludes` /
+     * `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license` matches any is dropped at ingest. Default (omitted)
+     * includes EVERYTHING — exclusion is a deliberate act, not a silent default. A proprietary-weights build passes the
+     * share-alike set (`--exclude-share-alike`).
+     */
+    excludeLicenses?: readonly RegExp[];
+}
+/**
+ * Top-level manifest tying every stage together.
+ */
+export interface BuildCorpusManifest {
+    corpus_version: string;
+    built_at: string;
+    adapters: AdapterRunManifest[];
+    skipped_adapters: string[];
+    splits: {
+        counts: SplitManifest["counts"];
+        holdouts: SplitManifest["holdouts"];
+    };
+    slices: {
+        counts: SliceManifest["counts"];
+        total_rows: number;
+    };
+    quarantine_count: number;
+    total_aligned_rows: number;
+    /**
+     * Resolved license set across all INCLUDED rows (license string → row count), + the count dropped by
+     * `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
+     */
+    licenses: Record<string, number>;
+    excluded_by_license: number;
+}
+/**
+ * Drive the full corpus build to completion.
+ *
+ * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → slice hand-off.
+ * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
+ * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
+ */
+export declare function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpusManifest>;
+//# sourceMappingURL=build.d.ts.map

package/out/build.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../lib/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAUH,OAAO,EAAoB,KAAK,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAEnE,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,QAAQ,CAAA;AAGrF,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,cAAc,CAAA;AAErB;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAyNxF"}