@mailwoman/corpus 2.0.0

package/out/src/runner.js

@@ -0,0 +1,183 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
+ *   per-shard manifest.
+ *
+ *   Output layout under `outputDir`:
+ *
+ *   ```
+ *   <outputDir>/<adapter.id>/
+ *   canonical.jsonl       # one row per line, in emission order
+ *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
+ * ```
+ *
+ *   The runner is responsible for everything an adapter is **not** responsible for:
+ *
+ *   - Stamping `corpus_version` on every row (adapters must NOT set it).
+ *   - Applying `canonicalDedupKey` and skipping duplicates.
+ *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
+ *   - Honoring backpressure on the output write stream.
+ *   - Counting + emitting periodic progress to an optional callback.
+ *   - Honoring `signal` (delegates to adapter's iteration boundary).
+ *
+ *   The runner does NOT perform alignment, tokenization, synthesis, or sharding into Parquet. Those
+ *   steps run later, consuming the JSONL shards this writes.
+ */
+import { createWriteStream } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { canonicalDedupKey, streamingSha256 } from "./adapter.js";
+/**
+ * Drive a single adapter to completion.
+ *
+ * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under
+ * `outputDir/<adapter.id>/`. Throws if the output directory cannot be created, if a row arrives
+ * with a missing required field, or if the abort signal fires.
+ */
+export async function runAdapter(opts) {
+    const { adapter, adapterOptions, outputDir, corpusVersion } = opts;
+    const progressEvery = opts.progressEvery ?? 1_000;
+    const adapterDir = join(outputDir, adapter.id);
+    await mkdir(adapterDir, { recursive: true });
+    const jsonlPath = join(adapterDir, "canonical.jsonl");
+    const manifestPath = join(adapterDir, "MANIFEST.json");
+    const startedAt = new Date();
+    const t0 = performance.now();
+    const stream = createWriteStream(jsonlPath, { encoding: "utf8" });
+    const hasher = streamingSha256();
+    const seen = new Set();
+    const DEDUP_MAX_SIZE = 10_000_000;
+    let dedupExhausted = false;
+    let yielded = 0;
+    let written = 0;
+    let bytes = 0;
+    const emitProgress = () => {
+        opts.onProgress?.({
+            adapterId: adapter.id,
+            yielded,
+            written,
+            bytes,
+            elapsed_ms: performance.now() - t0,
+        });
+    };
+    try {
+        for await (const row of adapter.rows(adapterOptions)) {
+            if (adapterOptions.signal?.aborted) {
+                throw new DOMException("Adapter run aborted by signal", "AbortError");
+            }
+            yielded++;
+            assertEmittedRow(adapter, row);
+            const stamped = { ...row, corpus_version: corpusVersion };
+            const key = canonicalDedupKey(stamped);
+            if (!dedupExhausted) {
+                if (seen.has(key)) {
+                    if (yielded % progressEvery === 0)
+                        emitProgress();
+                    continue;
+                }
+                if (seen.size >= DEDUP_MAX_SIZE) {
+                    dedupExhausted = true;
+                    process.stderr.write(`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`);
+                }
+                else {
+                    seen.add(key);
+                }
+            }
+            const line = `${JSON.stringify(stamped)}\n`;
+            hasher.update(line);
+            bytes += Buffer.byteLength(line, "utf8");
+            written++;
+            if (!stream.write(line)) {
+                await once(stream, "drain");
+            }
+            if (yielded % progressEvery === 0)
+                emitProgress();
+        }
+    }
+    finally {
+        stream.end();
+        await once(stream, "close");
+    }
+    const endedAt = new Date();
+    const elapsed_ms = performance.now() - t0;
+    emitProgress();
+    const manifest = {
+        adapter_id: adapter.id,
+        corpus_version: corpusVersion,
+        default_license: adapter.defaultLicense,
+        description: adapter.description,
+        yielded,
+        written,
+        deduped: yielded - written,
+        bytes,
+        sha256: hasher.digest(),
+        jsonl_path: jsonlPath,
+        started_at: startedAt.toISOString(),
+        ended_at: endedAt.toISOString(),
+        elapsed_ms,
+    };
+    await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+    return manifest;
+}
+/**
+ * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the
+ * registry before calling if partial-failure is desired).
+ *
+ * Returns the manifests in registry insertion order.
+ */
+export async function runAllAdapters(registry, common) {
+    const out = [];
+    for (const adapter of registry.list()) {
+        const adapterOptions = common.adapterOptionsFor?.(adapter) ?? common.adapterOptions;
+        out.push(await runAdapter({
+            ...common,
+            adapter,
+            adapterOptions,
+        }));
+    }
+    return out;
+}
+/**
+ * Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL
+ * doesn't end up half-malformed.
+ */
+function assertEmittedRow(adapter, row) {
+    if (row.source !== adapter.id) {
+        throw new Error(`adapter ${adapter.id}: row.source must equal adapter.id (got ${JSON.stringify(row.source)})`);
+    }
+    if (!row.source_id) {
+        throw new Error(`adapter ${adapter.id}: row.source_id is empty`);
+    }
+    if (!row.raw) {
+        throw new Error(`adapter ${adapter.id}: row.raw is empty for source_id=${row.source_id}`);
+    }
+    if (!row.country) {
+        throw new Error(`adapter ${adapter.id}: row.country is empty for source_id=${row.source_id}`);
+    }
+    if (!row.license) {
+        throw new Error(`adapter ${adapter.id}: row.license is empty for source_id=${row.source_id}`);
+    }
+}
+/** Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. */
+function once(emitter, event) {
+    return new Promise((resolve, reject) => {
+        const onEvent = () => {
+            emitter.off("error", onError);
+            resolve();
+        };
+        const onError = (err) => {
+            emitter.off(event, onEvent);
+            reject(err);
+        };
+        emitter.once(event, onEvent);
+        emitter.once("error", onError);
+    });
+}
+/** Convenience: ensure the parent directory of `filePath` exists. */
+export async function ensureParentDir(filePath) {
+    await mkdir(dirname(filePath), { recursive: true });
+}
+//# sourceMappingURL=runner.js.map

package/out/src/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AACzC,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAA8C,MAAM,cAAc,CAAA;AAiE7G;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAuB;IACvD,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,IAAI,CAAA;IAClE,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,KAAK,CAAA;IAEjD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAC9C,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAA;IACrD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;IAEtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAA;IAC5B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAE5B,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjE,MAAM,MAAM,GAAoB,eAAe,EAAE,CAAA;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,cAAc,GAAG,UAAU,CAAA;IACjC,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,YAAY,GAAG,GAAS,EAAE;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,OAAO;YACP,OAAO;YACP,KAAK;YACL,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;SAClC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACtD,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpC,MAAM,IAAI,YAAY,CAAC,+BAA+B,EAAE,YAAY,CAAC,CAAA;YACtE,CAAC;YAED,OAAO,EAAE,CAAA;YACT,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YAE9B,MAAM,OAAO,GAAiB,EAAE,GAAG,GAAG,EAAE,cAAc,EAAE,aAAa,EAAE,CAAA;YACvE,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;YACtC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC;wBAAE,YAAY,EAAE,CAAA;oBACjD,SAAQ;gBACT,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;oBACjC,cAAc,GAAG,IAAI,CAAA;oBACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,+BAA+B,cAAc,CAAC,cAAc,EAAE,wCAAwC,CACtG,CAAA;gBACF,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACd,CAAC;YACF,CAAC;YAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAA;YAC3C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACnB,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YACxC,OAAO,EAAE,CAAA;YAET,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAC5B,CAAC;YAED,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC;gBAAE,YAAY,EAAE,CAAA;QAClD,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAA;IACzC,YAAY,EAAE,CAAA;IAEd,MAAM,QAAQ,GAAuB;QACpC,UAAU,EAAE,OAAO,CAAC,EAAE;QACtB,cAAc,EAAE,aAAa;QAC7B,eAAe,EAAE,OAAO,CAAC,cAAc;QACvC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO;QACP,OAAO;QACP,OAAO,EAAE,OAAO,GAAG,OAAO;QAC1B,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QACvB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;QAC/B,UAAU;KACV,CAAA;IAED,MAAM,SAAS,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE/E,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,QAAyB,EACzB,MAAyG;IAEzG,MAAM,GAAG,GAAyB,EAAE,CAAA;IACpC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,cAAc,CAAA;QACnF,GAAG,CAAC,IAAI,CACP,MAAM,UAAU,CAAC;YAChB,GAAG,MAAM;YACT,OAAO;YACP,cAAc;SACd,CAAC,CACF,CAAA;IACF,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAAsB,EAAE,GAAiB;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,2CAA2C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC/G,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACjE,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,oCAAoC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1F,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;AACF,CAAC;AAED,gGAAgG;AAChG,SAAS,IAAI,CAAC,OAAoB,EAAE,KAAwB;IAC3D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,GAAS,EAAE;YAC1B,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7B,OAAO,EAAE,CAAA;QACV,CAAC,CAAA;QACD,MAAM,OAAO,GAAG,CAAC,GAAU,EAAQ,EAAE;YACpC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3B,MAAM,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACrD,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AACpD,CAAC"}

package/out/src/split.d.ts

@@ -0,0 +1,108 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ *
+ *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
+ *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
+ *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
+ *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
+ *   memorization, not by learning the underlying schema.
+ *
+ *   Phase 1 holdouts (chosen for low data density + administrative isolation):
+ *
+ *   - **US**: Vermont, Wyoming, North Dakota
+ *   - **FR**: Corse, Lozère, Creuse
+ *
+ *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
+ *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
+ *   the locality boundary, not the exact split percentages.
+ *
+ *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
+ *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ */
+import type { CanonicalRow, LabeledRow } from "./types.js";
+export type SplitName = "train" | "val" | "test";
+export interface SplitOptions {
+    /**
+     * Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the
+     * region-component strings the splitter looks for in `row.components.region`. Override to change
+     * the holdout for an experiment; defaults to `defaultHoldouts()`.
+     */
+    holdouts?: Record<string, readonly string[]>;
+}
+/** Output manifest: source_id lists per split. */
+export interface SplitManifest {
+    train: string[];
+    val: string[];
+    test: string[];
+    /** Echoes the holdouts used, so the manifest is self-describing. */
+    holdouts: Record<string, readonly string[]>;
+    /** Corpus version stamped onto the manifest. Read from the first row. */
+    corpus_version: string;
+    /** Counts for quick sanity checks. */
+    counts: {
+        train: number;
+        val: number;
+        test: number;
+        total: number;
+    };
+}
+/**
+ * Phase 1 default holdouts (per plan).
+ *
+ * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
+ * - FR: Corse, Lozère, Creuse (small departments / regions).
+ */
+export declare function defaultHoldouts(): Record<string, readonly string[]>;
+type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">;
+/**
+ * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming
+ * `buildCorpus` align loop (`build.ts`) to decide each row's split without retaining the row in
+ * heap. Identical hash bucketing semantics to the array-based path so the decision is stable
+ * regardless of caller.
+ */
+export declare function splitForRow(row: Pick<SplitInputRow, "source_id" | "country" | "components">, holdouts?: Record<string, readonly string[]>): SplitName;
+/**
+ * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are
+ * accepted — only `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ *
+ * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via
+ * `buildCorpus` use the streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to
+ * avoid materializing every aligned row's split membership in heap.
+ */
+export declare function splitRows(rows: Iterable<SplitInputRow>, opts?: SplitOptions): SplitManifest;
+/** Lightweight deterministic 0..(n-1) bucket from a string id. */
+export declare function hashBucket(id: string, n: number): number;
+/**
+ * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated
+ * source_id lists (one id per line) so they diff cleanly in git. Also writes
+ * `<outputDir>/MANIFEST.json` with the full structured manifest including holdouts + counts +
+ * corpus version.
+ *
+ * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ */
+export declare function writeSplitManifests(manifest: SplitManifest, outputDir: string): Promise<void>;
+/** Type re-export for callers that want to ingest LabeledRow specifically. */
+export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">;
+/**
+ * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests +
+ * `SPLIT_MANIFEST.json` by streaming three per-split labeled-row JSONL files (one per split).
+ * Memory cost is O(1) — `sort(1)` from coreutils handles the deterministic sort with disk spill for
+ * files that exceed in-memory thresholds.
+ *
+ * Used by `buildCorpus` after the align loop has already partitioned labeled rows into
+ * `labeled-{train,val,test}.jsonl` via `splitForRow`. Counts are pre-computed by the align loop and
+ * passed in (zero re-scan).
+ */
+export declare function writeSplitManifestsFromLabeledFiles(opts: {
+    labeledPaths: Record<SplitName, string>;
+    outputDir: string;
+    corpusVersion: string;
+    counts: Record<SplitName, number>;
+    holdouts?: Record<string, readonly string[]>;
+}): Promise<SplitManifest["counts"]>;
+export {};
+//# sourceMappingURL=split.d.ts.map

package/out/src/split.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../src/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAQH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE1D,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAA;AAEhD,MAAM,WAAW,YAAY;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C;AAED,kDAAkD;AAClD,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IAC3C,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAA;IACtB,sCAAsC;IACtC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACnE;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAKnE;AAED,KAAK,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAElG;;;;;GAKG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,GAAG,SAAS,GAAG,YAAY,CAAC,EAChE,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAqB,GAC7D,SAAS,CAOX;AAED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,GAAE,YAAiB,GAAG,aAAa,CAwB/F;AAED,kEAAkE;AAClE,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYnG;AAED,8EAA8E;AAC9E,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAE9G;;;;;;;;;GASG;AACH,wBAAsB,mCAAmC,CAAC,IAAI,EAAE;IAC/D,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACvC,SAAS,EAAE,MAAM,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAkBnC"}

package/out/src/split.js

@@ -0,0 +1,191 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ *
+ *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
+ *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
+ *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
+ *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
+ *   memorization, not by learning the underlying schema.
+ *
+ *   Phase 1 holdouts (chosen for low data density + administrative isolation):
+ *
+ *   - **US**: Vermont, Wyoming, North Dakota
+ *   - **FR**: Corse, Lozère, Creuse
+ *
+ *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
+ *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
+ *   the locality boundary, not the exact split percentages.
+ *
+ *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
+ *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ */
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { createReadStream, createWriteStream } from "node:fs";
+import { mkdir, unlink, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+/**
+ * Phase 1 default holdouts (per plan).
+ *
+ * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
+ * - FR: Corse, Lozère, Creuse (small departments / regions).
+ */
+export function defaultHoldouts() {
+    return {
+        US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
+        FR: ["Corse", "Lozère", "Lozere", "Creuse"],
+    };
+}
+/**
+ * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming
+ * `buildCorpus` align loop (`build.ts`) to decide each row's split without retaining the row in
+ * heap. Identical hash bucketing semantics to the array-based path so the decision is stable
+ * regardless of caller.
+ */
+export function splitForRow(row, holdouts = defaultHoldouts()) {
+    const region = row.components.region;
+    const countryHoldouts = holdouts[row.country] ?? [];
+    const isHeldOut = region !== undefined && countryHoldouts.includes(region);
+    if (!isHeldOut)
+        return "train";
+    // 50/50 deterministic by source_id hash. Same input always lands in the same split.
+    return hashBucket(row.source_id, 2) === 0 ? "val" : "test";
+}
+/**
+ * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are
+ * accepted — only `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ *
+ * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via
+ * `buildCorpus` use the streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to
+ * avoid materializing every aligned row's split membership in heap.
+ */
+export function splitRows(rows, opts = {}) {
+    const holdouts = opts.holdouts ?? defaultHoldouts();
+    const train = [];
+    const val = [];
+    const test = [];
+    let corpus_version = "";
+    for (const row of rows) {
+        if (!corpus_version && row.corpus_version)
+            corpus_version = row.corpus_version;
+        const split = splitForRow(row, holdouts);
+        if (split === "train")
+            train.push(row.source_id);
+        else if (split === "val")
+            val.push(row.source_id);
+        else
+            test.push(row.source_id);
+    }
+    const total = train.length + val.length + test.length;
+    return {
+        train,
+        val,
+        test,
+        holdouts,
+        corpus_version,
+        counts: { train: train.length, val: val.length, test: test.length, total },
+    };
+}
+/** Lightweight deterministic 0..(n-1) bucket from a string id. */
+export function hashBucket(id, n) {
+    const digest = createHash("sha256").update(id).digest();
+    // Read 4 bytes as uint32 to avoid bigint overhead.
+    const u = digest[0] * 0x01_00_00_00 + digest[1] * 0x01_00_00 + digest[2] * 0x01_00 + digest[3];
+    return u % n;
+}
+/**
+ * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated
+ * source_id lists (one id per line) so they diff cleanly in git. Also writes
+ * `<outputDir>/MANIFEST.json` with the full structured manifest including holdouts + counts +
+ * corpus version.
+ *
+ * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ */
+export async function writeSplitManifests(manifest, outputDir) {
+    await mkdir(outputDir, { recursive: true });
+    for (const name of ["train", "val", "test"]) {
+        const sorted = [...manifest[name]].sort();
+        await writeFile(join(outputDir, `${name}.txt`), sorted.join("\n") + (sorted.length ? "\n" : ""), "utf8");
+    }
+    const summary = {
+        corpus_version: manifest.corpus_version,
+        holdouts: manifest.holdouts,
+        counts: manifest.counts,
+    };
+    await writeFile(join(outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
+}
+/**
+ * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests +
+ * `SPLIT_MANIFEST.json` by streaming three per-split labeled-row JSONL files (one per split).
+ * Memory cost is O(1) — `sort(1)` from coreutils handles the deterministic sort with disk spill for
+ * files that exceed in-memory thresholds.
+ *
+ * Used by `buildCorpus` after the align loop has already partitioned labeled rows into
+ * `labeled-{train,val,test}.jsonl` via `splitForRow`. Counts are pre-computed by the align loop and
+ * passed in (zero re-scan).
+ */
+export async function writeSplitManifestsFromLabeledFiles(opts) {
+    await mkdir(opts.outputDir, { recursive: true });
+    const holdouts = opts.holdouts ?? defaultHoldouts();
+    for (const split of ["train", "val", "test"]) {
+        const labeledPath = opts.labeledPaths[split];
+        const outPath = join(opts.outputDir, `${split}.txt`);
+        await streamSortedSourceIds(labeledPath, outPath);
+    }
+    const total = opts.counts.train + opts.counts.val + opts.counts.test;
+    const summary = {
+        corpus_version: opts.corpusVersion,
+        holdouts,
+        counts: { ...opts.counts, total },
+    };
+    await writeFile(join(opts.outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
+    return summary.counts;
+}
+/**
+ * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input →
+ * empty output file (not absent). Uses `sort(1)` for disk-spilling external sort so peak memory
+ * stays O(1) regardless of labeled-row count.
+ */
+async function streamSortedSourceIds(labeledJsonlPath, outPath) {
+    const unsortedPath = `${outPath}.unsorted`;
+    const out = createWriteStream(unsortedPath, { encoding: "utf8" });
+    const rl = createInterface({ input: createReadStream(labeledJsonlPath, { encoding: "utf8" }), crlfDelay: Infinity });
+    await new Promise((resolve, reject) => {
+        rl.on("line", (line) => {
+            if (!line)
+                return;
+            try {
+                const obj = JSON.parse(line);
+                if (typeof obj.source_id === "string")
+                    out.write(`${obj.source_id}\n`);
+            }
+            catch (err) {
+                reject(err);
+            }
+        });
+        rl.on("close", () => {
+            out.end();
+        });
+        rl.on("error", reject);
+        out.on("close", () => resolve());
+        out.on("error", reject);
+    });
+    await new Promise((resolve, reject) => {
+        // LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
+        const proc = spawn("sort", [unsortedPath, "-o", outPath], { env: { ...process.env, LC_ALL: "C" } });
+        proc.on("error", reject);
+        proc.on("exit", (code) => {
+            if (code === 0)
+                resolve();
+            else
+                reject(new Error(`sort exited with code ${code}`));
+        });
+    });
+    await unlink(unsortedPath).catch(() => { });
+}
+//# sourceMappingURL=split.js.map

package/out/src/split.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AA2B/C;;;;;GAKG;AACH,MAAM,UAAU,eAAe;IAC9B,OAAO;QACN,EAAE,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC;QAC5D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC;KAC3C,CAAA;AACF,CAAC;AAID;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAC1B,GAAgE,EAChE,WAA8C,eAAe,EAAE;IAE/D,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAC1E,IAAI,CAAC,SAAS;QAAE,OAAO,OAAO,CAAA;IAC9B,oFAAoF;IACpF,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;AAC3D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,IAA6B,EAAE,OAAqB,EAAE;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IACnD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,cAAc,GAAG,EAAE,CAAA;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,cAAc;YAAE,cAAc,GAAG,GAAG,CAAC,cAAc,CAAA;QAC9E,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;QACxC,IAAI,KAAK,KAAK,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;aAC3C,IAAI,KAAK,KAAK,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;;YAC5C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IACrD,OAAO;QACN,KAAK;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,cAAc;QACd,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KAC1E,CAAA;AACF,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,UAAU,CAAC,EAAU,EAAE,CAAS;IAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IACvD,mDAAmD;IACnD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,aAAa,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;IAClG,OAAO,CAAC,GAAG,CAAC,CAAA;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAuB,EAAE,SAAiB;IACnF,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAC3C,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QACzC,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAA;IACzG,CAAC;IACD,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AACzG,CAAC;AAKD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,IAMzD;IACA,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IAEnD,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,MAAM,CAAC,CAAA;QACpD,MAAM,qBAAqB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IACpE,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KACjC,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7G,OAAO,OAAO,CAAC,MAAM,CAAA;AACtB,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,qBAAqB,CAAC,gBAAwB,EAAE,OAAe;IAC7E,MAAM,YAAY,GAAG,GAAG,OAAO,WAAW,CAAA;IAC1C,MAAM,GAAG,GAAG,iBAAiB,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjE,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEpH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,CAAC,IAAI;gBAAE,OAAM;YACjB,IAAI,CAAC;gBACJ,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA2B,CAAA;gBACtD,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ;oBAAE,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YACvE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,MAAM,CAAC,GAAY,CAAC,CAAA;YACrB,CAAC;QACF,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,GAAG,CAAC,GAAG,EAAE,CAAA;QACV,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QACtB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAChC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,wEAAwE;QACxE,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;QACnG,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QACxB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAA;;gBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC3C,CAAC"}

package/out/src/synthesize.d.ts

@@ -0,0 +1,146 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Synthesis / augmentation per Phase 1 task #6.
+ *
+ *   An `Augmentation` is a pure function that takes a `CanonicalRow` and either returns a new
+ *   `CanonicalRow` (with `raw` AND `components` transformed in lockstep so alignment still
+ *   succeeds) or `null` when the augmentation doesn't apply to the row's shape.
+ *
+ *   Synthesis runs **before** alignment: augmentations transform raw + components together, and the
+ *   runner reruns alignment on each augmented row to produce its labels. This keeps the synthesis
+ *   surface small (no token/label arithmetic) at the cost of a re-run.
+ *
+ *   Every augmented row carries the `synth` marker:
+ *
+ *   - `method`: the augmentation's stable id (e.g. `"case-upper"`, `"accent-strip"`).
+ *   - `base_source_id`: the source_id of the un-augmented (or upstream-augmented) row, so ancestry is
+ *       traceable.
+ *
+ *   Phase 1 implements the locale-agnostic + most useful US/FR augmentations. Typo injection and
+ *   other stochastic augmentations are intentionally deferred — they need a seed-aware API and are
+ *   most useful at training time, not corpus build time.
+ */
+import { type Tokenizer } from "./tokenize.js";
+import type { CanonicalRow, LabeledRow, QuarantinedRow } from "./types.js";
+/**
+ * An augmentation transforms a single row. Return `null` if the augmentation doesn't apply (e.g.
+ * accent-strip on a row that has no accents; particle-strip on a US row).
+ */
+export type Augmentation = (row: CanonicalRow) => CanonicalRow | null;
+/** Upper-case raw + every component value. Returns null if already all-upper. */
+export declare const caseUpper: Augmentation;
+/** Lower-case raw + every component value. Returns null if already all-lower. */
+export declare const caseLower: Augmentation;
+/** Drop commas from `raw`. Components unchanged (they didn't carry commas). */
+export declare const dropCommas: Augmentation;
+/**
+ * Replace single spaces with double spaces in `raw` AND in every component value. The component
+ * update is load-bearing for alignment: `alignRow` substring-searches each component's surface form
+ * inside `raw`, so doubling the spaces in `raw` only would leave single-spaced components
+ * unfindable (this was the bug behind v0.1.1's first build attempt — 99.9% of quarantined rows
+ * traced back to this augmentation). Doubling both keeps the substring contract intact.
+ */
+export declare const doubleSpace: Augmentation;
+/**
+ * Strip Unicode combining marks (accents, diacritics) from raw + components. "Hôtel" → "Hotel";
+ * "Île-de-France" → "Ile-de-France". Returns null if the row has no accents.
+ */
+export declare const accentStrip: Augmentation;
+/** US: substitute the full state name for its alpha-2 abbreviation. */
+export declare const stateExpand: Augmentation;
+/** US: substitute the alpha-2 abbreviation for the full state name. */
+export declare const stateAbbreviate: Augmentation;
+/** US: expand directional abbreviations in `street`/`street_suffix` (NW → Northwest). */
+export declare const directionalExpand: Augmentation;
+/** US: abbreviate directional words (Northwest → NW). */
+export declare const directionalAbbreviate: Augmentation;
+/**
+ * US: swap the trailing street-suffix word in `components.street` to its preferred USPS
+ * abbreviation, preserving case. `"5th Avenue"` → `"5th Ave"`; `"5TH AVENUE"` → `"5TH AVE"`; `"main
+ * street"` → `"main st"`. Returns null when no trailing suffix is recognized, when the trailing
+ * word is already the preferred abbreviation, or when the swap would leave `raw` un- touched
+ * (alignment requires both raw and components to move in lockstep).
+ *
+ * Targets the trailing word only to avoid mangling streets like "Avenue of the Americas" where the
+ * suffix-shaped word is part of the proper name rather than a USPS suffix.
+ */
+export declare const streetSuffixAbbreviate: Augmentation;
+/**
+ * US: swap the trailing street-suffix word in `components.street` to its full canonical form,
+ * preserving case. `"5th Ave"` → `"5th Avenue"`; `"5TH AVE"` → `"5TH AVENUE"`; `"main st"` → `"main
+ * street"`. Returns null when no trailing suffix is recognized, when the trailing word is already
+ * the canonical full form, or when the swap would leave `raw` untouched.
+ *
+ * Same trailing-word-only rule as `streetSuffixAbbreviate`.
+ */
+export declare const streetSuffixExpand: Augmentation;
+/** US: ZIP+4 form `12345-6789` → `123456789` (dash dropped). */
+export declare const zipPlus4DashDrop: Augmentation;
+/** FR: drop the article particle from a street ("Rue de la République" → "Rue République"). */
+export declare const particleStrip: Augmentation;
+/** Stable id → augmentation table. */
+export declare const AUGMENTATIONS: Record<string, Augmentation>;
+/** Default augmentation set, by country. Phase 1: US + FR; others get the locale-agnostic set. */
+export declare function defaultAugmentationsForCountry(country: string): readonly Augmentation[];
+/**
+ * Run every augmentation against a row; collect the non-null outputs. The augmentations are pure,
+ * so callers can compose them off this generator (e.g. nesting accent-strip ∘ state-abbreviate).
+ */
+export declare function synthesizeRow(row: CanonicalRow, augmentations?: readonly Augmentation[]): Generator<CanonicalRow>;
+/** Options accepted by `composeAdversarialRow`. */
+export interface ComposeAdversarialOptions {
+    /**
+     * Stable pattern label written into the emitted row's `synth.method` field (as
+     * `compose:<pattern>`). Free-form but should be one of a small set of canonical pattern names so
+     * downstream filtering / stratification can target individual patterns.
+     *
+     * Recommended values (Phase 1.6 §2.1):
+     *
+     * - `"place-name-venue"` — venue token shared with locality (`Buffalo Health Clinic, Buffalo NY`).
+     * - `"place-shaped-venue"` — venue contains a place-shaped substring (`New York, New York
+     *   Steakhouse, Las Vegas NV`).
+     * - `"particle-honorific"` — apostrophe + St./Saint ambiguity (`P'tit St. Denis Street Café`).
+     */
+    pattern: string;
+    /**
+     * Separator inserted between the venue and the address `raw`. Default `", "`. Single space (`"
+     * "`) produces the harder unpunctuated variant; newline (`"\n"`) the multi-line variant.
+     */
+    separator?: string;
+    /**
+     * Tokenizer to apply to the venue prefix. Default `whitespaceTokenizer()`. The address half uses
+     * the same tokenizer when re-aligned — pass a consistent one if customizing.
+     */
+    tokenizer?: Tokenizer;
+}
+/** Either a successful labeled composition or a quarantined attempt. */
+export type ComposeResult = {
+    kind: "labeled";
+    row: LabeledRow;
+} | {
+    kind: "quarantined";
+    row: QuarantinedRow;
+};
+/**
+ * Compose a venue string + an address row into a single adversarial `LabeledRow`.
+ *
+ * The emitted row's `raw` is `${venue}${separator}${address.raw}`. Tokens are produced by
+ * tokenizing the two halves independently and concatenating; labels are venue tokens → `B-venue` /
+ * `I-venue` followed by the address's labels (obtained by aligning the input address in isolation).
+ * This deterministic boundary is the entire point of the primitive: the embedded place-shaped
+ * tokens in the venue stay labeled as `venue`, never as the address's locality / region / etc.,
+ * even when they share surface forms.
+ *
+ * The address's components are forwarded as-is (alignment ran on them and they survived); `venue`
+ * is added on top with the trimmed venue string as its surface form.
+ *
+ * Returns `{ kind: "quarantined" }` when:
+ *
+ * - The venue is empty or whitespace-only.
+ * - The address row fails alignment in isolation (the underlying failure reason is propagated).
+ */
+export declare function composeAdversarialRow(venue: string, address: CanonicalRow, options: ComposeAdversarialOptions): ComposeResult;
+//# sourceMappingURL=synthesize.d.ts.map

package/out/src/synthesize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"synthesize.d.ts","sourceRoot":"","sources":["../../src/synthesize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAKH,OAAO,EAAuB,KAAK,SAAS,EAAE,MAAM,eAAe,CAAA;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAE1E;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,YAAY,KAAK,YAAY,GAAG,IAAI,CAAA;AAyBrE,iFAAiF;AACjF,eAAO,MAAM,SAAS,EAAE,YAQvB,CAAA;AAED,iFAAiF;AACjF,eAAO,MAAM,SAAS,EAAE,YAQvB,CAAA;AAED,+EAA+E;AAC/E,eAAO,MAAM,UAAU,EAAE,YAIxB,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,WAAW,EAAE,YAQzB,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,WAAW,EAAE,YAQzB,CAAA;AAqED,uEAAuE;AACvE,eAAO,MAAM,WAAW,EAAE,YAazB,CAAA;AAED,uEAAuE;AACvE,eAAO,MAAM,eAAe,EAAE,YAW7B,CAAA;AAgBD,yFAAyF;AACzF,eAAO,MAAM,iBAAiB,EAAE,YAkB/B,CAAA;AAED,yDAAyD;AACzD,eAAO,MAAM,qBAAqB,EAAE,YAqBnC,CAAA;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,sBAAsB,EAAE,YAkBpC,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,EAAE,YAiBhC,CAAA;AAED,gEAAgE;AAChE,eAAO,MAAM,gBAAgB,EAAE,YAQ9B,CAAA;AAMD,+FAA+F;AAC/F,eAAO,MAAM,aAAa,EAAE,YAW3B,CAAA;AAMD,sCAAsC;AACtC,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CActD,CAAA;AAED,kGAAkG;AAClG,wBAAgB,8BAA8B,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,YAAY,EAAE,CAmBvF;AAED;;;GAGG;AACH,wBAAiB,aAAa,CAC7B,GAAG,EAAE,YAAY,EACjB,aAAa,GAAE,SAAS,YAAY,EAAgD,GAClF,SAAS,CAAC,YAAY,CAAC,CAKzB;AAqCD,mDAAmD;AACnD,MAAM,WAAW,yBAAyB;IACzC;;;;;;;;;;;OAWG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,SAAS,CAAA;CACrB;AAED,wEAAwE;AACxE,MAAM,MAAM,aAAa,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,cAAc,CAAA;CAAE,CAAA;AAE/G;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,qBAAqB,CACpC,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,yBAAyB,GAChC,aAAa,CAsDf"}