@mailwoman/corpus 9.4.0 → 10.1.0

package/out/build/eligibility.js

@@ -0,0 +1,80 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file What the address-source register records about a row's source, read once per build.
+ */
+import { stringifyJSON } from "@mailwoman/core/json";
+import { ingestEligibilityProblems, readAddressSourceRegister } from "#source-register";
+/**
+ * The key a row is looked up under: the adapter that emitted it and the jurisdiction it describes.
+ *
+ * The register scopes a source to one publisher in one jurisdiction,
+ * so an adapter id alone cannot address a record.
+ * One adapter serves several: `ban` emits eleven jurisdictions from one schema
+ * and each is its own source under its own license decision.
+ */
+export function sourceEligibilityKey(adapterID, iso2) {
+    return `${adapterID}:${iso2.toUpperCase()}`;
+}
+/**
+ * Every register source's ingest-eligibility reasons, keyed by the adapter
+ * and jurisdiction of the rows that carry it.
+ *
+ * A source declaring no `adapterID` is absent from the map, because no adapter emits it
+ * and no row can arrive under it.
+ * An empty array is the only value a caller may read as permission.
+ * An absent key is refused rather than admitted.
+ */
+export async function readSourceEligibility() {
+    const register = await readAddressSourceRegister();
+    const keyed = new Map();
+    for (const source of register.sources) {
+        if (source.adapterID === undefined)
+            continue;
+        keyed.set(sourceEligibilityKey(source.adapterID, source.iso2), ingestEligibilityProblems(source, register));
+    }
+    return keyed;
+}
+/**
+ * The register's license decisions keyed by the label in a row.
+ *
+ * An unmatched adapter label records a `null` decision rather than inventing one.
+ */
+export async function readRegisterDecisions() {
+    const register = await readAddressSourceRegister();
+    return new Map(register.licenses.map((decision) => [decision.licenseID, decision]));
+}
+/**
+ * Reads why a row's source may be refused from a release-eligible corpus.
+ * Records each refused source.
+ *
+ * `refused` accumulates as rows arrive.
+ * The build manifest reports the collected values.
+ *
+ * A build whose `eligibility` is `null` runs under the exploratory profile and every row reads eligible.
+ */
+export function createIneligibilityReader(eligibility) {
+    const refused = new Map();
+    return {
+        refused,
+        read: (row) => {
+            if (!eligibility)
+                return null;
+            const key = sourceEligibilityKey(row.source, row.country);
+            const cached = refused.get(key);
+            if (cached)
+                return cached;
+            const problems = eligibility.get(key) ?? [
+                `no register source declares ${stringifyJSON(row.source)} as its adapter in ${stringifyJSON(row.country)}, ` +
+                    "so this row's publication has not been reviewed. Set `adapterID` on the source in " +
+                    "`source-resolutions.json` once a review has elected it.",
+            ];
+            if (!problems.length)
+                return null;
+            refused.set(key, problems);
+            return problems;
+        },
+    };
+}
+//# sourceMappingURL=eligibility.js.map

package/out/build/eligibility.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eligibility.js","sourceRoot":"","sources":["../../lib/build/eligibility.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,OAAO,EAAE,yBAAyB,EAAE,yBAAyB,EAAwB,MAAM,kBAAkB,CAAA;AAG7G;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAiB,EAAE,IAAY;IACnE,OAAO,GAAG,SAAS,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAA;AAC5C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IAC1C,MAAM,QAAQ,GAAG,MAAM,yBAAyB,EAAE,CAAA;IAClD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA6B,CAAA;IAElD,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS;YAAE,SAAQ;QAE5C,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,yBAAyB,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAA;IAC5G,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB;IAC1C,MAAM,QAAQ,GAAG,MAAM,yBAAyB,EAAE,CAAA;IAElD,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAA;AACpF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,yBAAyB,CAAC,WAA0D;IAInG,MAAM,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAA;IAEpD,OAAO;QACN,OAAO;QACP,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;YACb,IAAI,CAAC,WAAW;gBAAE,OAAO,IAAI,CAAA;YAE7B,MAAM,GAAG,GAAG,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,CAAA;YACzD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAE/B,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAA;YAEzB,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI;gBACxC,+BAA+B,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,sBAAsB,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI;oBAC3G,oFAAoF;oBACpF,yDAAyD;aAC1D,CAAA;YAED,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAA;YAEjC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YAE1B,OAAO,QAAQ,CAAA;QAChB,CAAC;KACD,CAAA;AACF,CAAC"}

package/out/build/heap.d.ts

@@ -0,0 +1,28 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The heap a build needs before a resident-record adapter may run.
+ */
+/**
+ * The heap a build needs before a resident-record adapter may run, in bytes.
+ *
+ * At Node's 4 GB default both WOF adapters abort with SIGABRT, `wof-admin` at 4,085 MB after 722 seconds.
+ * The floor sits under the measured 14,883 MiB peak by enough to admit a 16 GB cap
+ * and refuse the 4 GB default.
+ */
+export declare const RESIDENT_ADAPTER_HEAP_FLOOR_BYTES: number;
+/**
+ * Refuses a build whose heap cannot hold a resident-record adapter's dataset.
+ *
+ * The failure this replaces is a SIGABRT twelve minutes into an eight-hour build.
+ * That abort reports no cause and loses the adapter phase.
+ * A startup read of the limit costs one call.
+ *
+ * The `corpus build` command calls this rather than `buildCorpus`, because this package's
+ * own tests run `wof-admin` against a five-kilobyte fixture at the default heap.
+ *
+ * @throws When a resident-record adapter is configured and the heap limit is under the floor.
+ */
+export declare function assertHeapForAdapters(adapterIDs: readonly string[], heapLimitBytes: number): void;
+//# sourceMappingURL=heap.d.ts.map

package/out/build/heap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heap.d.ts","sourceRoot":"","sources":["../../lib/build/heap.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH;;;;;;GAMG;AACH,eAAO,MAAM,iCAAiC,QAA0B,CAAA;AAExE;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,CAcjG"}

package/out/build/heap.js

@@ -0,0 +1,47 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The heap a build needs before a resident-record adapter may run.
+ */
+/**
+ * Adapter ids whose run holds every source record resident before yielding a row.
+ *
+ * Both WOF adapters build a `Map` of the whole dataset and then sort its keys,
+ * so peak memory is a property of the source rather than of `limit`.
+ * Measured 2026-09-26: `wof-admin` peaks at 14,020 MiB.
+ * Both in one process peak at 14,883 MiB.
+ */
+const RESIDENT_RECORD_ADAPTERS = new Set(["wof-admin", "wof-postalcode"]);
+/**
+ * The heap a build needs before a resident-record adapter may run, in bytes.
+ *
+ * At Node's 4 GB default both WOF adapters abort with SIGABRT, `wof-admin` at 4,085 MB after 722 seconds.
+ * The floor sits under the measured 14,883 MiB peak by enough to admit a 16 GB cap
+ * and refuse the 4 GB default.
+ */
+export const RESIDENT_ADAPTER_HEAP_FLOOR_BYTES = 15 * 1024 * 1024 * 1024;
+/**
+ * Refuses a build whose heap cannot hold a resident-record adapter's dataset.
+ *
+ * The failure this replaces is a SIGABRT twelve minutes into an eight-hour build.
+ * That abort reports no cause and loses the adapter phase.
+ * A startup read of the limit costs one call.
+ *
+ * The `corpus build` command calls this rather than `buildCorpus`, because this package's
+ * own tests run `wof-admin` against a five-kilobyte fixture at the default heap.
+ *
+ * @throws When a resident-record adapter is configured and the heap limit is under the floor.
+ */
+export function assertHeapForAdapters(adapterIDs, heapLimitBytes) {
+    const resident = adapterIDs.filter((id) => RESIDENT_RECORD_ADAPTERS.has(id));
+    if (!resident.length || heapLimitBytes >= RESIDENT_ADAPTER_HEAP_FLOOR_BYTES)
+        return;
+    const gib = (bytes) => `${(bytes / 1024 / 1024 / 1024).toFixed(1)} GiB`;
+    throw new Error(`corpus build: the heap limit is ${gib(heapLimitBytes)} and ${resident.join(", ")} ` +
+        `${resident.length === 1 ? "holds" : "hold"} every source record resident. That peaked at 14,883 MiB ` +
+        `on 2026-09-26. Re-run with NODE_OPTIONS="--max-old-space-size=20480", or leave ` +
+        `${resident.join(", ")} out of --inputs. Under ${gib(RESIDENT_ADAPTER_HEAP_FLOOR_BYTES)} the run aborts ` +
+        `with SIGABRT partway through the adapter phase.`);
+}
+//# sourceMappingURL=heap.js.map

package/out/build/heap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"heap.js","sourceRoot":"","sources":["../../lib/build/heap.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;GAOG;AACH,MAAM,wBAAwB,GAAwB,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAA;AAE9F;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAA;AAExE;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CAAC,UAA6B,EAAE,cAAsB;IAC1F,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,wBAAwB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAE5E,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,cAAc,IAAI,iCAAiC;QAAE,OAAM;IAEnF,MAAM,GAAG,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAA;IAEvF,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,cAAc,CAAC,QAAQ,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QACnF,GAAG,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,2DAA2D;QACtG,iFAAiF;QACjF,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,GAAG,CAAC,iCAAiC,CAAC,kBAAkB;QACzG,iDAAiD,CAClD,CAAA;AACF,CAAC"}

package/out/build/inputs.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The committed record of which adapter read which input for one base corpus version.
+ *
+ *   Before this record existed, the only statement of a build's inputs was a JSON string in the launch
+ *   command, kept afterwards in a build-log directory under the data root. The predecessor of the
+ *   `v0.7.0-de-holdout` record names paths under `/mnt/playpen/mailwoman-data`. Those paths stopped resolving
+ *   when the data root moved on 2026-09-23. A corpus `MANIFEST.json` records `skipped_adapters` without any
+ *   adapter's input path, so a rebuild began by re-deriving the adapter set from that skip list.
+ *
+ *   Each `inputPath` is written relative to `$MAILWOMAN_DATA_ROOT` and resolved at launch, so the record
+ *   survives a data-root move. A path that has to sit outside the data root is written absolute and used as
+ *   written.
+ */
+import { type PathBuilderLike } from "path-ts";
+import type { AdapterOptions } from "#types";
+/**
+ * One adapter's input as the committed record states it.
+ *
+ * `inputPath` is data-root-relative.
+ * The remaining fields match {@linkcode AdapterOptions} and exist so a per-country
+ * or truncated run can be recorded as it ran.
+ */
+export interface BuildInputRecord {
+    inputPath: string;
+    country?: string;
+    limit?: number;
+}
+/**
+ * The adapter inputs of one base corpus build.
+ */
+export interface BuildInputsRecord {
+    corpusVersion: string;
+    builtAt?: string;
+    adapters: Record<string, BuildInputRecord>;
+}
+/**
+ * Resolve the committed inputs record for a corpus version through the package's `./data/*` export.
+ *
+ * Resolution happens on call, matching `addressSourceRegisterPath`.
+ */
+export declare function buildInputsPath(corpusVersion: string): string;
+/**
+ * Read a committed inputs record and resolve every `inputPath` against the data root.
+ *
+ * @throws When the record names no adapter, because a build with no inputs
+ * writes an empty corpus and reports success.
+ */
+export declare function readBuildInputs(path: PathBuilderLike): Promise<Record<string, AdapterOptions>>;
+//# sourceMappingURL=inputs.d.ts.map

package/out/build/inputs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inputs.d.ts","sourceRoot":"","sources":["../../lib/build/inputs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,EAAc,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,QAAQ,CAAA;AAE5C;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAA;CAC1C;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAmBpG"}

package/out/build/inputs.js

@@ -0,0 +1,49 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The committed record of which adapter read which input for one base corpus version.
+ *
+ *   Before this record existed, the only statement of a build's inputs was a JSON string in the launch
+ *   command, kept afterwards in a build-log directory under the data root. The predecessor of the
+ *   `v0.7.0-de-holdout` record names paths under `/mnt/playpen/mailwoman-data`. Those paths stopped resolving
+ *   when the data root moved on 2026-09-23. A corpus `MANIFEST.json` records `skipped_adapters` without any
+ *   adapter's input path, so a rebuild began by re-deriving the adapter set from that skip list.
+ *
+ *   Each `inputPath` is written relative to `$MAILWOMAN_DATA_ROOT` and resolved at launch, so the record
+ *   survives a data-root move. A path that has to sit outside the data root is written absolute and used as
+ *   written.
+ */
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { resolveModulePath } from "@mailwoman/core/module/resolvers";
+import { isAbsolute } from "path-ts";
+/**
+ * Resolve the committed inputs record for a corpus version through the package's `./data/*` export.
+ *
+ * Resolution happens on call, matching `addressSourceRegisterPath`.
+ */
+export function buildInputsPath(corpusVersion) {
+    return resolveModulePath(`@mailwoman/corpus/data/builds/${corpusVersion}/inputs.json`);
+}
+/**
+ * Read a committed inputs record and resolve every `inputPath` against the data root.
+ *
+ * @throws When the record names no adapter, because a build with no inputs
+ * writes an empty corpus and reports success.
+ */
+export async function readBuildInputs(path) {
+    const record = await readLocalJSONFile(path);
+    const entries = Object.entries(record.adapters ?? {});
+    if (!entries.length) {
+        throw new Error(`${path.toString()} names no adapter under \`adapters\`, so the build would read nothing.`);
+    }
+    return Object.fromEntries(entries.map(([id, input]) => {
+        if (typeof input?.inputPath !== "string" || !input.inputPath) {
+            throw new Error(`${path.toString()}: adapter ${id} has no \`inputPath\`.`);
+        }
+        const resolved = isAbsolute(input.inputPath) ? input.inputPath : dataRootPath(input.inputPath).toString();
+        return [id, { ...input, inputPath: resolved }];
+    }));
+}
+//# sourceMappingURL=inputs.js.map

package/out/build/inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inputs.js","sourceRoot":"","sources":["../../lib/build/inputs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAA;AACpE,OAAO,EAAE,UAAU,EAAwB,MAAM,SAAS,CAAA;AA0B1D;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,aAAqB;IACpD,OAAO,iBAAiB,CAAC,iCAAiC,aAAa,cAAc,CAAC,CAAA;AACvF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAqB;IAC1D,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAoB,IAAI,CAAC,CAAA;IAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IAErD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,wEAAwE,CAAC,CAAA;IAC5G,CAAC;IAED,OAAO,MAAM,CAAC,WAAW,CACxB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE;QAC3B,IAAI,OAAO,KAAK,EAAE,SAAS,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,EAAE,wBAAwB,CAAC,CAAA;QAC3E,CAAC;QAED,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAA;QAEzG,OAAO,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,QAAQ,EAA2B,CAAC,CAAA;IACxE,CAAC,CAAC,CACF,CAAA;AACF,CAAC"}

package/out/build/rows.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reading a build's own JSONL back, the one reader every phase after the adapters uses.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * Stream a JSONL file's rows.
+ *
+ * `JSONSpliterator` yields parsed rows and throws `SyntaxError` on a malformed row.
+ * `preferCompressed` and `delimitedSource` are called inline because each yields
+ * a source that may only be consumed once.
+ */
+export declare function streamJSONL<T>(path: PathBuilderLike): AsyncIterable<T>;
+//# sourceMappingURL=rows.d.ts.map

package/out/build/rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rows.d.ts","sourceRoot":"","sources":["../../lib/build/rows.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAG9C;;;;;;GAMG;AACH,wBAAuB,WAAW,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC,CAE7E"}

package/out/build/rows.js

@@ -0,0 +1,19 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reading a build's own JSONL back, the one reader every phase after the adapters uses.
+ */
+import { delimitedSource, preferCompressed } from "@mailwoman/core/fs/delimited";
+import { JSONSpliterator } from "spliterator";
+/**
+ * Stream a JSONL file's rows.
+ *
+ * `JSONSpliterator` yields parsed rows and throws `SyntaxError` on a malformed row.
+ * `preferCompressed` and `delimitedSource` are called inline because each yields
+ * a source that may only be consumed once.
+ */
+export async function* streamJSONL(path) {
+    yield* JSONSpliterator.fromAsync(delimitedSource(await preferCompressed(path)));
+}
+//# sourceMappingURL=rows.js.map

package/out/build/rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rows.js","sourceRoot":"","sources":["../../lib/build/rows.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAA;AAEhF,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAqB;IAC1D,KAAK,CAAC,CAAC,eAAe,CAAC,SAAS,CAAI,eAAe,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACnF,CAAC"}

package/out/build/types.d.ts

@@ -0,0 +1,220 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Defines the `buildCorpus` options and the profile that determines source eligibility.
+ *   It also defines the top-level manifest for each build stage.
+ *
+ *   These sit apart from `buildCorpus` because every phase module reads them and the orchestrator imports
+ *   each phase. These types live separately because placing them beside the orchestrator would make phases import it.
+ */
+import type { PathBuilderLike } from "path-ts";
+import type { ParquetManifest } from "#parquet/writers";
+import type { AdapterRunManifest } from "#runner";
+import type { AdapterOptions, CorpusAdapter } from "#types";
+import type { LicensePolicy, LicenseRefusalKind } from "#utils/license";
+import type { SplitManifest } from "#utils/split";
+/**
+ * Stage tags surfaced to `onProgress`.
+ */
+export type BuildStage = "adapter-run" | "align" | "split" | "parquet" | "manifest";
+/**
+ * The frozen source record's filename, beside the build's own `MANIFEST.json`.
+ */
+export declare const TRAINING_MANIFEST_FILE = "TRAINING_SOURCES.json";
+/**
+ * Defines the purpose of a corpus build.
+ *
+ * The profile determines whether a source must be eligible before its rows enter.
+ *
+ * `excludeLicenses` is orthogonal to this: it is a refusal rather than positive eligibility.
+ */
+export declare const BuildProfile: {
+    /**
+     * Include every row an adapter yields.
+     * A measurement runs under this profile.
+     */
+    readonly Exploratory: "exploratory";
+    /**
+     * Include a row only when the register marks its source eligible for ingest.
+     * A source whose terms remain unread is refused here.
+     */
+    readonly ReleaseEligible: "release-eligible";
+};
+export type BuildProfile = (typeof BuildProfile)[keyof typeof BuildProfile];
+/**
+ * Per-invocation options for `buildCorpus`.
+ */
+export interface BuildCorpusOptions {
+    /**
+     * Root output directory.
+     * Every build artifact lands beneath it.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Corpus version (e.g. `"0.1.0"`), stamped onto every row and into the output directory name.
+     */
+    corpusVersion: string;
+    /**
+     * Adapters to drive, in order.
+     *
+     * The default is `defaultAdapterRegistry.list()`, and an explicit list filters the run.
+     */
+    adapters?: readonly CorpusAdapter[];
+    /**
+     * Per-adapter `AdapterOptions` looked up by adapter id.
+     *
+     * An adapter whose id is missing is skipped and recorded in the manifest.
+     */
+    adapterInputs: Record<string, AdapterOptions>;
+    /**
+     * Enable the synthesis pass.
+     *
+     * The default is `true`, and a fixture-driven smoke test passes `false`.
+     */
+    synthesize?: boolean;
+    /**
+     * Max rows per `.parquet` file, forwarded to `writeParquetSplits`.
+     * The default is 1_000_000.
+     */
+    rowsPerFile?: number;
+    /**
+     * Rows held in memory while shuffling each split before it is written to parquet.
+     *
+     * Rows arrive in adapter order.
+     * Within one source, that order groups rows by country.
+     *
+     * An unshuffled row group therefore holds rows from one to eleven countries in its source.
+     * A bounded epoch draw reads only those countries.
+     *
+     * The default is `DEFAULT_SHUFFLE_WINDOW`, where `0` or `1` writes arrival
+     * order unchanged and consumes no random draw.
+     */
+    shuffleWindow?: number;
+    /**
+     * Seed for {@linkcode BuildCorpusOptions.shuffleWindow}'s draw.
+     *
+     * It is fixed by default, so two builds of one input write the same row order.
+     */
+    shuffleSeed?: number;
+    /**
+     * Reports each build stage.
+     * An error thrown from it aborts the build.
+     */
+    onProgress?: (stage: BuildStage, message: string) => void;
+    /**
+     * License prefixes supplied in `--exclude-licenses`.
+     *
+     * A row whose `license` starts with one is dropped at ingest.
+     * The default excludes no prefix.
+     */
+    excludeLicenses?: readonly RegExp[];
+    /**
+     * Which rows this build admits on the evidence of their license obligations.
+     *
+     * Defaults to `LicensePolicy.All`, admitting every row an adapter yields.
+     *
+     * A weights build that must have no share-alike obligation passes `LicensePolicy.ShareAlikeFree`.
+     */
+    licensePolicy?: LicensePolicy;
+    /**
+     * What this corpus is being built for.
+     * The default is {@linkcode BuildProfile.Exploratory}.
+     */
+    profile?: BuildProfile;
+}
+/**
+ * Top-level manifest tying every stage together.
+ */
+export interface BuildCorpusManifest {
+    corpus_version: string;
+    built_at: string;
+    adapters: AdapterRunManifest[];
+    skipped_adapters: string[];
+    splits: {
+        counts: SplitManifest["counts"];
+        holdouts: SplitManifest["holdouts"];
+    };
+    slices: {
+        counts: ParquetManifest["counts"];
+        total_rows: number;
+    };
+    quarantine_count: number;
+    total_aligned_rows: number;
+    /**
+     * Resolved license set across all included rows (license string → row count).
+     *
+     * The model card derives its data-attribution table from it.
+     *
+     * It covers the rows this build aligned.
+     * An overlay parquet added afterwards by `corpus overlay-manifest` contains
+     * its own rows and its own licenses.
+     *
+     * The overlay corpus's `MANIFEST.json` copies this set forward without them.
+     * {@linkcode licenses_cover} records this in the artifact, so a reader deriving an attribution
+     * table from a corpus with overlays knows the set is incomplete rather than complete.
+     */
+    licenses: Record<string, number>;
+    /**
+     * Which rows the `licenses` set was measured over.
+     *
+     * A base build writes `adapter-rows`, because no overlay had been assembled when it ran.
+     */
+    licenses_cover: string;
+    excluded_by_license: number;
+    /**
+     * The policy this build admitted rows under, so a consumer can tell a corpus that
+     * was never filtered from one filtered and found clean.
+     */
+    license_policy: LicensePolicy;
+    /**
+     * Rows refused per `LicenseRefusalKind`, summing to `excluded_by_license`.
+     *
+     * The classes rest on different evidence and a reader deciding what to do needs them apart.
+     */
+    refused_by_license_kind: Record<string, number>;
+    /**
+     * Every distinct license value this build refused, with the class it was refused under.
+     *
+     * A build that dropped rows records the values it dropped them for rather than only the count.
+     */
+    refused_license_values: Record<string, LicenseRefusalKind>;
+    /**
+     * Rows the license policy refused, counted per `source`.
+     *
+     * A source listed here contributed no row to this corpus, whatever `adapters[].written` reports
+     * for it: that field records what an adapter emitted, and the license filter runs after it.
+     * Model provenance is generated from what survived every filter, so a source present in corpus
+     * provenance may be absent from model provenance, and this field is where the difference reads.
+     */
+    refused_rows_by_source: Record<string, number>;
+    /**
+     * Admitted rows whose license resolves to no SPDX expression.
+     * Their obligations are unknown rather than empty.
+     *
+     * Under `LicensePolicy.ResolvedOnly` this reads zero because those rows are refused instead.
+     */
+    admitted_unresolved_license_rows: number;
+    /**
+     * The profile this build ran under, so a consumer can tell whether the corpus's sources were checked.
+     */
+    profile: BuildProfile;
+    /**
+     * Rows dropped because the register does not call their source eligible.
+     *
+     * This reads zero under {@linkcode BuildProfile.Exploratory}, applying no eligibility check.
+     */
+    excluded_by_eligibility: number;
+    /**
+     * Every source refused under {@linkcode BuildProfile.ReleaseEligible}, with the reasons the
+     * register gave, so a blocked build reports what to fix along with the fact that it stopped.
+     */
+    ineligible_sources: Record<string, readonly string[]>;
+    /**
+     * The `contentDigest` of the `TRAINING_SOURCES.json` written beside this manifest.
+     *
+     * That file is the frozen record of which sources contributed and under which terms.
+     */
+    training_manifest_digest: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/out/build/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../lib/build/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAA;AACjD,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAC3D,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AACvE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAEjD;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,UAAU,CAAA;AAEnF;;GAEG;AACH,eAAO,MAAM,sBAAsB,0BAA0B,CAAA;AAE7D;;;;;;GAMG;AACH,eAAO,MAAM,YAAY;IACxB;;;OAGG;aACH,WAAW,EAAE,aAAa;IAC1B;;;OAGG;aACH,eAAe,EAAE,kBAAkB;CAC1B,CAAA;AAEV,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAA;AAE3E;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAE1B;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;;OAIG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAEnC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,aAAa,CAAA;IAE7B;;;OAGG;IACH,OAAO,CAAC,EAAE,YAAY,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IACjE,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;;;;;;;;;;OAYG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC;;;;OAIG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB,mBAAmB,EAAE,MAAM,CAAA;IAC3B;;;OAGG;IACH,cAAc,EAAE,aAAa,CAAA;IAC7B;;;;OAIG;IACH,uBAAuB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/C;;;;OAIG;IACH,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;IAC1D;;;;;;;OAOG;IACH,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9C;;;;;OAKG;IACH,gCAAgC,EAAE,MAAM,CAAA;IACxC;;OAEG;IACH,OAAO,EAAE,YAAY,CAAA;IACrB;;;;OAIG;IACH,uBAAuB,EAAE,MAAM,CAAA;IAC/B;;;OAGG;IACH,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IACrD;;;;OAIG;IACH,wBAAwB,EAAE,MAAM,CAAA;CAChC"}

package/out/build/types.js

@@ -0,0 +1,34 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Defines the `buildCorpus` options and the profile that determines source eligibility.
+ *   It also defines the top-level manifest for each build stage.
+ *
+ *   These sit apart from `buildCorpus` because every phase module reads them and the orchestrator imports
+ *   each phase. These types live separately because placing them beside the orchestrator would make phases import it.
+ */
+/**
+ * The frozen source record's filename, beside the build's own `MANIFEST.json`.
+ */
+export const TRAINING_MANIFEST_FILE = "TRAINING_SOURCES.json";
+/**
+ * Defines the purpose of a corpus build.
+ *
+ * The profile determines whether a source must be eligible before its rows enter.
+ *
+ * `excludeLicenses` is orthogonal to this: it is a refusal rather than positive eligibility.
+ */
+export const BuildProfile = {
+    /**
+     * Include every row an adapter yields.
+     * A measurement runs under this profile.
+     */
+    Exploratory: "exploratory",
+    /**
+     * Include a row only when the register marks its source eligible for ingest.
+     * A source whose terms remain unread is refused here.
+     */
+    ReleaseEligible: "release-eligible",
+};
+//# sourceMappingURL=types.js.map

package/out/build/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../lib/build/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAeH;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,uBAAuB,CAAA;AAE7D;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC3B;;;OAGG;IACH,WAAW,EAAE,aAAa;IAC1B;;;OAGG;IACH,eAAe,EAAE,kBAAkB;CAC1B,CAAA"}