@mailwoman/corpus 9.4.0 → 10.1.0

package/out/utils/golden.d.ts

@@ -3,27 +3,16 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Golden eval-set validator (Phase 1 task #9 in the plan).
- *
- *   The golden set is hand-labeled ground truth for the neural classifier. Each entry must carry
- *   components whose surface forms actually occur in `raw` — otherwise the entry will silently rot
- *   the eval signal. This module:
- *
- *   - Defines `GoldenEntry` (schema check).
- *   - Loads `.jsonl` files (one entry per line).
- *   - Validates every entry: schema shape, ComponentTag membership, reachability of each component in
- *       `raw` via the same `reconcileComponents` helper alignment uses.
- *   - Returns a structured report of per-entry errors so the CLI / CI surface can act on it.
- *
- *   The 1000-entry target (500 US + 500 FR) is a human task. This module catches the regressions that
- *   creep in over time as new entries land.
+ *   Validates the hand-labeled golden eval set, where every labeled component must occur in `raw`.
  */
-import { type ComponentTag } from "@mailwoman/core/types";
+import { type ComponentTag } from "@mailwoman/codex/component";
+import { type PathBuilderLike } from "path-ts";
 /**
  * A golden-set candidate row, as `golden-expand` writes and `golden-promote` reads.
  *
- * `source` names the producer (`expand-golden:<provider>`); the seed/provenance fields trace the candidate back to the
- * corpus row and LLM call that produced it. A committed golden entry is the {@link GoldenEntry} narrowing.
+ * `source` identifies the producer, such as `expand-golden:<provider>`.
+ * The seed and provenance fields trace the candidate to its corpus row and LLM call.
+ * {@link GoldenEntry} narrows this type to a committed entry.
  */
 export interface GoldenCandidateEntry {
     raw: string;
@@ -46,7 +35,7 @@ export interface GoldenEntry extends GoldenCandidateEntry {
     source: "golden";
 }
 /**
- * Per-entry validation failure.
+ * A validation failure for one line of a golden file.
  */
 export interface GoldenIssue {
     file: string;
@@ -54,7 +43,7 @@ export interface GoldenIssue {
     reason: string;
 }
 /**
- * Aggregate report from `validateGoldenDir`.
+ * The aggregate report that `validateGoldenDir` returns.
  */
 export interface GoldenReport {
     entries: number;
@@ -62,24 +51,29 @@ export interface GoldenReport {
     issues: GoldenIssue[];
 }
 /**
- * Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
+ * Parses one JSONL line into a `GoldenEntry`.
+ *
+ * `validateGoldenFile` records the thrown message against the line number, so this parse must stay strict.
+ *
+ * @throws On schema violations.
  */
 export declare function parseGoldenLine(line: string): GoldenEntry;
 /**
- * Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent).
+ * Returns the tags in `entry` whose values do not occur in `entry.raw`.
+ *
+ * A golden `raw` is hand-written, so this check uses containment instead of
+ * reconciling against a rendered layout.
  */
 export declare function unreachableComponents(entry: GoldenEntry): ComponentTag[];
 /**
- * Validate one `.jsonl` file end-to-end, returning a list of issues.
+ * Validates one `.jsonl` file and returns its issues.
  *
- * Parses line by line over `TextSpliterator` rather than `JSONSpliterator`: every issue this returns carries the LINE
- * NUMBER it was found on, and a malformed line has to be REPORTED rather than thrown. `JSONSpliterator` parses each row
- * for you and throws on the first bad one — correct for consumers that want the rows, wrong for the validator whose
- * whole job is locating the bad ones.
+ * This reads lines with `TextSpliterator` because `JSONSpliterator` throws on the first malformed row.
+ * The validator must instead report every bad row with its line number.
  */
-export declare function validateGoldenFile(path: string): Promise<GoldenIssue[]>;
+export declare function validateGoldenFile(source: PathBuilderLike): Promise<GoldenIssue[]>;
 /**
- * Validate every `.jsonl` in a golden directory.
+ * Validates every `.jsonl` file directly inside a golden directory.
  */
-export declare function validateGoldenDir(dir: string): Promise<GoldenReport>;
+export declare function validateGoldenDir(dir: PathBuilderLike): Promise<GoldenReport>;
 //# sourceMappingURL=golden.d.ts.map

package/out/utils/golden.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"golden.d.ts","sourceRoot":"","sources":["../../lib/utils/golden.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,OAAO,EAAkB,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAOzE;;;;;GAKG;AACH,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IACjD,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B,UAAU,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,WAAY,SAAQ,oBAAoB;IACxD,MAAM,EAAE,QAAQ,CAAA;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,WAAW,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAkCzD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE,CAWxE;AAED;;;;;;;GAOG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CA6B7E;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAkB1E"}
+{"version":3,"file":"golden.d.ts","sourceRoot":"","sources":["../../lib/utils/golden.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAkB,KAAK,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAG9E,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAM3D;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IACjD,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B,UAAU,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,WAAY,SAAQ,oBAAoB;IACxD,MAAM,EAAE,QAAQ,CAAA;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,WAAW,EAAE,CAAA;CACrB;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAgCzD;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE,CAWxE;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CA6BxF;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,CAenF"}

package/out/utils/golden.js

@@ -3,43 +3,33 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Golden eval-set validator (Phase 1 task #9 in the plan).
- *
- *   The golden set is hand-labeled ground truth for the neural classifier. Each entry must carry
- *   components whose surface forms actually occur in `raw` — otherwise the entry will silently rot
- *   the eval signal. This module:
- *
- *   - Defines `GoldenEntry` (schema check).
- *   - Loads `.jsonl` files (one entry per line).
- *   - Validates every entry: schema shape, ComponentTag membership, reachability of each component in
- *       `raw` via the same `reconcileComponents` helper alignment uses.
- *   - Returns a structured report of per-entry errors so the CLI / CI surface can act on it.
- *
- *   The 1000-entry target (500 US + 500 FR) is a human task. This module catches the regressions that
- *   creep in over time as new entries land.
+ *   Validates the hand-labeled golden eval set, where every labeled component must occur in `raw`.
  */
-import { readDirectory } from "@mailwoman/core/fs/readers";
-import { parseJSONStrict } from "@mailwoman/core/json";
-import { COMPONENT_TAGS } from "@mailwoman/core/types";
-import { reconcileComponents } from "@mailwoman/formatter";
-import { join } from "path-ts";
+import { componentsPresentIn } from "@mailwoman/codex/address/format";
+import { COMPONENT_TAGS } from "@mailwoman/codex/component";
+import { isAlpha2CodeShape } from "@mailwoman/codex/country";
+import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json";
+import { PathBuilder } from "path-ts";
 import { TextSpliterator } from "spliterator";
+import { Globerator } from "spliterator/node/fs";
 const TAG_SET = new Set(COMPONENT_TAGS);
 /**
- * Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
+ * Parses one JSONL line into a `GoldenEntry`.
+ *
+ * `validateGoldenFile` records the thrown message against the line number, so this parse must stay strict.
+ *
+ * @throws On schema violations.
  */
 export function parseGoldenLine(line) {
-    // The throw IS the result: `validateGoldenFile` catches it and records the message against the line
-    // number, so a tolerant parse would report a corrupt row as valid.
     const obj = parseJSONStrict(line);
     if (typeof obj.raw !== "string" || !obj.raw.length) {
         throw new Error("missing/empty raw");
     }
-    if (typeof obj.country !== "string" || !/^[A-Z]{2}$/u.test(obj.country)) {
-        throw new Error(`country must be ISO 3166-1 alpha-2 (got ${JSON.stringify(obj.country)})`);
+    if (!isAlpha2CodeShape(obj.country)) {
+        throw new Error(`country must be ISO 3166-1 alpha-2 (got ${stringifyJSON(obj.country)})`);
     }
     if (obj.source !== "golden") {
-        throw new Error(`source must be "golden" (got ${JSON.stringify(obj.source)})`);
+        throw new Error(`source must be "golden" (got ${stringifyJSON(obj.source)})`);
     }
     const components = (obj.components ?? {});
     for (const [k, v] of Object.entries(components)) {
@@ -58,33 +48,34 @@ export function parseGoldenLine(line) {
     };
 }
 /**
- * Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent).
+ * Returns the tags in `entry` whose values do not occur in `entry.raw`.
+ *
+ * A golden `raw` is hand-written, so this check uses containment instead of
+ * reconciling against a rendered layout.
  */
 export function unreachableComponents(entry) {
-    const reconciled = reconcileComponents(entry.components, entry.raw);
+    const present = componentsPresentIn(entry.components, entry.raw);
     const missing = [];
     for (const tag of Object.keys(entry.components)) {
-        if (!(tag in reconciled)) {
+        if (!(tag in present)) {
             missing.push(tag);
         }
     }
     return missing;
 }
 /**
- * Validate one `.jsonl` file end-to-end, returning a list of issues.
+ * Validates one `.jsonl` file and returns its issues.
  *
- * Parses line by line over `TextSpliterator` rather than `JSONSpliterator`: every issue this returns carries the LINE
- * NUMBER it was found on, and a malformed line has to be REPORTED rather than thrown. `JSONSpliterator` parses each row
- * for you and throws on the first bad one — correct for consumers that want the rows, wrong for the validator whose
- * whole job is locating the bad ones.
+ * This reads lines with `TextSpliterator` because `JSONSpliterator` throws on the first malformed row.
+ * The validator must instead report every bad row with its line number.
  */
-export async function validateGoldenFile(path) {
+export async function validateGoldenFile(source) {
+    const path = source.toString();
     const issues = [];
-    // Counted over EVERY row including blanks, so the number matches what an editor shows.
+    // Blank lines count too, so the number matches the line an editor shows.
     let lineNumber = 0;
-    for await (const raw of TextSpliterator.fromAsync(path, { skipEmpty: false })) {
+    for await (const line of TextSpliterator.fromAsync(path, { skipEmpty: false })) {
         lineNumber++;
-        const line = raw.trim();
         if (!line)
             continue;
         const i = lineNumber - 1;
@@ -106,21 +97,18 @@ export async function validateGoldenFile(path) {
     return issues;
 }
 /**
- * Validate every `.jsonl` in a golden directory.
+ * Validates every `.jsonl` file directly inside a golden directory.
  */
 export async function validateGoldenDir(dir) {
-    const files = (await readDirectory(dir)).filter((n) => n.endsWith(".jsonl")).toSorted();
+    const root = PathBuilder.from(dir);
+    const files = await Globerator.files("jsonl", { cwd: root, absolute: false, recursive: false }).toSorted();
     const issues = [];
     let entries = 0;
     for (const name of files) {
-        const fullPath = join(dir, name);
+        const fullPath = root(name);
         const fileIssues = await validateGoldenFile(fullPath);
         issues.push(...fileIssues);
-        for await (const line of TextSpliterator.fromAsync(fullPath)) {
-            if (line.trim()) {
-                entries++;
-            }
-        }
+        entries += await TextSpliterator.countAsync(fullPath);
     }
     return { entries, files: files.length, issues };
 }

package/out/utils/golden.js.map

@@ -1 +1 @@
-{"version":3,"file":"golden.js","sourceRoot":"","sources":["../../lib/utils/golden.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAqB,MAAM,uBAAuB,CAAA;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,cAAmC,CAAC,CAAA;AA6CpE;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC3C,oGAAoG;IACpG,mEAAmE;IACnE,MAAM,GAAG,GAAG,eAAe,CAAiD,IAAI,CAAC,CAAA;IAEjF,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC3F,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC/E,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAA4B,CAAA;IAEpE,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAA;QAElE,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,6BAA6B,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,UAAU,EAAE,UAAuC;QACnD,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC5D,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACvD,MAAM,UAAU,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;IACnE,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAmB,EAAE,CAAC;QACnE,IAAI,CAAC,CAAC,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACpD,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,uFAAuF;IACvF,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC/E,UAAU,EAAE,CAAA;QACZ,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;QAEvB,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,CAAC,GAAG,UAAU,GAAG,CAAC,CAAA;QAExB,IAAI,CAAC;YACJ,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;YACnC,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAA;YAEhD,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,CAAC,GAAG,CAAC;oBACX,MAAM,EAAE,oCAAoC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;iBACpE,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IAClD,MAAM,KAAK,GAAG,CAAC,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;IACvF,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAChC,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QACrD,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;QAE1B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9D,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAChD,CAAC"}
+{"version":3,"file":"golden.js","sourceRoot":"","sources":["../../lib/utils/golden.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAA;AACrE,OAAO,EAAE,cAAc,EAAqB,MAAM,4BAA4B,CAAA;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACrE,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,cAAmC,CAAC,CAAA;AA8CpE;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC3C,MAAM,GAAG,GAAG,eAAe,CAAiD,IAAI,CAAC,CAAA;IAEjF,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,2CAA2C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,gCAAgC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC9E,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAA4B,CAAA;IAEpE,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAA;QAElE,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,6BAA6B,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,UAAU,EAAE,UAAuC;QACnD,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC5D,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACvD,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;IAChE,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAmB,EAAE,CAAC;QACnE,IAAI,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAuB;IAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC9B,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,yEAAyE;IACzE,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAChF,UAAU,EAAE,CAAA;QAEZ,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,CAAC,GAAG,UAAU,GAAG,CAAC,CAAA;QAExB,IAAI,CAAC;YACJ,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;YACnC,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAA;YAEhD,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,CAAC,GAAG,CAAC;oBACX,MAAM,EAAE,oCAAoC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;iBACpE,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAoB;IAC3D,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAClC,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAC1G,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAA;QAC3B,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QACrD,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;QAE1B,OAAO,IAAI,MAAM,eAAe,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;IACtD,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAChD,CAAC"}

package/out/utils/license.d.ts

@@ -3,36 +3,173 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Corpus licensing — the single source of truth for the training-data license policy (#26).
+ *   Corpus licensing — the training-data license policy, read from a row's obligations rather than from
+ *   the shape of its license string.
  *
- *   Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
- *   build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
- *   license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
- *   inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
- *   })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
- *   string unless the operator named it. This avoids the trap of silently dropping allowed data
- *   mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
- *   ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
- *   dropped 48M allowed rows).
+ *   Exclusion is a deliberate act rather than a silent default: the build includes every row an adapter
+ *   yields (stamping its `license`), and a build that needs a license set carrying no share-alike
+ *   obligation asks for {@linkcode LicensePolicy.ShareAlikeFree}, so no source is dropped unless the
+ *   operator selected the policy.
  *
- *   Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
- *   — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
- *   proprietary-weights build via `--exclude-share-alike`.
+ *   The refusal reads `readLicenseRecord`, which resolves a value to an SPDX expression and looks up
+ *   that expression's obligations. An anchored prefix match over the raw column cannot do this. Measured
+ *   over `v0.7.0-de-holdout` on 2026-09-26, the column holds 71 distinct values across 703,835,753 rows,
+ *   of which 4 are SPDX identifiers and 66 are prose. A row whose license reads "OpenStreetMap venue +
+ *   sub-venue names (ODbL, © OpenStreetMap contributors) …" is share-alike-derived and starts with
+ *   neither `ODbL` nor `CC-BY-SA`, so `/^ODbL/` admitted it.
+ *
+ *   Three refusal classes exist because they are different claims. A value that resolves to an
+ *   expression carrying share-alike states the obligation. A value whose prose mentions a share-alike
+ *   license while resolving to no expression is share-alike-derived without stating its own grant. A
+ *   value that resolves to no expression has unknown obligations. Unknown obligations differ from an empty
+ *   obligation set. Of these rows, 628,203,119 contain prose this repository writes about its own renderings. The
+ *   third class is therefore reported rather than refused under {@linkcode LicensePolicy.ShareAlikeFree},
+ *   and {@linkcode LicensePolicy.ResolvedOnly} refuses it for a caller that needs every grant stated.
+ */
+/**
+ * Which rows a corpus build admits on the evidence of their license values.
+ */
+export declare const LicensePolicy: {
+    /**
+     * Admit every row an adapter yields, whatever its license value states.
+     * The policy a measurement runs under.
+     */
+    readonly All: "all";
+    /**
+     * Refuse a row whose license imposes a share-alike obligation.
+     *
+     * Also refuse a row whose license text mentions a share-alike license while stating no grant of its own.
+     *
+     * A row whose license resolves to no expression and mentions no share-alike license is admitted
+     * and counted, because its obligations are unknown rather than known to include share-alike.
+     */
+    readonly ShareAlikeFree: "share-alike-free";
+    /**
+     * Apply all refusals from {@linkcode LicensePolicy.ShareAlikeFree}.
+     * Also refuse a row whose license resolves to no expression.
+     *
+     * Measured over `v0.7.0-de-holdout`, this refuses 628,203,119 of 703,835,753 rows,
+     * so a caller asking for it is asking for the subset whose grant is stated as an identifier.
+     */
+    readonly ResolvedOnly: "resolved-only";
+};
+/**
+ * One of the {@linkcode LicensePolicy} values.
+ */
+export type LicensePolicy = (typeof LicensePolicy)[keyof typeof LicensePolicy];
+/**
+ * Why a row's license value kept it out of the corpus, counted separately per class
+ * because the classes rest on different evidence.
  */
+export declare const LicenseRefusalKind: {
+    /**
+     * The operator supplied this license prefix in `--exclude-licenses`.
+     */
+    readonly OperatorExcluded: "operator-excluded";
+    /**
+     * The value resolves to an expression whose recorded obligations include share-alike.
+     */
+    readonly ShareAlikeCarried: "share-alike-carried";
+    /**
+     * The value's text mentions a share-alike license while resolving to no expression of its own.
+     */
+    readonly ShareAlikeMentioned: "share-alike-mentioned";
+    /**
+     * The value resolves to no expression, so its obligations are unknown.
+     */
+    readonly Unresolved: "unresolved";
+};
 /**
- * Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
- * `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use it.
+ * One of the {@linkcode LicenseRefusalKind} values.
  */
-export declare const SHARE_ALIKE_PATTERN: RegExp;
+export type LicenseRefusalKind = (typeof LicenseRefusalKind)[keyof typeof LicenseRefusalKind];
 /**
- * Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored, case-insensitive prefix
- * patterns. Each entry matches a license string that STARTS with it, so `CC-BY-SA` catches `CC-BY-SA-3.0`,
- * `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the spec is a literal license prefix, not a user-supplied
- * regex.
+ * What a license value means for a build, decided once per distinct value.
+ */
+export interface LicenseVerdict {
+    /**
+     * Why the row was refused, or `null` when the policy admits it.
+     */
+    refusal: LicenseRefusalKind | null;
+    /**
+     * Whether the value resolves to an expression whose obligations are recorded.
+     *
+     * An admitted row reading `false` here entered with unknown obligations.
+     */
+    resolved: boolean;
+    /**
+     * Whether the value's text mentions a share-alike license, whether or not it was refused for it.
+     */
+    mentionsShareAlike: boolean;
+}
+/**
+ * Compile an `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
+ * case-insensitive prefix patterns, so `CC-BY-SA` catches `CC-BY-SA-3.0`.
+ *
+ * The spec is a literal license prefix the operator typed.
+ * Regex metacharacters are escaped.
+ *
+ * This is the one place a prefix match is the right reading, because the operator
+ * is naming a spelling rather than asking about an obligation.
  */
 export declare function compileLicenseExcludes(spec: string): RegExp[];
 /**
- * True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded).
+ * True when `license` matches any of the operator's exclude `patterns`; empty patterns never exclude.
  */
 export declare function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean;
+/**
+ * Reads one license value against a policy.
+ *
+ * This resolves an expression and runs the mention patterns on every call.
+ * A corpus build asks about hundreds of millions of rows holding tens of distinct values,
+ * so a row loop calls {@linkcode createLicenseVerdictCache} instead.
+ */
+export declare function licenseVerdict(license: string | undefined, policy: LicensePolicy, excluded?: readonly RegExp[]): LicenseVerdict;
+/**
+ * One license value in a built corpus that a share-alike-free reading refuses, with its row count.
+ */
+export interface ShareAlikeFinding {
+    /**
+     * The value exactly as the corpus stores it.
+     */
+    license: string;
+    /**
+     * Rows the built manifest counts under this value.
+     */
+    rows: number;
+    kind: LicenseRefusalKind;
+}
+/**
+ * Every license value in a built corpus's license set that imposes or mentions share-alike.
+ *
+ * Takes the `licenses` map recorded by `BuildCorpusManifest`.
+ * The map counts the rows adapters yielded.
+ *
+ * A caller about to move or publish a corpus reads this rather than its own list of source ids,
+ * because the obligation is a property of the license value on the row.
+ *
+ * Ordered by row count, most rows first.
+ */
+export declare function shareAlikeFindings(licenses: Readonly<Record<string, number>>): ShareAlikeFinding[];
+/**
+ * A license reading memoized per distinct raw value, for a loop over corpus rows.
+ */
+export interface LicenseVerdictCache {
+    /**
+     * The verdict for this value under the policy this cache was built with.
+     */
+    read(license: string | undefined): LicenseVerdict;
+    /**
+     * Every distinct value refused so far, with the class it was refused under,
+     * so a build that dropped rows reports which values caused each drop.
+     */
+    refusedValues(): Map<string, LicenseRefusalKind>;
+}
+/**
+ * Builds a cache that reads each distinct license value once.
+ *
+ * The corpus holds tens of distinct values over hundreds of millions of rows,
+ * so the memo turns one expression resolution per row into one per value.
+ */
+export declare function createLicenseVerdictCache(policy: LicensePolicy, excluded?: readonly RegExp[]): LicenseVerdictCache;
 //# sourceMappingURL=license.d.ts.map

package/out/utils/license.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../lib/utils/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAAmD,CAAA;AAEnF;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM7D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAIjG"}
+{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../lib/utils/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH;;GAEG;AACH,eAAO,MAAM,aAAa;IACzB;;;OAGG;aACH,GAAG,EAAE,KAAK;IACV;;;;;;;OAOG;aACH,cAAc,EAAE,kBAAkB;IAClC;;;;;;OAMG;aACH,YAAY,EAAE,eAAe;CACpB,CAAA;AAEV;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,OAAO,aAAa,CAAC,CAAA;AAE9E;;;GAGG;AACH,eAAO,MAAM,kBAAkB;IAC9B;;OAEG;aACH,gBAAgB,EAAE,mBAAmB;IACrC;;OAEG;aACH,iBAAiB,EAAE,qBAAqB;IACxC;;OAEG;aACH,mBAAmB,EAAE,uBAAuB;IAC5C;;OAEG;aACH,UAAU,EAAE,YAAY;CACf,CAAA;AAEV;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,kBAAkB,CAAC,CAAC,MAAM,OAAO,kBAAkB,CAAC,CAAA;AAE7F;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,OAAO,EAAE,kBAAkB,GAAG,IAAI,CAAA;IAClC;;;;OAIG;IACH,QAAQ,EAAE,OAAO,CAAA;IACjB;;OAEG;IACH,kBAAkB,EAAE,OAAO,CAAA;CAC3B;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAE7D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAIjG;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC7B,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,MAAM,EAAE,aAAa,EACrB,QAAQ,GAAE,SAAS,MAAM,EAAO,GAC9B,cAAc,CAsBhB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,kBAAkB,CAAA;CACxB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,iBAAiB,EAAE,CAYlG;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,cAAc,CAAA;IACjD;;;OAGG;IACH,aAAa,IAAI,GAAG,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;CAChD;AAED;;;;;GAKG;AACH,wBAAgB,yBAAyB,CACxC,MAAM,EAAE,aAAa,EACrB,QAAQ,GAAE,SAAS,MAAM,EAAO,GAC9B,mBAAmB,CAyBrB"}