@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/lint-slice.js

@@ -1,322 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Corpus linter. Compares a new slice against pre-computed corpus statistics and flags patterns
- *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
- *   venue templates.
- *
- *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
- *
- *   1. **Token-label distribution outliers.** For each token in the new slice, compare the slice's
- *        majority label to the corpus's majority label. Flag when the corpus has a
- *        confidently-established majority (>66%) AND the slice's majority differs AND both have
- *        non-trivial counts (slice ≥ 50, corpus ≥ 200).
- *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
- *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
- *        we're introducing a novel association, not shifting a distribution.
- *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in slice while the
- *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
- *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
- *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
- *        mappings — flagging matches.
- *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of slice.
- *
- *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
- *   no errors, 1 if any errors (warnings don't check). Per the design, the MANIFEST entry for a
- *   flagged slice should require `lint_acknowledged: true` before training consumes it.
- *
- *   Usage: mailwoman dev lint corpus-slice\
- *   --slice <new-slice.parquet>\
- *   --stats <corpus-stats.json>\
- *   [--rules <rules.json>]\
- *   [--out-md /tmp/lint-report.md]\
- *   [--out-json /tmp/lint-report.json]
- */
-import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
-import { writeLocalFile, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
-import { resolvePackagePath } from "@mailwoman/core/module/resolvers";
-import { accumulateCooccurrences, createCooccurrenceStats, SLICE_STATS_SEP as SEP, streamTokenLabelRows, } from "#utils/slice-stats";
-/**
- * Occurrences of a forbidden label before it is reported — one or two are noise, five is a pattern.
- */
-const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5;
-/**
- * Examples printed per finding before the list is truncated.
- */
-const MAX_LISTED_EXAMPLES = 20;
-/**
- * Calibrated thresholds (DeepSeek turn 9). These can be tuned over time if new failure modes surface that the current
- * numbers miss.
- */
-const CORPUS_CONFIDENCE_FLOOR = 0.66;
-const SLICE_MIN_COUNT = 50;
-const CORPUS_MIN_COUNT = 200;
-const VACUUM_SLICE_MIN_COUNT = 20;
-const VACUUM_CORPUS_MIN_COUNT = 100;
-const BIGRAM_MIN_COUNT = 10;
-const ALL_O_RATIO_CEILING = 0.9;
-/**
- * Default `lint-rules.json` path — the rules ship beside this module in the source tree. tsc does not emit
- * readFileSync'd JSON into `out/`, so the compiled tree falls back to the source-tree copy (corpus/out/src/tools/ →
- * corpus/src/tools/). In-repo the `node` exports condition loads this module from source anyway, so the sibling URL is
- * the common path.
- */
-function defaultRulesPath() {
-    return resolvePackagePath("@mailwoman/corpus", "lib", "tools", "lint-rules.json");
-}
-async function statsFromSlice(rows) {
-    const co = createCooccurrenceStats();
-    const out = {
-        rowCount: 0,
-        tokens: co.tokens,
-        bigrams: co.bigrams,
-        truncatedRows: 0,
-        allORows: 0,
-    };
-    for await (const row of rows) {
-        out.rowCount++;
-        if (row.tokens.length !== row.labels.length) {
-            out.truncatedRows++;
-            continue;
-        }
-        if (row.labels.every((l) => l === "O")) {
-            out.allORows++;
-        }
-        accumulateCooccurrences(co, row.tokens, row.labels);
-    }
-    return out;
-}
-function majorityLabel(distribution) {
-    const entries = distribution instanceof Map ? [...distribution.entries()] : Object.entries(distribution);
-    let bestLabel = "";
-    let bestCount = 0;
-    let total = 0;
-    for (const [label, count] of entries) {
-        total += count;
-        if (count > bestCount) {
-            bestCount = count;
-            bestLabel = label;
-        }
-    }
-    return { label: bestLabel, count: bestCount, total, confidence: total === 0 ? 0 : bestCount / total };
-}
-function checkDistributionOutliers(slice, corpus) {
-    const flags = [];
-    for (const [token, sliceLabelMap] of slice.tokens) {
-        const corpusLabelMap = corpus.tokens[token];
-        if (!corpusLabelMap)
-            continue;
-        const sliceMaj = majorityLabel(sliceLabelMap);
-        const corpusMaj = majorityLabel(corpusLabelMap);
-        if (corpusMaj.confidence >= CORPUS_CONFIDENCE_FLOOR &&
-            sliceMaj.label !== corpusMaj.label &&
-            sliceMaj.count >= SLICE_MIN_COUNT &&
-            corpusMaj.total >= CORPUS_MIN_COUNT) {
-            flags.push({
-                check: "distribution-outlier",
-                severity: "error",
-                token,
-                sliceLabel: sliceMaj.label,
-                corpusLabel: corpusMaj.label,
-                sliceCount: sliceMaj.count,
-                corpusCount: corpusMaj.count,
-                detail: `Token "${token}": slice majority is ${sliceMaj.label} (${sliceMaj.count}/${sliceMaj.total}, ${(sliceMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
-            });
-        }
-    }
-    return flags;
-}
-function checkLabelVacuum(slice, corpus) {
-    const flags = [];
-    for (const [token, sliceLabelMap] of slice.tokens) {
-        const corpusLabelMap = corpus.tokens[token];
-        if (!corpusLabelMap)
-            continue;
-        const corpusTotal = Object.values(corpusLabelMap).reduce((a, b) => a + b, 0);
-        if (corpusTotal < VACUUM_CORPUS_MIN_COUNT)
-            continue;
-        for (const [label, sliceCount] of sliceLabelMap) {
-            if (sliceCount < VACUUM_SLICE_MIN_COUNT)
-                continue;
-            if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
-                flags.push({
-                    check: "label-vacuum",
-                    severity: "error",
-                    token,
-                    sliceLabel: label,
-                    sliceCount,
-                    corpusCount: corpusTotal,
-                    detail: `Token "${token}": slice labels it ${label} ${sliceCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
-                });
-            }
-        }
-    }
-    return flags;
-}
-function checkBigramCollisions(slice, corpus) {
-    const flags = [];
-    for (const [bigram, sliceLabelMap] of slice.bigrams) {
-        const corpusLabelMap = corpus.bigrams[bigram];
-        if (!corpusLabelMap)
-            continue;
-        const sliceMaj = majorityLabel(sliceLabelMap);
-        const corpusMaj = majorityLabel(corpusLabelMap);
-        if (sliceMaj.label !== corpusMaj.label &&
-            sliceMaj.count >= BIGRAM_MIN_COUNT &&
-            corpusMaj.count >= BIGRAM_MIN_COUNT) {
-            const renderBigram = bigram.split(SEP).join(" ");
-            const renderSliceLabel = sliceMaj.label.split(SEP).join(" → ");
-            const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ");
-            flags.push({
-                check: "bigram-collision",
-                severity: "error",
-                bigram: renderBigram,
-                sliceLabel: renderSliceLabel,
-                corpusLabel: renderCorpusLabel,
-                sliceCount: sliceMaj.count,
-                corpusCount: corpusMaj.count,
-                detail: `Bigram "${renderBigram}": slice label-bigram is [${renderSliceLabel}] (${sliceMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
-            });
-        }
-    }
-    return flags;
-}
-function checkRules(slice, rulesFile) {
-    const flags = [];
-    const compiled = rulesFile.rules.map((r) => ({
-        rule: r,
-        regex: new RegExp(r.pattern, r.pattern_case_sensitive ? "" : "i"),
-    }));
-    for (const [token, labelMap] of slice.tokens) {
-        for (const { rule, regex } of compiled) {
-            if (!regex.test(token))
-                continue;
-            for (const [label, count] of labelMap) {
-                if (rule.forbidden_labels.includes(label) && count >= FORBIDDEN_LABEL_REPORT_THRESHOLD) {
-                    flags.push({
-                        check: "anti-pattern-rule",
-                        severity: rule.severity,
-                        ruleID: rule.id,
-                        token,
-                        sliceLabel: label,
-                        sliceCount: count,
-                        detail: `Token "${token}" matched rule ${rule.id} and is labeled ${label} ${count} time(s). Rule message: ${rule.message}`,
-                    });
-                }
-            }
-        }
-    }
-    return flags;
-}
-function checkSanity(slice) {
-    const flags = [];
-    if (slice.truncatedRows > 0) {
-        flags.push({
-            check: "truncated-rows",
-            severity: "error",
-            detail: `${slice.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
-        });
-    }
-    const allORatio = slice.allORows / Math.max(1, slice.rowCount);
-    if (allORatio >= ALL_O_RATIO_CEILING) {
-        flags.push({
-            check: "all-O-slice",
-            severity: "warn",
-            detail: `${slice.allORows}/${slice.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. Slice contributes no signal.`,
-        });
-    }
-    return flags;
-}
-function renderReport(opts, slice, flags) {
-    const errors = flags.filter((f) => f.severity === "error");
-    const warns = flags.filter((f) => f.severity === "warn");
-    const verdict = !errors.length ? "**PASS** ✓" : "**FLAGGED** ⚠";
-    const lines = [
-        `# Corpus Lint: ${verdict}`,
-        "",
-        `- **Slice:** \`${opts.slicePath}\``,
-        `- **Corpus stats:** \`${opts.statsPath}\``,
-        `- **Rules:** \`${opts.rulesPath}\``,
-        `- **Slice rows:** ${slice.rowCount}`,
-        `- **Unique tokens:** ${slice.tokens.size}`,
-        `- **Unique bigrams:** ${slice.bigrams.size}`,
-        "",
-        `**Errors:** ${errors.length} (checks the slice's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
-        `**Warnings:** ${warns.length} (advisory)`,
-        "",
-    ];
-    if (!flags.length) {
-        lines.push("No anomalies detected.");
-        return lines.join("\n");
-    }
-    const byCheck = new Map();
-    for (const f of flags) {
-        const arr = byCheck.get(f.check) ?? [];
-        arr.push(f);
-        byCheck.set(f.check, arr);
-    }
-    for (const [check, list] of byCheck) {
-        lines.push(`## ${check} (${list.length})`);
-        lines.push("");
-        // Sort by sliceCount desc — highest-volume issues first
-        list.sort((a, b) => (b.sliceCount ?? 0) - (a.sliceCount ?? 0));
-        for (const f of list.slice(0, 20)) {
-            lines.push(`- **[${f.severity.toUpperCase()}]** ${f.detail}`);
-        }
-        if (list.length > MAX_LISTED_EXAMPLES) {
-            lines.push(`- ... and ${list.length - 20} more`);
-        }
-        lines.push("");
-    }
-    return lines.join("\n");
-}
-/**
- * Lint a slice against corpus stats + the anti-pattern rules; print the markdown report to stdout.
- */
-export async function lintCorpusSlice(options, report) {
-    const rulesPath = options.rulesPath ?? defaultRulesPath();
-    report?.(`Reading corpus stats from ${options.statsPath}...`);
-    const corpus = await readLocalJSONFile(options.statsPath);
-    report?.(`  ${corpus.row_count} rows from ${corpus.slice_paths.length} slice(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`);
-    report?.(`Reading slice from ${options.slicePath}...`);
-    const slice = await statsFromSlice(streamTokenLabelRows(options.slicePath));
-    report?.(`  ${slice.rowCount} rows`);
-    report?.(`Loading rules from ${rulesPath}...`);
-    const rulesFile = await readLocalJSONFile(rulesPath);
-    report?.(`Running checks...`);
-    const flags = [
-        ...checkDistributionOutliers(slice, corpus),
-        ...checkLabelVacuum(slice, corpus),
-        ...checkBigramCollisions(slice, corpus),
-        ...checkRules(slice, rulesFile),
-        ...checkSanity(slice),
-    ];
-    const rendered = renderReport({ slicePath: options.slicePath, statsPath: options.statsPath, rulesPath }, slice, flags);
-    console.log(rendered);
-    if (options.outMd) {
-        await writeLocalFile(rendered, options.outMd);
-    }
-    if (options.outJSON) {
-        await writeLocalJSONFile({
-            slice: options.slicePath,
-            stats: options.statsPath,
-            flags,
-            summary: {
-                errors: flags.filter((f) => f.severity === "error").length,
-                warnings: flags.filter((f) => f.severity === "warn").length,
-            },
-        }, options.outJSON);
-    }
-    const errorCount = flags.filter((f) => f.severity === "error").length;
-    const warningCount = flags.filter((f) => f.severity === "warn").length;
-    if (errorCount > 0) {
-        report?.(`LINT FAILED: ${errorCount} error(s).`);
-    }
-    else {
-        report?.("LINT PASSED.");
-    }
-    return { errors: errorCount, warnings: warningCount, findings: flags, report: rendered };
-}
-//# sourceMappingURL=lint-slice.js.map

package/out/tools/lint-slice.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-slice.js","sourceRoot":"","sources":["../../lib/tools/lint-slice.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAErE,OAAO,EACN,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,IAAI,GAAG,EACtB,oBAAoB,GAEpB,MAAM,oBAAoB,CAAA;AAE3B;;GAEG;AACH,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C;;GAEG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAE9B;;;GAGG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAA;AACpC,MAAM,eAAe,GAAG,EAAE,CAAA;AAC1B,MAAM,gBAAgB,GAAG,GAAG,CAAA;AAC5B,MAAM,sBAAsB,GAAG,EAAE,CAAA;AACjC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AACnC,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,mBAAmB,GAAG,GAAG,CAAA;AAE/B;;;;;GAKG;AACH,SAAS,gBAAgB;IACxB,OAAO,kBAAkB,CAAC,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAA;AAClF,CAAC;AAwDD,KAAK,UAAU,cAAc,CAAC,IAA6B;IAC1D,MAAM,EAAE,GAAG,uBAAuB,EAAE,CAAA;IAEpC,MAAM,GAAG,GAAe;QACvB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,OAAO,EAAE,EAAE,CAAC,OAAO;QACnB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;KACX,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,GAAG,CAAC,QAAQ,EAAE,CAAA;QAEd,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,GAAG,CAAC,aAAa,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACxC,GAAG,CAAC,QAAQ,EAAE,CAAA;QACf,CAAC;QAED,uBAAuB,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,aAAa,CAAC,YAA0D;IAMhF,MAAM,OAAO,GAAG,YAAY,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;IACxG,IAAI,SAAS,GAAG,EAAE,CAAA;IAClB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,KAAK,EAAE,CAAA;AACtG,CAAC;AA+BD,SAAS,yBAAyB,CAAC,KAAiB,EAAE,MAAmB;IACxE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,SAAS,CAAC,UAAU,IAAI,uBAAuB;YAC/C,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,eAAe;YACjC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,sBAAsB;gBAC7B,QAAQ,EAAE,OAAO;gBACjB,KAAK;gBACL,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,UAAU,KAAK,wBAAwB,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;aAC9Q,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAiB,EAAE,MAAmB;IAC/D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAE5E,IAAI,WAAW,GAAG,uBAAuB;YAAE,SAAQ;QAEnD,KAAK,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;YACjD,IAAI,UAAU,GAAG,sBAAsB;gBAAE,SAAQ;YAEjD,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxE,KAAK,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,cAAc;oBACrB,QAAQ,EAAE,OAAO;oBACjB,KAAK;oBACL,UAAU,EAAE,KAAK;oBACjB,UAAU;oBACV,WAAW,EAAE,WAAW;oBACxB,MAAM,EAAE,UAAU,KAAK,sBAAsB,KAAK,IAAI,UAAU,2BAA2B,WAAW,6DAA6D;iBACnK,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAiB,EAAE,MAAmB;IACpE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,gBAAgB;YAClC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9D,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEhE,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,kBAAkB;gBACzB,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,gBAAgB;gBAC5B,WAAW,EAAE,iBAAiB;gBAC9B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,WAAW,YAAY,6BAA6B,gBAAgB,MAAM,QAAQ,CAAC,KAAK,+BAA+B,iBAAiB,MAAM,SAAS,CAAC,KAAK,sDAAsD;aAC3N,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB,EAAE,SAAwB;IAC9D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KACjE,CAAC,CAAC,CAAA;IAEH,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9C,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAQ;YAEhC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,gCAAgC,EAAE,CAAC;oBACxF,KAAK,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,mBAAmB;wBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,KAAK;wBACL,UAAU,EAAE,KAAK;wBACjB,UAAU,EAAE,KAAK;wBACjB,MAAM,EAAE,UAAU,KAAK,kBAAkB,IAAI,CAAC,EAAE,mBAAmB,KAAK,IAAI,KAAK,2BAA2B,IAAI,CAAC,OAAO,EAAE;qBAC1H,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,WAAW,CAAC,KAAiB;IACrC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,gBAAgB;YACvB,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,aAAa,uEAAuE;SACrG,CAAC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE9D,IAAI,SAAS,IAAI,mBAAmB,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,aAAa;YACpB,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,UAAU,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yDAAyD;SAC1I,CAAC,CAAA;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CACpB,IAAiE,EACjE,KAAiB,EACjB,KAAsB;IAEtB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAA;IACxD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe,CAAA;IAE/D,MAAM,KAAK,GAAa;QACvB,kBAAkB,OAAO,EAAE;QAC3B,EAAE;QACF,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,yBAAyB,IAAI,CAAC,SAAS,IAAI;QAC3C,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,qBAAqB,KAAK,CAAC,QAAQ,EAAE;QACrC,wBAAwB,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;QAC3C,yBAAyB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAC7C,EAAE;QACF,eAAe,MAAM,CAAC,MAAM,kFAAkF;QAC9G,iBAAiB,KAAK,CAAC,MAAM,aAAa;QAC1C,EAAE;KACF,CAAA;IAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAA;QAEpC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAA;IAElD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QACtC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,wDAAwD;QACxD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAA;QAE9D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAA;QACjD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACf,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAA;IACzD,MAAM,EAAE,CAAC,6BAA6B,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAC7D,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAc,OAAO,CAAC,SAAS,CAAC,CAAA;IAEtE,MAAM,EAAE,CACP,KAAK,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,WAAW,CAAC,MAAM,cAAc,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,UAAU,CACnK,CAAA;IAED,MAAM,EAAE,CAAC,sBAAsB,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAEtD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAA;IAE3E,MAAM,EAAE,CAAC,KAAK,KAAK,CAAC,QAAQ,OAAO,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,sBAAsB,SAAS,KAAK,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAgB,SAAS,CAAC,CAAA;IAEnE,MAAM,EAAE,CAAC,mBAAmB,CAAC,CAAA;IAE7B,MAAM,KAAK,GAAoB;QAC9B,GAAG,yBAAyB,CAAC,KAAK,EAAE,MAAM,CAAC;QAC3C,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC;QAClC,GAAG,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC;QACvC,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC;QAC/B,GAAG,WAAW,CAAC,KAAK,CAAC;KACrB,CAAA;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAEtH,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAErB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC9C,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,kBAAkB,CACvB;YACC,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK;YACL,OAAO,EAAE;gBACR,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM;gBAC1D,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM;aAC3D;SACD,EACD,OAAO,CAAC,OAAO,CACf,CAAA;IACF,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM,CAAA;IACrE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IAEtE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,gBAAgB,UAAU,YAAY,CAAC,CAAA;IACjD,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACzF,CAAC"}

package/out/tools/overlay-manifest.d.ts

@@ -1,39 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
- *   ADDS one slice parquet to a base corpus, keeping every base slice VERBATIM (pure overlay ADD),
- *   and re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet +
- *   --source so it works for any overlay slice (the fr-admin-split one is the original; #148's
- *   overture-multilocale is the second user).
- *
- *   Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
- *   read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
- *
- *   Pipeline (the recipe rides the result): node scripts/build-overture-multilocale-canonical.mjs
- *   --cap 150000 --out /tmp/ovl/overture-ml.canonical.jsonl node scripts/align-canonical-slice.ts
- *   --input <canonical> --output <labeled> --corpus-version 0.5.0 mailwoman dev jsonl-to-parquet
- *   --input <labeled> --output <NEW>/train/<slice-parquet> node
- *   scripts/assemble-overlay-manifest.ts --base <BASE>/MANIFEST.json --new-dir <NEW>\
- *   --modal-root /data/corpus/versioned/<ver>/<dir> --version <ver>\
- *   --slice-parquet <slice-parquet> --source <source> --note "..."
- *
- *   # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
- */
-export interface OverlayManifestOptions {
-    base: string;
-    newDir: string;
-    modalRoot: string;
-    version: string;
-    sliceParquet: string;
-    source: string;
-    note: string;
-}
-/**
- * Resolve a base manifest's slice path to the mounted corpus tree used by Modal.
- */
-export declare function rerootBaseSlicePath(path: string, baseManifestPath: string): string;
-export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void>;
-//# sourceMappingURL=overlay-manifest.d.ts.map

package/out/tools/overlay-manifest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"overlay-manifest.d.ts","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAwDH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAalF;AAED,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwCzF"}

package/out/tools/overlay-manifest.js

@@ -1,90 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
- *   ADDS one slice parquet to a base corpus, keeping every base slice VERBATIM (pure overlay ADD),
- *   and re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet +
- *   --source so it works for any overlay slice (the fr-admin-split one is the original; #148's
- *   overture-multilocale is the second user).
- *
- *   Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
- *   read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
- *
- *   Pipeline (the recipe rides the result): node scripts/build-overture-multilocale-canonical.mjs
- *   --cap 150000 --out /tmp/ovl/overture-ml.canonical.jsonl node scripts/align-canonical-slice.ts
- *   --input <canonical> --output <labeled> --corpus-version 0.5.0 mailwoman dev jsonl-to-parquet
- *   --input <labeled> --output <NEW>/train/<slice-parquet> node
- *   scripts/assemble-overlay-manifest.ts --base <BASE>/MANIFEST.json --new-dir <NEW>\
- *   --modal-root /data/corpus/versioned/<ver>/<dir> --version <ver>\
- *   --slice-parquet <slice-parquet> --source <source> --note "..."
- *
- *   # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
- */
-import { readLocalBuffer, readLocalJSONFile, statPath } from "@mailwoman/core/fs/readers";
-import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
-import { sha256Hex } from "@mailwoman/core/hash";
-import { basename, dirname, join } from "path-ts";
-import { connectDuckDB, escapeSQLString } from "#utils/parquet";
-async function descriptor(localPath, modalPath, split, source) {
-    const db = await connectDuckDB();
-    const result = await db.runAndReadAll(`SELECT source_id FROM read_parquet('${escapeSQLString(localPath)}')`);
-    const sids = result.getRowObjects().map((r) => r.source_id);
-    return {
-        split,
-        path: modalPath,
-        format: "parquet",
-        compression: "SNAPPY",
-        rows: sids.length,
-        bytes: (await statPath(localPath)).size,
-        sha256: sha256Hex(await readLocalBuffer(localPath)),
-        first_source_id: sids[0],
-        last_source_id: sids.at(-1),
-        source,
-    };
-}
-/**
- * Resolve a base manifest's slice path to the mounted corpus tree used by Modal.
- */
-export function rerootBaseSlicePath(path, baseManifestPath) {
-    const versionedIndex = path.indexOf("/corpus/versioned/");
-    if (versionedIndex !== -1)
-        return "/data" + path.slice(versionedIndex);
-    if (/^\/data\/(?:train|val|test)\//u.test(path)) {
-        const localBaseDir = dirname(baseManifestPath);
-        const baseModalRoot = `/data/corpus/versioned/${basename(dirname(localBaseDir))}/${basename(localBaseDir)}`;
-        return baseModalRoot + path.slice("/data".length);
-    }
-    return path;
-}
-export async function assembleOverlayManifest(args) {
-    const base = await readLocalJSONFile(args.base);
-    if (base.slices.some((s) => s.source === args.source)) {
-        console.log(`WARN: base already contains source '${args.source}' — is this the right base?`);
-    }
-    const kept = base.slices.map((s) => ({ ...s, path: rerootBaseSlicePath(s.path, args.base) }));
-    const newTrain = await descriptor(join(args.newDir, "train", args.sliceParquet), `${args.modalRoot}/train/${args.sliceParquet}`, "train", args.source);
-    const manifest = {
-        corpus_version: args.version,
-        overlay_base: base.corpus_version ?? null,
-        note: args.note || `${base.corpus_version} slices (all kept verbatim) + the ${args.source} slice. Pure overlay add.`,
-        schema: base.schema,
-        rows_per_slice: base.rows_per_slice,
-        row_group_size: base.row_group_size,
-        slices: [...kept, newTrain],
-        counts: {
-            train: base.counts.train + newTrain.rows,
-            val: base.counts.val,
-            test: base.counts.test,
-        },
-        total_rows: base.total_rows + newTrain.rows,
-    };
-    const out = join(args.newDir, "MANIFEST.json");
-    await writeLocalJSONFile(manifest, out);
-    console.log(`wrote ${out}`);
-    console.log(`  slices: ${manifest.slices.length} (${kept.length} base kept, +1 ${args.source})`);
-    console.log(`  counts: ${JSON.stringify(manifest.counts)}  total: ${manifest.total_rows}`);
-    console.log(`  ${args.source} train: ${newTrain.rows} rows (${newTrain.bytes} bytes)`);
-}
-//# sourceMappingURL=overlay-manifest.js.map

package/out/tools/overlay-manifest.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"overlay-manifest.js","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEjD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAyB/D,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,SAAiB,EACjB,KAAa,EACb,MAAc;IAEd,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,uCAAuC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC5G,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAA;IAErE,OAAO;QACN,KAAK;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,QAAQ;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACnD,eAAe,EAAE,IAAI,CAAC,CAAC,CAAE;QACzB,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QAC5B,MAAM;KACN,CAAA;AACF,CAAC;AAYD;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,gBAAwB;IACzE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAEzD,IAAI,cAAc,KAAK,CAAC,CAAC;QAAE,OAAO,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAEtE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,0BAA0B,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAA;QAE3G,OAAO,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAA4B;IACzE,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAe,IAAI,CAAC,IAAI,CAAC,CAAA;IAE7D,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,uCAAuC,IAAI,CAAC,MAAM,6BAA6B,CAAC,CAAA;IAC7F,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7F,MAAM,QAAQ,GAAG,MAAM,UAAU,CAChC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,EAC7C,GAAG,IAAI,CAAC,SAAS,UAAU,IAAI,CAAC,YAAY,EAAE,EAC9C,OAAO,EACP,IAAI,CAAC,MAAM,CACX,CAAA;IAED,MAAM,QAAQ,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,OAAO;QAC5B,YAAY,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACzC,IAAI,EACH,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,qCAAqC,IAAI,CAAC,MAAM,2BAA2B;QAC/G,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC;QAC3B,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI;YACxC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,IAAI;KAC3C,CAAA;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAC9C,MAAM,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAEvC,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,CAAA;IAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,kBAAkB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IAChG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAC1F,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,WAAW,QAAQ,CAAC,IAAI,UAAU,QAAQ,CAAC,KAAK,SAAS,CAAC,CAAA;AACvF,CAAC"}

package/out/tools/slice-kryptonite.d.ts

@@ -1,32 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Build a parquet slice from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
- *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every slice from
- *   v0.3.0 plus the new kryptonite slice(s). No v0.3.0 bytes are touched or re-shuffled.
- *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
- *   DeepSeek model version + prompt versions used to produce the JSONL.
- *
- *   Invoke via `mailwoman corpus slice kryptonite \
- *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
- *   --out-dir /data/corpus/versioned/v0.4.0`
- */
-export interface SliceKryptoniteOptions {
-    jsonl: string;
-    baseManifest: string;
-    outDir: string;
-    /**
-     * Default `"0.4.0"`.
-     */
-    corpusVersion?: string;
-    /**
-     * Default `"deepseek-kryptonite"`.
-     */
-    source?: string;
-}
-export declare function buildKryptoniteSlice(options: SliceKryptoniteOptions, report?: (line: string) => void): Promise<void>;
-//# sourceMappingURL=slice-kryptonite.d.ts.map

package/out/tools/slice-kryptonite.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"slice-kryptonite.d.ts","sourceRoot":"","sources":["../../lib/tools/slice-kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAWH,MAAM,WAAW,sBAAsB;IACtC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAiCD,wBAAsB,oBAAoB,CACzC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CA6Df"}

package/out/tools/slice-kryptonite.js

@@ -1,99 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Build a parquet slice from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
- *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every slice from
- *   v0.3.0 plus the new kryptonite slice(s). No v0.3.0 bytes are touched or re-shuffled.
- *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
- *   DeepSeek model version + prompt versions used to produce the JSONL.
- *
- *   Invoke via `mailwoman corpus slice kryptonite \
- *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
- *   --out-dir /data/corpus/versioned/v0.4.0`
- */
-import { pathExists, readLocalJSONFile } from "@mailwoman/core/fs/readers";
-import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers";
-import { join } from "path-ts";
-import { JSONSpliterator } from "spliterator";
-import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, SLICE_COMPRESSION, writeSlices } from "#utils";
-async function* canonicalRows(jsonl, corpusVersion) {
-    for await (const raw of JSONSpliterator.fromAsync(jsonl)) {
-        // Strip sidecar underscore-prefixed fields the generator left behind for debugging.
-        const components = raw["components"];
-        yield {
-            raw: raw["raw"],
-            components,
-            country: raw["country"] ?? "US",
-            locale: raw["locale"] ?? undefined,
-            source: raw["source"] ?? "deepseek-kryptonite",
-            source_id: raw["source_id"],
-            corpus_version: corpusVersion,
-            license: raw["license"] ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-            synth: raw["synth"],
-        };
-    }
-}
-async function* labeledRows(jsonl, corpusVersion, quarantineLog) {
-    for await (const row of canonicalRows(jsonl, corpusVersion)) {
-        const result = alignRow(row);
-        if (result.kind === "labeled") {
-            yield result.row;
-        }
-        else {
-            quarantineLog.push(`${row.source_id}\t${result.row.reason}`);
-        }
-    }
-}
-export async function buildKryptoniteSlice(options, report) {
-    const corpusVersion = options.corpusVersion ?? "0.4.0";
-    const source = options.source ?? "deepseek-kryptonite";
-    if (!(await pathExists(options.jsonl)))
-        throw new Error(`jsonl not found: ${options.jsonl}`);
-    if (!(await pathExists(options.baseManifest)))
-        throw new Error(`base-manifest not found: ${options.baseManifest}`);
-    await makeDirectories(options.outDir);
-    const quarantine = [];
-    const newManifest = await writeSlices({ train: labeledRows(options.jsonl, corpusVersion, quarantine) }, { outputDir: options.outDir, corpusVersion });
-    report?.(`wrote ${newManifest.total_rows} rows into ${newManifest.slices.length} slice(s); ` +
-        `quarantined ${quarantine.length}`);
-    if (quarantine.length) {
-        const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv");
-        await writeLocalTextFile(quarantine.join("\n") + "\n", qPath);
-        report?.(`quarantine log → ${qPath}`);
-    }
-    // Stamp the new slice's source field for audit.ts (which prefers slice.source over
-    // first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
-    // to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
-    for (const sh of newManifest.slices) {
-        sh.source = source;
-    }
-    // Compose the final corpus-v0.4.0 manifest: every slice from base + the new slice(s).
-    const base = await readLocalJSONFile(options.baseManifest);
-    const combined = {
-        corpus_version: corpusVersion,
-        schema: PARQUET_COLUMNS,
-        rows_per_slice: base.rows_per_slice,
-        row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
-        slices: [...base.slices, ...newManifest.slices],
-        counts: {
-            train: base.counts.train + (newManifest.counts.train ?? 0),
-            val: base.counts.val,
-            test: base.counts.test,
-        },
-        total_rows: base.total_rows + newManifest.total_rows,
-    };
-    // Stamp source on the legacy v0.3.0 slices too, so audit's slice.source path is the
-    // authoritative one. v0.3.0 slices mix sources; we use the first_source_id-prefix
-    // inference for them (audit.ts will re-derive on its own when slice.source is absent).
-    const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json");
-    await writeLocalJSONFile(combined, combinedPath);
-    report?.(`wrote combined manifest → ${combinedPath}`);
-    report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`);
-    report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newManifest.slices.length})`);
-    report?.(`  compression=${SLICE_COMPRESSION}`);
-}
-//# sourceMappingURL=slice-kryptonite.js.map

package/out/tools/slice-kryptonite.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"slice-kryptonite.js","sourceRoot":"","sources":["../../lib/tools/slice-kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACpG,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAG7C,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAiBlG,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,KAAa,EAAE,aAAqB;IACjE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,KAAK,CAAC,EAAE,CAAC;QACnF,oFAAoF;QACpF,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAA2B,CAAA;QAE9D,MAAM;YACL,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;YACzB,UAAU;YACV,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;YAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;YAC9C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,qBAAqB;YAC1D,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;YACrC,cAAc,EAAE,aAAa;YAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;YACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;SAC5C,CAAA;IACF,CAAC;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa,EAAE,aAAqB,EAAE,aAAuB;IACxF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,CAAC,GAAG,CAAA;QACjB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAA;IAEtD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAE5F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAElH,MAAM,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAErC,MAAM,UAAU,GAAa,EAAE,CAAA;IAE/B,MAAM,WAAW,GAAG,MAAM,WAAW,CACpC,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,CAC5C,CAAA;IAED,MAAM,EAAE,CACP,SAAS,WAAW,CAAC,UAAU,cAAc,WAAW,CAAC,MAAM,CAAC,MAAM,aAAa;QAClF,eAAe,UAAU,CAAC,MAAM,EAAE,CACnC,CAAA;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,2BAA2B,CAAC,CAAA;QAC3F,MAAM,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAA;QAC7D,MAAM,EAAE,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAA;IACtC,CAAC;IAED,mFAAmF;IACnF,sFAAsF;IACtF,yFAAyF;IACzF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACrC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAA;IACnB,CAAC;IAED,sFAAsF;IACtF,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAgB,OAAO,CAAC,YAAY,CAAC,CAAA;IAEzE,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;YAC1D,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU;KACpD,CAAA;IAED,oFAAoF;IACpF,kFAAkF;IAClF,uFAAuF;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,eAAe,CAAC,CAAA;IACtF,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAChD,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,WAAW,CAAC,UAAU,GAAG,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;AAC/C,CAAC"}