@hviana/sema 0.5.9 → 0.6.0

package/dist/example/train_base/corpora/soda.d.ts

@@ -0,0 +1,12 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+import type { TaskmasterTurn } from "./taskmaster.js";
+/** Normalize a SODA row into its turns, or null when it carries no usable
+ *  dialogue. Speakers are optional (they only drive merging); an implausibly
+ *  long turn rejects the dialogue as corrupt. */
+export declare function toSodaTurns(row: unknown, maxChars?: number): TaskmasterTurn[] | null;
+/** Translate ONE SODA dialogue into SEMA items: the cumulative walk over its
+ *  speaker-merged turns. Shares `mergeSpeakerTurns` because the rule is the
+ *  same one — consecutive turns by one speaker are one contribution. */
+export declare function sodaDialogueToItems(turns: TaskmasterTurn[]): TrainingItem[];
+export declare const soda: Corpus;

package/dist/example/train_base/corpora/soda.js

@@ -0,0 +1,113 @@
+// train_base/corpora/soda.ts — allenai/soda social dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { accumulate, mergeSpeakerTurns, refineItems, } from "../items.js";
+import { parquet } from "../readers.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// ── allenai/soda (social dialogue) and AmazonScience/massive (short intents) ──
+// Both are read from Hugging Face's auto-converted `refs/convert/parquet`
+// branch. For SODA that is mandatory, not cosmetic: its main-branch
+// train.parquet is ONE 1,191,582-row group (1.19 GB uncompressed), and a
+// Parquet column chunk is per-group, so any read of it materialises the whole
+// file — measured at 100% of a 689 MB file and 2 GB of heap for a 500-row read.
+// The converted branch uses uniform 10,000-row groups.
+//
+// BOTH STAGES ARE BUDGETED, and that is a curriculum decision rather than an
+// algorithmic cap. SODA's train split holds 1,191,582 dialogues which the
+// cumulative walk would turn into ~8 MILLION episodes — against the 662,221
+// deposits of the entire current corpus. Trained whole it would not join the
+// mix, it would BE the mix, and corpus size is the quantity every scale problem
+// in this engine is measured against. The default takes the first
+// SODA_MAX_DIALOGS of them; set it to 0 to lift the budget.
+const SODA = env("SODA", "1") !== "0";
+const SODA_DATASET = env("SODA_DATASET", "allenai/soda");
+const SODA_SPLITS = env("SODA_SPLITS", "train")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// ~6.3 episodes per dialogue, so this budgets ~750k episodes — comparable to
+// the Taskmaster stage and to Aya, which is the intended balance. 0 = no budget.
+const SODA_MAX_DIALOGS = Math.max(0, Math.floor(Number(env("SODA_MAX_DIALOGS", "120000"))) || 0);
+const MAX_SODA_TURN_CHARS = Math.max(1_000, Math.floor(Number(env("MAX_SODA_TURN_KB", "32")) * 1000) || 32_000);
+// ═══════════════════════════════════════════════════════════════════════
+// §6e‴  SODA parsing — a social dialogue row → SEMA items
+//
+// Each row carries `dialogue` (an array of turn strings) and `speakers` (the
+// speaker name per turn). The deposit is the cumulative walk over speaker-merged
+// turns, identical in shape to Taskmaster and oasst2 — turns are short (mean
+// 87 B) and dialogues average 7.3 turns, so the accumulated context stays well
+// inside the healthy range.
+//
+// `narrative`, `literal` and the ATOMIC-style `head`/`relation`/`tail` columns
+// are NOT deposited: they are the generation scaffolding SODA was distilled
+// from, they restate the dialogue in the third person, and depositing both a
+// dialogue and its paraphrased summary gives one meaning two shapes — which is
+// measured to SUPPRESS composition rather than help it.
+// ═══════════════════════════════════════════════════════════════════════
+/** Normalize a SODA row into its turns, or null when it carries no usable
+ *  dialogue. Speakers are optional (they only drive merging); an implausibly
+ *  long turn rejects the dialogue as corrupt. */
+export function toSodaTurns(row, maxChars = MAX_SODA_TURN_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    const dialogue = r.dialogue;
+    if (!Array.isArray(dialogue))
+        return null;
+    const speakers = Array.isArray(r.speakers) ? r.speakers : [];
+    const turns = [];
+    for (let i = 0; i < dialogue.length; i++) {
+        const text = typeof dialogue[i] === "string"
+            ? dialogue[i].trim()
+            : "";
+        if (!text)
+            continue;
+        if (text.length > maxChars)
+            return null;
+        turns.push({
+            speaker: String(speakers[i] ?? "").trim().toUpperCase(),
+            text,
+        });
+    }
+    return turns.length ? turns : null;
+}
+/** Translate ONE SODA dialogue into SEMA items: the cumulative walk over its
+ *  speaker-merged turns. Shares `mergeSpeakerTurns` because the rule is the
+ *  same one — consecutive turns by one speaker are one contribution. */
+export function sodaDialogueToItems(turns) {
+    const texts = mergeSpeakerTurns(turns);
+    if (texts.length < 2)
+        return []; // not an exchange
+    return refineItems(accumulate(texts));
+}
+export const soda = {
+    id: "soda",
+    label: "SODA",
+    kind: "social dialogue",
+    enabled: SODA,
+    maxRows: SODA_MAX_DIALOGS,
+    // Two of sixteen columns. The `narrative`/`literal`/`head`/`relation`/`tail`
+    // scaffolding the note above declines to deposit is now also never decoded:
+    // measured on the converted train shard, 449 MB uncompressed across all
+    // sixteen against 306 MB for these two (68.1%).
+    read: parquet({ columns: ["dialogue", "speakers"] }),
+    toItems: (row) => {
+        const turns = toSodaTurns(row);
+        if (!turns)
+            return null;
+        const items = sodaDialogueToItems(turns);
+        return items.length ? items : null;
+    },
+    unitNoun: "shard(s)",
+    log: { rows: "row(s)" },
+    discover: convertedParquetUnits({
+        id: "soda",
+        label: "SODA",
+        dataset: SODA_DATASET,
+        config: "default",
+        splits: SODA_SPLITS,
+        localSub: "soda",
+    }),
+};

package/dist/example/train_base/corpora/taskmaster.d.ts

@@ -0,0 +1,15 @@
+import { type SpeakerTurn, type TrainingItem } from "../items.js";
+import { type Corpus } from "../corpus.js";
+/** One utterance of a Taskmaster conversation — the shared dialogue-turn shape,
+ *  under the name this corpus's adapters have always used. */
+export type TaskmasterTurn = SpeakerTurn;
+/** Normalize ONE element of a Taskmaster data file into its turns, or null when
+ *  it carries no usable utterance. Empty/whitespace-only utterances are dropped
+ *  (TM-3 has a few); a single implausibly long utterance rejects the whole
+ *  conversation as corrupt rather than depositing a dump. */
+export declare function toTaskmasterTurns(row: unknown, maxChars?: number): TaskmasterTurn[] | null;
+/** Translate ONE Taskmaster conversation into SEMA training items: the
+ *  cumulative walk over its merged turns. Returns [] for a conversation below
+ *  TASKMASTER_MIN_TURNS, so callers can simply skip empties. */
+export declare function taskmasterConversationToItems(turns: TaskmasterTurn[], minTurns?: number): TrainingItem[];
+export declare const taskmaster: Corpus;

package/dist/example/train_base/corpora/taskmaster.js

@@ -0,0 +1,144 @@
+// train_base/corpora/taskmaster.ts — google-research-datasets/Taskmaster 1–4 dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env, LOCAL_PATH } from "../config.js";
+import { accumulate, mergeSpeakerTurns, refineItems, } from "../items.js";
+import { githubContents, localFiles } from "../discovery.js";
+import { jsonArray } from "../readers.js";
+import { localDir } from "../corpus.js";
+import { DIM, R } from "../ui.js";
+import { basename, join } from "node:path";
+// ── google-research-datasets/Taskmaster 1–4 (the dialogue stages) ──
+// Four corpora of task-oriented dialogue, one shape between them: each file is a
+// JSON ARRAY of conversations and each conversation carries
+// `utterances: [{speaker, text, …}]`. TM-1 ships two files directly under its
+// directory (self-dialogs, woz-dialogs); TM-2/3/4 ship theirs under `<set>/data`.
+// They are the best-scoring corpora on the fold-unit recurrence benchmark that
+// selects for halo health (TM-3 85.1%, TM-4 78.8%, TM-2 68.7%, TM-1 51.8%,
+// against 23.2% for the incumbent SmolSent), and they are genuinely multi-turn
+// where the incumbent multi-turn stage is not (TM-3 median 20 turns of ~43 B,
+// against oasst2's median turn of 529 B).
+//
+// Served from GitHub raw, not Hugging Face: the HF mirrors are loading-script
+// repos with no data files, and the official copies carry the CC BY 4.0 notice.
+const TASKMASTER = env("TASKMASTER", "1") !== "0";
+// Which sets to train, in order. Each is a directory in the Taskmaster repo.
+const TASKMASTER_SETS = env("TASKMASTER_SETS", "TM-1-2019,TM-2-2020,TM-3-2020,TM-4-2024").split(",").map((s) => s.trim()).filter(Boolean);
+const TASKMASTER_REPO = env("TASKMASTER_REPO", "google-research-datasets/Taskmaster");
+const TASKMASTER_RAW = `https://raw.githubusercontent.com/${TASKMASTER_REPO}/master`;
+// A conversation must have at least this many turns AFTER same-speaker merging.
+// The default of 2 keeps every real exchange: unlike oasst2 — where a lone Q→A
+// tree merely replicates the Aya stage's shape and is dropped — a two-turn
+// task-oriented exchange is still task-oriented dialogue, and TM-4's dialogues
+// are short by design (median 3.7 turns), so a higher bar would discard most of
+// that set.
+const TASKMASTER_MIN_TURNS = Math.max(2, Math.floor(Number(env("TASKMASTER_MIN_TURNS", "2"))) || 2);
+// Skip a conversation carrying an implausibly long utterance (corruption). The
+// measured maximum across TM-1/2/3/4 is 1,897 bytes, so this only guards.
+const MAX_TASKMASTER_TURN_CHARS = Math.max(1_000, Math.floor(Number(env("MAX_TASKMASTER_TURN_KB", "32")) * 1000) || 32_000);
+/** Normalize ONE element of a Taskmaster data file into its turns, or null when
+ *  it carries no usable utterance. Empty/whitespace-only utterances are dropped
+ *  (TM-3 has a few); a single implausibly long utterance rejects the whole
+ *  conversation as corrupt rather than depositing a dump. */
+export function toTaskmasterTurns(row, maxChars = MAX_TASKMASTER_TURN_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const utterances = row.utterances;
+    if (!Array.isArray(utterances))
+        return null;
+    const turns = [];
+    for (const u of utterances) {
+        if (!u || typeof u !== "object")
+            continue;
+        const r = u;
+        const text = typeof r.text === "string" ? r.text.trim() : "";
+        if (!text)
+            continue;
+        if (text.length > maxChars)
+            return null;
+        turns.push({
+            speaker: String(r.speaker ?? "").trim().toUpperCase(),
+            text,
+        });
+    }
+    return turns.length ? turns : null;
+}
+/** Translate ONE Taskmaster conversation into SEMA training items: the
+ *  cumulative walk over its merged turns. Returns [] for a conversation below
+ *  TASKMASTER_MIN_TURNS, so callers can simply skip empties. */
+export function taskmasterConversationToItems(turns, minTurns = TASKMASTER_MIN_TURNS) {
+    const texts = mergeSpeakerTurns(turns);
+    if (texts.length < minTurns)
+        return [];
+    return refineItems(accumulate(texts));
+}
+/** List the Taskmaster data files to train, in TASKMASTER_SETS order. Returns
+ *  repo-relative paths, e.g. "TM-3-2020/data/data_00.json".
+ *
+ *  TM-2/3/4 keep their dialogue files under `<set>/data`, so everything there is
+ *  fair game. TM-1 has no `data` directory: its two dialogue files sit at the
+ *  set root NEXT TO `ontology.json` (a slot schema) and `sample.json` (a small
+ *  excerpt of self-dialogs). Neither is an array of conversations, and training
+ *  the excerpt would deposit a subset of TM-1 twice, so TM-1 is filtered to the
+ *  `*-dialogs.json` pair (self-dialogs, woz-dialogs). */
+async function listFiles(ctx) {
+    const out = [];
+    for (const set of TASKMASTER_SETS) {
+        const rootOnly = /^TM-1\b/i.test(set);
+        const dir = rootOnly ? set : `${set}/data`;
+        const names = await githubContents(TASKMASTER_REPO, dir, /\.json$/i, `GET Taskmaster ${dir}`, ctx.http);
+        for (const { path: name, size } of names) {
+            if (rootOnly && !/-dialogs\.json$/i.test(name))
+                continue;
+            out.push({ set, path: `${dir}/${name}`, size });
+        }
+    }
+    return out;
+}
+const unit = (key, name) => ({
+    key,
+    name,
+    display: `Taskmaster ${name}`,
+});
+export const taskmaster = {
+    id: "taskmaster",
+    label: "Taskmaster",
+    kind: "task dialogue",
+    enabled: TASKMASTER,
+    unitNoun: "dialogue file(s)",
+    read: jsonArray(),
+    toItems: (row) => {
+        const turns = toTaskmasterTurns(row);
+        if (!turns)
+            return null;
+        const items = taskmasterConversationToItems(turns); // [] when too short
+        return items.length ? items : null;
+    },
+    log: { bad: "unusable conversation(s)" },
+    async discover(ctx) {
+        // LOCAL_PATH/taskmaster/ — a subdirectory, because these share the .json
+        // extension with the General-Knowledge source and must not be confused
+        // with it.
+        if (LOCAL_PATH) {
+            const dir = localDir("taskmaster");
+            const names = localFiles(dir, /\.json$/i);
+            if (names.length === 0) {
+                ctx.progress.log(`  ${DIM}· no Taskmaster *.json in ${dir} — skipping${R}`);
+                return null;
+            }
+            return names.map((n) => ({
+                ...unit(n.path, n.path),
+                local: join(dir, n.path),
+                bytes: n.size,
+            }));
+        }
+        return (await listFiles(ctx)).map((f) => ({
+            ...unit(f.path, `${f.set}/${basename(f.path)}`),
+            url: `${TASKMASTER_RAW}/${f.path}`,
+            bytes: f.size,
+        }));
+    },
+};

package/dist/example/train_base/corpora/wiki2.d.ts

@@ -0,0 +1,23 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+/** One (subject, relation, object) triple from a 2Wiki `evidences` cell. */
+export interface WikiTriple {
+    subject: string;
+    relation: string;
+    object: string;
+}
+/** Normalize a 2Wiki row into its evidence triples, or null when it carries
+ *  none usable. `evidences` is a JSON STRING holding an array of 3-element
+ *  arrays; a row whose cell is absent, unparseable, or empty yields null.
+ *  Individual malformed or oversized triples are dropped without discarding the
+ *  row — one bad triple should not cost the others. */
+export declare function toWikiTriples(row: unknown, maxChars?: number): WikiTriple[] | null;
+/** Render ONE triple as the prose fact Sema stores. Kept separate so the two
+ *  deposits below are guaranteed to share a byte-identical continuation: the
+ *  pivot fact only works if it leads to the SAME node the relation fact does. */
+export declare function wikiTripleSentence(t: WikiTriple): string;
+/** Translate a row's triples into SEMA items: per triple, the relation fact and
+ *  the bare-subject PIVOT fact (see the section note above). refineItems drops
+ *  the duplicates this produces when a row states the same triple twice. */
+export declare function wikiTriplesToItems(triples: WikiTriple[]): TrainingItem[];
+export declare const wiki2: Corpus;

package/dist/example/train_base/corpora/wiki2.js

@@ -0,0 +1,132 @@
+// train_base/corpora/wiki2.ts — 2WikiMultihopQA evidence triples
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { refineItems } from "../items.js";
+import { parquet } from "../readers.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// ── 2WikiMultihopQA — the `evidences` TRIPLES only (the composition stage) ──
+// Each row carries `evidences`: a JSON string of (subject, relation, object)
+// triples that CHAIN — one triple's object is the next's subject. 72.5% of rows
+// carry such a chain (measured over 4,000 rows), and those triples are the only
+// representation measured to make Sema compose a two-hop answer at all.
+//
+// TWO COLUMNS ARE DELIBERATELY NOT READ, one for licence reasons and one for
+// capability reasons:
+//   • `context` holds Wikipedia PROSE. The repo is Apache-2.0 but Wikipedia text
+//     is CC BY-SA, and a Sema store keeps text verbatim, so ingesting the
+//     passages would attach ShareAlike to every distributed store. The triples
+//     originate in Wikidata (CC0). See DATASETS.md §3.2/§4.
+//   • `question`/`answer` are the composed multi-hop QUESTION. Depositing those
+//     teaches the answer to that exact question and nothing else — it memorises
+//     rather than composes. They are used to EVALUATE this adapter, never as
+//     training input.
+//
+// Read from Hugging Face's auto-converted `refs/convert/parquet` branch, not
+// from main: the main-branch train.parquet is written as ONE 167,454-row
+// group (666 MB uncompressed) and a Parquet column chunk is per-group, so any
+// read of it materialises the whole file. The converted branch uses uniform
+// 10,000-row groups, which `parquetBatchRows` then subdivides by BYTES.
+const WIKI2 = env("WIKI2", "1") !== "0";
+const WIKI2_DATASET = env("WIKI2_DATASET", "xanhho/2WikiMultihopQA");
+// Splits to train, in order. Only `train` by default: `validation`/`test` are
+// the dataset's held-out sets and are what an honest evaluation of this
+// adapter's composition rate has to be measured on.
+const WIKI2_SPLITS = env("WIKI2_SPLITS", "train")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// Reject a triple with an implausibly long field (corruption); real subjects and
+// objects are entity names, and relations are Wikidata property labels.
+// 0 = every row. The train split holds 167,454 rows at ~4.95 deposits each
+// (~830k facts), so this is the knob that keeps 2Wiki proportionate to the rest
+// of the curriculum in the same way SODA_MAX_DIALOGS does.
+const WIKI2_MAX_ROWS = Math.max(0, Math.floor(Number(env("WIKI2_MAX_ROWS", "0"))) || 0);
+const MAX_WIKI2_FIELD_CHARS = Math.max(100, Math.floor(Number(env("MAX_WIKI2_FIELD_KB", "2")) * 1000) || 2_000);
+/** Normalize a 2Wiki row into its evidence triples, or null when it carries
+ *  none usable. `evidences` is a JSON STRING holding an array of 3-element
+ *  arrays; a row whose cell is absent, unparseable, or empty yields null.
+ *  Individual malformed or oversized triples are dropped without discarding the
+ *  row — one bad triple should not cost the others. */
+export function toWikiTriples(row, maxChars = MAX_WIKI2_FIELD_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const cell = row.evidences;
+    let parsed = cell;
+    if (typeof cell === "string") {
+        try {
+            parsed = JSON.parse(cell);
+        }
+        catch {
+            return null;
+        }
+    }
+    if (!Array.isArray(parsed))
+        return null;
+    const out = [];
+    for (const e of parsed) {
+        if (!Array.isArray(e) || e.length < 3)
+            continue;
+        const subject = typeof e[0] === "string" ? e[0].trim() : "";
+        const relation = typeof e[1] === "string" ? e[1].trim() : "";
+        const object = typeof e[2] === "string" ? e[2].trim() : "";
+        if (!subject || !relation || !object)
+            continue;
+        if (subject.length > maxChars ||
+            relation.length > maxChars ||
+            object.length > maxChars)
+            continue;
+        out.push({ subject, relation, object });
+    }
+    return out.length ? out : null;
+}
+/** Render ONE triple as the prose fact Sema stores. Kept separate so the two
+ *  deposits below are guaranteed to share a byte-identical continuation: the
+ *  pivot fact only works if it leads to the SAME node the relation fact does. */
+export function wikiTripleSentence(t) {
+    return `The ${t.relation} of ${t.subject} is ${t.object}.`;
+}
+/** Translate a row's triples into SEMA items: per triple, the relation fact and
+ *  the bare-subject PIVOT fact (see the section note above). refineItems drops
+ *  the duplicates this produces when a row states the same triple twice. */
+export function wikiTriplesToItems(triples) {
+    const items = [];
+    for (const t of triples) {
+        const fact = wikiTripleSentence(t);
+        items.push({ context: `${t.subject} ${t.relation}`, continuation: fact });
+        items.push({ context: t.subject, continuation: fact });
+    }
+    return refineItems(items);
+}
+export const wiki2 = {
+    id: "2wiki",
+    label: "2Wiki",
+    kind: "relation triples",
+    enabled: WIKI2,
+    maxRows: WIKI2_MAX_ROWS,
+    // Read ONE of the file's seven columns. This is where the "context is not
+    // read" rule above stops being a property of the adapter and becomes a
+    // property of the read: the Wikipedia prose is never decoded at all. It is
+    // also almost the whole file — measured on the converted train shard, the
+    // seven columns hold 518 MB uncompressed and `evidences` is 18 MB of it
+    // (3.5%), so the projection cuts the decode work by ~28x.
+    read: parquet({ columns: ["evidences"] }),
+    toItems: (row) => {
+        const triples = toWikiTriples(row);
+        if (!triples)
+            return null;
+        const items = wikiTriplesToItems(triples);
+        return items.length ? items : null;
+    },
+    unitNoun: "shard(s)",
+    log: { rows: "row(s)" },
+    discover: convertedParquetUnits({
+        id: "2wiki",
+        label: "2Wiki",
+        dataset: WIKI2_DATASET,
+        config: "default",
+        splits: WIKI2_SPLITS,
+        localSub: "2wiki",
+    }),
+};

package/dist/example/train_base/corpus.d.ts

@@ -0,0 +1,88 @@
+import type { Reader, RowAdapter } from "./readers.js";
+import type { TrainCtx } from "./runtime.js";
+/** One file to train: a shard, a per-language file, or a whole single-file
+ *  corpus. Exactly one of `url` / `local` is set. */
+export interface Unit {
+    /** Resume-id suffix — the corpus id and this form `${id}::${key}`. Part of
+     *  the store's compatibility surface; see the file header. */
+    key: string;
+    /** How the run log names this unit once it is read. */
+    name: string;
+    /** How the live panel names it, and (unless `acquireLabel` overrides) how the
+     *  download is labelled. Conventionally `${corpus.label} ${name}`. */
+    display: string;
+    url?: string;
+    local?: string;
+    /** Size in bytes when the listing said, else 0/absent. Summed BEFORE the
+     *  stage reads anything, so the corpus progress bar has a denominator that
+     *  does not grow underneath it. */
+    bytes?: number;
+    /** Cache filename. Defaults to the resume id with unsafe characters folded. */
+    dest?: string;
+    /** Download label, when it differs from `display`. */
+    acquireLabel?: string;
+}
+/** How one unit's outcome reads in the run log. All optional: the defaults are
+ *  what every fact-shaped corpus prints. */
+export interface LogStyle {
+    /** What one deposit is called. Default "facts". */
+    deposits?: string;
+    /** When set, the line reports "from N <rows>" — the count of rows that
+     *  actually produced deposits. */
+    rows?: string;
+    /** What an unusable record is called. Default "unusable row(s)". */
+    bad?: string;
+    /** Report only the reader's `skipped` (malformed records), not the rows the
+     *  adapter declined. For a corpus that DECLINES records by design — oasst2
+     *  drops every single-turn tree — counting those as damage would be a lie. */
+    malformedOnly?: boolean;
+}
+export interface Corpus {
+    /** Tally key AND resume-id prefix. Compatibility surface — see the header. */
+    id: string;
+    /** Human name: the panel, the skip notices, the listing-failure message. */
+    label: string;
+    /** The dim tag in the log line, e.g. "translation", "social dialogue". */
+    kind: string;
+    enabled: boolean;
+    /** The work-list. Return [] for "nothing found" (the runner says so), or
+     *  null when the corpus has already logged a more specific reason. */
+    discover(ctx: TrainCtx): Promise<Unit[] | null>;
+    read: Reader;
+    toItems: RowAdapter;
+    /** Stage-wide row budget; 0/absent = unbounded. See the budget notes in
+     *  stage.ts. */
+    maxRows?: number;
+    /** Noun for the "N/M ___ to train" announcement. Absent ⇒ no announcement,
+     *  which is what a single-unit corpus has always done. */
+    unitNoun?: string;
+    /** Keep a file that came from the CACHE after a complete read. Only oasst2
+     *  does this: every other corpus deletes whatever acquire() handed it. */
+    keepCached?: boolean;
+    log?: LogStyle;
+}
+/** The string a store records once this unit is finished. ONE rule, no
+ *  exceptions — see the file header for why there used to be two. */
+export declare const unitIdOf: (corpus: Corpus, unit: Unit) => string;
+/** Local files live under `LOCAL_PATH/<sub>`, or directly in LOCAL_PATH when
+ *  `sub` is empty. Kept here because the layout is a user-facing convention:
+ *  the corpora that share an extension (.json, .parquet) are kept apart by a
+ *  subdirectory so a local run cannot feed one corpus's files to another. */
+export declare const localDir: (sub: string) => string;
+/** A single-unit corpus: one fixed URL, or one local file matched by pattern.
+ *  Factored out because the three corpora that are ONE file resolve it the same
+ *  way. `key` is what the store records this corpus under — `aya::dataset` is
+ *  `key: "dataset"` — so it is required rather than defaulted: a resume id is
+ *  the one thing here that must never be guessed. */
+export declare function singleUnit(opts: {
+    key: string;
+    label: string;
+    display: string;
+    url: string;
+    dest: string;
+    acquireLabel?: string;
+    /** Patterns tried, in order, against LOCAL_PATH. */
+    localMatch: RegExp[];
+    /** How the "no local copy" notice describes what it looked for. */
+    localWhat: string;
+}): (ctx: TrainCtx) => Promise<Unit[] | null>;

package/dist/example/train_base/corpus.js

@@ -0,0 +1,65 @@
+// train_base/corpus.ts — WHAT A CORPUS IS: the contract every corpus file
+// declares, and the resume identity derived from it.
+//
+// Separated from the loop that consumes it (stage.ts) so a corpus file never
+// imports the loop. A corpus declares three things the loop does not know —
+// where its work-list comes from, what container its bytes are in, and how a row
+// becomes deposits — and everything here exists to state those three.
+//
+// THE RESUME IDS ARE A COMPATIBILITY SURFACE. A store records the units it has
+// finished as strings, and a store trained by an earlier version must keep
+// resuming, so `unitIdOf` below has to reproduce the ids that store already
+// holds. It does, by one rule with no exceptions: `${corpus.id}::${unit.key}`.
+//
+// That single rule is a recent simplification, and the evidence for it is worth
+// keeping. There used to be a second mechanism — a `Corpus.unitId` field
+// carrying a FIXED id for the three single-unit corpora, which were believed
+// irregular — and with it a trap: those corpora emitted `key: ""` on the
+// assumption that the fixed id would always override, so a new single-unit
+// corpus that forgot the field silently got the id "foo::". Reading the ids out
+// of a real 2.5 GB store settled it: the three are `aya::dataset`,
+// `oasst2::trees` and `genknow::qa`, which decompose EXACTLY into corpus id and
+// key. They were never irregular, so the escape hatch and its trap are gone and
+// one rule covers all 239 recorded ids.
+import { LOCAL_PATH } from "./config.js";
+import { localFind } from "./discovery.js";
+import { DIM, R } from "./ui.js";
+import { join } from "node:path";
+/** The string a store records once this unit is finished. ONE rule, no
+ *  exceptions — see the file header for why there used to be two. */
+export const unitIdOf = (corpus, unit) => `${corpus.id}::${unit.key}`;
+/** Local files live under `LOCAL_PATH/<sub>`, or directly in LOCAL_PATH when
+ *  `sub` is empty. Kept here because the layout is a user-facing convention:
+ *  the corpora that share an extension (.json, .parquet) are kept apart by a
+ *  subdirectory so a local run cannot feed one corpus's files to another. */
+export const localDir = (sub) => sub ? join(LOCAL_PATH, sub) : LOCAL_PATH;
+/** A single-unit corpus: one fixed URL, or one local file matched by pattern.
+ *  Factored out because the three corpora that are ONE file resolve it the same
+ *  way. `key` is what the store records this corpus under — `aya::dataset` is
+ *  `key: "dataset"` — so it is required rather than defaulted: a resume id is
+ *  the one thing here that must never be guessed. */
+export function singleUnit(opts) {
+    return async (ctx) => {
+        if (LOCAL_PATH) {
+            const hit = localFind(LOCAL_PATH, ...opts.localMatch);
+            if (!hit) {
+                ctx.progress.log(`  ${DIM}· no ${opts.localWhat} in ${LOCAL_PATH} — skipping${R}`);
+                return null;
+            }
+            return [{
+                    key: opts.key,
+                    name: opts.label,
+                    display: opts.display,
+                    local: join(LOCAL_PATH, hit),
+                }];
+        }
+        return [{
+                key: opts.key,
+                name: opts.label,
+                display: opts.display,
+                url: opts.url,
+                dest: opts.dest,
+                acquireLabel: opts.acquireLabel,
+            }];
+    };
+}

package/dist/example/train_base/discovery.d.ts

@@ -0,0 +1,48 @@
+import { type HttpOptions } from "./http.js";
+/** A note a listing needs to make about ITSELF — an accepted-but-unexpected
+ *  split name, say. Distinct from a failure: the run continues, but silently
+ *  continuing would hide why fewer units appeared than expected. */
+export type Note = (msg: string) => void;
+/** One listed file: where it is, and how big it is.
+ *
+ *  The SIZE is carried because the panel is otherwise dishonest. Corpus
+ *  progress used to be measured against a total that GREW as each file was
+ *  opened, so the bar ran to 100% at the end of every file and then fell back
+ *  when the next one was added — 100%, 50%, 100%, 66%… Both listing APIs
+ *  already return the size, so the denominator can simply be known before the
+ *  first byte is read. */
+export interface Listed {
+    path: string;
+    /** Bytes, or 0 when the source did not say. */
+    size: number;
+}
+/** Files under `path` in a Hugging Face dataset repo's main branch, filtered to
+ *  an extension. Returns repo-relative paths (e.g. "smolsent/ha_en.jsonl"),
+ *  sorted, so a run's unit order is stable across machines. */
+export declare function hfTree(dataset: string, path: string, ext: RegExp, label: string, opts: HttpOptions): Promise<Listed[]>;
+/** A dataset's Parquet shards on Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch, restricted to `config` and to `splits`.
+ *
+ *  The converted branch is used rather than `main` because a dataset's own
+ *  Parquet may be written as ONE giant row-group (SODA's is 1,191,582 rows),
+ *  and a column chunk is per-group, so reading any part of it materialises all
+ *  of it. The converted branch is uniformly 10,000-row groups.
+ *
+ *  Paths look like "<config>/<split>/0000.parquet". The BRANCH name is a single
+ *  path SEGMENT here, so its "/" is percent-encoded — unlike a dataset id,
+ *  whose "/" must not be. */
+export declare function hfConvertedParquet(dataset: string, config: string, splits: string[], label: string, opts: HttpOptions, note?: Note): Promise<Listed[]>;
+/** File NAMES (not paths) in one directory of a GitHub repo, filtered to an
+ *  extension and sorted. Used for corpora served from GitHub raw rather than
+ *  Hugging Face — where the HF mirrors are loading-script repos with no data
+ *  files, the official GitHub copy is the one carrying the licence notice. */
+export declare function githubContents(repo: string, dir: string, ext: RegExp, label: string, opts: HttpOptions): Promise<Listed[]>;
+/** Every file in a local directory matching `ext`, sorted. A missing directory
+ *  is an empty list, not an error: LOCAL_PATH is an offline convenience and a
+ *  stage with no local copy simply reports that and moves on. */
+export declare function localFiles(dir: string, ext: RegExp): Listed[];
+/** The FIRST file in a local directory matching any of `exts`, in directory
+ *  order (deliberately NOT sorted — this mirrors the single-file stages, which
+ *  take whichever copy the filesystem hands back first). Null when none match
+ *  or the directory is absent. */
+export declare function localFind(dir: string, ...exts: RegExp[]): string | null;