@hviana/sema 0.5.9 → 0.6.0

package/dist/example/train_base/corpora/genknow.js

@@ -0,0 +1,83 @@
+// train_base/corpora/genknow.ts — MuskumPillerum/General-Knowledge Q&A
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { refineItems } from "../items.js";
+import { jsonArray } from "../readers.js";
+import { singleUnit } from "../corpus.js";
+// ── MuskumPillerum/General-Knowledge (the fourth training stage, after oasst2) ──
+// A ~37.6k-row general-knowledge Q&A set: each row is a single {Question, Answer}
+// pair. A row is a pure RELATION (question → answer), so it becomes exactly ONE
+// FACT, identical in shape to the Aya stage. It ships as a single JSON array
+// file (output.json); we DOWNLOAD it and stream the array. GENKNOW_URL overrides
+// the source.
+//
+// DISABLED BY DEFAULT ON LICENCE GROUNDS (2026-08-13). The HF repo carries NO
+// licence tag and no licence in its card — an earlier header in this file
+// claimed MIT without support — and its own dataset card states it "contains a
+// subset of the alpaca dataset". Alpaca is CC BY-NC 4.0: NonCommercial, which
+// conflicts with Sema's commercial licence. Because a Sema store retains its
+// training text VERBATIM, an unlicensed corpus inside it makes the whole
+// artifact undistributable. See DATASETS.md §3.2. GENKNOW=1 re-enables the
+// stage for local, non-distributed experiments only.
+const GENKNOW = env("GENKNOW", "0") !== "0";
+const GENKNOW_URL = env("GENKNOW_URL", "https://huggingface.co/datasets/MuskumPillerum/General-Knowledge/resolve/main/output.json");
+// A Question/Answer longer than this is skipped (answers run to a few hundred
+// chars; this only guards against a corrupt/runaway field).
+const MAX_GENKNOW_CHARS = Math.max(4_000, Math.floor(Number(env("MAX_GENKNOW_KB", "64")) * 1000) || 64_000);
+/** Turn a source value into clean prose: decode the literal "\n"/"\t"/"\r"
+ *  two-character escapes the source JSON left in the text, collapse the runs of
+ *  whitespace that creates, and trim. */
+function unescapePlain(s) {
+    return s
+        .replace(/\\r\\n|\\n|\\r/g, "\n")
+        .replace(/\\t/g, " ")
+        .replace(/[ \t]+/g, " ")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
+ *  a usable question/answer or a side is implausibly large (corruption). */
+export function toGenKnowRow(row, maxChars = MAX_GENKNOW_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    const question = typeof r.Question === "string"
+        ? unescapePlain(r.Question)
+        : "";
+    const answer = typeof r.Answer === "string" ? unescapePlain(r.Answer) : "";
+    if (!question || !answer)
+        return null;
+    if (question.length > maxChars || answer.length > maxChars)
+        return null;
+    return { question, answer };
+}
+/** Translate ONE General-Knowledge row into SEMA items: exactly one
+ *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
+export function genKnowRowToItems(row) {
+    return refineItems([{ context: row.question, continuation: row.answer }]);
+}
+export const genknow = {
+    id: "genknow",
+    label: "General-Knowledge",
+    kind: "Q&A facts",
+    enabled: GENKNOW,
+    read: jsonArray(),
+    toItems: (row) => {
+        const r = toGenKnowRow(row);
+        return r ? genKnowRowToItems(r) : null;
+    },
+    discover: singleUnit({
+        // Resume id "genknow::qa" — the string this store already records.
+        key: "qa",
+        label: "General-Knowledge",
+        display: "General-Knowledge",
+        url: GENKNOW_URL,
+        dest: "general_knowledge.json",
+        localMatch: [/general.*knowledge.*\.json$/i, /output\.json$/i],
+        localWhat: "General-Knowledge *.json",
+    }),
+};

package/dist/example/train_base/corpora/index.d.ts

@@ -0,0 +1,29 @@
+import type { Corpus } from "../corpus.js";
+export declare const CURRICULUM: Corpus[];
+/** The corpora this RUN will train, for the panel header — derived, so it can
+ *  never go stale the way the hand-written string it replaces had. */
+export declare const enabledLabels: () => string;
+/** The corpora a STORE contains: everything this run will train PLUS everything
+ *  an earlier run already did, in curriculum order.
+ *
+ *  Not the same question as `enabledLabels`, and conflating them wrote a false
+ *  statement into every store that was ever resumed with a different set of
+ *  stages enabled. Observed on a real store: `train.dataset` read
+ *  "SmolSent+Aya+oasst2" while the tally recorded 37,623 General-Knowledge
+ *  deposits sitting in it. That is not cosmetic — a Sema store retains its
+ *  training text VERBATIM, so `train.dataset` is the record of whose licence
+ *  terms travel with the artifact, and General-Knowledge is precisely the
+ *  corpus DATASETS.md §3.2 disables on NonCommercial grounds.
+ *
+ *  `trainedIds` are the corpus ids an earlier run deposited under (the keys of
+ *  the per-corpus tally). An id no longer in the curriculum cannot be named and
+ *  is dropped — the tally still carries it, which is where that evidence lives. */
+export declare const storedLabels: (trainedIds: Iterable<string>) => string;
+export * from "./smolsent.js";
+export * from "./aya.js";
+export * from "./oasst2.js";
+export * from "./taskmaster.js";
+export * from "./wiki2.js";
+export * from "./soda.js";
+export * from "./massive.js";
+export * from "./genknow.js";

package/dist/example/train_base/corpora/index.js

@@ -0,0 +1,81 @@
+// train_base/corpora/index.ts — THE CURRICULUM.
+//
+// The order is the curriculum, and it is load-bearing: each stage runs only
+// after the previous one finishes, and every stage records itself in the same
+// completed-set, so a single store resumes the whole sequence.
+//
+//   1. SmolSent    sentence-level TRANSLATION pairs across 100+ low-resource
+//                  languages. Each pair is "two names for one meaning" → a
+//                  foreign→English translation FACT, so every language's
+//                  rendering of a meaning converges on ONE English node (cf.
+//                  test/05-concepts.test.mjs).
+//   2. Aya         ~204k human prompt→completion pairs, 70+ languages → one
+//                  (question → answer) FACT each.
+//   3. oasst2      MULTI-TURN human↔assistant conversation trees → the
+//                  accumulated-context walk (single-turn trees are skipped).
+//   4. Taskmaster  task-oriented DIALOGUE, the best-scoring corpora on the
+//                  fold-unit recurrence benchmark that predicts halo health.
+//   5. 2Wiki       the `evidences` TRIPLES — the one stage aimed at
+//                  COMPOSITION. Its Wikipedia passages and its composed
+//                  questions are deliberately NOT read.
+//   6. SODA        social/commonsense DIALOGUE, budgeted.
+//   7. MASSIVE     short intent utterances → ONE bare experience each.
+//                  DISABLED BY DEFAULT — edge-less content was measured to
+//                  manufacture answers where the store should stay silent.
+//   8. GenKnow     ~37.6k {Question, Answer} pairs → one FACT each. DISABLED
+//                  BY DEFAULT on licence grounds; see DATASETS.md §3.2.
+import { smolsent } from "./smolsent.js";
+import { aya } from "./aya.js";
+import { oasst2 } from "./oasst2.js";
+import { taskmaster } from "./taskmaster.js";
+import { wiki2 } from "./wiki2.js";
+import { soda } from "./soda.js";
+import { massive } from "./massive.js";
+import { genknow } from "./genknow.js";
+export const CURRICULUM = [
+    smolsent,
+    aya,
+    oasst2,
+    taskmaster,
+    wiki2,
+    soda,
+    massive,
+    genknow,
+];
+/** The corpora this RUN will train, for the panel header — derived, so it can
+ *  never go stale the way the hand-written string it replaces had. */
+export const enabledLabels = () => CURRICULUM.filter((c) => c.enabled).map((c) => c.label).join("+");
+/** The corpora a STORE contains: everything this run will train PLUS everything
+ *  an earlier run already did, in curriculum order.
+ *
+ *  Not the same question as `enabledLabels`, and conflating them wrote a false
+ *  statement into every store that was ever resumed with a different set of
+ *  stages enabled. Observed on a real store: `train.dataset` read
+ *  "SmolSent+Aya+oasst2" while the tally recorded 37,623 General-Knowledge
+ *  deposits sitting in it. That is not cosmetic — a Sema store retains its
+ *  training text VERBATIM, so `train.dataset` is the record of whose licence
+ *  terms travel with the artifact, and General-Knowledge is precisely the
+ *  corpus DATASETS.md §3.2 disables on NonCommercial grounds.
+ *
+ *  `trainedIds` are the corpus ids an earlier run deposited under (the keys of
+ *  the per-corpus tally). An id no longer in the curriculum cannot be named and
+ *  is dropped — the tally still carries it, which is where that evidence lives. */
+export const storedLabels = (trainedIds) => {
+    const trained = new Set(trainedIds);
+    return CURRICULUM
+        .filter((c) => c.enabled || trained.has(c.id))
+        .map((c) => c.label)
+        .join("+");
+};
+// One re-export per corpus, carrying the descriptor AND its adapters. The
+// explicit `export { aya, genknow, … }` list that used to sit here named the
+// eight descriptors a second time; the star exports below already provide them,
+// and a hand-kept list of everything is exactly the thing that goes stale.
+export * from "./smolsent.js";
+export * from "./aya.js";
+export * from "./oasst2.js";
+export * from "./taskmaster.js";
+export * from "./wiki2.js";
+export * from "./soda.js";
+export * from "./massive.js";
+export * from "./genknow.js";

package/dist/example/train_base/corpora/massive.d.ts

@@ -0,0 +1,7 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+/** Translate ONE MASSIVE row into SEMA items: its bare utterance, as an
+ *  experience. `annot_utt` (slot-annotated) is deliberately not used — its
+ *  "[date : ...]" markup is not prose. Returns [] for an unusable row. */
+export declare function massiveRowToItems(row: unknown, maxChars?: number): TrainingItem[];
+export declare const massive: Corpus;

package/dist/example/train_base/corpora/massive.js

@@ -0,0 +1,98 @@
+// train_base/corpora/massive.ts — AmazonScience/massive short intents
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { refineItems } from "../items.js";
+import { parquet } from "../readers.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// MASSIVE deposits BARE UTTERANCES — an experience, not an episode — and that
+// is the only shape its data supports. Two richer shapes were considered and
+// rejected on evidence:
+//   • Same-intent pairs as paraphrases. 49.1% of consecutive rows share
+//     (locale, intent), but they are NOT meaning-equivalent: intent 48 in mn-MN
+//     runs "wake me at nine on the fifth" next to "set an alarm two hours from
+//     now". Depositing that pair as an episode teaches a continuation that does
+//     not exist.
+//   • Same-id rows across locales. Those ARE translations of one another —
+//     which is exactly SmolSent's relation, and SmolSent scores worst of every
+//     corpus measured on fold-unit recurrence (23.2%) because cross-lingual
+//     pairs share no units.
+// So the stage contributes recurring fold units and lexical coverage (65.1%
+// recurring unit mass, median 29 B) and nothing relational. `annot_utt` carries
+// slot markup ("[date : tavdahad] ...") and is never read.
+// DISABLED BY DEFAULT, on evidence gathered after the stage was written. A bare
+// experience deposits content with NO EDGE, and that cuts both ways. Measured on
+// a three-pair dialogue store with and without six MASSIVE-style utterances:
+//
+//   "set an alarm"            without: "Sure, what size would you like?"  (wrong)
+//                             with:    "set an alarm for seven"           (better)
+//   "play music"              without: ""                                 (correct silence)
+//                             with:    "Yes, sweetened or unsweetened?"   (wrong)
+//
+// So it displaces some wrong answers and manufactures others, INCLUDING turning
+// a correct silence into a wrong answer — and honest silence is a stated
+// property of this engine (AGENTS §2.13). On the mixed-curriculum store the
+// same shape produced the fragment "nus" for "wake me up at nine am".
+//
+// That evidence is four probes on toy stores and is NOT conclusive; it is,
+// however, the only evidence there is, and it points the wrong way. The stage
+// stays implemented and one env var away. Turn it on (MASSIVE=1) once there is
+// a real measurement showing the recurring fold units it contributes (72.3% of
+// deposited unit mass) buy more than the spurious answers cost.
+const MASSIVE = env("MASSIVE", "0") !== "0";
+const MASSIVE_DATASET = env("MASSIVE_DATASET", "AmazonScience/massive");
+// "all" is the config covering every locale in one set of shards.
+const MASSIVE_CONFIG = env("MASSIVE_CONFIG", "all");
+const MASSIVE_SPLITS = env("MASSIVE_SPLITS", "train")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// 0 = every row (587,214 in `all`/train, ~17 MB of content).
+const MASSIVE_MAX_ROWS = Math.max(0, Math.floor(Number(env("MASSIVE_MAX_ROWS", "0"))) || 0);
+const MAX_MASSIVE_UTT_CHARS = Math.max(100, Math.floor(Number(env("MAX_MASSIVE_UTT_KB", "2")) * 1000) || 2_000);
+// ═══════════════════════════════════════════════════════════════════════
+// §6e⁗  MASSIVE parsing — one short utterance → ONE SEMA experience
+//
+// See the constants note for why this deposits a bare experience and not a
+// relation: the two relational shapes this corpus appears to offer are both
+// false (same-intent rows are not paraphrases; same-id rows across locales are
+// translations, SmolSent's worst-scoring relation).
+// ═══════════════════════════════════════════════════════════════════════
+/** Translate ONE MASSIVE row into SEMA items: its bare utterance, as an
+ *  experience. `annot_utt` (slot-annotated) is deliberately not used — its
+ *  "[date : ...]" markup is not prose. Returns [] for an unusable row. */
+export function massiveRowToItems(row, maxChars = MAX_MASSIVE_UTT_CHARS) {
+    if (!row || typeof row !== "object")
+        return [];
+    const utt = row.utt;
+    const text = typeof utt === "string" ? utt.trim() : "";
+    if (!text || text.length > maxChars)
+        return [];
+    return refineItems([text]);
+}
+export const massive = {
+    id: "massive",
+    label: "MASSIVE",
+    kind: "short intents",
+    enabled: MASSIVE,
+    maxRows: MASSIVE_MAX_ROWS,
+    // One of ten columns — `annot_utt`, whose slot markup the note above rejects,
+    // is not decoded either. Measured on the converted train shard: 92 MB
+    // uncompressed across all ten, 36 MB for `utt` (38.7%).
+    read: parquet({ columns: ["utt"] }),
+    toItems: (row) => {
+        const items = massiveRowToItems(row);
+        return items.length ? items : null;
+    },
+    unitNoun: "shard(s)",
+    log: { rows: "row(s)" },
+    discover: convertedParquetUnits({
+        id: "massive",
+        label: "MASSIVE",
+        dataset: MASSIVE_DATASET,
+        config: MASSIVE_CONFIG,
+        splits: MASSIVE_SPLITS,
+        localSub: "massive",
+    }),
+};

package/dist/example/train_base/corpora/oasst2.d.ts

@@ -0,0 +1,52 @@
+import { type TrainingItem } from "../items.js";
+import { type Corpus } from "../corpus.js";
+/** A single oasst2 message node (the fields we use; the tree nests via replies). */
+interface OasstNode {
+    role?: string;
+    text?: string;
+    rank?: number | null;
+    deleted?: boolean;
+    replies?: OasstNode[];
+}
+/** One conversational turn extracted from a tree. */
+export interface OasstTurn {
+    role: string;
+    text: string;
+}
+/** Collapse a conversation tree to ONE linear path: at each node, descend into
+ *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
+ *  Returns the ordered turns (already strictly alternating in this corpus). */
+export declare function bestOasstPath(root: OasstNode): OasstTurn[];
+/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
+ *
+ *  This is the ONE stage where cumulative continuous context is truly necessary:
+ *  the data is a real multi-turn dialogue, and what must be learned is how each
+ *  turn follows from the WHOLE conversation so far — not from the previous turn
+ *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
+ *  turn experiences and local adjacent-pair facts are NOT emitted (they are
+ *  subsumed by it and would merely replicate the content).
+ *
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
+ *  alternate by position in an oasst2 best-path (the root is a prompter), so a
+ *  label adds nothing the position does not, while a clean continuation matches
+ *  the test's recall (predictNext queries bare prior turns) and lets a turn share
+ *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
+ *
+ *  Returns [] for a conversation below the multi-turn threshold, so callers can
+ *  simply skip empties. */
+export declare function oasstConversationToItems(turns: OasstTurn[], minTurns?: number): TrainingItem[];
+/** The row adapter: ONE line of the tree dump → its deposits. Returns null for
+ *  a tree with no prompt and for every single-turn tree — the latter is the
+ *  stage's design, not a defect, which is why the reader counts it `unusable`
+ *  rather than `skipped`. */
+export declare function oasstTreeToItems(row: unknown): TrainingItem[] | null;
+export declare const oasst2: Corpus;
+export {};

package/dist/example/train_base/corpora/oasst2.js

@@ -0,0 +1,120 @@
+// train_base/corpora/oasst2.ts — OpenAssistant/oasst2 conversation trees
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { accumulate, refineItems } from "../items.js";
+import { lines } from "../readers.js";
+import { singleUnit } from "../corpus.js";
+// ── OpenAssistant/oasst2 (the fourth training stage, after Aya) ──
+// oasst2 is a corpus of human↔assistant conversation TREES. Its richest, most
+// stream-friendly artifact is "<date>_oasst2_ready.trees.jsonl.gz": one JSON
+// conversation tree PER LINE, gzip-compressed (a web standard — Decompression
+// Stream("gzip")). Each tree is {message_tree_id, prompt:{role,text,replies:[…]}}
+// where `replies` nests recursively and a prompt can have several ranked
+// assistant replies (rank 0 = best). We follow the best-ranked, non-deleted
+// reply at each step to get ONE linear, strictly-alternating conversation per
+// tree, then keep only the MULTI-TURN ones (≥ OASST_MIN_TURNS messages, i.e. at
+// least two full user→assistant exchanges) — single Q→A trees are skipped, by
+// design. OASST=0 disables the stage; OASST_URL overrides the source.
+const OASST = env("OASST", "1") !== "0";
+const OASST_URL = env("OASST_URL", "https://huggingface.co/datasets/OpenAssistant/oasst2/resolve/main/2023-11-05_oasst2_ready.trees.jsonl.gz");
+// Multi-turn threshold: a conversation must have at least this many turns to be
+// trained (4 = user→assistant→user→assistant, the smallest real multi-turn).
+const OASST_MIN_TURNS = Math.max(2, Math.floor(Number(env("OASST_MIN_TURNS", "4"))) || 4);
+// Skip a tree whose decoded JSON line exceeds this (a pathological record); the
+// real maximum is far smaller, so this only guards against corruption.
+const MAX_OASST_LINE_CHARS = Math.max(100_000, Math.floor(Number(env("MAX_OASST_LINE_MB", "8")) * 1_000_000) || 8_000_000);
+/** Collapse a conversation tree to ONE linear path: at each node, descend into
+ *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
+ *  Returns the ordered turns (already strictly alternating in this corpus). */
+export function bestOasstPath(root) {
+    const turns = [];
+    let node = root;
+    while (node) {
+        const text = typeof node.text === "string" ? node.text.trim() : "";
+        if (text)
+            turns.push({ role: String(node.role ?? "?"), text });
+        const live = (node.replies ?? []).filter((r) => r && !r.deleted && typeof r.text === "string" && r.text.trim() !== "");
+        if (live.length === 0)
+            break;
+        live.sort((a, b) => (a.rank ?? Number.MAX_SAFE_INTEGER) - (b.rank ?? Number.MAX_SAFE_INTEGER));
+        node = live[0];
+    }
+    return turns;
+}
+/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
+ *
+ *  This is the ONE stage where cumulative continuous context is truly necessary:
+ *  the data is a real multi-turn dialogue, and what must be learned is how each
+ *  turn follows from the WHOLE conversation so far — not from the previous turn
+ *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
+ *  turn experiences and local adjacent-pair facts are NOT emitted (they are
+ *  subsumed by it and would merely replicate the content).
+ *
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
+ *  alternate by position in an oasst2 best-path (the root is a prompter), so a
+ *  label adds nothing the position does not, while a clean continuation matches
+ *  the test's recall (predictNext queries bare prior turns) and lets a turn share
+ *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
+ *
+ *  Returns [] for a conversation below the multi-turn threshold, so callers can
+ *  simply skip empties. */
+export function oasstConversationToItems(turns, minTurns = OASST_MIN_TURNS) {
+    if (turns.length < minTurns)
+        return []; // not multi-turn — skip
+    return refineItems(accumulate(turns.map((t) => t.text)));
+}
+/** The row adapter: ONE line of the tree dump → its deposits. Returns null for
+ *  a tree with no prompt and for every single-turn tree — the latter is the
+ *  stage's design, not a defect, which is why the reader counts it `unusable`
+ *  rather than `skipped`. */
+export function oasstTreeToItems(row) {
+    if (!row || typeof row !== "object")
+        return null;
+    const tree = row;
+    if (!tree.prompt)
+        return null;
+    const items = oasstConversationToItems(bestOasstPath(tree.prompt));
+    return items.length ? items : null;
+}
+export const oasst2 = {
+    id: "oasst2",
+    label: "oasst2",
+    kind: "multi-turn chat",
+    enabled: OASST,
+    read: lines({ gzip: true, maxLineChars: MAX_OASST_LINE_CHARS }),
+    toItems: oasstTreeToItems,
+    // The one corpus that KEEPS a cached file after a complete read: a copy left
+    // behind by a previous interrupted run is not this stage's to reclaim.
+    keepCached: true,
+    log: {
+        deposits: "examples",
+        // A tree that yields items is exactly a tree that cleared OASST_MIN_TURNS.
+        rows: "conversation(s)",
+        // The single-turn trees this stage drops BY DESIGN must not be reported as
+        // damage — only genuinely malformed lines are.
+        malformedOnly: true,
+    },
+    discover: singleUnit({
+        // Resume id "oasst2::trees" — the string this store already records.
+        key: "trees",
+        label: "oasst2",
+        display: "oasst2 (multi-turn)",
+        url: OASST_URL,
+        dest: "oasst2_ready.trees.jsonl.gz",
+        acquireLabel: "oasst2 trees",
+        localMatch: [/oasst.*trees.*\.jsonl\.gz$/i, /oasst.*\.jsonl\.gz$/i],
+        localWhat: "oasst2 *trees*.jsonl.gz",
+    }),
+};

package/dist/example/train_base/corpora/smolsent.d.ts

@@ -0,0 +1,23 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+/** One normalized SmolSent row. */
+export interface SmolSentRow {
+    src: string;
+    trg: string;
+    sl: string;
+    tl: string;
+}
+/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
+ *  both sides or a side is implausibly large (a dump, not a sentence). */
+export declare function toSmolSentRow(row: unknown, maxChars?: number): SmolSentRow | null;
+/** Translate ONE SmolSent pair into SEMA facts. The two sentences are one
+ *  meaning in two languages, but the two BINDINGS are not equally sound —
+ *  SmolSent's English side is a shared pool translated into every language, so
+ *  `trg -> src` gives one English context a different answer in every language
+ *  file. See SMOLSENT_DIRECTIONS. refineItems drops the degenerate case where
+ *  src === trg. */
+export declare function smolSentRowToItems(row: SmolSentRow, dirs?: {
+    src2trg: boolean;
+    trg2src: boolean;
+}): TrainingItem[];
+export declare const smolsent: Corpus;

package/dist/example/train_base/corpora/smolsent.js

@@ -0,0 +1,156 @@
+// train_base/corpora/smolsent.ts — google/smol sentence pairs
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env, LOCAL_PATH } from "../config.js";
+import { refineItems } from "../items.js";
+import { hfTree, localFiles } from "../discovery.js";
+import { lines } from "../readers.js";
+import { basename, join } from "node:path";
+// ── google/smol · SmolSent (the first training stage) ──
+// SmolSent is Google's sentence-level translation set: ~863 human sentence pairs
+// per language pair across 100+ low-resource languages, cc-by-4.0 (commercial-
+// friendly). Each row is {sl, tl, src, trg, …} — a source sentence and its
+// translation. A pair is "two names for one meaning", which is exactly the
+// cross-language concept SEMA fuses (see test/05-concepts.test.mjs), so each row
+// becomes FACTS that bind the two phrasings as one concept at recall time.
+//
+// The corpus ships as one plain JSONL file PER language pair under smolsent/ in
+// the HF repo (e.g. smolsent/ha_en.jsonl). We DOWNLOAD each file and stream its
+// lines — far faster and free of the rate-limiting that per-row API paging hit.
+// The file list is discovered from the HF repo tree. SMOLSENT=0 disables the
+// stage; SMOLSENT_PAIRS (comma-separated basenames without .jsonl, e.g.
+// "ha_en,zu_en") restricts to a chosen subset.
+const SMOLSENT = env("SMOLSENT", "1") !== "0";
+const SMOLSENT_DATASET = env("SMOLSENT_DATASET", "google/smol");
+const SMOLSENT_PAIRS = (process.env.SMOLSENT_PAIRS ?? "")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// The resume id PREFIX for the SmolSent stage; one completed-files entry per
+// file (e.g. "smolsent::ha_en.jsonl").
+const SMOLSENT_ID = "smolsent";
+// Which direction(s) of a translation pair to deposit. Was effectively "both",
+// and that is now the default NO longer, for a reason measured rather than
+// assumed.
+//
+// SmolSent's English side is a SHARED POOL translated into every language: row
+// id 0 of smolsent/ha_en.jsonl, zu_en.jsonl and am_en.jsonl all carry the SAME
+// `trg` ("It allows me to work by following my vibes and ..."). The two
+// directions are therefore not symmetric at all:
+//
+//   src2trg  (foreign -> English)  many distinct contexts -> ONE shared
+//                                  continuation. Every language's rendering of
+//                                  a meaning converges on the same English
+//                                  node — the cross-language concept fusion
+//                                  this stage exists for.
+//   trg2src  (English -> foreign)  ONE context -> 100+ DIFFERENT continuations,
+//                                  one per language file. The same English
+//                                  sentence is deposited over and over with a
+//                                  different answer each time.
+//
+// So dropping trg2src is not merely a corpus-size economy (it halves the
+// largest stage, which was 60.9% of all examples in the last trained store); it
+// removes a genuine ambiguity pathology. Set SMOLSENT_DIRECTIONS=both to
+// restore the old behaviour, or trg2src for English->foreign only.
+//
+// WHAT THE CUT DOES NOT DO, measured on a three-pair store: asking the English
+// sentence still ANSWERS with a foreign rendering, because the engine can reach
+// a shared continuation's predecessors on its own. What is removed is the
+// DEPOSITED forward ambiguity — one context carrying ~100 competing
+// continuations — not every reverse association.
+const SMOLSENT_DIRECTIONS = env("SMOLSENT_DIRECTIONS", "src2trg")
+    .trim().toLowerCase();
+const SMOLSENT_SRC2TRG = SMOLSENT_DIRECTIONS !== "trg2src";
+const SMOLSENT_TRG2SRC = SMOLSENT_DIRECTIONS === "trg2src" ||
+    SMOLSENT_DIRECTIONS === "both";
+// A SmolSent side longer than this is skipped (a sentence pair is short; a huge
+// value is corruption, not a sentence).
+const MAX_SMOLSENT_CHARS = Math.max(2_000, Math.floor(Number(env("MAX_SMOLSENT_KB", "16")) * 1000) || 16_000);
+/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
+ *  both sides or a side is implausibly large (a dump, not a sentence). */
+export function toSmolSentRow(row, maxChars = MAX_SMOLSENT_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    const src = typeof r.src === "string" ? r.src.trim() : "";
+    // `trg` is a single string in smolsent; tolerate a list form defensively.
+    const trgRaw = Array.isArray(r.trgs) ? r.trgs[0] : r.trg;
+    const trg = typeof trgRaw === "string" ? trgRaw.trim() : "";
+    if (!src || !trg)
+        return null;
+    if (src.length > maxChars || trg.length > maxChars)
+        return null;
+    const sl = typeof r.sl === "string" ? r.sl.trim() : "";
+    const tl = typeof r.tl === "string" ? r.tl.trim() : "";
+    return { src, trg, sl, tl };
+}
+/** Translate ONE SmolSent pair into SEMA facts. The two sentences are one
+ *  meaning in two languages, but the two BINDINGS are not equally sound —
+ *  SmolSent's English side is a shared pool translated into every language, so
+ *  `trg -> src` gives one English context a different answer in every language
+ *  file. See SMOLSENT_DIRECTIONS. refineItems drops the degenerate case where
+ *  src === trg. */
+export function smolSentRowToItems(row, dirs = {
+    src2trg: SMOLSENT_SRC2TRG,
+    trg2src: SMOLSENT_TRG2SRC,
+}) {
+    const { src, trg } = row;
+    const items = [];
+    if (dirs.src2trg)
+        items.push({ context: src, continuation: trg });
+    if (dirs.trg2src)
+        items.push({ context: trg, continuation: src });
+    return refineItems(items);
+}
+/** Discover the SmolSent per-pair JSONL files from the HF repo tree, restricted
+ *  to SMOLSENT_PAIRS (basenames without .jsonl) when set. Each entry is the
+ *  repo-relative path, e.g. "smolsent/ha_en.jsonl". */
+async function listFiles(ctx) {
+    const paths = await hfTree(SMOLSENT_DATASET, "smolsent", /\.jsonl$/i, `GET smol tree`, ctx.http);
+    if (!SMOLSENT_PAIRS.length)
+        return paths;
+    const want = new Set(SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")));
+    return paths.filter((p) => want.has(basename(p.path).replace(/\.jsonl$/i, "")));
+}
+const unit = (name) => ({
+    key: name,
+    // The log names a pair by its language code alone; the panel keeps the file.
+    name: name.replace(/\.jsonl$/i, ""),
+    display: `SmolSent ${name}`,
+});
+export const smolsent = {
+    id: SMOLSENT_ID,
+    label: "SmolSent",
+    kind: "translation",
+    enabled: SMOLSENT,
+    unitNoun: "translation file(s)",
+    // The *4 is chars-to-a-generous-JSON-envelope: the guard bounds ONE side of
+    // a pair, the line carries both plus its keys.
+    read: lines({ maxLineChars: MAX_SMOLSENT_CHARS * 4 }),
+    toItems: (row) => {
+        const r = toSmolSentRow(row);
+        return r ? smolSentRowToItems(r) : null;
+    },
+    async discover(ctx) {
+        // Work-list: local *.jsonl in LOCAL_PATH, else the repo's smolsent/ files.
+        if (LOCAL_PATH) {
+            let names = localFiles(LOCAL_PATH, /\.jsonl$/i);
+            if (SMOLSENT_PAIRS.length) {
+                const want = new Set(SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")));
+                names = names.filter((n) => want.has(n.path.replace(/\.jsonl$/i, "")));
+            }
+            return names.map((n) => ({
+                ...unit(n.path),
+                local: join(LOCAL_PATH, n.path),
+                bytes: n.size,
+            }));
+        }
+        return (await listFiles(ctx)).map(({ path, size }) => ({
+            ...unit(basename(path)),
+            // owner/name and the file path are URL PATH segments — do not encode "/".
+            url: `https://huggingface.co/datasets/${SMOLSENT_DATASET}/resolve/main/${path}`,
+            bytes: size,
+        }));
+    },
+};