@hviana/sema 0.5.8 → 0.6.0

package/dist/example/train_base/corpora/aya.js

@@ -0,0 +1,76 @@
+// train_base/corpora/aya.ts — CohereLabs/aya_dataset prompt→completion pairs
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { refineItems } from "../items.js";
+import { parquet } from "../readers.js";
+import { singleUnit } from "../corpus.js";
+// ── CohereLabs/aya_dataset (the second training stage, after SmolSent) ──
+// The Aya Dataset is ~204k HUMAN-annotated prompt→completion pairs across 70+
+// languages, each a clean (inputs → targets) fact in a named language. It ships
+// ONLY as Snappy-compressed Parquet (no JSONL/CSV). We DOWNLOAD the one train
+// Parquet file and read it row-group by row-group with `hyparquet` (a pure-JS,
+// dependency-free Parquet reader) + `hyparquet-compressors` (Snappy) over a
+// web-standard Blob byte source — no whole-file-in-memory load. AYA=0 disables
+// the stage; AYA_URL overrides the Parquet source.
+const AYA = env("AYA", "1") !== "0";
+const AYA_URL = env("AYA_URL", "https://huggingface.co/datasets/CohereLabs/aya_dataset/resolve/main/data/train-00000-of-00001.parquet");
+// A single Aya field this many chars or longer is skipped: inputs/targets range
+// up to ~3.3M chars, and a multi-MB "pair" is documentation/dump noise, not a
+// cognitive example.
+const MAX_AYA_FIELD_CHARS = Math.max(10_000, Math.floor(Number(env("MAX_AYA_FIELD_KB", "256")) * 1000) || 256_000);
+/** Normalize a raw datasets-server row object into an AyaRow, or null when it
+ *  lacks a usable prompt/answer or a field is implausibly large (a dump, not a
+ *  cognitive example). Trims surrounding whitespace; keeps inner text verbatim
+ *  (human prose, possibly multi-paragraph). */
+export function toAyaRow(row, maxChars = MAX_AYA_FIELD_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    const inputs = typeof r.inputs === "string" ? r.inputs.trim() : "";
+    const targets = typeof r.targets === "string" ? r.targets.trim() : "";
+    if (!inputs || !targets)
+        return null;
+    if (inputs.length > maxChars || targets.length > maxChars)
+        return null;
+    const language = typeof r.language === "string" ? r.language.trim() : "";
+    return { inputs, targets, language };
+}
+/** Translate ONE Aya row into SEMA training items. A row is a single human
+ *  (question → answer) exchange — exactly one FACT, the (inputs → targets) edge.
+ *  No standalone-answer experience and no one-exchange "cumulative" walk: a lone
+ *  Q→A is not multi-turn, and both would only replicate the same edge. */
+export function ayaRowToItems(row) {
+    const { inputs, targets } = row;
+    return refineItems([{ context: inputs, continuation: targets }]);
+}
+export const aya = {
+    id: "aya",
+    label: "Aya Dataset",
+    kind: "multilingual chat",
+    enabled: AYA,
+    // The three columns toAyaRow reads, out of six. Kept for the same reason as
+    // the other Parquet stages — the read states what the adapter uses — though
+    // here it is nearly free rather than a saving: measured on the train file,
+    // 238 MB uncompressed across all six and 233 MB for these three (98.2%). The
+    // dropped columns are ids and annotation metadata, so there is little to
+    // drop.
+    read: parquet({ columns: ["inputs", "targets", "language"] }),
+    toItems: (row) => {
+        const r = toAyaRow(row);
+        return r ? ayaRowToItems(r) : null;
+    },
+    discover: singleUnit({
+        // Resume id "aya::dataset" — the string this store already records.
+        key: "dataset",
+        label: "Aya Dataset",
+        display: "Aya Dataset",
+        url: AYA_URL,
+        dest: "aya_train.parquet",
+        localMatch: [/aya.*\.parquet$/i, /\.parquet$/i],
+        localWhat: "Aya *.parquet",
+    }),
+};

package/dist/example/train_base/corpora/converted-parquet.d.ts

@@ -0,0 +1,14 @@
+import { type Unit } from "../corpus.js";
+import type { TrainCtx } from "../runtime.js";
+/** The work-list of a corpus read from Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch. Shared by 2Wiki, SODA and MASSIVE: they
+ *  differ in their dataset, their budget and their adapter — nothing else. */
+export declare function convertedParquetUnits(opts: {
+    id: string;
+    label: string;
+    dataset: string;
+    config: string;
+    splits: string[];
+    /** Subdirectory of LOCAL_PATH holding pre-downloaded shards. */
+    localSub: string;
+}): (ctx: TrainCtx) => Promise<Unit[] | null>;

package/dist/example/train_base/corpora/converted-parquet.js

@@ -0,0 +1,44 @@
+// train_base/corpora/converted-parquet.ts — the work-list shape three corpora
+// share.
+//
+// Read from Hugging Face's auto-converted `refs/convert/parquet` branch, not
+// from main: a dataset's own Parquet may be written as ONE giant row-group
+// (SODA's is 1,191,582 rows) and a column chunk is per-group, so any read of it
+// materialises the whole file. The converted branch uses uniform 10,000-row
+// groups.
+import { LOCAL_PATH } from "../config.js";
+import { hfConvertedParquet, localFiles } from "../discovery.js";
+import { localDir } from "../corpus.js";
+import { DIM, R, YEL } from "../ui.js";
+import { join } from "node:path";
+/** The work-list of a corpus read from Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch. Shared by 2Wiki, SODA and MASSIVE: they
+ *  differ in their dataset, their budget and their adapter — nothing else. */
+export function convertedParquetUnits(opts) {
+    return async (ctx) => {
+        if (LOCAL_PATH) {
+            const dir = localDir(opts.localSub);
+            const names = localFiles(dir, /\.parquet$/i);
+            if (names.length === 0) {
+                ctx.progress.log(`  ${DIM}· no ${opts.label} *.parquet in ${dir} — skipping${R}`);
+                return null;
+            }
+            return names.map((n) => ({
+                key: n.path,
+                name: n.path,
+                display: `${opts.label} ${n.path}`,
+                local: join(dir, n.path),
+                bytes: n.size,
+            }));
+        }
+        const paths = await hfConvertedParquet(opts.dataset, opts.config, opts.splits, opts.label, ctx.http, (m) => ctx.progress.log(`  ${YEL}⚠${R} ${m}`));
+        return paths.map(({ path, size }) => ({
+            key: path,
+            name: path,
+            display: `${opts.label} ${path}`,
+            url: `https://huggingface.co/datasets/${opts.dataset}` +
+                `/resolve/refs%2Fconvert%2Fparquet/${path}`,
+            bytes: size,
+        }));
+    };
+}

package/dist/example/train_base/corpora/genknow.d.ts

@@ -0,0 +1,14 @@
+import { type TrainingItem } from "../items.js";
+import { type Corpus } from "../corpus.js";
+/** One normalized General-Knowledge row. */
+export interface GenKnowRow {
+    question: string;
+    answer: string;
+}
+/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
+ *  a usable question/answer or a side is implausibly large (corruption). */
+export declare function toGenKnowRow(row: unknown, maxChars?: number): GenKnowRow | null;
+/** Translate ONE General-Knowledge row into SEMA items: exactly one
+ *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
+export declare function genKnowRowToItems(row: GenKnowRow): TrainingItem[];
+export declare const genknow: Corpus;

package/dist/example/train_base/corpora/genknow.js

@@ -0,0 +1,83 @@
+// train_base/corpora/genknow.ts — MuskumPillerum/General-Knowledge Q&A
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { refineItems } from "../items.js";
+import { jsonArray } from "../readers.js";
+import { singleUnit } from "../corpus.js";
+// ── MuskumPillerum/General-Knowledge (the fourth training stage, after oasst2) ──
+// A ~37.6k-row general-knowledge Q&A set: each row is a single {Question, Answer}
+// pair. A row is a pure RELATION (question → answer), so it becomes exactly ONE
+// FACT, identical in shape to the Aya stage. It ships as a single JSON array
+// file (output.json); we DOWNLOAD it and stream the array. GENKNOW_URL overrides
+// the source.
+//
+// DISABLED BY DEFAULT ON LICENCE GROUNDS (2026-08-13). The HF repo carries NO
+// licence tag and no licence in its card — an earlier header in this file
+// claimed MIT without support — and its own dataset card states it "contains a
+// subset of the alpaca dataset". Alpaca is CC BY-NC 4.0: NonCommercial, which
+// conflicts with Sema's commercial licence. Because a Sema store retains its
+// training text VERBATIM, an unlicensed corpus inside it makes the whole
+// artifact undistributable. See DATASETS.md §3.2. GENKNOW=1 re-enables the
+// stage for local, non-distributed experiments only.
+const GENKNOW = env("GENKNOW", "0") !== "0";
+const GENKNOW_URL = env("GENKNOW_URL", "https://huggingface.co/datasets/MuskumPillerum/General-Knowledge/resolve/main/output.json");
+// A Question/Answer longer than this is skipped (answers run to a few hundred
+// chars; this only guards against a corrupt/runaway field).
+const MAX_GENKNOW_CHARS = Math.max(4_000, Math.floor(Number(env("MAX_GENKNOW_KB", "64")) * 1000) || 64_000);
+/** Turn a source value into clean prose: decode the literal "\n"/"\t"/"\r"
+ *  two-character escapes the source JSON left in the text, collapse the runs of
+ *  whitespace that creates, and trim. */
+function unescapePlain(s) {
+    return s
+        .replace(/\\r\\n|\\n|\\r/g, "\n")
+        .replace(/\\t/g, " ")
+        .replace(/[ \t]+/g, " ")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+}
+/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
+ *  a usable question/answer or a side is implausibly large (corruption). */
+export function toGenKnowRow(row, maxChars = MAX_GENKNOW_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    const question = typeof r.Question === "string"
+        ? unescapePlain(r.Question)
+        : "";
+    const answer = typeof r.Answer === "string" ? unescapePlain(r.Answer) : "";
+    if (!question || !answer)
+        return null;
+    if (question.length > maxChars || answer.length > maxChars)
+        return null;
+    return { question, answer };
+}
+/** Translate ONE General-Knowledge row into SEMA items: exactly one
+ *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
+export function genKnowRowToItems(row) {
+    return refineItems([{ context: row.question, continuation: row.answer }]);
+}
+export const genknow = {
+    id: "genknow",
+    label: "General-Knowledge",
+    kind: "Q&A facts",
+    enabled: GENKNOW,
+    read: jsonArray(),
+    toItems: (row) => {
+        const r = toGenKnowRow(row);
+        return r ? genKnowRowToItems(r) : null;
+    },
+    discover: singleUnit({
+        // Resume id "genknow::qa" — the string this store already records.
+        key: "qa",
+        label: "General-Knowledge",
+        display: "General-Knowledge",
+        url: GENKNOW_URL,
+        dest: "general_knowledge.json",
+        localMatch: [/general.*knowledge.*\.json$/i, /output\.json$/i],
+        localWhat: "General-Knowledge *.json",
+    }),
+};

package/dist/example/train_base/corpora/index.d.ts

@@ -0,0 +1,29 @@
+import type { Corpus } from "../corpus.js";
+export declare const CURRICULUM: Corpus[];
+/** The corpora this RUN will train, for the panel header — derived, so it can
+ *  never go stale the way the hand-written string it replaces had. */
+export declare const enabledLabels: () => string;
+/** The corpora a STORE contains: everything this run will train PLUS everything
+ *  an earlier run already did, in curriculum order.
+ *
+ *  Not the same question as `enabledLabels`, and conflating them wrote a false
+ *  statement into every store that was ever resumed with a different set of
+ *  stages enabled. Observed on a real store: `train.dataset` read
+ *  "SmolSent+Aya+oasst2" while the tally recorded 37,623 General-Knowledge
+ *  deposits sitting in it. That is not cosmetic — a Sema store retains its
+ *  training text VERBATIM, so `train.dataset` is the record of whose licence
+ *  terms travel with the artifact, and General-Knowledge is precisely the
+ *  corpus DATASETS.md §3.2 disables on NonCommercial grounds.
+ *
+ *  `trainedIds` are the corpus ids an earlier run deposited under (the keys of
+ *  the per-corpus tally). An id no longer in the curriculum cannot be named and
+ *  is dropped — the tally still carries it, which is where that evidence lives. */
+export declare const storedLabels: (trainedIds: Iterable<string>) => string;
+export * from "./smolsent.js";
+export * from "./aya.js";
+export * from "./oasst2.js";
+export * from "./taskmaster.js";
+export * from "./wiki2.js";
+export * from "./soda.js";
+export * from "./massive.js";
+export * from "./genknow.js";

package/dist/example/train_base/corpora/index.js

@@ -0,0 +1,81 @@
+// train_base/corpora/index.ts — THE CURRICULUM.
+//
+// The order is the curriculum, and it is load-bearing: each stage runs only
+// after the previous one finishes, and every stage records itself in the same
+// completed-set, so a single store resumes the whole sequence.
+//
+//   1. SmolSent    sentence-level TRANSLATION pairs across 100+ low-resource
+//                  languages. Each pair is "two names for one meaning" → a
+//                  foreign→English translation FACT, so every language's
+//                  rendering of a meaning converges on ONE English node (cf.
+//                  test/05-concepts.test.mjs).
+//   2. Aya         ~204k human prompt→completion pairs, 70+ languages → one
+//                  (question → answer) FACT each.
+//   3. oasst2      MULTI-TURN human↔assistant conversation trees → the
+//                  accumulated-context walk (single-turn trees are skipped).
+//   4. Taskmaster  task-oriented DIALOGUE, the best-scoring corpora on the
+//                  fold-unit recurrence benchmark that predicts halo health.
+//   5. 2Wiki       the `evidences` TRIPLES — the one stage aimed at
+//                  COMPOSITION. Its Wikipedia passages and its composed
+//                  questions are deliberately NOT read.
+//   6. SODA        social/commonsense DIALOGUE, budgeted.
+//   7. MASSIVE     short intent utterances → ONE bare experience each.
+//                  DISABLED BY DEFAULT — edge-less content was measured to
+//                  manufacture answers where the store should stay silent.
+//   8. GenKnow     ~37.6k {Question, Answer} pairs → one FACT each. DISABLED
+//                  BY DEFAULT on licence grounds; see DATASETS.md §3.2.
+import { smolsent } from "./smolsent.js";
+import { aya } from "./aya.js";
+import { oasst2 } from "./oasst2.js";
+import { taskmaster } from "./taskmaster.js";
+import { wiki2 } from "./wiki2.js";
+import { soda } from "./soda.js";
+import { massive } from "./massive.js";
+import { genknow } from "./genknow.js";
+export const CURRICULUM = [
+    smolsent,
+    aya,
+    oasst2,
+    taskmaster,
+    wiki2,
+    soda,
+    massive,
+    genknow,
+];
+/** The corpora this RUN will train, for the panel header — derived, so it can
+ *  never go stale the way the hand-written string it replaces had. */
+export const enabledLabels = () => CURRICULUM.filter((c) => c.enabled).map((c) => c.label).join("+");
+/** The corpora a STORE contains: everything this run will train PLUS everything
+ *  an earlier run already did, in curriculum order.
+ *
+ *  Not the same question as `enabledLabels`, and conflating them wrote a false
+ *  statement into every store that was ever resumed with a different set of
+ *  stages enabled. Observed on a real store: `train.dataset` read
+ *  "SmolSent+Aya+oasst2" while the tally recorded 37,623 General-Knowledge
+ *  deposits sitting in it. That is not cosmetic — a Sema store retains its
+ *  training text VERBATIM, so `train.dataset` is the record of whose licence
+ *  terms travel with the artifact, and General-Knowledge is precisely the
+ *  corpus DATASETS.md §3.2 disables on NonCommercial grounds.
+ *
+ *  `trainedIds` are the corpus ids an earlier run deposited under (the keys of
+ *  the per-corpus tally). An id no longer in the curriculum cannot be named and
+ *  is dropped — the tally still carries it, which is where that evidence lives. */
+export const storedLabels = (trainedIds) => {
+    const trained = new Set(trainedIds);
+    return CURRICULUM
+        .filter((c) => c.enabled || trained.has(c.id))
+        .map((c) => c.label)
+        .join("+");
+};
+// One re-export per corpus, carrying the descriptor AND its adapters. The
+// explicit `export { aya, genknow, … }` list that used to sit here named the
+// eight descriptors a second time; the star exports below already provide them,
+// and a hand-kept list of everything is exactly the thing that goes stale.
+export * from "./smolsent.js";
+export * from "./aya.js";
+export * from "./oasst2.js";
+export * from "./taskmaster.js";
+export * from "./wiki2.js";
+export * from "./soda.js";
+export * from "./massive.js";
+export * from "./genknow.js";

package/dist/example/train_base/corpora/massive.d.ts

@@ -0,0 +1,7 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+/** Translate ONE MASSIVE row into SEMA items: its bare utterance, as an
+ *  experience. `annot_utt` (slot-annotated) is deliberately not used — its
+ *  "[date : ...]" markup is not prose. Returns [] for an unusable row. */
+export declare function massiveRowToItems(row: unknown, maxChars?: number): TrainingItem[];
+export declare const massive: Corpus;

package/dist/example/train_base/corpora/massive.js

@@ -0,0 +1,98 @@
+// train_base/corpora/massive.ts — AmazonScience/massive short intents
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { refineItems } from "../items.js";
+import { parquet } from "../readers.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// MASSIVE deposits BARE UTTERANCES — an experience, not an episode — and that
+// is the only shape its data supports. Two richer shapes were considered and
+// rejected on evidence:
+//   • Same-intent pairs as paraphrases. 49.1% of consecutive rows share
+//     (locale, intent), but they are NOT meaning-equivalent: intent 48 in mn-MN
+//     runs "wake me at nine on the fifth" next to "set an alarm two hours from
+//     now". Depositing that pair as an episode teaches a continuation that does
+//     not exist.
+//   • Same-id rows across locales. Those ARE translations of one another —
+//     which is exactly SmolSent's relation, and SmolSent scores worst of every
+//     corpus measured on fold-unit recurrence (23.2%) because cross-lingual
+//     pairs share no units.
+// So the stage contributes recurring fold units and lexical coverage (65.1%
+// recurring unit mass, median 29 B) and nothing relational. `annot_utt` carries
+// slot markup ("[date : tavdahad] ...") and is never read.
+// DISABLED BY DEFAULT, on evidence gathered after the stage was written. A bare
+// experience deposits content with NO EDGE, and that cuts both ways. Measured on
+// a three-pair dialogue store with and without six MASSIVE-style utterances:
+//
+//   "set an alarm"            without: "Sure, what size would you like?"  (wrong)
+//                             with:    "set an alarm for seven"           (better)
+//   "play music"              without: ""                                 (correct silence)
+//                             with:    "Yes, sweetened or unsweetened?"   (wrong)
+//
+// So it displaces some wrong answers and manufactures others, INCLUDING turning
+// a correct silence into a wrong answer — and honest silence is a stated
+// property of this engine (AGENTS §2.13). On the mixed-curriculum store the
+// same shape produced the fragment "nus" for "wake me up at nine am".
+//
+// That evidence is four probes on toy stores and is NOT conclusive; it is,
+// however, the only evidence there is, and it points the wrong way. The stage
+// stays implemented and one env var away. Turn it on (MASSIVE=1) once there is
+// a real measurement showing the recurring fold units it contributes (72.3% of
+// deposited unit mass) buy more than the spurious answers cost.
+const MASSIVE = env("MASSIVE", "0") !== "0";
+const MASSIVE_DATASET = env("MASSIVE_DATASET", "AmazonScience/massive");
+// "all" is the config covering every locale in one set of shards.
+const MASSIVE_CONFIG = env("MASSIVE_CONFIG", "all");
+const MASSIVE_SPLITS = env("MASSIVE_SPLITS", "train")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// 0 = every row (587,214 in `all`/train, ~17 MB of content).
+const MASSIVE_MAX_ROWS = Math.max(0, Math.floor(Number(env("MASSIVE_MAX_ROWS", "0"))) || 0);
+const MAX_MASSIVE_UTT_CHARS = Math.max(100, Math.floor(Number(env("MAX_MASSIVE_UTT_KB", "2")) * 1000) || 2_000);
+// ═══════════════════════════════════════════════════════════════════════
+// §6e⁗  MASSIVE parsing — one short utterance → ONE SEMA experience
+//
+// See the constants note for why this deposits a bare experience and not a
+// relation: the two relational shapes this corpus appears to offer are both
+// false (same-intent rows are not paraphrases; same-id rows across locales are
+// translations, SmolSent's worst-scoring relation).
+// ═══════════════════════════════════════════════════════════════════════
+/** Translate ONE MASSIVE row into SEMA items: its bare utterance, as an
+ *  experience. `annot_utt` (slot-annotated) is deliberately not used — its
+ *  "[date : ...]" markup is not prose. Returns [] for an unusable row. */
+export function massiveRowToItems(row, maxChars = MAX_MASSIVE_UTT_CHARS) {
+    if (!row || typeof row !== "object")
+        return [];
+    const utt = row.utt;
+    const text = typeof utt === "string" ? utt.trim() : "";
+    if (!text || text.length > maxChars)
+        return [];
+    return refineItems([text]);
+}
+export const massive = {
+    id: "massive",
+    label: "MASSIVE",
+    kind: "short intents",
+    enabled: MASSIVE,
+    maxRows: MASSIVE_MAX_ROWS,
+    // One of ten columns — `annot_utt`, whose slot markup the note above rejects,
+    // is not decoded either. Measured on the converted train shard: 92 MB
+    // uncompressed across all ten, 36 MB for `utt` (38.7%).
+    read: parquet({ columns: ["utt"] }),
+    toItems: (row) => {
+        const items = massiveRowToItems(row);
+        return items.length ? items : null;
+    },
+    unitNoun: "shard(s)",
+    log: { rows: "row(s)" },
+    discover: convertedParquetUnits({
+        id: "massive",
+        label: "MASSIVE",
+        dataset: MASSIVE_DATASET,
+        config: MASSIVE_CONFIG,
+        splits: MASSIVE_SPLITS,
+        localSub: "massive",
+    }),
+};

package/dist/example/train_base/corpora/oasst2.d.ts

@@ -0,0 +1,52 @@
+import { type TrainingItem } from "../items.js";
+import { type Corpus } from "../corpus.js";
+/** A single oasst2 message node (the fields we use; the tree nests via replies). */
+interface OasstNode {
+    role?: string;
+    text?: string;
+    rank?: number | null;
+    deleted?: boolean;
+    replies?: OasstNode[];
+}
+/** One conversational turn extracted from a tree. */
+export interface OasstTurn {
+    role: string;
+    text: string;
+}
+/** Collapse a conversation tree to ONE linear path: at each node, descend into
+ *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
+ *  Returns the ordered turns (already strictly alternating in this corpus). */
+export declare function bestOasstPath(root: OasstNode): OasstTurn[];
+/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
+ *
+ *  This is the ONE stage where cumulative continuous context is truly necessary:
+ *  the data is a real multi-turn dialogue, and what must be learned is how each
+ *  turn follows from the WHOLE conversation so far — not from the previous turn
+ *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
+ *  turn experiences and local adjacent-pair facts are NOT emitted (they are
+ *  subsumed by it and would merely replicate the content).
+ *
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
+ *  alternate by position in an oasst2 best-path (the root is a prompter), so a
+ *  label adds nothing the position does not, while a clean continuation matches
+ *  the test's recall (predictNext queries bare prior turns) and lets a turn share
+ *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
+ *
+ *  Returns [] for a conversation below the multi-turn threshold, so callers can
+ *  simply skip empties. */
+export declare function oasstConversationToItems(turns: OasstTurn[], minTurns?: number): TrainingItem[];
+/** The row adapter: ONE line of the tree dump → its deposits. Returns null for
+ *  a tree with no prompt and for every single-turn tree — the latter is the
+ *  stage's design, not a defect, which is why the reader counts it `unusable`
+ *  rather than `skipped`. */
+export declare function oasstTreeToItems(row: unknown): TrainingItem[] | null;
+export declare const oasst2: Corpus;
+export {};

package/dist/example/train_base/corpora/oasst2.js

@@ -0,0 +1,120 @@
+// train_base/corpora/oasst2.ts — OpenAssistant/oasst2 conversation trees
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { accumulate, refineItems } from "../items.js";
+import { lines } from "../readers.js";
+import { singleUnit } from "../corpus.js";
+// ── OpenAssistant/oasst2 (the fourth training stage, after Aya) ──
+// oasst2 is a corpus of human↔assistant conversation TREES. Its richest, most
+// stream-friendly artifact is "<date>_oasst2_ready.trees.jsonl.gz": one JSON
+// conversation tree PER LINE, gzip-compressed (a web standard — Decompression
+// Stream("gzip")). Each tree is {message_tree_id, prompt:{role,text,replies:[…]}}
+// where `replies` nests recursively and a prompt can have several ranked
+// assistant replies (rank 0 = best). We follow the best-ranked, non-deleted
+// reply at each step to get ONE linear, strictly-alternating conversation per
+// tree, then keep only the MULTI-TURN ones (≥ OASST_MIN_TURNS messages, i.e. at
+// least two full user→assistant exchanges) — single Q→A trees are skipped, by
+// design. OASST=0 disables the stage; OASST_URL overrides the source.
+const OASST = env("OASST", "1") !== "0";
+const OASST_URL = env("OASST_URL", "https://huggingface.co/datasets/OpenAssistant/oasst2/resolve/main/2023-11-05_oasst2_ready.trees.jsonl.gz");
+// Multi-turn threshold: a conversation must have at least this many turns to be
+// trained (4 = user→assistant→user→assistant, the smallest real multi-turn).
+const OASST_MIN_TURNS = Math.max(2, Math.floor(Number(env("OASST_MIN_TURNS", "4"))) || 4);
+// Skip a tree whose decoded JSON line exceeds this (a pathological record); the
+// real maximum is far smaller, so this only guards against corruption.
+const MAX_OASST_LINE_CHARS = Math.max(100_000, Math.floor(Number(env("MAX_OASST_LINE_MB", "8")) * 1_000_000) || 8_000_000);
+/** Collapse a conversation tree to ONE linear path: at each node, descend into
+ *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
+ *  Returns the ordered turns (already strictly alternating in this corpus). */
+export function bestOasstPath(root) {
+    const turns = [];
+    let node = root;
+    while (node) {
+        const text = typeof node.text === "string" ? node.text.trim() : "";
+        if (text)
+            turns.push({ role: String(node.role ?? "?"), text });
+        const live = (node.replies ?? []).filter((r) => r && !r.deleted && typeof r.text === "string" && r.text.trim() !== "");
+        if (live.length === 0)
+            break;
+        live.sort((a, b) => (a.rank ?? Number.MAX_SAFE_INTEGER) - (b.rank ?? Number.MAX_SAFE_INTEGER));
+        node = live[0];
+    }
+    return turns;
+}
+/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
+ *
+ *  This is the ONE stage where cumulative continuous context is truly necessary:
+ *  the data is a real multi-turn dialogue, and what must be learned is how each
+ *  turn follows from the WHOLE conversation so far — not from the previous turn
+ *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
+ *  turn experiences and local adjacent-pair facts are NOT emitted (they are
+ *  subsumed by it and would merely replicate the content).
+ *
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
+ *  alternate by position in an oasst2 best-path (the root is a prompter), so a
+ *  label adds nothing the position does not, while a clean continuation matches
+ *  the test's recall (predictNext queries bare prior turns) and lets a turn share
+ *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
+ *
+ *  Returns [] for a conversation below the multi-turn threshold, so callers can
+ *  simply skip empties. */
+export function oasstConversationToItems(turns, minTurns = OASST_MIN_TURNS) {
+    if (turns.length < minTurns)
+        return []; // not multi-turn — skip
+    return refineItems(accumulate(turns.map((t) => t.text)));
+}
+/** The row adapter: ONE line of the tree dump → its deposits. Returns null for
+ *  a tree with no prompt and for every single-turn tree — the latter is the
+ *  stage's design, not a defect, which is why the reader counts it `unusable`
+ *  rather than `skipped`. */
+export function oasstTreeToItems(row) {
+    if (!row || typeof row !== "object")
+        return null;
+    const tree = row;
+    if (!tree.prompt)
+        return null;
+    const items = oasstConversationToItems(bestOasstPath(tree.prompt));
+    return items.length ? items : null;
+}
+export const oasst2 = {
+    id: "oasst2",
+    label: "oasst2",
+    kind: "multi-turn chat",
+    enabled: OASST,
+    read: lines({ gzip: true, maxLineChars: MAX_OASST_LINE_CHARS }),
+    toItems: oasstTreeToItems,
+    // The one corpus that KEEPS a cached file after a complete read: a copy left
+    // behind by a previous interrupted run is not this stage's to reclaim.
+    keepCached: true,
+    log: {
+        deposits: "examples",
+        // A tree that yields items is exactly a tree that cleared OASST_MIN_TURNS.
+        rows: "conversation(s)",
+        // The single-turn trees this stage drops BY DESIGN must not be reported as
+        // damage — only genuinely malformed lines are.
+        malformedOnly: true,
+    },
+    discover: singleUnit({
+        // Resume id "oasst2::trees" — the string this store already records.
+        key: "trees",
+        label: "oasst2",
+        display: "oasst2 (multi-turn)",
+        url: OASST_URL,
+        dest: "oasst2_ready.trees.jsonl.gz",
+        acquireLabel: "oasst2 trees",
+        localMatch: [/oasst.*trees.*\.jsonl\.gz$/i, /oasst.*\.jsonl\.gz$/i],
+        localWhat: "oasst2 *trees*.jsonl.gz",
+    }),
+};

package/dist/example/train_base/corpora/smolsent.d.ts

@@ -0,0 +1,23 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+/** One normalized SmolSent row. */
+export interface SmolSentRow {
+    src: string;
+    trg: string;
+    sl: string;
+    tl: string;
+}
+/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
+ *  both sides or a side is implausibly large (a dump, not a sentence). */
+export declare function toSmolSentRow(row: unknown, maxChars?: number): SmolSentRow | null;
+/** Translate ONE SmolSent pair into SEMA facts. The two sentences are one
+ *  meaning in two languages, but the two BINDINGS are not equally sound —
+ *  SmolSent's English side is a shared pool translated into every language, so
+ *  `trg -> src` gives one English context a different answer in every language
+ *  file. See SMOLSENT_DIRECTIONS. refineItems drops the degenerate case where
+ *  src === trg. */
+export declare function smolSentRowToItems(row: SmolSentRow, dirs?: {
+    src2trg: boolean;
+    trg2src: boolean;
+}): TrainingItem[];
+export declare const smolsent: Corpus;