@hviana/sema 0.5.9 → 0.6.0

package/example/train_base/corpora/index.ts

@@ -0,0 +1,88 @@
+// train_base/corpora/index.ts — THE CURRICULUM.
+//
+// The order is the curriculum, and it is load-bearing: each stage runs only
+// after the previous one finishes, and every stage records itself in the same
+// completed-set, so a single store resumes the whole sequence.
+//
+//   1. SmolSent    sentence-level TRANSLATION pairs across 100+ low-resource
+//                  languages. Each pair is "two names for one meaning" → a
+//                  foreign→English translation FACT, so every language's
+//                  rendering of a meaning converges on ONE English node (cf.
+//                  test/05-concepts.test.mjs).
+//   2. Aya         ~204k human prompt→completion pairs, 70+ languages → one
+//                  (question → answer) FACT each.
+//   3. oasst2      MULTI-TURN human↔assistant conversation trees → the
+//                  accumulated-context walk (single-turn trees are skipped).
+//   4. Taskmaster  task-oriented DIALOGUE, the best-scoring corpora on the
+//                  fold-unit recurrence benchmark that predicts halo health.
+//   5. 2Wiki       the `evidences` TRIPLES — the one stage aimed at
+//                  COMPOSITION. Its Wikipedia passages and its composed
+//                  questions are deliberately NOT read.
+//   6. SODA        social/commonsense DIALOGUE, budgeted.
+//   7. MASSIVE     short intent utterances → ONE bare experience each.
+//                  DISABLED BY DEFAULT — edge-less content was measured to
+//                  manufacture answers where the store should stay silent.
+//   8. GenKnow     ~37.6k {Question, Answer} pairs → one FACT each. DISABLED
+//                  BY DEFAULT on licence grounds; see DATASETS.md §3.2.
+
+import type { Corpus } from "../corpus.js";
+import { smolsent } from "./smolsent.js";
+import { aya } from "./aya.js";
+import { oasst2 } from "./oasst2.js";
+import { taskmaster } from "./taskmaster.js";
+import { wiki2 } from "./wiki2.js";
+import { soda } from "./soda.js";
+import { massive } from "./massive.js";
+import { genknow } from "./genknow.js";
+
+export const CURRICULUM: Corpus[] = [
+  smolsent,
+  aya,
+  oasst2,
+  taskmaster,
+  wiki2,
+  soda,
+  massive,
+  genknow,
+];
+
+/** The corpora this RUN will train, for the panel header — derived, so it can
+ *  never go stale the way the hand-written string it replaces had. */
+export const enabledLabels = (): string =>
+  CURRICULUM.filter((c) => c.enabled).map((c) => c.label).join("+");
+
+/** The corpora a STORE contains: everything this run will train PLUS everything
+ *  an earlier run already did, in curriculum order.
+ *
+ *  Not the same question as `enabledLabels`, and conflating them wrote a false
+ *  statement into every store that was ever resumed with a different set of
+ *  stages enabled. Observed on a real store: `train.dataset` read
+ *  "SmolSent+Aya+oasst2" while the tally recorded 37,623 General-Knowledge
+ *  deposits sitting in it. That is not cosmetic — a Sema store retains its
+ *  training text VERBATIM, so `train.dataset` is the record of whose licence
+ *  terms travel with the artifact, and General-Knowledge is precisely the
+ *  corpus DATASETS.md §3.2 disables on NonCommercial grounds.
+ *
+ *  `trainedIds` are the corpus ids an earlier run deposited under (the keys of
+ *  the per-corpus tally). An id no longer in the curriculum cannot be named and
+ *  is dropped — the tally still carries it, which is where that evidence lives. */
+export const storedLabels = (trainedIds: Iterable<string>): string => {
+  const trained = new Set(trainedIds);
+  return CURRICULUM
+    .filter((c) => c.enabled || trained.has(c.id))
+    .map((c) => c.label)
+    .join("+");
+};
+
+// One re-export per corpus, carrying the descriptor AND its adapters. The
+// explicit `export { aya, genknow, … }` list that used to sit here named the
+// eight descriptors a second time; the star exports below already provide them,
+// and a hand-kept list of everything is exactly the thing that goes stale.
+export * from "./smolsent.js";
+export * from "./aya.js";
+export * from "./oasst2.js";
+export * from "./taskmaster.js";
+export * from "./wiki2.js";
+export * from "./soda.js";
+export * from "./massive.js";
+export * from "./genknow.js";

package/example/train_base/corpora/massive.ts

@@ -0,0 +1,111 @@
+// train_base/corpora/massive.ts — AmazonScience/massive short intents
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { parquet } from "../readers.js";
+import type { Corpus } from "../corpus.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+
+// MASSIVE deposits BARE UTTERANCES — an experience, not an episode — and that
+// is the only shape its data supports. Two richer shapes were considered and
+// rejected on evidence:
+//   • Same-intent pairs as paraphrases. 49.1% of consecutive rows share
+//     (locale, intent), but they are NOT meaning-equivalent: intent 48 in mn-MN
+//     runs "wake me at nine on the fifth" next to "set an alarm two hours from
+//     now". Depositing that pair as an episode teaches a continuation that does
+//     not exist.
+//   • Same-id rows across locales. Those ARE translations of one another —
+//     which is exactly SmolSent's relation, and SmolSent scores worst of every
+//     corpus measured on fold-unit recurrence (23.2%) because cross-lingual
+//     pairs share no units.
+// So the stage contributes recurring fold units and lexical coverage (65.1%
+// recurring unit mass, median 29 B) and nothing relational. `annot_utt` carries
+// slot markup ("[date : tavdahad] ...") and is never read.
+// DISABLED BY DEFAULT, on evidence gathered after the stage was written. A bare
+// experience deposits content with NO EDGE, and that cuts both ways. Measured on
+// a three-pair dialogue store with and without six MASSIVE-style utterances:
+//
+//   "set an alarm"            without: "Sure, what size would you like?"  (wrong)
+//                             with:    "set an alarm for seven"           (better)
+//   "play music"              without: ""                                 (correct silence)
+//                             with:    "Yes, sweetened or unsweetened?"   (wrong)
+//
+// So it displaces some wrong answers and manufactures others, INCLUDING turning
+// a correct silence into a wrong answer — and honest silence is a stated
+// property of this engine (AGENTS §2.13). On the mixed-curriculum store the
+// same shape produced the fragment "nus" for "wake me up at nine am".
+//
+// That evidence is four probes on toy stores and is NOT conclusive; it is,
+// however, the only evidence there is, and it points the wrong way. The stage
+// stays implemented and one env var away. Turn it on (MASSIVE=1) once there is
+// a real measurement showing the recurring fold units it contributes (72.3% of
+// deposited unit mass) buy more than the spurious answers cost.
+const MASSIVE = env("MASSIVE", "0") !== "0";
+const MASSIVE_DATASET = env("MASSIVE_DATASET", "AmazonScience/massive");
+// "all" is the config covering every locale in one set of shards.
+const MASSIVE_CONFIG = env("MASSIVE_CONFIG", "all");
+const MASSIVE_SPLITS = env("MASSIVE_SPLITS", "train")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// 0 = every row (587,214 in `all`/train, ~17 MB of content).
+const MASSIVE_MAX_ROWS = Math.max(
+  0,
+  Math.floor(Number(env("MASSIVE_MAX_ROWS", "0"))) || 0,
+);
+const MAX_MASSIVE_UTT_CHARS = Math.max(
+  100,
+  Math.floor(Number(env("MAX_MASSIVE_UTT_KB", "2")) * 1000) || 2_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e⁗  MASSIVE parsing — one short utterance → ONE SEMA experience
+//
+// See the constants note for why this deposits a bare experience and not a
+// relation: the two relational shapes this corpus appears to offer are both
+// false (same-intent rows are not paraphrases; same-id rows across locales are
+// translations, SmolSent's worst-scoring relation).
+// ═══════════════════════════════════════════════════════════════════════
+
+/** Translate ONE MASSIVE row into SEMA items: its bare utterance, as an
+ *  experience. `annot_utt` (slot-annotated) is deliberately not used — its
+ *  "[date : ...]" markup is not prose. Returns [] for an unusable row. */
+export function massiveRowToItems(
+  row: unknown,
+  maxChars = MAX_MASSIVE_UTT_CHARS,
+): TrainingItem[] {
+  if (!row || typeof row !== "object") return [];
+  const utt = (row as Record<string, unknown>).utt;
+  const text = typeof utt === "string" ? utt.trim() : "";
+  if (!text || text.length > maxChars) return [];
+  return refineItems([text]);
+}
+
+export const massive: Corpus = {
+  id: "massive",
+  label: "MASSIVE",
+  kind: "short intents",
+  enabled: MASSIVE,
+  maxRows: MASSIVE_MAX_ROWS,
+  // One of ten columns — `annot_utt`, whose slot markup the note above rejects,
+  // is not decoded either. Measured on the converted train shard: 92 MB
+  // uncompressed across all ten, 36 MB for `utt` (38.7%).
+  read: parquet({ columns: ["utt"] }),
+  toItems: (row) => {
+    const items = massiveRowToItems(row);
+    return items.length ? items : null;
+  },
+  unitNoun: "shard(s)",
+  log: { rows: "row(s)" },
+  discover: convertedParquetUnits({
+    id: "massive",
+    label: "MASSIVE",
+    dataset: MASSIVE_DATASET,
+    config: MASSIVE_CONFIG,
+    splits: MASSIVE_SPLITS,
+    localSub: "massive",
+  }),
+};

package/example/train_base/corpora/oasst2.ts

@@ -0,0 +1,163 @@
+// train_base/corpora/oasst2.ts — OpenAssistant/oasst2 conversation trees
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { accumulate, refineItems, type TrainingItem } from "../items.js";
+import { lines } from "../readers.js";
+import { type Corpus, singleUnit } from "../corpus.js";
+
+// ── OpenAssistant/oasst2 (the fourth training stage, after Aya) ──
+// oasst2 is a corpus of human↔assistant conversation TREES. Its richest, most
+// stream-friendly artifact is "<date>_oasst2_ready.trees.jsonl.gz": one JSON
+// conversation tree PER LINE, gzip-compressed (a web standard — Decompression
+// Stream("gzip")). Each tree is {message_tree_id, prompt:{role,text,replies:[…]}}
+// where `replies` nests recursively and a prompt can have several ranked
+// assistant replies (rank 0 = best). We follow the best-ranked, non-deleted
+// reply at each step to get ONE linear, strictly-alternating conversation per
+// tree, then keep only the MULTI-TURN ones (≥ OASST_MIN_TURNS messages, i.e. at
+// least two full user→assistant exchanges) — single Q→A trees are skipped, by
+// design. OASST=0 disables the stage; OASST_URL overrides the source.
+const OASST = env("OASST", "1") !== "0";
+const OASST_URL = env(
+  "OASST_URL",
+  "https://huggingface.co/datasets/OpenAssistant/oasst2/resolve/main/2023-11-05_oasst2_ready.trees.jsonl.gz",
+);
+// Multi-turn threshold: a conversation must have at least this many turns to be
+// trained (4 = user→assistant→user→assistant, the smallest real multi-turn).
+const OASST_MIN_TURNS = Math.max(
+  2,
+  Math.floor(Number(env("OASST_MIN_TURNS", "4"))) || 4,
+);
+// Skip a tree whose decoded JSON line exceeds this (a pathological record); the
+// real maximum is far smaller, so this only guards against corruption.
+const MAX_OASST_LINE_CHARS = Math.max(
+  100_000,
+  Math.floor(Number(env("MAX_OASST_LINE_MB", "8")) * 1_000_000) || 8_000_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e  OpenAssistant/oasst2 parsing — a conversation TREE → SEMA items
+//
+// Each tree is {prompt:{role,text,replies:[…]}}, replies nested recursively. A
+// prompt can have several ranked assistant replies; we collapse the tree to ONE
+// linear conversation by following the best-ranked (rank 0), non-deleted reply
+// at each step. The result strictly alternates prompter/assistant. Only MULTI-
+// TURN conversations (≥ OASST_MIN_TURNS messages) are kept — the explicit focus
+// of this stage; single Q→A trees are dropped.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** A single oasst2 message node (the fields we use; the tree nests via replies). */
+interface OasstNode {
+  role?: string;
+  text?: string;
+  rank?: number | null;
+  deleted?: boolean;
+  replies?: OasstNode[];
+}
+
+/** One conversational turn extracted from a tree. */
+export interface OasstTurn {
+  role: string; // "prompter" | "assistant"
+  text: string;
+}
+
+/** Collapse a conversation tree to ONE linear path: at each node, descend into
+ *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
+ *  Returns the ordered turns (already strictly alternating in this corpus). */
+export function bestOasstPath(root: OasstNode): OasstTurn[] {
+  const turns: OasstTurn[] = [];
+  let node: OasstNode | undefined = root;
+  while (node) {
+    const text = typeof node.text === "string" ? node.text.trim() : "";
+    if (text) turns.push({ role: String(node.role ?? "?"), text });
+    const live: OasstNode[] = (node.replies ?? []).filter((r: OasstNode) =>
+      r && !r.deleted && typeof r.text === "string" && r.text.trim() !== ""
+    );
+    if (live.length === 0) break;
+    live.sort((a: OasstNode, b: OasstNode) =>
+      (a.rank ?? Number.MAX_SAFE_INTEGER) - (b.rank ?? Number.MAX_SAFE_INTEGER)
+    );
+    node = live[0];
+  }
+  return turns;
+}
+
+/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
+ *
+ *  This is the ONE stage where cumulative continuous context is truly necessary:
+ *  the data is a real multi-turn dialogue, and what must be learned is how each
+ *  turn follows from the WHOLE conversation so far — not from the previous turn
+ *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
+ *  turn experiences and local adjacent-pair facts are NOT emitted (they are
+ *  subsumed by it and would merely replicate the content).
+ *
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
+ *  alternate by position in an oasst2 best-path (the root is a prompter), so a
+ *  label adds nothing the position does not, while a clean continuation matches
+ *  the test's recall (predictNext queries bare prior turns) and lets a turn share
+ *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
+ *
+ *  Returns [] for a conversation below the multi-turn threshold, so callers can
+ *  simply skip empties. */
+export function oasstConversationToItems(
+  turns: OasstTurn[],
+  minTurns = OASST_MIN_TURNS,
+): TrainingItem[] {
+  if (turns.length < minTurns) return []; // not multi-turn — skip
+  return refineItems(accumulate(turns.map((t) => t.text)));
+}
+
+/** The row adapter: ONE line of the tree dump → its deposits. Returns null for
+ *  a tree with no prompt and for every single-turn tree — the latter is the
+ *  stage's design, not a defect, which is why the reader counts it `unusable`
+ *  rather than `skipped`. */
+export function oasstTreeToItems(row: unknown): TrainingItem[] | null {
+  if (!row || typeof row !== "object") return null;
+  const tree = row as { prompt?: OasstNode };
+  if (!tree.prompt) return null;
+  const items = oasstConversationToItems(bestOasstPath(tree.prompt));
+  return items.length ? items : null;
+}
+
+export const oasst2: Corpus = {
+  id: "oasst2",
+  label: "oasst2",
+  kind: "multi-turn chat",
+  enabled: OASST,
+  read: lines({ gzip: true, maxLineChars: MAX_OASST_LINE_CHARS }),
+  toItems: oasstTreeToItems,
+  // The one corpus that KEEPS a cached file after a complete read: a copy left
+  // behind by a previous interrupted run is not this stage's to reclaim.
+  keepCached: true,
+  log: {
+    deposits: "examples",
+    // A tree that yields items is exactly a tree that cleared OASST_MIN_TURNS.
+    rows: "conversation(s)",
+    // The single-turn trees this stage drops BY DESIGN must not be reported as
+    // damage — only genuinely malformed lines are.
+    malformedOnly: true,
+  },
+  discover: singleUnit({
+    // Resume id "oasst2::trees" — the string this store already records.
+    key: "trees",
+    label: "oasst2",
+    display: "oasst2 (multi-turn)",
+    url: OASST_URL,
+    dest: "oasst2_ready.trees.jsonl.gz",
+    acquireLabel: "oasst2 trees",
+    localMatch: [/oasst.*trees.*\.jsonl\.gz$/i, /oasst.*\.jsonl\.gz$/i],
+    localWhat: "oasst2 *trees*.jsonl.gz",
+  }),
+};

package/example/train_base/corpora/smolsent.ts

@@ -0,0 +1,203 @@
+// train_base/corpora/smolsent.ts — google/smol sentence pairs
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env, LOCAL_PATH } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { hfTree, type Listed, localFiles } from "../discovery.js";
+import { lines } from "../readers.js";
+import type { Corpus, Unit } from "../corpus.js";
+import type { TrainCtx } from "../runtime.js";
+import { basename, join } from "node:path";
+
+// ── google/smol · SmolSent (the first training stage) ──
+// SmolSent is Google's sentence-level translation set: ~863 human sentence pairs
+// per language pair across 100+ low-resource languages, cc-by-4.0 (commercial-
+// friendly). Each row is {sl, tl, src, trg, …} — a source sentence and its
+// translation. A pair is "two names for one meaning", which is exactly the
+// cross-language concept SEMA fuses (see test/05-concepts.test.mjs), so each row
+// becomes FACTS that bind the two phrasings as one concept at recall time.
+//
+// The corpus ships as one plain JSONL file PER language pair under smolsent/ in
+// the HF repo (e.g. smolsent/ha_en.jsonl). We DOWNLOAD each file and stream its
+// lines — far faster and free of the rate-limiting that per-row API paging hit.
+// The file list is discovered from the HF repo tree. SMOLSENT=0 disables the
+// stage; SMOLSENT_PAIRS (comma-separated basenames without .jsonl, e.g.
+// "ha_en,zu_en") restricts to a chosen subset.
+const SMOLSENT = env("SMOLSENT", "1") !== "0";
+const SMOLSENT_DATASET = env("SMOLSENT_DATASET", "google/smol");
+const SMOLSENT_PAIRS = (process.env.SMOLSENT_PAIRS ?? "")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// The resume id PREFIX for the SmolSent stage; one completed-files entry per
+// file (e.g. "smolsent::ha_en.jsonl").
+const SMOLSENT_ID = "smolsent";
+// Which direction(s) of a translation pair to deposit. Was effectively "both",
+// and that is now the default NO longer, for a reason measured rather than
+// assumed.
+//
+// SmolSent's English side is a SHARED POOL translated into every language: row
+// id 0 of smolsent/ha_en.jsonl, zu_en.jsonl and am_en.jsonl all carry the SAME
+// `trg` ("It allows me to work by following my vibes and ..."). The two
+// directions are therefore not symmetric at all:
+//
+//   src2trg  (foreign -> English)  many distinct contexts -> ONE shared
+//                                  continuation. Every language's rendering of
+//                                  a meaning converges on the same English
+//                                  node — the cross-language concept fusion
+//                                  this stage exists for.
+//   trg2src  (English -> foreign)  ONE context -> 100+ DIFFERENT continuations,
+//                                  one per language file. The same English
+//                                  sentence is deposited over and over with a
+//                                  different answer each time.
+//
+// So dropping trg2src is not merely a corpus-size economy (it halves the
+// largest stage, which was 60.9% of all examples in the last trained store); it
+// removes a genuine ambiguity pathology. Set SMOLSENT_DIRECTIONS=both to
+// restore the old behaviour, or trg2src for English->foreign only.
+//
+// WHAT THE CUT DOES NOT DO, measured on a three-pair store: asking the English
+// sentence still ANSWERS with a foreign rendering, because the engine can reach
+// a shared continuation's predecessors on its own. What is removed is the
+// DEPOSITED forward ambiguity — one context carrying ~100 competing
+// continuations — not every reverse association.
+const SMOLSENT_DIRECTIONS = env("SMOLSENT_DIRECTIONS", "src2trg")
+  .trim().toLowerCase();
+const SMOLSENT_SRC2TRG = SMOLSENT_DIRECTIONS !== "trg2src";
+const SMOLSENT_TRG2SRC = SMOLSENT_DIRECTIONS === "trg2src" ||
+  SMOLSENT_DIRECTIONS === "both";
+// A SmolSent side longer than this is skipped (a sentence pair is short; a huge
+// value is corruption, not a sentence).
+const MAX_SMOLSENT_CHARS = Math.max(
+  2_000,
+  Math.floor(Number(env("MAX_SMOLSENT_KB", "16")) * 1000) || 16_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6c  SmolSent parsing — a translation pair → SEMA facts
+//
+// Each SmolSent row is {sl, tl, src, trg, …}: a source sentence and its
+// translation into another language — "two names for one meaning". This is the
+// cross-language concept SEMA fuses (see test/05-concepts.test.mjs: "ice" and
+// "hielo" become one concept because they share company, so a fact about one
+// transfers to the other). So a pair is rendered as BIDIRECTIONAL translation
+// FACTS — (src → trg) and (trg → src) — binding the two phrasings as one
+// concept at recall time, in both directions. Two facts, no experiences and no
+// cumulative walk: a sentence pair is not multi-turn, and a bare sentence on its
+// own carries no relation to point at.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One normalized SmolSent row. */
+export interface SmolSentRow {
+  src: string; // source sentence
+  trg: string; // its translation
+  sl: string; // source language code
+  tl: string; // target language code
+}
+
+/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
+ *  both sides or a side is implausibly large (a dump, not a sentence). */
+export function toSmolSentRow(
+  row: unknown,
+  maxChars = MAX_SMOLSENT_CHARS,
+): SmolSentRow | null {
+  if (!row || typeof row !== "object") return null;
+  const r = row as Record<string, unknown>;
+  const src = typeof r.src === "string" ? r.src.trim() : "";
+  // `trg` is a single string in smolsent; tolerate a list form defensively.
+  const trgRaw = Array.isArray(r.trgs) ? r.trgs[0] : r.trg;
+  const trg = typeof trgRaw === "string" ? trgRaw.trim() : "";
+  if (!src || !trg) return null;
+  if (
+    src.length > maxChars || trg.length > maxChars
+  ) return null;
+  const sl = typeof r.sl === "string" ? r.sl.trim() : "";
+  const tl = typeof r.tl === "string" ? r.tl.trim() : "";
+  return { src, trg, sl, tl };
+}
+
+/** Translate ONE SmolSent pair into SEMA facts. The two sentences are one
+ *  meaning in two languages, but the two BINDINGS are not equally sound —
+ *  SmolSent's English side is a shared pool translated into every language, so
+ *  `trg -> src` gives one English context a different answer in every language
+ *  file. See SMOLSENT_DIRECTIONS. refineItems drops the degenerate case where
+ *  src === trg. */
+export function smolSentRowToItems(
+  row: SmolSentRow,
+  dirs: { src2trg: boolean; trg2src: boolean } = {
+    src2trg: SMOLSENT_SRC2TRG,
+    trg2src: SMOLSENT_TRG2SRC,
+  },
+): TrainingItem[] {
+  const { src, trg } = row;
+  const items: TrainingItem[] = [];
+  if (dirs.src2trg) items.push({ context: src, continuation: trg });
+  if (dirs.trg2src) items.push({ context: trg, continuation: src });
+  return refineItems(items);
+}
+
+/** Discover the SmolSent per-pair JSONL files from the HF repo tree, restricted
+ *  to SMOLSENT_PAIRS (basenames without .jsonl) when set. Each entry is the
+ *  repo-relative path, e.g. "smolsent/ha_en.jsonl". */
+async function listFiles(ctx: TrainCtx): Promise<Listed[]> {
+  const paths = await hfTree(
+    SMOLSENT_DATASET,
+    "smolsent",
+    /\.jsonl$/i,
+    `GET smol tree`,
+    ctx.http,
+  );
+  if (!SMOLSENT_PAIRS.length) return paths;
+  const want = new Set(SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")));
+  return paths.filter((p) =>
+    want.has(basename(p.path).replace(/\.jsonl$/i, ""))
+  );
+}
+
+const unit = (name: string): Unit => ({
+  key: name,
+  // The log names a pair by its language code alone; the panel keeps the file.
+  name: name.replace(/\.jsonl$/i, ""),
+  display: `SmolSent ${name}`,
+});
+
+export const smolsent: Corpus = {
+  id: SMOLSENT_ID,
+  label: "SmolSent",
+  kind: "translation",
+  enabled: SMOLSENT,
+  unitNoun: "translation file(s)",
+  // The *4 is chars-to-a-generous-JSON-envelope: the guard bounds ONE side of
+  // a pair, the line carries both plus its keys.
+  read: lines({ maxLineChars: MAX_SMOLSENT_CHARS * 4 }),
+  toItems: (row) => {
+    const r = toSmolSentRow(row);
+    return r ? smolSentRowToItems(r) : null;
+  },
+  async discover(ctx) {
+    // Work-list: local *.jsonl in LOCAL_PATH, else the repo's smolsent/ files.
+    if (LOCAL_PATH) {
+      let names = localFiles(LOCAL_PATH, /\.jsonl$/i);
+      if (SMOLSENT_PAIRS.length) {
+        const want = new Set(
+          SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")),
+        );
+        names = names.filter((n) => want.has(n.path.replace(/\.jsonl$/i, "")));
+      }
+      return names.map((n) => ({
+        ...unit(n.path),
+        local: join(LOCAL_PATH, n.path),
+        bytes: n.size,
+      }));
+    }
+    return (await listFiles(ctx)).map(({ path, size }) => ({
+      ...unit(basename(path)),
+      // owner/name and the file path are URL PATH segments — do not encode "/".
+      url:
+        `https://huggingface.co/datasets/${SMOLSENT_DATASET}/resolve/main/${path}`,
+      bytes: size,
+    }));
+  },
+};