@hviana/sema 0.5.8 → 0.6.0

package/dist/example/train_base/corpora/smolsent.js

@@ -0,0 +1,156 @@
+// train_base/corpora/smolsent.ts — google/smol sentence pairs
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env, LOCAL_PATH } from "../config.js";
+import { refineItems } from "../items.js";
+import { hfTree, localFiles } from "../discovery.js";
+import { lines } from "../readers.js";
+import { basename, join } from "node:path";
+// ── google/smol · SmolSent (the first training stage) ──
+// SmolSent is Google's sentence-level translation set: ~863 human sentence pairs
+// per language pair across 100+ low-resource languages, cc-by-4.0 (commercial-
+// friendly). Each row is {sl, tl, src, trg, …} — a source sentence and its
+// translation. A pair is "two names for one meaning", which is exactly the
+// cross-language concept SEMA fuses (see test/05-concepts.test.mjs), so each row
+// becomes FACTS that bind the two phrasings as one concept at recall time.
+//
+// The corpus ships as one plain JSONL file PER language pair under smolsent/ in
+// the HF repo (e.g. smolsent/ha_en.jsonl). We DOWNLOAD each file and stream its
+// lines — far faster and free of the rate-limiting that per-row API paging hit.
+// The file list is discovered from the HF repo tree. SMOLSENT=0 disables the
+// stage; SMOLSENT_PAIRS (comma-separated basenames without .jsonl, e.g.
+// "ha_en,zu_en") restricts to a chosen subset.
+const SMOLSENT = env("SMOLSENT", "1") !== "0";
+const SMOLSENT_DATASET = env("SMOLSENT_DATASET", "google/smol");
+const SMOLSENT_PAIRS = (process.env.SMOLSENT_PAIRS ?? "")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// The resume id PREFIX for the SmolSent stage; one completed-files entry per
+// file (e.g. "smolsent::ha_en.jsonl").
+const SMOLSENT_ID = "smolsent";
+// Which direction(s) of a translation pair to deposit. Was effectively "both",
+// and that is now the default NO longer, for a reason measured rather than
+// assumed.
+//
+// SmolSent's English side is a SHARED POOL translated into every language: row
+// id 0 of smolsent/ha_en.jsonl, zu_en.jsonl and am_en.jsonl all carry the SAME
+// `trg` ("It allows me to work by following my vibes and ..."). The two
+// directions are therefore not symmetric at all:
+//
+//   src2trg  (foreign -> English)  many distinct contexts -> ONE shared
+//                                  continuation. Every language's rendering of
+//                                  a meaning converges on the same English
+//                                  node — the cross-language concept fusion
+//                                  this stage exists for.
+//   trg2src  (English -> foreign)  ONE context -> 100+ DIFFERENT continuations,
+//                                  one per language file. The same English
+//                                  sentence is deposited over and over with a
+//                                  different answer each time.
+//
+// So dropping trg2src is not merely a corpus-size economy (it halves the
+// largest stage, which was 60.9% of all examples in the last trained store); it
+// removes a genuine ambiguity pathology. Set SMOLSENT_DIRECTIONS=both to
+// restore the old behaviour, or trg2src for English->foreign only.
+//
+// WHAT THE CUT DOES NOT DO, measured on a three-pair store: asking the English
+// sentence still ANSWERS with a foreign rendering, because the engine can reach
+// a shared continuation's predecessors on its own. What is removed is the
+// DEPOSITED forward ambiguity — one context carrying ~100 competing
+// continuations — not every reverse association.
+const SMOLSENT_DIRECTIONS = env("SMOLSENT_DIRECTIONS", "src2trg")
+    .trim().toLowerCase();
+const SMOLSENT_SRC2TRG = SMOLSENT_DIRECTIONS !== "trg2src";
+const SMOLSENT_TRG2SRC = SMOLSENT_DIRECTIONS === "trg2src" ||
+    SMOLSENT_DIRECTIONS === "both";
+// A SmolSent side longer than this is skipped (a sentence pair is short; a huge
+// value is corruption, not a sentence).
+const MAX_SMOLSENT_CHARS = Math.max(2_000, Math.floor(Number(env("MAX_SMOLSENT_KB", "16")) * 1000) || 16_000);
+/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
+ *  both sides or a side is implausibly large (a dump, not a sentence). */
+export function toSmolSentRow(row, maxChars = MAX_SMOLSENT_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    const src = typeof r.src === "string" ? r.src.trim() : "";
+    // `trg` is a single string in smolsent; tolerate a list form defensively.
+    const trgRaw = Array.isArray(r.trgs) ? r.trgs[0] : r.trg;
+    const trg = typeof trgRaw === "string" ? trgRaw.trim() : "";
+    if (!src || !trg)
+        return null;
+    if (src.length > maxChars || trg.length > maxChars)
+        return null;
+    const sl = typeof r.sl === "string" ? r.sl.trim() : "";
+    const tl = typeof r.tl === "string" ? r.tl.trim() : "";
+    return { src, trg, sl, tl };
+}
+/** Translate ONE SmolSent pair into SEMA facts. The two sentences are one
+ *  meaning in two languages, but the two BINDINGS are not equally sound —
+ *  SmolSent's English side is a shared pool translated into every language, so
+ *  `trg -> src` gives one English context a different answer in every language
+ *  file. See SMOLSENT_DIRECTIONS. refineItems drops the degenerate case where
+ *  src === trg. */
+export function smolSentRowToItems(row, dirs = {
+    src2trg: SMOLSENT_SRC2TRG,
+    trg2src: SMOLSENT_TRG2SRC,
+}) {
+    const { src, trg } = row;
+    const items = [];
+    if (dirs.src2trg)
+        items.push({ context: src, continuation: trg });
+    if (dirs.trg2src)
+        items.push({ context: trg, continuation: src });
+    return refineItems(items);
+}
+/** Discover the SmolSent per-pair JSONL files from the HF repo tree, restricted
+ *  to SMOLSENT_PAIRS (basenames without .jsonl) when set. Each entry is the
+ *  repo-relative path, e.g. "smolsent/ha_en.jsonl". */
+async function listFiles(ctx) {
+    const paths = await hfTree(SMOLSENT_DATASET, "smolsent", /\.jsonl$/i, `GET smol tree`, ctx.http);
+    if (!SMOLSENT_PAIRS.length)
+        return paths;
+    const want = new Set(SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")));
+    return paths.filter((p) => want.has(basename(p.path).replace(/\.jsonl$/i, "")));
+}
+const unit = (name) => ({
+    key: name,
+    // The log names a pair by its language code alone; the panel keeps the file.
+    name: name.replace(/\.jsonl$/i, ""),
+    display: `SmolSent ${name}`,
+});
+export const smolsent = {
+    id: SMOLSENT_ID,
+    label: "SmolSent",
+    kind: "translation",
+    enabled: SMOLSENT,
+    unitNoun: "translation file(s)",
+    // The *4 is chars-to-a-generous-JSON-envelope: the guard bounds ONE side of
+    // a pair, the line carries both plus its keys.
+    read: lines({ maxLineChars: MAX_SMOLSENT_CHARS * 4 }),
+    toItems: (row) => {
+        const r = toSmolSentRow(row);
+        return r ? smolSentRowToItems(r) : null;
+    },
+    async discover(ctx) {
+        // Work-list: local *.jsonl in LOCAL_PATH, else the repo's smolsent/ files.
+        if (LOCAL_PATH) {
+            let names = localFiles(LOCAL_PATH, /\.jsonl$/i);
+            if (SMOLSENT_PAIRS.length) {
+                const want = new Set(SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")));
+                names = names.filter((n) => want.has(n.path.replace(/\.jsonl$/i, "")));
+            }
+            return names.map((n) => ({
+                ...unit(n.path),
+                local: join(LOCAL_PATH, n.path),
+                bytes: n.size,
+            }));
+        }
+        return (await listFiles(ctx)).map(({ path, size }) => ({
+            ...unit(basename(path)),
+            // owner/name and the file path are URL PATH segments — do not encode "/".
+            url: `https://huggingface.co/datasets/${SMOLSENT_DATASET}/resolve/main/${path}`,
+            bytes: size,
+        }));
+    },
+};

package/dist/example/train_base/corpora/soda.d.ts

@@ -0,0 +1,12 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+import type { TaskmasterTurn } from "./taskmaster.js";
+/** Normalize a SODA row into its turns, or null when it carries no usable
+ *  dialogue. Speakers are optional (they only drive merging); an implausibly
+ *  long turn rejects the dialogue as corrupt. */
+export declare function toSodaTurns(row: unknown, maxChars?: number): TaskmasterTurn[] | null;
+/** Translate ONE SODA dialogue into SEMA items: the cumulative walk over its
+ *  speaker-merged turns. Shares `mergeSpeakerTurns` because the rule is the
+ *  same one — consecutive turns by one speaker are one contribution. */
+export declare function sodaDialogueToItems(turns: TaskmasterTurn[]): TrainingItem[];
+export declare const soda: Corpus;

package/dist/example/train_base/corpora/soda.js

@@ -0,0 +1,113 @@
+// train_base/corpora/soda.ts — allenai/soda social dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { accumulate, mergeSpeakerTurns, refineItems, } from "../items.js";
+import { parquet } from "../readers.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// ── allenai/soda (social dialogue) and AmazonScience/massive (short intents) ──
+// Both are read from Hugging Face's auto-converted `refs/convert/parquet`
+// branch. For SODA that is mandatory, not cosmetic: its main-branch
+// train.parquet is ONE 1,191,582-row group (1.19 GB uncompressed), and a
+// Parquet column chunk is per-group, so any read of it materialises the whole
+// file — measured at 100% of a 689 MB file and 2 GB of heap for a 500-row read.
+// The converted branch uses uniform 10,000-row groups.
+//
+// BOTH STAGES ARE BUDGETED, and that is a curriculum decision rather than an
+// algorithmic cap. SODA's train split holds 1,191,582 dialogues which the
+// cumulative walk would turn into ~8 MILLION episodes — against the 662,221
+// deposits of the entire current corpus. Trained whole it would not join the
+// mix, it would BE the mix, and corpus size is the quantity every scale problem
+// in this engine is measured against. The default takes the first
+// SODA_MAX_DIALOGS of them; set it to 0 to lift the budget.
+const SODA = env("SODA", "1") !== "0";
+const SODA_DATASET = env("SODA_DATASET", "allenai/soda");
+const SODA_SPLITS = env("SODA_SPLITS", "train")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// ~6.3 episodes per dialogue, so this budgets ~750k episodes — comparable to
+// the Taskmaster stage and to Aya, which is the intended balance. 0 = no budget.
+const SODA_MAX_DIALOGS = Math.max(0, Math.floor(Number(env("SODA_MAX_DIALOGS", "120000"))) || 0);
+const MAX_SODA_TURN_CHARS = Math.max(1_000, Math.floor(Number(env("MAX_SODA_TURN_KB", "32")) * 1000) || 32_000);
+// ═══════════════════════════════════════════════════════════════════════
+// §6e‴  SODA parsing — a social dialogue row → SEMA items
+//
+// Each row carries `dialogue` (an array of turn strings) and `speakers` (the
+// speaker name per turn). The deposit is the cumulative walk over speaker-merged
+// turns, identical in shape to Taskmaster and oasst2 — turns are short (mean
+// 87 B) and dialogues average 7.3 turns, so the accumulated context stays well
+// inside the healthy range.
+//
+// `narrative`, `literal` and the ATOMIC-style `head`/`relation`/`tail` columns
+// are NOT deposited: they are the generation scaffolding SODA was distilled
+// from, they restate the dialogue in the third person, and depositing both a
+// dialogue and its paraphrased summary gives one meaning two shapes — which is
+// measured to SUPPRESS composition rather than help it.
+// ═══════════════════════════════════════════════════════════════════════
+/** Normalize a SODA row into its turns, or null when it carries no usable
+ *  dialogue. Speakers are optional (they only drive merging); an implausibly
+ *  long turn rejects the dialogue as corrupt. */
+export function toSodaTurns(row, maxChars = MAX_SODA_TURN_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const r = row;
+    const dialogue = r.dialogue;
+    if (!Array.isArray(dialogue))
+        return null;
+    const speakers = Array.isArray(r.speakers) ? r.speakers : [];
+    const turns = [];
+    for (let i = 0; i < dialogue.length; i++) {
+        const text = typeof dialogue[i] === "string"
+            ? dialogue[i].trim()
+            : "";
+        if (!text)
+            continue;
+        if (text.length > maxChars)
+            return null;
+        turns.push({
+            speaker: String(speakers[i] ?? "").trim().toUpperCase(),
+            text,
+        });
+    }
+    return turns.length ? turns : null;
+}
+/** Translate ONE SODA dialogue into SEMA items: the cumulative walk over its
+ *  speaker-merged turns. Shares `mergeSpeakerTurns` because the rule is the
+ *  same one — consecutive turns by one speaker are one contribution. */
+export function sodaDialogueToItems(turns) {
+    const texts = mergeSpeakerTurns(turns);
+    if (texts.length < 2)
+        return []; // not an exchange
+    return refineItems(accumulate(texts));
+}
+export const soda = {
+    id: "soda",
+    label: "SODA",
+    kind: "social dialogue",
+    enabled: SODA,
+    maxRows: SODA_MAX_DIALOGS,
+    // Two of sixteen columns. The `narrative`/`literal`/`head`/`relation`/`tail`
+    // scaffolding the note above declines to deposit is now also never decoded:
+    // measured on the converted train shard, 449 MB uncompressed across all
+    // sixteen against 306 MB for these two (68.1%).
+    read: parquet({ columns: ["dialogue", "speakers"] }),
+    toItems: (row) => {
+        const turns = toSodaTurns(row);
+        if (!turns)
+            return null;
+        const items = sodaDialogueToItems(turns);
+        return items.length ? items : null;
+    },
+    unitNoun: "shard(s)",
+    log: { rows: "row(s)" },
+    discover: convertedParquetUnits({
+        id: "soda",
+        label: "SODA",
+        dataset: SODA_DATASET,
+        config: "default",
+        splits: SODA_SPLITS,
+        localSub: "soda",
+    }),
+};

package/dist/example/train_base/corpora/taskmaster.d.ts

@@ -0,0 +1,15 @@
+import { type SpeakerTurn, type TrainingItem } from "../items.js";
+import { type Corpus } from "../corpus.js";
+/** One utterance of a Taskmaster conversation — the shared dialogue-turn shape,
+ *  under the name this corpus's adapters have always used. */
+export type TaskmasterTurn = SpeakerTurn;
+/** Normalize ONE element of a Taskmaster data file into its turns, or null when
+ *  it carries no usable utterance. Empty/whitespace-only utterances are dropped
+ *  (TM-3 has a few); a single implausibly long utterance rejects the whole
+ *  conversation as corrupt rather than depositing a dump. */
+export declare function toTaskmasterTurns(row: unknown, maxChars?: number): TaskmasterTurn[] | null;
+/** Translate ONE Taskmaster conversation into SEMA training items: the
+ *  cumulative walk over its merged turns. Returns [] for a conversation below
+ *  TASKMASTER_MIN_TURNS, so callers can simply skip empties. */
+export declare function taskmasterConversationToItems(turns: TaskmasterTurn[], minTurns?: number): TrainingItem[];
+export declare const taskmaster: Corpus;

package/dist/example/train_base/corpora/taskmaster.js

@@ -0,0 +1,144 @@
+// train_base/corpora/taskmaster.ts — google-research-datasets/Taskmaster 1–4 dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env, LOCAL_PATH } from "../config.js";
+import { accumulate, mergeSpeakerTurns, refineItems, } from "../items.js";
+import { githubContents, localFiles } from "../discovery.js";
+import { jsonArray } from "../readers.js";
+import { localDir } from "../corpus.js";
+import { DIM, R } from "../ui.js";
+import { basename, join } from "node:path";
+// ── google-research-datasets/Taskmaster 1–4 (the dialogue stages) ──
+// Four corpora of task-oriented dialogue, one shape between them: each file is a
+// JSON ARRAY of conversations and each conversation carries
+// `utterances: [{speaker, text, …}]`. TM-1 ships two files directly under its
+// directory (self-dialogs, woz-dialogs); TM-2/3/4 ship theirs under `<set>/data`.
+// They are the best-scoring corpora on the fold-unit recurrence benchmark that
+// selects for halo health (TM-3 85.1%, TM-4 78.8%, TM-2 68.7%, TM-1 51.8%,
+// against 23.2% for the incumbent SmolSent), and they are genuinely multi-turn
+// where the incumbent multi-turn stage is not (TM-3 median 20 turns of ~43 B,
+// against oasst2's median turn of 529 B).
+//
+// Served from GitHub raw, not Hugging Face: the HF mirrors are loading-script
+// repos with no data files, and the official copies carry the CC BY 4.0 notice.
+const TASKMASTER = env("TASKMASTER", "1") !== "0";
+// Which sets to train, in order. Each is a directory in the Taskmaster repo.
+const TASKMASTER_SETS = env("TASKMASTER_SETS", "TM-1-2019,TM-2-2020,TM-3-2020,TM-4-2024").split(",").map((s) => s.trim()).filter(Boolean);
+const TASKMASTER_REPO = env("TASKMASTER_REPO", "google-research-datasets/Taskmaster");
+const TASKMASTER_RAW = `https://raw.githubusercontent.com/${TASKMASTER_REPO}/master`;
+// A conversation must have at least this many turns AFTER same-speaker merging.
+// The default of 2 keeps every real exchange: unlike oasst2 — where a lone Q→A
+// tree merely replicates the Aya stage's shape and is dropped — a two-turn
+// task-oriented exchange is still task-oriented dialogue, and TM-4's dialogues
+// are short by design (median 3.7 turns), so a higher bar would discard most of
+// that set.
+const TASKMASTER_MIN_TURNS = Math.max(2, Math.floor(Number(env("TASKMASTER_MIN_TURNS", "2"))) || 2);
+// Skip a conversation carrying an implausibly long utterance (corruption). The
+// measured maximum across TM-1/2/3/4 is 1,897 bytes, so this only guards.
+const MAX_TASKMASTER_TURN_CHARS = Math.max(1_000, Math.floor(Number(env("MAX_TASKMASTER_TURN_KB", "32")) * 1000) || 32_000);
+/** Normalize ONE element of a Taskmaster data file into its turns, or null when
+ *  it carries no usable utterance. Empty/whitespace-only utterances are dropped
+ *  (TM-3 has a few); a single implausibly long utterance rejects the whole
+ *  conversation as corrupt rather than depositing a dump. */
+export function toTaskmasterTurns(row, maxChars = MAX_TASKMASTER_TURN_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const utterances = row.utterances;
+    if (!Array.isArray(utterances))
+        return null;
+    const turns = [];
+    for (const u of utterances) {
+        if (!u || typeof u !== "object")
+            continue;
+        const r = u;
+        const text = typeof r.text === "string" ? r.text.trim() : "";
+        if (!text)
+            continue;
+        if (text.length > maxChars)
+            return null;
+        turns.push({
+            speaker: String(r.speaker ?? "").trim().toUpperCase(),
+            text,
+        });
+    }
+    return turns.length ? turns : null;
+}
+/** Translate ONE Taskmaster conversation into SEMA training items: the
+ *  cumulative walk over its merged turns. Returns [] for a conversation below
+ *  TASKMASTER_MIN_TURNS, so callers can simply skip empties. */
+export function taskmasterConversationToItems(turns, minTurns = TASKMASTER_MIN_TURNS) {
+    const texts = mergeSpeakerTurns(turns);
+    if (texts.length < minTurns)
+        return [];
+    return refineItems(accumulate(texts));
+}
+/** List the Taskmaster data files to train, in TASKMASTER_SETS order. Returns
+ *  repo-relative paths, e.g. "TM-3-2020/data/data_00.json".
+ *
+ *  TM-2/3/4 keep their dialogue files under `<set>/data`, so everything there is
+ *  fair game. TM-1 has no `data` directory: its two dialogue files sit at the
+ *  set root NEXT TO `ontology.json` (a slot schema) and `sample.json` (a small
+ *  excerpt of self-dialogs). Neither is an array of conversations, and training
+ *  the excerpt would deposit a subset of TM-1 twice, so TM-1 is filtered to the
+ *  `*-dialogs.json` pair (self-dialogs, woz-dialogs). */
+async function listFiles(ctx) {
+    const out = [];
+    for (const set of TASKMASTER_SETS) {
+        const rootOnly = /^TM-1\b/i.test(set);
+        const dir = rootOnly ? set : `${set}/data`;
+        const names = await githubContents(TASKMASTER_REPO, dir, /\.json$/i, `GET Taskmaster ${dir}`, ctx.http);
+        for (const { path: name, size } of names) {
+            if (rootOnly && !/-dialogs\.json$/i.test(name))
+                continue;
+            out.push({ set, path: `${dir}/${name}`, size });
+        }
+    }
+    return out;
+}
+const unit = (key, name) => ({
+    key,
+    name,
+    display: `Taskmaster ${name}`,
+});
+export const taskmaster = {
+    id: "taskmaster",
+    label: "Taskmaster",
+    kind: "task dialogue",
+    enabled: TASKMASTER,
+    unitNoun: "dialogue file(s)",
+    read: jsonArray(),
+    toItems: (row) => {
+        const turns = toTaskmasterTurns(row);
+        if (!turns)
+            return null;
+        const items = taskmasterConversationToItems(turns); // [] when too short
+        return items.length ? items : null;
+    },
+    log: { bad: "unusable conversation(s)" },
+    async discover(ctx) {
+        // LOCAL_PATH/taskmaster/ — a subdirectory, because these share the .json
+        // extension with the General-Knowledge source and must not be confused
+        // with it.
+        if (LOCAL_PATH) {
+            const dir = localDir("taskmaster");
+            const names = localFiles(dir, /\.json$/i);
+            if (names.length === 0) {
+                ctx.progress.log(`  ${DIM}· no Taskmaster *.json in ${dir} — skipping${R}`);
+                return null;
+            }
+            return names.map((n) => ({
+                ...unit(n.path, n.path),
+                local: join(dir, n.path),
+                bytes: n.size,
+            }));
+        }
+        return (await listFiles(ctx)).map((f) => ({
+            ...unit(f.path, `${f.set}/${basename(f.path)}`),
+            url: `${TASKMASTER_RAW}/${f.path}`,
+            bytes: f.size,
+        }));
+    },
+};

package/dist/example/train_base/corpora/wiki2.d.ts

@@ -0,0 +1,23 @@
+import { type TrainingItem } from "../items.js";
+import type { Corpus } from "../corpus.js";
+/** One (subject, relation, object) triple from a 2Wiki `evidences` cell. */
+export interface WikiTriple {
+    subject: string;
+    relation: string;
+    object: string;
+}
+/** Normalize a 2Wiki row into its evidence triples, or null when it carries
+ *  none usable. `evidences` is a JSON STRING holding an array of 3-element
+ *  arrays; a row whose cell is absent, unparseable, or empty yields null.
+ *  Individual malformed or oversized triples are dropped without discarding the
+ *  row — one bad triple should not cost the others. */
+export declare function toWikiTriples(row: unknown, maxChars?: number): WikiTriple[] | null;
+/** Render ONE triple as the prose fact Sema stores. Kept separate so the two
+ *  deposits below are guaranteed to share a byte-identical continuation: the
+ *  pivot fact only works if it leads to the SAME node the relation fact does. */
+export declare function wikiTripleSentence(t: WikiTriple): string;
+/** Translate a row's triples into SEMA items: per triple, the relation fact and
+ *  the bare-subject PIVOT fact (see the section note above). refineItems drops
+ *  the duplicates this produces when a row states the same triple twice. */
+export declare function wikiTriplesToItems(triples: WikiTriple[]): TrainingItem[];
+export declare const wiki2: Corpus;

package/dist/example/train_base/corpora/wiki2.js

@@ -0,0 +1,132 @@
+// train_base/corpora/wiki2.ts — 2WikiMultihopQA evidence triples
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+import { env } from "../config.js";
+import { refineItems } from "../items.js";
+import { parquet } from "../readers.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// ── 2WikiMultihopQA — the `evidences` TRIPLES only (the composition stage) ──
+// Each row carries `evidences`: a JSON string of (subject, relation, object)
+// triples that CHAIN — one triple's object is the next's subject. 72.5% of rows
+// carry such a chain (measured over 4,000 rows), and those triples are the only
+// representation measured to make Sema compose a two-hop answer at all.
+//
+// TWO COLUMNS ARE DELIBERATELY NOT READ, one for licence reasons and one for
+// capability reasons:
+//   • `context` holds Wikipedia PROSE. The repo is Apache-2.0 but Wikipedia text
+//     is CC BY-SA, and a Sema store keeps text verbatim, so ingesting the
+//     passages would attach ShareAlike to every distributed store. The triples
+//     originate in Wikidata (CC0). See DATASETS.md §3.2/§4.
+//   • `question`/`answer` are the composed multi-hop QUESTION. Depositing those
+//     teaches the answer to that exact question and nothing else — it memorises
+//     rather than composes. They are used to EVALUATE this adapter, never as
+//     training input.
+//
+// Read from Hugging Face's auto-converted `refs/convert/parquet` branch, not
+// from main: the main-branch train.parquet is written as ONE 167,454-row
+// group (666 MB uncompressed) and a Parquet column chunk is per-group, so any
+// read of it materialises the whole file. The converted branch uses uniform
+// 10,000-row groups, which `parquetBatchRows` then subdivides by BYTES.
+const WIKI2 = env("WIKI2", "1") !== "0";
+const WIKI2_DATASET = env("WIKI2_DATASET", "xanhho/2WikiMultihopQA");
+// Splits to train, in order. Only `train` by default: `validation`/`test` are
+// the dataset's held-out sets and are what an honest evaluation of this
+// adapter's composition rate has to be measured on.
+const WIKI2_SPLITS = env("WIKI2_SPLITS", "train")
+    .split(",").map((s) => s.trim()).filter(Boolean);
+// Reject a triple with an implausibly long field (corruption); real subjects and
+// objects are entity names, and relations are Wikidata property labels.
+// 0 = every row. The train split holds 167,454 rows at ~4.95 deposits each
+// (~830k facts), so this is the knob that keeps 2Wiki proportionate to the rest
+// of the curriculum in the same way SODA_MAX_DIALOGS does.
+const WIKI2_MAX_ROWS = Math.max(0, Math.floor(Number(env("WIKI2_MAX_ROWS", "0"))) || 0);
+const MAX_WIKI2_FIELD_CHARS = Math.max(100, Math.floor(Number(env("MAX_WIKI2_FIELD_KB", "2")) * 1000) || 2_000);
+/** Normalize a 2Wiki row into its evidence triples, or null when it carries
+ *  none usable. `evidences` is a JSON STRING holding an array of 3-element
+ *  arrays; a row whose cell is absent, unparseable, or empty yields null.
+ *  Individual malformed or oversized triples are dropped without discarding the
+ *  row — one bad triple should not cost the others. */
+export function toWikiTriples(row, maxChars = MAX_WIKI2_FIELD_CHARS) {
+    if (!row || typeof row !== "object")
+        return null;
+    const cell = row.evidences;
+    let parsed = cell;
+    if (typeof cell === "string") {
+        try {
+            parsed = JSON.parse(cell);
+        }
+        catch {
+            return null;
+        }
+    }
+    if (!Array.isArray(parsed))
+        return null;
+    const out = [];
+    for (const e of parsed) {
+        if (!Array.isArray(e) || e.length < 3)
+            continue;
+        const subject = typeof e[0] === "string" ? e[0].trim() : "";
+        const relation = typeof e[1] === "string" ? e[1].trim() : "";
+        const object = typeof e[2] === "string" ? e[2].trim() : "";
+        if (!subject || !relation || !object)
+            continue;
+        if (subject.length > maxChars ||
+            relation.length > maxChars ||
+            object.length > maxChars)
+            continue;
+        out.push({ subject, relation, object });
+    }
+    return out.length ? out : null;
+}
+/** Render ONE triple as the prose fact Sema stores. Kept separate so the two
+ *  deposits below are guaranteed to share a byte-identical continuation: the
+ *  pivot fact only works if it leads to the SAME node the relation fact does. */
+export function wikiTripleSentence(t) {
+    return `The ${t.relation} of ${t.subject} is ${t.object}.`;
+}
+/** Translate a row's triples into SEMA items: per triple, the relation fact and
+ *  the bare-subject PIVOT fact (see the section note above). refineItems drops
+ *  the duplicates this produces when a row states the same triple twice. */
+export function wikiTriplesToItems(triples) {
+    const items = [];
+    for (const t of triples) {
+        const fact = wikiTripleSentence(t);
+        items.push({ context: `${t.subject} ${t.relation}`, continuation: fact });
+        items.push({ context: t.subject, continuation: fact });
+    }
+    return refineItems(items);
+}
+export const wiki2 = {
+    id: "2wiki",
+    label: "2Wiki",
+    kind: "relation triples",
+    enabled: WIKI2,
+    maxRows: WIKI2_MAX_ROWS,
+    // Read ONE of the file's seven columns. This is where the "context is not
+    // read" rule above stops being a property of the adapter and becomes a
+    // property of the read: the Wikipedia prose is never decoded at all. It is
+    // also almost the whole file — measured on the converted train shard, the
+    // seven columns hold 518 MB uncompressed and `evidences` is 18 MB of it
+    // (3.5%), so the projection cuts the decode work by ~28x.
+    read: parquet({ columns: ["evidences"] }),
+    toItems: (row) => {
+        const triples = toWikiTriples(row);
+        if (!triples)
+            return null;
+        const items = wikiTriplesToItems(triples);
+        return items.length ? items : null;
+    },
+    unitNoun: "shard(s)",
+    log: { rows: "row(s)" },
+    discover: convertedParquetUnits({
+        id: "2wiki",
+        label: "2Wiki",
+        dataset: WIKI2_DATASET,
+        config: "default",
+        splits: WIKI2_SPLITS,
+        localSub: "2wiki",
+    }),
+};