@hviana/sema 0.5.9 → 0.6.0

package/example/train_base/corpora/soda.ts

@@ -0,0 +1,130 @@
+// train_base/corpora/soda.ts — allenai/soda social dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import {
+  accumulate,
+  mergeSpeakerTurns,
+  refineItems,
+  type TrainingItem,
+} from "../items.js";
+import { parquet } from "../readers.js";
+import type { Corpus } from "../corpus.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// SODA turns are the same shape as Taskmaster's, and merge by the same rule.
+import type { TaskmasterTurn } from "./taskmaster.js";
+
+// ── allenai/soda (social dialogue) and AmazonScience/massive (short intents) ──
+// Both are read from Hugging Face's auto-converted `refs/convert/parquet`
+// branch. For SODA that is mandatory, not cosmetic: its main-branch
+// train.parquet is ONE 1,191,582-row group (1.19 GB uncompressed), and a
+// Parquet column chunk is per-group, so any read of it materialises the whole
+// file — measured at 100% of a 689 MB file and 2 GB of heap for a 500-row read.
+// The converted branch uses uniform 10,000-row groups.
+//
+// BOTH STAGES ARE BUDGETED, and that is a curriculum decision rather than an
+// algorithmic cap. SODA's train split holds 1,191,582 dialogues which the
+// cumulative walk would turn into ~8 MILLION episodes — against the 662,221
+// deposits of the entire current corpus. Trained whole it would not join the
+// mix, it would BE the mix, and corpus size is the quantity every scale problem
+// in this engine is measured against. The default takes the first
+// SODA_MAX_DIALOGS of them; set it to 0 to lift the budget.
+const SODA = env("SODA", "1") !== "0";
+const SODA_DATASET = env("SODA_DATASET", "allenai/soda");
+const SODA_SPLITS = env("SODA_SPLITS", "train")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// ~6.3 episodes per dialogue, so this budgets ~750k episodes — comparable to
+// the Taskmaster stage and to Aya, which is the intended balance. 0 = no budget.
+const SODA_MAX_DIALOGS = Math.max(
+  0,
+  Math.floor(Number(env("SODA_MAX_DIALOGS", "120000"))) || 0,
+);
+const MAX_SODA_TURN_CHARS = Math.max(
+  1_000,
+  Math.floor(Number(env("MAX_SODA_TURN_KB", "32")) * 1000) || 32_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e‴  SODA parsing — a social dialogue row → SEMA items
+//
+// Each row carries `dialogue` (an array of turn strings) and `speakers` (the
+// speaker name per turn). The deposit is the cumulative walk over speaker-merged
+// turns, identical in shape to Taskmaster and oasst2 — turns are short (mean
+// 87 B) and dialogues average 7.3 turns, so the accumulated context stays well
+// inside the healthy range.
+//
+// `narrative`, `literal` and the ATOMIC-style `head`/`relation`/`tail` columns
+// are NOT deposited: they are the generation scaffolding SODA was distilled
+// from, they restate the dialogue in the third person, and depositing both a
+// dialogue and its paraphrased summary gives one meaning two shapes — which is
+// measured to SUPPRESS composition rather than help it.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** Normalize a SODA row into its turns, or null when it carries no usable
+ *  dialogue. Speakers are optional (they only drive merging); an implausibly
+ *  long turn rejects the dialogue as corrupt. */
+export function toSodaTurns(
+  row: unknown,
+  maxChars = MAX_SODA_TURN_CHARS,
+): TaskmasterTurn[] | null {
+  if (!row || typeof row !== "object") return null;
+  const r = row as Record<string, unknown>;
+  const dialogue = r.dialogue;
+  if (!Array.isArray(dialogue)) return null;
+  const speakers = Array.isArray(r.speakers) ? r.speakers : [];
+  const turns: TaskmasterTurn[] = [];
+  for (let i = 0; i < dialogue.length; i++) {
+    const text = typeof dialogue[i] === "string"
+      ? (dialogue[i] as string).trim()
+      : "";
+    if (!text) continue;
+    if (text.length > maxChars) return null;
+    turns.push({
+      speaker: String(speakers[i] ?? "").trim().toUpperCase(),
+      text,
+    });
+  }
+  return turns.length ? turns : null;
+}
+
+/** Translate ONE SODA dialogue into SEMA items: the cumulative walk over its
+ *  speaker-merged turns. Shares `mergeSpeakerTurns` because the rule is the
+ *  same one — consecutive turns by one speaker are one contribution. */
+export function sodaDialogueToItems(turns: TaskmasterTurn[]): TrainingItem[] {
+  const texts = mergeSpeakerTurns(turns);
+  if (texts.length < 2) return []; // not an exchange
+  return refineItems(accumulate(texts));
+}
+
+export const soda: Corpus = {
+  id: "soda",
+  label: "SODA",
+  kind: "social dialogue",
+  enabled: SODA,
+  maxRows: SODA_MAX_DIALOGS,
+  // Two of sixteen columns. The `narrative`/`literal`/`head`/`relation`/`tail`
+  // scaffolding the note above declines to deposit is now also never decoded:
+  // measured on the converted train shard, 449 MB uncompressed across all
+  // sixteen against 306 MB for these two (68.1%).
+  read: parquet({ columns: ["dialogue", "speakers"] }),
+  toItems: (row) => {
+    const turns = toSodaTurns(row);
+    if (!turns) return null;
+    const items = sodaDialogueToItems(turns);
+    return items.length ? items : null;
+  },
+  unitNoun: "shard(s)",
+  log: { rows: "row(s)" },
+  discover: convertedParquetUnits({
+    id: "soda",
+    label: "SODA",
+    dataset: SODA_DATASET,
+    config: "default",
+    splits: SODA_SPLITS,
+    localSub: "soda",
+  }),
+};

package/example/train_base/corpora/taskmaster.ts

@@ -0,0 +1,217 @@
+// train_base/corpora/taskmaster.ts — google-research-datasets/Taskmaster 1–4 dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env, LOCAL_PATH } from "../config.js";
+import {
+  accumulate,
+  mergeSpeakerTurns,
+  refineItems,
+  type SpeakerTurn,
+  type TrainingItem,
+} from "../items.js";
+import { githubContents, localFiles } from "../discovery.js";
+import { jsonArray } from "../readers.js";
+import { type Corpus, localDir, type Unit } from "../corpus.js";
+import type { TrainCtx } from "../runtime.js";
+import { DIM, R } from "../ui.js";
+import { basename, join } from "node:path";
+
+// ── google-research-datasets/Taskmaster 1–4 (the dialogue stages) ──
+// Four corpora of task-oriented dialogue, one shape between them: each file is a
+// JSON ARRAY of conversations and each conversation carries
+// `utterances: [{speaker, text, …}]`. TM-1 ships two files directly under its
+// directory (self-dialogs, woz-dialogs); TM-2/3/4 ship theirs under `<set>/data`.
+// They are the best-scoring corpora on the fold-unit recurrence benchmark that
+// selects for halo health (TM-3 85.1%, TM-4 78.8%, TM-2 68.7%, TM-1 51.8%,
+// against 23.2% for the incumbent SmolSent), and they are genuinely multi-turn
+// where the incumbent multi-turn stage is not (TM-3 median 20 turns of ~43 B,
+// against oasst2's median turn of 529 B).
+//
+// Served from GitHub raw, not Hugging Face: the HF mirrors are loading-script
+// repos with no data files, and the official copies carry the CC BY 4.0 notice.
+const TASKMASTER = env("TASKMASTER", "1") !== "0";
+// Which sets to train, in order. Each is a directory in the Taskmaster repo.
+const TASKMASTER_SETS = env(
+  "TASKMASTER_SETS",
+  "TM-1-2019,TM-2-2020,TM-3-2020,TM-4-2024",
+).split(",").map((s) => s.trim()).filter(Boolean);
+const TASKMASTER_REPO = env(
+  "TASKMASTER_REPO",
+  "google-research-datasets/Taskmaster",
+);
+const TASKMASTER_RAW =
+  `https://raw.githubusercontent.com/${TASKMASTER_REPO}/master`;
+// A conversation must have at least this many turns AFTER same-speaker merging.
+// The default of 2 keeps every real exchange: unlike oasst2 — where a lone Q→A
+// tree merely replicates the Aya stage's shape and is dropped — a two-turn
+// task-oriented exchange is still task-oriented dialogue, and TM-4's dialogues
+// are short by design (median 3.7 turns), so a higher bar would discard most of
+// that set.
+const TASKMASTER_MIN_TURNS = Math.max(
+  2,
+  Math.floor(Number(env("TASKMASTER_MIN_TURNS", "2"))) || 2,
+);
+// Skip a conversation carrying an implausibly long utterance (corruption). The
+// measured maximum across TM-1/2/3/4 is 1,897 bytes, so this only guards.
+const MAX_TASKMASTER_TURN_CHARS = Math.max(
+  1_000,
+  Math.floor(Number(env("MAX_TASKMASTER_TURN_KB", "32")) * 1000) || 32_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e′  Taskmaster 1–4 parsing — a conversation ARRAY ELEMENT → SEMA items
+//
+// One adapter serves all four sets: every Taskmaster conversation, in every
+// set, is `{conversation_id, …, utterances: [{speaker, text, …}]}`.
+//
+// ONLY `utterances[].text` IS READ, and that is a licence-adjacent correctness
+// property, not a stylistic one. TM-3 and TM-4 also carry an `instructions`
+// field holding the crowd-worker's task template — page after page of
+// `{{HIDE movie_1 name.movie No Time To Die}}`, `{{CHECK confirm_natural …}}`
+// and `var_theater_1` placeholders. That is authoring scaffolding, not
+// dialogue, and depositing it would teach the store template noise as prose.
+// Reading only `utterances[].text` excludes it structurally. Verified against
+// the real files: across TM-2 (13,953 turns), TM-3 (24,059) and TM-4 (786),
+// utterance text contains ZERO `var_*` placeholders and ZERO `{{ }}` markers —
+// the scaffolding never leaks out of `instructions`.
+//
+// CONSECUTIVE SAME-SPEAKER TURNS ARE MERGED. Taskmaster splits one speaker's
+// contribution across several indexed utterances ("I can help you with your
+// movie search." / "Where are you located?" are two ASSISTANT rows), which is
+// an artifact of the collection UI. Left unmerged, the cumulative walk deposits
+// a turn boundary in the middle of one speaker's contribution and teaches it as
+// a hand-off. Measured share of turns absorbed by merging: TM-1 17.7%,
+// TM-2 11.9%, TM-3 0.8%, TM-4 0.0% — so this is load-bearing for the older sets
+// and a no-op for the newer ones. Speaker names are compared case-insensitively
+// because TM-1/2 use USER/ASSISTANT and TM-3/4 use user/assistant.
+//
+// The deposit shape is the cumulative walk (§6e's `accumulate`), identical to
+// oasst2: each turn is the continuation of ALL prior turns, bare text, no role
+// labels. It is the right shape here for the same reason and at a far healthier
+// size — merged turns run p50 34–45 B (p90 ~100 B) and the accumulated context
+// p50 301–532 B (p90 ~1.1 KB), against oasst2's median SINGLE turn of 529 B.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One utterance of a Taskmaster conversation — the shared dialogue-turn shape,
+ *  under the name this corpus's adapters have always used. */
+export type TaskmasterTurn = SpeakerTurn;
+
+/** Normalize ONE element of a Taskmaster data file into its turns, or null when
+ *  it carries no usable utterance. Empty/whitespace-only utterances are dropped
+ *  (TM-3 has a few); a single implausibly long utterance rejects the whole
+ *  conversation as corrupt rather than depositing a dump. */
+export function toTaskmasterTurns(
+  row: unknown,
+  maxChars = MAX_TASKMASTER_TURN_CHARS,
+): TaskmasterTurn[] | null {
+  if (!row || typeof row !== "object") return null;
+  const utterances = (row as Record<string, unknown>).utterances;
+  if (!Array.isArray(utterances)) return null;
+  const turns: TaskmasterTurn[] = [];
+  for (const u of utterances) {
+    if (!u || typeof u !== "object") continue;
+    const r = u as Record<string, unknown>;
+    const text = typeof r.text === "string" ? r.text.trim() : "";
+    if (!text) continue;
+    if (text.length > maxChars) return null;
+    turns.push({
+      speaker: String(r.speaker ?? "").trim().toUpperCase(),
+      text,
+    });
+  }
+  return turns.length ? turns : null;
+}
+
+/** Translate ONE Taskmaster conversation into SEMA training items: the
+ *  cumulative walk over its merged turns. Returns [] for a conversation below
+ *  TASKMASTER_MIN_TURNS, so callers can simply skip empties. */
+export function taskmasterConversationToItems(
+  turns: TaskmasterTurn[],
+  minTurns = TASKMASTER_MIN_TURNS,
+): TrainingItem[] {
+  const texts = mergeSpeakerTurns(turns);
+  if (texts.length < minTurns) return [];
+  return refineItems(accumulate(texts));
+}
+
+/** List the Taskmaster data files to train, in TASKMASTER_SETS order. Returns
+ *  repo-relative paths, e.g. "TM-3-2020/data/data_00.json".
+ *
+ *  TM-2/3/4 keep their dialogue files under `<set>/data`, so everything there is
+ *  fair game. TM-1 has no `data` directory: its two dialogue files sit at the
+ *  set root NEXT TO `ontology.json` (a slot schema) and `sample.json` (a small
+ *  excerpt of self-dialogs). Neither is an array of conversations, and training
+ *  the excerpt would deposit a subset of TM-1 twice, so TM-1 is filtered to the
+ *  `*-dialogs.json` pair (self-dialogs, woz-dialogs). */
+async function listFiles(
+  ctx: TrainCtx,
+): Promise<Array<{ set: string; path: string; size: number }>> {
+  const out: Array<{ set: string; path: string; size: number }> = [];
+  for (const set of TASKMASTER_SETS) {
+    const rootOnly = /^TM-1\b/i.test(set);
+    const dir = rootOnly ? set : `${set}/data`;
+    const names = await githubContents(
+      TASKMASTER_REPO,
+      dir,
+      /\.json$/i,
+      `GET Taskmaster ${dir}`,
+      ctx.http,
+    );
+    for (const { path: name, size } of names) {
+      if (rootOnly && !/-dialogs\.json$/i.test(name)) continue;
+      out.push({ set, path: `${dir}/${name}`, size });
+    }
+  }
+  return out;
+}
+
+const unit = (key: string, name: string): Unit => ({
+  key,
+  name,
+  display: `Taskmaster ${name}`,
+});
+
+export const taskmaster: Corpus = {
+  id: "taskmaster",
+  label: "Taskmaster",
+  kind: "task dialogue",
+  enabled: TASKMASTER,
+  unitNoun: "dialogue file(s)",
+  read: jsonArray(),
+  toItems: (row) => {
+    const turns = toTaskmasterTurns(row);
+    if (!turns) return null;
+    const items = taskmasterConversationToItems(turns); // [] when too short
+    return items.length ? items : null;
+  },
+  log: { bad: "unusable conversation(s)" },
+  async discover(ctx) {
+    // LOCAL_PATH/taskmaster/ — a subdirectory, because these share the .json
+    // extension with the General-Knowledge source and must not be confused
+    // with it.
+    if (LOCAL_PATH) {
+      const dir = localDir("taskmaster");
+      const names = localFiles(dir, /\.json$/i);
+      if (names.length === 0) {
+        ctx.progress.log(
+          `  ${DIM}· no Taskmaster *.json in ${dir} — skipping${R}`,
+        );
+        return null;
+      }
+      return names.map((n) => ({
+        ...unit(n.path, n.path),
+        local: join(dir, n.path),
+        bytes: n.size,
+      }));
+    }
+    return (await listFiles(ctx)).map((f) => ({
+      ...unit(f.path, `${f.set}/${basename(f.path)}`),
+      url: `${TASKMASTER_RAW}/${f.path}`,
+      bytes: f.size,
+    }));
+  },
+};

package/example/train_base/corpora/wiki2.ts

@@ -0,0 +1,190 @@
+// train_base/corpora/wiki2.ts — 2WikiMultihopQA evidence triples
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { parquet } from "../readers.js";
+import type { Corpus } from "../corpus.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+
+// ── 2WikiMultihopQA — the `evidences` TRIPLES only (the composition stage) ──
+// Each row carries `evidences`: a JSON string of (subject, relation, object)
+// triples that CHAIN — one triple's object is the next's subject. 72.5% of rows
+// carry such a chain (measured over 4,000 rows), and those triples are the only
+// representation measured to make Sema compose a two-hop answer at all.
+//
+// TWO COLUMNS ARE DELIBERATELY NOT READ, one for licence reasons and one for
+// capability reasons:
+//   • `context` holds Wikipedia PROSE. The repo is Apache-2.0 but Wikipedia text
+//     is CC BY-SA, and a Sema store keeps text verbatim, so ingesting the
+//     passages would attach ShareAlike to every distributed store. The triples
+//     originate in Wikidata (CC0). See DATASETS.md §3.2/§4.
+//   • `question`/`answer` are the composed multi-hop QUESTION. Depositing those
+//     teaches the answer to that exact question and nothing else — it memorises
+//     rather than composes. They are used to EVALUATE this adapter, never as
+//     training input.
+//
+// Read from Hugging Face's auto-converted `refs/convert/parquet` branch, not
+// from main: the main-branch train.parquet is written as ONE 167,454-row
+// group (666 MB uncompressed) and a Parquet column chunk is per-group, so any
+// read of it materialises the whole file. The converted branch uses uniform
+// 10,000-row groups, which `parquetBatchRows` then subdivides by BYTES.
+const WIKI2 = env("WIKI2", "1") !== "0";
+const WIKI2_DATASET = env("WIKI2_DATASET", "xanhho/2WikiMultihopQA");
+// Splits to train, in order. Only `train` by default: `validation`/`test` are
+// the dataset's held-out sets and are what an honest evaluation of this
+// adapter's composition rate has to be measured on.
+const WIKI2_SPLITS = env("WIKI2_SPLITS", "train")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// Reject a triple with an implausibly long field (corruption); real subjects and
+// objects are entity names, and relations are Wikidata property labels.
+// 0 = every row. The train split holds 167,454 rows at ~4.95 deposits each
+// (~830k facts), so this is the knob that keeps 2Wiki proportionate to the rest
+// of the curriculum in the same way SODA_MAX_DIALOGS does.
+const WIKI2_MAX_ROWS = Math.max(
+  0,
+  Math.floor(Number(env("WIKI2_MAX_ROWS", "0"))) || 0,
+);
+const MAX_WIKI2_FIELD_CHARS = Math.max(
+  100,
+  Math.floor(Number(env("MAX_WIKI2_FIELD_KB", "2")) * 1000) || 2_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e″  2WikiMultihopQA parsing — `evidences` TRIPLES → SEMA facts
+//
+// This is the only stage whose purpose is COMPOSITION: answering a question
+// whose answer no single deposited fact contains. Sema composes by grounding
+// hop 1, then pivoting on the longest unconsumed learnt context that the
+// grounded answer CONTAINS (`reason`/`pivotStep`), so the pivot target must
+// itself be a deposited context. Each triple therefore deposits TWO facts:
+//
+//     "<subject> <relation>"  →  "The <relation> of <subject> is <object>."
+//     "<subject>"             →  "The <relation> of <subject> is <object>."
+//
+// The second is the PIVOT FACT. Without it the bare entity naming hop 2's
+// subject is not a learnt context, so the chain is structurally unreachable no
+// matter what the rest of the pipeline does.
+//
+// MEASURED on 200 real chained dev rows, depositing triples only and asking the
+// dataset's own composed questions (D = 1024, seed 7):
+//
+//   relation fact only          240 deposits    5/120 ( 4%)   pivotStep  0
+//   relation + pivot fact       800 deposits   44/200 (22%)   pivotStep 31
+//
+// A 5x improvement, and the only variant where the second hop fires at all.
+//
+// REJECTED ALTERNATIVE, so it is not re-tried blind: depositing the pivot fact
+// only for subjects that also appear as an OBJECT within the same row's
+// evidences (a row-local "something can pivot into this" test) cut deposits 25%
+// (800 → 600) but cost composition — 41/200 (20.5%) with pivotStep down to 21,
+// because real chains also run BETWEEN rows. Composition is this stage's entire
+// justification, so the deposits are worth keeping.
+//
+// The residual ~78% is a KNOWN, previously-recorded limitation and not a defect
+// in this adapter: the climb elects a topic rather than a relation, so a
+// question phrased "When did X's father die?" does not align with the Wikidata
+// property label "date of death". Failure is dominated by hop 2 never firing,
+// not by a wrong hop 2. Answer-shape breakdown at N = 120: entity answers
+// 23/106, date answers 2/14 — dates are worse, but not the cliff an earlier
+// note suggested, which is why no object-shape filter is applied here.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One (subject, relation, object) triple from a 2Wiki `evidences` cell. */
+export interface WikiTriple {
+  subject: string;
+  relation: string;
+  object: string;
+}
+
+/** Normalize a 2Wiki row into its evidence triples, or null when it carries
+ *  none usable. `evidences` is a JSON STRING holding an array of 3-element
+ *  arrays; a row whose cell is absent, unparseable, or empty yields null.
+ *  Individual malformed or oversized triples are dropped without discarding the
+ *  row — one bad triple should not cost the others. */
+export function toWikiTriples(
+  row: unknown,
+  maxChars = MAX_WIKI2_FIELD_CHARS,
+): WikiTriple[] | null {
+  if (!row || typeof row !== "object") return null;
+  const cell = (row as Record<string, unknown>).evidences;
+  let parsed: unknown = cell;
+  if (typeof cell === "string") {
+    try {
+      parsed = JSON.parse(cell);
+    } catch {
+      return null;
+    }
+  }
+  if (!Array.isArray(parsed)) return null;
+  const out: WikiTriple[] = [];
+  for (const e of parsed) {
+    if (!Array.isArray(e) || e.length < 3) continue;
+    const subject = typeof e[0] === "string" ? e[0].trim() : "";
+    const relation = typeof e[1] === "string" ? e[1].trim() : "";
+    const object = typeof e[2] === "string" ? e[2].trim() : "";
+    if (!subject || !relation || !object) continue;
+    if (
+      subject.length > maxChars ||
+      relation.length > maxChars ||
+      object.length > maxChars
+    ) continue;
+    out.push({ subject, relation, object });
+  }
+  return out.length ? out : null;
+}
+
+/** Render ONE triple as the prose fact Sema stores. Kept separate so the two
+ *  deposits below are guaranteed to share a byte-identical continuation: the
+ *  pivot fact only works if it leads to the SAME node the relation fact does. */
+export function wikiTripleSentence(t: WikiTriple): string {
+  return `The ${t.relation} of ${t.subject} is ${t.object}.`;
+}
+
+/** Translate a row's triples into SEMA items: per triple, the relation fact and
+ *  the bare-subject PIVOT fact (see the section note above). refineItems drops
+ *  the duplicates this produces when a row states the same triple twice. */
+export function wikiTriplesToItems(triples: WikiTriple[]): TrainingItem[] {
+  const items: TrainingItem[] = [];
+  for (const t of triples) {
+    const fact = wikiTripleSentence(t);
+    items.push({ context: `${t.subject} ${t.relation}`, continuation: fact });
+    items.push({ context: t.subject, continuation: fact });
+  }
+  return refineItems(items);
+}
+
+export const wiki2: Corpus = {
+  id: "2wiki",
+  label: "2Wiki",
+  kind: "relation triples",
+  enabled: WIKI2,
+  maxRows: WIKI2_MAX_ROWS,
+  // Read ONE of the file's seven columns. This is where the "context is not
+  // read" rule above stops being a property of the adapter and becomes a
+  // property of the read: the Wikipedia prose is never decoded at all. It is
+  // also almost the whole file — measured on the converted train shard, the
+  // seven columns hold 518 MB uncompressed and `evidences` is 18 MB of it
+  // (3.5%), so the projection cuts the decode work by ~28x.
+  read: parquet({ columns: ["evidences"] }),
+  toItems: (row) => {
+    const triples = toWikiTriples(row);
+    if (!triples) return null;
+    const items = wikiTriplesToItems(triples);
+    return items.length ? items : null;
+  },
+  unitNoun: "shard(s)",
+  log: { rows: "row(s)" },
+  discover: convertedParquetUnits({
+    id: "2wiki",
+    label: "2Wiki",
+    dataset: WIKI2_DATASET,
+    config: "default",
+    splits: WIKI2_SPLITS,
+    localSub: "2wiki",
+  }),
+};