@hviana/sema 0.5.8 → 0.6.0

package/example/train_base/corpora/smolsent.ts

@@ -0,0 +1,203 @@
+// train_base/corpora/smolsent.ts — google/smol sentence pairs
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env, LOCAL_PATH } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { hfTree, type Listed, localFiles } from "../discovery.js";
+import { lines } from "../readers.js";
+import type { Corpus, Unit } from "../corpus.js";
+import type { TrainCtx } from "../runtime.js";
+import { basename, join } from "node:path";
+
+// ── google/smol · SmolSent (the first training stage) ──
+// SmolSent is Google's sentence-level translation set: ~863 human sentence pairs
+// per language pair across 100+ low-resource languages, cc-by-4.0 (commercial-
+// friendly). Each row is {sl, tl, src, trg, …} — a source sentence and its
+// translation. A pair is "two names for one meaning", which is exactly the
+// cross-language concept SEMA fuses (see test/05-concepts.test.mjs), so each row
+// becomes FACTS that bind the two phrasings as one concept at recall time.
+//
+// The corpus ships as one plain JSONL file PER language pair under smolsent/ in
+// the HF repo (e.g. smolsent/ha_en.jsonl). We DOWNLOAD each file and stream its
+// lines — far faster and free of the rate-limiting that per-row API paging hit.
+// The file list is discovered from the HF repo tree. SMOLSENT=0 disables the
+// stage; SMOLSENT_PAIRS (comma-separated basenames without .jsonl, e.g.
+// "ha_en,zu_en") restricts to a chosen subset.
+const SMOLSENT = env("SMOLSENT", "1") !== "0";
+const SMOLSENT_DATASET = env("SMOLSENT_DATASET", "google/smol");
+const SMOLSENT_PAIRS = (process.env.SMOLSENT_PAIRS ?? "")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// The resume id PREFIX for the SmolSent stage; one completed-files entry per
+// file (e.g. "smolsent::ha_en.jsonl").
+const SMOLSENT_ID = "smolsent";
+// Which direction(s) of a translation pair to deposit. Was effectively "both",
+// and that is now the default NO longer, for a reason measured rather than
+// assumed.
+//
+// SmolSent's English side is a SHARED POOL translated into every language: row
+// id 0 of smolsent/ha_en.jsonl, zu_en.jsonl and am_en.jsonl all carry the SAME
+// `trg` ("It allows me to work by following my vibes and ..."). The two
+// directions are therefore not symmetric at all:
+//
+//   src2trg  (foreign -> English)  many distinct contexts -> ONE shared
+//                                  continuation. Every language's rendering of
+//                                  a meaning converges on the same English
+//                                  node — the cross-language concept fusion
+//                                  this stage exists for.
+//   trg2src  (English -> foreign)  ONE context -> 100+ DIFFERENT continuations,
+//                                  one per language file. The same English
+//                                  sentence is deposited over and over with a
+//                                  different answer each time.
+//
+// So dropping trg2src is not merely a corpus-size economy (it halves the
+// largest stage, which was 60.9% of all examples in the last trained store); it
+// removes a genuine ambiguity pathology. Set SMOLSENT_DIRECTIONS=both to
+// restore the old behaviour, or trg2src for English->foreign only.
+//
+// WHAT THE CUT DOES NOT DO, measured on a three-pair store: asking the English
+// sentence still ANSWERS with a foreign rendering, because the engine can reach
+// a shared continuation's predecessors on its own. What is removed is the
+// DEPOSITED forward ambiguity — one context carrying ~100 competing
+// continuations — not every reverse association.
+const SMOLSENT_DIRECTIONS = env("SMOLSENT_DIRECTIONS", "src2trg")
+  .trim().toLowerCase();
+const SMOLSENT_SRC2TRG = SMOLSENT_DIRECTIONS !== "trg2src";
+const SMOLSENT_TRG2SRC = SMOLSENT_DIRECTIONS === "trg2src" ||
+  SMOLSENT_DIRECTIONS === "both";
+// A SmolSent side longer than this is skipped (a sentence pair is short; a huge
+// value is corruption, not a sentence).
+const MAX_SMOLSENT_CHARS = Math.max(
+  2_000,
+  Math.floor(Number(env("MAX_SMOLSENT_KB", "16")) * 1000) || 16_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6c  SmolSent parsing — a translation pair → SEMA facts
+//
+// Each SmolSent row is {sl, tl, src, trg, …}: a source sentence and its
+// translation into another language — "two names for one meaning". This is the
+// cross-language concept SEMA fuses (see test/05-concepts.test.mjs: "ice" and
+// "hielo" become one concept because they share company, so a fact about one
+// transfers to the other). So a pair is rendered as BIDIRECTIONAL translation
+// FACTS — (src → trg) and (trg → src) — binding the two phrasings as one
+// concept at recall time, in both directions. Two facts, no experiences and no
+// cumulative walk: a sentence pair is not multi-turn, and a bare sentence on its
+// own carries no relation to point at.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One normalized SmolSent row. */
+export interface SmolSentRow {
+  src: string; // source sentence
+  trg: string; // its translation
+  sl: string; // source language code
+  tl: string; // target language code
+}
+
+/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
+ *  both sides or a side is implausibly large (a dump, not a sentence). */
+export function toSmolSentRow(
+  row: unknown,
+  maxChars = MAX_SMOLSENT_CHARS,
+): SmolSentRow | null {
+  if (!row || typeof row !== "object") return null;
+  const r = row as Record<string, unknown>;
+  const src = typeof r.src === "string" ? r.src.trim() : "";
+  // `trg` is a single string in smolsent; tolerate a list form defensively.
+  const trgRaw = Array.isArray(r.trgs) ? r.trgs[0] : r.trg;
+  const trg = typeof trgRaw === "string" ? trgRaw.trim() : "";
+  if (!src || !trg) return null;
+  if (
+    src.length > maxChars || trg.length > maxChars
+  ) return null;
+  const sl = typeof r.sl === "string" ? r.sl.trim() : "";
+  const tl = typeof r.tl === "string" ? r.tl.trim() : "";
+  return { src, trg, sl, tl };
+}
+
+/** Translate ONE SmolSent pair into SEMA facts. The two sentences are one
+ *  meaning in two languages, but the two BINDINGS are not equally sound —
+ *  SmolSent's English side is a shared pool translated into every language, so
+ *  `trg -> src` gives one English context a different answer in every language
+ *  file. See SMOLSENT_DIRECTIONS. refineItems drops the degenerate case where
+ *  src === trg. */
+export function smolSentRowToItems(
+  row: SmolSentRow,
+  dirs: { src2trg: boolean; trg2src: boolean } = {
+    src2trg: SMOLSENT_SRC2TRG,
+    trg2src: SMOLSENT_TRG2SRC,
+  },
+): TrainingItem[] {
+  const { src, trg } = row;
+  const items: TrainingItem[] = [];
+  if (dirs.src2trg) items.push({ context: src, continuation: trg });
+  if (dirs.trg2src) items.push({ context: trg, continuation: src });
+  return refineItems(items);
+}
+
+/** Discover the SmolSent per-pair JSONL files from the HF repo tree, restricted
+ *  to SMOLSENT_PAIRS (basenames without .jsonl) when set. Each entry is the
+ *  repo-relative path, e.g. "smolsent/ha_en.jsonl". */
+async function listFiles(ctx: TrainCtx): Promise<Listed[]> {
+  const paths = await hfTree(
+    SMOLSENT_DATASET,
+    "smolsent",
+    /\.jsonl$/i,
+    `GET smol tree`,
+    ctx.http,
+  );
+  if (!SMOLSENT_PAIRS.length) return paths;
+  const want = new Set(SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")));
+  return paths.filter((p) =>
+    want.has(basename(p.path).replace(/\.jsonl$/i, ""))
+  );
+}
+
+const unit = (name: string): Unit => ({
+  key: name,
+  // The log names a pair by its language code alone; the panel keeps the file.
+  name: name.replace(/\.jsonl$/i, ""),
+  display: `SmolSent ${name}`,
+});
+
+export const smolsent: Corpus = {
+  id: SMOLSENT_ID,
+  label: "SmolSent",
+  kind: "translation",
+  enabled: SMOLSENT,
+  unitNoun: "translation file(s)",
+  // The *4 is chars-to-a-generous-JSON-envelope: the guard bounds ONE side of
+  // a pair, the line carries both plus its keys.
+  read: lines({ maxLineChars: MAX_SMOLSENT_CHARS * 4 }),
+  toItems: (row) => {
+    const r = toSmolSentRow(row);
+    return r ? smolSentRowToItems(r) : null;
+  },
+  async discover(ctx) {
+    // Work-list: local *.jsonl in LOCAL_PATH, else the repo's smolsent/ files.
+    if (LOCAL_PATH) {
+      let names = localFiles(LOCAL_PATH, /\.jsonl$/i);
+      if (SMOLSENT_PAIRS.length) {
+        const want = new Set(
+          SMOLSENT_PAIRS.map((p) => p.replace(/\.jsonl$/i, "")),
+        );
+        names = names.filter((n) => want.has(n.path.replace(/\.jsonl$/i, "")));
+      }
+      return names.map((n) => ({
+        ...unit(n.path),
+        local: join(LOCAL_PATH, n.path),
+        bytes: n.size,
+      }));
+    }
+    return (await listFiles(ctx)).map(({ path, size }) => ({
+      ...unit(basename(path)),
+      // owner/name and the file path are URL PATH segments — do not encode "/".
+      url:
+        `https://huggingface.co/datasets/${SMOLSENT_DATASET}/resolve/main/${path}`,
+      bytes: size,
+    }));
+  },
+};

package/example/train_base/corpora/soda.ts

@@ -0,0 +1,130 @@
+// train_base/corpora/soda.ts — allenai/soda social dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import {
+  accumulate,
+  mergeSpeakerTurns,
+  refineItems,
+  type TrainingItem,
+} from "../items.js";
+import { parquet } from "../readers.js";
+import type { Corpus } from "../corpus.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+// SODA turns are the same shape as Taskmaster's, and merge by the same rule.
+import type { TaskmasterTurn } from "./taskmaster.js";
+
+// ── allenai/soda (social dialogue) and AmazonScience/massive (short intents) ──
+// Both are read from Hugging Face's auto-converted `refs/convert/parquet`
+// branch. For SODA that is mandatory, not cosmetic: its main-branch
+// train.parquet is ONE 1,191,582-row group (1.19 GB uncompressed), and a
+// Parquet column chunk is per-group, so any read of it materialises the whole
+// file — measured at 100% of a 689 MB file and 2 GB of heap for a 500-row read.
+// The converted branch uses uniform 10,000-row groups.
+//
+// BOTH STAGES ARE BUDGETED, and that is a curriculum decision rather than an
+// algorithmic cap. SODA's train split holds 1,191,582 dialogues which the
+// cumulative walk would turn into ~8 MILLION episodes — against the 662,221
+// deposits of the entire current corpus. Trained whole it would not join the
+// mix, it would BE the mix, and corpus size is the quantity every scale problem
+// in this engine is measured against. The default takes the first
+// SODA_MAX_DIALOGS of them; set it to 0 to lift the budget.
+const SODA = env("SODA", "1") !== "0";
+const SODA_DATASET = env("SODA_DATASET", "allenai/soda");
+const SODA_SPLITS = env("SODA_SPLITS", "train")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// ~6.3 episodes per dialogue, so this budgets ~750k episodes — comparable to
+// the Taskmaster stage and to Aya, which is the intended balance. 0 = no budget.
+const SODA_MAX_DIALOGS = Math.max(
+  0,
+  Math.floor(Number(env("SODA_MAX_DIALOGS", "120000"))) || 0,
+);
+const MAX_SODA_TURN_CHARS = Math.max(
+  1_000,
+  Math.floor(Number(env("MAX_SODA_TURN_KB", "32")) * 1000) || 32_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e‴  SODA parsing — a social dialogue row → SEMA items
+//
+// Each row carries `dialogue` (an array of turn strings) and `speakers` (the
+// speaker name per turn). The deposit is the cumulative walk over speaker-merged
+// turns, identical in shape to Taskmaster and oasst2 — turns are short (mean
+// 87 B) and dialogues average 7.3 turns, so the accumulated context stays well
+// inside the healthy range.
+//
+// `narrative`, `literal` and the ATOMIC-style `head`/`relation`/`tail` columns
+// are NOT deposited: they are the generation scaffolding SODA was distilled
+// from, they restate the dialogue in the third person, and depositing both a
+// dialogue and its paraphrased summary gives one meaning two shapes — which is
+// measured to SUPPRESS composition rather than help it.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** Normalize a SODA row into its turns, or null when it carries no usable
+ *  dialogue. Speakers are optional (they only drive merging); an implausibly
+ *  long turn rejects the dialogue as corrupt. */
+export function toSodaTurns(
+  row: unknown,
+  maxChars = MAX_SODA_TURN_CHARS,
+): TaskmasterTurn[] | null {
+  if (!row || typeof row !== "object") return null;
+  const r = row as Record<string, unknown>;
+  const dialogue = r.dialogue;
+  if (!Array.isArray(dialogue)) return null;
+  const speakers = Array.isArray(r.speakers) ? r.speakers : [];
+  const turns: TaskmasterTurn[] = [];
+  for (let i = 0; i < dialogue.length; i++) {
+    const text = typeof dialogue[i] === "string"
+      ? (dialogue[i] as string).trim()
+      : "";
+    if (!text) continue;
+    if (text.length > maxChars) return null;
+    turns.push({
+      speaker: String(speakers[i] ?? "").trim().toUpperCase(),
+      text,
+    });
+  }
+  return turns.length ? turns : null;
+}
+
+/** Translate ONE SODA dialogue into SEMA items: the cumulative walk over its
+ *  speaker-merged turns. Shares `mergeSpeakerTurns` because the rule is the
+ *  same one — consecutive turns by one speaker are one contribution. */
+export function sodaDialogueToItems(turns: TaskmasterTurn[]): TrainingItem[] {
+  const texts = mergeSpeakerTurns(turns);
+  if (texts.length < 2) return []; // not an exchange
+  return refineItems(accumulate(texts));
+}
+
+export const soda: Corpus = {
+  id: "soda",
+  label: "SODA",
+  kind: "social dialogue",
+  enabled: SODA,
+  maxRows: SODA_MAX_DIALOGS,
+  // Two of sixteen columns. The `narrative`/`literal`/`head`/`relation`/`tail`
+  // scaffolding the note above declines to deposit is now also never decoded:
+  // measured on the converted train shard, 449 MB uncompressed across all
+  // sixteen against 306 MB for these two (68.1%).
+  read: parquet({ columns: ["dialogue", "speakers"] }),
+  toItems: (row) => {
+    const turns = toSodaTurns(row);
+    if (!turns) return null;
+    const items = sodaDialogueToItems(turns);
+    return items.length ? items : null;
+  },
+  unitNoun: "shard(s)",
+  log: { rows: "row(s)" },
+  discover: convertedParquetUnits({
+    id: "soda",
+    label: "SODA",
+    dataset: SODA_DATASET,
+    config: "default",
+    splits: SODA_SPLITS,
+    localSub: "soda",
+  }),
+};

package/example/train_base/corpora/taskmaster.ts

@@ -0,0 +1,217 @@
+// train_base/corpora/taskmaster.ts — google-research-datasets/Taskmaster 1–4 dialogue
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env, LOCAL_PATH } from "../config.js";
+import {
+  accumulate,
+  mergeSpeakerTurns,
+  refineItems,
+  type SpeakerTurn,
+  type TrainingItem,
+} from "../items.js";
+import { githubContents, localFiles } from "../discovery.js";
+import { jsonArray } from "../readers.js";
+import { type Corpus, localDir, type Unit } from "../corpus.js";
+import type { TrainCtx } from "../runtime.js";
+import { DIM, R } from "../ui.js";
+import { basename, join } from "node:path";
+
+// ── google-research-datasets/Taskmaster 1–4 (the dialogue stages) ──
+// Four corpora of task-oriented dialogue, one shape between them: each file is a
+// JSON ARRAY of conversations and each conversation carries
+// `utterances: [{speaker, text, …}]`. TM-1 ships two files directly under its
+// directory (self-dialogs, woz-dialogs); TM-2/3/4 ship theirs under `<set>/data`.
+// They are the best-scoring corpora on the fold-unit recurrence benchmark that
+// selects for halo health (TM-3 85.1%, TM-4 78.8%, TM-2 68.7%, TM-1 51.8%,
+// against 23.2% for the incumbent SmolSent), and they are genuinely multi-turn
+// where the incumbent multi-turn stage is not (TM-3 median 20 turns of ~43 B,
+// against oasst2's median turn of 529 B).
+//
+// Served from GitHub raw, not Hugging Face: the HF mirrors are loading-script
+// repos with no data files, and the official copies carry the CC BY 4.0 notice.
+const TASKMASTER = env("TASKMASTER", "1") !== "0";
+// Which sets to train, in order. Each is a directory in the Taskmaster repo.
+const TASKMASTER_SETS = env(
+  "TASKMASTER_SETS",
+  "TM-1-2019,TM-2-2020,TM-3-2020,TM-4-2024",
+).split(",").map((s) => s.trim()).filter(Boolean);
+const TASKMASTER_REPO = env(
+  "TASKMASTER_REPO",
+  "google-research-datasets/Taskmaster",
+);
+const TASKMASTER_RAW =
+  `https://raw.githubusercontent.com/${TASKMASTER_REPO}/master`;
+// A conversation must have at least this many turns AFTER same-speaker merging.
+// The default of 2 keeps every real exchange: unlike oasst2 — where a lone Q→A
+// tree merely replicates the Aya stage's shape and is dropped — a two-turn
+// task-oriented exchange is still task-oriented dialogue, and TM-4's dialogues
+// are short by design (median 3.7 turns), so a higher bar would discard most of
+// that set.
+const TASKMASTER_MIN_TURNS = Math.max(
+  2,
+  Math.floor(Number(env("TASKMASTER_MIN_TURNS", "2"))) || 2,
+);
+// Skip a conversation carrying an implausibly long utterance (corruption). The
+// measured maximum across TM-1/2/3/4 is 1,897 bytes, so this only guards.
+const MAX_TASKMASTER_TURN_CHARS = Math.max(
+  1_000,
+  Math.floor(Number(env("MAX_TASKMASTER_TURN_KB", "32")) * 1000) || 32_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e′  Taskmaster 1–4 parsing — a conversation ARRAY ELEMENT → SEMA items
+//
+// One adapter serves all four sets: every Taskmaster conversation, in every
+// set, is `{conversation_id, …, utterances: [{speaker, text, …}]}`.
+//
+// ONLY `utterances[].text` IS READ, and that is a licence-adjacent correctness
+// property, not a stylistic one. TM-3 and TM-4 also carry an `instructions`
+// field holding the crowd-worker's task template — page after page of
+// `{{HIDE movie_1 name.movie No Time To Die}}`, `{{CHECK confirm_natural …}}`
+// and `var_theater_1` placeholders. That is authoring scaffolding, not
+// dialogue, and depositing it would teach the store template noise as prose.
+// Reading only `utterances[].text` excludes it structurally. Verified against
+// the real files: across TM-2 (13,953 turns), TM-3 (24,059) and TM-4 (786),
+// utterance text contains ZERO `var_*` placeholders and ZERO `{{ }}` markers —
+// the scaffolding never leaks out of `instructions`.
+//
+// CONSECUTIVE SAME-SPEAKER TURNS ARE MERGED. Taskmaster splits one speaker's
+// contribution across several indexed utterances ("I can help you with your
+// movie search." / "Where are you located?" are two ASSISTANT rows), which is
+// an artifact of the collection UI. Left unmerged, the cumulative walk deposits
+// a turn boundary in the middle of one speaker's contribution and teaches it as
+// a hand-off. Measured share of turns absorbed by merging: TM-1 17.7%,
+// TM-2 11.9%, TM-3 0.8%, TM-4 0.0% — so this is load-bearing for the older sets
+// and a no-op for the newer ones. Speaker names are compared case-insensitively
+// because TM-1/2 use USER/ASSISTANT and TM-3/4 use user/assistant.
+//
+// The deposit shape is the cumulative walk (§6e's `accumulate`), identical to
+// oasst2: each turn is the continuation of ALL prior turns, bare text, no role
+// labels. It is the right shape here for the same reason and at a far healthier
+// size — merged turns run p50 34–45 B (p90 ~100 B) and the accumulated context
+// p50 301–532 B (p90 ~1.1 KB), against oasst2's median SINGLE turn of 529 B.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One utterance of a Taskmaster conversation — the shared dialogue-turn shape,
+ *  under the name this corpus's adapters have always used. */
+export type TaskmasterTurn = SpeakerTurn;
+
+/** Normalize ONE element of a Taskmaster data file into its turns, or null when
+ *  it carries no usable utterance. Empty/whitespace-only utterances are dropped
+ *  (TM-3 has a few); a single implausibly long utterance rejects the whole
+ *  conversation as corrupt rather than depositing a dump. */
+export function toTaskmasterTurns(
+  row: unknown,
+  maxChars = MAX_TASKMASTER_TURN_CHARS,
+): TaskmasterTurn[] | null {
+  if (!row || typeof row !== "object") return null;
+  const utterances = (row as Record<string, unknown>).utterances;
+  if (!Array.isArray(utterances)) return null;
+  const turns: TaskmasterTurn[] = [];
+  for (const u of utterances) {
+    if (!u || typeof u !== "object") continue;
+    const r = u as Record<string, unknown>;
+    const text = typeof r.text === "string" ? r.text.trim() : "";
+    if (!text) continue;
+    if (text.length > maxChars) return null;
+    turns.push({
+      speaker: String(r.speaker ?? "").trim().toUpperCase(),
+      text,
+    });
+  }
+  return turns.length ? turns : null;
+}
+
+/** Translate ONE Taskmaster conversation into SEMA training items: the
+ *  cumulative walk over its merged turns. Returns [] for a conversation below
+ *  TASKMASTER_MIN_TURNS, so callers can simply skip empties. */
+export function taskmasterConversationToItems(
+  turns: TaskmasterTurn[],
+  minTurns = TASKMASTER_MIN_TURNS,
+): TrainingItem[] {
+  const texts = mergeSpeakerTurns(turns);
+  if (texts.length < minTurns) return [];
+  return refineItems(accumulate(texts));
+}
+
+/** List the Taskmaster data files to train, in TASKMASTER_SETS order. Returns
+ *  repo-relative paths, e.g. "TM-3-2020/data/data_00.json".
+ *
+ *  TM-2/3/4 keep their dialogue files under `<set>/data`, so everything there is
+ *  fair game. TM-1 has no `data` directory: its two dialogue files sit at the
+ *  set root NEXT TO `ontology.json` (a slot schema) and `sample.json` (a small
+ *  excerpt of self-dialogs). Neither is an array of conversations, and training
+ *  the excerpt would deposit a subset of TM-1 twice, so TM-1 is filtered to the
+ *  `*-dialogs.json` pair (self-dialogs, woz-dialogs). */
+async function listFiles(
+  ctx: TrainCtx,
+): Promise<Array<{ set: string; path: string; size: number }>> {
+  const out: Array<{ set: string; path: string; size: number }> = [];
+  for (const set of TASKMASTER_SETS) {
+    const rootOnly = /^TM-1\b/i.test(set);
+    const dir = rootOnly ? set : `${set}/data`;
+    const names = await githubContents(
+      TASKMASTER_REPO,
+      dir,
+      /\.json$/i,
+      `GET Taskmaster ${dir}`,
+      ctx.http,
+    );
+    for (const { path: name, size } of names) {
+      if (rootOnly && !/-dialogs\.json$/i.test(name)) continue;
+      out.push({ set, path: `${dir}/${name}`, size });
+    }
+  }
+  return out;
+}
+
+const unit = (key: string, name: string): Unit => ({
+  key,
+  name,
+  display: `Taskmaster ${name}`,
+});
+
+export const taskmaster: Corpus = {
+  id: "taskmaster",
+  label: "Taskmaster",
+  kind: "task dialogue",
+  enabled: TASKMASTER,
+  unitNoun: "dialogue file(s)",
+  read: jsonArray(),
+  toItems: (row) => {
+    const turns = toTaskmasterTurns(row);
+    if (!turns) return null;
+    const items = taskmasterConversationToItems(turns); // [] when too short
+    return items.length ? items : null;
+  },
+  log: { bad: "unusable conversation(s)" },
+  async discover(ctx) {
+    // LOCAL_PATH/taskmaster/ — a subdirectory, because these share the .json
+    // extension with the General-Knowledge source and must not be confused
+    // with it.
+    if (LOCAL_PATH) {
+      const dir = localDir("taskmaster");
+      const names = localFiles(dir, /\.json$/i);
+      if (names.length === 0) {
+        ctx.progress.log(
+          `  ${DIM}· no Taskmaster *.json in ${dir} — skipping${R}`,
+        );
+        return null;
+      }
+      return names.map((n) => ({
+        ...unit(n.path, n.path),
+        local: join(dir, n.path),
+        bytes: n.size,
+      }));
+    }
+    return (await listFiles(ctx)).map((f) => ({
+      ...unit(f.path, `${f.set}/${basename(f.path)}`),
+      url: `${TASKMASTER_RAW}/${f.path}`,
+      bytes: f.size,
+    }));
+  },
+};