@hviana/sema 0.5.8 → 0.6.0

package/example/train_base/corpora/genknow.ts

@@ -0,0 +1,114 @@
+// train_base/corpora/genknow.ts — MuskumPillerum/General-Knowledge Q&A
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { jsonArray } from "../readers.js";
+import { type Corpus, singleUnit } from "../corpus.js";
+
+// ── MuskumPillerum/General-Knowledge (the fourth training stage, after oasst2) ──
+// A ~37.6k-row general-knowledge Q&A set: each row is a single {Question, Answer}
+// pair. A row is a pure RELATION (question → answer), so it becomes exactly ONE
+// FACT, identical in shape to the Aya stage. It ships as a single JSON array
+// file (output.json); we DOWNLOAD it and stream the array. GENKNOW_URL overrides
+// the source.
+//
+// DISABLED BY DEFAULT ON LICENCE GROUNDS (2026-08-13). The HF repo carries NO
+// licence tag and no licence in its card — an earlier header in this file
+// claimed MIT without support — and its own dataset card states it "contains a
+// subset of the alpaca dataset". Alpaca is CC BY-NC 4.0: NonCommercial, which
+// conflicts with Sema's commercial licence. Because a Sema store retains its
+// training text VERBATIM, an unlicensed corpus inside it makes the whole
+// artifact undistributable. See DATASETS.md §3.2. GENKNOW=1 re-enables the
+// stage for local, non-distributed experiments only.
+const GENKNOW = env("GENKNOW", "0") !== "0";
+const GENKNOW_URL = env(
+  "GENKNOW_URL",
+  "https://huggingface.co/datasets/MuskumPillerum/General-Knowledge/resolve/main/output.json",
+);
+// A Question/Answer longer than this is skipped (answers run to a few hundred
+// chars; this only guards against a corrupt/runaway field).
+const MAX_GENKNOW_CHARS = Math.max(
+  4_000,
+  Math.floor(Number(env("MAX_GENKNOW_KB", "64")) * 1000) || 64_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6f  General-Knowledge parsing — a {Question, Answer} row → SEMA fact
+//
+// Each row is a single general-knowledge question with one answer — a pure
+// RELATION (question → answer), so it becomes exactly ONE FACT, like the Aya
+// stage. No experience (a fact is possible) and no cumulative walk (a lone Q&A
+// is not multi-turn). The source over-escapes newlines (a literal "\n" two-char
+// sequence) and leaves trailing whitespace, so answers are un-escaped and
+// trimmed to plain prose before deposit.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One normalized General-Knowledge row. */
+export interface GenKnowRow {
+  question: string;
+  answer: string;
+}
+
+/** Turn a source value into clean prose: decode the literal "\n"/"\t"/"\r"
+ *  two-character escapes the source JSON left in the text, collapse the runs of
+ *  whitespace that creates, and trim. */
+function unescapePlain(s: string): string {
+  return s
+    .replace(/\\r\\n|\\n|\\r/g, "\n")
+    .replace(/\\t/g, " ")
+    .replace(/[ \t]+/g, " ")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
+ *  a usable question/answer or a side is implausibly large (corruption). */
+export function toGenKnowRow(
+  row: unknown,
+  maxChars = MAX_GENKNOW_CHARS,
+): GenKnowRow | null {
+  if (!row || typeof row !== "object") return null;
+  const r = row as Record<string, unknown>;
+  const question = typeof r.Question === "string"
+    ? unescapePlain(r.Question)
+    : "";
+  const answer = typeof r.Answer === "string" ? unescapePlain(r.Answer) : "";
+  if (!question || !answer) return null;
+  if (
+    question.length > maxChars || answer.length > maxChars
+  ) return null;
+  return { question, answer };
+}
+
+/** Translate ONE General-Knowledge row into SEMA items: exactly one
+ *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
+export function genKnowRowToItems(row: GenKnowRow): TrainingItem[] {
+  return refineItems([{ context: row.question, continuation: row.answer }]);
+}
+
+export const genknow: Corpus = {
+  id: "genknow",
+  label: "General-Knowledge",
+  kind: "Q&A facts",
+  enabled: GENKNOW,
+  read: jsonArray(),
+  toItems: (row) => {
+    const r = toGenKnowRow(row);
+    return r ? genKnowRowToItems(r) : null;
+  },
+  discover: singleUnit({
+    // Resume id "genknow::qa" — the string this store already records.
+    key: "qa",
+    label: "General-Knowledge",
+    display: "General-Knowledge",
+    url: GENKNOW_URL,
+    dest: "general_knowledge.json",
+    localMatch: [/general.*knowledge.*\.json$/i, /output\.json$/i],
+    localWhat: "General-Knowledge *.json",
+  }),
+};

package/example/train_base/corpora/index.ts

@@ -0,0 +1,88 @@
+// train_base/corpora/index.ts — THE CURRICULUM.
+//
+// The order is the curriculum, and it is load-bearing: each stage runs only
+// after the previous one finishes, and every stage records itself in the same
+// completed-set, so a single store resumes the whole sequence.
+//
+//   1. SmolSent    sentence-level TRANSLATION pairs across 100+ low-resource
+//                  languages. Each pair is "two names for one meaning" → a
+//                  foreign→English translation FACT, so every language's
+//                  rendering of a meaning converges on ONE English node (cf.
+//                  test/05-concepts.test.mjs).
+//   2. Aya         ~204k human prompt→completion pairs, 70+ languages → one
+//                  (question → answer) FACT each.
+//   3. oasst2      MULTI-TURN human↔assistant conversation trees → the
+//                  accumulated-context walk (single-turn trees are skipped).
+//   4. Taskmaster  task-oriented DIALOGUE, the best-scoring corpora on the
+//                  fold-unit recurrence benchmark that predicts halo health.
+//   5. 2Wiki       the `evidences` TRIPLES — the one stage aimed at
+//                  COMPOSITION. Its Wikipedia passages and its composed
+//                  questions are deliberately NOT read.
+//   6. SODA        social/commonsense DIALOGUE, budgeted.
+//   7. MASSIVE     short intent utterances → ONE bare experience each.
+//                  DISABLED BY DEFAULT — edge-less content was measured to
+//                  manufacture answers where the store should stay silent.
+//   8. GenKnow     ~37.6k {Question, Answer} pairs → one FACT each. DISABLED
+//                  BY DEFAULT on licence grounds; see DATASETS.md §3.2.
+
+import type { Corpus } from "../corpus.js";
+import { smolsent } from "./smolsent.js";
+import { aya } from "./aya.js";
+import { oasst2 } from "./oasst2.js";
+import { taskmaster } from "./taskmaster.js";
+import { wiki2 } from "./wiki2.js";
+import { soda } from "./soda.js";
+import { massive } from "./massive.js";
+import { genknow } from "./genknow.js";
+
+export const CURRICULUM: Corpus[] = [
+  smolsent,
+  aya,
+  oasst2,
+  taskmaster,
+  wiki2,
+  soda,
+  massive,
+  genknow,
+];
+
+/** The corpora this RUN will train, for the panel header — derived, so it can
+ *  never go stale the way the hand-written string it replaces had. */
+export const enabledLabels = (): string =>
+  CURRICULUM.filter((c) => c.enabled).map((c) => c.label).join("+");
+
+/** The corpora a STORE contains: everything this run will train PLUS everything
+ *  an earlier run already did, in curriculum order.
+ *
+ *  Not the same question as `enabledLabels`, and conflating them wrote a false
+ *  statement into every store that was ever resumed with a different set of
+ *  stages enabled. Observed on a real store: `train.dataset` read
+ *  "SmolSent+Aya+oasst2" while the tally recorded 37,623 General-Knowledge
+ *  deposits sitting in it. That is not cosmetic — a Sema store retains its
+ *  training text VERBATIM, so `train.dataset` is the record of whose licence
+ *  terms travel with the artifact, and General-Knowledge is precisely the
+ *  corpus DATASETS.md §3.2 disables on NonCommercial grounds.
+ *
+ *  `trainedIds` are the corpus ids an earlier run deposited under (the keys of
+ *  the per-corpus tally). An id no longer in the curriculum cannot be named and
+ *  is dropped — the tally still carries it, which is where that evidence lives. */
+export const storedLabels = (trainedIds: Iterable<string>): string => {
+  const trained = new Set(trainedIds);
+  return CURRICULUM
+    .filter((c) => c.enabled || trained.has(c.id))
+    .map((c) => c.label)
+    .join("+");
+};
+
+// One re-export per corpus, carrying the descriptor AND its adapters. The
+// explicit `export { aya, genknow, … }` list that used to sit here named the
+// eight descriptors a second time; the star exports below already provide them,
+// and a hand-kept list of everything is exactly the thing that goes stale.
+export * from "./smolsent.js";
+export * from "./aya.js";
+export * from "./oasst2.js";
+export * from "./taskmaster.js";
+export * from "./wiki2.js";
+export * from "./soda.js";
+export * from "./massive.js";
+export * from "./genknow.js";

package/example/train_base/corpora/massive.ts

@@ -0,0 +1,111 @@
+// train_base/corpora/massive.ts — AmazonScience/massive short intents
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { parquet } from "../readers.js";
+import type { Corpus } from "../corpus.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+
+// MASSIVE deposits BARE UTTERANCES — an experience, not an episode — and that
+// is the only shape its data supports. Two richer shapes were considered and
+// rejected on evidence:
+//   • Same-intent pairs as paraphrases. 49.1% of consecutive rows share
+//     (locale, intent), but they are NOT meaning-equivalent: intent 48 in mn-MN
+//     runs "wake me at nine on the fifth" next to "set an alarm two hours from
+//     now". Depositing that pair as an episode teaches a continuation that does
+//     not exist.
+//   • Same-id rows across locales. Those ARE translations of one another —
+//     which is exactly SmolSent's relation, and SmolSent scores worst of every
+//     corpus measured on fold-unit recurrence (23.2%) because cross-lingual
+//     pairs share no units.
+// So the stage contributes recurring fold units and lexical coverage (65.1%
+// recurring unit mass, median 29 B) and nothing relational. `annot_utt` carries
+// slot markup ("[date : tavdahad] ...") and is never read.
+// DISABLED BY DEFAULT, on evidence gathered after the stage was written. A bare
+// experience deposits content with NO EDGE, and that cuts both ways. Measured on
+// a three-pair dialogue store with and without six MASSIVE-style utterances:
+//
+//   "set an alarm"            without: "Sure, what size would you like?"  (wrong)
+//                             with:    "set an alarm for seven"           (better)
+//   "play music"              without: ""                                 (correct silence)
+//                             with:    "Yes, sweetened or unsweetened?"   (wrong)
+//
+// So it displaces some wrong answers and manufactures others, INCLUDING turning
+// a correct silence into a wrong answer — and honest silence is a stated
+// property of this engine (AGENTS §2.13). On the mixed-curriculum store the
+// same shape produced the fragment "nus" for "wake me up at nine am".
+//
+// That evidence is four probes on toy stores and is NOT conclusive; it is,
+// however, the only evidence there is, and it points the wrong way. The stage
+// stays implemented and one env var away. Turn it on (MASSIVE=1) once there is
+// a real measurement showing the recurring fold units it contributes (72.3% of
+// deposited unit mass) buy more than the spurious answers cost.
+const MASSIVE = env("MASSIVE", "0") !== "0";
+const MASSIVE_DATASET = env("MASSIVE_DATASET", "AmazonScience/massive");
+// "all" is the config covering every locale in one set of shards.
+const MASSIVE_CONFIG = env("MASSIVE_CONFIG", "all");
+const MASSIVE_SPLITS = env("MASSIVE_SPLITS", "train")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// 0 = every row (587,214 in `all`/train, ~17 MB of content).
+const MASSIVE_MAX_ROWS = Math.max(
+  0,
+  Math.floor(Number(env("MASSIVE_MAX_ROWS", "0"))) || 0,
+);
+const MAX_MASSIVE_UTT_CHARS = Math.max(
+  100,
+  Math.floor(Number(env("MAX_MASSIVE_UTT_KB", "2")) * 1000) || 2_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e⁗  MASSIVE parsing — one short utterance → ONE SEMA experience
+//
+// See the constants note for why this deposits a bare experience and not a
+// relation: the two relational shapes this corpus appears to offer are both
+// false (same-intent rows are not paraphrases; same-id rows across locales are
+// translations, SmolSent's worst-scoring relation).
+// ═══════════════════════════════════════════════════════════════════════
+
+/** Translate ONE MASSIVE row into SEMA items: its bare utterance, as an
+ *  experience. `annot_utt` (slot-annotated) is deliberately not used — its
+ *  "[date : ...]" markup is not prose. Returns [] for an unusable row. */
+export function massiveRowToItems(
+  row: unknown,
+  maxChars = MAX_MASSIVE_UTT_CHARS,
+): TrainingItem[] {
+  if (!row || typeof row !== "object") return [];
+  const utt = (row as Record<string, unknown>).utt;
+  const text = typeof utt === "string" ? utt.trim() : "";
+  if (!text || text.length > maxChars) return [];
+  return refineItems([text]);
+}
+
+export const massive: Corpus = {
+  id: "massive",
+  label: "MASSIVE",
+  kind: "short intents",
+  enabled: MASSIVE,
+  maxRows: MASSIVE_MAX_ROWS,
+  // One of ten columns — `annot_utt`, whose slot markup the note above rejects,
+  // is not decoded either. Measured on the converted train shard: 92 MB
+  // uncompressed across all ten, 36 MB for `utt` (38.7%).
+  read: parquet({ columns: ["utt"] }),
+  toItems: (row) => {
+    const items = massiveRowToItems(row);
+    return items.length ? items : null;
+  },
+  unitNoun: "shard(s)",
+  log: { rows: "row(s)" },
+  discover: convertedParquetUnits({
+    id: "massive",
+    label: "MASSIVE",
+    dataset: MASSIVE_DATASET,
+    config: MASSIVE_CONFIG,
+    splits: MASSIVE_SPLITS,
+    localSub: "massive",
+  }),
+};

package/example/train_base/corpora/oasst2.ts

@@ -0,0 +1,163 @@
+// train_base/corpora/oasst2.ts — OpenAssistant/oasst2 conversation trees
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { accumulate, refineItems, type TrainingItem } from "../items.js";
+import { lines } from "../readers.js";
+import { type Corpus, singleUnit } from "../corpus.js";
+
+// ── OpenAssistant/oasst2 (the fourth training stage, after Aya) ──
+// oasst2 is a corpus of human↔assistant conversation TREES. Its richest, most
+// stream-friendly artifact is "<date>_oasst2_ready.trees.jsonl.gz": one JSON
+// conversation tree PER LINE, gzip-compressed (a web standard — Decompression
+// Stream("gzip")). Each tree is {message_tree_id, prompt:{role,text,replies:[…]}}
+// where `replies` nests recursively and a prompt can have several ranked
+// assistant replies (rank 0 = best). We follow the best-ranked, non-deleted
+// reply at each step to get ONE linear, strictly-alternating conversation per
+// tree, then keep only the MULTI-TURN ones (≥ OASST_MIN_TURNS messages, i.e. at
+// least two full user→assistant exchanges) — single Q→A trees are skipped, by
+// design. OASST=0 disables the stage; OASST_URL overrides the source.
+const OASST = env("OASST", "1") !== "0";
+const OASST_URL = env(
+  "OASST_URL",
+  "https://huggingface.co/datasets/OpenAssistant/oasst2/resolve/main/2023-11-05_oasst2_ready.trees.jsonl.gz",
+);
+// Multi-turn threshold: a conversation must have at least this many turns to be
+// trained (4 = user→assistant→user→assistant, the smallest real multi-turn).
+const OASST_MIN_TURNS = Math.max(
+  2,
+  Math.floor(Number(env("OASST_MIN_TURNS", "4"))) || 4,
+);
+// Skip a tree whose decoded JSON line exceeds this (a pathological record); the
+// real maximum is far smaller, so this only guards against corruption.
+const MAX_OASST_LINE_CHARS = Math.max(
+  100_000,
+  Math.floor(Number(env("MAX_OASST_LINE_MB", "8")) * 1_000_000) || 8_000_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e  OpenAssistant/oasst2 parsing — a conversation TREE → SEMA items
+//
+// Each tree is {prompt:{role,text,replies:[…]}}, replies nested recursively. A
+// prompt can have several ranked assistant replies; we collapse the tree to ONE
+// linear conversation by following the best-ranked (rank 0), non-deleted reply
+// at each step. The result strictly alternates prompter/assistant. Only MULTI-
+// TURN conversations (≥ OASST_MIN_TURNS messages) are kept — the explicit focus
+// of this stage; single Q→A trees are dropped.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** A single oasst2 message node (the fields we use; the tree nests via replies). */
+interface OasstNode {
+  role?: string;
+  text?: string;
+  rank?: number | null;
+  deleted?: boolean;
+  replies?: OasstNode[];
+}
+
+/** One conversational turn extracted from a tree. */
+export interface OasstTurn {
+  role: string; // "prompter" | "assistant"
+  text: string;
+}
+
+/** Collapse a conversation tree to ONE linear path: at each node, descend into
+ *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
+ *  Returns the ordered turns (already strictly alternating in this corpus). */
+export function bestOasstPath(root: OasstNode): OasstTurn[] {
+  const turns: OasstTurn[] = [];
+  let node: OasstNode | undefined = root;
+  while (node) {
+    const text = typeof node.text === "string" ? node.text.trim() : "";
+    if (text) turns.push({ role: String(node.role ?? "?"), text });
+    const live: OasstNode[] = (node.replies ?? []).filter((r: OasstNode) =>
+      r && !r.deleted && typeof r.text === "string" && r.text.trim() !== ""
+    );
+    if (live.length === 0) break;
+    live.sort((a: OasstNode, b: OasstNode) =>
+      (a.rank ?? Number.MAX_SAFE_INTEGER) - (b.rank ?? Number.MAX_SAFE_INTEGER)
+    );
+    node = live[0];
+  }
+  return turns;
+}
+
+/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
+ *
+ *  This is the ONE stage where cumulative continuous context is truly necessary:
+ *  the data is a real multi-turn dialogue, and what must be learned is how each
+ *  turn follows from the WHOLE conversation so far — not from the previous turn
+ *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
+ *  turn experiences and local adjacent-pair facts are NOT emitted (they are
+ *  subsumed by it and would merely replicate the content).
+ *
+ *  The walk is the pattern proven in test/13-conversation.test.mjs
+ *  ("teachConversation"): each turn is the continuation of all prior turns,
+ *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
+ *  (cumulative context → next turn); the join string is not, and does not need
+ *  to be — that file joins with nothing and this corpus joins with "\n" (see
+ *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
+ *  reading that the two must agree on a separator.  They must not agree,
+ *  because there is nothing to agree about: turn boundaries are offsets, and
+ *  the join string is just corpus text. Roles already
+ *  alternate by position in an oasst2 best-path (the root is a prompter), so a
+ *  label adds nothing the position does not, while a clean continuation matches
+ *  the test's recall (predictNext queries bare prior turns) and lets a turn share
+ *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
+ *
+ *  Returns [] for a conversation below the multi-turn threshold, so callers can
+ *  simply skip empties. */
+export function oasstConversationToItems(
+  turns: OasstTurn[],
+  minTurns = OASST_MIN_TURNS,
+): TrainingItem[] {
+  if (turns.length < minTurns) return []; // not multi-turn — skip
+  return refineItems(accumulate(turns.map((t) => t.text)));
+}
+
+/** The row adapter: ONE line of the tree dump → its deposits. Returns null for
+ *  a tree with no prompt and for every single-turn tree — the latter is the
+ *  stage's design, not a defect, which is why the reader counts it `unusable`
+ *  rather than `skipped`. */
+export function oasstTreeToItems(row: unknown): TrainingItem[] | null {
+  if (!row || typeof row !== "object") return null;
+  const tree = row as { prompt?: OasstNode };
+  if (!tree.prompt) return null;
+  const items = oasstConversationToItems(bestOasstPath(tree.prompt));
+  return items.length ? items : null;
+}
+
+export const oasst2: Corpus = {
+  id: "oasst2",
+  label: "oasst2",
+  kind: "multi-turn chat",
+  enabled: OASST,
+  read: lines({ gzip: true, maxLineChars: MAX_OASST_LINE_CHARS }),
+  toItems: oasstTreeToItems,
+  // The one corpus that KEEPS a cached file after a complete read: a copy left
+  // behind by a previous interrupted run is not this stage's to reclaim.
+  keepCached: true,
+  log: {
+    deposits: "examples",
+    // A tree that yields items is exactly a tree that cleared OASST_MIN_TURNS.
+    rows: "conversation(s)",
+    // The single-turn trees this stage drops BY DESIGN must not be reported as
+    // damage — only genuinely malformed lines are.
+    malformedOnly: true,
+  },
+  discover: singleUnit({
+    // Resume id "oasst2::trees" — the string this store already records.
+    key: "trees",
+    label: "oasst2",
+    display: "oasst2 (multi-turn)",
+    url: OASST_URL,
+    dest: "oasst2_ready.trees.jsonl.gz",
+    acquireLabel: "oasst2 trees",
+    localMatch: [/oasst.*trees.*\.jsonl\.gz$/i, /oasst.*\.jsonl\.gz$/i],
+    localWhat: "oasst2 *trees*.jsonl.gz",
+  }),
+};