@hviana/sema 0.5.9 → 0.6.0

package/example/train_base/cache.ts

@@ -0,0 +1,251 @@
+// train_base/cache.ts — the durable disk cache and the download sink.
+//
+// This is the ONE place the trainer needs Node rather than the web platform:
+// every other byte in the pipeline moves through fetch, WHATWG streams,
+// DecompressionStream, TextDecoderStream and Blob, but writing a file is the
+// single capability the web platform does not expose. So the sink below wraps a
+// raw fs descriptor, and nothing else here does.
+//
+// Two invariants the rest of the trainer relies on:
+//   • ATOMIC — a download streams to "<file>.part", is fsync'd, then renamed
+//     into place. A file at its final path is, by construction, complete, so an
+//     interrupted download can never be mistaken for a cached one.
+//   • BOUNDED — a download blocks under the MAX_CACHE_GB ceiling, and a fully
+//     processed file is deleted by its caller immediately.
+
+import {
+  CACHE_DIR,
+  CACHE_WAIT_MS,
+  MAX_CACHE_BYTES,
+  PART_SUFFIX,
+} from "./config.js";
+import { httpError, retry, waitMs } from "./http.js";
+import {
+  closeSync,
+  existsSync,
+  fsyncSync,
+  mkdirSync,
+  openSync,
+  readdirSync,
+  renameSync,
+  statSync,
+  unlinkSync,
+  writeSync,
+} from "node:fs";
+import { basename, join } from "node:path";
+
+/** Delete every orphaned "<file>.part" in the cache, returning how many were
+ *  removed and the bytes they held.
+ *
+ *  A .part file at rest is by definition the debris of a download that never
+ *  finished — the rename that promotes one is the last step of `downloadFile`,
+ *  so a live .part exists only while THIS process is writing it. Sweeping at
+ *  startup is therefore safe, and it is load-bearing rather than cosmetic:
+ *  `cacheSize` deliberately counts .part files (an in-flight download really
+ *  does occupy the disk), so debris left by a killed run consumes ceiling
+ *  budget that nothing would ever free, and `ensureCacheRoom` would wait for
+ *  room that cannot appear.
+ *
+ *  The one assumption is that a cache directory belongs to ONE run at a time.
+ *  That was already true — two trainers sharing CACHE_DIR would write the same
+ *  .part path — so this adds no constraint that did not exist. */
+export function sweepPartials(): { files: number; bytes: number } {
+  const out = { files: 0, bytes: 0 };
+  if (!existsSync(CACHE_DIR)) return out;
+  for (const name of readdirSync(CACHE_DIR)) {
+    if (!name.endsWith(PART_SUFFIX)) continue;
+    const p = join(CACHE_DIR, name);
+    try {
+      const size = statSync(p).size;
+      unlinkSync(p);
+      out.files++;
+      out.bytes += size;
+    } catch { /* raced with another delete — nothing to reclaim */ }
+  }
+  return out;
+}
+
+/** Total bytes currently held in the cache directory — INCLUDING any .part
+ *  file, because an in-flight download occupies the disk like any other file.
+ *  Orphaned ones are removed by {@link sweepPartials} at startup. */
+export function cacheSize(): number {
+  if (!existsSync(CACHE_DIR)) return 0;
+  let total = 0;
+  for (const name of readdirSync(CACHE_DIR)) {
+    try {
+      total += statSync(join(CACHE_DIR, name)).size;
+    } catch { /* raced with a delete */ }
+  }
+  return total;
+}
+
+/** Block until there is room for a file of `fileBytes` under the ceiling.
+ *  A single file larger than the whole ceiling can never "fit", so we let it
+ *  through (it is deleted right after processing) rather than wait forever. */
+export async function ensureCacheRoom(
+  fileBytes: number,
+  signal: AbortSignal,
+  warn?: (msg: string) => void,
+  maxWaitMs = CACHE_WAIT_MS,
+): Promise<void> {
+  mkdirSync(CACHE_DIR, { recursive: true });
+  if (fileBytes >= MAX_CACHE_BYTES) return;
+  let warned = false;
+  const until = Date.now() + maxWaitMs;
+  // Stop waiting the moment a shutdown is requested — the abort signal unblocks
+  // a long cache-full wait so Ctrl+C is never swallowed by the ceiling.
+  while (!signal.aborted && cacheSize() + fileBytes > MAX_CACHE_BYTES) {
+    // BOUNDED. Room appears when this run consumes and deletes a file, so a
+    // cache already over the ceiling with nothing left to consume — stale files
+    // from another run, a ceiling set below one corpus — would otherwise wait
+    // for room that cannot arrive, forever, after a single warning line.
+    if (Date.now() >= until) {
+      throw new Error(
+        `cache still full after ${Math.round(maxWaitMs / 60_000)} min ` +
+          `(${(cacheSize() / 1e9).toFixed(1)} GB of a ` +
+          `${(MAX_CACHE_BYTES / 1e9).toFixed(0)} GB ceiling) — raise ` +
+          `MAX_CACHE_GB or clear ${CACHE_DIR}`,
+      );
+    }
+    if (!warned) {
+      warn?.(
+        `cache at ${
+          (MAX_CACHE_BYTES / 1e9).toFixed(0)
+        } GB ceiling — waiting for room…`,
+      );
+      warned = true;
+    }
+    await waitMs(5_000, signal);
+  }
+}
+
+export interface DownloadOptions {
+  signal: AbortSignal;
+  tries: number;
+  onFail?: (attempt: number, err: Error) => void;
+  onProgress?: (done: number, total: number) => void;
+}
+
+/** Stream `url` to `destPath`, atomically and with backpressure. */
+export async function downloadFile(
+  url: string,
+  destPath: string,
+  opts: DownloadOptions,
+): Promise<void> {
+  const partPath = destPath + PART_SUFFIX;
+  await retry(
+    `download ${basename(destPath)}`,
+    async () => {
+      // Abort promptly on shutdown rather than waiting out a slow socket.
+      if (opts.signal.aborted) {
+        const e: Error & { fatal?: boolean } = new Error("aborted");
+        e.fatal = true;
+        throw e;
+      }
+      const res = await fetch(url, { signal: opts.signal });
+      if (!res.ok) throw httpError(res);
+      if (!res.body) throw new Error("empty response body");
+
+      // `content-length` describes the bytes ON THE WIRE. When the server
+      // applied a content-coding, fetch hands us the DECODED body, so the
+      // header no longer describes what gets written to disk and the integrity
+      // guard below must not use it. Measured: raw.githubusercontent.com sends
+      // `content-encoding: gzip` with content-length 110,928 for a file that
+      // decodes to 1,607,931 bytes — a size check against that rejects every
+      // healthy download. (The bug stayed latent because Hugging Face sends
+      // `content-encoding: br` and NO content-length, leaving total = 0, which
+      // already disables the guard.)
+      const encoding = (res.headers.get("content-encoding") ?? "").trim()
+        .toLowerCase();
+      const decoded = encoding !== "" && encoding !== "identity";
+      const total = decoded
+        ? 0
+        : Number(res.headers.get("content-length")) || 0;
+      let done = 0;
+
+      // Stream straight to a ".part" sibling using pure WHATWG streams. A
+      // TransformStream meters progress; pipeTo into a WritableStream gives REAL
+      // backpressure natively — the sink's write() returns a promise the
+      // readable side awaits, so a fast server can never outrun the disk (no
+      // whole-file heap buffering). The sink wraps a single raw fs descriptor
+      // (the one capability the web platform lacks); writing to disk is the only
+      // Node operation in the whole pipeline. The final, valid file only ever
+      // appears via the atomic rename below, so a crash mid-transfer can never
+      // leave a truncated file at the real path.
+      const meter = new TransformStream<Uint8Array, Uint8Array>({
+        transform(chunk, controller) {
+          done += chunk.length;
+          opts.onProgress?.(done, total);
+          controller.enqueue(chunk);
+        },
+      });
+
+      const fd = openSync(partPath, "w");
+      let closed = false;
+      const closeFd = () => {
+        if (closed) return;
+        closed = true;
+        try {
+          closeSync(fd);
+        } catch { /* already closed */ }
+      };
+      const sink = new WritableStream<Uint8Array>({
+        write(chunk) {
+          // writeSync drains the whole chunk before returning, so the readable
+          // side is paused for exactly as long as the disk needs — backpressure.
+          let off = 0;
+          while (off < chunk.length) {
+            off += writeSync(fd, chunk, off, chunk.length - off);
+          }
+        },
+        close() {
+          fsyncSync(fd); // durable bytes before the rename promotes them
+          closeFd();
+        },
+        abort() {
+          closeFd();
+        },
+      });
+
+      try {
+        await res.body.pipeThrough(meter).pipeTo(sink, {
+          signal: opts.signal,
+        });
+      } catch (e) {
+        // pipeTo's abort() ran the sink's abort() (closing the descriptor); if
+        // it didn't (a non-abort throw), make sure the descriptor is not leaked.
+        closeFd();
+        try {
+          unlinkSync(partPath);
+        } catch { /* best effort */ }
+        throw e;
+      }
+
+      // Optional integrity guard: when the server advertised a size FOR THE
+      // BYTES WE WRITE (see the content-encoding note above — `total` is 0 for
+      // a decoded body, which disables this), a complete file must match it. A
+      // short read (silent truncation) is retried rather than promoted, so the
+      // parser never sees a partial file.
+      try {
+        const got = statSync(partPath).size;
+        if (total > 0 && got !== total) {
+          try {
+            unlinkSync(partPath);
+          } catch { /* best effort */ }
+          throw new Error(`size mismatch: got ${got}, expected ${total}`);
+        }
+      } catch (e) {
+        if (e instanceof Error && e.message.startsWith("size mismatch")) {
+          throw e;
+        }
+        // statSync failure is non-fatal here; the rename below will surface it.
+      }
+
+      // Atomic publish: rename is atomic within a filesystem, so the final path
+      // flips from "absent" to "complete" in one step — never an in-between.
+      renameSync(partPath, destPath);
+    },
+    opts.tries,
+    { signal: opts.signal, onFail: opts.onFail },
+  );
+}

package/example/train_base/config.ts

@@ -0,0 +1,128 @@
+// train_base/config.ts — RUN-LEVEL configuration, all from the environment.
+//
+// Only knobs that describe the RUN live here: the store, the checkpoint cadence,
+// the cache ceiling, the read budgets, the caps. A knob that describes ONE
+// CORPUS (which pairs of SmolSent, how many SODA dialogues, how long an Aya
+// field may be) belongs next to that corpus's adapter, together with the
+// evidence that fixed its default — see AGENTS.md §2.16: a comment carries the
+// constraint, and a constraint is only readable beside the code it constrains.
+
+import { join } from "node:path";
+
+/** Read an environment variable, or `d` when it is unset. */
+export const env = (k: string, d: string): string => process.env[k] ?? d;
+
+export const DB_PATH = env("DB_PATH", "sema"); // → {DB_PATH}.sqlite
+export const D = Number(env("D", "1024"));
+export const SEED = Number(env("SEED", "7"));
+
+// Checkpoint cadence is measured in LEARNED CONTENT, not deposits: a snapshot
+// every CHECKPOINT_MB megabytes of trained UTF-8 content (decimal MB, matching
+// the bytes() helper). A floor of 1 MB: a zero/NaN value must not make every
+// deposit checkpoint, nor silently disable checkpointing. The tail (a run that
+// learns less than one interval, or the remainder past the last interval) is
+// always saved by finish() at exit — a complete point.
+export const CHECKPOINT_BYTES = Math.max(
+  1_000_000,
+  Math.floor(Number(env("CHECKPOINT_MB", "100")) * 1_000_000) || 100_000_000,
+);
+
+// Target size of ONE materialised Parquet read, in uncompressed source bytes.
+// A row-GROUP is a layout choice made by whoever wrote the file, not a memory
+// budget: Aya ships 203 groups of 1,000 rows (~1 MB each), while SODA ships ONE
+// group of 1,191,582 rows (1.19 GB uncompressed) and 2Wiki ONE of 167,454
+// (666 MB). Reading "exactly one row-group" is therefore safe for the first and
+// fatal for the others, so reads are sized in BYTES instead — see
+// `parquetBatchRows`. Materialised JS objects cost several times their source
+// bytes, hence a default well under available memory.
+export const PARQUET_BATCH_BYTES = Math.max(
+  1_000_000,
+  Math.floor(Number(env("PARQUET_BATCH_MB", "32")) * 1_000_000) || 32_000_000,
+);
+
+export const LOCAL_PATH = env("LOCAL_PATH", ""); // train from a local dir
+export const CACHE_DIR = env("CACHE_DIR", join(process.cwd(), "cache"));
+export const MAX_CACHE_BYTES = Number(env("MAX_CACHE_GB", "100")) * 1e9;
+export const PROGRESS_MS = Number(env("PROGRESS_MS", "250")); // panel cadence
+
+// Index maintenance at checkpoints: compact (remove garbage), repair (fill
+// gaps), then refresh the canonical-form index (equivalence-class resolution —
+// src/canon.ts). All three are idempotent batch operations (the canon build is
+// additionally incremental via the store's `canon.upto` cursor);
+// INDEX_MAINTENANCE=0 disables.
+export const INDEX_MAINTENANCE = env("INDEX_MAINTENANCE", "1") !== "0";
+export const DOWNLOAD_TRIES = 5;
+
+// In-progress downloads are written to a sibling "<dest>.part" and atomically
+// renamed into place only after the bytes are fully flushed to disk. The cache
+// invariant is therefore absolute: a file at its final path is, by definition,
+// complete. Partial transfers (a crash, a kill, a dropped socket) leave only a
+// .part file, which is never fed to the parser and is swept at startup by
+// cache.ts's sweepPartials() — without which the debris would consume cache
+// ceiling that nothing frees.
+export const PART_SUFFIX = ".part";
+
+// The checkpoint recall is a best-effort diagnostic — it must NEVER stall
+// training. We bound it so a slow/large store cannot freeze the deposit loop.
+export const INFER_TIMEOUT_MS = Number(env("INFER_TIMEOUT_MS", "15000"));
+
+// How long the run may make NO progress before it gives up and exits non-zero.
+//
+// A long training run's worst failure is not a crash — a crash resumes. It is a
+// HANG: the uncaught-exception handler deliberately swallows dropped-connection
+// errors so a long run survives them, and the keep-alive timer deliberately
+// holds the process open; together, an error that escapes and leaves an await
+// unsettled produces a live process that will never do anything again. No error,
+// no exit, and a supervisor that sees a healthy pid. Exiting instead turns that
+// into a resume, which costs at most the work since the last checkpoint.
+//
+// "Progress" is any deposit, downloaded chunk, or rate-limit wait; time inside
+// index maintenance and the checkpoint recall does not count against it, since
+// those legitimately deposit nothing. Generous by default — this is a
+// last-resort backstop, not a latency budget. 0 disables it.
+export const STALL_MS = Math.max(
+  0,
+  Math.floor(Number(env("STALL_MIN", "15")) * 60_000) || 900_000,
+);
+
+// How long a download may wait for room under the cache ceiling before failing
+// the unit instead of waiting forever. The wait exists so a bounded cache can
+// throttle a fast source; it is not meant to outlast the run. The unit stays
+// resumable, so a genuine ceiling problem costs a retry, not the corpus.
+export const CACHE_WAIT_MS = Math.max(
+  60_000,
+  Math.floor(Number(env("CACHE_WAIT_MIN", "10")) * 60_000) || 600_000,
+);
+
+// The vector indices' memory knob (MiB) — each index's SQLite page cache.
+// The IVF index routes inserts through a RAM-resident pivot table and
+// appends to chunk blobs, so this cache mostly serves query-time cluster
+// scans; 256 MiB comfortably covers the probed working set of a trained
+// store.  Override with VECTOR_CACHE_MB (64 is the library default).
+export const VECTOR_CACHE_MB = Math.max(
+  0,
+  Number(env("VECTOR_CACHE_MB", "256")),
+);
+
+// Page cache for the MAIN DAG database (node/kid/edge/contain tables).
+// Training issues millions of content-addressed point probes per session
+// against a GB-scale file; the library default (64 MiB) is sized for a
+// small machine — a training box affords more.  Override with
+// SQLITE_CACHE_MB.
+export const SQLITE_CACHE_MB = Math.max(
+  0,
+  Number(env("SQLITE_CACHE_MB", "256")),
+);
+
+// Optional ceiling on how much LEARNED CONTENT to train, in megabytes (decimal,
+// like CHECKPOINT_MB). Default Infinity = unbounded. The cap is checked against
+// trainedContentBytes after each deposit, so a run stops at the first item that
+// carries the running total to/past the ceiling (that item is still counted).
+export const MAX_MB = Number(env("MAX_MB", "Infinity"));
+if (isNaN(MAX_MB) || MAX_MB < 0) {
+  process.stderr.write(
+    `fatal: MAX_MB must be a non-negative number or "Infinity"\n`,
+  );
+  process.exit(1);
+}
+export const MAX_BYTES = MAX_MB * 1_000_000; // Infinity stays Infinity

package/example/train_base/corpora/aya.ts

@@ -0,0 +1,106 @@
+// train_base/corpora/aya.ts — CohereLabs/aya_dataset prompt→completion pairs
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { parquet } from "../readers.js";
+import { type Corpus, singleUnit } from "../corpus.js";
+
+// ── CohereLabs/aya_dataset (the second training stage, after SmolSent) ──
+// The Aya Dataset is ~204k HUMAN-annotated prompt→completion pairs across 70+
+// languages, each a clean (inputs → targets) fact in a named language. It ships
+// ONLY as Snappy-compressed Parquet (no JSONL/CSV). We DOWNLOAD the one train
+// Parquet file and read it row-group by row-group with `hyparquet` (a pure-JS,
+// dependency-free Parquet reader) + `hyparquet-compressors` (Snappy) over a
+// web-standard Blob byte source — no whole-file-in-memory load. AYA=0 disables
+// the stage; AYA_URL overrides the Parquet source.
+const AYA = env("AYA", "1") !== "0";
+const AYA_URL = env(
+  "AYA_URL",
+  "https://huggingface.co/datasets/CohereLabs/aya_dataset/resolve/main/data/train-00000-of-00001.parquet",
+);
+// A single Aya field this many chars or longer is skipped: inputs/targets range
+// up to ~3.3M chars, and a multi-MB "pair" is documentation/dump noise, not a
+// cognitive example.
+const MAX_AYA_FIELD_CHARS = Math.max(
+  10_000,
+  Math.floor(Number(env("MAX_AYA_FIELD_KB", "256")) * 1000) || 256_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6d  Aya Dataset parsing — a human prompt→completion row → SEMA items
+//
+// Each Aya row is a single human-written (inputs → targets) pair in a named
+// language, e.g. {inputs:"Qual é a capital da Índia?", targets:"Nova Déli.",
+// language:"Portuguese", …}. That is already the canonical SEMA fact (ask →
+// answer), so the translation is direct: exactly ONE (question → answer) fact.
+// reasoning/scratch-work fields do not exist in this corpus, so nothing is
+// stripped; the text is human prose already.
+
+/** One normalized Aya row. */
+export interface AyaRow {
+  inputs: string;
+  targets: string;
+  language: string;
+}
+
+/** Normalize a raw datasets-server row object into an AyaRow, or null when it
+ *  lacks a usable prompt/answer or a field is implausibly large (a dump, not a
+ *  cognitive example). Trims surrounding whitespace; keeps inner text verbatim
+ *  (human prose, possibly multi-paragraph). */
+export function toAyaRow(
+  row: unknown,
+  maxChars = MAX_AYA_FIELD_CHARS,
+): AyaRow | null {
+  if (!row || typeof row !== "object") return null;
+  const r = row as Record<string, unknown>;
+  const inputs = typeof r.inputs === "string" ? r.inputs.trim() : "";
+  const targets = typeof r.targets === "string" ? r.targets.trim() : "";
+  if (!inputs || !targets) return null;
+  if (
+    inputs.length > maxChars || targets.length > maxChars
+  ) return null;
+  const language = typeof r.language === "string" ? r.language.trim() : "";
+  return { inputs, targets, language };
+}
+
+/** Translate ONE Aya row into SEMA training items. A row is a single human
+ *  (question → answer) exchange — exactly one FACT, the (inputs → targets) edge.
+ *  No standalone-answer experience and no one-exchange "cumulative" walk: a lone
+ *  Q→A is not multi-turn, and both would only replicate the same edge. */
+export function ayaRowToItems(row: AyaRow): TrainingItem[] {
+  const { inputs, targets } = row;
+  return refineItems([{ context: inputs, continuation: targets }]);
+}
+
+export const aya: Corpus = {
+  id: "aya",
+  label: "Aya Dataset",
+  kind: "multilingual chat",
+  enabled: AYA,
+  // The three columns toAyaRow reads, out of six. Kept for the same reason as
+  // the other Parquet stages — the read states what the adapter uses — though
+  // here it is nearly free rather than a saving: measured on the train file,
+  // 238 MB uncompressed across all six and 233 MB for these three (98.2%). The
+  // dropped columns are ids and annotation metadata, so there is little to
+  // drop.
+  read: parquet({ columns: ["inputs", "targets", "language"] }),
+  toItems: (row) => {
+    const r = toAyaRow(row);
+    return r ? ayaRowToItems(r) : null;
+  },
+  discover: singleUnit({
+    // Resume id "aya::dataset" — the string this store already records.
+    key: "dataset",
+    label: "Aya Dataset",
+    display: "Aya Dataset",
+    url: AYA_URL,
+    dest: "aya_train.parquet",
+    localMatch: [/aya.*\.parquet$/i, /\.parquet$/i],
+    localWhat: "Aya *.parquet",
+  }),
+};

package/example/train_base/corpora/converted-parquet.ts

@@ -0,0 +1,64 @@
+// train_base/corpora/converted-parquet.ts — the work-list shape three corpora
+// share.
+//
+// Read from Hugging Face's auto-converted `refs/convert/parquet` branch, not
+// from main: a dataset's own Parquet may be written as ONE giant row-group
+// (SODA's is 1,191,582 rows) and a column chunk is per-group, so any read of it
+// materialises the whole file. The converted branch uses uniform 10,000-row
+// groups.
+
+import { LOCAL_PATH } from "../config.js";
+import { hfConvertedParquet, localFiles } from "../discovery.js";
+import { localDir, type Unit } from "../corpus.js";
+import type { TrainCtx } from "../runtime.js";
+import { DIM, R, YEL } from "../ui.js";
+import { join } from "node:path";
+
+/** The work-list of a corpus read from Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch. Shared by 2Wiki, SODA and MASSIVE: they
+ *  differ in their dataset, their budget and their adapter — nothing else. */
+export function convertedParquetUnits(opts: {
+  id: string;
+  label: string;
+  dataset: string;
+  config: string;
+  splits: string[];
+  /** Subdirectory of LOCAL_PATH holding pre-downloaded shards. */
+  localSub: string;
+}): (ctx: TrainCtx) => Promise<Unit[] | null> {
+  return async (ctx: TrainCtx) => {
+    if (LOCAL_PATH) {
+      const dir = localDir(opts.localSub);
+      const names = localFiles(dir, /\.parquet$/i);
+      if (names.length === 0) {
+        ctx.progress.log(
+          `  ${DIM}· no ${opts.label} *.parquet in ${dir} — skipping${R}`,
+        );
+        return null;
+      }
+      return names.map((n) => ({
+        key: n.path,
+        name: n.path,
+        display: `${opts.label} ${n.path}`,
+        local: join(dir, n.path),
+        bytes: n.size,
+      }));
+    }
+    const paths = await hfConvertedParquet(
+      opts.dataset,
+      opts.config,
+      opts.splits,
+      opts.label,
+      ctx.http,
+      (m) => ctx.progress.log(`  ${YEL}⚠${R} ${m}`),
+    );
+    return paths.map(({ path, size }) => ({
+      key: path,
+      name: path,
+      display: `${opts.label} ${path}`,
+      url: `https://huggingface.co/datasets/${opts.dataset}` +
+        `/resolve/refs%2Fconvert%2Fparquet/${path}`,
+      bytes: size,
+    }));
+  };
+}

package/example/train_base/corpora/genknow.ts

@@ -0,0 +1,114 @@
+// train_base/corpora/genknow.ts — MuskumPillerum/General-Knowledge Q&A
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { jsonArray } from "../readers.js";
+import { type Corpus, singleUnit } from "../corpus.js";
+
+// ── MuskumPillerum/General-Knowledge (the fourth training stage, after oasst2) ──
+// A ~37.6k-row general-knowledge Q&A set: each row is a single {Question, Answer}
+// pair. A row is a pure RELATION (question → answer), so it becomes exactly ONE
+// FACT, identical in shape to the Aya stage. It ships as a single JSON array
+// file (output.json); we DOWNLOAD it and stream the array. GENKNOW_URL overrides
+// the source.
+//
+// DISABLED BY DEFAULT ON LICENCE GROUNDS (2026-08-13). The HF repo carries NO
+// licence tag and no licence in its card — an earlier header in this file
+// claimed MIT without support — and its own dataset card states it "contains a
+// subset of the alpaca dataset". Alpaca is CC BY-NC 4.0: NonCommercial, which
+// conflicts with Sema's commercial licence. Because a Sema store retains its
+// training text VERBATIM, an unlicensed corpus inside it makes the whole
+// artifact undistributable. See DATASETS.md §3.2. GENKNOW=1 re-enables the
+// stage for local, non-distributed experiments only.
+const GENKNOW = env("GENKNOW", "0") !== "0";
+const GENKNOW_URL = env(
+  "GENKNOW_URL",
+  "https://huggingface.co/datasets/MuskumPillerum/General-Knowledge/resolve/main/output.json",
+);
+// A Question/Answer longer than this is skipped (answers run to a few hundred
+// chars; this only guards against a corrupt/runaway field).
+const MAX_GENKNOW_CHARS = Math.max(
+  4_000,
+  Math.floor(Number(env("MAX_GENKNOW_KB", "64")) * 1000) || 64_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6f  General-Knowledge parsing — a {Question, Answer} row → SEMA fact
+//
+// Each row is a single general-knowledge question with one answer — a pure
+// RELATION (question → answer), so it becomes exactly ONE FACT, like the Aya
+// stage. No experience (a fact is possible) and no cumulative walk (a lone Q&A
+// is not multi-turn). The source over-escapes newlines (a literal "\n" two-char
+// sequence) and leaves trailing whitespace, so answers are un-escaped and
+// trimmed to plain prose before deposit.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One normalized General-Knowledge row. */
+export interface GenKnowRow {
+  question: string;
+  answer: string;
+}
+
+/** Turn a source value into clean prose: decode the literal "\n"/"\t"/"\r"
+ *  two-character escapes the source JSON left in the text, collapse the runs of
+ *  whitespace that creates, and trim. */
+function unescapePlain(s: string): string {
+  return s
+    .replace(/\\r\\n|\\n|\\r/g, "\n")
+    .replace(/\\t/g, " ")
+    .replace(/[ \t]+/g, " ")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
+ *  a usable question/answer or a side is implausibly large (corruption). */
+export function toGenKnowRow(
+  row: unknown,
+  maxChars = MAX_GENKNOW_CHARS,
+): GenKnowRow | null {
+  if (!row || typeof row !== "object") return null;
+  const r = row as Record<string, unknown>;
+  const question = typeof r.Question === "string"
+    ? unescapePlain(r.Question)
+    : "";
+  const answer = typeof r.Answer === "string" ? unescapePlain(r.Answer) : "";
+  if (!question || !answer) return null;
+  if (
+    question.length > maxChars || answer.length > maxChars
+  ) return null;
+  return { question, answer };
+}
+
+/** Translate ONE General-Knowledge row into SEMA items: exactly one
+ *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
+export function genKnowRowToItems(row: GenKnowRow): TrainingItem[] {
+  return refineItems([{ context: row.question, continuation: row.answer }]);
+}
+
+export const genknow: Corpus = {
+  id: "genknow",
+  label: "General-Knowledge",
+  kind: "Q&A facts",
+  enabled: GENKNOW,
+  read: jsonArray(),
+  toItems: (row) => {
+    const r = toGenKnowRow(row);
+    return r ? genKnowRowToItems(r) : null;
+  },
+  discover: singleUnit({
+    // Resume id "genknow::qa" — the string this store already records.
+    key: "qa",
+    label: "General-Knowledge",
+    display: "General-Knowledge",
+    url: GENKNOW_URL,
+    dest: "general_knowledge.json",
+    localMatch: [/general.*knowledge.*\.json$/i, /output\.json$/i],
+    localWhat: "General-Knowledge *.json",
+  }),
+};