@hviana/sema 0.5.9 → 0.7.1

package/example/train_base/corpora/wiki2.ts

@@ -0,0 +1,190 @@
+// train_base/corpora/wiki2.ts — 2WikiMultihopQA evidence triples
+//
+// Knobs, the row adapter, and the stage descriptor for ONE corpus, together
+// with the evidence that fixed each default. A corpus file owns everything
+// source-specific; the contract it fills is ../corpus.ts and the loop that runs
+// it is ../stage.ts.
+
+import { env } from "../config.js";
+import { refineItems, type TrainingItem } from "../items.js";
+import { parquet } from "../readers.js";
+import type { Corpus } from "../corpus.js";
+import { convertedParquetUnits } from "./converted-parquet.js";
+
+// ── 2WikiMultihopQA — the `evidences` TRIPLES only (the composition stage) ──
+// Each row carries `evidences`: a JSON string of (subject, relation, object)
+// triples that CHAIN — one triple's object is the next's subject. 72.5% of rows
+// carry such a chain (measured over 4,000 rows), and those triples are the only
+// representation measured to make Sema compose a two-hop answer at all.
+//
+// TWO COLUMNS ARE DELIBERATELY NOT READ, one for licence reasons and one for
+// capability reasons:
+//   • `context` holds Wikipedia PROSE. The repo is Apache-2.0 but Wikipedia text
+//     is CC BY-SA, and a Sema store keeps text verbatim, so ingesting the
+//     passages would attach ShareAlike to every distributed store. The triples
+//     originate in Wikidata (CC0). See DATASETS.md §3.2/§4.
+//   • `question`/`answer` are the composed multi-hop QUESTION. Depositing those
+//     teaches the answer to that exact question and nothing else — it memorises
+//     rather than composes. They are used to EVALUATE this adapter, never as
+//     training input.
+//
+// Read from Hugging Face's auto-converted `refs/convert/parquet` branch, not
+// from main: the main-branch train.parquet is written as ONE 167,454-row
+// group (666 MB uncompressed) and a Parquet column chunk is per-group, so any
+// read of it materialises the whole file. The converted branch uses uniform
+// 10,000-row groups, which `parquetBatchRows` then subdivides by BYTES.
+const WIKI2 = env("WIKI2", "1") !== "0";
+const WIKI2_DATASET = env("WIKI2_DATASET", "xanhho/2WikiMultihopQA");
+// Splits to train, in order. Only `train` by default: `validation`/`test` are
+// the dataset's held-out sets and are what an honest evaluation of this
+// adapter's composition rate has to be measured on.
+const WIKI2_SPLITS = env("WIKI2_SPLITS", "train")
+  .split(",").map((s) => s.trim()).filter(Boolean);
+// Reject a triple with an implausibly long field (corruption); real subjects and
+// objects are entity names, and relations are Wikidata property labels.
+// 0 = every row. The train split holds 167,454 rows at ~4.95 deposits each
+// (~830k facts), so this is the knob that keeps 2Wiki proportionate to the rest
+// of the curriculum in the same way SODA_MAX_DIALOGS does.
+const WIKI2_MAX_ROWS = Math.max(
+  0,
+  Math.floor(Number(env("WIKI2_MAX_ROWS", "0"))) || 0,
+);
+const MAX_WIKI2_FIELD_CHARS = Math.max(
+  100,
+  Math.floor(Number(env("MAX_WIKI2_FIELD_KB", "2")) * 1000) || 2_000,
+);
+
+// ═══════════════════════════════════════════════════════════════════════
+// §6e″  2WikiMultihopQA parsing — `evidences` TRIPLES → SEMA facts
+//
+// This is the only stage whose purpose is COMPOSITION: answering a question
+// whose answer no single deposited fact contains. Sema composes by grounding
+// hop 1, then pivoting on the longest unconsumed learnt context that the
+// grounded answer CONTAINS (`reason`/`pivotStep`), so the pivot target must
+// itself be a deposited context. Each triple therefore deposits TWO facts:
+//
+//     "<subject> <relation>"  →  "The <relation> of <subject> is <object>."
+//     "<subject>"             →  "The <relation> of <subject> is <object>."
+//
+// The second is the PIVOT FACT. Without it the bare entity naming hop 2's
+// subject is not a learnt context, so the chain is structurally unreachable no
+// matter what the rest of the pipeline does.
+//
+// MEASURED on 200 real chained dev rows, depositing triples only and asking the
+// dataset's own composed questions (D = 1024, seed 7):
+//
+//   relation fact only          240 deposits    5/120 ( 4%)   pivotStep  0
+//   relation + pivot fact       800 deposits   44/200 (22%)   pivotStep 31
+//
+// A 5x improvement, and the only variant where the second hop fires at all.
+//
+// REJECTED ALTERNATIVE, so it is not re-tried blind: depositing the pivot fact
+// only for subjects that also appear as an OBJECT within the same row's
+// evidences (a row-local "something can pivot into this" test) cut deposits 25%
+// (800 → 600) but cost composition — 41/200 (20.5%) with pivotStep down to 21,
+// because real chains also run BETWEEN rows. Composition is this stage's entire
+// justification, so the deposits are worth keeping.
+//
+// The residual ~78% is a KNOWN, previously-recorded limitation and not a defect
+// in this adapter: the climb elects a topic rather than a relation, so a
+// question phrased "When did X's father die?" does not align with the Wikidata
+// property label "date of death". Failure is dominated by hop 2 never firing,
+// not by a wrong hop 2. Answer-shape breakdown at N = 120: entity answers
+// 23/106, date answers 2/14 — dates are worse, but not the cliff an earlier
+// note suggested, which is why no object-shape filter is applied here.
+// ═══════════════════════════════════════════════════════════════════════
+
+/** One (subject, relation, object) triple from a 2Wiki `evidences` cell. */
+export interface WikiTriple {
+  subject: string;
+  relation: string;
+  object: string;
+}
+
+/** Normalize a 2Wiki row into its evidence triples, or null when it carries
+ *  none usable. `evidences` is a JSON STRING holding an array of 3-element
+ *  arrays; a row whose cell is absent, unparseable, or empty yields null.
+ *  Individual malformed or oversized triples are dropped without discarding the
+ *  row — one bad triple should not cost the others. */
+export function toWikiTriples(
+  row: unknown,
+  maxChars = MAX_WIKI2_FIELD_CHARS,
+): WikiTriple[] | null {
+  if (!row || typeof row !== "object") return null;
+  const cell = (row as Record<string, unknown>).evidences;
+  let parsed: unknown = cell;
+  if (typeof cell === "string") {
+    try {
+      parsed = JSON.parse(cell);
+    } catch {
+      return null;
+    }
+  }
+  if (!Array.isArray(parsed)) return null;
+  const out: WikiTriple[] = [];
+  for (const e of parsed) {
+    if (!Array.isArray(e) || e.length < 3) continue;
+    const subject = typeof e[0] === "string" ? e[0].trim() : "";
+    const relation = typeof e[1] === "string" ? e[1].trim() : "";
+    const object = typeof e[2] === "string" ? e[2].trim() : "";
+    if (!subject || !relation || !object) continue;
+    if (
+      subject.length > maxChars ||
+      relation.length > maxChars ||
+      object.length > maxChars
+    ) continue;
+    out.push({ subject, relation, object });
+  }
+  return out.length ? out : null;
+}
+
+/** Render ONE triple as the prose fact Sema stores. Kept separate so the two
+ *  deposits below are guaranteed to share a byte-identical continuation: the
+ *  pivot fact only works if it leads to the SAME node the relation fact does. */
+export function wikiTripleSentence(t: WikiTriple): string {
+  return `The ${t.relation} of ${t.subject} is ${t.object}.`;
+}
+
+/** Translate a row's triples into SEMA items: per triple, the relation fact and
+ *  the bare-subject PIVOT fact (see the section note above). refineItems drops
+ *  the duplicates this produces when a row states the same triple twice. */
+export function wikiTriplesToItems(triples: WikiTriple[]): TrainingItem[] {
+  const items: TrainingItem[] = [];
+  for (const t of triples) {
+    const fact = wikiTripleSentence(t);
+    items.push({ context: `${t.subject} ${t.relation}`, continuation: fact });
+    items.push({ context: t.subject, continuation: fact });
+  }
+  return refineItems(items);
+}
+
+export const wiki2: Corpus = {
+  id: "2wiki",
+  label: "2Wiki",
+  kind: "relation triples",
+  enabled: WIKI2,
+  maxRows: WIKI2_MAX_ROWS,
+  // Read ONE of the file's seven columns. This is where the "context is not
+  // read" rule above stops being a property of the adapter and becomes a
+  // property of the read: the Wikipedia prose is never decoded at all. It is
+  // also almost the whole file — measured on the converted train shard, the
+  // seven columns hold 518 MB uncompressed and `evidences` is 18 MB of it
+  // (3.5%), so the projection cuts the decode work by ~28x.
+  read: parquet({ columns: ["evidences"] }),
+  toItems: (row) => {
+    const triples = toWikiTriples(row);
+    if (!triples) return null;
+    const items = wikiTriplesToItems(triples);
+    return items.length ? items : null;
+  },
+  unitNoun: "shard(s)",
+  log: { rows: "row(s)" },
+  discover: convertedParquetUnits({
+    id: "2wiki",
+    label: "2Wiki",
+    dataset: WIKI2_DATASET,
+    config: "default",
+    splits: WIKI2_SPLITS,
+    localSub: "2wiki",
+  }),
+};

package/example/train_base/corpus.ts

@@ -0,0 +1,150 @@
+// train_base/corpus.ts — WHAT A CORPUS IS: the contract every corpus file
+// declares, and the resume identity derived from it.
+//
+// Separated from the loop that consumes it (stage.ts) so a corpus file never
+// imports the loop. A corpus declares three things the loop does not know —
+// where its work-list comes from, what container its bytes are in, and how a row
+// becomes deposits — and everything here exists to state those three.
+//
+// THE RESUME IDS ARE A COMPATIBILITY SURFACE. A store records the units it has
+// finished as strings, and a store trained by an earlier version must keep
+// resuming, so `unitIdOf` below has to reproduce the ids that store already
+// holds. It does, by one rule with no exceptions: `${corpus.id}::${unit.key}`.
+//
+// That single rule is a recent simplification, and the evidence for it is worth
+// keeping. There used to be a second mechanism — a `Corpus.unitId` field
+// carrying a FIXED id for the three single-unit corpora, which were believed
+// irregular — and with it a trap: those corpora emitted `key: ""` on the
+// assumption that the fixed id would always override, so a new single-unit
+// corpus that forgot the field silently got the id "foo::". Reading the ids out
+// of a real 2.5 GB store settled it: the three are `aya::dataset`,
+// `oasst2::trees` and `genknow::qa`, which decompose EXACTLY into corpus id and
+// key. They were never irregular, so the escape hatch and its trap are gone and
+// one rule covers all 239 recorded ids.
+
+import { LOCAL_PATH } from "./config.js";
+import { localFind } from "./discovery.js";
+import type { Reader, RowAdapter } from "./readers.js";
+import type { TrainCtx } from "./runtime.js";
+import { DIM, R } from "./ui.js";
+import { join } from "node:path";
+
+/** One file to train: a shard, a per-language file, or a whole single-file
+ *  corpus. Exactly one of `url` / `local` is set. */
+export interface Unit {
+  /** Resume-id suffix — the corpus id and this form `${id}::${key}`. Part of
+   *  the store's compatibility surface; see the file header. */
+  key: string;
+  /** How the run log names this unit once it is read. */
+  name: string;
+  /** How the live panel names it, and (unless `acquireLabel` overrides) how the
+   *  download is labelled. Conventionally `${corpus.label} ${name}`. */
+  display: string;
+  url?: string;
+  local?: string;
+  /** Size in bytes when the listing said, else 0/absent. Summed BEFORE the
+   *  stage reads anything, so the corpus progress bar has a denominator that
+   *  does not grow underneath it. */
+  bytes?: number;
+  /** Cache filename. Defaults to the resume id with unsafe characters folded. */
+  dest?: string;
+  /** Download label, when it differs from `display`. */
+  acquireLabel?: string;
+}
+
+/** How one unit's outcome reads in the run log. All optional: the defaults are
+ *  what every fact-shaped corpus prints. */
+export interface LogStyle {
+  /** What one deposit is called. Default "facts". */
+  deposits?: string;
+  /** When set, the line reports "from N <rows>" — the count of rows that
+   *  actually produced deposits. */
+  rows?: string;
+  /** What an unusable record is called. Default "unusable row(s)". */
+  bad?: string;
+  /** Report only the reader's `skipped` (malformed records), not the rows the
+   *  adapter declined. For a corpus that DECLINES records by design — oasst2
+   *  drops every single-turn tree — counting those as damage would be a lie. */
+  malformedOnly?: boolean;
+}
+
+export interface Corpus {
+  /** Tally key AND resume-id prefix. Compatibility surface — see the header. */
+  id: string;
+  /** Human name: the panel, the skip notices, the listing-failure message. */
+  label: string;
+  /** The dim tag in the log line, e.g. "translation", "social dialogue". */
+  kind: string;
+  enabled: boolean;
+  /** The work-list. Return [] for "nothing found" (the runner says so), or
+   *  null when the corpus has already logged a more specific reason. */
+  discover(ctx: TrainCtx): Promise<Unit[] | null>;
+  read: Reader;
+  toItems: RowAdapter;
+  /** Stage-wide row budget; 0/absent = unbounded. See the budget notes in
+   *  stage.ts. */
+  maxRows?: number;
+  /** Noun for the "N/M ___ to train" announcement. Absent ⇒ no announcement,
+   *  which is what a single-unit corpus has always done. */
+  unitNoun?: string;
+  /** Keep a file that came from the CACHE after a complete read. Only oasst2
+   *  does this: every other corpus deletes whatever acquire() handed it. */
+  keepCached?: boolean;
+  log?: LogStyle;
+}
+
+/** The string a store records once this unit is finished. ONE rule, no
+ *  exceptions — see the file header for why there used to be two. */
+export const unitIdOf = (corpus: Corpus, unit: Unit): string =>
+  `${corpus.id}::${unit.key}`;
+
+/** Local files live under `LOCAL_PATH/<sub>`, or directly in LOCAL_PATH when
+ *  `sub` is empty. Kept here because the layout is a user-facing convention:
+ *  the corpora that share an extension (.json, .parquet) are kept apart by a
+ *  subdirectory so a local run cannot feed one corpus's files to another. */
+export const localDir = (sub: string): string =>
+  sub ? join(LOCAL_PATH, sub) : LOCAL_PATH;
+
+/** A single-unit corpus: one fixed URL, or one local file matched by pattern.
+ *  Factored out because the three corpora that are ONE file resolve it the same
+ *  way. `key` is what the store records this corpus under — `aya::dataset` is
+ *  `key: "dataset"` — so it is required rather than defaulted: a resume id is
+ *  the one thing here that must never be guessed. */
+export function singleUnit(opts: {
+  key: string;
+  label: string;
+  display: string;
+  url: string;
+  dest: string;
+  acquireLabel?: string;
+  /** Patterns tried, in order, against LOCAL_PATH. */
+  localMatch: RegExp[];
+  /** How the "no local copy" notice describes what it looked for. */
+  localWhat: string;
+}): (ctx: TrainCtx) => Promise<Unit[] | null> {
+  return async (ctx: TrainCtx) => {
+    if (LOCAL_PATH) {
+      const hit = localFind(LOCAL_PATH, ...opts.localMatch);
+      if (!hit) {
+        ctx.progress.log(
+          `  ${DIM}· no ${opts.localWhat} in ${LOCAL_PATH} — skipping${R}`,
+        );
+        return null;
+      }
+      return [{
+        key: opts.key,
+        name: opts.label,
+        display: opts.display,
+        local: join(LOCAL_PATH, hit),
+      }];
+    }
+    return [{
+      key: opts.key,
+      name: opts.label,
+      display: opts.display,
+      url: opts.url,
+      dest: opts.dest,
+      acquireLabel: opts.acquireLabel,
+    }];
+  };
+}

package/example/train_base/discovery.ts

@@ -0,0 +1,203 @@
+// train_base/discovery.ts — where a stage's WORK-LIST comes from.
+//
+// Four remote strategies and two local ones, each generic over the dataset it
+// is pointed at. What stays with a corpus is its POLICY — which subset of the
+// listing to keep, how to name the resume unit — because that is a curriculum
+// decision, not a protocol one.
+//
+// A note that has bitten this code twice, in both directions: a dataset id
+// ("owner/name") is a PATH here and its "/" must NOT be percent-encoded, while
+// a branch name ("refs/convert/parquet") is a single path SEGMENT and its "/"
+// MUST be.
+
+import { getJson, getJsonPaged, type HttpOptions } from "./http.js";
+import { readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+/** A note a listing needs to make about ITSELF — an accepted-but-unexpected
+ *  split name, say. Distinct from a failure: the run continues, but silently
+ *  continuing would hide why fewer units appeared than expected. */
+export type Note = (msg: string) => void;
+
+/** One listed file: where it is, and how big it is.
+ *
+ *  The SIZE is carried because the panel is otherwise dishonest. Corpus
+ *  progress used to be measured against a total that GREW as each file was
+ *  opened, so the bar ran to 100% at the end of every file and then fell back
+ *  when the next one was added — 100%, 50%, 100%, 66%… Both listing APIs
+ *  already return the size, so the denominator can simply be known before the
+ *  first byte is read. */
+export interface Listed {
+  path: string;
+  /** Bytes, or 0 when the source did not say. */
+  size: number;
+}
+
+const sizeOf = (e: any): number => {
+  const n = Number(e?.size ?? e?.lfs?.size ?? 0);
+  return Number.isFinite(n) && n > 0 ? n : 0;
+};
+
+/** Files under `path` in a Hugging Face dataset repo's main branch, filtered to
+ *  an extension. Returns repo-relative paths (e.g. "smolsent/ha_en.jsonl"),
+ *  sorted, so a run's unit order is stable across machines. */
+export async function hfTree(
+  dataset: string,
+  path: string,
+  ext: RegExp,
+  label: string,
+  opts: HttpOptions,
+): Promise<Listed[]> {
+  // The dataset id is a PATH here, so its "/" must not be percent-encoded.
+  // `recursive=true` returns every file under `path`, 1,000 at a time — hence
+  // the PAGED fetch: a truncated work-list would train part of a corpus and
+  // then call it finished.
+  const url = `https://huggingface.co/api/datasets/${dataset}` +
+    `/tree/main/${path}?recursive=true`;
+  const body = await getJsonPaged(url, label, opts);
+  const out: Listed[] = body
+    .filter((e: any) => e?.type === "file" && ext.test(e?.path))
+    .map((e: any) => ({ path: String(e.path), size: sizeOf(e) }));
+  out.sort((a, b) => a.path.localeCompare(b.path));
+  return out;
+}
+
+/** A dataset's Parquet shards on Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch, restricted to `config` and to `splits`.
+ *
+ *  The converted branch is used rather than `main` because a dataset's own
+ *  Parquet may be written as ONE giant row-group (SODA's is 1,191,582 rows),
+ *  and a column chunk is per-group, so reading any part of it materialises all
+ *  of it. The converted branch is uniformly 10,000-row groups.
+ *
+ *  Paths look like "<config>/<split>/0000.parquet". The BRANCH name is a single
+ *  path SEGMENT here, so its "/" is percent-encoded — unlike a dataset id,
+ *  whose "/" must not be. */
+export async function hfConvertedParquet(
+  dataset: string,
+  config: string,
+  splits: string[],
+  label: string,
+  opts: HttpOptions,
+  note?: Note,
+): Promise<Listed[]> {
+  const body = await getJsonPaged(
+    `https://huggingface.co/api/datasets/${dataset}` +
+      `/tree/refs%2Fconvert%2Fparquet/${config}?recursive=true`,
+    `GET ${label} tree`,
+    opts,
+  );
+  const paths: Listed[] = body
+    .filter((e: any) => e?.type === "file" && /\.parquet$/i.test(e?.path))
+    .map((e: any) => ({ path: String(e.path), size: sizeOf(e) }));
+  paths.sort((a, b) => a.path.localeCompare(b.path));
+
+  // The tree is rooted at `config`, so the split is the second-to-last part.
+  const splitOf = (p: string) => p.split("/").slice(-2)[0] ?? "";
+  const present = new Set(paths.map((p) => splitOf(p.path)));
+
+  // MATCH THE SPLIT AGAINST WHAT THE BRANCH ACTUALLY CARRIES. An exact name is
+  // not guaranteed: the converter renames a split it could not finish, and
+  // shards a very large one. Observed on real datasets today —
+  //   allenai/c4            → partial-train, partial-validation
+  //   HuggingFaceFW/fineweb → train-part0
+  // — neither of which equals "train". The old exact-match filter returned []
+  // for both, and an empty work-list is reported as "no files found — skipping",
+  // which reads like a normal outcome rather than a corpus being dropped whole.
+  const wanted = new Set<string>();
+  for (const want of splits) {
+    const pick = present.has(want)
+      ? want
+      : [...present].find((s) => s === `partial-${want}`) ??
+        [...present].find((s) => s.startsWith(`${want}-part`));
+    if (!pick) {
+      // Loud, not empty: a requested split that simply is not there is a
+      // configuration error, and silence would hide the whole corpus.
+      throw new Error(
+        `${label}: split "${want}" is not on the converted branch — it ` +
+          `carries ${[...present].join(", ") || "no parquet at all"}`,
+      );
+    }
+    if (pick !== want) {
+      note?.(
+        `${label}: split "${want}" is published as "${pick}"` +
+          (pick.startsWith("partial-")
+            ? " — Hugging Face has only PARTIALLY converted this dataset, so " +
+              "the shards below are not the whole split"
+            : ""),
+      );
+    }
+    wanted.add(pick);
+  }
+  return paths.filter((p) => wanted.has(splitOf(p.path)));
+}
+
+/** File NAMES (not paths) in one directory of a GitHub repo, filtered to an
+ *  extension and sorted. Used for corpora served from GitHub raw rather than
+ *  Hugging Face — where the HF mirrors are loading-script repos with no data
+ *  files, the official GitHub copy is the one carrying the licence notice. */
+export async function githubContents(
+  repo: string,
+  dir: string,
+  ext: RegExp,
+  label: string,
+  opts: HttpOptions,
+): Promise<Listed[]> {
+  const body = await getJson(
+    `https://api.github.com/repos/${repo}/contents/${dir}`,
+    label,
+    opts,
+  );
+  const entries: unknown[] = Array.isArray(body) ? body : [];
+  // The contents API returns at most 1,000 entries for a directory and does NOT
+  // paginate them — it simply stops, with no Link header and no error. A
+  // directory at that boundary is therefore indistinguishable from a truncated
+  // one, so the only honest response is to refuse rather than train part of it
+  // and record the part as the whole.
+  if (entries.length >= 1000) {
+    throw new Error(
+      `${label}: GitHub returned ${entries.length} entries, the point at ` +
+        `which the contents API truncates without saying so — this listing ` +
+        `cannot be trusted to be complete`,
+    );
+  }
+  const names: Listed[] = entries
+    .filter((e: any) => e?.type === "file" && ext.test(e?.name))
+    .map((e: any) => ({ path: String(e.name), size: sizeOf(e) }));
+  names.sort((a, b) => a.path.localeCompare(b.path));
+  return names;
+}
+
+/** Every file in a local directory matching `ext`, sorted. A missing directory
+ *  is an empty list, not an error: LOCAL_PATH is an offline convenience and a
+ *  stage with no local copy simply reports that and moves on. */
+export function localFiles(dir: string, ext: RegExp): Listed[] {
+  try {
+    return readdirSync(dir)
+      .filter((f: string) => ext.test(f))
+      .sort()
+      .map((f: string) => {
+        let size = 0;
+        try {
+          size = statSync(join(dir, f)).size;
+        } catch { /* unreadable — the read will report it */ }
+        return { path: f, size };
+      });
+  } catch {
+    return []; // no such directory
+  }
+}
+
+/** The FIRST file in a local directory matching any of `exts`, in directory
+ *  order (deliberately NOT sorted — this mirrors the single-file stages, which
+ *  take whichever copy the filesystem hands back first). Null when none match
+ *  or the directory is absent. */
+export function localFind(dir: string, ...exts: RegExp[]): string | null {
+  try {
+    return readdirSync(dir).find((f: string) =>
+      exts.some((re) => re.test(f))
+    ) ?? null;
+  } catch {
+    return null; // no such directory
+  }
+}