@hviana/sema 0.5.8 → 0.6.0

package/example/train_base/http.ts

@@ -0,0 +1,284 @@
+// train_base/http.ts — the shared network policy: wait out throttling, retry
+// what is transient, give up at once on what is not.
+//
+// A leaf module by construction: everything it needs to cancel (the run's
+// AbortSignal) and everything it needs to report (a throttle notice) arrives
+// as a parameter. There is no module-level shutdown handle and no module-level
+// log hook — those were globals precisely because this code used to live in the
+// same file as the run that owned them.
+
+import { DOWNLOAD_TRIES } from "./config.js";
+
+/** Sleep `ms`, but wake early if `signal` fires — so a long back-off (e.g. a
+ *  rate-limit wait) never swallows Ctrl+C. Resolves either way. */
+export const waitMs = (ms: number, signal: AbortSignal): Promise<void> =>
+  new Promise((resolve) => {
+    if (signal.aborted) return resolve();
+    // NOTE: the timer is deliberately NOT unref'd — an unref'd timer does not
+    // keep the event loop alive, so a pending wait (e.g. the pace between page
+    // requests, or a rate-limit back-off) would let Node exit early and the run
+    // would "do nothing and close". The listener lets a shutdown wake it early.
+    const t = setTimeout(done, ms);
+    function done() {
+      clearTimeout(t);
+      signal.removeEventListener("abort", done);
+      resolve();
+    }
+    signal.addEventListener("abort", done, { once: true });
+  });
+
+/** Resolve `p`, but reject with a TimeoutError if it takes longer than `ms`.
+ *  The underlying promise is left to settle on its own (we just stop waiting),
+ *  so a slow black-box call can never wedge the caller. */
+export function withTimeout<T>(
+  p: Promise<T>,
+  ms: number,
+  label = "operation",
+): Promise<T> {
+  return new Promise<T>((resolve, reject) => {
+    const t = setTimeout(() => {
+      const e: Error & { name: string } = new Error(
+        `${label} timed out after ${ms}ms`,
+      );
+      e.name = "TimeoutError";
+      reject(e);
+    }, ms);
+    if (typeof (t as any).unref === "function") (t as any).unref();
+    p.then(
+      (v) => {
+        clearTimeout(t);
+        resolve(v);
+      },
+      (e) => {
+        clearTimeout(t);
+        reject(e);
+      },
+    );
+  });
+}
+
+/** An HTTP error the caller tagged as transient. `.fatal` skips all retries;
+ *  `.throttle` (a 429/503 rate-limit or overload) is retried indefinitely and
+ *  does NOT consume the bounded attempt budget — the server told us to wait, not
+ *  to give up. `.retryAfterMs` carries a server-suggested delay when present. */
+export type HttpError = Error & {
+  fatal?: boolean;
+  throttle?: boolean;
+  retryAfterMs?: number;
+};
+
+/** What every network call in this trainer needs from its caller: how to be
+ *  cancelled, and (optionally) where to report a rate-limit wait. */
+export interface HttpOptions {
+  signal: AbortSignal;
+  /** Called after each throttle wait, so a 429 back-off reads as "waiting"
+   *  rather than a silent hang. Omitted ⇒ the wait is silent. */
+  onThrottle?: (waitMsAmount: number, label: string) => void;
+}
+
+/** Wrap a throttle notice so a STORM of 429s logs at most one notice every
+ *  `minGapMs`. Without the gap a busy server produces a wall of identical
+ *  "waiting…" lines that pushes the real log out of the scrollback. */
+export function throttleNotifier(
+  fn: (waitMsAmount: number, label: string) => void,
+  minGapMs = 3000,
+): (waitMsAmount: number, label: string) => void {
+  let last = 0;
+  return (ms, label) => {
+    const now = Date.now();
+    if (now - last <= minGapMs) return;
+    last = now;
+    fn(ms, label);
+  };
+}
+
+/** Retry `fn` with exponential backoff.
+ *
+ *  Three error classes:
+ *   • `.fatal` / AbortError  → rethrown immediately (never retried).
+ *   • `.throttle` (429/503)  → the server is rate-limiting/overloaded. We are
+ *     NOT failing — we WAIT (honouring Retry-After, else capped exponential
+ *     back-off with jitter) and retry WITHOUT consuming an attempt, so a
+ *     throttled request holds on until it succeeds rather than being dropped.
+ *     Only a shutdown breaks this loop.
+ *   • anything else          → a genuine transient error, retried up to `tries`
+ *     with exponential back-off before giving up.
+ *
+ *  `onFail` is called after each non-throttle failed attempt; `onThrottle` after
+ *  each throttle wait (for a "waiting…" notice). */
+export async function retry<T>(
+  label: string,
+  fn: () => Promise<T>,
+  tries: number,
+  opts: HttpOptions & { onFail?: (attempt: number, err: Error) => void },
+): Promise<T> {
+  const { signal, onFail, onThrottle } = opts;
+  let wait = 1000, last = "", throttleWait = 1000;
+  for (let attempt = 1; attempt <= tries;) {
+    if (signal.aborted) {
+      const e: HttpError = new Error("aborted");
+      e.fatal = true;
+      throw e;
+    }
+    try {
+      return await fn();
+    } catch (e) {
+      const err = e as HttpError;
+      if (err.name === "AbortError" || err.fatal) throw err;
+
+      // Rate-limited / overloaded: wait it out. Does NOT advance `attempt`, so a
+      // busy server can never exhaust the retry budget and drop the request.
+      if (err.throttle && !signal.aborted) {
+        // Honour Retry-After when the server sent one; else exponential back-off
+        // with jitter, capped, so a fleet of requests does not resynchronise.
+        const base = err.retryAfterMs && err.retryAfterMs > 0
+          ? err.retryAfterMs
+          : throttleWait;
+        const ms = Math.min(base, 60_000) +
+          Math.floor(base * 0.25 * Math.random());
+        onThrottle?.(ms, label);
+        await waitMs(ms, signal);
+        throttleWait = Math.min(throttleWait * 2, 60_000);
+        continue;
+      }
+
+      last = err.message;
+      onFail?.(attempt, err);
+      attempt++;
+      if (attempt <= tries) {
+        await waitMs(wait, signal);
+        wait = Math.min(wait * 2, 30_000);
+      }
+    }
+  }
+  throw new Error(`${label} failed after ${tries} attempts: ${last}`);
+}
+
+/** Classify a non-OK HTTP response into an {@link HttpError} for {@link retry}:
+ *   • 429 / 503  → THROTTLE (rate-limited / overloaded): retried indefinitely,
+ *     honouring a Retry-After header (seconds or an HTTP-date) when present.
+ *   • other 5xx  → transient: retried up to the caller's attempt budget.
+ *   • other 4xx  → FATAL: a real client error (404, 401, …) — not retried.
+ *  Never throttles forever silently: the wait is interruptible by shutdown. */
+export function httpError(res: Response): HttpError {
+  const err: HttpError = new Error(`HTTP ${res.status}`);
+  if (res.status === 429 || res.status === 503) {
+    err.throttle = true;
+    const ra = res.headers.get("retry-after");
+    if (ra) {
+      const secs = Number(ra);
+      if (Number.isFinite(secs)) err.retryAfterMs = Math.max(0, secs * 1000);
+      else {
+        const when = Date.parse(ra);
+        if (Number.isFinite(when)) {
+          err.retryAfterMs = Math.max(0, when - Date.now());
+        }
+      }
+    }
+  } else if (res.status < 500) {
+    err.fatal = true; // genuine client error — do not retry
+  } // other 5xx: neither fatal nor throttle → ordinary bounded retry
+  return err;
+}
+
+/** GET a URL and parse JSON, with the shared retry policy: rate-limits (429/503)
+ *  WAIT indefinitely (surfaced through `opts.onThrottle`), other 4xx is fatal,
+ *  other 5xx retried up to DOWNLOAD_TRIES. Used by every dataset LISTING call so
+ *  all share the same never-drop-on-throttle behaviour. */
+export async function getJson(
+  url: string,
+  label: string,
+  opts: HttpOptions,
+): Promise<any> {
+  return retry(
+    label,
+    async () => {
+      const res = await fetch(url, { signal: opts.signal });
+      if (res.ok) return res.json();
+      throw httpError(res);
+    },
+    DOWNLOAD_TRIES,
+    opts,
+  );
+}
+
+/** The `rel="next"` URL of an RFC 5988 Link header, or null. */
+export function nextLink(header: string | null): string | null {
+  if (!header) return null;
+  for (const part of header.split(",")) {
+    const m = part.match(/<([^>]+)>\s*;\s*rel\s*=\s*"?next"?/i);
+    if (m) return m[1];
+  }
+  return null;
+}
+
+/** GET a paginated JSON ARRAY, following `Link: rel="next"` to the end.
+ *
+ *  A LISTING THAT STOPS EARLY IS INVISIBLE, and that is why this exists.
+ *  Hugging Face caps a tree listing at 1,000 entries and hands back a next
+ *  link (verified: allenai/c4 returns exactly 1,000 plus a link). A caller that
+ *  ignores it gets a work-list silently missing everything past the first page,
+ *  trains it, marks those units complete, and thereafter reports the corpus
+ *  "already trained". No error at any point. Following the links is the only
+ *  way the work-list can be trusted to be the whole work-list.
+ *
+ *  `maxPages` is a runaway guard, not a limit anyone should hit; exceeding it
+ *  throws rather than returning a partial list, for exactly the reason above. */
+export async function getJsonPaged(
+  url: string,
+  label: string,
+  opts: HttpOptions,
+  maxPages = 500,
+): Promise<unknown[]> {
+  const out: unknown[] = [];
+  let next: string | null = url;
+  let pages = 0;
+  while (next !== null) {
+    const at: string = next;
+    const { body, link } = await retry(
+      label,
+      async () => {
+        const res = await fetch(at, { signal: opts.signal });
+        if (!res.ok) throw httpError(res);
+        return { body: await res.json(), link: res.headers.get("link") };
+      },
+      DOWNLOAD_TRIES,
+      opts,
+    );
+    if (!Array.isArray(body)) break; // not a listing — nothing to page through
+    out.push(...body);
+    next = nextLink(link);
+    if (++pages >= maxPages && next) {
+      throw new Error(
+        `${label}: more than ${maxPages} pages of listing — refusing to ` +
+          `continue with a work-list that may be incomplete`,
+      );
+    }
+  }
+  return out;
+}
+
+/** Advertised transfer size of `url`, used only to reserve cache room. Like any
+ *  `content-length` this is the ON-THE-WIRE size, so for a content-coded source
+ *  (GitHub raw gzips JSON ~14x) it UNDER-estimates the file that lands on disk.
+ *  That is tolerable here because the cache ceiling is a budget, not a
+ *  correctness property — a run may overshoot MAX_CACHE_GB by the compression
+ *  ratio of one in-flight file, and each file is deleted as soon as it is
+ *  consumed. It must NOT be reused as an integrity check; see downloadFile.
+ *
+ *  Rate-limits wait; other 4xx is fatal; total failure → the caller's catch. */
+export async function headSize(
+  url: string,
+  opts: HttpOptions,
+): Promise<number> {
+  return retry(
+    `HEAD ${url}`,
+    async () => {
+      const res = await fetch(url, { method: "HEAD", signal: opts.signal });
+      if (res.ok) return Number(res.headers.get("content-length")) || 0;
+      throw httpError(res);
+    },
+    4,
+    opts,
+  );
+}

package/example/train_base/items.ts

@@ -0,0 +1,118 @@
+// train_base/items.ts — the REPRESENTATION core: what a training item is, and
+// the three shapes a corpus datum may take.
+//
+// REPRESENTATION POLICY (one datum → one form; no replication):
+//   • FACTS are the default. A datum that is a RELATION (translation pair,
+//     question → answer) is emitted as a (context → continuation) edge SEMA
+//     points at and, by example across the corpus, generalizes from (cf.
+//     example/demo.ts).
+//   • EXPERIENCES (bare statements) are used only when a fact is NOT possible —
+//     content with no natural relational split.
+//   • CUMULATIVE CONTINUOUS CONTEXT is used only when truly necessary — genuine
+//     MULTI-TURN dialogue, where a turn follows from the whole conversation so
+//     far. The fact stages do NOT synthesize a multi-turn walk, which would just
+//     replicate the facts (repetition SEMA avoids).
+//
+// Nothing here reads the environment or touches I/O: these are the pure
+// functions every corpus adapter is built out of.
+
+export interface Episode {
+  context: string;
+  continuation: string;
+}
+export type TrainingItem = string | Episode;
+
+export const isEpisode = (it: TrainingItem): it is Episode =>
+  typeof it !== "string";
+
+/** One turn of a dialogue, attributed to a speaker. The speaker is only ever
+ *  used to decide MERGING (see `mergeSpeakerTurns`); it is never deposited. */
+export interface SpeakerTurn {
+  // Upper-cased by the adapters, so USER/ASSISTANT (TM-1/2) and user/assistant
+  // (TM-3/4) compare equal.
+  speaker: string;
+  text: string;
+}
+
+/** Build the accumulated-context episodes of a turn sequence: each successive
+ *  turn is the continuation of ALL the turns before it joined together. This is
+ *  the same cumulative-context shape a multi-turn conversation deposits, so the
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
+export function accumulate(turns: string[]): Episode[] {
+  const out: Episode[] = [];
+  for (let i = 1; i < turns.length; i++) {
+    out.push({ context: turns.slice(0, i).join("\n"), continuation: turns[i] });
+  }
+  return out;
+}
+
+/** Collapse consecutive same-speaker turns into one, joining with a space, and
+ *  return the bare texts in order. A turn with no speaker never merges with its
+ *  neighbour: an unlabelled row is of unknown origin, and joining two of them
+ *  would invent a contribution that may span two speakers.
+ *
+ *  Load-bearing for corpora that split one contribution across several indexed
+ *  utterances (an artifact of the collection UI). Left unmerged, the cumulative
+ *  walk deposits a turn boundary in the middle of one speaker's contribution
+ *  and teaches it as a hand-off. Measured share of turns absorbed by merging:
+ *  TM-1 17.7%, TM-2 11.9%, TM-3 0.8%, TM-4 0.0%. */
+export function mergeSpeakerTurns(turns: SpeakerTurn[]): string[] {
+  const out: string[] = [];
+  let prev = "";
+  for (const t of turns) {
+    if (out.length > 0 && t.speaker !== "" && t.speaker === prev) {
+      out[out.length - 1] += " " + t.text;
+    } else {
+      out.push(t.text);
+    }
+    prev = t.speaker;
+  }
+  return out;
+}
+
+/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
+ *  repeats so a concept never deposits the same form twice. */
+export function refineItems(items: TrainingItem[]): TrainingItem[] {
+  const out: TrainingItem[] = [];
+  const seen = new Set<string>();
+  for (const it of items) {
+    if (!isEpisode(it)) {
+      const exp = it.trim();
+      const key = "E:" + exp;
+      if (exp && !seen.has(key)) {
+        seen.add(key);
+        out.push(exp);
+      }
+      continue;
+    }
+    const ctx = it.context.trim();
+    const cont = it.continuation.trim();
+    if (!ctx || !cont || ctx === cont) continue;
+    const key = "P:" + ctx + "\u0000" + cont;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push({ context: ctx, continuation: cont });
+  }
+  return out;
+}
+
+const ENC = new TextEncoder();
+
+/** Content size of a training item in UTF-8 bytes — the same quantity the
+ *  scaling suite (14-scaling.test.mjs) measures as KB/s: for an episode the
+ *  context plus the continuation, for a bare experience its own text. */
+export const itemBytes = (it: TrainingItem): number =>
+  isEpisode(it)
+    ? ENC.encode(it.context).length + ENC.encode(it.continuation).length
+    : ENC.encode(it).length;

package/example/train_base/main.ts

@@ -0,0 +1,240 @@
+//This file uses the Google SMOL dataset, made available under the CC BY 4.0 license.
+//This file uses Aya and oasst2 datasets, made available under the apache-2.0 license.
+//
+//A trained Sema store retains its training text VERBATIM, so distributing a
+//store distributes these corpora and every upstream licence applies to it in
+//full. Read DATASETS.md before adding a corpus here or publishing a store:
+//it carries the per-corpus attribution a distributed store is required to
+//travel with, and the two rules a candidate corpus must pass (no NonCommercial
+//term, no ShareAlike term — checked against what the corpus was BUILT FROM,
+//not merely against the repository's licence tag).
+
+//This file is a more appropriate training example for Sema.
+//Sema does not learn through repetition;
+//it does not require a massive database.
+//It needs fundamental datasets that teach basic cognitive concepts such as conversation, logic, relationships, behaviors and feelings.
+//The focus is on covering fundamental patterns, not repetition.
+//Tip: ontology-based adapted training datasets could be an interesting path.
+
+// train_base/main.ts — streaming trainer for the SmolSent + Aya + oasst2 +
+//                  Taskmaster + 2Wiki + SODA base.
+//
+// Training IS deposition: every source datum is translated into SEMA facts (or,
+// for genuine dialogue, accumulated-context episodes), then stored in one pass.
+// There are no gradients or epochs, and there is no LLM in the loop — the only
+// "model" is the SEMA store itself.
+//
+// Every source here is commercially licensable (cc-by-4.0 / apache-2.0).
+//
+// WHERE THINGS LIVE (this file is the folder's entry point and nothing else):
+//
+//   config.ts     RUN-level knobs — store, checkpoint cadence, cache ceiling,
+//                 MAX_MB, the Parquet read budget.
+//   items.ts      the REPRESENTATION core: what a training item is (fact /
+//                 experience / accumulated walk) and the policy governing
+//                 which shape a datum may take.
+//   http.ts       network policy: wait out throttling, retry what is
+//                 transient, give up at once on what is not.
+//   cache.ts      the durable disk cache and the atomic download sink — the
+//                 sole irreducible Node dependency.
+//   readers.ts    container formats: newline-delimited JSON (plain or
+//                 gzipped), JSON arrays, Parquet.
+//   discovery.ts  where a work-list comes from: an HF repo tree, the
+//                 auto-converted Parquet branch, a GitHub directory, a local
+//                 directory.
+//   progress.ts   the resume record inside the store, and the index passes
+//                 that keep a checkpoint queryable.
+//   ui.ts         the live panel, the formatters, the recall box.
+//   runtime.ts    the RUN: counters, the deposit gate, file acquisition,
+//                 checkpointing, shutdown.
+//   corpus.ts     WHAT A CORPUS IS — the contract a corpus declares, and the
+//                 one rule that turns it into a resume id.
+//   stage.ts      ONE loop, run once per corpus.
+//   corpora/      ONE FILE PER CORPUS: its knobs, its row adapter, its stage
+//                 descriptor, and the evidence that fixed each default.
+//                 corpora/index.ts is the curriculum, in order.
+//
+// Adding a corpus is therefore one new file in corpora/ plus its import, its
+// place in CURRICULUM and its re-export in corpora/index.ts — and no change at
+// all to the loop, the readers, or the run.
+//
+// Every source is DOWNLOADED as a file and streamed from disk (never paged
+// row-by-row over an HTTP API — that was slow and rate-limited). Resume is
+// per-file: a fully-consumed file is marked complete; an interrupted one
+// re-reads from the top (re-deposition is idempotent). LOCAL_PATH may hold
+// pre-downloaded files.
+//
+// The store IS the model: memories, training metadata, and the config snapshot
+// all live in {DB_PATH}.sqlite, so a run resumes from the store alone.
+//
+// Built on web standards. All I/O except the durable disk cache uses platform
+// primitives — fetch, WHATWG ReadableStream/WritableStream/TransformStream,
+// DecompressionStream, TextDecoderStream, Blob, AbortController. The sole
+// third-party code is hyparquet (+ its Snappy codec): a DEV dependency of this
+// example, never of the library, and loaded by a dynamic import inside
+// readers.ts the first time a Parquet corpus is read — so a curriculum with no
+// Parquet stage needs it not at all. Consistency guarantees:
+//   • Resume from the store alone — completed stage-units, example count,
+//     learned-content bytes, and processed-byte total are persisted in
+//     {DB_PATH}.sqlite and reloaded.
+//   • Atomic cache — a download streams to "<file>.part", is fsync'd, then
+//     renamed into place; a file at its final path is, by construction,
+//     complete, so an interrupted download can never be mistaken for a cached
+//     one.
+//   • Bounded cache — a download blocks under the MAX_CACHE_GB ceiling and the
+//     fully-processed file is deleted immediately.
+//   • Interruptible — Ctrl+C (SIGINT/SIGTERM) aborts in-flight network at once,
+//     stops at the next item boundary, writes a final checkpoint, and exits; an
+//     un-finished stage-unit is NOT marked complete, so resume re-reads it (re-
+//     deposition is idempotent). A second Ctrl+C, or a 60s watchdog, force-exits.
+//
+// Run:
+//   npx tsc && node dist/example/train_base/main.js
+//   MAX_MB=500 node dist/example/train_base/main.js
+//   CHECKPOINT_MB=250 node dist/example/train_base/main.js
+//   SMOLSENT_PAIRS=ha_en,zu_en node dist/example/train_base/main.js  # a subset of pairs
+//   SMOLSENT_DIRECTIONS=both node dist/example/train_base/main.js  # also English->foreign
+//   SMOLSENT=0 node dist/example/train_base/main.js           # skip SmolSent stage
+//   AYA=0 node dist/example/train_base/main.js                # skip Aya stage
+//   OASST=0 node dist/example/train_base/main.js              # skip oasst2 stage
+//   OASST_MIN_TURNS=6 node dist/example/train_base/main.js    # deeper multi-turn only
+//   GENKNOW=1 node dist/example/train_base/main.js            # General-Knowledge (see DATASETS.md §3.2)
+//   PARQUET_BATCH_MB=8 node dist/example/train_base/main.js   # smaller Parquet reads on a tight host
+//   TASKMASTER=0 node dist/example/train_base/main.js         # skip Taskmaster stage
+//   TASKMASTER_SETS=TM-3-2020 node dist/example/train_base/main.js  # one Taskmaster set
+//   WIKI2=0 node dist/example/train_base/main.js               # skip 2Wiki triples stage
+//   SODA=0 node dist/example/train_base/main.js                # skip the SODA stage
+//   MASSIVE=1 node dist/example/train_base/main.js             # enable MASSIVE (off by default)
+//   SODA_MAX_DIALOGS=0 node dist/example/train_base/main.js    # lift the SODA budget
+//   WIKI2_MAX_ROWS=50000 node dist/example/train_base/main.js  # budget the 2Wiki stage
+//   LOCAL_PATH=./base node dist/example/train_base/main.js    # offline: *.jsonl/.parquet/.jsonl.gz/.json
+//   DB_PATH=./data/sema node dist/example/train_base/main.js
+
+import { CachedIngest, Mind, SQliteStore } from "../../src/index.js";
+import {
+  D,
+  DB_PATH,
+  SEED,
+  SQLITE_CACHE_MB,
+  VECTOR_CACHE_MB,
+} from "./config.js";
+import { createRuntime } from "./runtime.js";
+import { runStage } from "./stage.js";
+import { sweepPartials } from "./cache.js";
+import { CURRICULUM, enabledLabels, storedLabels } from "./corpora/index.js";
+import { bytes, DIM, dur, num, R, RED, SHOW } from "./ui.js";
+
+// The parser/representation surface this module used to define itself. Kept
+// exported from here so importing `example/train_base/main.js` still reaches every
+// row adapter (toSmolSentRow, wikiTriplesToItems, …) and the shapes they build.
+export * from "./items.js";
+export * from "./corpora/index.js";
+export { parquetBatchRows } from "./readers.js";
+// The name this helper had while it lived here. It is `mergeSpeakerTurns` now,
+// because SODA merges by the same rule and the Taskmaster-specific name was a
+// lie — but the old name stays reachable so nothing importing it breaks.
+export { mergeSpeakerTurns as mergeTaskmasterTurns } from "./items.js";
+
+async function main(): Promise<void> {
+  const store = new SQliteStore({
+    path: DB_PATH,
+    D,
+    vectorCacheMb: VECTOR_CACHE_MB,
+    sqliteCacheMb: SQLITE_CACHE_MB,
+  });
+
+  // The store IS the model: memories, progress, and metadata all persist in
+  // it, so a resumed run just reopens the same store and continues. Guard
+  // against a changed D/SEED by comparing against what a previous run recorded.
+  const mind = new Mind({ seed: SEED, store });
+
+  // Pre-fill the vector indices' RAM caches with sequential scans (bounded by
+  // VECTOR_CACHE_MB).  A resumed run over a large store otherwise spends its
+  // first minutes warming those caches through random point reads — the
+  // ingest hot path is cache-miss bound until then.  Seconds, once, up front.
+  if (VECTOR_CACHE_MB > 0) {
+    const t = Date.now();
+    const warmed = await store.warmVectorCaches();
+    if (warmed > 0) {
+      process.stderr.write(
+        `  warmed vector caches: ${num(warmed)} rows in ${
+          dur((Date.now() - t) / 1000)
+        }\n`,
+      );
+    }
+  }
+  const ci = new CachedIngest(mind);
+  const prevD = await store.getMeta("train.D");
+  const prevSeed = await store.getMeta("train.seed");
+  if (
+    (prevD && Number(prevD) !== D) || (prevSeed && Number(prevSeed) !== SEED)
+  ) {
+    process.stderr.write(
+      `fatal: D/SEED changed (store has D=${prevD} seed=${prevSeed}, ` +
+        `requested D=${D} seed=${SEED}). Delete ${DB_PATH}.sqlite ` +
+        `to start fresh.\n`,
+    );
+    process.exit(1);
+  }
+
+  const dataset = enabledLabels();
+  await store.setMeta("train.D", String(D));
+  await store.setMeta("train.seed", String(SEED));
+  // createdAt is when this STORE was first trained into, so a resume must not
+  // overwrite it — the previous behaviour left every store claiming it was
+  // created at its most recent resume. When the current run started is a
+  // separate, also-useful fact, so it gets its own key rather than displacing
+  // this one.
+  if ((await store.getMeta("train.createdAt")) === null) {
+    await store.setMeta("train.createdAt", new Date().toISOString());
+  }
+  await store.setMeta("train.startedAt", new Date().toISOString());
+
+  const ctx = createRuntime({ store, mind, ci, title: dataset });
+  ctx.tick(true);
+
+  // Reclaim the debris of any download killed mid-transfer. It occupies cache
+  // ceiling that nothing else would ever free (see cache.ts sweepPartials).
+  const swept = sweepPartials();
+  if (swept.files > 0) {
+    ctx.progress.log(
+      `  ${DIM}· swept ${swept.files} interrupted download(s), ` +
+        `${bytes(swept.bytes)} reclaimed${R}`,
+    );
+  }
+
+  // ── resume — restore counters and the per-source tally from the store ──
+  await ctx.restore();
+
+  // What the STORE contains, which is only knowable after the resume: this
+  // run's stages plus whatever earlier runs deposited. The panel header above
+  // names this RUN's stages, which is a different and also-true fact.
+  await store.setMeta(
+    "train.dataset",
+    storedLabels(Object.keys(ctx.counters.langTally)),
+  );
+
+  // Walk the curriculum. Each stage skips itself on a resume that already
+  // finished it, and the walk stops at the first requested stop.
+  for (const corpus of CURRICULUM) {
+    if (ctx.stopRequested) break;
+    await runStage(ctx, corpus);
+  }
+
+  // Nothing is in flight any more; each stage announces itself as it starts, so
+  // this is the one transition that has no next stage to correct it.
+  ctx.state.activity = "idle";
+  await ctx.finish(ctx.stopRequested ? ctx.stopReason : "done");
+}
+
+// Only run when invoked directly, so importing the row adapters above (e.g.
+// for a fixture check) never starts training.
+const isMain = import.meta.url === `file://${process.argv[1]}` ||
+  process.argv[1]?.endsWith("train_base/main.js");
+if (isMain) {
+  main().catch((e) => {
+    process.stderr.write(SHOW);
+    console.error(`\n${RED}fatal:${R}`, e);
+    process.exit(1);
+  });
+}