@hviana/sema 0.5.8 → 0.6.0

package/dist/example/train_base/items.d.ts

@@ -0,0 +1,46 @@
+export interface Episode {
+    context: string;
+    continuation: string;
+}
+export type TrainingItem = string | Episode;
+export declare const isEpisode: (it: TrainingItem) => it is Episode;
+/** One turn of a dialogue, attributed to a speaker. The speaker is only ever
+ *  used to decide MERGING (see `mergeSpeakerTurns`); it is never deposited. */
+export interface SpeakerTurn {
+    speaker: string;
+    text: string;
+}
+/** Build the accumulated-context episodes of a turn sequence: each successive
+ *  turn is the continuation of ALL the turns before it joined together. This is
+ *  the same cumulative-context shape a multi-turn conversation deposits, so the
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
+export declare function accumulate(turns: string[]): Episode[];
+/** Collapse consecutive same-speaker turns into one, joining with a space, and
+ *  return the bare texts in order. A turn with no speaker never merges with its
+ *  neighbour: an unlabelled row is of unknown origin, and joining two of them
+ *  would invent a contribution that may span two speakers.
+ *
+ *  Load-bearing for corpora that split one contribution across several indexed
+ *  utterances (an artifact of the collection UI). Left unmerged, the cumulative
+ *  walk deposits a turn boundary in the middle of one speaker's contribution
+ *  and teaches it as a hand-off. Measured share of turns absorbed by merging:
+ *  TM-1 17.7%, TM-2 11.9%, TM-3 0.8%, TM-4 0.0%. */
+export declare function mergeSpeakerTurns(turns: SpeakerTurn[]): string[];
+/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
+ *  repeats so a concept never deposits the same form twice. */
+export declare function refineItems(items: TrainingItem[]): TrainingItem[];
+/** Content size of a training item in UTF-8 bytes — the same quantity the
+ *  scaling suite (14-scaling.test.mjs) measures as KB/s: for an episode the
+ *  context plus the continuation, for a bare experience its own text. */
+export declare const itemBytes: (it: TrainingItem) => number;

package/dist/example/train_base/items.js

@@ -0,0 +1,98 @@
+// train_base/items.ts — the REPRESENTATION core: what a training item is, and
+// the three shapes a corpus datum may take.
+//
+// REPRESENTATION POLICY (one datum → one form; no replication):
+//   • FACTS are the default. A datum that is a RELATION (translation pair,
+//     question → answer) is emitted as a (context → continuation) edge SEMA
+//     points at and, by example across the corpus, generalizes from (cf.
+//     example/demo.ts).
+//   • EXPERIENCES (bare statements) are used only when a fact is NOT possible —
+//     content with no natural relational split.
+//   • CUMULATIVE CONTINUOUS CONTEXT is used only when truly necessary — genuine
+//     MULTI-TURN dialogue, where a turn follows from the whole conversation so
+//     far. The fact stages do NOT synthesize a multi-turn walk, which would just
+//     replicate the facts (repetition SEMA avoids).
+//
+// Nothing here reads the environment or touches I/O: these are the pure
+// functions every corpus adapter is built out of.
+export const isEpisode = (it) => typeof it !== "string";
+/** Build the accumulated-context episodes of a turn sequence: each successive
+ *  turn is the continuation of ALL the turns before it joined together. This is
+ *  the same cumulative-context shape a multi-turn conversation deposits, so the
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
+export function accumulate(turns) {
+    const out = [];
+    for (let i = 1; i < turns.length; i++) {
+        out.push({ context: turns.slice(0, i).join("\n"), continuation: turns[i] });
+    }
+    return out;
+}
+/** Collapse consecutive same-speaker turns into one, joining with a space, and
+ *  return the bare texts in order. A turn with no speaker never merges with its
+ *  neighbour: an unlabelled row is of unknown origin, and joining two of them
+ *  would invent a contribution that may span two speakers.
+ *
+ *  Load-bearing for corpora that split one contribution across several indexed
+ *  utterances (an artifact of the collection UI). Left unmerged, the cumulative
+ *  walk deposits a turn boundary in the middle of one speaker's contribution
+ *  and teaches it as a hand-off. Measured share of turns absorbed by merging:
+ *  TM-1 17.7%, TM-2 11.9%, TM-3 0.8%, TM-4 0.0%. */
+export function mergeSpeakerTurns(turns) {
+    const out = [];
+    let prev = "";
+    for (const t of turns) {
+        if (out.length > 0 && t.speaker !== "" && t.speaker === prev) {
+            out[out.length - 1] += " " + t.text;
+        }
+        else {
+            out.push(t.text);
+        }
+        prev = t.speaker;
+    }
+    return out;
+}
+/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
+ *  repeats so a concept never deposits the same form twice. */
+export function refineItems(items) {
+    const out = [];
+    const seen = new Set();
+    for (const it of items) {
+        if (!isEpisode(it)) {
+            const exp = it.trim();
+            const key = "E:" + exp;
+            if (exp && !seen.has(key)) {
+                seen.add(key);
+                out.push(exp);
+            }
+            continue;
+        }
+        const ctx = it.context.trim();
+        const cont = it.continuation.trim();
+        if (!ctx || !cont || ctx === cont)
+            continue;
+        const key = "P:" + ctx + "\u0000" + cont;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push({ context: ctx, continuation: cont });
+    }
+    return out;
+}
+const ENC = new TextEncoder();
+/** Content size of a training item in UTF-8 bytes — the same quantity the
+ *  scaling suite (14-scaling.test.mjs) measures as KB/s: for an episode the
+ *  context plus the continuation, for a bare experience its own text. */
+export const itemBytes = (it) => isEpisode(it)
+    ? ENC.encode(it.context).length + ENC.encode(it.continuation).length
+    : ENC.encode(it).length;

package/dist/example/train_base/main.d.ts

@@ -0,0 +1,4 @@
+export * from "./items.js";
+export * from "./corpora/index.js";
+export { parquetBatchRows } from "./readers.js";
+export { mergeSpeakerTurns as mergeTaskmasterTurns } from "./items.js";

package/dist/example/train_base/main.js

@@ -0,0 +1,207 @@
+//This file uses the Google SMOL dataset, made available under the CC BY 4.0 license.
+//This file uses Aya and oasst2 datasets, made available under the apache-2.0 license.
+//
+//A trained Sema store retains its training text VERBATIM, so distributing a
+//store distributes these corpora and every upstream licence applies to it in
+//full. Read DATASETS.md before adding a corpus here or publishing a store:
+//it carries the per-corpus attribution a distributed store is required to
+//travel with, and the two rules a candidate corpus must pass (no NonCommercial
+//term, no ShareAlike term — checked against what the corpus was BUILT FROM,
+//not merely against the repository's licence tag).
+//This file is a more appropriate training example for Sema.
+//Sema does not learn through repetition;
+//it does not require a massive database.
+//It needs fundamental datasets that teach basic cognitive concepts such as conversation, logic, relationships, behaviors and feelings.
+//The focus is on covering fundamental patterns, not repetition.
+//Tip: ontology-based adapted training datasets could be an interesting path.
+// train_base/main.ts — streaming trainer for the SmolSent + Aya + oasst2 +
+//                  Taskmaster + 2Wiki + SODA base.
+//
+// Training IS deposition: every source datum is translated into SEMA facts (or,
+// for genuine dialogue, accumulated-context episodes), then stored in one pass.
+// There are no gradients or epochs, and there is no LLM in the loop — the only
+// "model" is the SEMA store itself.
+//
+// Every source here is commercially licensable (cc-by-4.0 / apache-2.0).
+//
+// WHERE THINGS LIVE (this file is the folder's entry point and nothing else):
+//
+//   config.ts     RUN-level knobs — store, checkpoint cadence, cache ceiling,
+//                 MAX_MB, the Parquet read budget.
+//   items.ts      the REPRESENTATION core: what a training item is (fact /
+//                 experience / accumulated walk) and the policy governing
+//                 which shape a datum may take.
+//   http.ts       network policy: wait out throttling, retry what is
+//                 transient, give up at once on what is not.
+//   cache.ts      the durable disk cache and the atomic download sink — the
+//                 sole irreducible Node dependency.
+//   readers.ts    container formats: newline-delimited JSON (plain or
+//                 gzipped), JSON arrays, Parquet.
+//   discovery.ts  where a work-list comes from: an HF repo tree, the
+//                 auto-converted Parquet branch, a GitHub directory, a local
+//                 directory.
+//   progress.ts   the resume record inside the store, and the index passes
+//                 that keep a checkpoint queryable.
+//   ui.ts         the live panel, the formatters, the recall box.
+//   runtime.ts    the RUN: counters, the deposit gate, file acquisition,
+//                 checkpointing, shutdown.
+//   corpus.ts     WHAT A CORPUS IS — the contract a corpus declares, and the
+//                 one rule that turns it into a resume id.
+//   stage.ts      ONE loop, run once per corpus.
+//   corpora/      ONE FILE PER CORPUS: its knobs, its row adapter, its stage
+//                 descriptor, and the evidence that fixed each default.
+//                 corpora/index.ts is the curriculum, in order.
+//
+// Adding a corpus is therefore one new file in corpora/ plus its import, its
+// place in CURRICULUM and its re-export in corpora/index.ts — and no change at
+// all to the loop, the readers, or the run.
+//
+// Every source is DOWNLOADED as a file and streamed from disk (never paged
+// row-by-row over an HTTP API — that was slow and rate-limited). Resume is
+// per-file: a fully-consumed file is marked complete; an interrupted one
+// re-reads from the top (re-deposition is idempotent). LOCAL_PATH may hold
+// pre-downloaded files.
+//
+// The store IS the model: memories, training metadata, and the config snapshot
+// all live in {DB_PATH}.sqlite, so a run resumes from the store alone.
+//
+// Built on web standards. All I/O except the durable disk cache uses platform
+// primitives — fetch, WHATWG ReadableStream/WritableStream/TransformStream,
+// DecompressionStream, TextDecoderStream, Blob, AbortController. The sole
+// third-party code is hyparquet (+ its Snappy codec): a DEV dependency of this
+// example, never of the library, and loaded by a dynamic import inside
+// readers.ts the first time a Parquet corpus is read — so a curriculum with no
+// Parquet stage needs it not at all. Consistency guarantees:
+//   • Resume from the store alone — completed stage-units, example count,
+//     learned-content bytes, and processed-byte total are persisted in
+//     {DB_PATH}.sqlite and reloaded.
+//   • Atomic cache — a download streams to "<file>.part", is fsync'd, then
+//     renamed into place; a file at its final path is, by construction,
+//     complete, so an interrupted download can never be mistaken for a cached
+//     one.
+//   • Bounded cache — a download blocks under the MAX_CACHE_GB ceiling and the
+//     fully-processed file is deleted immediately.
+//   • Interruptible — Ctrl+C (SIGINT/SIGTERM) aborts in-flight network at once,
+//     stops at the next item boundary, writes a final checkpoint, and exits; an
+//     un-finished stage-unit is NOT marked complete, so resume re-reads it (re-
+//     deposition is idempotent). A second Ctrl+C, or a 60s watchdog, force-exits.
+//
+// Run:
+//   npx tsc && node dist/example/train_base/main.js
+//   MAX_MB=500 node dist/example/train_base/main.js
+//   CHECKPOINT_MB=250 node dist/example/train_base/main.js
+//   SMOLSENT_PAIRS=ha_en,zu_en node dist/example/train_base/main.js  # a subset of pairs
+//   SMOLSENT_DIRECTIONS=both node dist/example/train_base/main.js  # also English->foreign
+//   SMOLSENT=0 node dist/example/train_base/main.js           # skip SmolSent stage
+//   AYA=0 node dist/example/train_base/main.js                # skip Aya stage
+//   OASST=0 node dist/example/train_base/main.js              # skip oasst2 stage
+//   OASST_MIN_TURNS=6 node dist/example/train_base/main.js    # deeper multi-turn only
+//   GENKNOW=1 node dist/example/train_base/main.js            # General-Knowledge (see DATASETS.md §3.2)
+//   PARQUET_BATCH_MB=8 node dist/example/train_base/main.js   # smaller Parquet reads on a tight host
+//   TASKMASTER=0 node dist/example/train_base/main.js         # skip Taskmaster stage
+//   TASKMASTER_SETS=TM-3-2020 node dist/example/train_base/main.js  # one Taskmaster set
+//   WIKI2=0 node dist/example/train_base/main.js               # skip 2Wiki triples stage
+//   SODA=0 node dist/example/train_base/main.js                # skip the SODA stage
+//   MASSIVE=1 node dist/example/train_base/main.js             # enable MASSIVE (off by default)
+//   SODA_MAX_DIALOGS=0 node dist/example/train_base/main.js    # lift the SODA budget
+//   WIKI2_MAX_ROWS=50000 node dist/example/train_base/main.js  # budget the 2Wiki stage
+//   LOCAL_PATH=./base node dist/example/train_base/main.js    # offline: *.jsonl/.parquet/.jsonl.gz/.json
+//   DB_PATH=./data/sema node dist/example/train_base/main.js
+import { CachedIngest, Mind, SQliteStore } from "../../src/index.js";
+import { D, DB_PATH, SEED, SQLITE_CACHE_MB, VECTOR_CACHE_MB, } from "./config.js";
+import { createRuntime } from "./runtime.js";
+import { runStage } from "./stage.js";
+import { sweepPartials } from "./cache.js";
+import { CURRICULUM, enabledLabels, storedLabels } from "./corpora/index.js";
+import { bytes, DIM, dur, num, R, RED, SHOW } from "./ui.js";
+// The parser/representation surface this module used to define itself. Kept
+// exported from here so importing `example/train_base/main.js` still reaches every
+// row adapter (toSmolSentRow, wikiTriplesToItems, …) and the shapes they build.
+export * from "./items.js";
+export * from "./corpora/index.js";
+export { parquetBatchRows } from "./readers.js";
+// The name this helper had while it lived here. It is `mergeSpeakerTurns` now,
+// because SODA merges by the same rule and the Taskmaster-specific name was a
+// lie — but the old name stays reachable so nothing importing it breaks.
+export { mergeSpeakerTurns as mergeTaskmasterTurns } from "./items.js";
+async function main() {
+    const store = new SQliteStore({
+        path: DB_PATH,
+        D,
+        vectorCacheMb: VECTOR_CACHE_MB,
+        sqliteCacheMb: SQLITE_CACHE_MB,
+    });
+    // The store IS the model: memories, progress, and metadata all persist in
+    // it, so a resumed run just reopens the same store and continues. Guard
+    // against a changed D/SEED by comparing against what a previous run recorded.
+    const mind = new Mind({ seed: SEED, store });
+    // Pre-fill the vector indices' RAM caches with sequential scans (bounded by
+    // VECTOR_CACHE_MB).  A resumed run over a large store otherwise spends its
+    // first minutes warming those caches through random point reads — the
+    // ingest hot path is cache-miss bound until then.  Seconds, once, up front.
+    if (VECTOR_CACHE_MB > 0) {
+        const t = Date.now();
+        const warmed = await store.warmVectorCaches();
+        if (warmed > 0) {
+            process.stderr.write(`  warmed vector caches: ${num(warmed)} rows in ${dur((Date.now() - t) / 1000)}\n`);
+        }
+    }
+    const ci = new CachedIngest(mind);
+    const prevD = await store.getMeta("train.D");
+    const prevSeed = await store.getMeta("train.seed");
+    if ((prevD && Number(prevD) !== D) || (prevSeed && Number(prevSeed) !== SEED)) {
+        process.stderr.write(`fatal: D/SEED changed (store has D=${prevD} seed=${prevSeed}, ` +
+            `requested D=${D} seed=${SEED}). Delete ${DB_PATH}.sqlite ` +
+            `to start fresh.\n`);
+        process.exit(1);
+    }
+    const dataset = enabledLabels();
+    await store.setMeta("train.D", String(D));
+    await store.setMeta("train.seed", String(SEED));
+    // createdAt is when this STORE was first trained into, so a resume must not
+    // overwrite it — the previous behaviour left every store claiming it was
+    // created at its most recent resume. When the current run started is a
+    // separate, also-useful fact, so it gets its own key rather than displacing
+    // this one.
+    if ((await store.getMeta("train.createdAt")) === null) {
+        await store.setMeta("train.createdAt", new Date().toISOString());
+    }
+    await store.setMeta("train.startedAt", new Date().toISOString());
+    const ctx = createRuntime({ store, mind, ci, title: dataset });
+    ctx.tick(true);
+    // Reclaim the debris of any download killed mid-transfer. It occupies cache
+    // ceiling that nothing else would ever free (see cache.ts sweepPartials).
+    const swept = sweepPartials();
+    if (swept.files > 0) {
+        ctx.progress.log(`  ${DIM}· swept ${swept.files} interrupted download(s), ` +
+            `${bytes(swept.bytes)} reclaimed${R}`);
+    }
+    // ── resume — restore counters and the per-source tally from the store ──
+    await ctx.restore();
+    // What the STORE contains, which is only knowable after the resume: this
+    // run's stages plus whatever earlier runs deposited. The panel header above
+    // names this RUN's stages, which is a different and also-true fact.
+    await store.setMeta("train.dataset", storedLabels(Object.keys(ctx.counters.langTally)));
+    // Walk the curriculum. Each stage skips itself on a resume that already
+    // finished it, and the walk stops at the first requested stop.
+    for (const corpus of CURRICULUM) {
+        if (ctx.stopRequested)
+            break;
+        await runStage(ctx, corpus);
+    }
+    // Nothing is in flight any more; each stage announces itself as it starts, so
+    // this is the one transition that has no next stage to correct it.
+    ctx.state.activity = "idle";
+    await ctx.finish(ctx.stopRequested ? ctx.stopReason : "done");
+}
+// Only run when invoked directly, so importing the row adapters above (e.g.
+// for a fixture check) never starts training.
+const isMain = import.meta.url === `file://${process.argv[1]}` ||
+    process.argv[1]?.endsWith("train_base/main.js");
+if (isMain) {
+    main().catch((e) => {
+        process.stderr.write(SHOW);
+        console.error(`\n${RED}fatal:${R}`, e);
+        process.exit(1);
+    });
+}

package/dist/example/train_base/progress.d.ts

@@ -0,0 +1,34 @@
+import type { Mind, Store } from "../../src/index.js";
+export interface SavedProgress {
+    completedFiles: string[];
+    depositCount: number;
+    trainedContentBytes: number;
+    totalBytesProcessed: number;
+    totalCorpusBytes: number;
+}
+export declare function loadProgress(store: Store): Promise<SavedProgress>;
+export declare function saveProgress(store: Store, p: SavedProgress): Promise<void>;
+/** Run index maintenance: compact (remove garbage), repair (fill gaps),
+ *  then refresh the canonical-form index (see below).  All three are
+ *  idempotent — running twice produces the same result as once.
+ *  Compaction frees index space first; repair then adds back every
+ *  edge/halo-bearing node whose gist was evicted from the pending cache
+ *  before it reached the content index, completing the coverage that
+ *  incremental promotion alone cannot guarantee.
+ *
+ *  repair runs with minParents = 0, NOT the library default of 2.  The
+ *  default repairs only structural BRIDGES (≥2 parents), but this
+ *  trainer's fact deposits also leave answer-side DEPOSIT ROOTS with 0
+ *  structural parents ("The capital of France is Paris." as the dst of a
+ *  Q→A edge is a root of its own tree, contained in nothing).  Those are
+ *  resonance targets recall depends on — a trained store shipped without
+ *  them cannot ground statement-shaped queries against its own answers
+ *  (observed: 33 such roots missing after a full curriculum, including
+ *  high-traffic conversation replies).  minParents = 0 admits every
+ *  edge/halo bearer; the candidate set is still corpus-of-experiences-
+ *  sized, so the pass stays cheap.
+ *
+ *  Logs the number of entries removed/added so a run that silently degrades
+ *  (growing compaction count, or repair never recovering anything) is
+ *  visible in the training log. */
+export declare function runIndexMaintenance(mind: Mind, log: (msg: string) => void): Promise<void>;

package/dist/example/train_base/progress.js

@@ -0,0 +1,114 @@
+// train_base/progress.ts — resume state, and the index passes that keep a
+// checkpointed store queryable.
+//
+// The store IS the model: memories, training metadata and the config snapshot
+// all live in {DB_PATH}.sqlite, so a run resumes from the store alone. The keys
+// below are that resume record, and they are a COMPATIBILITY SURFACE — a store
+// trained by an earlier version must keep resuming, so neither the key names
+// nor the shape of `completedFiles` may drift.
+import { INDEX_MAINTENANCE } from "./config.js";
+import { DIM, GRN, int, R, YEL } from "./ui.js";
+const META_COMPLETED = "train.completedFiles";
+const META_DEPOSITS = "train.depositCount";
+const META_TRAINED_BYTES = "train.trainedContentBytes";
+const META_BYTES = "train.totalBytesProcessed";
+const META_CORPUS_BYTES = "train.totalCorpusBytes";
+export async function loadProgress(store) {
+    try {
+        const raw = await store.getMeta(META_COMPLETED);
+        const deps = await store.getMeta(META_DEPOSITS);
+        const b = await store.getMeta(META_BYTES);
+        if (raw !== null && deps !== null && b !== null) {
+            const completedFiles = JSON.parse(raw);
+            if (Array.isArray(completedFiles)) {
+                const trained = await store.getMeta(META_TRAINED_BYTES);
+                const corpus = await store.getMeta(META_CORPUS_BYTES);
+                return {
+                    completedFiles,
+                    depositCount: Number(deps) || 0,
+                    trainedContentBytes: Number(trained) || 0,
+                    totalBytesProcessed: Number(b) || 0,
+                    totalCorpusBytes: Number(corpus) || 0,
+                };
+            }
+        }
+    }
+    catch { /* corrupt/missing — start fresh */ }
+    return {
+        completedFiles: [],
+        depositCount: 0,
+        trainedContentBytes: 0,
+        totalBytesProcessed: 0,
+        totalCorpusBytes: 0,
+    };
+}
+export async function saveProgress(store, p) {
+    await store.setMeta(META_COMPLETED, JSON.stringify(p.completedFiles));
+    await store.setMeta(META_DEPOSITS, String(p.depositCount));
+    await store.setMeta(META_TRAINED_BYTES, String(p.trainedContentBytes));
+    await store.setMeta(META_BYTES, String(p.totalBytesProcessed));
+    await store.setMeta(META_CORPUS_BYTES, String(p.totalCorpusBytes));
+    await store.setMeta("train.updatedAt", new Date().toISOString());
+    store.commit();
+}
+/** Run index maintenance: compact (remove garbage), repair (fill gaps),
+ *  then refresh the canonical-form index (see below).  All three are
+ *  idempotent — running twice produces the same result as once.
+ *  Compaction frees index space first; repair then adds back every
+ *  edge/halo-bearing node whose gist was evicted from the pending cache
+ *  before it reached the content index, completing the coverage that
+ *  incremental promotion alone cannot guarantee.
+ *
+ *  repair runs with minParents = 0, NOT the library default of 2.  The
+ *  default repairs only structural BRIDGES (≥2 parents), but this
+ *  trainer's fact deposits also leave answer-side DEPOSIT ROOTS with 0
+ *  structural parents ("The capital of France is Paris." as the dst of a
+ *  Q→A edge is a root of its own tree, contained in nothing).  Those are
+ *  resonance targets recall depends on — a trained store shipped without
+ *  them cannot ground statement-shaped queries against its own answers
+ *  (observed: 33 such roots missing after a full curriculum, including
+ *  high-traffic conversation replies).  minParents = 0 admits every
+ *  edge/halo bearer; the candidate set is still corpus-of-experiences-
+ *  sized, so the pass stays cheap.
+ *
+ *  Logs the number of entries removed/added so a run that silently degrades
+ *  (growing compaction count, or repair never recovering anything) is
+ *  visible in the training log. */
+export async function runIndexMaintenance(mind, log) {
+    if (!INDEX_MAINTENANCE)
+        return;
+    try {
+        const removed = await mind.store.compactContentIndex();
+        if (removed > 0) {
+            log(`  ${DIM}index compact: removed ${int(removed)} isolated entries${R}`);
+        }
+    }
+    catch (err) {
+        log(`  ${YEL}⚠ index compact failed${R}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    try {
+        const added = await mind.repairContentIndex(0);
+        if (added > 0) {
+            log(`  ${GRN}index repair: added ${int(added)} missing resonance targets${R}`);
+        }
+    }
+    catch (err) {
+        log(`  ${YEL}⚠ index repair failed${R}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    // Canonical-form index (src/canon.ts): lets resolution find stored forms
+    // across surface variation (case, width, whitespace).  Incremental and
+    // idempotent by construction — the `canon.upto` meta cursor scans only
+    // nodes newer than the last pass, and the (h, id) primary key ignores
+    // re-inserted rows — so it composes with the resume model exactly like
+    // compact/repair: every checkpoint (and finish) leaves the index
+    // covering all content trained so far.
+    try {
+        const added = await mind.buildCanonIndex();
+        if (added > 0) {
+            log(`  ${GRN}canon index: added ${int(added)} canonical-form entries${R}`);
+        }
+    }
+    catch (err) {
+        log(`  ${YEL}⚠ canon index build failed${R}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+}

package/dist/example/train_base/readers.d.ts

@@ -0,0 +1,125 @@
+import type { CachedIngest } from "../../src/index.js";
+import { type TrainingItem } from "./items.js";
+/** Turn ONE raw row into deposits, or null/[] when the row carries nothing
+ *  usable. Pure: no I/O, no counters, no logging. */
+export type RowAdapter = (row: unknown) => TrainingItem[] | null;
+/** What a read produced. `skipped` and `unusable` are deliberately SEPARATE:
+ *  a line that failed to parse is a defect in the file, while a row the adapter
+ *  declined is a normal, expected outcome for a corpus being filtered (oasst2
+ *  drops every single-turn tree by design). Collapsing them — as the two
+ *  original line readers each did, in opposite directions — makes one of the
+ *  two log lines a lie. */
+export interface FileResult {
+    examples: number;
+    rowsUsed: number;
+    skipped: number;
+    unusable: number;
+    stopped: boolean;
+}
+/** Everything a reader needs from the run: where to deposit, what to count,
+ *  and how to be stopped. */
+export interface ReadContext {
+    ci: CachedIngest;
+    /** Called once per deposit with its UTF-8 content size. Returns false to
+     *  stop the read (the MAX_MB cap, or a pending shutdown). */
+    onExample: (contentBytes: number) => Promise<boolean>;
+    /** Feeds the reservoir behind the checkpoint recall box. */
+    sample: (it: TrainingItem) => void;
+    signal: AbortSignal;
+    /** A stage-level budget. Checked per row and before each Parquet batch is
+     *  decoded — a budget must STOP the read rather than reject rows: left to
+     *  reject, a budgeted stage still DECODES every remaining row-group (143,346
+     *  rows of one 86.7 MB SODA shard) and reports them as "unusable" when
+     *  nothing was wrong with them, which is a lie in the run log.
+     *
+     *  Measured honestly: on that shard the wall time did NOT improve (2m 35s ->
+     *  2m 37s), because a budgeted run is dominated by depositing the rows it DID
+     *  take, not by scanning past the ones it did not. The win here is a truthful
+     *  log and the CPU/allocation of ~143k skipped row decodes, not elapsed time.
+     *  A larger shard past a small budget is where the decode cost would show. */
+    shouldStop?: () => boolean;
+    /** Rows to SKIP before depositing anything — the position a previous run
+     *  reached, taken from the durable cursor (see runtime.ts). Resume used to
+     *  mean "re-read this unit from the top", which was safe but re-deposited
+     *  everything already stored and counted it a second time; the store then
+     *  reported up to 77% more examples than it held.
+     *
+     *  Skipping is only sound because the cursor is written in the SAME COMMIT
+     *  that flushes the deposits it counts, so a row before the cursor is
+     *  necessarily durable. A skipped row is neither parsed nor counted, so a
+     *  resumed read's log line describes what THIS read did and nothing else. */
+    startRow?: number;
+    /** "Row `rows` is FULLY dealt with" — every item it produced is deposited, or
+     *  it produced none. Called at ROW BOUNDARIES ONLY, and never for a row the
+     *  read stopped in the middle of.
+     *
+     *  That boundary is the whole point. A checkpoint fires per DEPOSIT, and a row
+     *  can produce many (2Wiki emits ~5 facts per row, a dialogue one per turn),
+     *  so a position recorded when a row STARTS would mark it consumed while some
+     *  of its items were still unwritten — and the resume would skip them. Data
+     *  loss, silently. Advancing only here means the worst case is re-depositing
+     *  one row, which is idempotent and counted once. */
+    onRowDone?: (rows: number) => void;
+}
+/** A reader: read `filePath`, deposit every row `toItems` accepts. */
+export type Reader = (filePath: string, toItems: RowAdapter, rc: ReadContext) => Promise<FileResult>;
+/** Deposit a row's items: an experience via ingest(text), an episode via
+ *  ingest(context, continuation). After each, the per-example callback receives
+ *  the item's UTF-8 content size — the quantity the scaling suite
+ *  (14-scaling.test.mjs) reports as a constant KB/s — then gates the global
+ *  example count and checkpointing (returns false to stop). */
+export declare function ingestItems(ci: CachedIngest, items: TrainingItem[], onItem: (contentBytes: number) => Promise<boolean>, sample?: (it: TrainingItem) => void): Promise<boolean>;
+/** Newline-delimited JSON, optionally gzipped.
+ *
+ *  ONE reader serves both the plain JSONL sources and the gzipped oasst2 tree
+ *  dump: the only difference between them is a `DecompressionStream("gzip")` in
+ *  the pipeline, and duplicating an 80-line splitter to express that was how
+ *  the two copies drifted apart in the first place.
+ *
+ *  Lines are split without buffering the whole file OR an unbounded line: a
+ *  record longer than `maxLineChars` is dropped (counted `skipped`) and the
+ *  stream continues at the next newline, so a corrupt record can never exhaust
+ *  memory or abort a good file. */
+export declare const lines: (opts: {
+    gzip?: boolean;
+    maxLineChars: number;
+}) => Reader;
+/** A whole-file JSON ARRAY of rows. The arrays this reads are small enough
+ *  (~16 MB) to parse whole; a huge file would be rejected by the cache ceiling
+ *  long before this. */
+export declare const jsonArray: () => Reader;
+/** How many rows to materialise in one read from a row-group of `rgRows` rows
+ *  occupying `groupBytes` uncompressed bytes, under a `budgetBytes` target.
+ *
+ *  The group's own footer statistics give the mean row width, so the batch
+ *  follows the CORPUS's row size rather than the writer's layout: wide rows
+ *  (SODA carries a whole dialogue per row) batch smaller than narrow ones at
+ *  the same memory cost. Never exceeds the group — a batch is a subdivision of
+ *  a group, never a span across two, because `parquetReadObjects` is given an
+ *  absolute row range and column chunks are per-group. Never returns 0, or the
+ *  read loop could not advance.
+ *
+ *  A writer that omits `total_byte_size` yields `groupBytes <= 0`; the batch is
+ *  then the whole group, which is exactly the behaviour this replaced. That
+ *  fallback is safe for every file we read today (all three report it) and
+ *  degrades to the old memory profile rather than to a wrong result. */
+export declare function parquetBatchRows(rgRows: number, groupBytes: number, budgetBytes: number): number;
+/** Parquet, read in bounded row batches with hyparquet (+Snappy from
+ *  hyparquet-compressors) over a web-standard Blob byte source. At most
+ *  `batchBytes` of source rows are materialised at a time, so neither a
+ *  multi-hundred-MB file nor a file written as ONE giant row-group loads whole
+ *  into memory.
+ *
+ *  Batching also makes a single-group file INTERRUPTIBLE: the abort check runs
+ *  per batch, where before a 1.19M-row group could not be cancelled at all.
+ *
+ *  `columns` PROJECTS the read down to the columns the adapter actually uses.
+ *  That is not only a memory economy: 2Wiki's `context` column holds the
+ *  Wikipedia prose the adapter exists to avoid depositing, and naming the
+ *  columns makes that exclusion structural — the bytes are never decoded at
+ *  all — in the same way reading only `utterances[].text` structurally excludes
+ *  Taskmaster's `instructions` scaffolding. Absent ⇒ every column, as before. */
+export declare const parquet: (opts?: {
+    batchBytes?: number;
+    columns?: string[];
+}) => Reader;