@hviana/sema 0.5.9 → 0.6.0

package/example/train_base/items.ts

@@ -0,0 +1,118 @@
+// train_base/items.ts — the REPRESENTATION core: what a training item is, and
+// the three shapes a corpus datum may take.
+//
+// REPRESENTATION POLICY (one datum → one form; no replication):
+//   • FACTS are the default. A datum that is a RELATION (translation pair,
+//     question → answer) is emitted as a (context → continuation) edge SEMA
+//     points at and, by example across the corpus, generalizes from (cf.
+//     example/demo.ts).
+//   • EXPERIENCES (bare statements) are used only when a fact is NOT possible —
+//     content with no natural relational split.
+//   • CUMULATIVE CONTINUOUS CONTEXT is used only when truly necessary — genuine
+//     MULTI-TURN dialogue, where a turn follows from the whole conversation so
+//     far. The fact stages do NOT synthesize a multi-turn walk, which would just
+//     replicate the facts (repetition SEMA avoids).
+//
+// Nothing here reads the environment or touches I/O: these are the pure
+// functions every corpus adapter is built out of.
+
+export interface Episode {
+  context: string;
+  continuation: string;
+}
+export type TrainingItem = string | Episode;
+
+export const isEpisode = (it: TrainingItem): it is Episode =>
+  typeof it !== "string";
+
+/** One turn of a dialogue, attributed to a speaker. The speaker is only ever
+ *  used to decide MERGING (see `mergeSpeakerTurns`); it is never deposited. */
+export interface SpeakerTurn {
+  // Upper-cased by the adapters, so USER/ASSISTANT (TM-1/2) and user/assistant
+  // (TM-3/4) compare equal.
+  speaker: string;
+  text: string;
+}
+
+/** Build the accumulated-context episodes of a turn sequence: each successive
+ *  turn is the continuation of ALL the turns before it joined together. This is
+ *  the same cumulative-context shape a multi-turn conversation deposits, so the
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
+export function accumulate(turns: string[]): Episode[] {
+  const out: Episode[] = [];
+  for (let i = 1; i < turns.length; i++) {
+    out.push({ context: turns.slice(0, i).join("\n"), continuation: turns[i] });
+  }
+  return out;
+}
+
+/** Collapse consecutive same-speaker turns into one, joining with a space, and
+ *  return the bare texts in order. A turn with no speaker never merges with its
+ *  neighbour: an unlabelled row is of unknown origin, and joining two of them
+ *  would invent a contribution that may span two speakers.
+ *
+ *  Load-bearing for corpora that split one contribution across several indexed
+ *  utterances (an artifact of the collection UI). Left unmerged, the cumulative
+ *  walk deposits a turn boundary in the middle of one speaker's contribution
+ *  and teaches it as a hand-off. Measured share of turns absorbed by merging:
+ *  TM-1 17.7%, TM-2 11.9%, TM-3 0.8%, TM-4 0.0%. */
+export function mergeSpeakerTurns(turns: SpeakerTurn[]): string[] {
+  const out: string[] = [];
+  let prev = "";
+  for (const t of turns) {
+    if (out.length > 0 && t.speaker !== "" && t.speaker === prev) {
+      out[out.length - 1] += " " + t.text;
+    } else {
+      out.push(t.text);
+    }
+    prev = t.speaker;
+  }
+  return out;
+}
+
+/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
+ *  repeats so a concept never deposits the same form twice. */
+export function refineItems(items: TrainingItem[]): TrainingItem[] {
+  const out: TrainingItem[] = [];
+  const seen = new Set<string>();
+  for (const it of items) {
+    if (!isEpisode(it)) {
+      const exp = it.trim();
+      const key = "E:" + exp;
+      if (exp && !seen.has(key)) {
+        seen.add(key);
+        out.push(exp);
+      }
+      continue;
+    }
+    const ctx = it.context.trim();
+    const cont = it.continuation.trim();
+    if (!ctx || !cont || ctx === cont) continue;
+    const key = "P:" + ctx + "\u0000" + cont;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push({ context: ctx, continuation: cont });
+  }
+  return out;
+}
+
+const ENC = new TextEncoder();
+
+/** Content size of a training item in UTF-8 bytes — the same quantity the
+ *  scaling suite (14-scaling.test.mjs) measures as KB/s: for an episode the
+ *  context plus the continuation, for a bare experience its own text. */
+export const itemBytes = (it: TrainingItem): number =>
+  isEpisode(it)
+    ? ENC.encode(it.context).length + ENC.encode(it.continuation).length
+    : ENC.encode(it).length;

package/example/train_base/main.ts

@@ -0,0 +1,240 @@
+//This file uses the Google SMOL dataset, made available under the CC BY 4.0 license.
+//This file uses Aya and oasst2 datasets, made available under the apache-2.0 license.
+//
+//A trained Sema store retains its training text VERBATIM, so distributing a
+//store distributes these corpora and every upstream licence applies to it in
+//full. Read DATASETS.md before adding a corpus here or publishing a store:
+//it carries the per-corpus attribution a distributed store is required to
+//travel with, and the two rules a candidate corpus must pass (no NonCommercial
+//term, no ShareAlike term — checked against what the corpus was BUILT FROM,
+//not merely against the repository's licence tag).
+
+//This file is a more appropriate training example for Sema.
+//Sema does not learn through repetition;
+//it does not require a massive database.
+//It needs fundamental datasets that teach basic cognitive concepts such as conversation, logic, relationships, behaviors and feelings.
+//The focus is on covering fundamental patterns, not repetition.
+//Tip: ontology-based adapted training datasets could be an interesting path.
+
+// train_base/main.ts — streaming trainer for the SmolSent + Aya + oasst2 +
+//                  Taskmaster + 2Wiki + SODA base.
+//
+// Training IS deposition: every source datum is translated into SEMA facts (or,
+// for genuine dialogue, accumulated-context episodes), then stored in one pass.
+// There are no gradients or epochs, and there is no LLM in the loop — the only
+// "model" is the SEMA store itself.
+//
+// Every source here is commercially licensable (cc-by-4.0 / apache-2.0).
+//
+// WHERE THINGS LIVE (this file is the folder's entry point and nothing else):
+//
+//   config.ts     RUN-level knobs — store, checkpoint cadence, cache ceiling,
+//                 MAX_MB, the Parquet read budget.
+//   items.ts      the REPRESENTATION core: what a training item is (fact /
+//                 experience / accumulated walk) and the policy governing
+//                 which shape a datum may take.
+//   http.ts       network policy: wait out throttling, retry what is
+//                 transient, give up at once on what is not.
+//   cache.ts      the durable disk cache and the atomic download sink — the
+//                 sole irreducible Node dependency.
+//   readers.ts    container formats: newline-delimited JSON (plain or
+//                 gzipped), JSON arrays, Parquet.
+//   discovery.ts  where a work-list comes from: an HF repo tree, the
+//                 auto-converted Parquet branch, a GitHub directory, a local
+//                 directory.
+//   progress.ts   the resume record inside the store, and the index passes
+//                 that keep a checkpoint queryable.
+//   ui.ts         the live panel, the formatters, the recall box.
+//   runtime.ts    the RUN: counters, the deposit gate, file acquisition,
+//                 checkpointing, shutdown.
+//   corpus.ts     WHAT A CORPUS IS — the contract a corpus declares, and the
+//                 one rule that turns it into a resume id.
+//   stage.ts      ONE loop, run once per corpus.
+//   corpora/      ONE FILE PER CORPUS: its knobs, its row adapter, its stage
+//                 descriptor, and the evidence that fixed each default.
+//                 corpora/index.ts is the curriculum, in order.
+//
+// Adding a corpus is therefore one new file in corpora/ plus its import, its
+// place in CURRICULUM and its re-export in corpora/index.ts — and no change at
+// all to the loop, the readers, or the run.
+//
+// Every source is DOWNLOADED as a file and streamed from disk (never paged
+// row-by-row over an HTTP API — that was slow and rate-limited). Resume is
+// per-file: a fully-consumed file is marked complete; an interrupted one
+// re-reads from the top (re-deposition is idempotent). LOCAL_PATH may hold
+// pre-downloaded files.
+//
+// The store IS the model: memories, training metadata, and the config snapshot
+// all live in {DB_PATH}.sqlite, so a run resumes from the store alone.
+//
+// Built on web standards. All I/O except the durable disk cache uses platform
+// primitives — fetch, WHATWG ReadableStream/WritableStream/TransformStream,
+// DecompressionStream, TextDecoderStream, Blob, AbortController. The sole
+// third-party code is hyparquet (+ its Snappy codec): a DEV dependency of this
+// example, never of the library, and loaded by a dynamic import inside
+// readers.ts the first time a Parquet corpus is read — so a curriculum with no
+// Parquet stage needs it not at all. Consistency guarantees:
+//   • Resume from the store alone — completed stage-units, example count,
+//     learned-content bytes, and processed-byte total are persisted in
+//     {DB_PATH}.sqlite and reloaded.
+//   • Atomic cache — a download streams to "<file>.part", is fsync'd, then
+//     renamed into place; a file at its final path is, by construction,
+//     complete, so an interrupted download can never be mistaken for a cached
+//     one.
+//   • Bounded cache — a download blocks under the MAX_CACHE_GB ceiling and the
+//     fully-processed file is deleted immediately.
+//   • Interruptible — Ctrl+C (SIGINT/SIGTERM) aborts in-flight network at once,
+//     stops at the next item boundary, writes a final checkpoint, and exits; an
+//     un-finished stage-unit is NOT marked complete, so resume re-reads it (re-
+//     deposition is idempotent). A second Ctrl+C, or a 60s watchdog, force-exits.
+//
+// Run:
+//   npx tsc && node dist/example/train_base/main.js
+//   MAX_MB=500 node dist/example/train_base/main.js
+//   CHECKPOINT_MB=250 node dist/example/train_base/main.js
+//   SMOLSENT_PAIRS=ha_en,zu_en node dist/example/train_base/main.js  # a subset of pairs
+//   SMOLSENT_DIRECTIONS=both node dist/example/train_base/main.js  # also English->foreign
+//   SMOLSENT=0 node dist/example/train_base/main.js           # skip SmolSent stage
+//   AYA=0 node dist/example/train_base/main.js                # skip Aya stage
+//   OASST=0 node dist/example/train_base/main.js              # skip oasst2 stage
+//   OASST_MIN_TURNS=6 node dist/example/train_base/main.js    # deeper multi-turn only
+//   GENKNOW=1 node dist/example/train_base/main.js            # General-Knowledge (see DATASETS.md §3.2)
+//   PARQUET_BATCH_MB=8 node dist/example/train_base/main.js   # smaller Parquet reads on a tight host
+//   TASKMASTER=0 node dist/example/train_base/main.js         # skip Taskmaster stage
+//   TASKMASTER_SETS=TM-3-2020 node dist/example/train_base/main.js  # one Taskmaster set
+//   WIKI2=0 node dist/example/train_base/main.js               # skip 2Wiki triples stage
+//   SODA=0 node dist/example/train_base/main.js                # skip the SODA stage
+//   MASSIVE=1 node dist/example/train_base/main.js             # enable MASSIVE (off by default)
+//   SODA_MAX_DIALOGS=0 node dist/example/train_base/main.js    # lift the SODA budget
+//   WIKI2_MAX_ROWS=50000 node dist/example/train_base/main.js  # budget the 2Wiki stage
+//   LOCAL_PATH=./base node dist/example/train_base/main.js    # offline: *.jsonl/.parquet/.jsonl.gz/.json
+//   DB_PATH=./data/sema node dist/example/train_base/main.js
+
+import { CachedIngest, Mind, SQliteStore } from "../../src/index.js";
+import {
+  D,
+  DB_PATH,
+  SEED,
+  SQLITE_CACHE_MB,
+  VECTOR_CACHE_MB,
+} from "./config.js";
+import { createRuntime } from "./runtime.js";
+import { runStage } from "./stage.js";
+import { sweepPartials } from "./cache.js";
+import { CURRICULUM, enabledLabels, storedLabels } from "./corpora/index.js";
+import { bytes, DIM, dur, num, R, RED, SHOW } from "./ui.js";
+
+// The parser/representation surface this module used to define itself. Kept
+// exported from here so importing `example/train_base/main.js` still reaches every
+// row adapter (toSmolSentRow, wikiTriplesToItems, …) and the shapes they build.
+export * from "./items.js";
+export * from "./corpora/index.js";
+export { parquetBatchRows } from "./readers.js";
+// The name this helper had while it lived here. It is `mergeSpeakerTurns` now,
+// because SODA merges by the same rule and the Taskmaster-specific name was a
+// lie — but the old name stays reachable so nothing importing it breaks.
+export { mergeSpeakerTurns as mergeTaskmasterTurns } from "./items.js";
+
+async function main(): Promise<void> {
+  const store = new SQliteStore({
+    path: DB_PATH,
+    D,
+    vectorCacheMb: VECTOR_CACHE_MB,
+    sqliteCacheMb: SQLITE_CACHE_MB,
+  });
+
+  // The store IS the model: memories, progress, and metadata all persist in
+  // it, so a resumed run just reopens the same store and continues. Guard
+  // against a changed D/SEED by comparing against what a previous run recorded.
+  const mind = new Mind({ seed: SEED, store });
+
+  // Pre-fill the vector indices' RAM caches with sequential scans (bounded by
+  // VECTOR_CACHE_MB).  A resumed run over a large store otherwise spends its
+  // first minutes warming those caches through random point reads — the
+  // ingest hot path is cache-miss bound until then.  Seconds, once, up front.
+  if (VECTOR_CACHE_MB > 0) {
+    const t = Date.now();
+    const warmed = await store.warmVectorCaches();
+    if (warmed > 0) {
+      process.stderr.write(
+        `  warmed vector caches: ${num(warmed)} rows in ${
+          dur((Date.now() - t) / 1000)
+        }\n`,
+      );
+    }
+  }
+  const ci = new CachedIngest(mind);
+  const prevD = await store.getMeta("train.D");
+  const prevSeed = await store.getMeta("train.seed");
+  if (
+    (prevD && Number(prevD) !== D) || (prevSeed && Number(prevSeed) !== SEED)
+  ) {
+    process.stderr.write(
+      `fatal: D/SEED changed (store has D=${prevD} seed=${prevSeed}, ` +
+        `requested D=${D} seed=${SEED}). Delete ${DB_PATH}.sqlite ` +
+        `to start fresh.\n`,
+    );
+    process.exit(1);
+  }
+
+  const dataset = enabledLabels();
+  await store.setMeta("train.D", String(D));
+  await store.setMeta("train.seed", String(SEED));
+  // createdAt is when this STORE was first trained into, so a resume must not
+  // overwrite it — the previous behaviour left every store claiming it was
+  // created at its most recent resume. When the current run started is a
+  // separate, also-useful fact, so it gets its own key rather than displacing
+  // this one.
+  if ((await store.getMeta("train.createdAt")) === null) {
+    await store.setMeta("train.createdAt", new Date().toISOString());
+  }
+  await store.setMeta("train.startedAt", new Date().toISOString());
+
+  const ctx = createRuntime({ store, mind, ci, title: dataset });
+  ctx.tick(true);
+
+  // Reclaim the debris of any download killed mid-transfer. It occupies cache
+  // ceiling that nothing else would ever free (see cache.ts sweepPartials).
+  const swept = sweepPartials();
+  if (swept.files > 0) {
+    ctx.progress.log(
+      `  ${DIM}· swept ${swept.files} interrupted download(s), ` +
+        `${bytes(swept.bytes)} reclaimed${R}`,
+    );
+  }
+
+  // ── resume — restore counters and the per-source tally from the store ──
+  await ctx.restore();
+
+  // What the STORE contains, which is only knowable after the resume: this
+  // run's stages plus whatever earlier runs deposited. The panel header above
+  // names this RUN's stages, which is a different and also-true fact.
+  await store.setMeta(
+    "train.dataset",
+    storedLabels(Object.keys(ctx.counters.langTally)),
+  );
+
+  // Walk the curriculum. Each stage skips itself on a resume that already
+  // finished it, and the walk stops at the first requested stop.
+  for (const corpus of CURRICULUM) {
+    if (ctx.stopRequested) break;
+    await runStage(ctx, corpus);
+  }
+
+  // Nothing is in flight any more; each stage announces itself as it starts, so
+  // this is the one transition that has no next stage to correct it.
+  ctx.state.activity = "idle";
+  await ctx.finish(ctx.stopRequested ? ctx.stopReason : "done");
+}
+
+// Only run when invoked directly, so importing the row adapters above (e.g.
+// for a fixture check) never starts training.
+const isMain = import.meta.url === `file://${process.argv[1]}` ||
+  process.argv[1]?.endsWith("train_base/main.js");
+if (isMain) {
+  main().catch((e) => {
+    process.stderr.write(SHOW);
+    console.error(`\n${RED}fatal:${R}`, e);
+    process.exit(1);
+  });
+}

package/example/train_base/progress.ts

@@ -0,0 +1,149 @@
+// train_base/progress.ts — resume state, and the index passes that keep a
+// checkpointed store queryable.
+//
+// The store IS the model: memories, training metadata and the config snapshot
+// all live in {DB_PATH}.sqlite, so a run resumes from the store alone. The keys
+// below are that resume record, and they are a COMPATIBILITY SURFACE — a store
+// trained by an earlier version must keep resuming, so neither the key names
+// nor the shape of `completedFiles` may drift.
+
+import type { Mind, Store } from "../../src/index.js";
+import { INDEX_MAINTENANCE } from "./config.js";
+import { DIM, GRN, int, R, YEL } from "./ui.js";
+
+const META_COMPLETED = "train.completedFiles";
+const META_DEPOSITS = "train.depositCount";
+const META_TRAINED_BYTES = "train.trainedContentBytes";
+const META_BYTES = "train.totalBytesProcessed";
+const META_CORPUS_BYTES = "train.totalCorpusBytes";
+
+export interface SavedProgress {
+  completedFiles: string[];
+  depositCount: number;
+  trainedContentBytes: number;
+  totalBytesProcessed: number;
+  totalCorpusBytes: number;
+}
+
+export async function loadProgress(store: Store): Promise<SavedProgress> {
+  try {
+    const raw = await store.getMeta(META_COMPLETED);
+    const deps = await store.getMeta(META_DEPOSITS);
+    const b = await store.getMeta(META_BYTES);
+    if (raw !== null && deps !== null && b !== null) {
+      const completedFiles = JSON.parse(raw);
+      if (Array.isArray(completedFiles)) {
+        const trained = await store.getMeta(META_TRAINED_BYTES);
+        const corpus = await store.getMeta(META_CORPUS_BYTES);
+        return {
+          completedFiles,
+          depositCount: Number(deps) || 0,
+          trainedContentBytes: Number(trained) || 0,
+          totalBytesProcessed: Number(b) || 0,
+          totalCorpusBytes: Number(corpus) || 0,
+        };
+      }
+    }
+  } catch { /* corrupt/missing — start fresh */ }
+  return {
+    completedFiles: [],
+    depositCount: 0,
+    trainedContentBytes: 0,
+    totalBytesProcessed: 0,
+    totalCorpusBytes: 0,
+  };
+}
+
+export async function saveProgress(
+  store: Store,
+  p: SavedProgress,
+): Promise<void> {
+  await store.setMeta(META_COMPLETED, JSON.stringify(p.completedFiles));
+  await store.setMeta(META_DEPOSITS, String(p.depositCount));
+  await store.setMeta(META_TRAINED_BYTES, String(p.trainedContentBytes));
+  await store.setMeta(META_BYTES, String(p.totalBytesProcessed));
+  await store.setMeta(META_CORPUS_BYTES, String(p.totalCorpusBytes));
+  await store.setMeta("train.updatedAt", new Date().toISOString());
+  store.commit();
+}
+
+/** Run index maintenance: compact (remove garbage), repair (fill gaps),
+ *  then refresh the canonical-form index (see below).  All three are
+ *  idempotent — running twice produces the same result as once.
+ *  Compaction frees index space first; repair then adds back every
+ *  edge/halo-bearing node whose gist was evicted from the pending cache
+ *  before it reached the content index, completing the coverage that
+ *  incremental promotion alone cannot guarantee.
+ *
+ *  repair runs with minParents = 0, NOT the library default of 2.  The
+ *  default repairs only structural BRIDGES (≥2 parents), but this
+ *  trainer's fact deposits also leave answer-side DEPOSIT ROOTS with 0
+ *  structural parents ("The capital of France is Paris." as the dst of a
+ *  Q→A edge is a root of its own tree, contained in nothing).  Those are
+ *  resonance targets recall depends on — a trained store shipped without
+ *  them cannot ground statement-shaped queries against its own answers
+ *  (observed: 33 such roots missing after a full curriculum, including
+ *  high-traffic conversation replies).  minParents = 0 admits every
+ *  edge/halo bearer; the candidate set is still corpus-of-experiences-
+ *  sized, so the pass stays cheap.
+ *
+ *  Logs the number of entries removed/added so a run that silently degrades
+ *  (growing compaction count, or repair never recovering anything) is
+ *  visible in the training log. */
+export async function runIndexMaintenance(
+  mind: Mind,
+  log: (msg: string) => void,
+): Promise<void> {
+  if (!INDEX_MAINTENANCE) return;
+  try {
+    const removed = await mind.store.compactContentIndex();
+    if (removed > 0) {
+      log(
+        `  ${DIM}index compact: removed ${int(removed)} isolated entries${R}`,
+      );
+    }
+  } catch (err) {
+    log(
+      `  ${YEL}⚠ index compact failed${R}: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+  try {
+    const added = await mind.repairContentIndex(0);
+    if (added > 0) {
+      log(
+        `  ${GRN}index repair: added ${
+          int(added)
+        } missing resonance targets${R}`,
+      );
+    }
+  } catch (err) {
+    log(
+      `  ${YEL}⚠ index repair failed${R}: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+  // Canonical-form index (src/canon.ts): lets resolution find stored forms
+  // across surface variation (case, width, whitespace).  Incremental and
+  // idempotent by construction — the `canon.upto` meta cursor scans only
+  // nodes newer than the last pass, and the (h, id) primary key ignores
+  // re-inserted rows — so it composes with the resume model exactly like
+  // compact/repair: every checkpoint (and finish) leaves the index
+  // covering all content trained so far.
+  try {
+    const added = await mind.buildCanonIndex();
+    if (added > 0) {
+      log(
+        `  ${GRN}canon index: added ${int(added)} canonical-form entries${R}`,
+      );
+    }
+  } catch (err) {
+    log(
+      `  ${YEL}⚠ canon index build failed${R}: ${
+        err instanceof Error ? err.message : String(err)
+      }`,
+    );
+  }
+}