@hviana/sema 0.5.9 → 0.6.0

package/dist/example/train_base/stage.js

@@ -0,0 +1,246 @@
+// train_base/stage.ts — ONE loop, run once per corpus.
+//
+// Every stage in this trainer was the same seven steps — resume-check, build a
+// work-list, acquire each unit, read it, tally it, mark it complete, persist —
+// differing only in where the work-list comes from, what container the bytes
+// are in, and how a row becomes deposits. Those three are now DATA (a
+// `Corpus`, declared in corpus.ts), and this file is the loop they are fed to.
+//
+// Nothing but the loop lives here, so no corpus file needs to import it.
+import { MAX_BYTES } from "./config.js";
+import { unitIdOf } from "./corpus.js";
+import { loadProgress } from "./progress.js";
+import { DIM, dur, GRN, int, R, RED, YEL } from "./ui.js";
+import { statSync, unlinkSync } from "node:fs";
+/** Rows a read could not use, as the log has always reported them: one number.
+ *  The reader keeps malformed records and adapter-declined rows apart (see
+ *  FileResult) because they mean different things, but only a corpus that
+ *  declines records BY DESIGN needs the distinction on screen. */
+const unusedRows = (r, style) => style?.malformedOnly ? r.skipped : r.skipped + r.unusable;
+export async function runStage(ctx, corpus) {
+    const { progress, state, store, tick } = ctx;
+    const c = ctx.counters;
+    if (!corpus.enabled)
+        return;
+    if (c.trainedContentBytes >= MAX_BYTES || ctx.stopRequested)
+        return;
+    // Say what is actually happening. Discovery is a network call that can wait
+    // out a rate limit for minutes, and until it is announced the panel keeps
+    // displaying the PREVIOUS stage's file as though it were still processing.
+    state.activity = "list";
+    state.filePath = corpus.label;
+    state.fileExamples = 0;
+    // The unit counter belongs to the stage that is running, so it is cleared
+    // here rather than left showing the previous stage's totals through this one.
+    state.fileIndex = 0;
+    state.fileTotal = 0;
+    tick(true);
+    let units;
+    try {
+        units = await corpus.discover(ctx);
+    }
+    catch (e) {
+        if (ctx.stopRequested || e?.name === "AbortError")
+            return;
+        progress.log(`  ${RED}✗${R} ${corpus.label} file listing failed: ${e.message}`);
+        return;
+    }
+    if (units === null)
+        return; // discover already said why
+    if (units.length === 0) {
+        progress.log(`  ${DIM}· no ${corpus.label} files found — skipping${R}`);
+        return;
+    }
+    const idOf = (u) => unitIdOf(corpus, u);
+    const maxRows = corpus.maxRows ?? 0;
+    const p = await loadProgress(store);
+    const done = new Set(p.completedFiles);
+    // A budget-limited stage never marks its later shards complete — that is
+    // what lets a raised budget resume instead of restarting. But it also means
+    // "every shard complete" is NOT how such a stage finishes, so without a
+    // marker of its own a satisfied budget would re-read and re-deposit its
+    // rows on every subsequent run: harmless to the store (deposition is
+    // idempotent) but it repeats the work and double-counts langTally.
+    //
+    // The marker carries the budget it was satisfied AT, so raising the budget
+    // still resumes: a bigger budget does not match the marker and the stage
+    // runs again, picking up from the rows it has already taken (below) rather
+    // than from zero.
+    const budgetMark = maxRows > 0 ? `${corpus.id}::budget=${maxRows}` : "";
+    if (budgetMark && done.has(budgetMark)) {
+        progress.log(`  ${DIM}· ${corpus.label} budget of ${int(maxRows)} row(s) already met — skipping${R}`);
+        return;
+    }
+    const remaining = units.filter((u) => !done.has(idOf(u)));
+    if (remaining.length === 0) {
+        progress.log(`  ${DIM}· ${corpus.label} already trained — skipping${R}`);
+        return;
+    }
+    state.fileTotal = units.length;
+    state.unitNoun = corpus.unitNoun ?? "unit(s)";
+    // Fix the corpus denominator BEFORE reading anything. It used to grow as each
+    // file was opened, so the bar ran to 100% at the end of every file and fell
+    // back when the next was added — 100%, 50%, 100%, 66%. Both listings report
+    // sizes, so the pending work is knowable up front. Units of unknown size
+    // (a single-file corpus, whose size arrives with its HEAD) contribute 0 here
+    // and are added when they are opened, as before.
+    const pendingBytes = units
+        .filter((u) => !done.has(idOf(u)))
+        .reduce((n, u) => n + (u.bytes ?? 0), 0);
+    c.totalCorpusBytes = c.totalBytesProcessed + pendingBytes;
+    if (corpus.unitNoun) {
+        const taken = c.rowsTaken[corpus.id] ?? 0;
+        progress.log(`  ${GRN}✓${R} ${corpus.label}: ${remaining.length}/${units.length} ${corpus.unitNoun} to train` +
+            (maxRows > 0
+                ? ` ${DIM}(budget ${int(maxRows)} rows` +
+                    (taken > 0 ? `, ${int(taken)} already taken` : "") + `)${R}`
+                : ""));
+    }
+    // The budget spans the whole stage AND the whole STORE, not one shard and not
+    // one run. It starts from the rows already taken by units this store has
+    // finished, so an interrupted run resumes into the same budget instead of
+    // being granted a fresh one — without that, every Ctrl+C between two shards
+    // let a budgeted corpus deposit up to a full budget more than asked for, and
+    // for SODA the budget IS the curriculum balance (see corpora/soda.ts).
+    //
+    // The persisted figure counts FINISHED units only. Rows read from a unit that
+    // was cut short are deliberately not committed, because a resume re-reads
+    // that unit from the top and re-deposits exactly those rows (idempotent); had
+    // they been committed, the re-read would count them twice and the stage would
+    // stop short of its budget.
+    let rowsTaken = c.rowsTaken[corpus.id] ?? 0;
+    const spent = () => maxRows > 0 && rowsTaken >= maxRows;
+    const adapt = maxRows > 0
+        ? (row) => {
+            const items = corpus.toItems(row);
+            if (!items || items.length === 0)
+                return null;
+            rowsTaken++;
+            return items;
+        }
+        : corpus.toItems;
+    let idx = 0;
+    for (const u of units) {
+        if (c.trainedContentBytes >= MAX_BYTES || ctx.stopRequested)
+            break;
+        if (spent())
+            break;
+        idx++;
+        if (done.has(idOf(u)))
+            continue;
+        // Acquire (download or reuse), then read.
+        let path = u.local ?? "";
+        let downloaded = false;
+        if (!path) {
+            const got = await ctx.acquire(u.url, u.dest ?? idOf(u).replace(/[^A-Za-z0-9._-]+/g, "_"), u.acquireLabel ?? u.display);
+            if (!got) {
+                if (ctx.stopRequested)
+                    break;
+                continue; // a single failed unit never aborts the stage
+            }
+            path = got.path;
+            // A file that came from the cache is the corpus's to keep or to reclaim;
+            // only oasst2 keeps it. See Corpus.keepCached.
+            downloaded = corpus.keepCached ? !got.cached : true;
+        }
+        // Only for a unit the listing could not size (a single-file corpus). One
+        // whose size was declared is already in the denominator set above, and
+        // adding it again is exactly the drift that made the bar bounce.
+        if (!u.bytes) {
+            try {
+                c.totalCorpusBytes += statSync(path).size;
+            }
+            catch { /* best effort */ }
+        }
+        state.activity = "process";
+        state.fileIndex = idx;
+        state.filePath = u.display;
+        state.fileExamples = 0;
+        tick(true);
+        const p0 = Date.now();
+        let res;
+        // Pick up inside this unit if the last run stopped inside THIS one. A
+        // cursor is never applied to a different unit: the row count means whatever
+        // that unit's reader counts, and only there.
+        const cur = ctx.resumeCursor;
+        const startRow = cur && cur.unitId === idOf(u) ? cur.rows : 0;
+        try {
+            res = await corpus.read(path, adapt, ctx.readCtx({
+                unitId: idOf(u),
+                corpusId: corpus.id,
+                startRow,
+                shouldStop: maxRows > 0 ? spent : undefined,
+                // The LIVE count, for the cursor snapshot only. It must not be
+                // written into ctx.counters: those are what persist() records, and
+                // recording a budget figure for a unit that has not finished would
+                // charge the store for rows a resume is about to re-read.
+                rowsTakenNow: maxRows > 0 ? () => rowsTaken : undefined,
+            }));
+        }
+        catch (e) {
+            if (ctx.stopRequested || e?.name === "AbortError")
+                break;
+            progress.log(`  ${RED}✗${R} ${u.display} parse failed: ${e.message}`);
+            if (downloaded) {
+                try {
+                    unlinkSync(path);
+                }
+                catch { /* best effort */ }
+            }
+            continue;
+        }
+        // A shard cut short by the BUDGET is not "done" — leave it resumable so a
+        // later run with a bigger budget continues instead of starting over.
+        const hitBudget = spent();
+        // The tally is accrued per deposit by the runtime now (see onDeposit), so
+        // that a cursor taken mid-unit carries a true one. Adding res.examples here
+        // as well would count this unit twice.
+        const style = corpus.log;
+        const bad = unusedRows(res, style);
+        const badNoun = style?.malformedOnly
+            ? "malformed line(s)"
+            : style?.bad ?? "unusable row(s)";
+        const fromRows = style?.rows
+            ? `from ${int(res.rowsUsed)} ${style.rows} `
+            : "";
+        progress.log(`  ${GRN}✓${R} ${u.name} ${DIM}[${corpus.kind}]${R} → ${int(res.examples)} ${style?.deposits ?? "facts"} ${DIM}${fromRows}in ${dur((Date.now() - p0) / 1000)}${R}` +
+            (bad ? ` ${YEL}· ${int(bad)} ${badNoun} skipped${R}` : "") +
+            (hitBudget
+                ? ` ${YEL}(budget reached)${R}`
+                : res.stopped
+                    ? ` ${YEL}(stopped early)${R}`
+                    : ""));
+        if (!res.stopped && !hitBudget) {
+            try {
+                c.totalBytesProcessed += statSync(path).size;
+            }
+            catch { /* best effort */ }
+            if (downloaded) {
+                try {
+                    unlinkSync(path);
+                }
+                catch { /* best effort */ }
+            }
+            done.add(idOf(u));
+            p.completedFiles.push(idOf(u));
+            // Commit this unit's rows against the budget — see the note above: only
+            // a FINISHED unit's rows are committed, so a resume never counts a
+            // re-read prefix twice.
+            if (maxRows > 0)
+                c.rowsTaken[corpus.id] = rowsTaken;
+        }
+        await ctx.persist(p.completedFiles, !res.stopped && !hitBudget);
+        // A budget stop is not a cap/signal stop: the stage is finished, so fall
+        // out of the loop rather than treating it as an interruption.
+        if (res.stopped && !hitBudget)
+            break;
+    }
+    // Record a satisfied budget so the next run skips this stage instead of
+    // re-reading it. Only when the budget was actually reached: a stage that
+    // ran out of shards first is complete by the normal per-shard rule, and a
+    // stage cut short by MAX_MB or Ctrl+C must stay resumable.
+    if (budgetMark && spent() && !ctx.stopRequested && !done.has(budgetMark)) {
+        p.completedFiles.push(budgetMark);
+        await ctx.persist(p.completedFiles);
+    }
+}

package/dist/example/train_base/ui.d.ts

@@ -0,0 +1,88 @@
+import { type TrainingItem } from "./items.js";
+export declare const CSI = "\u001B[";
+export declare const B = "\u001B[1m", DIM = "\u001B[2m", R = "\u001B[0m";
+export declare const GREY = "\u001B[90m", CYAN = "\u001B[36m", GRN = "\u001B[32m";
+export declare const YEL = "\u001B[33m", RED = "\u001B[31m";
+export declare const HIDE = "\u001B[?25l", SHOW = "\u001B[?25h";
+/** Human-readable duration from seconds. */
+export declare function dur(seconds: number): string;
+/** Human-readable byte size. */
+export declare function bytes(n: number): string;
+/** Short count: 1234567 → "1.23M". */
+export declare function num(n: number): string;
+export declare const int: (n: number) => string;
+export declare const clamp01: (f: number) => number;
+export declare const pct: (f: number) => string;
+/** A progress bar of width `w` filled to fraction `frac`. */
+export declare function bar(w: number, frac: number): string;
+/** Collapse whitespace and clip to `max` chars with an ellipsis. */
+export declare function clip(text: string, max: number): string;
+export interface ProgState {
+    exampleCount: number;
+    target: number;
+    elapsedS: number;
+    trainedBytes: number;
+    trainedRate: number;
+    bytesDone: number;
+    bytesTotal: number;
+    bytesRate: number;
+    fileIndex: number;
+    fileTotal: number;
+    /** What this stage's units ARE ("translation file(s)", "shard(s)"). The panel
+     *  used to say "languages" for every corpus — true only of SmolSent, and
+     *  plainly wrong while reading Parquet shards or dialogue files. */
+    unitNoun: string;
+    filePath: string;
+    fileSize: number;
+    fileExamples: number;
+    /** "list" is the work-list call — an HF/GitHub tree fetch, which can wait
+     *  minutes behind a rate limit and had no way to say so: the panel simply
+     *  kept showing the PREVIOUS stage's file as though it were still being
+     *  processed. */
+    activity: "download" | "process" | "list" | "idle";
+    dlSpeed: number;
+    dlDone: number;
+    dlTotal: number;
+    storeEntries: number;
+    cacheBytes: number;
+    lastSample: string | null;
+}
+/** A prompt/expected pair to display for an item. */
+export declare function promptOf(it: TrainingItem): {
+    prompt: string;
+    expected: string | null;
+    kind: "episode" | "experience";
+};
+/** A coarse, honest similarity between an expected continuation and SEMA's
+ *  recall. Both are normalized (lowercased, whitespace-collapsed) and compared
+ *  by the longest shared leading run plus token overlap, so the verdict is a
+ *  heuristic signal of recall quality rather than a brittle fixed-prefix test. */
+export declare function recallSimilarity(expected: string, response: string): number;
+/** A framed recall sample. Pinned in the panel on a TTY (so the most recent
+ *  example is always on screen) and logged once per checkpoint when piped. */
+export declare function renderInferenceBox(prompt: string, expected: string | null, response: string, kind: "episode" | "experience", checkpointN: number): string;
+/** Render the whole panel. `title` names the curriculum being trained; the run
+ *  supplies it, so the panel never has to know which corpora exist. */
+export declare function renderPanel(s: ProgState, title: string): string;
+/** A live panel pinned to the bottom of stderr. On a TTY it redraws in place,
+ *  clearing only its own lines; logs are flushed into the scrollback above it.
+ *  Off a TTY (piped/CI) the panel is suppressed and a plain status line is
+ *  emitted occasionally, so logs stay clean and parseable. */
+export declare class Progress {
+    private readonly title;
+    private lines;
+    private lastPaint;
+    private lastStatus;
+    private last;
+    private readonly tty;
+    constructor(title: string);
+    /** True when attached to an interactive terminal (panel is live). */
+    get interactive(): boolean;
+    /** Cursor sequence that returns to the top of the panel and clears it. */
+    private clearPanel;
+    render(s: ProgState, force?: boolean): void;
+    /** Emit a line (or block) into the scrollback above the panel; the panel is
+     *  redrawn immediately beneath it so it never disappears between frames. */
+    log(msg: string): void;
+    dispose(): void;
+}

package/dist/example/train_base/ui.js

@@ -0,0 +1,272 @@
+// train_base/ui.ts — everything the run puts on the screen: the formatters, the
+// live panel pinned to the bottom of stderr, and the checkpoint recall box.
+//
+// Nothing here decides anything. It renders a ProgState the run owns, and the
+// run never reads a value back out of it — so the whole display can be swapped
+// (or silenced) without touching a line of training logic.
+import { CHECKPOINT_BYTES, D, DB_PATH, PROGRESS_MS, SEED } from "./config.js";
+import { isEpisode } from "./items.js";
+import { basename } from "node:path";
+// ── colours ──
+export const CSI = "\x1b[";
+export const B = `${CSI}1m`, DIM = `${CSI}2m`, R = `${CSI}0m`;
+export const GREY = `${CSI}90m`, CYAN = `${CSI}36m`, GRN = `${CSI}32m`;
+export const YEL = `${CSI}33m`, RED = `${CSI}31m`;
+export const HIDE = `${CSI}?25l`, SHOW = `${CSI}?25h`;
+// ── formatters ──
+/** Human-readable duration from seconds. */
+export function dur(seconds) {
+    if (!isFinite(seconds) || seconds < 0)
+        return "--";
+    const h = Math.floor(seconds / 3600);
+    const m = Math.floor((seconds % 3600) / 60);
+    const s = Math.floor(seconds % 60);
+    if (h > 0)
+        return `${h}h ${m}m ${s}s`;
+    if (m > 0)
+        return `${m}m ${s}s`;
+    return `${s}s`;
+}
+/** Human-readable byte size. */
+export function bytes(n) {
+    if (!isFinite(n) || n < 0)
+        return "--";
+    if (n < 1024)
+        return `${n} B`;
+    if (n < 1e6)
+        return `${(n / 1024).toFixed(1)} KB`;
+    if (n < 1e9)
+        return `${(n / 1e6).toFixed(1)} MB`;
+    return `${(n / 1e9).toFixed(2)} GB`;
+}
+/** Short count: 1234567 → "1.23M". */
+export function num(n) {
+    if (n >= 1e9)
+        return `${(n / 1e9).toFixed(2)}B`;
+    if (n >= 1e6)
+        return `${(n / 1e6).toFixed(2)}M`;
+    if (n >= 1e3)
+        return `${(n / 1e3).toFixed(1)}K`;
+    return String(n);
+}
+export const int = (n) => Math.round(n).toLocaleString("en-US");
+export const clamp01 = (f) => Math.max(0, Math.min(1, f));
+export const pct = (f) => `${(clamp01(f) * 100).toFixed(1)}%`;
+/** A progress bar of width `w` filled to fraction `frac`. */
+export function bar(w, frac) {
+    const filled = Math.round(clamp01(frac) * w);
+    return `${GRN}${"█".repeat(filled)}${GREY}${"░".repeat(w - filled)}${R}`;
+}
+/** Collapse whitespace and clip to `max` chars with an ellipsis. */
+export function clip(text, max) {
+    const t = text.replace(/\s+/g, " ").trim();
+    if (max < 1)
+        return "";
+    return t.length <= max ? t : t.slice(0, max - 1) + "…";
+}
+/** A prompt/expected pair to display for an item. */
+export function promptOf(it) {
+    return isEpisode(it)
+        ? { prompt: it.context, expected: it.continuation, kind: "episode" }
+        : { prompt: it.slice(0, 200), expected: null, kind: "experience" };
+}
+/** A coarse, honest similarity between an expected continuation and SEMA's
+ *  recall. Both are normalized (lowercased, whitespace-collapsed) and compared
+ *  by the longest shared leading run plus token overlap, so the verdict is a
+ *  heuristic signal of recall quality rather than a brittle fixed-prefix test. */
+export function recallSimilarity(expected, response) {
+    const norm = (s) => s.toLowerCase().replace(/\s+/g, " ").trim();
+    const a = norm(expected), b = norm(response);
+    if (!a || !b)
+        return 0;
+    let lead = 0;
+    const lim = Math.min(a.length, b.length);
+    while (lead < lim && a[lead] === b[lead])
+        lead++;
+    const leadFrac = lead / Math.max(1, Math.min(a.length, b.length));
+    const ta = new Set(a.split(" ")), tb = new Set(b.split(" "));
+    let inter = 0;
+    for (const w of ta)
+        if (tb.has(w))
+            inter++;
+    const jac = inter / Math.max(1, ta.size + tb.size - inter);
+    return Math.max(leadFrac, jac);
+}
+/** A framed recall sample. Pinned in the panel on a TTY (so the most recent
+ *  example is always on screen) and logged once per checkpoint when piped. */
+export function renderInferenceBox(prompt, expected, response, kind, checkpointN) {
+    const W = 68;
+    const hr = `${DIM}${"─".repeat(W)}${R}`;
+    const title = kind === "episode"
+        ? "latest recall"
+        : "latest recall (experience)";
+    const head = `${title} · checkpoint #${checkpointN} `;
+    const shown = response.trim() ? response : "(empty)";
+    const lines = [
+        `${B}╭─ ${head}${"─".repeat(Math.max(0, W - 2 - head.length))}╮${R}`,
+        `${B}│${R} ${hr}`,
+        `${B}│${R}  ${CYAN}${B}Context:${R}  ${clip(prompt, W - 13)}`,
+    ];
+    if (expected) {
+        lines.push(`${B}│${R}  ${YEL}${B}Expected:${R} ${clip(expected, W - 13)}`);
+    }
+    lines.push(`${B}│${R}  ${GRN}${B}SEMA:${R}     ${clip(shown, W - 13)}`);
+    lines.push(`${B}│${R} ${hr}`);
+    let verdict;
+    if (expected) {
+        const sim = recallSimilarity(expected, response);
+        const pctStr = `${Math.round(sim * 100)}%`;
+        verdict = sim >= 0.6
+            ? `${GRN}✓${R}  recall close to expected ${DIM}(~${pctStr} overlap)${R}`
+            : sim >= 0.25
+                ? `${YEL}△${R}  partial recall ${DIM}(~${pctStr} overlap)${R}`
+                : `${RED}✗${R}  recall diverges ${DIM}(~${pctStr} overlap)${R}`;
+    }
+    else {
+        verdict = `${DIM}·${R}  plain experience — no expected answer`;
+    }
+    lines.push(`${B}│${R}  ${verdict}`);
+    lines.push(`${B}╰${"─".repeat(W)}╯${R}`);
+    return lines.join("\n");
+}
+/** Render the whole panel. `title` names the curriculum being trained; the run
+ *  supplies it, so the panel never has to know which corpora exist. */
+export function renderPanel(s, title) {
+    const targetKnown = isFinite(s.target);
+    // Primary progress: by learned-content bytes when a MAX_MB target is set,
+    // else by how far we are through the corpus on disk (bytes) — so the default
+    // unbounded run still shows a real fraction and a real ETA.
+    const frac = targetKnown
+        ? (s.target > 0 ? s.trainedBytes / s.target : 0)
+        : (s.bytesTotal > 0 ? s.bytesDone / s.bytesTotal : 0);
+    const etaStr = (() => {
+        if (targetKnown) {
+            return s.trainedRate > 0
+                ? dur((s.target - s.trainedBytes) / s.trainedRate)
+                : "∞";
+        }
+        if (s.bytesTotal > 0 && s.bytesRate > 0) {
+            return dur((s.bytesTotal - s.bytesDone) / s.bytesRate);
+        }
+        return "∞";
+    })();
+    const fileFrac = s.fileTotal > 0 ? s.fileIndex / s.fileTotal : 0;
+    let actIcon = `${DIM}·${R}`, actText = "waiting…";
+    if (s.activity === "download") {
+        actIcon = `${CYAN}⬇${R}`;
+        const name = s.filePath;
+        const total = s.dlTotal > 0 ? s.dlTotal : s.fileSize;
+        if (total > 0 && s.dlDone > 0) {
+            const dlFrac = clamp01(s.dlDone / total);
+            actText =
+                `downloading ${name}  ${bar(18, dlFrac)} ${B}${pct(dlFrac)}${R}` +
+                    ` ${DIM}${bytes(s.dlDone)}/${bytes(total)}${R}`;
+            if (s.dlSpeed > 0)
+                actText += ` ${DIM}@ ${bytes(s.dlSpeed)}/s${R}`;
+        }
+        else {
+            actText = total > 0
+                ? `downloading ${name} · ${bytes(total)}…`
+                : `downloading ${name}…`;
+        }
+    }
+    else if (s.activity === "process") {
+        actIcon = `${GRN}✓${R}`;
+        actText = `processing ${s.filePath} · ${int(s.fileExamples)} examples so far`;
+    }
+    else if (s.activity === "list") {
+        actIcon = `${CYAN}⋯${R}`;
+        actText = `listing ${s.filePath} files…`;
+    }
+    const targetStr = targetKnown ? bytes(s.target) : "∞";
+    const headExamples = targetKnown
+        ? `${CYAN}${bytes(s.trainedBytes)}${R} / ${targetStr} learned ${DIM}·${R} ${int(s.exampleCount)} examples`
+        : `${CYAN}${int(s.exampleCount)}${R} examples`;
+    const corpusInfo = s.bytesTotal > 0
+        ? `${B}📦${R} ${bytes(s.bytesDone)}/${bytes(s.bytesTotal)} (${pct(s.bytesDone / s.bytesTotal)})`
+        : `${B}📦${R} ${bytes(s.bytesDone)} processed`;
+    const fileInfo = s.fileTotal > 0
+        ? `${B}🌐${R} ${s.fileIndex}/${s.fileTotal} ${s.unitNoun} (${pct(fileFrac)})`
+        : `${B}🌐${R} ${s.unitNoun}`;
+    const panel = [
+        `${B}╭${R}${B} sema train${R} ${DIM}·${R} ${title} ${DIM}·${R} ` +
+            `D=${D} ${DIM}·${R} seed=${SEED} ${DIM}·${R} ` +
+            `store=${basename(DB_PATH)}.sqlite\n${B}╰${R} target=${CYAN}${targetStr}${R} ` +
+            `learned ${DIM}·${R} checkpoint every ${bytes(CHECKPOINT_BYTES)}`,
+        `\n${bar(40, frac)}  ${B}${pct(frac)}${R}  ${headExamples}`,
+        `\n${B}⚡${R} ${bytes(s.trainedRate)}/s learned  ${B}🧠${R} ${bytes(s.trainedBytes)} content  ${B}⏱${R} ${dur(s.elapsedS)} elapsed  ${B}🕐${R} ${etaStr} ETA`,
+        `${fileInfo}  ${corpusInfo}  ${B}🗄${R} ${num(s.storeEntries)} entries  ` +
+            `${B}💾${R} cache ${bytes(s.cacheBytes)}`,
+        `\n${actIcon} ${actText}`,
+    ].join("");
+    return s.lastSample ? `${panel}\n${s.lastSample}` : panel;
+}
+/** A live panel pinned to the bottom of stderr. On a TTY it redraws in place,
+ *  clearing only its own lines; logs are flushed into the scrollback above it.
+ *  Off a TTY (piped/CI) the panel is suppressed and a plain status line is
+ *  emitted occasionally, so logs stay clean and parseable. */
+export class Progress {
+    title;
+    lines = 0; // height of the panel currently on screen
+    lastPaint = 0;
+    lastStatus = 0;
+    last = null;
+    tty = process.stderr.isTTY === true;
+    constructor(title) {
+        this.title = title;
+    }
+    /** True when attached to an interactive terminal (panel is live). */
+    get interactive() {
+        return this.tty;
+    }
+    /** Cursor sequence that returns to the top of the panel and clears it. */
+    clearPanel() {
+        if (this.lines <= 0)
+            return "";
+        const up = this.lines - 1; // cursor is on the panel's last line
+        return (up > 0 ? `${CSI}${up}F` : "\r") + `${CSI}0J`;
+    }
+    render(s, force = false) {
+        this.last = s;
+        const now = Date.now();
+        if (!force && now - this.lastPaint < PROGRESS_MS)
+            return;
+        this.lastPaint = now;
+        if (!this.tty) {
+            if (force || now - this.lastStatus >= 10_000) {
+                this.lastStatus = now;
+                const targetKnown = isFinite(s.target);
+                const where = s.bytesTotal > 0
+                    ? ` ${pct(s.bytesDone / s.bytesTotal)} of corpus`
+                    : "";
+                process.stderr.write(`[sema] ${bytes(s.trainedBytes)}${targetKnown ? "/" + bytes(s.target) : ""} learned · ${int(s.exampleCount)} examples · ` +
+                    `${bytes(s.trainedRate)}/s · ${s.fileIndex}/${s.fileTotal} ${s.unitNoun}${where} · ` +
+                    `${num(s.storeEntries)} entries\n`);
+            }
+            return;
+        }
+        const text = renderPanel(s, this.title);
+        process.stderr.write(`${this.clearPanel()}${HIDE}${text}`);
+        this.lines = text.split("\n").length;
+    }
+    /** Emit a line (or block) into the scrollback above the panel; the panel is
+     *  redrawn immediately beneath it so it never disappears between frames. */
+    log(msg) {
+        if (!this.tty) {
+            process.stderr.write(`${msg}\n`);
+            return;
+        }
+        let out = `${this.clearPanel()}${msg}\n`;
+        this.lines = 0;
+        if (this.last) {
+            const text = renderPanel(this.last, this.title);
+            out += `${HIDE}${text}`;
+            this.lines = text.split("\n").length;
+        }
+        process.stderr.write(out);
+    }
+    dispose() {
+        if (this.tty)
+            process.stderr.write(`${SHOW}\n`);
+    }
+}

package/dist/src/mind/mind.d.ts

@@ -282,7 +282,7 @@ export declare class Mind implements MindContext {
      *     "what byte separates two turns?" because nothing downstream finds
      *     boundaries by looking at content at all.
      *  2. A separator in a CORPUS is ordinary content.  If a trainer joins
-     *     turns with "\n" (example/train_base.ts does), those newlines are
+     *     turns with "\n" (example/train_base does), those newlines are
      *     simply bytes inside the stream, folded like every other byte.  They
      *     are a property of that corpus, not of this API and not of the fold.
      *  3. This API can therefore reproduce ANY corpus exactly, with no

package/dist/src/mind/mind.js

@@ -479,7 +479,7 @@ export class Mind {
      *     "what byte separates two turns?" because nothing downstream finds
      *     boundaries by looking at content at all.
      *  2. A separator in a CORPUS is ordinary content.  If a trainer joins
-     *     turns with "\n" (example/train_base.ts does), those newlines are
+     *     turns with "\n" (example/train_base does), those newlines are
      *     simply bytes inside the stream, folded like every other byte.  They
      *     are a property of that corpus, not of this API and not of the fold.
      *  3. This API can therefore reproduce ANY corpus exactly, with no