@hviana/sema 0.5.9 → 0.6.0

package/dist/example/train_base/discovery.js

@@ -0,0 +1,143 @@
+// train_base/discovery.ts — where a stage's WORK-LIST comes from.
+//
+// Four remote strategies and two local ones, each generic over the dataset it
+// is pointed at. What stays with a corpus is its POLICY — which subset of the
+// listing to keep, how to name the resume unit — because that is a curriculum
+// decision, not a protocol one.
+//
+// A note that has bitten this code twice, in both directions: a dataset id
+// ("owner/name") is a PATH here and its "/" must NOT be percent-encoded, while
+// a branch name ("refs/convert/parquet") is a single path SEGMENT and its "/"
+// MUST be.
+import { getJson, getJsonPaged } from "./http.js";
+import { readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+const sizeOf = (e) => {
+    const n = Number(e?.size ?? e?.lfs?.size ?? 0);
+    return Number.isFinite(n) && n > 0 ? n : 0;
+};
+/** Files under `path` in a Hugging Face dataset repo's main branch, filtered to
+ *  an extension. Returns repo-relative paths (e.g. "smolsent/ha_en.jsonl"),
+ *  sorted, so a run's unit order is stable across machines. */
+export async function hfTree(dataset, path, ext, label, opts) {
+    // The dataset id is a PATH here, so its "/" must not be percent-encoded.
+    // `recursive=true` returns every file under `path`, 1,000 at a time — hence
+    // the PAGED fetch: a truncated work-list would train part of a corpus and
+    // then call it finished.
+    const url = `https://huggingface.co/api/datasets/${dataset}` +
+        `/tree/main/${path}?recursive=true`;
+    const body = await getJsonPaged(url, label, opts);
+    const out = body
+        .filter((e) => e?.type === "file" && ext.test(e?.path))
+        .map((e) => ({ path: String(e.path), size: sizeOf(e) }));
+    out.sort((a, b) => a.path.localeCompare(b.path));
+    return out;
+}
+/** A dataset's Parquet shards on Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch, restricted to `config` and to `splits`.
+ *
+ *  The converted branch is used rather than `main` because a dataset's own
+ *  Parquet may be written as ONE giant row-group (SODA's is 1,191,582 rows),
+ *  and a column chunk is per-group, so reading any part of it materialises all
+ *  of it. The converted branch is uniformly 10,000-row groups.
+ *
+ *  Paths look like "<config>/<split>/0000.parquet". The BRANCH name is a single
+ *  path SEGMENT here, so its "/" is percent-encoded — unlike a dataset id,
+ *  whose "/" must not be. */
+export async function hfConvertedParquet(dataset, config, splits, label, opts, note) {
+    const body = await getJsonPaged(`https://huggingface.co/api/datasets/${dataset}` +
+        `/tree/refs%2Fconvert%2Fparquet/${config}?recursive=true`, `GET ${label} tree`, opts);
+    const paths = body
+        .filter((e) => e?.type === "file" && /\.parquet$/i.test(e?.path))
+        .map((e) => ({ path: String(e.path), size: sizeOf(e) }));
+    paths.sort((a, b) => a.path.localeCompare(b.path));
+    // The tree is rooted at `config`, so the split is the second-to-last part.
+    const splitOf = (p) => p.split("/").slice(-2)[0] ?? "";
+    const present = new Set(paths.map((p) => splitOf(p.path)));
+    // MATCH THE SPLIT AGAINST WHAT THE BRANCH ACTUALLY CARRIES. An exact name is
+    // not guaranteed: the converter renames a split it could not finish, and
+    // shards a very large one. Observed on real datasets today —
+    //   allenai/c4            → partial-train, partial-validation
+    //   HuggingFaceFW/fineweb → train-part0
+    // — neither of which equals "train". The old exact-match filter returned []
+    // for both, and an empty work-list is reported as "no files found — skipping",
+    // which reads like a normal outcome rather than a corpus being dropped whole.
+    const wanted = new Set();
+    for (const want of splits) {
+        const pick = present.has(want)
+            ? want
+            : [...present].find((s) => s === `partial-${want}`) ??
+                [...present].find((s) => s.startsWith(`${want}-part`));
+        if (!pick) {
+            // Loud, not empty: a requested split that simply is not there is a
+            // configuration error, and silence would hide the whole corpus.
+            throw new Error(`${label}: split "${want}" is not on the converted branch — it ` +
+                `carries ${[...present].join(", ") || "no parquet at all"}`);
+        }
+        if (pick !== want) {
+            note?.(`${label}: split "${want}" is published as "${pick}"` +
+                (pick.startsWith("partial-")
+                    ? " — Hugging Face has only PARTIALLY converted this dataset, so " +
+                        "the shards below are not the whole split"
+                    : ""));
+        }
+        wanted.add(pick);
+    }
+    return paths.filter((p) => wanted.has(splitOf(p.path)));
+}
+/** File NAMES (not paths) in one directory of a GitHub repo, filtered to an
+ *  extension and sorted. Used for corpora served from GitHub raw rather than
+ *  Hugging Face — where the HF mirrors are loading-script repos with no data
+ *  files, the official GitHub copy is the one carrying the licence notice. */
+export async function githubContents(repo, dir, ext, label, opts) {
+    const body = await getJson(`https://api.github.com/repos/${repo}/contents/${dir}`, label, opts);
+    const entries = Array.isArray(body) ? body : [];
+    // The contents API returns at most 1,000 entries for a directory and does NOT
+    // paginate them — it simply stops, with no Link header and no error. A
+    // directory at that boundary is therefore indistinguishable from a truncated
+    // one, so the only honest response is to refuse rather than train part of it
+    // and record the part as the whole.
+    if (entries.length >= 1000) {
+        throw new Error(`${label}: GitHub returned ${entries.length} entries, the point at ` +
+            `which the contents API truncates without saying so — this listing ` +
+            `cannot be trusted to be complete`);
+    }
+    const names = entries
+        .filter((e) => e?.type === "file" && ext.test(e?.name))
+        .map((e) => ({ path: String(e.name), size: sizeOf(e) }));
+    names.sort((a, b) => a.path.localeCompare(b.path));
+    return names;
+}
+/** Every file in a local directory matching `ext`, sorted. A missing directory
+ *  is an empty list, not an error: LOCAL_PATH is an offline convenience and a
+ *  stage with no local copy simply reports that and moves on. */
+export function localFiles(dir, ext) {
+    try {
+        return readdirSync(dir)
+            .filter((f) => ext.test(f))
+            .sort()
+            .map((f) => {
+            let size = 0;
+            try {
+                size = statSync(join(dir, f)).size;
+            }
+            catch { /* unreadable — the read will report it */ }
+            return { path: f, size };
+        });
+    }
+    catch {
+        return []; // no such directory
+    }
+}
+/** The FIRST file in a local directory matching any of `exts`, in directory
+ *  order (deliberately NOT sorted — this mirrors the single-file stages, which
+ *  take whichever copy the filesystem hands back first). Null when none match
+ *  or the directory is absent. */
+export function localFind(dir, ...exts) {
+    try {
+        return readdirSync(dir).find((f) => exts.some((re) => re.test(f))) ?? null;
+    }
+    catch {
+        return null; // no such directory
+    }
+}

package/dist/example/train_base/http.d.ts

@@ -0,0 +1,82 @@
+/** Sleep `ms`, but wake early if `signal` fires — so a long back-off (e.g. a
+ *  rate-limit wait) never swallows Ctrl+C. Resolves either way. */
+export declare const waitMs: (ms: number, signal: AbortSignal) => Promise<void>;
+/** Resolve `p`, but reject with a TimeoutError if it takes longer than `ms`.
+ *  The underlying promise is left to settle on its own (we just stop waiting),
+ *  so a slow black-box call can never wedge the caller. */
+export declare function withTimeout<T>(p: Promise<T>, ms: number, label?: string): Promise<T>;
+/** An HTTP error the caller tagged as transient. `.fatal` skips all retries;
+ *  `.throttle` (a 429/503 rate-limit or overload) is retried indefinitely and
+ *  does NOT consume the bounded attempt budget — the server told us to wait, not
+ *  to give up. `.retryAfterMs` carries a server-suggested delay when present. */
+export type HttpError = Error & {
+    fatal?: boolean;
+    throttle?: boolean;
+    retryAfterMs?: number;
+};
+/** What every network call in this trainer needs from its caller: how to be
+ *  cancelled, and (optionally) where to report a rate-limit wait. */
+export interface HttpOptions {
+    signal: AbortSignal;
+    /** Called after each throttle wait, so a 429 back-off reads as "waiting"
+     *  rather than a silent hang. Omitted ⇒ the wait is silent. */
+    onThrottle?: (waitMsAmount: number, label: string) => void;
+}
+/** Wrap a throttle notice so a STORM of 429s logs at most one notice every
+ *  `minGapMs`. Without the gap a busy server produces a wall of identical
+ *  "waiting…" lines that pushes the real log out of the scrollback. */
+export declare function throttleNotifier(fn: (waitMsAmount: number, label: string) => void, minGapMs?: number): (waitMsAmount: number, label: string) => void;
+/** Retry `fn` with exponential backoff.
+ *
+ *  Three error classes:
+ *   • `.fatal` / AbortError  → rethrown immediately (never retried).
+ *   • `.throttle` (429/503)  → the server is rate-limiting/overloaded. We are
+ *     NOT failing — we WAIT (honouring Retry-After, else capped exponential
+ *     back-off with jitter) and retry WITHOUT consuming an attempt, so a
+ *     throttled request holds on until it succeeds rather than being dropped.
+ *     Only a shutdown breaks this loop.
+ *   • anything else          → a genuine transient error, retried up to `tries`
+ *     with exponential back-off before giving up.
+ *
+ *  `onFail` is called after each non-throttle failed attempt; `onThrottle` after
+ *  each throttle wait (for a "waiting…" notice). */
+export declare function retry<T>(label: string, fn: () => Promise<T>, tries: number, opts: HttpOptions & {
+    onFail?: (attempt: number, err: Error) => void;
+}): Promise<T>;
+/** Classify a non-OK HTTP response into an {@link HttpError} for {@link retry}:
+ *   • 429 / 503  → THROTTLE (rate-limited / overloaded): retried indefinitely,
+ *     honouring a Retry-After header (seconds or an HTTP-date) when present.
+ *   • other 5xx  → transient: retried up to the caller's attempt budget.
+ *   • other 4xx  → FATAL: a real client error (404, 401, …) — not retried.
+ *  Never throttles forever silently: the wait is interruptible by shutdown. */
+export declare function httpError(res: Response): HttpError;
+/** GET a URL and parse JSON, with the shared retry policy: rate-limits (429/503)
+ *  WAIT indefinitely (surfaced through `opts.onThrottle`), other 4xx is fatal,
+ *  other 5xx retried up to DOWNLOAD_TRIES. Used by every dataset LISTING call so
+ *  all share the same never-drop-on-throttle behaviour. */
+export declare function getJson(url: string, label: string, opts: HttpOptions): Promise<any>;
+/** The `rel="next"` URL of an RFC 5988 Link header, or null. */
+export declare function nextLink(header: string | null): string | null;
+/** GET a paginated JSON ARRAY, following `Link: rel="next"` to the end.
+ *
+ *  A LISTING THAT STOPS EARLY IS INVISIBLE, and that is why this exists.
+ *  Hugging Face caps a tree listing at 1,000 entries and hands back a next
+ *  link (verified: allenai/c4 returns exactly 1,000 plus a link). A caller that
+ *  ignores it gets a work-list silently missing everything past the first page,
+ *  trains it, marks those units complete, and thereafter reports the corpus
+ *  "already trained". No error at any point. Following the links is the only
+ *  way the work-list can be trusted to be the whole work-list.
+ *
+ *  `maxPages` is a runaway guard, not a limit anyone should hit; exceeding it
+ *  throws rather than returning a partial list, for exactly the reason above. */
+export declare function getJsonPaged(url: string, label: string, opts: HttpOptions, maxPages?: number): Promise<unknown[]>;
+/** Advertised transfer size of `url`, used only to reserve cache room. Like any
+ *  `content-length` this is the ON-THE-WIRE size, so for a content-coded source
+ *  (GitHub raw gzips JSON ~14x) it UNDER-estimates the file that lands on disk.
+ *  That is tolerable here because the cache ceiling is a budget, not a
+ *  correctness property — a run may overshoot MAX_CACHE_GB by the compression
+ *  ratio of one in-flight file, and each file is deleted as soon as it is
+ *  consumed. It must NOT be reused as an integrity check; see downloadFile.
+ *
+ *  Rate-limits wait; other 4xx is fatal; total failure → the caller's catch. */
+export declare function headSize(url: string, opts: HttpOptions): Promise<number>;

package/dist/example/train_base/http.js

@@ -0,0 +1,219 @@
+// train_base/http.ts — the shared network policy: wait out throttling, retry
+// what is transient, give up at once on what is not.
+//
+// A leaf module by construction: everything it needs to cancel (the run's
+// AbortSignal) and everything it needs to report (a throttle notice) arrives
+// as a parameter. There is no module-level shutdown handle and no module-level
+// log hook — those were globals precisely because this code used to live in the
+// same file as the run that owned them.
+import { DOWNLOAD_TRIES } from "./config.js";
+/** Sleep `ms`, but wake early if `signal` fires — so a long back-off (e.g. a
+ *  rate-limit wait) never swallows Ctrl+C. Resolves either way. */
+export const waitMs = (ms, signal) => new Promise((resolve) => {
+    if (signal.aborted)
+        return resolve();
+    // NOTE: the timer is deliberately NOT unref'd — an unref'd timer does not
+    // keep the event loop alive, so a pending wait (e.g. the pace between page
+    // requests, or a rate-limit back-off) would let Node exit early and the run
+    // would "do nothing and close". The listener lets a shutdown wake it early.
+    const t = setTimeout(done, ms);
+    function done() {
+        clearTimeout(t);
+        signal.removeEventListener("abort", done);
+        resolve();
+    }
+    signal.addEventListener("abort", done, { once: true });
+});
+/** Resolve `p`, but reject with a TimeoutError if it takes longer than `ms`.
+ *  The underlying promise is left to settle on its own (we just stop waiting),
+ *  so a slow black-box call can never wedge the caller. */
+export function withTimeout(p, ms, label = "operation") {
+    return new Promise((resolve, reject) => {
+        const t = setTimeout(() => {
+            const e = new Error(`${label} timed out after ${ms}ms`);
+            e.name = "TimeoutError";
+            reject(e);
+        }, ms);
+        if (typeof t.unref === "function")
+            t.unref();
+        p.then((v) => {
+            clearTimeout(t);
+            resolve(v);
+        }, (e) => {
+            clearTimeout(t);
+            reject(e);
+        });
+    });
+}
+/** Wrap a throttle notice so a STORM of 429s logs at most one notice every
+ *  `minGapMs`. Without the gap a busy server produces a wall of identical
+ *  "waiting…" lines that pushes the real log out of the scrollback. */
+export function throttleNotifier(fn, minGapMs = 3000) {
+    let last = 0;
+    return (ms, label) => {
+        const now = Date.now();
+        if (now - last <= minGapMs)
+            return;
+        last = now;
+        fn(ms, label);
+    };
+}
+/** Retry `fn` with exponential backoff.
+ *
+ *  Three error classes:
+ *   • `.fatal` / AbortError  → rethrown immediately (never retried).
+ *   • `.throttle` (429/503)  → the server is rate-limiting/overloaded. We are
+ *     NOT failing — we WAIT (honouring Retry-After, else capped exponential
+ *     back-off with jitter) and retry WITHOUT consuming an attempt, so a
+ *     throttled request holds on until it succeeds rather than being dropped.
+ *     Only a shutdown breaks this loop.
+ *   • anything else          → a genuine transient error, retried up to `tries`
+ *     with exponential back-off before giving up.
+ *
+ *  `onFail` is called after each non-throttle failed attempt; `onThrottle` after
+ *  each throttle wait (for a "waiting…" notice). */
+export async function retry(label, fn, tries, opts) {
+    const { signal, onFail, onThrottle } = opts;
+    let wait = 1000, last = "", throttleWait = 1000;
+    for (let attempt = 1; attempt <= tries;) {
+        if (signal.aborted) {
+            const e = new Error("aborted");
+            e.fatal = true;
+            throw e;
+        }
+        try {
+            return await fn();
+        }
+        catch (e) {
+            const err = e;
+            if (err.name === "AbortError" || err.fatal)
+                throw err;
+            // Rate-limited / overloaded: wait it out. Does NOT advance `attempt`, so a
+            // busy server can never exhaust the retry budget and drop the request.
+            if (err.throttle && !signal.aborted) {
+                // Honour Retry-After when the server sent one; else exponential back-off
+                // with jitter, capped, so a fleet of requests does not resynchronise.
+                const base = err.retryAfterMs && err.retryAfterMs > 0
+                    ? err.retryAfterMs
+                    : throttleWait;
+                const ms = Math.min(base, 60_000) +
+                    Math.floor(base * 0.25 * Math.random());
+                onThrottle?.(ms, label);
+                await waitMs(ms, signal);
+                throttleWait = Math.min(throttleWait * 2, 60_000);
+                continue;
+            }
+            last = err.message;
+            onFail?.(attempt, err);
+            attempt++;
+            if (attempt <= tries) {
+                await waitMs(wait, signal);
+                wait = Math.min(wait * 2, 30_000);
+            }
+        }
+    }
+    throw new Error(`${label} failed after ${tries} attempts: ${last}`);
+}
+/** Classify a non-OK HTTP response into an {@link HttpError} for {@link retry}:
+ *   • 429 / 503  → THROTTLE (rate-limited / overloaded): retried indefinitely,
+ *     honouring a Retry-After header (seconds or an HTTP-date) when present.
+ *   • other 5xx  → transient: retried up to the caller's attempt budget.
+ *   • other 4xx  → FATAL: a real client error (404, 401, …) — not retried.
+ *  Never throttles forever silently: the wait is interruptible by shutdown. */
+export function httpError(res) {
+    const err = new Error(`HTTP ${res.status}`);
+    if (res.status === 429 || res.status === 503) {
+        err.throttle = true;
+        const ra = res.headers.get("retry-after");
+        if (ra) {
+            const secs = Number(ra);
+            if (Number.isFinite(secs))
+                err.retryAfterMs = Math.max(0, secs * 1000);
+            else {
+                const when = Date.parse(ra);
+                if (Number.isFinite(when)) {
+                    err.retryAfterMs = Math.max(0, when - Date.now());
+                }
+            }
+        }
+    }
+    else if (res.status < 500) {
+        err.fatal = true; // genuine client error — do not retry
+    } // other 5xx: neither fatal nor throttle → ordinary bounded retry
+    return err;
+}
+/** GET a URL and parse JSON, with the shared retry policy: rate-limits (429/503)
+ *  WAIT indefinitely (surfaced through `opts.onThrottle`), other 4xx is fatal,
+ *  other 5xx retried up to DOWNLOAD_TRIES. Used by every dataset LISTING call so
+ *  all share the same never-drop-on-throttle behaviour. */
+export async function getJson(url, label, opts) {
+    return retry(label, async () => {
+        const res = await fetch(url, { signal: opts.signal });
+        if (res.ok)
+            return res.json();
+        throw httpError(res);
+    }, DOWNLOAD_TRIES, opts);
+}
+/** The `rel="next"` URL of an RFC 5988 Link header, or null. */
+export function nextLink(header) {
+    if (!header)
+        return null;
+    for (const part of header.split(",")) {
+        const m = part.match(/<([^>]+)>\s*;\s*rel\s*=\s*"?next"?/i);
+        if (m)
+            return m[1];
+    }
+    return null;
+}
+/** GET a paginated JSON ARRAY, following `Link: rel="next"` to the end.
+ *
+ *  A LISTING THAT STOPS EARLY IS INVISIBLE, and that is why this exists.
+ *  Hugging Face caps a tree listing at 1,000 entries and hands back a next
+ *  link (verified: allenai/c4 returns exactly 1,000 plus a link). A caller that
+ *  ignores it gets a work-list silently missing everything past the first page,
+ *  trains it, marks those units complete, and thereafter reports the corpus
+ *  "already trained". No error at any point. Following the links is the only
+ *  way the work-list can be trusted to be the whole work-list.
+ *
+ *  `maxPages` is a runaway guard, not a limit anyone should hit; exceeding it
+ *  throws rather than returning a partial list, for exactly the reason above. */
+export async function getJsonPaged(url, label, opts, maxPages = 500) {
+    const out = [];
+    let next = url;
+    let pages = 0;
+    while (next !== null) {
+        const at = next;
+        const { body, link } = await retry(label, async () => {
+            const res = await fetch(at, { signal: opts.signal });
+            if (!res.ok)
+                throw httpError(res);
+            return { body: await res.json(), link: res.headers.get("link") };
+        }, DOWNLOAD_TRIES, opts);
+        if (!Array.isArray(body))
+            break; // not a listing — nothing to page through
+        out.push(...body);
+        next = nextLink(link);
+        if (++pages >= maxPages && next) {
+            throw new Error(`${label}: more than ${maxPages} pages of listing — refusing to ` +
+                `continue with a work-list that may be incomplete`);
+        }
+    }
+    return out;
+}
+/** Advertised transfer size of `url`, used only to reserve cache room. Like any
+ *  `content-length` this is the ON-THE-WIRE size, so for a content-coded source
+ *  (GitHub raw gzips JSON ~14x) it UNDER-estimates the file that lands on disk.
+ *  That is tolerable here because the cache ceiling is a budget, not a
+ *  correctness property — a run may overshoot MAX_CACHE_GB by the compression
+ *  ratio of one in-flight file, and each file is deleted as soon as it is
+ *  consumed. It must NOT be reused as an integrity check; see downloadFile.
+ *
+ *  Rate-limits wait; other 4xx is fatal; total failure → the caller's catch. */
+export async function headSize(url, opts) {
+    return retry(`HEAD ${url}`, async () => {
+        const res = await fetch(url, { method: "HEAD", signal: opts.signal });
+        if (res.ok)
+            return Number(res.headers.get("content-length")) || 0;
+        throw httpError(res);
+    }, 4, opts);
+}

package/dist/example/train_base/items.d.ts

@@ -0,0 +1,46 @@
+export interface Episode {
+    context: string;
+    continuation: string;
+}
+export type TrainingItem = string | Episode;
+export declare const isEpisode: (it: TrainingItem) => it is Episode;
+/** One turn of a dialogue, attributed to a speaker. The speaker is only ever
+ *  used to decide MERGING (see `mergeSpeakerTurns`); it is never deposited. */
+export interface SpeakerTurn {
+    speaker: string;
+    text: string;
+}
+/** Build the accumulated-context episodes of a turn sequence: each successive
+ *  turn is the continuation of ALL the turns before it joined together. This is
+ *  the same cumulative-context shape a multi-turn conversation deposits, so the
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
+export declare function accumulate(turns: string[]): Episode[];
+/** Collapse consecutive same-speaker turns into one, joining with a space, and
+ *  return the bare texts in order. A turn with no speaker never merges with its
+ *  neighbour: an unlabelled row is of unknown origin, and joining two of them
+ *  would invent a contribution that may span two speakers.
+ *
+ *  Load-bearing for corpora that split one contribution across several indexed
+ *  utterances (an artifact of the collection UI). Left unmerged, the cumulative
+ *  walk deposits a turn boundary in the middle of one speaker's contribution
+ *  and teaches it as a hand-off. Measured share of turns absorbed by merging:
+ *  TM-1 17.7%, TM-2 11.9%, TM-3 0.8%, TM-4 0.0%. */
+export declare function mergeSpeakerTurns(turns: SpeakerTurn[]): string[];
+/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
+ *  repeats so a concept never deposits the same form twice. */
+export declare function refineItems(items: TrainingItem[]): TrainingItem[];
+/** Content size of a training item in UTF-8 bytes — the same quantity the
+ *  scaling suite (14-scaling.test.mjs) measures as KB/s: for an episode the
+ *  context plus the continuation, for a bare experience its own text. */
+export declare const itemBytes: (it: TrainingItem) => number;

package/dist/example/train_base/items.js

@@ -0,0 +1,98 @@
+// train_base/items.ts — the REPRESENTATION core: what a training item is, and
+// the three shapes a corpus datum may take.
+//
+// REPRESENTATION POLICY (one datum → one form; no replication):
+//   • FACTS are the default. A datum that is a RELATION (translation pair,
+//     question → answer) is emitted as a (context → continuation) edge SEMA
+//     points at and, by example across the corpus, generalizes from (cf.
+//     example/demo.ts).
+//   • EXPERIENCES (bare statements) are used only when a fact is NOT possible —
+//     content with no natural relational split.
+//   • CUMULATIVE CONTINUOUS CONTEXT is used only when truly necessary — genuine
+//     MULTI-TURN dialogue, where a turn follows from the whole conversation so
+//     far. The fact stages do NOT synthesize a multi-turn walk, which would just
+//     replicate the facts (repetition SEMA avoids).
+//
+// Nothing here reads the environment or touches I/O: these are the pure
+// functions every corpus adapter is built out of.
+export const isEpisode = (it) => typeof it !== "string";
+/** Build the accumulated-context episodes of a turn sequence: each successive
+ *  turn is the continuation of ALL the turns before it joined together. This is
+ *  the same cumulative-context shape a multi-turn conversation deposits, so the
+ *  store learns to continue a growing context.
+ *
+ *  The "\n" below is a CORPUS choice, not a protocol.  oasst2 turns are
+ *  paragraphs, and reading them back with the newlines kept is how this corpus
+ *  reads naturally; a different corpus may join with nothing, and
+ *  test/13-conversation.test.mjs does exactly that.  Neither has to match the
+ *  other, because Sema never scans content for turn boundaries — those are
+ *  offsets the Conversation API carries beside the bytes (see Mind.addTurn's
+ *  "ON SEPARATORS" note).  The newline here is simply part of the text this
+ *  store learnt, so anything replaying this corpus feeds it back as part of
+ *  the turn: `addTurn(conv, "\n" + turnText)`.  It is not a convention the
+ *  engine, the API, or the tests have to agree on. */
+export function accumulate(turns) {
+    const out = [];
+    for (let i = 1; i < turns.length; i++) {
+        out.push({ context: turns.slice(0, i).join("\n"), continuation: turns[i] });
+    }
+    return out;
+}
+/** Collapse consecutive same-speaker turns into one, joining with a space, and
+ *  return the bare texts in order. A turn with no speaker never merges with its
+ *  neighbour: an unlabelled row is of unknown origin, and joining two of them
+ *  would invent a contribution that may span two speakers.
+ *
+ *  Load-bearing for corpora that split one contribution across several indexed
+ *  utterances (an artifact of the collection UI). Left unmerged, the cumulative
+ *  walk deposits a turn boundary in the middle of one speaker's contribution
+ *  and teaches it as a hand-off. Measured share of turns absorbed by merging:
+ *  TM-1 17.7%, TM-2 11.9%, TM-3 0.8%, TM-4 0.0%. */
+export function mergeSpeakerTurns(turns) {
+    const out = [];
+    let prev = "";
+    for (const t of turns) {
+        if (out.length > 0 && t.speaker !== "" && t.speaker === prev) {
+            out[out.length - 1] += " " + t.text;
+        }
+        else {
+            out.push(t.text);
+        }
+        prev = t.speaker;
+    }
+    return out;
+}
+/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
+ *  repeats so a concept never deposits the same form twice. */
+export function refineItems(items) {
+    const out = [];
+    const seen = new Set();
+    for (const it of items) {
+        if (!isEpisode(it)) {
+            const exp = it.trim();
+            const key = "E:" + exp;
+            if (exp && !seen.has(key)) {
+                seen.add(key);
+                out.push(exp);
+            }
+            continue;
+        }
+        const ctx = it.context.trim();
+        const cont = it.continuation.trim();
+        if (!ctx || !cont || ctx === cont)
+            continue;
+        const key = "P:" + ctx + "\u0000" + cont;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push({ context: ctx, continuation: cont });
+    }
+    return out;
+}
+const ENC = new TextEncoder();
+/** Content size of a training item in UTF-8 bytes — the same quantity the
+ *  scaling suite (14-scaling.test.mjs) measures as KB/s: for an episode the
+ *  context plus the continuation, for a bare experience its own text. */
+export const itemBytes = (it) => isEpisode(it)
+    ? ENC.encode(it.context).length + ENC.encode(it.continuation).length
+    : ENC.encode(it).length;

package/dist/example/train_base/main.d.ts

@@ -0,0 +1,4 @@
+export * from "./items.js";
+export * from "./corpora/index.js";
+export { parquetBatchRows } from "./readers.js";
+export { mergeSpeakerTurns as mergeTaskmasterTurns } from "./items.js";