@hviana/sema 0.5.9 → 0.7.1

package/.github/workflows/release.yml

@@ -0,0 +1,80 @@
+name: release
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      version:
+        description: version to release, e.g. 0.6.1
+        type: string
+        required: true
+
+permissions:
+  # Trusted publishing (OIDC): GitHub mints a short-lived identity token that
+  # npm and JSR both exchange against the trusted publisher on the package.
+  # No registry token is stored anywhere.
+  id-token: write
+  # The release path bumps package.json + jsr.json, commits, and pushes the tag.
+  contents: write
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: bump version and tag
+        if: github.event_name == 'workflow_dispatch'
+        env:
+          VERSION: ${{ inputs.version }}
+        run: |
+          git config user.name  "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          current="$(node -p "require('./package.json').version")"
+          if [ "$current" != "$VERSION" ]; then
+            npm version "$VERSION" --no-git-tag-version
+            node -e 'const fs=require("fs");const p=JSON.parse(fs.readFileSync("jsr.json","utf8"));p.version=process.env.VERSION;fs.writeFileSync("jsr.json",JSON.stringify(p,null,2)+"\n")'
+            git add package.json jsr.json
+            git commit -m "$VERSION"
+            git tag -a "v$VERSION" -m "v$VERSION"
+            git push origin HEAD --follow-tags
+          fi
+
+      # The two manifests must not drift: a dual npm+JSR publish ships the SAME
+      # version to both, so a mismatch here would tag one and ship another.
+      - name: confirm package.json and jsr.json agree on version
+        run: |
+          pkg="$(node -p "require('./package.json').version")"
+          jsr="$(node -p "require('./jsr.json').version")"
+          if [ "$pkg" != "$jsr" ]; then
+            echo "package.json version $pkg != jsr.json version $jsr" >&2
+            exit 1
+          fi
+          echo "publishing version $pkg"
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "24"
+          registry-url: https://registry.npmjs.org
+
+      - name: install
+        run: npm install
+
+      # Trusted publishing (OIDC) needs npm >= 11.5.1; Node 24 bundles an older npm.
+      - name: update npm for trusted publishing
+        run: npm install -g npm@latest
+
+      # The same run publishes: a tag pushed with GITHUB_TOKEN does NOT
+      # re-trigger workflows, so the publish cannot depend on a second run.
+      # prepublishOnly (npm test) verifies and builds dist/ before packing.
+      # dist/ is gitignored (so it stays off GitHub and JSR) but ships to npm
+      # because .npmignore deliberately omits the `dist/` entry.
+      - name: publish to npm
+        run: npm publish --access public
+
+      # JSR publishes the SOURCE (jsr.json exports → src/index.ts), not dist/.
+      # dist/ is gitignored and JSR honours .gitignore, so it is excluded here
+      # with no --allow-dirty needed (the tree is clean after npm's build).
+      - name: publish to jsr
+        run: npx jsr publish

package/AGENTS.md

@@ -455,8 +455,7 @@ Asking never writes, which is the only reason per-response memos are sound.
 eager fields (recognition, computed spans, guide, the evidence-breadth constant
 `k`) plus **lazily-cached methods** for expensive analyses (`attention()` — the
 consensus climb, `weave()`, `resonance()` — the response's ONE top-k
-content-index read, `wideResonance()` — the one WIDE candidate list every
-past-the-top-k mechanism reads, `frames()` — the frame/slot inventory,
+content-index read, `frames()` — the frame/slot inventory,
 `spanShapedOf`/`spanShapedAll`, `queryWindows`, `queryResolved`, `windowsOf`,
 `reachMemo`) — each computed at most once, shared by mechanisms and
 post-grounding stages, and never computed if nobody asks. The async ones are
@@ -464,10 +463,13 @@ cached **by promise**, so a second caller awaits the first computation rather
 than starting another.
 
 Mind-level memos (`climbMemo`, `recogniseMemo`, `perceiveMemo`, `canonMemo`,
-`_resolvedSubtrees`, `_edgeChoice`, `_gistCache`) are created in
-`beginResponse()` and torn down in `endResponse()` — a new memo must be added to
-both. A conversation supplies its own maps for the first four, so they persist
-across turns.
+`_resolvedSubtrees`) are assigned in `beginResponse()` and nulled in
+`endResponse()` — a new per-response memo must be added to both. A conversation
+supplies its own maps for the first four, so they persist across turns.
+`_edgeChoice` is a Mind field CLEARED in `endResponse()` (not re-created).
+`_gistCache` is a SESSION-lifetime Mind field (32 MB, `mind.ts`) never touched
+by begin/endResponse: a node's bytes are immutable and perception is pure, so a
+cached gist is valid for the store's lifetime.
 
 **Which memos a trace bypasses, and why the answer is "almost none".** Only
 `_edgeChoice` (via `guidedNext`) and `sharedReachMemo` are trace-bypassed —
@@ -546,9 +548,9 @@ answer was chosen, the meter says what it cost. Four contracts:
    so two runs are diffable and a work regression is visible without a
    stopwatch. Only `elapsedMs` and the phase millisecond totals are not.
 3. **Phases nest, and carry their own counter deltas.** `think` ⊃ `<mech>.run` ⊃
-   `substitutionBridge` ⊃ `recall.exhaustiveResonate`. Inclusive, never summed —
-   but each phase reports the work done inside it (`PhaseCost.counters`), which
-   is what makes "which phase did those byte reads?" answerable at all.
+   `substitutionBridge`. Inclusive, never summed — but each phase reports the
+   work done inside it (`PhaseCost.counters`), which is what makes "which phase
+   did those byte reads?" answerable at all.
 4. **Count a logical operation once.** A recursive read (`bytesPrefix`
    descending a branch) is charged at the public entry point only — the private
    `_prefix` body is uncharged. Counting the recursion made one read of an
@@ -608,6 +610,48 @@ corruption via phrase-interior chunks) and the `couldGrow` liveness rule (O(N²)
 chart growth). When you fix a subtle bug, leave the constraint behind, not the
 story of the fix.
 
+### 2.17 Saturation — every walk decides, none drifts
+
+Saturation is a first-class control, not a secondary nicety — and it is TWO
+things, distinguished deliberately:
+
+- a CAP — a derived bound (√N per read, √N·W per walk) that exists only to stop
+  magic constants (§2.2) — is a SAFETY NET. It bounds the walk when its question
+  never decides (a side too common to ever settle). Derived, never tuned; but it
+  is not itself a decision.
+- a REAL saturation — a named, derived stop that DECIDES the walk's question and
+  terminates the moment it is decided. The cap remains as the backstop; the
+  saturation ends the walk. A walk with only a cap drifts to the cap every time;
+  a walk with a real saturation stops where the answer is already known.
+
+`edgeAncestors` is the model (EXPAND-UNTIL-DECIDED): a reach is consumed either
+as a VOTE (needs `contextsReached` exactly, only while ≤ √N) or as an ABSTENTION
+(`saturated`), so it stops at the FIRST of its five named stops and no consumer
+reads a saturated reach's roots or counts. `pivotInto`'s candidate scan is the
+second: "longest valid wins" is DECIDED at the first valid candidate in
+descending length, so it reads one winner's bytes, never every shorter
+candidate. Saturation is a DECISION about the answer — never a cache, never a
+budget.
+
+The junction walk's per-node hub guards are real per-node saturations; its
+`√N·W` budget is the NET, not a saturation. REFUTED (test/16 bridge synthesis,
+test/34 n-ary binding): a "stop once one side's cone is exhausted" early stop is
+WRONG, in both a hub-guarded and a hub-flagged form. The junction test is a BYTE
+containment over the UNION of the two cones, and a junction can be structurally
+reachable from only ONE side — the side whose seed is a FOLD sub-node of the
+container. test/16: "cold or hot" is reached from the window "cold", but the
+3-byte answer "hot" is not a 4-byte window of it, so "hot"'s cone empties after
+one pop while the junction still lies ahead in "cold"'s cone. "One cone
+exhausted" therefore never proves "no junction left", and the budget stays the
+net that backs the per-node saturations.
+
+_Follow it:_ when a new walk measures commonality against the corpus, name its
+saturation condition — the answer it may stop producing — beside its read cap.
+No code may traverse the inference uncontrolled from the corpus, nor lack the
+saturation its question admits. A cap without a saturation is a drift, and a
+drifting walk is a bug, not a tuning choice; do not mask it with a cache (§2.12)
+— a cache hides a drift on a warm store, saturation removes it.
+
 ---
 
 ## 3. Where things live
@@ -768,10 +812,12 @@ memoises perceive+intern of repeated inputs and routes through the same
 `store.commit()` at checkpoints; run `compactContentIndex` /
 `repairContentIndex` post-training if eviction was heavy, and
 `mind.buildCanonIndex()` if queries will carry a canonicalizer (2.9). See
-`example/train_base.ts`. Profiling note: the first `resonate` after a big ingest
-pays the pending index flush; the dominant query-side ANN cost is connector
-pre-resolution (bounded by recognised-site count — don't add another loop over
-site pairs).
+`example/train_base/` — a folder, entry `main.ts`: the run context lives in
+`runtime.ts`, the one per-corpus loop in `stage.ts`, and each corpus (knobs, row
+adapter, stage descriptor) in `corpora/<name>.ts`. Profiling note: the first
+`resonate` after a big ingest pays the pending index flush; the dominant
+query-side ANN cost is connector pre-resolution (bounded by recognised-site
+count — don't add another loop over site pairs).
 
 ---
 
@@ -811,6 +857,20 @@ vendor code under licenses incompatible with dual distribution, and do not add
 runtime dependencies casually — the near-zero-dependency footprint is a product
 feature.
 
+**The library has NO runtime dependencies at all**, and that is now pinned by
+`test/88-dependency-footprint.test.mjs`: the built `dist/src` may import only
+`node:` builtins and relative paths, `package.json` may declare no
+`dependencies`, and the published entry points may not reach outside `dist/src`.
+The rule an EXAMPLE follows is different and looser — it may use what it needs,
+as a **dev** dependency, loaded **lazily** so it is a requirement only of the
+code path that uses it. `example/train_base` is the reference: `hyparquet` (+
+its Snappy codec) is the sole third-party package in this repository, it is
+dev-only, and `readers.ts` resolves it by dynamic import the first time a
+Parquet corpus is actually read — so a curriculum with no Parquet stage runs
+with the package absent. It used to sit in `dependencies`, which installed a
+Parquet reader on every consumer of Sema for the sake of one example; that is
+the mistake the suite exists to catch.
+
 **Training corpora are governed by the same rule, and more strictly.** Sema is
 non-parametric: a trained store retains its training text VERBATIM (read any
 content node back and the original sentence comes out). A store is therefore a

package/DATASETS.md

@@ -77,17 +77,17 @@ changes to be stated. The modification statement above satisfies the latter.
 | [Taskmaster-1/2/3/4](https://github.com/google-research-datasets/Taskmaster)                             | CC BY 4.0                                             | Google LLC                         | Task-oriented dialogue. Only `utterances[].text` is ingested; the `instructions` / `scenario` / `vertical` fields are never read                                                                                                                                      |
 | [2WikiMultihopQA](https://huggingface.co/datasets/xanhho/2WikiMultihopQA) — **`evidences` triples only** | Apache-2.0 (repo); triples originate in Wikidata, CC0 | Ho et al.; Wikidata contributors   | Only the `evidences` column is ingested. The `context` column (Wikipedia prose, CC BY-SA) is **never read** — see §4. The `question`/`answer` columns are also never deposited, for a capability reason rather than a licence one: they memorise instead of composing |
 | [allenai/soda](https://huggingface.co/datasets/allenai/soda)                                             | CC BY 4.0                                             | Allen Institute for AI             | Social dialogue. Only the `dialogue` column is ingested; `narrative` / `literal` / `head` / `relation` / `tail` are never read. Model-generated provenance — see §5                                                                                                   |
-| [AmazonScience/massive](https://huggingface.co/datasets/AmazonScience/massive)                           | CC BY 4.0                                             | Amazon Science                     | Short multilingual intents. Only the `utt` column is ingested; the slot-annotated `annot_utt` is never read. **Disabled by default** on capability grounds (not licence) — see `MASSIVE` in `example/train_base.ts`                                                   |
+| [AmazonScience/massive](https://huggingface.co/datasets/AmazonScience/massive)                           | CC BY 4.0                                             | Amazon Science                     | Short multilingual intents. Only the `utt` column is ingested; the slot-annotated `annot_utt` is never read. **Disabled by default** on capability grounds (not licence) — see `MASSIVE` in `example/train_base/corpora/massive.ts`                                   |
 
 ### 3.2 Excluded, and why
 
-| Corpus                                                            | Reason                                                                                                                                                                                                                                                                                                                                          |
-| :---------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **MuskumPillerum/General-Knowledge**                              | **No licence at all.** The HF repo carries no licence tag and no licence in its card; an earlier header in `example/train_base.ts` claimed MIT without support. Its own card states it "contains a subset of the alpaca dataset", and Alpaca is CC BY-NC 4.0 — **NonCommercial**, incompatible with Sema's commercial licence. See §6.          |
-| **PAWS**                                                          | Google's own grant is maximally permissive ("may be freely used for any purpose"), but PAWS-Wiki sentences derive from Wikipedia (CC BY-**SA**) and PAWS-QQP from Quora question pairs under Quora's terms. Because Sema retains text verbatim, the upstream terms would attach to the distributed store. Excluded despite strong measured fit. |
-| **Schema-Guided Dialogue (SGD/dstc8)**, **HotpotQA**, **MuSiQue** | CC BY-SA 4.0 — ShareAlike conflicts with dual distribution.                                                                                                                                                                                                                                                                                     |
-| **2WikiMultihopQA passages**                                      | Wikipedia prose, CC BY-SA. The repo's Apache-2.0 tag does not relicense the text it was built from. Only the Wikidata-derived `evidences` triples are ingested.                                                                                                                                                                                 |
-| **Alpaca** and derivatives                                        | CC BY-NC 4.0, and generated from OpenAI model outputs.                                                                                                                                                                                                                                                                                          |
+| Corpus                                                            | Reason                                                                                                                                                                                                                                                                                                                                                            |
+| :---------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **MuskumPillerum/General-Knowledge**                              | **No licence at all.** The HF repo carries no licence tag and no licence in its card; an earlier header in `example/train_base.ts` (now `corpora/genknow.ts`) claimed MIT without support. Its own card states it "contains a subset of the alpaca dataset", and Alpaca is CC BY-NC 4.0 — **NonCommercial**, incompatible with Sema's commercial licence. See §6. |
+| **PAWS**                                                          | Google's own grant is maximally permissive ("may be freely used for any purpose"), but PAWS-Wiki sentences derive from Wikipedia (CC BY-**SA**) and PAWS-QQP from Quora question pairs under Quora's terms. Because Sema retains text verbatim, the upstream terms would attach to the distributed store. Excluded despite strong measured fit.                   |
+| **Schema-Guided Dialogue (SGD/dstc8)**, **HotpotQA**, **MuSiQue** | CC BY-SA 4.0 — ShareAlike conflicts with dual distribution.                                                                                                                                                                                                                                                                                                       |
+| **2WikiMultihopQA passages**                                      | Wikipedia prose, CC BY-SA. The repo's Apache-2.0 tag does not relicense the text it was built from. Only the Wikidata-derived `evidences` triples are ingested.                                                                                                                                                                                                   |
+| **Alpaca** and derivatives                                        | CC BY-NC 4.0, and generated from OpenAI model outputs.                                                                                                                                                                                                                                                                                                            |
 
 ---
 
@@ -154,6 +154,7 @@ Stores published before this file was written — including those under
 rows), whose licence status is described in §3.2. Those artifacts should be
 treated as **not redistributable** until retrained without that stage.
 
-The stage is now **disabled by default** in `example/train_base.ts`
-(`GENKNOW=0`). The adapter code remains so the stage can be re-enabled for local
-experiments; a store trained with `GENKNOW=1` must not be distributed.
+The stage is now **disabled by default** in
+`example/train_base/corpora/genknow.ts` (`GENKNOW=0`). The adapter code remains
+so the stage can be re-enabled for local experiments; a store trained with
+`GENKNOW=1` must not be distributed.

package/dist/example/train_base/cache.d.ts

@@ -0,0 +1,35 @@
+/** Delete every orphaned "<file>.part" in the cache, returning how many were
+ *  removed and the bytes they held.
+ *
+ *  A .part file at rest is by definition the debris of a download that never
+ *  finished — the rename that promotes one is the last step of `downloadFile`,
+ *  so a live .part exists only while THIS process is writing it. Sweeping at
+ *  startup is therefore safe, and it is load-bearing rather than cosmetic:
+ *  `cacheSize` deliberately counts .part files (an in-flight download really
+ *  does occupy the disk), so debris left by a killed run consumes ceiling
+ *  budget that nothing would ever free, and `ensureCacheRoom` would wait for
+ *  room that cannot appear.
+ *
+ *  The one assumption is that a cache directory belongs to ONE run at a time.
+ *  That was already true — two trainers sharing CACHE_DIR would write the same
+ *  .part path — so this adds no constraint that did not exist. */
+export declare function sweepPartials(): {
+    files: number;
+    bytes: number;
+};
+/** Total bytes currently held in the cache directory — INCLUDING any .part
+ *  file, because an in-flight download occupies the disk like any other file.
+ *  Orphaned ones are removed by {@link sweepPartials} at startup. */
+export declare function cacheSize(): number;
+/** Block until there is room for a file of `fileBytes` under the ceiling.
+ *  A single file larger than the whole ceiling can never "fit", so we let it
+ *  through (it is deleted right after processing) rather than wait forever. */
+export declare function ensureCacheRoom(fileBytes: number, signal: AbortSignal, warn?: (msg: string) => void, maxWaitMs?: number): Promise<void>;
+export interface DownloadOptions {
+    signal: AbortSignal;
+    tries: number;
+    onFail?: (attempt: number, err: Error) => void;
+    onProgress?: (done: number, total: number) => void;
+}
+/** Stream `url` to `destPath`, atomically and with backpressure. */
+export declare function downloadFile(url: string, destPath: string, opts: DownloadOptions): Promise<void>;

package/dist/example/train_base/cache.js

@@ -0,0 +1,211 @@
+// train_base/cache.ts — the durable disk cache and the download sink.
+//
+// This is the ONE place the trainer needs Node rather than the web platform:
+// every other byte in the pipeline moves through fetch, WHATWG streams,
+// DecompressionStream, TextDecoderStream and Blob, but writing a file is the
+// single capability the web platform does not expose. So the sink below wraps a
+// raw fs descriptor, and nothing else here does.
+//
+// Two invariants the rest of the trainer relies on:
+//   • ATOMIC — a download streams to "<file>.part", is fsync'd, then renamed
+//     into place. A file at its final path is, by construction, complete, so an
+//     interrupted download can never be mistaken for a cached one.
+//   • BOUNDED — a download blocks under the MAX_CACHE_GB ceiling, and a fully
+//     processed file is deleted by its caller immediately.
+import { CACHE_DIR, CACHE_WAIT_MS, MAX_CACHE_BYTES, PART_SUFFIX, } from "./config.js";
+import { httpError, retry, waitMs } from "./http.js";
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readdirSync, renameSync, statSync, unlinkSync, writeSync, } from "node:fs";
+import { basename, join } from "node:path";
+/** Delete every orphaned "<file>.part" in the cache, returning how many were
+ *  removed and the bytes they held.
+ *
+ *  A .part file at rest is by definition the debris of a download that never
+ *  finished — the rename that promotes one is the last step of `downloadFile`,
+ *  so a live .part exists only while THIS process is writing it. Sweeping at
+ *  startup is therefore safe, and it is load-bearing rather than cosmetic:
+ *  `cacheSize` deliberately counts .part files (an in-flight download really
+ *  does occupy the disk), so debris left by a killed run consumes ceiling
+ *  budget that nothing would ever free, and `ensureCacheRoom` would wait for
+ *  room that cannot appear.
+ *
+ *  The one assumption is that a cache directory belongs to ONE run at a time.
+ *  That was already true — two trainers sharing CACHE_DIR would write the same
+ *  .part path — so this adds no constraint that did not exist. */
+export function sweepPartials() {
+    const out = { files: 0, bytes: 0 };
+    if (!existsSync(CACHE_DIR))
+        return out;
+    for (const name of readdirSync(CACHE_DIR)) {
+        if (!name.endsWith(PART_SUFFIX))
+            continue;
+        const p = join(CACHE_DIR, name);
+        try {
+            const size = statSync(p).size;
+            unlinkSync(p);
+            out.files++;
+            out.bytes += size;
+        }
+        catch { /* raced with another delete — nothing to reclaim */ }
+    }
+    return out;
+}
+/** Total bytes currently held in the cache directory — INCLUDING any .part
+ *  file, because an in-flight download occupies the disk like any other file.
+ *  Orphaned ones are removed by {@link sweepPartials} at startup. */
+export function cacheSize() {
+    if (!existsSync(CACHE_DIR))
+        return 0;
+    let total = 0;
+    for (const name of readdirSync(CACHE_DIR)) {
+        try {
+            total += statSync(join(CACHE_DIR, name)).size;
+        }
+        catch { /* raced with a delete */ }
+    }
+    return total;
+}
+/** Block until there is room for a file of `fileBytes` under the ceiling.
+ *  A single file larger than the whole ceiling can never "fit", so we let it
+ *  through (it is deleted right after processing) rather than wait forever. */
+export async function ensureCacheRoom(fileBytes, signal, warn, maxWaitMs = CACHE_WAIT_MS) {
+    mkdirSync(CACHE_DIR, { recursive: true });
+    if (fileBytes >= MAX_CACHE_BYTES)
+        return;
+    let warned = false;
+    const until = Date.now() + maxWaitMs;
+    // Stop waiting the moment a shutdown is requested — the abort signal unblocks
+    // a long cache-full wait so Ctrl+C is never swallowed by the ceiling.
+    while (!signal.aborted && cacheSize() + fileBytes > MAX_CACHE_BYTES) {
+        // BOUNDED. Room appears when this run consumes and deletes a file, so a
+        // cache already over the ceiling with nothing left to consume — stale files
+        // from another run, a ceiling set below one corpus — would otherwise wait
+        // for room that cannot arrive, forever, after a single warning line.
+        if (Date.now() >= until) {
+            throw new Error(`cache still full after ${Math.round(maxWaitMs / 60_000)} min ` +
+                `(${(cacheSize() / 1e9).toFixed(1)} GB of a ` +
+                `${(MAX_CACHE_BYTES / 1e9).toFixed(0)} GB ceiling) — raise ` +
+                `MAX_CACHE_GB or clear ${CACHE_DIR}`);
+        }
+        if (!warned) {
+            warn?.(`cache at ${(MAX_CACHE_BYTES / 1e9).toFixed(0)} GB ceiling — waiting for room…`);
+            warned = true;
+        }
+        await waitMs(5_000, signal);
+    }
+}
+/** Stream `url` to `destPath`, atomically and with backpressure. */
+export async function downloadFile(url, destPath, opts) {
+    const partPath = destPath + PART_SUFFIX;
+    await retry(`download ${basename(destPath)}`, async () => {
+        // Abort promptly on shutdown rather than waiting out a slow socket.
+        if (opts.signal.aborted) {
+            const e = new Error("aborted");
+            e.fatal = true;
+            throw e;
+        }
+        const res = await fetch(url, { signal: opts.signal });
+        if (!res.ok)
+            throw httpError(res);
+        if (!res.body)
+            throw new Error("empty response body");
+        // `content-length` describes the bytes ON THE WIRE. When the server
+        // applied a content-coding, fetch hands us the DECODED body, so the
+        // header no longer describes what gets written to disk and the integrity
+        // guard below must not use it. Measured: raw.githubusercontent.com sends
+        // `content-encoding: gzip` with content-length 110,928 for a file that
+        // decodes to 1,607,931 bytes — a size check against that rejects every
+        // healthy download. (The bug stayed latent because Hugging Face sends
+        // `content-encoding: br` and NO content-length, leaving total = 0, which
+        // already disables the guard.)
+        const encoding = (res.headers.get("content-encoding") ?? "").trim()
+            .toLowerCase();
+        const decoded = encoding !== "" && encoding !== "identity";
+        const total = decoded
+            ? 0
+            : Number(res.headers.get("content-length")) || 0;
+        let done = 0;
+        // Stream straight to a ".part" sibling using pure WHATWG streams. A
+        // TransformStream meters progress; pipeTo into a WritableStream gives REAL
+        // backpressure natively — the sink's write() returns a promise the
+        // readable side awaits, so a fast server can never outrun the disk (no
+        // whole-file heap buffering). The sink wraps a single raw fs descriptor
+        // (the one capability the web platform lacks); writing to disk is the only
+        // Node operation in the whole pipeline. The final, valid file only ever
+        // appears via the atomic rename below, so a crash mid-transfer can never
+        // leave a truncated file at the real path.
+        const meter = new TransformStream({
+            transform(chunk, controller) {
+                done += chunk.length;
+                opts.onProgress?.(done, total);
+                controller.enqueue(chunk);
+            },
+        });
+        const fd = openSync(partPath, "w");
+        let closed = false;
+        const closeFd = () => {
+            if (closed)
+                return;
+            closed = true;
+            try {
+                closeSync(fd);
+            }
+            catch { /* already closed */ }
+        };
+        const sink = new WritableStream({
+            write(chunk) {
+                // writeSync drains the whole chunk before returning, so the readable
+                // side is paused for exactly as long as the disk needs — backpressure.
+                let off = 0;
+                while (off < chunk.length) {
+                    off += writeSync(fd, chunk, off, chunk.length - off);
+                }
+            },
+            close() {
+                fsyncSync(fd); // durable bytes before the rename promotes them
+                closeFd();
+            },
+            abort() {
+                closeFd();
+            },
+        });
+        try {
+            await res.body.pipeThrough(meter).pipeTo(sink, {
+                signal: opts.signal,
+            });
+        }
+        catch (e) {
+            // pipeTo's abort() ran the sink's abort() (closing the descriptor); if
+            // it didn't (a non-abort throw), make sure the descriptor is not leaked.
+            closeFd();
+            try {
+                unlinkSync(partPath);
+            }
+            catch { /* best effort */ }
+            throw e;
+        }
+        // Optional integrity guard: when the server advertised a size FOR THE
+        // BYTES WE WRITE (see the content-encoding note above — `total` is 0 for
+        // a decoded body, which disables this), a complete file must match it. A
+        // short read (silent truncation) is retried rather than promoted, so the
+        // parser never sees a partial file.
+        try {
+            const got = statSync(partPath).size;
+            if (total > 0 && got !== total) {
+                try {
+                    unlinkSync(partPath);
+                }
+                catch { /* best effort */ }
+                throw new Error(`size mismatch: got ${got}, expected ${total}`);
+            }
+        }
+        catch (e) {
+            if (e instanceof Error && e.message.startsWith("size mismatch")) {
+                throw e;
+            }
+            // statSync failure is non-fatal here; the rename below will surface it.
+        }
+        // Atomic publish: rename is atomic within a filesystem, so the final path
+        // flips from "absent" to "complete" in one step — never an in-between.
+        renameSync(partPath, destPath);
+    }, opts.tries, { signal: opts.signal, onFail: opts.onFail });
+}

package/dist/example/train_base/config.d.ts

@@ -0,0 +1,21 @@
+/** Read an environment variable, or `d` when it is unset. */
+export declare const env: (k: string, d: string) => string;
+export declare const DB_PATH: string;
+export declare const D: number;
+export declare const SEED: number;
+export declare const CHECKPOINT_BYTES: number;
+export declare const PARQUET_BATCH_BYTES: number;
+export declare const LOCAL_PATH: string;
+export declare const CACHE_DIR: string;
+export declare const MAX_CACHE_BYTES: number;
+export declare const PROGRESS_MS: number;
+export declare const INDEX_MAINTENANCE: boolean;
+export declare const DOWNLOAD_TRIES = 5;
+export declare const PART_SUFFIX = ".part";
+export declare const INFER_TIMEOUT_MS: number;
+export declare const STALL_MS: number;
+export declare const CACHE_WAIT_MS: number;
+export declare const VECTOR_CACHE_MB: number;
+export declare const SQLITE_CACHE_MB: number;
+export declare const MAX_MB: number;
+export declare const MAX_BYTES: number;

package/dist/example/train_base/config.js

@@ -0,0 +1,94 @@
+// train_base/config.ts — RUN-LEVEL configuration, all from the environment.
+//
+// Only knobs that describe the RUN live here: the store, the checkpoint cadence,
+// the cache ceiling, the read budgets, the caps. A knob that describes ONE
+// CORPUS (which pairs of SmolSent, how many SODA dialogues, how long an Aya
+// field may be) belongs next to that corpus's adapter, together with the
+// evidence that fixed its default — see AGENTS.md §2.16: a comment carries the
+// constraint, and a constraint is only readable beside the code it constrains.
+import { join } from "node:path";
+/** Read an environment variable, or `d` when it is unset. */
+export const env = (k, d) => process.env[k] ?? d;
+export const DB_PATH = env("DB_PATH", "sema"); // → {DB_PATH}.sqlite
+export const D = Number(env("D", "1024"));
+export const SEED = Number(env("SEED", "7"));
+// Checkpoint cadence is measured in LEARNED CONTENT, not deposits: a snapshot
+// every CHECKPOINT_MB megabytes of trained UTF-8 content (decimal MB, matching
+// the bytes() helper). A floor of 1 MB: a zero/NaN value must not make every
+// deposit checkpoint, nor silently disable checkpointing. The tail (a run that
+// learns less than one interval, or the remainder past the last interval) is
+// always saved by finish() at exit — a complete point.
+export const CHECKPOINT_BYTES = Math.max(1_000_000, Math.floor(Number(env("CHECKPOINT_MB", "100")) * 1_000_000) || 100_000_000);
+// Target size of ONE materialised Parquet read, in uncompressed source bytes.
+// A row-GROUP is a layout choice made by whoever wrote the file, not a memory
+// budget: Aya ships 203 groups of 1,000 rows (~1 MB each), while SODA ships ONE
+// group of 1,191,582 rows (1.19 GB uncompressed) and 2Wiki ONE of 167,454
+// (666 MB). Reading "exactly one row-group" is therefore safe for the first and
+// fatal for the others, so reads are sized in BYTES instead — see
+// `parquetBatchRows`. Materialised JS objects cost several times their source
+// bytes, hence a default well under available memory.
+export const PARQUET_BATCH_BYTES = Math.max(1_000_000, Math.floor(Number(env("PARQUET_BATCH_MB", "32")) * 1_000_000) || 32_000_000);
+export const LOCAL_PATH = env("LOCAL_PATH", ""); // train from a local dir
+export const CACHE_DIR = env("CACHE_DIR", join(process.cwd(), "cache"));
+export const MAX_CACHE_BYTES = Number(env("MAX_CACHE_GB", "100")) * 1e9;
+export const PROGRESS_MS = Number(env("PROGRESS_MS", "250")); // panel cadence
+// Index maintenance at checkpoints: compact (remove garbage), repair (fill
+// gaps), then refresh the canonical-form index (equivalence-class resolution —
+// src/canon.ts). All three are idempotent batch operations (the canon build is
+// additionally incremental via the store's `canon.upto` cursor);
+// INDEX_MAINTENANCE=0 disables.
+export const INDEX_MAINTENANCE = env("INDEX_MAINTENANCE", "1") !== "0";
+export const DOWNLOAD_TRIES = 5;
+// In-progress downloads are written to a sibling "<dest>.part" and atomically
+// renamed into place only after the bytes are fully flushed to disk. The cache
+// invariant is therefore absolute: a file at its final path is, by definition,
+// complete. Partial transfers (a crash, a kill, a dropped socket) leave only a
+// .part file, which is never fed to the parser and is swept at startup by
+// cache.ts's sweepPartials() — without which the debris would consume cache
+// ceiling that nothing frees.
+export const PART_SUFFIX = ".part";
+// The checkpoint recall is a best-effort diagnostic — it must NEVER stall
+// training. We bound it so a slow/large store cannot freeze the deposit loop.
+export const INFER_TIMEOUT_MS = Number(env("INFER_TIMEOUT_MS", "15000"));
+// How long the run may make NO progress before it gives up and exits non-zero.
+//
+// A long training run's worst failure is not a crash — a crash resumes. It is a
+// HANG: the uncaught-exception handler deliberately swallows dropped-connection
+// errors so a long run survives them, and the keep-alive timer deliberately
+// holds the process open; together, an error that escapes and leaves an await
+// unsettled produces a live process that will never do anything again. No error,
+// no exit, and a supervisor that sees a healthy pid. Exiting instead turns that
+// into a resume, which costs at most the work since the last checkpoint.
+//
+// "Progress" is any deposit, downloaded chunk, or rate-limit wait; time inside
+// index maintenance and the checkpoint recall does not count against it, since
+// those legitimately deposit nothing. Generous by default — this is a
+// last-resort backstop, not a latency budget. 0 disables it.
+export const STALL_MS = Math.max(0, Math.floor(Number(env("STALL_MIN", "15")) * 60_000) || 900_000);
+// How long a download may wait for room under the cache ceiling before failing
+// the unit instead of waiting forever. The wait exists so a bounded cache can
+// throttle a fast source; it is not meant to outlast the run. The unit stays
+// resumable, so a genuine ceiling problem costs a retry, not the corpus.
+export const CACHE_WAIT_MS = Math.max(60_000, Math.floor(Number(env("CACHE_WAIT_MIN", "10")) * 60_000) || 600_000);
+// The vector indices' memory knob (MiB) — each index's SQLite page cache.
+// The IVF index routes inserts through a RAM-resident pivot table and
+// appends to chunk blobs, so this cache mostly serves query-time cluster
+// scans; 256 MiB comfortably covers the probed working set of a trained
+// store.  Override with VECTOR_CACHE_MB (64 is the library default).
+export const VECTOR_CACHE_MB = Math.max(0, Number(env("VECTOR_CACHE_MB", "256")));
+// Page cache for the MAIN DAG database (node/kid/edge/contain tables).
+// Training issues millions of content-addressed point probes per session
+// against a GB-scale file; the library default (64 MiB) is sized for a
+// small machine — a training box affords more.  Override with
+// SQLITE_CACHE_MB.
+export const SQLITE_CACHE_MB = Math.max(0, Number(env("SQLITE_CACHE_MB", "256")));
+// Optional ceiling on how much LEARNED CONTENT to train, in megabytes (decimal,
+// like CHECKPOINT_MB). Default Infinity = unbounded. The cap is checked against
+// trainedContentBytes after each deposit, so a run stops at the first item that
+// carries the running total to/past the ceiling (that item is still counted).
+export const MAX_MB = Number(env("MAX_MB", "Infinity"));
+if (isNaN(MAX_MB) || MAX_MB < 0) {
+    process.stderr.write(`fatal: MAX_MB must be a non-negative number or "Infinity"\n`);
+    process.exit(1);
+}
+export const MAX_BYTES = MAX_MB * 1_000_000; // Infinity stays Infinity

package/dist/example/train_base/corpora/aya.d.ts

@@ -0,0 +1,19 @@
+import { type TrainingItem } from "../items.js";
+import { type Corpus } from "../corpus.js";
+/** One normalized Aya row. */
+export interface AyaRow {
+    inputs: string;
+    targets: string;
+    language: string;
+}
+/** Normalize a raw datasets-server row object into an AyaRow, or null when it
+ *  lacks a usable prompt/answer or a field is implausibly large (a dump, not a
+ *  cognitive example). Trims surrounding whitespace; keeps inner text verbatim
+ *  (human prose, possibly multi-paragraph). */
+export declare function toAyaRow(row: unknown, maxChars?: number): AyaRow | null;
+/** Translate ONE Aya row into SEMA training items. A row is a single human
+ *  (question → answer) exchange — exactly one FACT, the (inputs → targets) edge.
+ *  No standalone-answer experience and no one-exchange "cumulative" walk: a lone
+ *  Q→A is not multi-turn, and both would only replicate the same edge. */
+export declare function ayaRowToItems(row: AyaRow): TrainingItem[];
+export declare const aya: Corpus;