@hviana/sema 0.5.8 → 0.6.0

package/dist/example/train_base/corpus.d.ts

@@ -0,0 +1,88 @@
+import type { Reader, RowAdapter } from "./readers.js";
+import type { TrainCtx } from "./runtime.js";
+/** One file to train: a shard, a per-language file, or a whole single-file
+ *  corpus. Exactly one of `url` / `local` is set. */
+export interface Unit {
+    /** Resume-id suffix — the corpus id and this form `${id}::${key}`. Part of
+     *  the store's compatibility surface; see the file header. */
+    key: string;
+    /** How the run log names this unit once it is read. */
+    name: string;
+    /** How the live panel names it, and (unless `acquireLabel` overrides) how the
+     *  download is labelled. Conventionally `${corpus.label} ${name}`. */
+    display: string;
+    url?: string;
+    local?: string;
+    /** Size in bytes when the listing said, else 0/absent. Summed BEFORE the
+     *  stage reads anything, so the corpus progress bar has a denominator that
+     *  does not grow underneath it. */
+    bytes?: number;
+    /** Cache filename. Defaults to the resume id with unsafe characters folded. */
+    dest?: string;
+    /** Download label, when it differs from `display`. */
+    acquireLabel?: string;
+}
+/** How one unit's outcome reads in the run log. All optional: the defaults are
+ *  what every fact-shaped corpus prints. */
+export interface LogStyle {
+    /** What one deposit is called. Default "facts". */
+    deposits?: string;
+    /** When set, the line reports "from N <rows>" — the count of rows that
+     *  actually produced deposits. */
+    rows?: string;
+    /** What an unusable record is called. Default "unusable row(s)". */
+    bad?: string;
+    /** Report only the reader's `skipped` (malformed records), not the rows the
+     *  adapter declined. For a corpus that DECLINES records by design — oasst2
+     *  drops every single-turn tree — counting those as damage would be a lie. */
+    malformedOnly?: boolean;
+}
+export interface Corpus {
+    /** Tally key AND resume-id prefix. Compatibility surface — see the header. */
+    id: string;
+    /** Human name: the panel, the skip notices, the listing-failure message. */
+    label: string;
+    /** The dim tag in the log line, e.g. "translation", "social dialogue". */
+    kind: string;
+    enabled: boolean;
+    /** The work-list. Return [] for "nothing found" (the runner says so), or
+     *  null when the corpus has already logged a more specific reason. */
+    discover(ctx: TrainCtx): Promise<Unit[] | null>;
+    read: Reader;
+    toItems: RowAdapter;
+    /** Stage-wide row budget; 0/absent = unbounded. See the budget notes in
+     *  stage.ts. */
+    maxRows?: number;
+    /** Noun for the "N/M ___ to train" announcement. Absent ⇒ no announcement,
+     *  which is what a single-unit corpus has always done. */
+    unitNoun?: string;
+    /** Keep a file that came from the CACHE after a complete read. Only oasst2
+     *  does this: every other corpus deletes whatever acquire() handed it. */
+    keepCached?: boolean;
+    log?: LogStyle;
+}
+/** The string a store records once this unit is finished. ONE rule, no
+ *  exceptions — see the file header for why there used to be two. */
+export declare const unitIdOf: (corpus: Corpus, unit: Unit) => string;
+/** Local files live under `LOCAL_PATH/<sub>`, or directly in LOCAL_PATH when
+ *  `sub` is empty. Kept here because the layout is a user-facing convention:
+ *  the corpora that share an extension (.json, .parquet) are kept apart by a
+ *  subdirectory so a local run cannot feed one corpus's files to another. */
+export declare const localDir: (sub: string) => string;
+/** A single-unit corpus: one fixed URL, or one local file matched by pattern.
+ *  Factored out because the three corpora that are ONE file resolve it the same
+ *  way. `key` is what the store records this corpus under — `aya::dataset` is
+ *  `key: "dataset"` — so it is required rather than defaulted: a resume id is
+ *  the one thing here that must never be guessed. */
+export declare function singleUnit(opts: {
+    key: string;
+    label: string;
+    display: string;
+    url: string;
+    dest: string;
+    acquireLabel?: string;
+    /** Patterns tried, in order, against LOCAL_PATH. */
+    localMatch: RegExp[];
+    /** How the "no local copy" notice describes what it looked for. */
+    localWhat: string;
+}): (ctx: TrainCtx) => Promise<Unit[] | null>;

package/dist/example/train_base/corpus.js

@@ -0,0 +1,65 @@
+// train_base/corpus.ts — WHAT A CORPUS IS: the contract every corpus file
+// declares, and the resume identity derived from it.
+//
+// Separated from the loop that consumes it (stage.ts) so a corpus file never
+// imports the loop. A corpus declares three things the loop does not know —
+// where its work-list comes from, what container its bytes are in, and how a row
+// becomes deposits — and everything here exists to state those three.
+//
+// THE RESUME IDS ARE A COMPATIBILITY SURFACE. A store records the units it has
+// finished as strings, and a store trained by an earlier version must keep
+// resuming, so `unitIdOf` below has to reproduce the ids that store already
+// holds. It does, by one rule with no exceptions: `${corpus.id}::${unit.key}`.
+//
+// That single rule is a recent simplification, and the evidence for it is worth
+// keeping. There used to be a second mechanism — a `Corpus.unitId` field
+// carrying a FIXED id for the three single-unit corpora, which were believed
+// irregular — and with it a trap: those corpora emitted `key: ""` on the
+// assumption that the fixed id would always override, so a new single-unit
+// corpus that forgot the field silently got the id "foo::". Reading the ids out
+// of a real 2.5 GB store settled it: the three are `aya::dataset`,
+// `oasst2::trees` and `genknow::qa`, which decompose EXACTLY into corpus id and
+// key. They were never irregular, so the escape hatch and its trap are gone and
+// one rule covers all 239 recorded ids.
+import { LOCAL_PATH } from "./config.js";
+import { localFind } from "./discovery.js";
+import { DIM, R } from "./ui.js";
+import { join } from "node:path";
+/** The string a store records once this unit is finished. ONE rule, no
+ *  exceptions — see the file header for why there used to be two. */
+export const unitIdOf = (corpus, unit) => `${corpus.id}::${unit.key}`;
+/** Local files live under `LOCAL_PATH/<sub>`, or directly in LOCAL_PATH when
+ *  `sub` is empty. Kept here because the layout is a user-facing convention:
+ *  the corpora that share an extension (.json, .parquet) are kept apart by a
+ *  subdirectory so a local run cannot feed one corpus's files to another. */
+export const localDir = (sub) => sub ? join(LOCAL_PATH, sub) : LOCAL_PATH;
+/** A single-unit corpus: one fixed URL, or one local file matched by pattern.
+ *  Factored out because the three corpora that are ONE file resolve it the same
+ *  way. `key` is what the store records this corpus under — `aya::dataset` is
+ *  `key: "dataset"` — so it is required rather than defaulted: a resume id is
+ *  the one thing here that must never be guessed. */
+export function singleUnit(opts) {
+    return async (ctx) => {
+        if (LOCAL_PATH) {
+            const hit = localFind(LOCAL_PATH, ...opts.localMatch);
+            if (!hit) {
+                ctx.progress.log(`  ${DIM}· no ${opts.localWhat} in ${LOCAL_PATH} — skipping${R}`);
+                return null;
+            }
+            return [{
+                    key: opts.key,
+                    name: opts.label,
+                    display: opts.display,
+                    local: join(LOCAL_PATH, hit),
+                }];
+        }
+        return [{
+                key: opts.key,
+                name: opts.label,
+                display: opts.display,
+                url: opts.url,
+                dest: opts.dest,
+                acquireLabel: opts.acquireLabel,
+            }];
+    };
+}

package/dist/example/train_base/discovery.d.ts

@@ -0,0 +1,48 @@
+import { type HttpOptions } from "./http.js";
+/** A note a listing needs to make about ITSELF — an accepted-but-unexpected
+ *  split name, say. Distinct from a failure: the run continues, but silently
+ *  continuing would hide why fewer units appeared than expected. */
+export type Note = (msg: string) => void;
+/** One listed file: where it is, and how big it is.
+ *
+ *  The SIZE is carried because the panel is otherwise dishonest. Corpus
+ *  progress used to be measured against a total that GREW as each file was
+ *  opened, so the bar ran to 100% at the end of every file and then fell back
+ *  when the next one was added — 100%, 50%, 100%, 66%… Both listing APIs
+ *  already return the size, so the denominator can simply be known before the
+ *  first byte is read. */
+export interface Listed {
+    path: string;
+    /** Bytes, or 0 when the source did not say. */
+    size: number;
+}
+/** Files under `path` in a Hugging Face dataset repo's main branch, filtered to
+ *  an extension. Returns repo-relative paths (e.g. "smolsent/ha_en.jsonl"),
+ *  sorted, so a run's unit order is stable across machines. */
+export declare function hfTree(dataset: string, path: string, ext: RegExp, label: string, opts: HttpOptions): Promise<Listed[]>;
+/** A dataset's Parquet shards on Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch, restricted to `config` and to `splits`.
+ *
+ *  The converted branch is used rather than `main` because a dataset's own
+ *  Parquet may be written as ONE giant row-group (SODA's is 1,191,582 rows),
+ *  and a column chunk is per-group, so reading any part of it materialises all
+ *  of it. The converted branch is uniformly 10,000-row groups.
+ *
+ *  Paths look like "<config>/<split>/0000.parquet". The BRANCH name is a single
+ *  path SEGMENT here, so its "/" is percent-encoded — unlike a dataset id,
+ *  whose "/" must not be. */
+export declare function hfConvertedParquet(dataset: string, config: string, splits: string[], label: string, opts: HttpOptions, note?: Note): Promise<Listed[]>;
+/** File NAMES (not paths) in one directory of a GitHub repo, filtered to an
+ *  extension and sorted. Used for corpora served from GitHub raw rather than
+ *  Hugging Face — where the HF mirrors are loading-script repos with no data
+ *  files, the official GitHub copy is the one carrying the licence notice. */
+export declare function githubContents(repo: string, dir: string, ext: RegExp, label: string, opts: HttpOptions): Promise<Listed[]>;
+/** Every file in a local directory matching `ext`, sorted. A missing directory
+ *  is an empty list, not an error: LOCAL_PATH is an offline convenience and a
+ *  stage with no local copy simply reports that and moves on. */
+export declare function localFiles(dir: string, ext: RegExp): Listed[];
+/** The FIRST file in a local directory matching any of `exts`, in directory
+ *  order (deliberately NOT sorted — this mirrors the single-file stages, which
+ *  take whichever copy the filesystem hands back first). Null when none match
+ *  or the directory is absent. */
+export declare function localFind(dir: string, ...exts: RegExp[]): string | null;

package/dist/example/train_base/discovery.js

@@ -0,0 +1,143 @@
+// train_base/discovery.ts — where a stage's WORK-LIST comes from.
+//
+// Four remote strategies and two local ones, each generic over the dataset it
+// is pointed at. What stays with a corpus is its POLICY — which subset of the
+// listing to keep, how to name the resume unit — because that is a curriculum
+// decision, not a protocol one.
+//
+// A note that has bitten this code twice, in both directions: a dataset id
+// ("owner/name") is a PATH here and its "/" must NOT be percent-encoded, while
+// a branch name ("refs/convert/parquet") is a single path SEGMENT and its "/"
+// MUST be.
+import { getJson, getJsonPaged } from "./http.js";
+import { readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+const sizeOf = (e) => {
+    const n = Number(e?.size ?? e?.lfs?.size ?? 0);
+    return Number.isFinite(n) && n > 0 ? n : 0;
+};
+/** Files under `path` in a Hugging Face dataset repo's main branch, filtered to
+ *  an extension. Returns repo-relative paths (e.g. "smolsent/ha_en.jsonl"),
+ *  sorted, so a run's unit order is stable across machines. */
+export async function hfTree(dataset, path, ext, label, opts) {
+    // The dataset id is a PATH here, so its "/" must not be percent-encoded.
+    // `recursive=true` returns every file under `path`, 1,000 at a time — hence
+    // the PAGED fetch: a truncated work-list would train part of a corpus and
+    // then call it finished.
+    const url = `https://huggingface.co/api/datasets/${dataset}` +
+        `/tree/main/${path}?recursive=true`;
+    const body = await getJsonPaged(url, label, opts);
+    const out = body
+        .filter((e) => e?.type === "file" && ext.test(e?.path))
+        .map((e) => ({ path: String(e.path), size: sizeOf(e) }));
+    out.sort((a, b) => a.path.localeCompare(b.path));
+    return out;
+}
+/** A dataset's Parquet shards on Hugging Face's auto-converted
+ *  `refs/convert/parquet` branch, restricted to `config` and to `splits`.
+ *
+ *  The converted branch is used rather than `main` because a dataset's own
+ *  Parquet may be written as ONE giant row-group (SODA's is 1,191,582 rows),
+ *  and a column chunk is per-group, so reading any part of it materialises all
+ *  of it. The converted branch is uniformly 10,000-row groups.
+ *
+ *  Paths look like "<config>/<split>/0000.parquet". The BRANCH name is a single
+ *  path SEGMENT here, so its "/" is percent-encoded — unlike a dataset id,
+ *  whose "/" must not be. */
+export async function hfConvertedParquet(dataset, config, splits, label, opts, note) {
+    const body = await getJsonPaged(`https://huggingface.co/api/datasets/${dataset}` +
+        `/tree/refs%2Fconvert%2Fparquet/${config}?recursive=true`, `GET ${label} tree`, opts);
+    const paths = body
+        .filter((e) => e?.type === "file" && /\.parquet$/i.test(e?.path))
+        .map((e) => ({ path: String(e.path), size: sizeOf(e) }));
+    paths.sort((a, b) => a.path.localeCompare(b.path));
+    // The tree is rooted at `config`, so the split is the second-to-last part.
+    const splitOf = (p) => p.split("/").slice(-2)[0] ?? "";
+    const present = new Set(paths.map((p) => splitOf(p.path)));
+    // MATCH THE SPLIT AGAINST WHAT THE BRANCH ACTUALLY CARRIES. An exact name is
+    // not guaranteed: the converter renames a split it could not finish, and
+    // shards a very large one. Observed on real datasets today —
+    //   allenai/c4            → partial-train, partial-validation
+    //   HuggingFaceFW/fineweb → train-part0
+    // — neither of which equals "train". The old exact-match filter returned []
+    // for both, and an empty work-list is reported as "no files found — skipping",
+    // which reads like a normal outcome rather than a corpus being dropped whole.
+    const wanted = new Set();
+    for (const want of splits) {
+        const pick = present.has(want)
+            ? want
+            : [...present].find((s) => s === `partial-${want}`) ??
+                [...present].find((s) => s.startsWith(`${want}-part`));
+        if (!pick) {
+            // Loud, not empty: a requested split that simply is not there is a
+            // configuration error, and silence would hide the whole corpus.
+            throw new Error(`${label}: split "${want}" is not on the converted branch — it ` +
+                `carries ${[...present].join(", ") || "no parquet at all"}`);
+        }
+        if (pick !== want) {
+            note?.(`${label}: split "${want}" is published as "${pick}"` +
+                (pick.startsWith("partial-")
+                    ? " — Hugging Face has only PARTIALLY converted this dataset, so " +
+                        "the shards below are not the whole split"
+                    : ""));
+        }
+        wanted.add(pick);
+    }
+    return paths.filter((p) => wanted.has(splitOf(p.path)));
+}
+/** File NAMES (not paths) in one directory of a GitHub repo, filtered to an
+ *  extension and sorted. Used for corpora served from GitHub raw rather than
+ *  Hugging Face — where the HF mirrors are loading-script repos with no data
+ *  files, the official GitHub copy is the one carrying the licence notice. */
+export async function githubContents(repo, dir, ext, label, opts) {
+    const body = await getJson(`https://api.github.com/repos/${repo}/contents/${dir}`, label, opts);
+    const entries = Array.isArray(body) ? body : [];
+    // The contents API returns at most 1,000 entries for a directory and does NOT
+    // paginate them — it simply stops, with no Link header and no error. A
+    // directory at that boundary is therefore indistinguishable from a truncated
+    // one, so the only honest response is to refuse rather than train part of it
+    // and record the part as the whole.
+    if (entries.length >= 1000) {
+        throw new Error(`${label}: GitHub returned ${entries.length} entries, the point at ` +
+            `which the contents API truncates without saying so — this listing ` +
+            `cannot be trusted to be complete`);
+    }
+    const names = entries
+        .filter((e) => e?.type === "file" && ext.test(e?.name))
+        .map((e) => ({ path: String(e.name), size: sizeOf(e) }));
+    names.sort((a, b) => a.path.localeCompare(b.path));
+    return names;
+}
+/** Every file in a local directory matching `ext`, sorted. A missing directory
+ *  is an empty list, not an error: LOCAL_PATH is an offline convenience and a
+ *  stage with no local copy simply reports that and moves on. */
+export function localFiles(dir, ext) {
+    try {
+        return readdirSync(dir)
+            .filter((f) => ext.test(f))
+            .sort()
+            .map((f) => {
+            let size = 0;
+            try {
+                size = statSync(join(dir, f)).size;
+            }
+            catch { /* unreadable — the read will report it */ }
+            return { path: f, size };
+        });
+    }
+    catch {
+        return []; // no such directory
+    }
+}
+/** The FIRST file in a local directory matching any of `exts`, in directory
+ *  order (deliberately NOT sorted — this mirrors the single-file stages, which
+ *  take whichever copy the filesystem hands back first). Null when none match
+ *  or the directory is absent. */
+export function localFind(dir, ...exts) {
+    try {
+        return readdirSync(dir).find((f) => exts.some((re) => re.test(f))) ?? null;
+    }
+    catch {
+        return null; // no such directory
+    }
+}

package/dist/example/train_base/http.d.ts

@@ -0,0 +1,82 @@
+/** Sleep `ms`, but wake early if `signal` fires — so a long back-off (e.g. a
+ *  rate-limit wait) never swallows Ctrl+C. Resolves either way. */
+export declare const waitMs: (ms: number, signal: AbortSignal) => Promise<void>;
+/** Resolve `p`, but reject with a TimeoutError if it takes longer than `ms`.
+ *  The underlying promise is left to settle on its own (we just stop waiting),
+ *  so a slow black-box call can never wedge the caller. */
+export declare function withTimeout<T>(p: Promise<T>, ms: number, label?: string): Promise<T>;
+/** An HTTP error the caller tagged as transient. `.fatal` skips all retries;
+ *  `.throttle` (a 429/503 rate-limit or overload) is retried indefinitely and
+ *  does NOT consume the bounded attempt budget — the server told us to wait, not
+ *  to give up. `.retryAfterMs` carries a server-suggested delay when present. */
+export type HttpError = Error & {
+    fatal?: boolean;
+    throttle?: boolean;
+    retryAfterMs?: number;
+};
+/** What every network call in this trainer needs from its caller: how to be
+ *  cancelled, and (optionally) where to report a rate-limit wait. */
+export interface HttpOptions {
+    signal: AbortSignal;
+    /** Called after each throttle wait, so a 429 back-off reads as "waiting"
+     *  rather than a silent hang. Omitted ⇒ the wait is silent. */
+    onThrottle?: (waitMsAmount: number, label: string) => void;
+}
+/** Wrap a throttle notice so a STORM of 429s logs at most one notice every
+ *  `minGapMs`. Without the gap a busy server produces a wall of identical
+ *  "waiting…" lines that pushes the real log out of the scrollback. */
+export declare function throttleNotifier(fn: (waitMsAmount: number, label: string) => void, minGapMs?: number): (waitMsAmount: number, label: string) => void;
+/** Retry `fn` with exponential backoff.
+ *
+ *  Three error classes:
+ *   • `.fatal` / AbortError  → rethrown immediately (never retried).
+ *   • `.throttle` (429/503)  → the server is rate-limiting/overloaded. We are
+ *     NOT failing — we WAIT (honouring Retry-After, else capped exponential
+ *     back-off with jitter) and retry WITHOUT consuming an attempt, so a
+ *     throttled request holds on until it succeeds rather than being dropped.
+ *     Only a shutdown breaks this loop.
+ *   • anything else          → a genuine transient error, retried up to `tries`
+ *     with exponential back-off before giving up.
+ *
+ *  `onFail` is called after each non-throttle failed attempt; `onThrottle` after
+ *  each throttle wait (for a "waiting…" notice). */
+export declare function retry<T>(label: string, fn: () => Promise<T>, tries: number, opts: HttpOptions & {
+    onFail?: (attempt: number, err: Error) => void;
+}): Promise<T>;
+/** Classify a non-OK HTTP response into an {@link HttpError} for {@link retry}:
+ *   • 429 / 503  → THROTTLE (rate-limited / overloaded): retried indefinitely,
+ *     honouring a Retry-After header (seconds or an HTTP-date) when present.
+ *   • other 5xx  → transient: retried up to the caller's attempt budget.
+ *   • other 4xx  → FATAL: a real client error (404, 401, …) — not retried.
+ *  Never throttles forever silently: the wait is interruptible by shutdown. */
+export declare function httpError(res: Response): HttpError;
+/** GET a URL and parse JSON, with the shared retry policy: rate-limits (429/503)
+ *  WAIT indefinitely (surfaced through `opts.onThrottle`), other 4xx is fatal,
+ *  other 5xx retried up to DOWNLOAD_TRIES. Used by every dataset LISTING call so
+ *  all share the same never-drop-on-throttle behaviour. */
+export declare function getJson(url: string, label: string, opts: HttpOptions): Promise<any>;
+/** The `rel="next"` URL of an RFC 5988 Link header, or null. */
+export declare function nextLink(header: string | null): string | null;
+/** GET a paginated JSON ARRAY, following `Link: rel="next"` to the end.
+ *
+ *  A LISTING THAT STOPS EARLY IS INVISIBLE, and that is why this exists.
+ *  Hugging Face caps a tree listing at 1,000 entries and hands back a next
+ *  link (verified: allenai/c4 returns exactly 1,000 plus a link). A caller that
+ *  ignores it gets a work-list silently missing everything past the first page,
+ *  trains it, marks those units complete, and thereafter reports the corpus
+ *  "already trained". No error at any point. Following the links is the only
+ *  way the work-list can be trusted to be the whole work-list.
+ *
+ *  `maxPages` is a runaway guard, not a limit anyone should hit; exceeding it
+ *  throws rather than returning a partial list, for exactly the reason above. */
+export declare function getJsonPaged(url: string, label: string, opts: HttpOptions, maxPages?: number): Promise<unknown[]>;
+/** Advertised transfer size of `url`, used only to reserve cache room. Like any
+ *  `content-length` this is the ON-THE-WIRE size, so for a content-coded source
+ *  (GitHub raw gzips JSON ~14x) it UNDER-estimates the file that lands on disk.
+ *  That is tolerable here because the cache ceiling is a budget, not a
+ *  correctness property — a run may overshoot MAX_CACHE_GB by the compression
+ *  ratio of one in-flight file, and each file is deleted as soon as it is
+ *  consumed. It must NOT be reused as an integrity check; see downloadFile.
+ *
+ *  Rate-limits wait; other 4xx is fatal; total failure → the caller's catch. */
+export declare function headSize(url: string, opts: HttpOptions): Promise<number>;

package/dist/example/train_base/http.js

@@ -0,0 +1,219 @@
+// train_base/http.ts — the shared network policy: wait out throttling, retry
+// what is transient, give up at once on what is not.
+//
+// A leaf module by construction: everything it needs to cancel (the run's
+// AbortSignal) and everything it needs to report (a throttle notice) arrives
+// as a parameter. There is no module-level shutdown handle and no module-level
+// log hook — those were globals precisely because this code used to live in the
+// same file as the run that owned them.
+import { DOWNLOAD_TRIES } from "./config.js";
+/** Sleep `ms`, but wake early if `signal` fires — so a long back-off (e.g. a
+ *  rate-limit wait) never swallows Ctrl+C. Resolves either way. */
+export const waitMs = (ms, signal) => new Promise((resolve) => {
+    if (signal.aborted)
+        return resolve();
+    // NOTE: the timer is deliberately NOT unref'd — an unref'd timer does not
+    // keep the event loop alive, so a pending wait (e.g. the pace between page
+    // requests, or a rate-limit back-off) would let Node exit early and the run
+    // would "do nothing and close". The listener lets a shutdown wake it early.
+    const t = setTimeout(done, ms);
+    function done() {
+        clearTimeout(t);
+        signal.removeEventListener("abort", done);
+        resolve();
+    }
+    signal.addEventListener("abort", done, { once: true });
+});
+/** Resolve `p`, but reject with a TimeoutError if it takes longer than `ms`.
+ *  The underlying promise is left to settle on its own (we just stop waiting),
+ *  so a slow black-box call can never wedge the caller. */
+export function withTimeout(p, ms, label = "operation") {
+    return new Promise((resolve, reject) => {
+        const t = setTimeout(() => {
+            const e = new Error(`${label} timed out after ${ms}ms`);
+            e.name = "TimeoutError";
+            reject(e);
+        }, ms);
+        if (typeof t.unref === "function")
+            t.unref();
+        p.then((v) => {
+            clearTimeout(t);
+            resolve(v);
+        }, (e) => {
+            clearTimeout(t);
+            reject(e);
+        });
+    });
+}
+/** Wrap a throttle notice so a STORM of 429s logs at most one notice every
+ *  `minGapMs`. Without the gap a busy server produces a wall of identical
+ *  "waiting…" lines that pushes the real log out of the scrollback. */
+export function throttleNotifier(fn, minGapMs = 3000) {
+    let last = 0;
+    return (ms, label) => {
+        const now = Date.now();
+        if (now - last <= minGapMs)
+            return;
+        last = now;
+        fn(ms, label);
+    };
+}
+/** Retry `fn` with exponential backoff.
+ *
+ *  Three error classes:
+ *   • `.fatal` / AbortError  → rethrown immediately (never retried).
+ *   • `.throttle` (429/503)  → the server is rate-limiting/overloaded. We are
+ *     NOT failing — we WAIT (honouring Retry-After, else capped exponential
+ *     back-off with jitter) and retry WITHOUT consuming an attempt, so a
+ *     throttled request holds on until it succeeds rather than being dropped.
+ *     Only a shutdown breaks this loop.
+ *   • anything else          → a genuine transient error, retried up to `tries`
+ *     with exponential back-off before giving up.
+ *
+ *  `onFail` is called after each non-throttle failed attempt; `onThrottle` after
+ *  each throttle wait (for a "waiting…" notice). */
+export async function retry(label, fn, tries, opts) {
+    const { signal, onFail, onThrottle } = opts;
+    let wait = 1000, last = "", throttleWait = 1000;
+    for (let attempt = 1; attempt <= tries;) {
+        if (signal.aborted) {
+            const e = new Error("aborted");
+            e.fatal = true;
+            throw e;
+        }
+        try {
+            return await fn();
+        }
+        catch (e) {
+            const err = e;
+            if (err.name === "AbortError" || err.fatal)
+                throw err;
+            // Rate-limited / overloaded: wait it out. Does NOT advance `attempt`, so a
+            // busy server can never exhaust the retry budget and drop the request.
+            if (err.throttle && !signal.aborted) {
+                // Honour Retry-After when the server sent one; else exponential back-off
+                // with jitter, capped, so a fleet of requests does not resynchronise.
+                const base = err.retryAfterMs && err.retryAfterMs > 0
+                    ? err.retryAfterMs
+                    : throttleWait;
+                const ms = Math.min(base, 60_000) +
+                    Math.floor(base * 0.25 * Math.random());
+                onThrottle?.(ms, label);
+                await waitMs(ms, signal);
+                throttleWait = Math.min(throttleWait * 2, 60_000);
+                continue;
+            }
+            last = err.message;
+            onFail?.(attempt, err);
+            attempt++;
+            if (attempt <= tries) {
+                await waitMs(wait, signal);
+                wait = Math.min(wait * 2, 30_000);
+            }
+        }
+    }
+    throw new Error(`${label} failed after ${tries} attempts: ${last}`);
+}
+/** Classify a non-OK HTTP response into an {@link HttpError} for {@link retry}:
+ *   • 429 / 503  → THROTTLE (rate-limited / overloaded): retried indefinitely,
+ *     honouring a Retry-After header (seconds or an HTTP-date) when present.
+ *   • other 5xx  → transient: retried up to the caller's attempt budget.
+ *   • other 4xx  → FATAL: a real client error (404, 401, …) — not retried.
+ *  Never throttles forever silently: the wait is interruptible by shutdown. */
+export function httpError(res) {
+    const err = new Error(`HTTP ${res.status}`);
+    if (res.status === 429 || res.status === 503) {
+        err.throttle = true;
+        const ra = res.headers.get("retry-after");
+        if (ra) {
+            const secs = Number(ra);
+            if (Number.isFinite(secs))
+                err.retryAfterMs = Math.max(0, secs * 1000);
+            else {
+                const when = Date.parse(ra);
+                if (Number.isFinite(when)) {
+                    err.retryAfterMs = Math.max(0, when - Date.now());
+                }
+            }
+        }
+    }
+    else if (res.status < 500) {
+        err.fatal = true; // genuine client error — do not retry
+    } // other 5xx: neither fatal nor throttle → ordinary bounded retry
+    return err;
+}
+/** GET a URL and parse JSON, with the shared retry policy: rate-limits (429/503)
+ *  WAIT indefinitely (surfaced through `opts.onThrottle`), other 4xx is fatal,
+ *  other 5xx retried up to DOWNLOAD_TRIES. Used by every dataset LISTING call so
+ *  all share the same never-drop-on-throttle behaviour. */
+export async function getJson(url, label, opts) {
+    return retry(label, async () => {
+        const res = await fetch(url, { signal: opts.signal });
+        if (res.ok)
+            return res.json();
+        throw httpError(res);
+    }, DOWNLOAD_TRIES, opts);
+}
+/** The `rel="next"` URL of an RFC 5988 Link header, or null. */
+export function nextLink(header) {
+    if (!header)
+        return null;
+    for (const part of header.split(",")) {
+        const m = part.match(/<([^>]+)>\s*;\s*rel\s*=\s*"?next"?/i);
+        if (m)
+            return m[1];
+    }
+    return null;
+}
+/** GET a paginated JSON ARRAY, following `Link: rel="next"` to the end.
+ *
+ *  A LISTING THAT STOPS EARLY IS INVISIBLE, and that is why this exists.
+ *  Hugging Face caps a tree listing at 1,000 entries and hands back a next
+ *  link (verified: allenai/c4 returns exactly 1,000 plus a link). A caller that
+ *  ignores it gets a work-list silently missing everything past the first page,
+ *  trains it, marks those units complete, and thereafter reports the corpus
+ *  "already trained". No error at any point. Following the links is the only
+ *  way the work-list can be trusted to be the whole work-list.
+ *
+ *  `maxPages` is a runaway guard, not a limit anyone should hit; exceeding it
+ *  throws rather than returning a partial list, for exactly the reason above. */
+export async function getJsonPaged(url, label, opts, maxPages = 500) {
+    const out = [];
+    let next = url;
+    let pages = 0;
+    while (next !== null) {
+        const at = next;
+        const { body, link } = await retry(label, async () => {
+            const res = await fetch(at, { signal: opts.signal });
+            if (!res.ok)
+                throw httpError(res);
+            return { body: await res.json(), link: res.headers.get("link") };
+        }, DOWNLOAD_TRIES, opts);
+        if (!Array.isArray(body))
+            break; // not a listing — nothing to page through
+        out.push(...body);
+        next = nextLink(link);
+        if (++pages >= maxPages && next) {
+            throw new Error(`${label}: more than ${maxPages} pages of listing — refusing to ` +
+                `continue with a work-list that may be incomplete`);
+        }
+    }
+    return out;
+}
+/** Advertised transfer size of `url`, used only to reserve cache room. Like any
+ *  `content-length` this is the ON-THE-WIRE size, so for a content-coded source
+ *  (GitHub raw gzips JSON ~14x) it UNDER-estimates the file that lands on disk.
+ *  That is tolerable here because the cache ceiling is a budget, not a
+ *  correctness property — a run may overshoot MAX_CACHE_GB by the compression
+ *  ratio of one in-flight file, and each file is deleted as soon as it is
+ *  consumed. It must NOT be reused as an integrity check; see downloadFile.
+ *
+ *  Rate-limits wait; other 4xx is fatal; total failure → the caller's catch. */
+export async function headSize(url, opts) {
+    return retry(`HEAD ${url}`, async () => {
+        const res = await fetch(url, { method: "HEAD", signal: opts.signal });
+        if (res.ok)
+            return Number(res.headers.get("content-length")) || 0;
+        throw httpError(res);
+    }, 4, opts);
+}