@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/utils/xet-download.js

@@ -0,0 +1,526 @@
+/**
+ * Xet reconstruction downloader — the fetch/stream/retry/resume half of the
+ * Xet support. See `xet.ts` for the protocol overview and the pure decoders.
+ *
+ * The classic downloader (`download-with-retry.ts`) detects a Xet-backed file
+ * from the resolve 302's `X-Xet-Hash` header and hands the `{ xetHash, authUrl }`
+ * detection here. `downloadXet` drives the reconstruction: mint a CAS token,
+ * fetch the reconstruction manifest, then fetch each term's xorb byte range,
+ * decode + decompress the chunks, and stream them to `${destPath}.partial` in
+ * term order — with the same event/result contract, resume-from-partial,
+ * backoff retry, stall timeout, and abort handling as the classic downloader.
+ * The caller still sha256-verifies the finished file, so a corrupt resume is
+ * caught and the `.partial` is cleaned by the caller.
+ */
+import { createWriteStream } from 'node:fs';
+import { backoffDelayMs } from '@bendyline/gezel';
+import { DEFAULT_CHUNK_TIMEOUT_MS, DEFAULT_MAX_RETRIES, DownloadRetryBudget, GEZEL_DOWNLOAD_UA, downloadLog, existingPartialSize, friendlyFetchError, friendlyStatusError, friendlyStreamError, huggingFaceRequestHeaders, rawErrorString, sleepRespectingAbort, } from './download-common.js';
+import { decodeSegmentChunks } from './xet.js';
+const DEFAULT_XET_TERM_CONCURRENCY = 8;
+/** How often progress is reported while the writer waits on the next term. */
+const PROGRESS_TICK_MS = 500;
+const TICK = Symbol('tick');
+/**
+ * Compressed bytes received across every in-flight segment, against the total
+ * the manifest says this attempt will fetch. Bytes reach the file only in term
+ * order, and the eight prefetched terms share the link, so the first term lands
+ * after about eight terms' worth of transfer: a 2 GB model read 0% for minutes
+ * and then jumped (2026-10-08). Received bytes move the bar while that happens.
+ */
+class ReceiveMeter {
+    total;
+    received = 0;
+    constructor(total) {
+        this.total = total;
+    }
+    add(bytes) {
+        this.received += bytes;
+    }
+    fraction() {
+        return this.total > 0 ? Math.min(1, this.received / this.total) : 0;
+    }
+}
+/** Compressed bytes of every segment the terms still to be written will fetch. */
+function remainingFetchBytes(recon, termStarts, resumeFrom) {
+    const seen = new Set();
+    let total = 0;
+    recon.terms.forEach((term, index) => {
+        if (!term)
+            return;
+        const leadingTrim = index === 0 ? (recon.offset_into_first_range ?? 0) : 0;
+        const termEnd = (termStarts[index] ?? 0) + Math.max(0, term.unpacked_length - leadingTrim);
+        if (termEnd <= resumeFrom)
+            return;
+        for (const seg of recon.fetch_info[term.hash] ?? []) {
+            if (seg.range.end <= term.range.start || seg.range.start >= term.range.end)
+                continue;
+            const key = `${term.hash}:${seg.range.start}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            total += seg.url_range.end - seg.url_range.start + 1;
+        }
+    });
+    return total;
+}
+/** The settled value, or {@link TICK} if `ms` passes first. */
+async function settledOrTick(promise, ms) {
+    let timer;
+    const tick = new Promise((resolve) => {
+        timer = setTimeout(() => resolve(TICK), ms);
+    });
+    try {
+        return await Promise.race([promise, tick]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export async function* downloadXet(opts) {
+    const partialPath = `${opts.destPath}.partial`;
+    const maxAttempts = opts.maxRetries ?? DEFAULT_MAX_RETRIES;
+    if (opts.maxBytes !== undefined && existingPartialSize(partialPath) > opts.maxBytes) {
+        return {
+            kind: 'error',
+            error: byteLimitError(opts.maxBytes),
+            attemptsMade: 0,
+            bytesWritten: existingPartialSize(partialPath),
+        };
+    }
+    const budget = new DownloadRetryBudget(maxAttempts, existingPartialSize(partialPath));
+    let lastFriendly = 'Xet download failed';
+    while (budget.canAttempt()) {
+        budget.beginAttempt();
+        if (opts.signal?.aborted)
+            return aborted(partialPath);
+        const resumeFrom = existingPartialSize(partialPath);
+        const result = yield* runXetAttempt(opts, partialPath, resumeFrom);
+        if (result.kind === 'ok')
+            return { kind: 'ok', bytesWritten: result.bytesWritten, partialPath };
+        if (result.kind === 'aborted')
+            return aborted(partialPath);
+        if (result.kind === 'fatal') {
+            return {
+                kind: 'error',
+                error: result.friendlyError,
+                attemptsMade: budget.attemptsMade,
+                bytesWritten: result.bytesWritten,
+            };
+        }
+        lastFriendly = result.friendlyError;
+        budget.recordFailure(result.bytesWritten);
+        downloadLog.warn(`[xet] attempt ${budget.attemptsMade} failed at ${result.bytesWritten} bytes: ${result.friendlyError} (rawError=${result.rawError})`);
+        if (!budget.canAttempt())
+            break;
+        const delay = budget.nextDelayMs();
+        yield {
+            type: 'retrying',
+            attempt: budget.nextAttemptLabel,
+            maxAttempts: budget.maxAttempts,
+            delayMs: delay,
+            reason: result.friendlyError,
+            resumeFromBytes: result.bytesWritten,
+        };
+        if (await sleepRespectingAbort(delay, opts.signal))
+            return aborted(partialPath);
+    }
+    const attemptsMade = budget.attemptsMade;
+    return {
+        kind: 'error',
+        error: `${lastFriendly} (gave up after ${attemptsMade} attempt${attemptsMade === 1 ? '' : 's'})`,
+        attemptsMade,
+        bytesWritten: existingPartialSize(partialPath),
+    };
+}
+function byteLimitError(maxBytes) {
+    return `Download exceeds the ${maxBytes} byte limit`;
+}
+function aborted(partialPath) {
+    return { kind: 'aborted', bytesWritten: existingPartialSize(partialPath), partialPath };
+}
+/** A Xet fetch failure carrying its retryability + friendly copy. */
+class XetError extends Error {
+    friendly;
+    transient;
+    raw;
+    wasAborted;
+    constructor(friendly, transient, raw, wasAborted = false) {
+        super(friendly);
+        this.friendly = friendly;
+        this.transient = transient;
+        this.raw = raw;
+        this.wasAborted = wasAborted;
+    }
+}
+async function* runXetAttempt(opts, partialPath, resumeFrom) {
+    const { detection, fetchImpl, signal } = opts;
+    const chunkTimeoutMs = opts.chunkTimeoutMs ?? DEFAULT_CHUNK_TIMEOUT_MS;
+    // 1. Mint a CAS token (anonymous for public repos). A 4xx here (other than
+    //    rate-limit / timeout) means a gated repo — fatal, no point retrying.
+    let token;
+    try {
+        const tr = await fetchImpl(detection.authUrl, {
+            headers: huggingFaceRequestHeaders(detection.authUrl),
+            signal,
+        });
+        if (!tr.ok) {
+            const fatal = tr.status >= 400 && tr.status < 500 && tr.status !== 429 && tr.status !== 408;
+            return {
+                kind: fatal ? 'fatal' : 'transient',
+                friendlyError: friendlyStatusError(tr.status, tr.statusText, detection.authUrl),
+                rawError: `xet-auth HTTP ${tr.status}`,
+                bytesWritten: resumeFrom,
+            };
+        }
+        token = (await tr.json());
+    }
+    catch (err) {
+        if (signal?.aborted)
+            return { kind: 'aborted', bytesWritten: resumeFrom };
+        return {
+            kind: 'transient',
+            friendlyError: friendlyFetchError(err),
+            rawError: rawErrorString(err),
+            bytesWritten: resumeFrom,
+        };
+    }
+    // 2. Fetch the reconstruction manifest.
+    let recon;
+    try {
+        const rr = await fetchImpl(`${token.casUrl}/v1/reconstructions/${detection.xetHash}`, {
+            headers: { Authorization: `Bearer ${token.accessToken}`, 'User-Agent': GEZEL_DOWNLOAD_UA },
+            signal,
+        });
+        if (!rr.ok) {
+            // 403 with a just-minted token → treat as transient (re-mint next pass).
+            const transient = rr.status >= 500 || rr.status === 429 || rr.status === 408 || rr.status === 403;
+            return {
+                kind: transient ? 'transient' : 'fatal',
+                friendlyError: friendlyStatusError(rr.status, rr.statusText, 'reconstruction'),
+                rawError: `reconstruction HTTP ${rr.status}`,
+                bytesWritten: resumeFrom,
+            };
+        }
+        recon = (await rr.json());
+    }
+    catch (err) {
+        if (signal?.aborted)
+            return { kind: 'aborted', bytesWritten: resumeFrom };
+        return {
+            kind: 'transient',
+            friendlyError: friendlyFetchError(err),
+            rawError: rawErrorString(err),
+            bytesWritten: resumeFrom,
+        };
+    }
+    // Exact total from the manifest (drives an accurate progress bar).
+    const termStarts = [];
+    let totalBytes = 0;
+    for (let index = 0; index < recon.terms.length; index++) {
+        const term = recon.terms[index];
+        if (!term)
+            continue;
+        termStarts[index] = totalBytes;
+        const leadingTrim = index === 0 ? (recon.offset_into_first_range ?? 0) : 0;
+        totalBytes += Math.max(0, term.unpacked_length - leadingTrim);
+    }
+    if (opts.maxBytes !== undefined && totalBytes > opts.maxBytes) {
+        return {
+            kind: 'fatal',
+            friendlyError: byteLimitError(opts.maxBytes),
+            bytesWritten: resumeFrom,
+        };
+    }
+    // 3. Stream terms to `.partial`, resuming past bytes already on disk.
+    const stream = createWriteStream(partialPath, { flags: resumeFrom > 0 ? 'a' : 'w' });
+    let streamError = null;
+    stream.on('error', (e) => {
+        streamError ??= e;
+    });
+    const segCache = new Map();
+    const prefetched = new Map();
+    const meter = new ReceiveMeter(remainingFetchBytes(recon, termStarts, resumeFrom));
+    const onBytes = (bytes) => meter.add(bytes);
+    let reported = resumeFrom;
+    const progress = (bytes) => {
+        reported = Math.max(reported, Math.min(totalBytes, bytes));
+        return { type: 'progress', bytesWritten: reported, totalBytes };
+    };
+    const estimate = () => resumeFrom + Math.floor((totalBytes - resumeFrom) * meter.fraction());
+    const termConcurrency = Math.max(1, Math.floor(opts.termConcurrency ?? DEFAULT_XET_TERM_CONCURRENCY));
+    let nextToPrefetch = 0;
+    const fillPrefetchWindow = () => {
+        while (prefetched.size < termConcurrency && nextToPrefetch < recon.terms.length) {
+            const index = nextToPrefetch++;
+            const term = recon.terms[index];
+            if (!term)
+                continue;
+            const leadingTrim = index === 0 ? (recon.offset_into_first_range ?? 0) : 0;
+            const termStart = termStarts[index] ?? 0;
+            const termLength = Math.max(0, term.unpacked_length - leadingTrim);
+            if (termStart + termLength <= resumeFrom)
+                continue;
+            const pending = termData(term, recon, segCache, fetchImpl, chunkTimeoutMs, signal, onBytes).then((data) => ({
+                ok: true,
+                data: leadingTrim > 0 ? data.subarray(leadingTrim) : data,
+            }), (error) => ({ ok: false, error }));
+            prefetched.set(index, pending);
+        }
+    };
+    fillPrefetchWindow();
+    let written = 0; // logical output-file offset processed so far
+    let lastReport = Date.now();
+    try {
+        for (let termIndex = 0; termIndex < recon.terms.length; termIndex++) {
+            const term = recon.terms[termIndex];
+            if (!term)
+                continue;
+            if (signal?.aborted) {
+                await closeStream(stream);
+                return { kind: 'aborted', bytesWritten: Math.max(written, resumeFrom) };
+            }
+            // `offset_into_first_range` trims bytes from the first reconstruction
+            // term before it becomes file output. Because the manifest already
+            // carries every term's unpacked length, we can locate the resume
+            // boundary without downloading or decoding the preceding xorbs.
+            const leadingTrim = termIndex === 0 ? (recon.offset_into_first_range ?? 0) : 0;
+            const termLength = Math.max(0, term.unpacked_length - leadingTrim);
+            const termStart = termStarts[termIndex] ?? written;
+            const termEnd = termStart + termLength;
+            // The partial file already contains this entire term. The previous
+            // implementation still fetched + decoded it and merely discarded the
+            // bytes, making a 16 GB resume silently re-download 16 GB while the
+            // progress bar appeared frozen at the saved offset.
+            if (termEnd <= resumeFrom) {
+                written = termEnd;
+                continue;
+            }
+            // A manifest that lies about its total cannot over-deliver past the cap.
+            if (opts.maxBytes !== undefined && termEnd > opts.maxBytes) {
+                await closeStream(stream);
+                return {
+                    kind: 'fatal',
+                    friendlyError: byteLimitError(opts.maxBytes),
+                    bytesWritten: Math.max(written, resumeFrom),
+                };
+            }
+            const pending = prefetched.get(termIndex);
+            if (!pending)
+                throw new XetError('reconstruction term was not prefetched', false, 'term-missing');
+            let fetched = await settledOrTick(pending, PROGRESS_TICK_MS);
+            while (fetched === TICK) {
+                lastReport = Date.now();
+                yield progress(estimate());
+                fetched = await settledOrTick(pending, PROGRESS_TICK_MS);
+            }
+            prefetched.delete(termIndex);
+            fillPrefetchWindow();
+            if (!fetched.ok)
+                throw fetched.error;
+            const data = fetched.data;
+            const toWrite = termStart < resumeFrom ? data.subarray(resumeFrom - termStart) : data;
+            if (opts.maxBytes !== undefined &&
+                Math.max(written, resumeFrom) + toWrite.byteLength > opts.maxBytes) {
+                await closeStream(stream);
+                return {
+                    kind: 'fatal',
+                    friendlyError: byteLimitError(opts.maxBytes),
+                    bytesWritten: Math.max(written, resumeFrom),
+                };
+            }
+            await writeChunk(stream, toWrite);
+            if (streamError)
+                throw streamError;
+            written = termEnd;
+            // Evict fully-consumed segments of this xorb (terms are ordered).
+            for (const [key, entry] of segCache) {
+                if (key.startsWith(`${term.hash}:`) && entry.end <= term.range.end)
+                    segCache.delete(key);
+            }
+            if (segCache.size > 16)
+                segCache.clear();
+            const now = Date.now();
+            if (now - lastReport > 250) {
+                lastReport = now;
+                yield progress(Math.max(written, estimate()));
+            }
+        }
+    }
+    catch (err) {
+        await closeStream(stream);
+        if (signal?.aborted || (err instanceof XetError && err.wasAborted)) {
+            return { kind: 'aborted', bytesWritten: Math.max(written, resumeFrom) };
+        }
+        // The `.partial` genuinely holds `resumeFrom` bytes even when the failure
+        // landed inside the term that straddles the resume boundary, so report the
+        // larger of the two — the retry budget reads this as forward progress.
+        const onDisk = Math.max(written, resumeFrom);
+        if (err instanceof XetError) {
+            return err.transient
+                ? {
+                    kind: 'transient',
+                    friendlyError: err.friendly,
+                    rawError: err.raw,
+                    bytesWritten: onDisk,
+                }
+                : { kind: 'fatal', friendlyError: err.friendly, bytesWritten: onDisk };
+        }
+        return {
+            kind: 'transient',
+            friendlyError: friendlyStreamError(err),
+            rawError: rawErrorString(err),
+            bytesWritten: onDisk,
+        };
+    }
+    try {
+        await new Promise((resolve, reject) => {
+            stream.end((err) => (err ? reject(err) : resolve()));
+        });
+        if (streamError)
+            throw streamError;
+    }
+    catch (err) {
+        await closeStream(stream);
+        return {
+            kind: 'transient',
+            friendlyError: friendlyStreamError(err),
+            rawError: rawErrorString(err),
+            bytesWritten: written,
+        };
+    }
+    yield progress(totalBytes);
+    return { kind: 'ok', bytesWritten: written };
+}
+/** Decode one term's chunk range into a contiguous buffer. */
+async function termData(term, recon, segCache, fetchImpl, chunkTimeoutMs, signal, onBytes) {
+    const segs = recon.fetch_info[term.hash];
+    if (!segs)
+        throw new XetError(`reconstruction missing xorb ${term.hash.slice(0, 12)}`, false, 'missing-xorb');
+    const parts = [];
+    for (let idx = term.range.start; idx < term.range.end; idx++) {
+        const seg = segs.find((s) => s.range.start <= idx && idx < s.range.end);
+        if (!seg)
+            throw new XetError(`no segment covers chunk ${idx}`, false, 'missing-segment');
+        const key = `${term.hash}:${seg.range.start}`;
+        let entry = segCache.get(key);
+        if (!entry) {
+            entry = {
+                end: seg.range.end,
+                chunks: fetchXorbSegment(seg, fetchImpl, chunkTimeoutMs, signal, onBytes).then((buf) => decodeSegmentChunks(buf, seg.range.end - seg.range.start)),
+            };
+            segCache.set(key, entry);
+        }
+        const chunks = await entry.chunks;
+        parts.push(chunks[idx - seg.range.start]);
+    }
+    return Buffer.concat(parts);
+}
+/** In-place retries for one xorb segment before the failure escalates. */
+const SEGMENT_ATTEMPTS = 3;
+/** Tighter than the whole-file schedule: a segment is small and independently
+ * re-fetchable, so the useful move is to try again quickly rather than to sit
+ * out a multi-second backoff hundreds of times over one file. */
+const SEGMENT_BACKOFF_MS = [400, 1_200, 2_500];
+/**
+ * Fetch one xorb byte range, retrying transient failures in place.
+ *
+ * This is the difference between "one dropped packet costs a few seconds" and
+ * "one dropped packet costs a whole file attempt". A multi-GB Xet file is
+ * reconstructed from hundreds of these segment fetches; before the inner
+ * retry, a single blip unwound all the way to the outer loop, which re-minted
+ * the CAS token, re-fetched the reconstruction manifest, and re-walked the
+ * terms — and, worse, spent one of only five whole-file attempts. On a link
+ * that drops even occasionally, the outer budget was guaranteed to run out
+ * before the file finished. Segments are small and individually re-fetchable,
+ * so retrying at this granularity is nearly free.
+ */
+async function fetchXorbSegment(seg, fetchImpl, chunkTimeoutMs, signal, onBytes) {
+    let lastError;
+    for (let attempt = 1; attempt <= SEGMENT_ATTEMPTS; attempt++) {
+        try {
+            return await fetchXorbSegmentOnce(seg, fetchImpl, chunkTimeoutMs, signal, onBytes);
+        }
+        catch (err) {
+            if (!(err instanceof XetError) || !err.transient || err.wasAborted || signal?.aborted) {
+                throw err;
+            }
+            lastError = err;
+            if (attempt >= SEGMENT_ATTEMPTS)
+                break;
+            const delay = backoffDelayMs(attempt - 1, SEGMENT_BACKOFF_MS);
+            downloadLog.debug(`[xet] segment fetch attempt ${attempt}/${SEGMENT_ATTEMPTS} failed (${err.raw}); retrying in ${delay}ms`);
+            if (await sleepRespectingAbort(delay, signal)) {
+                throw new XetError(err.friendly, true, err.raw, true);
+            }
+        }
+    }
+    throw lastError ?? new XetError('xorb fetch failed', true, 'segment-exhausted');
+}
+/** One attempt at a xorb byte range, with a per-read stall timeout. */
+async function fetchXorbSegmentOnce(seg, fetchImpl, chunkTimeoutMs, signal, onBytes) {
+    const ac = new AbortController();
+    const forward = () => ac.abort();
+    signal?.addEventListener('abort', forward);
+    try {
+        let res;
+        try {
+            res = await fetchImpl(seg.url, {
+                headers: {
+                    Range: `bytes=${seg.url_range.start}-${seg.url_range.end}`,
+                    'User-Agent': GEZEL_DOWNLOAD_UA,
+                },
+                signal: ac.signal,
+            });
+        }
+        catch (err) {
+            throw new XetError(friendlyFetchError(err), true, rawErrorString(err), Boolean(signal?.aborted));
+        }
+        if (res.status !== 200 && res.status !== 206) {
+            // 403 mid-download ≈ expired CAS token → transient (re-mint next attempt).
+            const transient = res.status >= 500 || res.status === 429 || res.status === 408 || res.status === 403;
+            throw new XetError(friendlyStatusError(res.status, res.statusText, seg.url), transient, `xorb HTTP ${res.status}`);
+        }
+        if (!res.body)
+            throw new XetError('xorb response had no body', true, 'no-body');
+        const reader = res.body.getReader();
+        const parts = [];
+        while (true) {
+            let timeout;
+            const raced = await Promise.race([
+                reader.read(),
+                new Promise((resolve) => {
+                    timeout = setTimeout(() => resolve('timeout'), chunkTimeoutMs);
+                }),
+            ]).finally(() => {
+                if (timeout)
+                    clearTimeout(timeout);
+            });
+            if (raced === 'timeout') {
+                ac.abort();
+                throw new XetError(`connection stalled — no data for ${Math.round(chunkTimeoutMs / 1000)}s`, true, 'chunk-timeout');
+            }
+            if (raced.done)
+                break;
+            if (raced.value) {
+                parts.push(Buffer.from(raced.value));
+                onBytes?.(raced.value.byteLength);
+            }
+        }
+        return Buffer.concat(parts);
+    }
+    finally {
+        signal?.removeEventListener('abort', forward);
+    }
+}
+function writeChunk(stream, buf) {
+    return new Promise((resolve, reject) => {
+        stream.write(buf, (err) => (err ? reject(err) : resolve()));
+    });
+}
+async function closeStream(stream) {
+    if (!stream.destroyed)
+        stream.destroy();
+    if (stream.closed)
+        return;
+    await new Promise((resolve) => stream.once('close', () => resolve()));
+}

package/dist/utils/xet.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Pure decoders for HuggingFace's Xet content-addressed storage format.
+ *
+ * HF migrated model hosting to Xet (content-defined chunking + dedup). A
+ * `/resolve/<rev>/<file>` request for a Xet-backed file no longer 302s to a
+ * plain CDN blob — it redirects into a CAS bridge that a naive "follow the
+ * redirect and GET" client can't read (it 403s). The supported path is the
+ * reconstruction protocol:
+ *
+ *   1. resolve (redirect: manual) → the 302 carries `X-Xet-Hash` + a `link`
+ *      header with rel="xet-auth" (token mint) and rel="xet-reconstruction-info".
+ *   2. GET the xet-auth endpoint → { casUrl, accessToken, exp } (anonymous
+ *      works for public repos).
+ *   3. GET `${casUrl}/v1/reconstructions/${xetHash}` (Bearer token) → an
+ *      ordered `terms[]` (each = a xorb hash + a chunk-index range) plus
+ *      `fetch_info` (per-xorb presigned `transfer.xethub.hf.co` URLs with the
+ *      byte range to fetch for a given chunk range).
+ *   4. For each term, fetch its xorb byte range, walk the chunks, decompress,
+ *      and concatenate in term order (dropping `offset_into_first_range` bytes
+ *      at the very start). sha256 of the result matches the manifest pin.
+ *
+ * This file is the PURE half: no network, no fs. It parses chunk headers and
+ * decompresses the three schemes Xet emits. The I/O driver lives in
+ * `xet-download.ts`. Every decoder here was validated against real e2b files
+ * (tokenizer.json = 100% LZ4, model.safetensors = None+LZ4+bg4) by full-file
+ * sha256 before this shipped — do not "simplify" the LZ4 frame handling
+ * without re-checking against a bg4 file.
+ *
+ * Chunk header (8 bytes, little-endian):
+ *   [version:u8][compressedLen:u24][scheme:u8][uncompressedLen:u24]
+ * Compression schemes:
+ *   0 None · 1 LZ4 (frame) · 2 ByteGroupingLZ4 (bg4: 4 byte-planes, LZ4 frame)
+ *
+ * NB: scheme 1/2 payloads are LZ4 *frame* format (magic 0x184D2204), NOT raw
+ * blocks — the block decoder runs once per frame block.
+ */
+export declare const XET_CHUNK_HEADER_BYTES = 8;
+/** A single reconstruction term: chunks [range.start, range.end) of a xorb. */
+export interface XetTerm {
+    hash: string;
+    unpacked_length: number;
+    range: {
+        start: number;
+        end: number;
+    };
+}
+/** A fetchable byte segment of a xorb covering chunk range [start, end). */
+export interface XetFetchSegment {
+    range: {
+        start: number;
+        end: number;
+    };
+    url: string;
+    url_range: {
+        start: number;
+        end: number;
+    };
+}
+export interface XetReconstruction {
+    offset_into_first_range: number;
+    terms: XetTerm[];
+    fetch_info: Record<string, XetFetchSegment[]>;
+}
+/** Decompress an LZ4 frame to exactly `destSize` bytes. */
+export declare function lz4DecompressFrame(src: Buffer, destSize: number): Buffer;
+/**
+ * Undo bg4 byte-plane regrouping. The original data was split into 4 planes by
+ * index % 4 (plane sizes floor((len - k + 3) / 4)), the planes concatenated,
+ * then LZ4-compressed. We're given the already-decompressed regrouped buffer.
+ */
+export declare function bg4Unregroup(regrouped: Buffer, len: number): Buffer;
+export interface DecodedChunk {
+    data: Buffer;
+    /** Offset of the next chunk header within the source buffer. */
+    next: number;
+}
+/** Decode a single chunk at `buf[off]`. */
+export declare function decodeChunk(buf: Buffer, off: number): DecodedChunk;
+/**
+ * Decode every chunk in a fetched xorb byte segment → one decoded buffer per
+ * chunk, in order. `expectedChunks` (segment chunk-range width) guards against
+ * a short/over-read.
+ */
+export declare function decodeSegmentChunks(buf: Buffer, expectedChunks: number): Buffer[];
+/** Parse the `link` header from a resolve 302 into a rel → url map. */
+export declare function parseLinkHeader(header: string | null): Record<string, string>;