@elaraai/e3-api-server 1.0.78 → 1.0.80

package/dist/src/handlers/datasets.js

@@ -2,9 +2,9 @@
  * Copyright (c) 2025 Elara AI Pty Ltd
  * Licensed under BSL 1.1. See LICENSE for details.
  */
-import { NullType, ArrayType, StringType, compareFor, decodeBeast2, encodeBeast2For, openBeast2PagesFor, parseFor, readBeast2ExtentsRanged, carveBeast2Ranged, some, none, variant, toEastTypeValue, isVariant } from '@elaraai/east';
-import { workspaceListTree, workspaceGetDatasetHash, workspaceGetDatasetStatus, workspaceSetDataset, workspaceGetTree, } from '@elaraai/e3-core';
-import { BEAST2_CONTENT_TYPE } from '@elaraai/e3-core';
+import { NullType, ArrayType, StringType, compareFor, encodeBeast2For, openBeast2PagesFor, parseFor, some, none, variant, toEastTypeValue, isVariant } from '@elaraai/east';
+import { BEAST2_CONTENT_TYPE } from '@elaraai/e3-types';
+import { workspaceListTree, workspaceGetDatasetHash, workspaceGetDatasetStatus, workspaceSetDatasetBytes, workspaceGetTree, openDatasetObject, recordIndexNames, resolveRecordIndex, DatasetSegments, } from '@elaraai/e3-core';
 import { sendSuccess, sendError } from '../beast2.js';
 import { errorToVariant, sendJsonError } from '../errors.js';
 import { DatasetStatusDetailType, ListEntryType } from '../types.js';
@@ -20,15 +20,23 @@ export async function listDatasets(storage, repoPath, workspace, treePath) {
         return sendError(ArrayType(StringType), errorToVariant(err));
     }
 }
-const SIZE_THRESHOLD = 1 * 1024 * 1024; // 1 MB
+/** The size over which an object is answered with a URL to fetch it from
+ *  rather than inline, when a transfer backend is configured. */
+export const DOWNLOAD_REDIRECT_BYTES = 1 * 1024 * 1024;
 /**
  * Get dataset value as raw BEAST2 bytes.
  *
- * For objects > 1MB, returns a JSON response with a download URL
- * that the client can fetch directly. This avoids browser issues
- * with opaque redirect responses from `redirect: 'manual'`.
+ * A collection held as a segment manifest is streamed as its splice, or, for a
+ * caller that asks for its segments, answered with JSON `{ manifest }`, the
+ * manifest's hash — the primary's, for an indexed record — for a client that
+ * downloads the manifest and its segment objects itself and splices them. A
+ * host that cannot stream a response answers a large collection no other way.
+ * Any other object over {@link DOWNLOAD_REDIRECT_BYTES}, when a transfer
+ * backend is configured, is answered with a JSON response carrying a download
+ * URL the client fetches directly. This avoids browser issues with opaque
+ * redirect responses from `redirect: 'manual'`.
  */
-export async function getDataset(storage, repoPath, workspace, treePath, repo, requestUrl, transferBackend) {
+export async function getDataset(storage, repoPath, workspace, treePath, repo, requestUrl, transferBackend, segments = false) {
     try {
         if (treePath.length === 0) {
             return new Response(JSON.stringify({ error: { type: 'bad_request', message: 'Path required for get' } }), {
@@ -49,10 +57,49 @@ export async function getDataset(storage, repoPath, workspace, treePath, repo, r
                 headers: { 'Content-Type': 'application/json' },
             });
         }
+        // A collection held as a segment manifest is many objects, so the
+        // transfer backend — which serves ONE object by hash — cannot hand the
+        // client a value. The splice is streamed instead, a segment read only
+        // when the client takes the bytes before it: a host that streams the
+        // response holds one segment at a time, and one that buffers it into a
+        // single payload holds the value, as far as that payload's limit allows.
+        // The download redirect stays for objects that are the value.
+        const collection = await openDatasetObject(storage, repoPath, hash);
+        if (collection.manifest !== null) {
+            if (segments) {
+                return new Response(JSON.stringify({ manifest: collection.hash }), {
+                    status: 200,
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-Content-SHA256': hash,
+                    },
+                });
+            }
+            const chunks = (await DatasetSegments.open(storage, repoPath, hash)).splice()[Symbol.asyncIterator]();
+            const body = new ReadableStream({
+                async pull(controller) {
+                    const next = await chunks.next();
+                    if (next.done)
+                        controller.close();
+                    else
+                        controller.enqueue(next.value);
+                },
+                async cancel() {
+                    await chunks.return?.();
+                },
+            }, { highWaterMark: 0 });
+            return new Response(body, {
+                status: 200,
+                headers: {
+                    'Content-Type': BEAST2_CONTENT_TYPE,
+                    'X-Content-SHA256': hash,
+                },
+            });
+        }
         // When serving via API with a transfer backend, check size to decide whether to redirect
         if (transferBackend && repo && requestUrl) {
             const { size } = await storage.objects.stat(repoPath, hash);
-            if (size > SIZE_THRESHOLD) {
+            if (size > DOWNLOAD_REDIRECT_BYTES) {
                 let downloadUrl = await transferBackend.datasetDownload.getDownloadUrl(repo, hash);
                 // Resolve relative URL against the request origin
                 if (downloadUrl.startsWith('/')) {
@@ -94,53 +141,61 @@ const PAGE_MAX_LIMIT = 10_000;
  *  tighter response limits (e.g. Lambda proxy's 6 MB, base64-inflated) pass
  *  a smaller budget through the route options. */
 export const PAGE_BYTE_BUDGET_DEFAULT = 4 * 1024 * 1024;
-/** Largest blob the page endpoint will buffer when the storage backend
- *  cannot serve ranged reads (`objects.readRange` absent) and every page
- *  request must read the blob whole. Backends with ranged reads never
- *  buffer the blob, so no cap applies — per-window memory is O(window) at
- *  any blob size. */
-export const PAGE_READ_MAX_BYTES_DEFAULT = 512 * 1024 * 1024;
-const EXTENTS_CACHE_MAX_ENTRIES = 64;
+const SEGMENTS_CACHE_MAX_ENTRIES = 64;
 /** Cap on the cache's total retained bytes. The newest entry always stays
  *  (serving the request needs it regardless), so one giant index can still
  *  be held — but never alongside others. */
-const EXTENTS_CACHE_MAX_BYTES = 16 * 1024 * 1024;
-const extentsCache = new Map();
-let extentsCacheBytes = 0;
+const SEGMENTS_CACHE_MAX_BYTES = 16 * 1024 * 1024;
+const segmentsCache = new Map();
+let segmentsCacheBytes = 0;
 /** Evicts oldest entries until the count and byte budgets hold, always
  *  keeping at least the newest entry. */
-function evictExtents() {
-    while (extentsCache.size > 1 &&
-        (extentsCache.size > EXTENTS_CACHE_MAX_ENTRIES || extentsCacheBytes > EXTENTS_CACHE_MAX_BYTES)) {
-        const oldest = extentsCache.keys().next().value;
-        extentsCacheBytes -= extentsCache.get(oldest).bytes;
-        extentsCache.delete(oldest);
+function evictSegments() {
+    while (segmentsCache.size > 1 &&
+        (segmentsCache.size > SEGMENTS_CACHE_MAX_ENTRIES || segmentsCacheBytes > SEGMENTS_CACHE_MAX_BYTES)) {
+        const oldest = segmentsCache.keys().next().value;
+        segmentsCacheBytes -= segmentsCache.get(oldest).bytes;
+        segmentsCache.delete(oldest);
     }
 }
-/** Fetches (or reuses) a blob's ranged extents, LRU-cached per hash and
- *  bounded by retained bytes. */
-async function cachedRangedExtents(hash, size, readRange) {
-    const cached = extentsCache.get(hash);
-    if (cached) {
-        extentsCache.delete(hash);
-        extentsCache.set(hash, cached); // refresh recency
-        return cached;
+/** Identifies a storage backend for the cache key — lazily, and without
+ *  keeping it alive. */
+const storageIds = new WeakMap();
+let nextStorageId = 0;
+/** Opens (or reuses) a stored collection, LRU-cached per backend, repository
+ *  and content hash, and bounded by retained bytes.
+ *
+ *  All three key parts are load-bearing. An opened dataset holds the backend
+ *  it reads segment objects through, and a manifest names objects that exist
+ *  in the repository it was read from and nowhere else — so two backends, or
+ *  two repositories, that happen to hold the same bytes under the same name
+ *  must not share an entry. */
+async function cachedSegments(storage, repoPath, hash, size) {
+    let backendId = storageIds.get(storage);
+    if (backendId === undefined) {
+        backendId = nextStorageId++;
+        storageIds.set(storage, backendId);
     }
-    const extents = await readBeast2ExtentsRanged({ size, read: readRange });
-    const cumulative = new Array(extents.counts.length);
-    let running = 0;
-    for (let i = 0; i < extents.counts.length; i++) {
-        running += extents.counts[i];
-        cumulative[i] = running;
+    const key = `${backendId}\u0000${repoPath}\u0000${hash}`;
+    const cached = segmentsCache.get(key);
+    if (cached) {
+        segmentsCache.delete(key);
+        segmentsCache.set(key, cached); // refresh recency
+        return cached.segments;
     }
-    // offsets + counts + cumulative at 8 bytes per element, plus the header.
-    const bytes = extents.head.byteLength + 24 * extents.counts.length;
-    const entry = { extents, cumulative, fences: new Map(), bytes };
-    extentsCache.set(hash, entry);
-    extentsCacheBytes += bytes;
-    evictExtents();
-    return entry;
+    const segments = await DatasetSegments.open(storage, repoPath, hash, size);
+    // counts + prefix sums at 8 bytes each, plus the manifest's stored fences.
+    let bytes = 16 * segments.segmentCount;
+    for (const entry of segments.manifest.entries)
+        bytes += entry.fence.byteLength + 80;
+    segmentsCache.set(key, { segments, bytes });
+    segmentsCacheBytes += bytes;
+    evictSegments();
+    return segments;
 }
+/** The opener's refusal of a collection stored as one blob, which an older
+ *  e3 wrote: answered as `dataset_not_indexed`, in the refusal's own words. */
+const BLOB_REFUSAL = /^the collection [0-9a-f]{64} is stored as one blob:/;
 function pageError(type, message, status = 400, headers) {
     return new Response(JSON.stringify({ error: { type, message } }), {
         status,
@@ -149,7 +204,6 @@ function pageError(type, message, status = 400, headers) {
 }
 export async function getDatasetPage(storage, repoPath, workspace, treePath, window, limits) {
     const byteBudget = limits?.byteBudget ?? PAGE_BYTE_BUDGET_DEFAULT;
-    const readMaxBytes = limits?.readMaxBytes ?? PAGE_READ_MAX_BYTES_DEFAULT;
     try {
         if (treePath.length === 0) {
             return pageError('bad_request', 'Path required for paged get');
@@ -174,6 +228,9 @@ export async function getDatasetPage(storage, repoPath, workspace, treePath, win
         if (kind !== 'Array' && kind !== 'Set' && kind !== 'Dict') {
             return pageError('dataset_not_pageable', `Paged reads address Array, Set or Dict datasets; this dataset holds ${kind}`);
         }
+        if (window.index !== undefined) {
+            return await indexPage(storage, repoPath, workspace, treePath, status.hash, window, byteBudget);
+        }
         const segmentMode = window.segment !== undefined;
         if (segmentMode && (window.offset !== undefined || window.limit !== undefined)) {
             return pageError('bad_request', 'Pass either segment or offset/limit, not both');
@@ -186,26 +243,43 @@ export async function getDatasetPage(storage, repoPath, workspace, treePath, win
         if (!segmentMode && (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(requestedLimit) || requestedLimit < 1)) {
             return pageError('bad_request', `offset must be a non-negative integer and limit a positive integer, got offset=${offset} limit=${requestedLimit}`);
         }
-        const statSize = status.size ?? (await storage.objects.stat(repoPath, status.hash)).size;
+        const objectSize = status.size ?? (await storage.objects.stat(repoPath, status.hash)).size;
+        let segments;
+        try {
+            segments = await cachedSegments(storage, repoPath, status.hash, objectSize);
+        }
+        catch (err) {
+            // Only the opener's refusal of a blob-stored collection means "not
+            // indexed". Anything else (storage I/O, a missing or corrupt object) is
+            // a real failure and must surface as one.
+            const message = err instanceof Error ? err.message : String(err);
+            if (!BLOB_REFUSAL.test(message))
+                throw err;
+            return pageError('dataset_not_indexed', message);
+        }
+        const segmentCount = segments.segmentCount;
+        const totalElements = segments.elementCount;
+        const totalBytes = segments.bytes;
+        const cumulative = segments.cumulative;
         // The byte budget turns the requested element limit into an effective one
-        // using the blob's average element size, so pages stay bounded even for
+        // using the dataset's average element size, so pages stay bounded even for
         // very wide rows.
-        const effectiveLimit = (totalElements) => {
-            const avgBytes = totalElements > 0 ? statSize / totalElements : 1;
+        const effectiveLimit = () => {
+            const avgBytes = totalElements > 0 ? totalBytes / totalElements : 1;
             const byBudget = Math.max(1, Math.floor(byteBudget / Math.max(1, avgBytes)));
             return Math.max(1, Math.min(requestedLimit, PAGE_MAX_LIMIT, byBudget));
         };
-        const pageHeaders = (totalElements, segmentCount, pageOffset, pageCount) => ({
+        const pageHeaders = (pageOffset, pageCount) => ({
             'Content-Type': BEAST2_CONTENT_TYPE,
             // Hash-pinned windows are content-addressed: same URL ⇒ same bytes,
             // forever — cacheable at any layer with no invalidation. Unpinned
             // windows track the mutable current value and must not be cached.
             'Cache-Control': window.hash !== undefined ? 'public, max-age=31536000, immutable' : 'no-store',
             'X-Content-SHA256': status.hash,
-            // The stored blob's full byte size — the page endpoint is then
+            // The value's full stored byte size — the page endpoint is then
             // self-describing (no separate status call needed for the header
             // line); Content-Length remains the page's own bytes.
-            'X-Total-Bytes': String(statSize),
+            'X-Total-Bytes': String(totalBytes),
             'X-Total-Elements': String(totalElements),
             // Always exact: Array counts index stream order, and v5 Set/Dict
             // segments are disjoint ranges of the canonical value.
@@ -214,152 +288,54 @@ export async function getDatasetPage(storage, repoPath, workspace, treePath, win
             'X-Page-Offset': String(pageOffset),
             'X-Page-Count': String(pageCount),
         });
-        // Ranged path: backends with `objects.readRange` never buffer the blob.
-        // The tail (index) and head (header sections) are read once per content
-        // hash and cached; each window then reads only the byte ranges of the
-        // segments it touches — per-request memory is O(window) at any blob
-        // size, so no cap applies (#513).
-        const readRange = storage.objects.readRange?.bind(storage.objects);
-        if (readRange) {
-            let cached;
-            try {
-                cached = await cachedRangedExtents(status.hash, statSize, (offset, length) => readRange(repoPath, status.hash, offset, length));
-            }
-            catch (err) {
-                // Only the reader's own blob-shape refusals mean "not indexed" — a
-                // blob predating the stored-segmented contract (or injected raw).
-                // Anything else (storage I/O, missing object) is a real failure and
-                // must surface as one, not masquerade as a re-write suggestion.
-                const message = err instanceof Error ? err.message : String(err);
-                if (!/^(beast2 v5:|Data too short for Beast2|Invalid Beast2)/.test(message)) {
-                    throw err;
-                }
-                return pageError('dataset_not_indexed', 'Dataset blob carries no pageable segment index — re-write the dataset (re-run the producing task, or set the value again) to store it in the indexed form.');
-            }
-            const { extents, cumulative } = cached;
-            if (!extents.selfContained) {
-                return pageError('dataset_not_indexed', 'Dataset blob carries no pageable segment index — re-write the dataset (re-run the producing task, or set the value again) to store it in the indexed form.');
+        // The touched segment span [from, to) and the window placement.
+        let from;
+        let to;
+        let limit = 0;
+        if (segmentMode) {
+            const seg = window.segment;
+            if (seg >= segmentCount) {
+                return pageError('bad_request', `segment ${seg} out of range (${segmentCount} segments)`);
             }
-            const segmentCount = extents.offsets.length;
-            const totalElements = extents.elementCount;
-            // The touched segment span [from, to) and the window placement.
-            let from;
-            let to;
-            let limit = 0;
-            if (segmentMode) {
-                const seg = window.segment;
-                if (seg >= segmentCount) {
-                    return pageError('bad_request', `segment ${seg} out of range (${segmentCount} segments)`);
-                }
-                from = seg;
-                to = seg + 1;
+            from = seg;
+            to = seg + 1;
+        }
+        else {
+            limit = effectiveLimit();
+            if (offset >= totalElements) {
+                from = 0;
+                to = 0; // empty window past the end
             }
             else {
-                limit = effectiveLimit(totalElements);
-                if (offset >= totalElements) {
-                    from = 0;
-                    to = 0; // empty window past the end
-                }
-                else {
-                    from = 0;
-                    while (cumulative[from] <= offset)
-                        from++;
-                    const lastRow = Math.min(offset + limit, totalElements) - 1;
-                    to = from;
-                    while (cumulative[to] <= lastRow)
-                        to++;
+                from = 0;
+                while (cumulative[from] <= offset)
+                    from++;
+                const lastRow = Math.min(offset + limit, totalElements) - 1;
+                to = from;
+                while (cumulative[to] <= lastRow)
                     to++;
-                }
-            }
-            const spanStart = to > from ? extents.offsets[from] : 0;
-            const spanEnd = to > from
-                ? (to < segmentCount ? extents.offsets[to] : extents.segmentsEnd)
-                : 0;
-            const frames = to > from
-                ? await readRange(repoPath, status.hash, spanStart, spanEnd - spanStart)
-                : new Uint8Array(0);
-            const windowBlob = carveBeast2Ranged(extents, frames, from, to);
-            const pages = openBeast2PagesFor(typeValue)(windowBlob);
-            let windowValue;
-            let pageOffset;
-            let pageCount;
-            if (segmentMode) {
-                try {
-                    windowValue = pages.segment(0);
-                }
-                catch (err) {
-                    return pageError('dataset_not_segmented', err instanceof Error ? err.message : String(err));
-                }
-                pageCount = extents.counts[from];
-                pageOffset = from === 0 ? 0 : cumulative[from - 1];
+                to++;
             }
-            else {
-                const base = from > 0 ? cumulative[from - 1] : 0;
-                try {
-                    windowValue = pages.slice(to > from ? offset - base : 0, limit);
-                }
-                catch (err) {
-                    return pageError('dataset_not_canonical', err instanceof Error ? err.message : String(err));
-                }
-                pageCount = kind === 'Array'
-                    ? windowValue.length
-                    : windowValue.size;
-                pageOffset = offset;
-            }
-            const body = encodeBeast2For(typeValue)(windowValue);
-            return new Response(body, {
-                status: 200,
-                headers: {
-                    ...pageHeaders(totalElements, segmentCount, pageOffset, pageCount),
-                    'Content-Length': String(body.byteLength),
-                },
-            });
-        }
-        // Fallback path (no ranged reads): the whole blob is buffered per
-        // request, so the absolute cap protects the server process itself.
-        if (statSize > readMaxBytes) {
-            return pageError('dataset_too_large', `Dataset is ${Math.round(statSize / 1024 / 1024)} MB — beyond the ${Math.round(readMaxBytes / 1024 / 1024)} MB paging cap. Download it instead.`);
-        }
-        const data = await storage.objects.read(repoPath, status.hash);
-        let pages = null;
-        try {
-            pages = openBeast2PagesFor(typeValue)(data);
-        }
-        catch {
-            pages = null; // No index/footer — every existing whole-value blob.
         }
+        const windowBlob = await segments.span(from, to);
+        const pages = openBeast2PagesFor(typeValue)(windowBlob);
         let windowValue;
-        let totalElements;
         let pageOffset;
         let pageCount;
         if (segmentMode) {
-            if (!pages) {
-                return pageError('dataset_not_segmented', 'Blob carries no segment index — address it with offset/limit instead');
-            }
-            const seg = window.segment;
-            if (seg >= pages.segmentCount) {
-                return pageError('bad_request', `segment ${seg} out of range (${pages.segmentCount} segments)`);
-            }
             try {
-                windowValue = pages.segment(seg);
+                windowValue = pages.segment(0);
             }
             catch (err) {
                 return pageError('dataset_not_segmented', err instanceof Error ? err.message : String(err));
             }
-            totalElements = pages.elementCount;
-            pageCount = pages.counts[seg];
-            pageOffset = 0;
-            for (let i = 0; i < seg; i++)
-                pageOffset += pages.counts[i];
-        }
-        else if (pages !== null && pages.selfContained) {
-            // The indexed path, every collection kind: Array windows address
-            // stream order, Set/Dict windows the canonical key order (fences are
-            // verified on first access; a non-canonical blob is corrupt).
-            totalElements = pages.elementCount;
-            const limit = effectiveLimit(totalElements);
+            pageCount = segments.counts[from];
+            pageOffset = from === 0 ? 0 : cumulative[from - 1];
+        }
+        else {
+            const base = from > 0 ? cumulative[from - 1] : 0;
             try {
-                windowValue = pages.slice(offset, limit);
+                windowValue = pages.slice(to > from ? offset - base : 0, limit);
             }
             catch (err) {
                 return pageError('dataset_not_canonical', err instanceof Error ? err.message : String(err));
@@ -369,18 +345,11 @@ export async function getDatasetPage(storage, repoPath, workspace, treePath, win
                 : windowValue.size;
             pageOffset = offset;
         }
-        else {
-            // Collection datasets are stored segmented + indexed by every writer
-            // at every size; a blob without a pageable index predates that
-            // contract (or was injected raw) and is refused rather than
-            // whole-decoded — re-writing the dataset produces the indexed form.
-            return pageError('dataset_not_indexed', 'Dataset blob carries no pageable segment index — re-write the dataset (re-run the producing task, or set the value again) to store it in the indexed form.');
-        }
         const body = encodeBeast2For(typeValue)(windowValue);
         return new Response(body, {
             status: 200,
             headers: {
-                ...pageHeaders(totalElements, pages ? pages.segmentCount : 0, pageOffset, pageCount),
+                ...pageHeaders(pageOffset, pageCount),
                 'Content-Length': String(body.byteLength),
             },
         });
@@ -389,6 +358,154 @@ export async function getDatasetPage(storage, repoPath, workspace, treePath, win
         return sendJsonError(err);
     }
 }
+/**
+ * One window of a record read through one of its secondary indexes.
+ *
+ * @remarks
+ * The window comes from the index's OWN segments — the same fence bisect and
+ * window slice a primary page uses, over the index manifest — so with a
+ * covering projection the window IS the answer and the primary is never
+ * touched. `join` fills the rows: the window's primary keys are bucketed by
+ * owning primary segment, each distinct segment read once, and the rows
+ * projected out, so a page whose entries share an index key usually costs a
+ * handful of reads rather than one per row.
+ */
+async function indexPage(storage, repoPath, workspace, treePath, stateHash, window, byteBudget) {
+    const refPath = treePath.map((s) => s.value).join('/');
+    const resolved = await resolveRecordIndex(storage, repoPath, workspace, refPath, stateHash, window.index);
+    if (resolved === null) {
+        const declared = await recordIndexNames(storage, repoPath, stateHash);
+        return pageError('index_not_found', `Dataset '${refPath}' has no index '${window.index}'`
+            + (declared.length > 0 ? ` — it has ${declared.join(', ')}` : ' — it declares none'), 404);
+    }
+    const offset = window.offset ?? 0;
+    const requestedLimit = window.limit ?? PAGE_DEFAULT_LIMIT;
+    if (!Number.isInteger(offset) || offset < 0 || !Number.isInteger(requestedLimit) || requestedLimit < 1) {
+        return pageError('bad_request', `offset must be a non-negative integer and limit a positive integer, got offset=${offset} limit=${requestedLimit}`);
+    }
+    const index = await cachedSegments(storage, repoPath, resolved.manifest, (await storage.objects.stat(repoPath, resolved.manifest)).size);
+    const totalElements = index.elementCount;
+    // A joined page carries whole rows, so its budget is the PRIMARY's average
+    // row size, not the index entry's — a covering page of the same limit is far
+    // smaller and is clamped on its own terms.
+    const primary = window.join === true
+        ? await cachedSegments(storage, repoPath, resolved.primary, (await storage.objects.stat(repoPath, resolved.primary)).size)
+        : null;
+    const perRow = (primary !== null && primary.elementCount > 0 ? primary.bytes / primary.elementCount : 0)
+        + (totalElements > 0 ? index.bytes / totalElements : 1);
+    const limit = Math.max(1, Math.min(requestedLimit, PAGE_MAX_LIMIT, Math.max(1, Math.floor(byteBudget / Math.max(1, perRow)))));
+    // The index window, as the index's own collection.
+    let from = 0;
+    let to = 0;
+    if (offset < totalElements) {
+        while (index.cumulative[from] <= offset)
+            from++;
+        const lastRow = Math.min(offset + limit, totalElements) - 1;
+        to = from;
+        while (index.cumulative[to] <= lastRow)
+            to++;
+        to++;
+    }
+    const base = from > 0 ? index.cumulative[from - 1] : 0;
+    const slice = openBeast2PagesFor(resolved.collectionType)(await index.span(from, to))
+        .slice(to > from ? offset - base : 0, limit);
+    // The rows, if asked for. An index's order is not the record's, so a page's
+    // primary keys scatter across the record: they are grouped by the segment
+    // that holds them, and each segment is decoded once and dropped before the
+    // next. A page holds one decoded segment at a time however widely its keys
+    // scatter, and reads at most one segment per row it returns.
+    const entries = [...slice];
+    const rows = entries.map(() => none);
+    if (primary !== null) {
+        const bySegment = new Map();
+        for (let i = 0; i < entries.length; i++) {
+            const segment = await primary.segmentFor(entries[i][0].k);
+            const held = bySegment.get(segment);
+            if (held === undefined)
+                bySegment.set(segment, [i]);
+            else
+                held.push(i);
+        }
+        const decodeSegment = openBeast2PagesFor(primary.typeValue);
+        for (const [segment, indices] of bySegment) {
+            const decoded = decodeSegment(await primary.segment(segment)).segment(0);
+            for (const i of indices) {
+                const row = decoded.get(entries[i][0].k);
+                if (row !== undefined)
+                    rows[i] = some(row);
+            }
+        }
+    }
+    const windowValue = entries.map(([entry, value], i) => ({ ik: entry.ik, key: entry.k, value, row: rows[i] }));
+    const body = encodeBeast2For(resolved.windowType)(windowValue);
+    return new Response(body, {
+        status: 200,
+        headers: {
+            'Content-Type': BEAST2_CONTENT_TYPE,
+            'Cache-Control': window.hash !== undefined ? 'public, max-age=31536000, immutable' : 'no-store',
+            'X-Content-SHA256': stateHash,
+            'X-Total-Bytes': String(index.bytes),
+            'X-Total-Elements': String(totalElements),
+            'X-Total-Exactness': 'exact',
+            'X-Segment-Count': String(index.segmentCount),
+            'X-Page-Offset': String(offset),
+            'X-Page-Count': String(windowValue.length),
+            // The window is the index's shape, not the dataset's: a client decodes
+            // it as `Array<{ik, key, value, row}>`, never as the record's type.
+            'X-Window-Index': window.index,
+            ...(window.join === true && { 'X-Window-Joined': 'true' }),
+            'Content-Length': String(body.byteLength),
+        },
+    });
+}
+/**
+ * A key type's leaves, in declaration order, recursing into nested structs.
+ *
+ * @remarks
+ * Struct keys compare field by field in declaration order, so this flattening
+ * IS the key's sort order — which is what makes a bound on a leading prefix of
+ * it one contiguous row range. A scalar key flattens to one leaf with an empty
+ * path.
+ */
+function flattenKeyFields(keyType) {
+    if (keyType.type !== 'Struct')
+        return [{ path: [], type: keyType }];
+    const out = [];
+    for (const field of keyType.value) {
+        for (const leaf of flattenKeyFields(field.type)) {
+            out.push({ path: [field.name, ...leaf.path], type: leaf.type });
+        }
+    }
+    return out;
+}
+/** The value at a flattened field path. */
+function fieldAt(key, path) {
+    let value = key;
+    for (const segment of path)
+        value = value[segment];
+    return value;
+}
+/**
+ * A monotone "is this key at or past the bound?" predicate over a leading
+ * prefix of the flattened key.
+ *
+ * @remarks
+ * Monotone over the canonical key order because it reads only the leading
+ * fields, which are exactly the ones the order sorts by first — so one fence
+ * bisect finds its boundary row, and a pair of them bound one contiguous
+ * range.
+ */
+function boundPredicate(leaves, values) {
+    const comparators = leaves.slice(0, values.length).map((leaf) => compareFor(leaf.type));
+    return (key) => {
+        for (let i = 0; i < values.length; i++) {
+            const order = comparators[i](fieldAt(key, leaves[i].path), values[i]);
+            if (order !== 0)
+                return order > 0;
+        }
+        return true; // equal on the prefix: at the bound, so past it
+    };
+}
 /**
  * Locate a key (or string-prefix range) in a Set/Dict dataset by global
  * element row, without decoding the collection.
@@ -396,18 +513,17 @@ export async function getDatasetPage(storage, repoPath, workspace, treePath, win
  * Rows address the canonical East key order — the same row space
  * {@link getDatasetPage} element windows serve — so the result plugs
  * straight into the paged preview's scroll position. The search
- * binary-searches the blob's segment fences (each segment's first key,
- * probed and cached per content hash) with the key type's East comparator,
- * then decodes at most the one owning segment (exact key) or the two edge
- * segments (prefix range).
+ * binary-searches the manifest's segment fences (each segment's first key)
+ * with the key type's East comparator, then decodes at most the one owning
+ * segment (exact key) or the two edge segments (a range — which is what an
+ * exact key is through an index: every entry under it).
  *
  * Responds with JSON `{ found, row, count }`: `row` is the match's global
  * element index (for a prefix, the range's first row; for a miss, the
  * key's insertion row), `count` the number of matched rows. Hash-pinned
  * queries are immutable-cacheable exactly like page windows.
  */
-export async function findDatasetKey(storage, repoPath, workspace, treePath, query, limits) {
-    const readMaxBytes = limits?.readMaxBytes ?? PAGE_READ_MAX_BYTES_DEFAULT;
+export async function findDatasetKey(storage, repoPath, workspace, treePath, query) {
     try {
         if (treePath.length === 0) {
             return pageError('bad_request', 'Path required for key search');
@@ -422,9 +538,24 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
         if (query.hash !== undefined && query.hash !== status.hash) {
             return pageError('dataset_hash_mismatch', `Dataset content is ${status.hash}, not ${query.hash} — refetch status and retry`, 409, { 'X-Content-SHA256': status.hash });
         }
-        const typeValue = isVariant(status.datasetType)
+        // An index selector searches the index's own collection, whose rows the
+        // answer is in — the same row space an index page serves.
+        let typeValue = isVariant(status.datasetType)
             ? status.datasetType
             : toEastTypeValue(status.datasetType);
+        let searchHash = status.hash;
+        const indexed = query.index !== undefined;
+        if (query.index !== undefined) {
+            const refPath = treePath.map((seg) => seg.value).join('/');
+            const resolved = await resolveRecordIndex(storage, repoPath, workspace, refPath, status.hash, query.index);
+            if (resolved === null) {
+                const declared = await recordIndexNames(storage, repoPath, status.hash);
+                return pageError('index_not_found', `Dataset '${refPath}' has no index '${query.index}'`
+                    + (declared.length > 0 ? ` — it has ${declared.join(', ')}` : ' — it declares none'), 404);
+            }
+            typeValue = resolved.collectionType;
+            searchHash = resolved.manifest;
+        }
         const kind = typeValue.type;
         if (kind !== 'Set' && kind !== 'Dict') {
             return pageError('dataset_not_searchable', `Key search addresses Set or Dict datasets; this dataset holds ${kind}`);
@@ -432,16 +563,56 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
         const keyTypeValue = kind === 'Dict'
             ? typeValue.value.key
             : typeValue.value;
+        // Through an index, the key, prefix and fields forms address the index key
+        // `ik` — the key a client holds, as the record's signature names it — not
+        // the collection's `{ik, k}`, which pairs it with each row's own key. The
+        // collection sorts by `ik` first, so a predicate over it is monotone over
+        // the collection's order. The range form bounds the collection's
+        // flattened key, which begins inside `ik`.
+        const formKeyType = indexed
+            ? keyTypeValue.value.find((f) => f.name === 'ik').type
+            : keyTypeValue;
+        const formKey = indexed ? (k) => k.ik : (k) => k;
+        const keysNoun = indexed ? `the keys of index '${query.index}'` : "this dataset's keys";
         const fields = query.fields !== undefined && query.fields.length > 0 ? query.fields : undefined;
-        if ((query.key === undefined) === (query.prefix === undefined && fields === undefined)) {
-            return pageError('bad_request', 'Pass a key literal, or a prefix and/or leading fields');
-        }
-        const structMeta = keyTypeValue.type === 'Struct'
-            ? keyTypeValue.value
+        const ranged = (query.from !== undefined && query.from.length > 0) || (query.to !== undefined && query.to.length > 0);
+        const forms = [query.key !== undefined, query.prefix !== undefined || fields !== undefined, ranged];
+        if (forms.filter(Boolean).length !== 1) {
+            return pageError('bad_request', 'Pass a key literal, or a prefix and/or leading fields, or a from/to range — exactly one form');
+        }
+        // The range form's bounds are leading prefixes of the FLATTENED key, which
+        // is the order the collection is sorted in, so each bounds one contiguous
+        // row range.
+        const leaves = flattenKeyFields(keyTypeValue);
+        const parseBound = (label, literals) => {
+            if (literals === undefined || literals.length === 0)
+                return { values: [] };
+            if (literals.length > leaves.length) {
+                return pageError('bad_request', `${label} names ${literals.length} key fields, but the key flattens to ${leaves.length}: `
+                    + leaves.map((leaf) => leaf.path.join('.') || '(the key)').join(', '));
+            }
+            const values = [];
+            for (let i = 0; i < literals.length; i++) {
+                const parsed = parseFor(leaves[i].type)(literals[i]);
+                if (!parsed.success) {
+                    return pageError('key_parse_error', `${label} field '${leaves[i].path.join('.') || '(the key)'}': ${parsed.error}`);
+                }
+                values.push(parsed.value);
+            }
+            return { values };
+        };
+        const lowerBound = parseBound('from', query.from);
+        if (lowerBound instanceof Response)
+            return lowerBound;
+        const upperBound = parseBound('to', query.to);
+        if (upperBound instanceof Response)
+            return upperBound;
+        const structMeta = formKeyType.type === 'Struct'
+            ? formKeyType.value
             : null;
         if (fields !== undefined) {
             if (structMeta === null) {
-                return pageError('bad_request', `Leading-field search addresses Struct keys; this dataset's keys are ${keyTypeValue.type}`);
+                return pageError('bad_request', `Leading-field search addresses Struct keys; ${keysNoun} are ${formKeyType.type}`);
             }
             if (fields.length > structMeta.length) {
                 return pageError('bad_request', `Key has ${structMeta.length} fields, got ${fields.length}`);
@@ -461,106 +632,49 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
             const prefixIdx = fieldValues.length;
             const prefixFieldType = structMeta !== null
                 ? (prefixIdx < structMeta.length ? structMeta[prefixIdx].type : undefined)
-                : keyTypeValue;
+                : formKeyType;
             if (prefixFieldType === undefined) {
                 return pageError('bad_request', `All ${structMeta.length} key fields are exact — nothing left for a prefix`);
             }
             if (prefixFieldType.type !== 'String') {
                 return pageError('bad_request', structMeta !== null
                     ? `prefix continues key field '${structMeta[prefixIdx].name}', which is ${prefixFieldType.type}, not String`
-                    : `prefix search needs String keys; this dataset's keys are ${keyTypeValue.type}`);
+                    : `prefix search needs String keys; ${keysNoun} are ${formKeyType.type}`);
             }
         }
         let keyValue;
         if (query.key !== undefined) {
-            const parsed = parseFor(keyTypeValue)(query.key);
+            const parsed = parseFor(formKeyType)(query.key);
             if (!parsed.success) {
                 return pageError('key_parse_error', parsed.error);
             }
             keyValue = parsed.value;
         }
-        const cmp = compareFor(keyTypeValue);
-        const statSize = status.size ?? (await storage.objects.stat(repoPath, status.hash)).size;
-        const notIndexed = () => pageError('dataset_not_indexed', 'Dataset blob carries no pageable segment index — re-write the dataset (re-run the producing task, or set the value again) to store it in the indexed form.');
-        // Both data paths resolve to the same fence-search surface: the segment
-        // geometry plus a fence probe and a one-segment key iterator.
-        let segmentCount;
-        let elementCount;
-        let cumulative;
-        let fenceAt;
-        let keysAt;
-        const readRange = storage.objects.readRange?.bind(storage.objects);
-        if (readRange) {
-            // Ranged path: extents (and previously probed fences) are cached per
-            // content hash; each probe reads exactly one segment's frame bytes.
-            let cached;
-            try {
-                cached = await cachedRangedExtents(status.hash, statSize, (offset, length) => readRange(repoPath, status.hash, offset, length));
-            }
-            catch (err) {
-                const message = err instanceof Error ? err.message : String(err);
-                if (!/^(beast2 v5:|Data too short for Beast2|Invalid Beast2)/.test(message)) {
-                    throw err;
-                }
-                return notIndexed();
-            }
-            const { extents } = cached;
-            if (!extents.selfContained) {
-                return notIndexed();
-            }
-            segmentCount = extents.offsets.length;
-            elementCount = extents.elementCount;
-            cumulative = cached.cumulative;
-            const segmentBlob = async (i) => {
-                const start = extents.offsets[i];
-                const end = i + 1 < segmentCount ? extents.offsets[i + 1] : extents.segmentsEnd;
-                const frames = await readRange(repoPath, status.hash, start, end - start);
-                return carveBeast2Ranged(extents, frames, i, i + 1);
-            };
-            fenceAt = async (i) => {
-                if (cached.fences.has(i))
-                    return cached.fences.get(i);
-                const fence = openBeast2PagesFor(typeValue)(await segmentBlob(i)).fence(0);
-                cached.fences.set(i, fence);
-                return fence;
-            };
-            keysAt = async (i) => {
-                const segment = openBeast2PagesFor(typeValue)(await segmentBlob(i)).segment(0);
-                return kind === 'Set' ? segment : segment.keys();
-            };
-        }
-        else {
-            // Fallback path (no ranged reads): buffer the blob whole under the
-            // same cap as paging; fences are then bounded in-memory probes.
-            if (statSize > readMaxBytes) {
-                return pageError('dataset_too_large', `Dataset is ${Math.round(statSize / 1024 / 1024)} MB — beyond the ${Math.round(readMaxBytes / 1024 / 1024)} MB paging cap. Download it instead.`);
-            }
-            const data = await storage.objects.read(repoPath, status.hash);
-            let pages;
-            try {
-                pages = openBeast2PagesFor(typeValue)(data);
-            }
-            catch {
-                return notIndexed();
-            }
-            if (!pages.selfContained) {
-                return notIndexed();
-            }
-            segmentCount = pages.segmentCount;
-            elementCount = pages.elementCount;
-            const sums = new Array(pages.counts.length);
-            let running = 0;
-            for (let i = 0; i < pages.counts.length; i++) {
-                running += pages.counts[i];
-                sums[i] = running;
-            }
-            cumulative = sums;
-            fenceAt = (i) => Promise.resolve(pages.fence(i));
-            keysAt = (i) => {
-                const segment = pages.segment(i);
-                return Promise.resolve(kind === 'Set' ? segment : segment.keys());
-            };
-        }
+        const cmp = compareFor(formKeyType);
+        const objectSize = searchHash === status.hash
+            ? status.size ?? (await storage.objects.stat(repoPath, status.hash)).size
+            : (await storage.objects.stat(repoPath, searchHash)).size;
+        let segments;
+        try {
+            segments = await cachedSegments(storage, repoPath, searchHash, objectSize);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            if (!BLOB_REFUSAL.test(message))
+                throw err;
+            return pageError('dataset_not_indexed', message);
+        }
+        // The fence-search surface: the segment geometry, a fence probe, and a
+        // one-segment key iterator. A manifest answers every fence from its own
+        // entries, so a bisect over one reads no frames at all.
+        const segmentCount = segments.segmentCount;
+        const elementCount = segments.elementCount;
+        const cumulative = segments.cumulative;
+        const fenceAt = (i) => segments.fence(i);
+        const keysAt = async (i) => {
+            const segment = openBeast2PagesFor(typeValue)(await segments.segment(i)).segment(0);
+            return kind === 'Set' ? segment : segment.keys();
+        };
         // First row whose key satisfies a monotone predicate (false… then
         // true… over the canonical key order), plus that row's key. Fences
         // bound the candidate segment, so at most ONE segment decodes: when the
@@ -597,11 +711,33 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
         let found;
         let row;
         let count;
-        if (query.key !== undefined) {
-            const at = await locate((k) => cmp(k, keyValue) >= 0);
-            found = at.hasKey && cmp(at.key, keyValue) === 0;
-            row = at.row;
-            count = found ? 1 : 0;
+        if (ranged) {
+            // Two bisects, one contiguous range: `[from, to)` on the flattened
+            // prefix. An absent bound is the collection's own end.
+            const lower = query.from === undefined || query.from.length === 0
+                ? { row: 0 }
+                : await locate(boundPredicate(leaves, lowerBound.values));
+            const upper = query.to === undefined || query.to.length === 0
+                ? { row: elementCount }
+                : await locate(boundPredicate(leaves, upperBound.values));
+            row = lower.row;
+            count = Math.max(0, upper.row - lower.row);
+            found = count > 0;
+        }
+        else if (query.key !== undefined) {
+            const at = await locate((k) => cmp(formKey(k), keyValue) >= 0);
+            if (indexed) {
+                // An index holds an entry per row under a key, so a key is a run.
+                const past = await locate((k) => cmp(formKey(k), keyValue) > 0);
+                row = at.row;
+                count = past.row - at.row;
+                found = count > 0;
+            }
+            else {
+                found = at.hasKey && cmp(at.key, keyValue) === 0;
+                row = at.row;
+                count = found ? 1 : 0;
+            }
         }
         else {
             // Range query — a String prefix, or a struct key's exact leading
@@ -609,15 +745,16 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
             // depend only on the leading field tuple, and struct keys compare
             // field-by-field in declaration order, so they are monotone over the
             // canonical key order; prefix-extending strings form one contiguous
-            // interval in East (code-unit) order.
+            // interval in East (code-point) order.
             let lowerPred;
             let upperPred;
             const prefix = query.prefix;
             if (structMeta !== null) {
                 const fieldCmps = structMeta.map((f) => compareFor(f.type));
                 const lead = (k) => {
+                    const key = formKey(k);
                     for (let j = 0; j < fieldValues.length; j++) {
-                        const c = fieldCmps[j](k[structMeta[j].name], fieldValues[j]);
+                        const c = fieldCmps[j](key[structMeta[j].name], fieldValues[j]);
                         if (c !== 0)
                             return c;
                     }
@@ -633,13 +770,13 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
                     const prefixCmp = fieldCmps[prefixIdx];
                     lowerPred = (k) => {
                         const c = lead(k);
-                        return c !== 0 ? c > 0 : prefixCmp(k[prefixName], prefix) >= 0;
+                        return c !== 0 ? c > 0 : prefixCmp(formKey(k)[prefixName], prefix) >= 0;
                     };
                     upperPred = (k) => {
                         const c = lead(k);
                         if (c !== 0)
                             return c > 0;
-                        const field = k[prefixName];
+                        const field = formKey(k)[prefixName];
                         return prefixCmp(field, prefix) > 0 && !field.startsWith(prefix);
                     };
                 }
@@ -647,8 +784,8 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
             else {
                 // Scalar String keys — validation guarantees the prefix is set here.
                 const scalarPrefix = prefix;
-                lowerPred = (k) => cmp(k, scalarPrefix) >= 0;
-                upperPred = (k) => cmp(k, scalarPrefix) > 0 && !k.startsWith(scalarPrefix);
+                lowerPred = (k) => cmp(formKey(k), scalarPrefix) >= 0;
+                upperPred = (k) => cmp(formKey(k), scalarPrefix) > 0 && !formKey(k).startsWith(scalarPrefix);
             }
             const lower = await locate(lowerPred);
             const upper = await locate(upperPred);
@@ -672,16 +809,20 @@ export async function findDatasetKey(storage, repoPath, workspace, treePath, que
     }
 }
 /**
- * Set dataset value from raw BEAST2 bytes.
+ * Set a dataset value from raw BEAST2 bytes, read as they arrive.
+ *
+ * @remarks
+ * The body's type is read from its head and checked against the dataset's. A
+ * collection then goes into the store through its door a segment of the body
+ * at a time; any other value is collected and decoded whole, so the handler
+ * holds it.
  */
 export async function setDataset(storage, repoPath, workspace, treePath, body) {
     try {
         if (treePath.length === 0) {
             return sendError(NullType, errorToVariant(new Error('Path required for set')));
         }
-        // Body is raw BEAST2 - decode to get type and value
-        const { type, value } = decodeBeast2(body);
-        await workspaceSetDataset(storage, repoPath, workspace, treePath, value, type);
+        await workspaceSetDatasetBytes(storage, repoPath, workspace, treePath, body);
         return sendSuccess(NullType, null);
     }
     catch (err) {
@@ -736,7 +877,12 @@ export async function getDatasetStatus(storage, repoPath, workspace, treePath) {
             type: typeValue,
             refType: result.refType,
             hash: result.hash ? some(result.hash) : none,
-            size: result.size !== null ? some(BigInt(result.size)) : none,
+            // What the dataset costs in the store. For a collection held as a
+            // segment manifest that is the segments plus the manifest; the manifest
+            // object alone is a few dozen bytes per segment whatever the value
+            // weighs, and reporting that would say a 10 MiB dataset is 12 KiB.
+            size: result.storedBytes != null ? some(BigInt(result.storedBytes))
+                : result.size !== null ? some(BigInt(result.size)) : none,
             segments: result.segments != null ? some(BigInt(result.segments)) : none,
             rows: result.rows != null ? some(BigInt(result.rows)) : none,
         };