@elaraai/e3-api-server 1.0.79 → 1.0.80

package/dist/src/handlers/datasets.spec.js

@@ -4,11 +4,12 @@
  */
 import { describe, it } from 'node:test';
 import assert from 'node:assert/strict';
-import { ArrayType, DictType, IntegerType, SetType, StringType, StructType, decodeBeast2For, encodeBeast2For, encodeBeast2PagedFor, equalFor, none, toEastTypeValue, variant, } from '@elaraai/east';
-import { BEAST2_CONTENT_TYPE, computeHash, InMemoryTransferBackend } from '@elaraai/e3-core';
-import { InMemoryStorage } from '@elaraai/e3-core/test';
-import { PackageObjectType, WorkspaceStateType } from '@elaraai/e3-types';
-import { findDatasetKey, getDataset, getDatasetPage } from './datasets.js';
+import { ArrayType, DictType, IntegerType, NullType, RUN_MAX_BYTES, SetType, SortedMap, StringType, StructType, compareFor, decodeBeast2For, encodeBeast2For, encodeBeast2PagedFor, equalFor, none, some, toEastTypeValue, variant, } from '@elaraai/east';
+import { computeHash, datasetWrite, InMemoryTransferBackend, writeRecordState } from '@elaraai/e3-core';
+import { InMemoryStorage, encodeInSegmentsOf, storeSegmentsOf } from '@elaraai/e3-core/test';
+import { BEAST2_CONTENT_TYPE, PackageObjectType, RecordIndexObjectType, WorkspaceRecordType, decodeCollectionManifest, indexCollectionType, indexWindowType, } from '@elaraai/e3-types';
+import { ResponseType } from '../types.js';
+import { findDatasetKey, getDataset, getDatasetPage, setDataset } from './datasets.js';
 /**
  * ~`byteLength` bytes of high-entropy ASCII, deterministic across runs.
  *
@@ -139,6 +140,66 @@ describe('getDataset', () => {
         assert.equal(response.headers.get('Content-Type'), BEAST2_CONTENT_TYPE);
         assert.equal(response.headers.get('X-Content-SHA256'), hash);
     });
+    it('streams a manifest-backed collection, reading each segment as the client takes it', async () => {
+        // A collection is many objects, so there is no one object to redirect a
+        // download to — and splicing it into a buffer first held the whole value
+        // (twice) in the server for every download.
+        const storage = new InMemoryStorage();
+        await storage.repos.create(REPO);
+        const type = DictType(StringType, IntegerType);
+        const value = new Map(Array.from({ length: 20_000 }, (_, i) => [`k${String(i).padStart(6, '0')}`, BigInt(i)]));
+        const hash = await datasetWrite(storage, REPO, value, type);
+        const segments = decodeCollectionManifest(await storage.objects.read(REPO, hash)).entries.map((entry) => entry.hash);
+        assert.ok(segments.length > 3, `the value spans segments, got ${segments.length}`);
+        await storage.datasets.write(REPO, WS, 'inputs/lookup', variant('value', { hash, versions: new Map() }));
+        const objects = storage.objects;
+        const read = objects.read.bind(objects);
+        let segmentReads = 0;
+        objects.read = (repo, object) => {
+            if (segments.includes(object))
+                segmentReads++;
+            return read(repo, object);
+        };
+        try {
+            const response = await getDataset(storage, REPO, WS, [variant('field', 'inputs'), variant('field', 'lookup')]);
+            assert.equal(response.status, 200);
+            assert.equal(response.headers.get('Content-Type'), BEAST2_CONTENT_TYPE);
+            assert.equal(segmentReads, 0, 'nothing is read before the client takes the body');
+            const decoded = decodeBeast2For(type)(new Uint8Array(await response.arrayBuffer()));
+            assert.equal(decoded.size, 20_000);
+            assert.equal(decoded.get('k019999'), 19999n);
+            assert.equal(segmentReads, segments.length, 'each segment is read once, as its bytes are sent');
+        }
+        finally {
+            objects.read = read;
+        }
+    });
+    it('answers a caller asking for segments with the manifest to download, and any other value as before', async () => {
+        const storage = new InMemoryStorage();
+        await storage.repos.create(REPO);
+        const type = DictType(StringType, IntegerType);
+        const hash = await datasetWrite(storage, REPO, new Map([['a', 1n], ['b', 2n]]), type);
+        await storage.datasets.write(REPO, WS, 'inputs/lookup', variant('value', { hash, versions: new Map() }));
+        const small = await storage.objects.write(REPO, encodeBeast2For(StringType)('small value'));
+        await storage.datasets.write(REPO, WS, 'inputs/small', variant('value', { hash: small, versions: new Map() }));
+        const response = await getDataset(storage, REPO, WS, [variant('field', 'inputs'), variant('field', 'lookup')], undefined, undefined, undefined, true);
+        assert.equal(response.status, 200);
+        assert.equal(response.headers.get('Content-Type'), 'application/json');
+        assert.equal(response.headers.get('X-Content-SHA256'), hash);
+        assert.deepEqual(await response.json(), { manifest: hash });
+        const inline = await getDataset(storage, REPO, WS, [variant('field', 'inputs'), variant('field', 'small')], undefined, undefined, undefined, true);
+        assert.equal(inline.headers.get('Content-Type'), BEAST2_CONTENT_TYPE);
+        assert.equal(computeHash(new Uint8Array(await inline.arrayBuffer())), small);
+    });
+    it('names an indexed record\'s primary as the manifest to download', async () => {
+        const storage = new InMemoryStorage();
+        const { rows } = await seedIndexedRecord(storage, 500);
+        const state = await storage.datasets.read(REPO, WS, 'records/plans');
+        assert.ok(state?.type === 'value');
+        const response = await getDataset(storage, REPO, WS, plansPath, undefined, undefined, undefined, true);
+        assert.equal(response.headers.get('X-Content-SHA256'), state.value.hash, 'the dataset is the record\'s state');
+        assert.deepEqual(await response.json(), { manifest: await datasetWrite(storage, REPO, rows, PlansType) });
+    });
     it('returns 404 JSON error for null dataset', async () => {
         const storage = new InMemoryStorage();
         await storage.repos.create(REPO);
@@ -179,16 +240,18 @@ const rowsPath = [variant('field', 'inputs'), variant('field', 'rows')];
 function makeRows(n) {
     return Array.from({ length: n }, (_, i) => ({ id: BigInt(i), name: `row-${i % 97}` }));
 }
-/** Seeds a deployed workspace whose `.inputs.<name>` dataset holds `blob`.
- *  Reseeding the same storage replaces the workspace's package wholesale. */
-async function seedDataset(storage, blob, name, type) {
+/** Seeds a deployed workspace whose `.inputs.<name>` dataset holds `blob`'s
+ *  segments, as they stand, under a manifest — or, `bare`, the blob itself, as
+ *  an older e3 stored a collection. Reseeding the same storage replaces the
+ *  workspace's package wholesale. */
+async function seedDataset(storage, blob, name, type, options = {}) {
     try {
         await storage.repos.create(REPO);
     }
     catch {
         // Already created by an earlier seed into this storage.
     }
-    const hash = await storage.objects.write(REPO, blob);
+    const hash = options.bare === true ? await storage.objects.write(REPO, blob) : await storeSegmentsOf(storage, REPO, blob);
     const structure = variant('struct', new Map([
         ['inputs', variant('struct', new Map([
                 [name, variant('value', { type: toEastTypeValue(type), writable: true })],
@@ -200,9 +263,9 @@ async function seedDataset(storage, blob, name, type) {
         functions: new Map(),
         records: new Map(), sources: new Map(),
     }));
-    await storage.refs.workspaceWrite(REPO, WS, encodeBeast2For(WorkspaceStateType)({
+    await storage.refs.workspaceWrite(REPO, WS, encodeBeast2For(WorkspaceRecordType)(some({
         packageName: 'pages', packageVersion: '1.0.0', packageHash: pkgHash, deployedAt: new Date(0), currentRunId: none,
-    }));
+    })));
     await storage.datasets.write(REPO, WS, `inputs/${name}`, variant('value', { hash, versions: new Map() }));
     return hash;
 }
@@ -210,83 +273,113 @@ async function seedDataset(storage, blob, name, type) {
 async function seedRowsDataset(storage, blob) {
     return seedDataset(storage, blob, 'rows', RowsType);
 }
-/** Counts whole-object and ranged reads of `watchedHash` (the dataset blob —
- *  workspace/package object reads are expected and not counted). */
-function spyObjectReads(storage, watchedHash) {
+/** Counts whole-object reads of the watched objects — a dataset's segments, or
+ *  a bare blob; workspace/package object reads are expected and not counted. */
+function spyObjectReads(storage, watched) {
     const objects = storage.objects;
+    const hashes = new Set(watched);
     let whole = 0;
-    let ranged = 0;
-    let calls = 0;
     const origRead = objects.read.bind(objects);
-    const origRange = objects.readRange.bind(objects);
     objects.read = (repo, hash) => {
-        if (hash === watchedHash)
+        if (hashes.has(hash))
             whole++;
         return origRead(repo, hash);
     };
-    objects.readRange = (repo, hash, offset, length) => {
-        if (hash === watchedHash) {
-            ranged += length;
-            calls++;
-        }
-        return origRange(repo, hash, offset, length);
-    };
-    return { wholeReads: () => whole, rangedBytes: () => ranged, rangedCalls: () => calls };
+    return { wholeReads: () => whole };
 }
-describe('getDatasetPage (ranged reads)', () => {
-    it('serves element windows through ranged reads without buffering the blob', async () => {
-        const storage = new InMemoryStorage();
-        // Incompressible per-row names so the deflated blob comfortably exceeds
-        // the 64 KiB tail probe — otherwise the probe alone covers the blob and
-        // the read-volume assertion below measures nothing.
-        const rows = Array.from({ length: 8000 }, (_, i) => {
-            let seed = (Math.imul(i, 2654435761) + 1) >>> 0;
-            const chars = new Array(48);
-            for (let j = 0; j < 48; j++) {
-                seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0;
-                chars[j] = String.fromCharCode(33 + ((seed >>> 16) % 94));
-            }
-            return { id: BigInt(i), name: chars.join('') };
-        });
-        const blob = encodeBeast2PagedFor(RowsType, { batchSize: 100 })(rows);
-        assert.ok(blob.byteLength > 128 * 1024, `fixture (${blob.byteLength} bytes) must dwarf the 64 KiB tail probe`);
-        const hash = await seedRowsDataset(storage, blob);
-        const spy = spyObjectReads(storage, hash);
+/** The segment objects a stored collection's manifest names. */
+async function segmentsOf(storage, hash) {
+    return decodeCollectionManifest(await storage.objects.read(REPO, hash)).entries.map((entry) => entry.hash);
+}
+describe('setDataset (an upload, read as it arrives)', () => {
+    /** The body in pieces, as a request's arrives. */
+    function* pieces(bytes) {
+        for (let at = 0; at < bytes.length; at += 4096)
+            yield bytes.subarray(at, at + 4096);
+    }
+    /** The dataset's current hash. */
+    async function rowsHash(storage) {
+        const ref = await storage.datasets.read(REPO, WS, 'inputs/rows');
+        return ref?.type === 'value' ? ref.value.hash : null;
+    }
+    it('stores a collection upload as the manifest the value path writes, whatever layout the client sent', async () => {
+        const storage = new InMemoryStorage();
+        await seedRowsDataset(storage, encodeBeast2PagedFor(RowsType)(makeRows(10)));
+        const rows = makeRows(5_000);
+        const expected = await datasetWrite(storage, REPO, rows, RowsType);
+        const layouts = [
+            ['batched by the client', encodeInSegmentsOf(RowsType, 100)(rows)],
+            ['encoded whole', encodeBeast2For(RowsType)(rows)],
+        ];
+        for (const [layout, bytes] of layouts) {
+            const response = await setDataset(storage, REPO, WS, rowsPath, pieces(bytes));
+            const answer = decodeBeast2For(ResponseType(NullType))(new Uint8Array(await response.arrayBuffer()));
+            assert.equal(answer.type, 'success', `${layout}: ${JSON.stringify(answer.value)}`);
+            assert.equal(await rowsHash(storage), expected, layout);
+        }
+    });
+    it('refuses an upload of another type, and one holding a segment larger than a collection is read in, storing nothing', async () => {
+        const storage = new InMemoryStorage();
+        const seeded = await seedRowsDataset(storage, encodeBeast2PagedFor(RowsType)(makeRows(10)));
+        const objects = await storage.objects.count(REPO);
+        const drifted = await setDataset(storage, REPO, WS, rowsPath, pieces(encodeBeast2PagedFor(ArrayType(StringType))(['a'])));
+        const refusal = decodeBeast2For(ResponseType(NullType))(new Uint8Array(await drifted.arrayBuffer()));
+        assert.equal(refusal.type === 'error' ? refusal.value.type : refusal.type, 'dataset_type_mismatch');
+        // A whole-value encode is one frame; this one declares more logical bytes
+        // than the limit, and none of them follow.
+        const head = encodeBeast2For(RowsType, { codec: 'none' })([]);
+        const frameHeader = new Uint8Array(21);
+        let at = 0;
+        for (const n of [0, RUN_MAX_BYTES + 1, RUN_MAX_BYTES + 1]) {
+            let v = n;
+            for (; v >= 0x80; v = Math.floor(v / 128))
+                frameHeader[at++] = (v & 0x7f) | 0x80;
+            frameHeader[at++] = v;
+        }
+        const oversized = await setDataset(storage, REPO, WS, rowsPath, [head.subarray(0, head.length - 5), frameHeader.subarray(0, at)]);
+        const cap = decodeBeast2For(ResponseType(NullType))(new Uint8Array(await oversized.arrayBuffer()));
+        assert.ok(cap.type === 'error' && cap.value.type === 'internal'
+            && /more than the 67108864 a collection is read in at once/.test(cap.value.value.message), JSON.stringify(cap));
+        assert.equal(await rowsHash(storage), seeded, 'the dataset keeps its value');
+        assert.equal(await storage.objects.count(REPO), objects, 'nothing was stored');
+    });
+});
+describe('getDatasetPage (segment reads)', () => {
+    it('serves element windows from the segments they touch, and no others', async () => {
+        const storage = new InMemoryStorage();
+        const rows = makeRows(8000);
+        const hash = await seedRowsDataset(storage, encodeInSegmentsOf(RowsType, 100)(rows));
+        const spy = spyObjectReads(storage, await segmentsOf(storage, hash));
         const response = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 900, limit: 200 });
         assert.equal(response.status, 200);
         assert.equal(response.headers.get('X-Total-Elements'), '8000');
-        assert.equal(response.headers.get('X-Total-Bytes'), String(blob.byteLength));
         assert.equal(response.headers.get('X-Page-Offset'), '900');
         assert.equal(response.headers.get('X-Page-Count'), '200');
         const page = decodeBeast2For(RowsType)(new Uint8Array(await response.arrayBuffer()));
         assert.ok(equalFor(RowsType)(page, rows.slice(900, 1100)), 'window equals the expected slice');
-        assert.equal(spy.wholeReads(), 0, 'the ranged path must never read the blob whole');
-        assert.ok(spy.rangedBytes() < blob.byteLength / 2, `ranged reads (${spy.rangedBytes()} bytes) must stay well under the blob (${blob.byteLength} bytes)`);
-        // A second window on the same hash reuses the cached extents: only the
-        // window's own frame bytes are read.
-        const before = spy.rangedBytes();
+        assert.equal(spy.wholeReads(), 2, 'rows 900..1099 are segments 9 and 10, and only they are read');
+        // A second window on the same hash reads only its own segment.
         const deep = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 7500, limit: 100 });
         assert.equal(deep.status, 200);
         const deepPage = decodeBeast2For(RowsType)(new Uint8Array(await deep.arrayBuffer()));
         assert.ok(equalFor(RowsType)(deepPage, rows.slice(7500, 7600)));
-        assert.ok(spy.rangedBytes() - before < 64 * 1024, 'a cached-extents window reads only its own segments');
+        assert.equal(spy.wholeReads(), 3, 'the second window reads one segment');
     });
-    it('pages blobs beyond the fallback cap — no size limit on the ranged path', async () => {
+    it('clamps a window that runs past the end to the rows there are', async () => {
         const storage = new InMemoryStorage();
         const rows = makeRows(2500);
-        const blob = encodeBeast2PagedFor(RowsType, { batchSize: 100 })(rows);
+        const blob = encodeInSegmentsOf(RowsType, 100)(rows);
         await seedRowsDataset(storage, blob);
-        // A cap far below the blob size: the ranged path must ignore it.
-        const response = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 2400, limit: 1000 }, { readMaxBytes: 1024 });
+        const response = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 2400, limit: 1000 });
         assert.equal(response.status, 200);
-        assert.equal(response.headers.get('X-Page-Count'), '100', 'tail clamp still applies');
+        assert.equal(response.headers.get('X-Page-Count'), '100', 'the tail clamps the window');
         const page = decodeBeast2For(RowsType)(new Uint8Array(await response.arrayBuffer()));
         assert.ok(equalFor(RowsType)(page, rows.slice(2400)));
     });
     it('serves segment windows and empty past-the-end windows', async () => {
         const storage = new InMemoryStorage();
         const rows = makeRows(2500);
-        const blob = encodeBeast2PagedFor(RowsType, { batchSize: 100 })(rows);
+        const blob = encodeInSegmentsOf(RowsType, 100)(rows);
         await seedRowsDataset(storage, blob);
         const seg = await getDatasetPage(storage, REPO, WS, rowsPath, { segment: 1 });
         assert.equal(seg.status, 200);
@@ -300,52 +393,35 @@ describe('getDatasetPage (ranged reads)', () => {
         assert.equal(past.headers.get('X-Page-Offset'), '5000');
         assert.deepEqual(decodeBeast2For(RowsType)(new Uint8Array(await past.arrayBuffer())), []);
     });
-    it('refuses index-less blobs without buffering them', async () => {
+    it('refuses a collection an older e3 stored as one blob, naming the fix, without reading it whole', async () => {
         const storage = new InMemoryStorage();
-        // Whole-value v5 encode: no index — predates the stored-segmented contract.
         const blob = encodeBeast2For(RowsType)(makeRows(50));
-        const hash = await seedRowsDataset(storage, blob);
-        const spy = spyObjectReads(storage, hash);
+        const hash = await seedDataset(storage, blob, 'rows', RowsType, { bare: true });
+        const spy = spyObjectReads(storage, [hash]);
         const response = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 0, limit: 10 });
         assert.equal(response.status, 400);
         const body = await response.json();
         assert.equal(body.error.type, 'dataset_not_indexed');
-        assert.equal(spy.wholeReads(), 0, 'the refusal must come from the tail probe, not a whole read');
+        assert.equal(body.error.message, `the collection ${hash} is stored as one blob: ` +
+            'an older e3 wrote this repository — re-create it: deploy again and import its data again');
+        assert.equal(spy.wholeReads(), 0, 'the refusal comes from the head probe, not a whole read');
     });
     it('surfaces storage failures as errors, never as dataset_not_indexed', async () => {
         const storage = new InMemoryStorage();
         // Distinct row count → distinct content hash, so the module-level
-        // extents cache cannot mask the injected failure.
+        // cache of opened datasets cannot mask the injected failure.
         const rows = makeRows(600);
-        const blob = encodeBeast2PagedFor(RowsType, { batchSize: 100 })(rows);
+        const blob = encodeInSegmentsOf(RowsType, 100)(rows);
         await seedRowsDataset(storage, blob);
-        // The blob IS indexed; the BACKEND fails at read time. The old bare
-        // catch would answer dataset_not_indexed ("re-write the dataset") for
-        // what is really an I/O failure.
+        // The dataset is a manifest; the BACKEND fails at read time, and an I/O
+        // failure must not read as a refusal of the layout.
         storage.objects.readRange = () => Promise.reject(new Error('injected storage failure'));
         const response = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 0, limit: 10 });
         assert.notEqual(response.status, 200);
         const body = await response.json();
-        assert.notEqual(body.error.type, 'dataset_not_indexed', 'an I/O failure must not masquerade as a re-write suggestion');
+        assert.notEqual(body.error.type, 'dataset_not_indexed', 'an I/O failure must not masquerade as a refusal of the layout');
         assert.match(body.error.message, /injected storage failure/);
     });
-    it('falls back to whole reads — and keeps the cap — when the backend has no ranged reads', async () => {
-        const storage = new InMemoryStorage();
-        const rows = makeRows(500);
-        const blob = encodeBeast2PagedFor(RowsType, { batchSize: 100 })(rows);
-        await seedRowsDataset(storage, blob);
-        // Simulate a backend without ranged reads (e.g. a store that has not
-        // implemented the optional method yet).
-        storage.objects.readRange = undefined;
-        const capped = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 0, limit: 10 }, { readMaxBytes: 1024 });
-        assert.equal(capped.status, 400);
-        const body = await capped.json();
-        assert.equal(body.error.type, 'dataset_too_large');
-        const served = await getDatasetPage(storage, REPO, WS, rowsPath, { offset: 90, limit: 20 });
-        assert.equal(served.status, 200);
-        const page = decodeBeast2For(RowsType)(new Uint8Array(await served.arrayBuffer()));
-        assert.ok(equalFor(RowsType)(page, rows.slice(90, 110)), 'the fallback still pages correctly');
-    });
 });
 const LookupType = DictType(StringType, IntegerType);
 const IntLookupType = DictType(IntegerType, StringType);
@@ -360,11 +436,11 @@ async function findJson(response) {
     return await response.json();
 }
 describe('findDatasetKey', () => {
-    // batchSize 97 puts segment boundaries at 97, 194, … — deliberately off
-    // the decimal key grid, so prefix ranges span segment boundaries.
+    // Segments of 97 put boundaries at 97, 194, … — deliberately off the
+    // decimal key grid, so prefix ranges span segment boundaries.
     it('locates exact keys by fence bisect, including segment-fence rows and misses', async () => {
         const storage = new InMemoryStorage();
-        const blob = encodeBeast2PagedFor(LookupType, { batchSize: 97 })(lookupOf(2500));
+        const blob = encodeInSegmentsOf(LookupType, 97)(lookupOf(2500));
         await seedDataset(storage, blob, 'lookup', LookupType);
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, lookupPath, { key: '"k0150"' })), { found: true, row: 150, count: 1 });
         // A key that IS a segment fence — row 97 opens segment 1.
@@ -377,7 +453,7 @@ describe('findDatasetKey', () => {
     });
     it('prefix ranges are contiguous rows, spanning segment boundaries', async () => {
         const storage = new InMemoryStorage();
-        const blob = encodeBeast2PagedFor(LookupType, { batchSize: 97 })(lookupOf(2500));
+        const blob = encodeInSegmentsOf(LookupType, 97)(lookupOf(2500));
         await seedDataset(storage, blob, 'lookup', LookupType);
         // k01__ covers rows 100..199 — across the boundary at row 194.
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, lookupPath, { prefix: 'k01' })), { found: true, row: 100, count: 100 });
@@ -387,25 +463,22 @@ describe('findDatasetKey', () => {
     });
     it('decodes at most the touched segments: one for exact, two for a spanning prefix', async () => {
         const storage = new InMemoryStorage();
-        const blob = encodeBeast2PagedFor(LookupType, { batchSize: 97 })(lookupOf(2500));
+        const blob = encodeInSegmentsOf(LookupType, 97)(lookupOf(2500));
         const hash = await seedDataset(storage, blob, 'lookup', LookupType);
-        // Warm the per-hash extents + fence caches, then count reads: each
-        // segment decode is exactly one ranged read of that segment's frames.
+        // Warm the cache of opened datasets, then count reads: the fences come
+        // with the manifest, and each segment decode is one read of its object.
         await findDatasetKey(storage, REPO, WS, lookupPath, { key: '"k0150"' });
         await findDatasetKey(storage, REPO, WS, lookupPath, { prefix: 'k01' });
-        const spy = spyObjectReads(storage, hash);
+        const spy = spyObjectReads(storage, await segmentsOf(storage, hash));
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, lookupPath, { key: '"k0150"' })), { found: true, row: 150, count: 1 });
-        assert.equal(spy.wholeReads(), 0, 'the ranged path must never read the blob whole');
-        assert.equal(spy.rangedCalls(), 1, 'an exact find decodes exactly one segment');
-        const before = spy.rangedCalls();
+        assert.equal(spy.wholeReads(), 1, 'an exact find decodes exactly one segment');
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, lookupPath, { prefix: 'k01' })), { found: true, row: 100, count: 100 });
-        assert.equal(spy.rangedCalls() - before, 2, 'a boundary-spanning prefix decodes exactly the two edge segments');
-        assert.equal(spy.wholeReads(), 0);
+        assert.equal(spy.wholeReads(), 3, 'a boundary-spanning prefix decodes exactly the two edge segments');
     });
     it('scalar keys parse as .east literals; bad literals are key_parse_error', async () => {
         const storage = new InMemoryStorage();
         const entries = new Map(Array.from({ length: 500 }, (_, i) => [BigInt(i), `v${i}`]));
-        const blob = encodeBeast2PagedFor(IntLookupType, { batchSize: 97 })(entries);
+        const blob = encodeInSegmentsOf(IntLookupType, 97)(entries);
         await seedDataset(storage, blob, 'lookup', IntLookupType);
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, lookupPath, { key: '42' })), { found: true, row: 42, count: 1 });
         const bad = await findDatasetKey(storage, REPO, WS, lookupPath, { key: '"abc"' });
@@ -421,14 +494,14 @@ describe('findDatasetKey', () => {
     it('Set datasets search elements like dict keys', async () => {
         const storage = new InMemoryStorage();
         const tags = new Set(Array.from({ length: 300 }, (_, i) => `k${String(i).padStart(4, '0')}`));
-        const blob = encodeBeast2PagedFor(TagsType, { batchSize: 97 })(tags);
+        const blob = encodeInSegmentsOf(TagsType, 97)(tags);
         await seedDataset(storage, blob, 'tags', TagsType);
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, tagsPath, { key: '"k0123"' })), { found: true, row: 123, count: 1 });
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, tagsPath, { prefix: 'k00' })), { found: true, row: 0, count: 100 });
     });
     it('refuses non-keyed datasets and malformed queries', async () => {
         const storage = new InMemoryStorage();
-        const blob = encodeBeast2PagedFor(RowsType, { batchSize: 100 })(makeRows(50));
+        const blob = encodeInSegmentsOf(RowsType, 100)(makeRows(50));
         await seedDataset(storage, blob, 'rows', RowsType);
         const arr = await findDatasetKey(storage, REPO, WS, rowsPath, { key: '(id=1, name="x")' });
         assert.equal(arr.status, 400);
@@ -454,24 +527,17 @@ describe('findDatasetKey', () => {
         assert.equal(stale.status, 409);
         assert.equal(stale.headers.get('X-Content-SHA256'), hash);
     });
-    it('refuses index-less blobs; the whole-read fallback still searches under its cap', async () => {
+    it('refuses a collection an older e3 stored as one blob, naming the fix, without reading it whole', async () => {
         const storage = new InMemoryStorage();
-        // Whole-value v5 encode: no index — predates the stored-segmented contract.
         const raw = encodeBeast2For(LookupType)(lookupOf(60));
-        const hash = await seedDataset(storage, raw, 'lookup', LookupType);
-        const spy = spyObjectReads(storage, hash);
+        const hash = await seedDataset(storage, raw, 'lookup', LookupType, { bare: true });
+        const spy = spyObjectReads(storage, [hash]);
         const refused = await findDatasetKey(storage, REPO, WS, lookupPath, { key: '"k0001"' });
         assert.equal(refused.status, 400);
-        assert.equal((await refused.json()).error.type, 'dataset_not_indexed');
-        assert.equal(spy.wholeReads(), 0, 'the refusal must come from the tail probe, not a whole read');
-        const fallback = new InMemoryStorage();
-        const blob = encodeBeast2PagedFor(LookupType, { batchSize: 97 })(lookupOf(400));
-        await seedDataset(fallback, blob, 'lookup', LookupType);
-        fallback.objects.readRange = undefined;
-        assert.deepEqual(await findJson(await findDatasetKey(fallback, REPO, WS, lookupPath, { key: '"k0123"' })), { found: true, row: 123, count: 1 });
-        const capped = await findDatasetKey(fallback, REPO, WS, lookupPath, { key: '"k0123"' }, { readMaxBytes: 64 });
-        assert.equal(capped.status, 400);
-        assert.equal((await capped.json()).error.type, 'dataset_too_large');
+        const body = await refused.json();
+        assert.equal(body.error.type, 'dataset_not_indexed');
+        assert.match(body.error.message, /is stored as one blob: an older e3 wrote this repository — re-create it/);
+        assert.equal(spy.wholeReads(), 0, 'the refusal comes from the head probe, not a whole read');
     });
     it('an empty collection reports no match at row 0', async () => {
         const storage = new InMemoryStorage();
@@ -502,8 +568,8 @@ function machinesOf() {
 describe('findDatasetKey — struct keys', () => {
     it('leading fields and field prefixes address contiguous tuple ranges', async () => {
         const storage = new InMemoryStorage();
-        // batchSize 23 puts segment boundaries all over the tuple ranges.
-        const blob = encodeBeast2PagedFor(MachinesType, { batchSize: 23 })(machinesOf());
+        // Segments of 23 put boundaries all over the tuple ranges.
+        const blob = encodeInSegmentsOf(MachinesType, 23)(machinesOf());
         await seedDataset(storage, blob, 'machines', MachinesType);
         // A prefix alone types ahead on the FIRST field.
         assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, machinesPath, { prefix: 'p' })), { found: true, row: 100, count: 50 });
@@ -520,7 +586,7 @@ describe('findDatasetKey — struct keys', () => {
     });
     it('refuses malformed struct queries with typed errors', async () => {
         const storage = new InMemoryStorage();
-        const blob = encodeBeast2PagedFor(MachinesType, { batchSize: 23 })(machinesOf());
+        const blob = encodeInSegmentsOf(MachinesType, 23)(machinesOf());
         await seedDataset(storage, blob, 'machines', MachinesType);
         const intPrefix = await findDatasetKey(storage, REPO, WS, machinesPath, { fields: ['"press"', '"L2"'], prefix: '7' });
         assert.equal(intPrefix.status, 400);
@@ -543,4 +609,157 @@ describe('findDatasetKey — struct keys', () => {
         assert.match((await fieldsOnScalar.json()).error.message, /Struct keys/);
     });
 });
+const PlanRowType = StructType({ due: IntegerType, title: StringType });
+const PlansType = DictType(StringType, PlanRowType);
+/** The `by_status` index's key, which many rows share. */
+const PlanStatusKeyType = StructType({ status: StringType, week: IntegerType });
+const plansPath = [variant('field', 'records'), variant('field', 'plans')];
+/**
+ * Seeds a deployed workspace holding an indexed record: `n` plans keyed
+ * `p-000000…`, and two indexes. `by_due`'s order scatters them — `due` is a
+ * permutation of the rows, so consecutive index entries land in unrelated
+ * primary segments. `by_status` keys each row by a status and a week its
+ * `due` picks, so a key is a run of rows: for `n` a multiple of 30, a third
+ * of them per status, `done` then `late` then `open`, and a tenth of a
+ * status's per week. An index object carries only what a read resolves (its
+ * key and projection types); no program runs here.
+ */
+async function seedIndexedRecord(storage, n) {
+    await storage.repos.create(REPO);
+    const rows = new SortedMap(Array.from({ length: n }, (_, i) => [`p-${String(i).padStart(6, '0')}`,
+        { due: BigInt((i * 7919) % n), title: `Plan ${i}` }]), compareFor(StringType));
+    const primary = await datasetWrite(storage, REPO, rows, PlansType);
+    const EntryType = StructType({ ik: IntegerType, k: StringType });
+    const entries = new SortedMap([...rows].map(([k, row]) => [{ ik: row.due, k }, row.title]), compareFor(EntryType));
+    const index = await datasetWrite(storage, REPO, entries, indexCollectionType(StringType, IntegerType, StringType));
+    const declaration = await storage.objects.write(REPO, encodeBeast2For(RecordIndexObjectType)({
+        keyIr: '0'.repeat(64), multi: false, valueIr: some('0'.repeat(64)),
+        keyType: toEastTypeValue(IntegerType), valueType: toEastTypeValue(StringType),
+        buildIr: '0'.repeat(64), runner: variant('east_node', { platforms: [] }),
+    }));
+    const StatusEntryType = StructType({ ik: PlanStatusKeyType, k: StringType });
+    const statusEntries = new SortedMap([...rows].map(([k, row]) => [
+        { ik: { status: ['done', 'late', 'open'][Number(row.due % 3n)], week: row.due % 10n }, k }, row.title,
+    ]), compareFor(StatusEntryType));
+    const statusIndex = await datasetWrite(storage, REPO, statusEntries, indexCollectionType(StringType, PlanStatusKeyType, StringType));
+    const statusDeclaration = await storage.objects.write(REPO, encodeBeast2For(RecordIndexObjectType)({
+        keyIr: '0'.repeat(64), multi: false, valueIr: some('0'.repeat(64)),
+        keyType: toEastTypeValue(PlanStatusKeyType), valueType: toEastTypeValue(StringType),
+        buildIr: '0'.repeat(64), runner: variant('east_node', { platforms: [] }),
+    }));
+    const state = await writeRecordState(storage, REPO, {
+        primary, indexes: new Map([
+            ['by_due', { manifest: index, index: declaration }],
+            ['by_status', { manifest: statusIndex, index: statusDeclaration }],
+        ]),
+    });
+    const structure = variant('struct', new Map([
+        ['records', variant('struct', new Map([
+                ['plans', variant('value', { type: toEastTypeValue(PlansType), writable: false })],
+            ]))],
+    ]));
+    const pkgHash = await storage.objects.write(REPO, encodeBeast2For(PackageObjectType)({
+        tasks: new Map(),
+        data: { structure, refs: new Map([['records/plans', variant('value', { hash: state, versions: new Map() })]]) },
+        functions: new Map(),
+        records: new Map(), sources: new Map(),
+    }));
+    await storage.refs.workspaceWrite(REPO, WS, encodeBeast2For(WorkspaceRecordType)(some({
+        packageName: 'plans', packageVersion: '1.0.0', packageHash: pkgHash, deployedAt: new Date(0), currentRunId: none,
+    })));
+    await storage.datasets.write(REPO, WS, 'records/plans', variant('value', { hash: state, versions: new Map() }));
+    const manifest = decodeCollectionManifest(await storage.objects.read(REPO, primary));
+    return { rows, primarySegments: manifest.entries.map((entry) => entry.hash), statusSegments: await segmentsOf(storage, statusIndex) };
+}
+describe('getDatasetPage (index reads)', () => {
+    it('a joined page reads each primary segment it needs once, and serves the window whole', async () => {
+        const storage = new InMemoryStorage();
+        const { rows, primarySegments } = await seedIndexedRecord(storage, 12_000);
+        assert.ok(primarySegments.length > 3, `the record spans segments, got ${primarySegments.length}`);
+        const reads = new Map();
+        const objects = storage.objects;
+        const read = objects.read.bind(objects);
+        objects.read = (repo, hash) => {
+            if (primarySegments.includes(hash))
+                reads.set(hash, (reads.get(hash) ?? 0) + 1);
+            return read(repo, hash);
+        };
+        const response = await getDatasetPage(storage, REPO, WS, plansPath, { offset: 100, limit: 200, index: 'by_due', join: true });
+        objects.read = read;
+        assert.equal(response.status, 200, await response.clone().text());
+        // The window is the index's rows 100..299, in INDEX order, each joined to
+        // its row — never shortened to bound the join: a client concatenates
+        // windows at their offsets, so a short one would silently drop rows.
+        const window = decodeBeast2For(indexWindowType(StringType, IntegerType, StringType, PlanRowType))(new Uint8Array(await response.arrayBuffer()));
+        const expected = [...rows].sort(([ka, a], [kb, b]) => compareFor(IntegerType)(a.due, b.due) || compareFor(StringType)(ka, kb)).slice(100, 300);
+        assert.equal(window.length, 200);
+        assert.equal(response.headers.get('X-Page-Count'), '200');
+        window.forEach((entry, i) => {
+            const [key, row] = expected[i];
+            assert.equal(entry.key, key);
+            assert.equal(entry.ik, row.due);
+            assert.equal(entry.value, row.title);
+            assert.ok(entry.row.type === 'some' && equalFor(PlanRowType)(entry.row.value, row), `row ${i} joined`);
+        });
+        // Scattered keys touch many segments; each is read once, however many of
+        // the window's rows it holds.
+        assert.ok(reads.size > 1, 'the window scatters across the record');
+        assert.deepEqual([...reads.values()].filter((count) => count !== 1), [], 'every touched segment is read exactly once');
+    });
+    it('an unjoined page never touches the primary', async () => {
+        const storage = new InMemoryStorage();
+        const { primarySegments } = await seedIndexedRecord(storage, 3000);
+        const objects = storage.objects;
+        const read = objects.read.bind(objects);
+        const touched = [];
+        objects.read = (repo, hash) => {
+            if (primarySegments.includes(hash))
+                touched.push(hash);
+            return read(repo, hash);
+        };
+        const response = await getDatasetPage(storage, REPO, WS, plansPath, { offset: 0, limit: 50, index: 'by_due' });
+        objects.read = read;
+        assert.equal(response.status, 200, await response.clone().text());
+        assert.deepEqual(touched, [], 'a covering read is the index alone');
+    });
+});
+describe('findDatasetKey (through an index)', () => {
+    it('takes the index key: an exact key is every entry under it, and a prefix or leading fields a run of them', async () => {
+        const storage = new InMemoryStorage();
+        const { statusSegments } = await seedIndexedRecord(storage, 12_000);
+        assert.ok(statusSegments.length > 3, `the index spans segments, got ${statusSegments.length}`);
+        const byStatus = async (query) => findJson(await findDatasetKey(storage, REPO, WS, plansPath, { ...query, index: 'by_status' }));
+        // `done` is rows 0..3999, `late` 4000..7999 and `open` 8000..11999, and
+        // a status's weeks are 400 rows apiece.
+        assert.deepEqual(await byStatus({ key: '(status="late", week=3)' }), { found: true, row: 5200, count: 400 });
+        assert.deepEqual(await byStatus({ key: '(status="late", week=12)' }), { found: false, row: 8000, count: 0 });
+        assert.deepEqual(await byStatus({ fields: ['"late"'] }), { found: true, row: 4000, count: 4000 });
+        assert.deepEqual(await byStatus({ fields: ['"late"', '3'] }), { found: true, row: 5200, count: 400 });
+        // A prefix types ahead on the index key's first field.
+        assert.deepEqual(await byStatus({ prefix: 'l' }), { found: true, row: 4000, count: 4000 });
+        // A range still bounds the collection's flattened key, `ik`'s fields first.
+        assert.deepEqual(await byStatus({ from: ['"late"', '3'], to: ['"late"', '5'] }), { found: true, row: 5200, count: 800 });
+        // `due` is a permutation, so each of its keys is one entry, at the row of its own value.
+        assert.deepEqual(await findJson(await findDatasetKey(storage, REPO, WS, plansPath, { key: '42', index: 'by_due' })), { found: true, row: 42, count: 1 });
+    });
+    it('refuses a form the index key cannot take, naming the index, and an index the record does not declare', async () => {
+        const storage = new InMemoryStorage();
+        await seedIndexedRecord(storage, 300);
+        const refusal = async (query) => {
+            const response = await findDatasetKey(storage, REPO, WS, plansPath, query);
+            return { status: response.status, ...await response.json() };
+        };
+        // A prefix continues the index key's own fields, never the collection's `ik`.
+        const intPrefix = await refusal({ fields: ['"late"'], prefix: '3', index: 'by_status' });
+        assert.equal(intPrefix.status, 400);
+        assert.match(intPrefix.error.message, /prefix continues key field 'week', which is Integer, not String/);
+        assert.match((await refusal({ prefix: '4', index: 'by_due' })).error.message, /prefix search needs String keys; the keys of index 'by_due' are Integer/);
+        // The collection's own key is not the index's.
+        assert.equal((await refusal({ key: '(ik=42, k="p-000042")', index: 'by_due' })).error.type, 'key_parse_error');
+        const missing = await refusal({ key: '42', index: 'nope' });
+        assert.equal(missing.status, 404);
+        assert.equal(missing.error.type, 'index_not_found');
+        assert.match(missing.error.message, /has no index 'nope' — it has by_due, by_status/);
+    });
+});
 //# sourceMappingURL=datasets.spec.js.map