@jarenjs/db 0.73.0 → 0.83.2

package/src/query.js

@@ -31,6 +31,8 @@
  * the whole document runs over them.
  */
 
+import { physicalSelection, columnCodec } from './physical.js';
+
 import { createSemanticCache } from '@jarenjs/core/cache';
 import { analyzeQuery } from '@jarenjs/json/query';
 
@@ -47,7 +49,7 @@ import {
 } from './residual.js';
 import { createCursor, createSyncCursor, drainPage, utf8Length, PAGE_LIMIT_DEFAULT, rowClassOf } from './cursor.js';
 import { deepFreeze } from '@jarenjs/core/object';
-import { derivedSlotValue, probeBox, probeVector, columnScore } from './derive.js';
+import { derivedSlotValue, probeVector, columnScore } from './derive.js';
 import { cutCandidates, identityBatches } from './knn.js';
 import {
   deterministicFragment, registerFragment, registerAggregateOperator,
@@ -149,7 +151,8 @@ function slotValue(slot, externals, anchors = null) {
     return value === undefined ? undefined : JSON.stringify(value);
   }
   if ('derived' in slot)
-    return derivedSlotValue(slot.derived, externals[slot.derived.external]);
+    return derivedSlotValue(slot.derived, slot.derived.kind === 'bboxAxis'
+      ? externals[slot.derived.external] : externals);
   if ('typed' in slot) {
     // a typed slot is only ever emitted beside the seek that fills it,
     // so a bind that never resolved the seeks is a defect in the
@@ -179,8 +182,14 @@ function externalSlotKinds(slots, rank) {
   const kinds = new Map();
   for (const slot of slots) {
     if ('external' in slot) kinds.set(slot.external, 'plain');
-    else if ('derived' in slot && !kinds.has(slot.derived.external))
-      kinds.set(slot.derived.external, 'derived');
+    else if ('derived' in slot) {
+      const inputs = slot.derived.kind === 'bboxAxis'
+        ? [slot.derived] : [slot.derived.centre, slot.derived.radius];
+      for (const input of inputs) {
+        if ('external' in input && !kinds.has(input.external))
+          kinds.set(input.external, 'derived');
+      }
+    }
   }
   if (rank !== null && 'ext' in rank.probe && !kinds.has(rank.probe.ext))
     kinds.set(rank.probe.ext, 'probe');
@@ -285,7 +294,11 @@ function segmentsOf(ref) {
  * @param {string} suffix
  * @returns {any[]} the item, or nothing where the member is absent
  */
-function leafItems(row, suffix) {
+function leafItems(row, suffix, column = undefined) {
+  if (column !== undefined) {
+    const value = columnCodec(column).decode(row[`v${suffix}`]);
+    return value === undefined ? [] : [value];
+  }
   const type = row[`t${suffix}`];
   if (type === null || type === undefined) return [];
   const value = row[`v${suffix}`];
@@ -314,7 +327,8 @@ function leafItems(row, suffix) {
 function projectedTreeItems(project, row, prefix = '') {
   const build = (node) => {
     if (node.p === 'lit') return [node.value];
-    if (node.p === 'leaf') return leafItems(row, `${prefix}${node.index}`);
+    if (node.p === 'leaf') return leafItems(row, `${prefix}${node.index}`,
+      project.leaves[node.index].ref?.codecColumn);
     if (node.p === 'object') {
       /** @type {any} */
       const out = {};
@@ -331,6 +345,47 @@ function projectedTreeItems(project, row, prefix = '') {
   return build(project.tree);
 }
 
+  /**
+   * The items a GENERAL grouping answers: one per group row, built from
+   * the group's keys and aggregates through the plan's own tree. A key
+   * comes back with its JSON type beside it, so an absent key leaves
+   * its member out exactly as the object constructor does; an aggregate
+   * over no values follows the mapping the plan recorded — `0` for a
+   * count or a sum, the empty sequence for the other three, which is
+   * the ENGINE's answer, not SQL's `NULL`.
+   * @param {any} entry
+   * @param {any[]} rows
+   * @returns {any[]}
+   */
+  function groupItems(group, rows) {
+    const aggregateItems = (row, index) => {
+      const value = row[`a${index}`] ?? null;
+      const empty = group.aggregates[index].empty;
+      if (value === null) return empty === 'omit' ? [] : [empty === 'zero' ? 0 : null];
+      const aggregate = group.aggregates[index];
+      return [aggregate.ref?.codecColumn && ['min', 'max'].includes(aggregate.fn)
+        ? columnCodec(aggregate.ref.codecColumn).decode(value) : value];
+    };
+    const build = (node, row) => {
+      if (node.p === 'lit') return [node.value];
+      if (node.p === 'key') return leafItems(row, `k${node.index}`, group.keys[node.index].ref.codecColumn);
+      if (node.p === 'agg') return aggregateItems(row, node.index);
+      if (node.p === 'object') {
+        /** @type {any} */
+        const out = {};
+        for (const member of node.members) {
+          const items = build(member.node, row);
+          if (items.length > 0) out[member.name] = items[0];
+        }
+        return [out];
+      }
+      const items = [];
+      for (const item of node.items) items.push(...build(item, row));
+      return [items];
+    };
+    return rows.flatMap((row) => build(group.tree, row));
+  }
+
 /**
  * The query engine for one collection.
  * @param {{ connection: any, state: any, collection: any,
@@ -510,7 +565,6 @@ export function createQueryEngine(context) {
           : { ...planned.series, mode: 'engine', index: null, prefix: [] },
       };
     }
-
     if (profile !== null) {
       // the member allow-list: what the caller may OBTAIN, checked
       // against every member path the document references, before a
@@ -792,7 +846,15 @@ export function createQueryEngine(context) {
   const divertingExternal = (entry, externals) =>
     entry.externalNames.find((name) => {
       const kind = entry.externalSlotKinds.get(name);
-      if (kind === 'derived') return probeBox(externals[name]) === null;
+      const invalidDerived = entry.slots.some((slot) => {
+        if (!('derived' in slot)) return false;
+        const inputs = slot.derived.kind === 'bboxAxis'
+          ? [slot.derived] : [slot.derived.centre, slot.derived.radius];
+        return inputs.some((input) => 'external' in input && input.external === name)
+          && !bindable(slotValue(slot, externals));
+      });
+      if (invalidDerived) return true;
+      if (kind === 'derived') return false;
       // a probe binds when SOME declared width takes it; a width the
       // model does not declare is the diversion it always was
       if (kind === 'probe') return rankAlternativeFor(entry, externals[name]) === null;
@@ -834,46 +896,6 @@ export function createQueryEngine(context) {
     return items;
   };
 
-  /**
-   * The items a GENERAL grouping answers: one per group row, built from
-   * the group's keys and aggregates through the plan's own tree. A key
-   * comes back with its JSON type beside it, so an absent key leaves
-   * its member out exactly as the object constructor does; an aggregate
-   * over no values follows the mapping the plan recorded — `0` for a
-   * count or a sum, the empty sequence for the other three, which is
-   * the ENGINE's answer, not SQL's `NULL`.
-   * @param {any} entry
-   * @param {any[]} rows
-   * @returns {any[]}
-   */
-  const groupItems = (entry, rows) => {
-    const group = entry.plan.group;
-    const aggregateItems = (row, index) => {
-      const value = row[`a${index}`] ?? null;
-      const empty = group.aggregates[index].empty;
-      if (value === null) return empty === 'omit' ? [] : [empty === 'zero' ? 0 : null];
-      return [value];
-    };
-    const build = (node, row) => {
-      if (node.p === 'lit') return [node.value];
-      if (node.p === 'key') return leafItems(row, `k${node.index}`);
-      if (node.p === 'agg') return aggregateItems(row, node.index);
-      if (node.p === 'object') {
-        /** @type {any} */
-        const out = {};
-        for (const member of node.members) {
-          const items = build(member.node, row);
-          if (items.length > 0) out[member.name] = items[0];
-        }
-        return [out];
-      }
-      const items = [];
-      for (const item of node.items) items.push(...build(item, row));
-      return [items];
-    };
-    return rows.flatMap((row) => build(group.tree, row));
-  };
-
   /** How many ITEMS an engine result carries (its own shape rule). */
   const itemCount = (answer) => (answer === undefined ? 0
     : Array.isArray(answer) ? answer.length : 1);
@@ -1096,7 +1118,7 @@ export function createQueryEngine(context) {
         }
         if (entry.plan.group !== null) {
           return chain(runAll(entry, externals, statement), (rows) =>
-            answerOf(entry, groupItems(entry, checkRowBound(entry, rows))));
+            answerOf(entry, groupItems(entry.plan.group, checkRowBound(entry, rows))));
         }
         if (entry.plan.bucket !== null) {
           return chain(runAll(entry, externals, statement), (rows) => {
@@ -1193,12 +1215,12 @@ export function createQueryEngine(context) {
             return items;
           })) });
     }
-    if (entry.plan.group !== null) {
+    if (entry.plan.group !== null && entry.plan.aggregate === null) {
       // a native grouping is a barrier: the groups are the answer
       return createCursor({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
         materialize: () => chain(guardScan(entry), () => chain(statementOf(entry), (statement) =>
           chain(runAll(entry, externals, statement), (rows) =>
-            groupItems(entry, checkRowBound(entry, rows))))) });
+            groupItems(entry.plan.group, checkRowBound(entry, rows))))) });
     }
     if (entry.plan.bucket !== null) {
       // a native bucket is a barrier: the groups are the answer
@@ -1295,6 +1317,10 @@ export function createQueryEngine(context) {
       else if (pred.p === 'bboxRtree') touchedVirtual.add(pred.table);
       else if (pred.p === 'cellIn' || pred.p === 'cellPrefix')
         touchedColumns.add(pred.column);
+      else if (pred.p === 'refCmp') {
+        if (pred.left.column) touchedColumns.add(pred.left.column);
+        if (pred.right.column) touchedColumns.add(pred.right.column);
+      }
       else if ('ref' in pred && pred.ref?.column) touchedColumns.add(pred.ref.column);
     };
     collectColumns(entry.plan.filter);
@@ -1383,7 +1409,10 @@ export function createQueryEngine(context) {
             fn: entry2.fn, path: entry2.ref === null ? null : segmentsOf(entry2.ref) })),
           order: entry.plan.group.order === 'first-seen' ? 'first-seen'
             : entry.plan.group.order.map((term) => ({
-              key: entry.plan.group.keys[term.index].as, desc: term.desc })),
+              ...(term.aggregate === undefined ? { key: entry.plan.group.keys[term.index].as }
+                : { aggregate: entry.plan.group.aggregates[term.aggregate].fn,
+                  path: entry.plan.group.aggregates[term.aggregate].ref?.segments ?? null }),
+              desc: term.desc })),
         },
         // a chain's element window (`[<phrase>]`): the phrase planned as
         // if bare, its rows answered as the one array item
@@ -1577,7 +1606,11 @@ export function createEntityQueryEngine(context) {
   const q = dialect.quoteIdentifier;
   const physicalOf = (name) => ({ table: mapping.entities[name].table,
     // a join-table root has no document column of its own (§10.7)
-    document: mapping.entities[name].document !== false });
+    document: mapping.entities[name].document !== false,
+    ...(entities.get(name)?.physical == null ? {} : {
+      mapping: mapping.entities[name],
+      keys: mapping.entities[name].keys.map((key) => mapping.entities[name].columns.find((c) => c.name === key).physical),
+    }) });
   // the relation tables of every root this engine serves (§10.1): the
   // engine is the scope every entity set of the store shares, so a
   // producer holding one set can follow a hop into another root
@@ -1620,6 +1653,10 @@ export function createEntityQueryEngine(context) {
       planned = { ...planned, mode: 'set', plan: null,
         reasons: [{ construct: 'pushdown', reason: BIND_REASONS.pushdown }] };
     }
+    if (planned.plan?.group && dialect.name !== 'sqlite') {
+      planned = { ...planned, mode: 'set', plan: null,
+        reasons: [{ construct: '$groupby', reason: 'entity grouping runtime guards are qualified for SQLite' }] };
+    }
     const docPath = entities.get(planned.referenced[0])?.docPath;
     // the profile, applied exactly as on a collection (MODEL-FORMAT §8):
     // the roots a document may read, the references it may make, the
@@ -1678,6 +1715,8 @@ export function createEntityQueryEngine(context) {
       residualLimits: profile === null ? undefined : profile.limits,
       needsScanCheck: profile !== null && profile.refuseFullScan === true,
       scanChecked: false,
+      admitted: null,
+      runtimeReason: null,
     };
     if (planned.mode === 'native') {
       let plan = planned.plan;
@@ -1687,6 +1726,9 @@ export function createEntityQueryEngine(context) {
           const predicate = mandatory.get(binding.entity);
           return predicate === undefined ? entry : { ...entry, filter: conjoin(entry.filter, predicate) };
         }) };
+        if (plan.project) plan = { ...plan, project: { ...plan.project, leaves: plan.project.leaves.map((leaf) =>
+          leaf.count && mandatory.has(leaf.count.entity) ? { ...leaf, count: { ...leaf.count,
+            filter: conjoin(leaf.count.filter, mandatory.get(leaf.count.entity)) } } : leaf) } };
       }
       if (entry.rowBound !== null && plan.aggregate === null) {
         const cap = entry.rowBound + 1;
@@ -1712,6 +1754,13 @@ export function createEntityQueryEngine(context) {
     }
     return rows;
   };
+  const admittedRows = (entry, rows) => {
+    entry.admitted ??= { statements: 0, rows: 0, bytes: 0 };
+    entry.admitted.statements++;
+    entry.admitted.rows += rows.length;
+    entry.admitted.bytes += utf8Length(JSON.stringify(rows));
+    return rows;
+  };
   /** One merged entity document against the profile's byte bound
    * (JD2076): measured after the merge, because the mapped scalars live
    * in columns and the row's own JSON text holds only the rest. */
@@ -1767,11 +1816,15 @@ export function createEntityQueryEngine(context) {
         const limit = entry.rowBound === null ? '' : ` ${dialect.limitClause(entry.rowBound + 1, undefined)}`;
         return {
           name,
-          sql: `SELECT ${q('t')}.*, ${mapping.entities[name].document === false
-            ? dialect.stringLiteral('{}')
-            : dialect.jsonText(`${q('t')}.${q('doc')}`)} AS ${q('__doc')} `
+          sql: `SELECT ${entities.get(name)?.physical != null ? physicalSelection(mapping.entities[name], dialect, `${q('t')}.`)
+            : `${q('t')}.*, ${mapping.entities[name].document === false ? dialect.stringLiteral('{}')
+              : dialect.jsonText(`${q('t')}.${q('doc')}`)} AS ${q('__doc')}`} `
             + `FROM ${q(mapping.entities[name].table)} AS ${q('t')}${where} `
-            + `ORDER BY ${q('t')}.${dialect.rowIdentity()}${limit}`,
+            + `ORDER BY ${entities.get(name)?.physical != null ? mapping.entities[name].keys.map((k) => {
+              const column = mapping.entities[name].columns.find((c) => c.name === k);
+              const value = `${q('t')}.${q(column.physical)}`;
+              return column.codec === 'text' ? dialect.codepoint(value) : value;
+            }).join(', ') : `${q('t')}.${dialect.rowIdentity()}`}${limit}`,
           params: slots.map((slot) => slot.literal),
           statement: null,
         };
@@ -1785,6 +1838,7 @@ export function createEntityQueryEngine(context) {
       if (fetcher.statement === null) fetcher.statement = connection.prepare(fetcher.sql, { readOnly: true });
       return chain(fetcher.statement, (statement) =>
         chain(statement.all(fetcher.params), (rows) => {
+          admittedRows(entry, rows);
           root[fetcher.name] = checkRows(entry, rows, fetcher.name).map((row) =>
             checkBytes(entry, mergeEntityRow(mapping.entities[fetcher.name], row, '__doc'), fetcher.name));
           return next(i + 1);
@@ -1803,6 +1857,8 @@ export function createEntityQueryEngine(context) {
     requireCallable(options, state.now);
     const { externals, strict, pushdown, profile } = callState(options);
     const entry = entryFor(document, pushdown, profile);
+    entry.admitted = { statements: 0, rows: 0, bytes: 0 };
+    entry.runtimeReason = null;
     if (entry.planned.mode !== 'native') {
       if (strict) {
         const forcing = entry.planned.reasons[0];
@@ -1822,12 +1878,30 @@ export function createEntityQueryEngine(context) {
     if (entry.statement === null) entry.statement = connection.prepare(entry.sql, { readOnly: true });
     return chain(guardEntityScan(entry), () => chain(entry.statement, (statement) => {
       if (entry.planned.plan.aggregate === 'count')
-        return chain(statement.get(params), (row) => wrapValue(entry, row?.value ?? 0));
+        return chain(statement.get(params), (row) => { admittedRows(entry, row ? [row] : []); return wrapValue(entry, row?.value ?? 0); });
       return chain(statement.all(params), (rows) => {
+        admittedRows(entry, rows);
+        checkRows(entry, rows, entry.planned.referenced[0]);
+        if (entry.planned.plan.group) {
+          const group = entry.planned.plan.group;
+          if (rows.some((row) => row._valid === 0))
+            throw new DbRuntimeError('JD2003', 'a grouped physical column refuses a lossy or invalid value');
+          if (rows.some((row) => group.aggregates.some((aggregate, i) =>
+            ['sum', 'avg'].includes(aggregate.fn) && row[`_safe${i}`] !== 1))) {
+            entry.runtimeReason = { construct: '$groupby', reason: 'integer accumulation exceeded its runtime exactness bound' };
+            if (strict) throw new DbCompileError('JD0010', entry.runtimeReason.reason);
+            if (profile?.refuseFullScan) throw profileEntityRefusal(
+              'the profile refuses the decoded scan required by the integer accumulation bound', '/entities');
+            return runResidual(entry, document, externals);
+          }
+          return answerOf(entry, groupItems(group, rows)
+            .map((row) => checkBytes(entry, row, entry.planned.referenced[0])));
+        }
         const project = entry.planned.plan.project;
         if (project != null) {
           return answerOf(entry,
-            rows.flatMap((row) => projectedTreeItems(project, row, 'p')));
+            rows.flatMap((row) => projectedTreeItems(project, row, 'p'))
+              .map((row) => checkBytes(entry, row, entry.planned.referenced[0])));
         }
         const retEntity = entry.planned.plan.bindings
           .find((binding) => binding.name === entry.planned.plan.ret).entity;
@@ -1856,7 +1930,8 @@ export function createEntityQueryEngine(context) {
     for (const slot of entry.slots) {
       if (bindable(slotValue(slot, externals))) continue;
       const name = 'external' in slot ? slot.external
-        : 'derived' in slot ? slot.derived.external : null;
+        : 'derived' in slot ? (slot.derived.kind === 'bboxAxis' ? slot.derived.external
+          : [slot.derived.centre, slot.derived.radius].find((input) => 'external' in input)?.external) : null;
       return { construct: 'external', reason: BIND_REASONS.external(name, 'root') };
     }
     return null;
@@ -1877,6 +1952,8 @@ export function createEntityQueryEngine(context) {
     if (entry.planned.wrapped === true) {
       return buffered({ construct: 'window', reason: BIND_REASONS.wrappedWindow });
     }
+    if (entry.planned.plan?.group)
+      return buffered({ construct: '$groupby', reason: 'group results are validated and reconstructed together' });
     if (entry.planned.mode !== 'native') {
       const forcing = entry.planned.reasons[0]
         ?? { construct: 'residual', reason: BIND_REASONS.untranslated };
@@ -1924,14 +2001,17 @@ export function createEntityQueryEngine(context) {
     refuseBuffered(options, classified, entities.get(entry.planned.retEntity ?? '')?.docPath);
     const signal = options?.signal;
     const deadline = options?.deadline;
-    if (entry.planned.wrapped === true) {
+    if (entry.planned.wrapped === true || entry.planned.plan?.group) {
       return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
-        materialize: () => chain(execute(document, options), (value) => [value]) });
+        materialize: () => chain(execute(document, options), (value) => entry.planned.wrapped === true
+          ? [value] : value === undefined ? [] : Array.isArray(value) ? value : [value]) });
     }
     if (classified.barrier !== null) {
       return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
-        materialize: () => chain(fetchRoot(entry), (root) =>
-          packedResidualOf(entry, document)(root, externals).map(each)) });
+        materialize: () => {
+          entry.admitted = { statements: 0, rows: 0, bytes: 0 };
+          return chain(fetchRoot(entry), (root) => packedResidualOf(entry, document)(root, externals).map(each));
+        } });
     }
     const params = entry.slots.map((slot) => slotValue(slot, externals));
     // the statement is prepared by the first PULL, not here: a root
@@ -1944,7 +2024,11 @@ export function createEntityQueryEngine(context) {
     if (entry.planned.plan.aggregate === 'count') {
       return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
         materialize: () => chain(guardEntityScan(entry), () => chain(prepared(), (statement) =>
-          chain(statement.get(params), (row) => [row?.value ?? 0]))) });
+          chain(statement.get(params), (row) => {
+            entry.admitted = { statements: 0, rows: 0, bytes: 0 };
+            admittedRows(entry, row ? [row] : []);
+            return [row?.value ?? 0];
+          }))) });
     }
     const rowEntity = entry.planned.plan.ret === null ? null
       : entry.planned.plan.bindings
@@ -1953,17 +2037,23 @@ export function createEntityQueryEngine(context) {
     // a statement of its own per cursor: two live iterators over one
     // cached statement invalidate each other at the driver
     return cursorFactory({ ...classified, signal, deadline, now: state.now, wrap: driverWrap,
-      open: () => chain(guardEntityScan(entry), () => chain(connection.prepare(entry.sql, { readOnly: true, ephemeral: true }),
-        (statement) => statement.iterate(params))),
+      open: () => {
+        entry.admitted = { statements: 1, rows: 0, bytes: 2 };
+        return chain(guardEntityScan(entry), () => chain(connection.prepare(entry.sql, { readOnly: true, ephemeral: true }),
+          (statement) => statement.iterate(params)));
+      },
       items: (row) => {
         pulledRows++;
+        entry.admitted.bytes += utf8Length(JSON.stringify(row)) + (entry.admitted.rows ? 1 : 0);
+        entry.admitted.rows++;
         if (entry.rowBound !== null && pulledRows > entry.rowBound) {
           throw new DbRuntimeError('JD2007',
             `the fetch crossed the profile's maxRows bound of ${entry.rowBound}`,
             { docPath: entities.get(rowEntity)?.docPath, collection: rowEntity });
         }
         const project = entry.planned.plan.project;
-        if (project != null) return projectedTreeItems(project, row, 'p');
+        if (project != null) return projectedTreeItems(project, row, 'p')
+          .map((item) => checkBytes(entry, item, entry.planned.referenced[0]));
         return [each(checkBytes(entry, mergeEntityRow(mapping.entities[rowEntity], row, '__doc'), rowEntity))];
       } });
   };
@@ -1978,21 +2068,23 @@ export function createEntityQueryEngine(context) {
     const classified = cursorClass(entry, options?.externals ?? null);
     const diverted = entry.planned.mode === 'native' && classified.barrier !== null
       && classified.barrier.construct === 'external';
-    const mode = diverted ? 'set' : entry.planned.mode;
-    const reasons = diverted && classified.barrier !== null
+    const runtime = entry.runtimeReason;
+    const mode = diverted || runtime ? 'set' : entry.planned.mode;
+    const reasons = runtime ? [runtime, ...entry.planned.reasons] : diverted && classified.barrier !== null
       ? [classified.barrier, ...entry.planned.reasons] : entry.planned.reasons;
     const base = {
       mode,
       streaming: classified.streaming,
-      barrier: classified.barrier,
+      barrier: runtime ?? classified.barrier,
       budget: budgetOf(profile, profileSource, connection.capabilities),
       wrapped: entry.planned.wrapped === true,
       referenced: [...entry.planned.referenced],
+      admitted: entry.admitted === null ? null : { ...entry.admitted },
       reasons,
       // the same effective-order vocabulary the collection engine and
       // the graph loader report; `null` when no statement answers
-      order: diverted ? null : planOrder(entry.planned.plan),
-      sql: diverted ? null : entry.sql,
+      order: mode === 'native' ? planOrder(entry.planned.plan) : null,
+      sql: mode === 'native' ? entry.sql : null,
       residual: mode === 'native'
         ? null
         : { mode: 'set', reasons },
@@ -2426,6 +2518,8 @@ export function createLoadEngine(context, entityName) {
       if (spec?.[member] !== undefined && !isWindowBound(spec[member]))
         throw refuse(`${member} must be a non-negative integer`, []);
     }
+    if (entities.get(entityName)?.physical != null && (keyset || spec?.after !== undefined))
+      throw refuse('column codecs require decoded identities; physical keyset continuation is not qualified', []);
     const tree = buildTree(entityName, spec ?? {}, 0, maxDepth, [], new Set(), profile);
     const rendered = render(tree, 'r', param, emitters);
 
@@ -2515,7 +2609,8 @@ export function createLoadEngine(context, entityName) {
       pagination = 'offset';
     }
 
-    let sql = `SELECT ${rendered.aliasSql}.*, ${dialect.jsonText(rendered.docSql)} AS ${q('__doc')}`
+    let sql = `SELECT ${tree.entity.physical != null ? physicalSelection(tree.entityMapping, dialect, `${rendered.aliasSql}.`)
+      : `${rendered.aliasSql}.*, ${tree.entityMapping.document === false ? dialect.stringLiteral('{}') : dialect.jsonText(rendered.docSql)} AS ${q('__doc')}`}`
       + includeSql
       + ` FROM ${q(tree.entityMapping.table)} AS ${rendered.aliasSql}`;
     if (conditions.length > 0) sql += ` WHERE ${conditions.join(' AND ')}`;
@@ -2529,7 +2624,7 @@ export function createLoadEngine(context, entityName) {
         const nullsFirst = term.emptyGreatest === term.desc;
         return `${value} ${term.desc ? 'DESC' : 'ASC'}${dialect.orderNulls(nullsFirst)}`;
       });
-    if (identity === null) orderSql.push(`${rendered.aliasSql}.${dialect.rowIdentity()}`);
+    if (identity === null) orderSql.push(...(tree.entity.physical != null ? tree.entityMapping.keys.map((k) => `${rendered.aliasSql}.${q(tree.entityMapping.columns.find((c) => c.name === k).physical)}`) : [`${rendered.aliasSql}.${dialect.rowIdentity()}`]));
     sql += ` ORDER BY ${orderSql.join(', ')}`;
     // the profile's row bound rides the root as LIMIT maxRows + 1, so
     // a load past it is detected at the bound and refused (JD2007)
@@ -2681,7 +2776,7 @@ export function createLoadEngine(context, entityName) {
      * the truth either way.
      * @param {any} spec
      * @param {{ limit?: number, after?: any, maxBytes?: number | null,
-     *   consistency?: 'live' | 'snapshot', signal?: AbortSignal }} [options]
+     *   consistency?: 'live' | 'snapshot', signal?: AbortSignal, lookahead?: boolean }} [options]
      * @param {((tree: any, doc: any) => any) | undefined} [register]
      */
     page(spec, options = undefined, register = undefined, cursorFactory = createCursor) {
@@ -2700,7 +2795,9 @@ export function createLoadEngine(context, entityName) {
       if (spec?.take !== undefined || spec?.skip !== undefined)
         throw refuse('page() windows by its limit and continuation — a take or skip in the spec is refused', []);
       const after = options?.after ?? spec?.after ?? undefined;
-      const paged = { ...(spec ?? {}), take: limit + 1 };
+      if (options?.lookahead !== undefined && typeof options.lookahead !== 'boolean')
+        throw refuse('page() lookahead must be boolean', []);
+      const paged = { ...(spec ?? {}), take: limit + (options?.lookahead === false ? 0 : 1) };
       if (after === undefined) delete paged.after;
       else paged.after = after;
       const entry = buildLoad(paged, true, profileOf(options));
@@ -2718,7 +2815,7 @@ export function createLoadEngine(context, entityName) {
       // of work
       const cursor = openCursor(entry, options?.signal, undefined, options?.deadline, cursorFactory);
       return chain(drainPage(cursor, {
-        limit, maxBytes, after: after ?? null,
+        limit, maxBytes, after: after ?? null, lookahead: options?.lookahead,
         sizeOf: (doc) => utf8Length(JSON.stringify(doc)),
         continuationOf: (doc) => continuationOf(entry, doc),
       }), (page) => ({