@jarenjs/db 0.73.0 → 0.83.2

package/src/plan.js

@@ -95,11 +95,12 @@ const KIND_REASONS = {
 
 /** Why one PREDICATE stayed in the engine. */
 const PREDICATE_REASONS = {
+  physicalCodec: 'this codec or NULL policy requires decoded-row evaluation',
   notPredicate: 'not a predicate the planner translates',
   negatedPrefilter: 'a negated predicate cannot ride an implied pre-filter '
     + '(negating a superset drops rows)',
   existence: 'existence tests translate only over a singular member path on the binding',
-  joinTerritory: 'comparisons where both sides are paths are join territory',
+  joinTerritory: 'path comparisons require the same non-null numeric or string family',
   operands: 'comparisons translate only between a singular member path and a literal or external',
   compoundLiteral: 'array and object literals have no guarded native comparison form',
   stringSubject: 'string operators translate only over schema-typed string paths '
@@ -130,8 +131,8 @@ const PLAN_REASONS = {
   countProjection: 'count translates only over the bare binding or one member path '
     + '(a projected return can change the item count)',
   groupedAggregate: 'an aggregate over a grouped phrase folds its groups, which the engine does',
-  windowedGroup: 'a window over the GROUPS is engine work: the plan groups whole, '
-    + 'and a LIMIT over the groups would cut a different set',
+  distinctProjection: 'distinct translates over one typed scalar member path, unordered or ordered by that same path',
+  windowedGroup: 'a window over groups whose return can omit an item needs the engine cardinality',
   aggregatePath: 'aggregates translate only over a singular schema-typed path '
     + '(the engine ERRORS on non-conforming operands)',
 };
@@ -187,7 +188,8 @@ const ENTITY_REASONS = {
     + 'a binding nothing connects is a cartesian product, which is engine work',
   conjunctBinding: 'a conjunct must belong to one binding (or be the single join equality)',
   external: 'externals compare only against entity columns in this version',
-  projection: 'entity queries return one bare binding natively; projections run in the engine',
+  projection: 'this entity return or grouping needs decoded-row evaluation',
+  groupOrder: 'first-seen grouping over a compound physical key requires tuple ordering',
   order: 'ordering translates only over typed entity paths (never a boolean, never a document '
     + 'path that admits null)',
 };
@@ -756,7 +758,8 @@ function planDistanceBound(node, itSlot, shape) {
   const distanceNode = upperOnLeft ? node.args[0] : node.args[1];
   const radiusNode = upperOnLeft ? node.args[1] : node.args[0];
   const radius = constantOf(radiusNode);
-  if (radius === null || typeof radius.value !== 'number')
+  const radiusExternal = radiusNode.kind === 'var' && radiusNode.external === true;
+  if (!radiusExternal && (radius === null || typeof radius.value !== 'number'))
     return { refusal: refusal('$distance', SPATIAL_REASONS.operand) };
 
   let subjectAt = 0;
@@ -769,21 +772,33 @@ function planDistanceBound(node, itSlot, shape) {
 
   const probeNode = distanceNode.args[subjectAt === 0 ? 1 : 0];
   const constant = constantOf(probeNode);
-  // an EXTERNAL centre would need a slot that composes the bound value
-  // with the radius, and the derived slot kind is closed at one axis of
-  // one bound value; such a query diverts to the full scan as before
-  if (constant === null) return { refusal: refusal('$distance', SPATIAL_REASONS.operand) };
-  const at = probePosition(constant.value);
-  if (at === null) return { refusal: refusal('$distance', SPATIAL_REASONS.unbounded) };
-  if (!Number.isFinite(radius.value) || radius.value < 0) {
+  const centreExternal = probeNode.kind === 'var' && probeNode.external === true;
+  if (!centreExternal && constant === null)
+    return { refusal: refusal('$distance', SPATIAL_REASONS.operand) };
+  if (!radiusExternal && (!Number.isFinite(radius.value) || radius.value < 0)) {
     return { refusal: refusal('$distance', SPATIAL_REASONS.radius) };
   }
-  const box = probeCircleBox(at, radius.value);
-  if (box === null) return { refusal: refusal('$distance', SPATIAL_REASONS.pole) };
-  if (box[0] < -180 || box[2] > 180)
-    return { refusal: refusal('$distance', SPATIAL_REASONS.wrapped) };
+  let probe;
+  if (centreExternal || radiusExternal) {
+    // Both inputs belong to one closed derived slot. Bind-time failure
+    // diverts the whole query, preserving the engine's errors and the
+    // candidates on the far side of a pole or the antimeridian.
+    probe = { circle: {
+      centre: centreExternal ? { external: probeNode.name } : { literal: constant.value },
+      radius: radiusExternal ? { external: radiusNode.name } : { literal: radius.value },
+    } };
+  }
+  else {
+    const at = probePosition(constant.value);
+    if (at === null) return { refusal: refusal('$distance', SPATIAL_REASONS.unbounded) };
+    const box = probeCircleBox(at, radius.value);
+    if (box === null) return { refusal: refusal('$distance', SPATIAL_REASONS.pole) };
+    if (box[0] < -180 || box[2] > 180)
+      return { refusal: refusal('$distance', SPATIAL_REASONS.wrapped) };
+    probe = { box };
+  }
 
-  const conjunct = boxConjunct(columns, rtreeTableOf(shape, subject.canonical), { box });
+  const conjunct = boxConjunct(columns, rtreeTableOf(shape, subject.canonical), probe);
   return promotion(conjunct.pred,
     { construct: '$distance', via: conjunct.via, columns: conjunct.columns, exact: false },
     [refusal('$distance', SPATIAL_REASONS.distance)]);
@@ -1305,8 +1320,14 @@ function planPredicate(node, itSlot, shape) {
     const ref = pathRef(left, itSlot, shape);
     const operand = operandOf(right);
     if (ref === null || operand === null) {
-      if (pathRef(left, itSlot, shape) !== null && pathRef(right, itSlot, shape) !== null)
+      const other = pathRef(right, itSlot, shape);
+      if (ref !== null && other !== null) {
+        const family = (item) => isNumericType(item.type) ? 'number' : item.type;
+        if (orderable(ref, shape.schema) && orderable(other, shape.schema)
+          && family(ref) === family(other))
+          return exactly({ p: 'refCmp', op, left: ref, right: other });
         return { refusal: refusal(node.name, PREDICATE_REASONS.joinTerritory) };
+      }
       return { refusal: refusal(node.name, PREDICATE_REASONS.operands) };
     }
     if ('lit' in operand) {
@@ -2057,7 +2078,7 @@ function planSeriesOperator(root, shape) {
  * The rules that make it agree with the engine, each one a refusal
  * rather than an approximation:
  *
- * - a key is a singular schema-typed path that cannot hold `null`, so
+ * - a key is a singular schema-typed scalar path, so
  *   the group SQL forms is the group the engine forms. An ABSENT key is
  *   its own group, and its value comes back with its JSON type beside
  *   it, so the decoder can leave the member out exactly as the object
@@ -2074,13 +2095,13 @@ function planSeriesOperator(root, shape) {
  * @param {any} shape
  * @returns {any | null}
  */
-function planGeneralGrouping(node, itSlot, shape) {
+function planGeneralGrouping(node, itSlot, shape, resolve = pathRef) {
   const keys = [];
   /** @type {Map<number, number>} */
   const keySlot = new Map();
   for (const key of node.groupby.keys) {
-    const ref = pathRef(key.expr, itSlot, shape);
-    if (ref === null || ref.type === 'unknown' || admitsNull(shape.schema, ref.segments))
+    const ref = resolve(key.expr, itSlot, shape);
+    if (ref === null || ref.type === 'unknown')
       return null;
     keySlot.set(key.slot, keys.length);
     keys.push({ as: key.name, ref });
@@ -2094,7 +2115,7 @@ function planGeneralGrouping(node, itSlot, shape) {
     if (child.kind === 'var' && child.external !== true && keySlot.has(child.slot))
       return { p: 'key', index: keySlot.get(child.slot) };
     if (child.kind === 'op') {
-      const entry = groupAggregate(child, itSlot, shape);
+      const entry = groupAggregate(child, itSlot, shape, resolve);
       if (entry === null) return null;
       // one aggregate per distinct (function, path): two members that
       // ask the same question are one SQL aggregate
@@ -2129,8 +2150,12 @@ function planGeneralGrouping(node, itSlot, shape) {
   };
   const tree = build(node.ret);
   if (tree === null) return null;
-  const order = groupOrder(node.orderby, keySlot);
+  const order = groupOrder(node.orderby, keySlot, build);
   if (order === null) return null;
+  // Group equality accepts null and boolean keys; ordering them raises
+  // JQ2005. SQL ordering must not hide that engine error.
+  if (order !== 'first-seen' && order.some((term) =>
+    term.aggregate === undefined && !orderable(keys[term.index].ref, shape.schema))) return null;
   return { keys, aggregates, tree, order };
 }
 
@@ -2144,14 +2169,36 @@ function planGeneralGrouping(node, itSlot, shape) {
  * @param {any} shape
  * @returns {{ as: string, fn: string, ref: any, empty: string } | null}
  */
-function groupAggregate(node, itSlot, shape) {
+function groupAggregate(node, itSlot, shape, resolve = pathRef) {
+  // SQL's nullable sum is authored explicitly: test for any non-null
+  // values, sum exactly that sequence, otherwise return a literal null.
+  if (node.name === '$if' && node.args[0]?.name === '$exists'
+    && node.args[1]?.name === '$sum' && node.args[2]?.kind === 'literal' && node.args[2].value === null) {
+    const filtered = (phrase) => {
+      if (phrase?.kind !== 'flwor' || phrase.forBindings.length !== 1 || phrase.letBindings.length
+        || phrase.groupby !== null || phrase.orderby !== null || phrase.fold !== null
+        || phrase.count !== null || phrase.asChecks !== null) return null;
+      const binding = phrase.forBindings[0];
+      if (!isItVar(unpacked(binding.expr), itSlot) || binding.window !== null || binding.atSlot !== -1 || binding.allowingEmpty) return null;
+      const ref = resolve(phrase.ret, binding.slot, shape);
+      const where = phrase.where;
+      if (ref === null || !isNumericType(ref.type) || where?.name !== '$ne') return null;
+      const other = resolve(where.args[0], binding.slot, shape);
+      return other !== null && canonicalOf(other.segments) === canonicalOf(ref.segments)
+        && where.args[1]?.kind === 'literal' && where.args[1].value === null ? ref : null;
+    };
+    const test = filtered(node.args[0].args[0]);
+    const sum = filtered(node.args[1].args[0]);
+    return test !== null && sum !== null && canonicalOf(test.segments) === canonicalOf(sum.segments)
+      ? { fn: 'sum', ref: sum, empty: 'null' } : null;
+  }
   if (node.name === '$count') {
     return isItVar(node.args[0], itSlot)
       ? { fn: 'rows', ref: null, empty: 'zero' } : null;
   }
   const fn = AGGREGATES.get(node.name);
   if (fn === undefined || fn === 'count') return null;
-  const ref = pathRef(node.args[0], itSlot, shape);
+  const ref = resolve(node.args[0], itSlot, shape);
   const numeric = fn === 'sum' || fn === 'avg';
   const acceptable = ref !== null
     && (numeric ? isNumericType(ref.type) : ref.type !== 'unknown')
@@ -2164,21 +2211,32 @@ function groupAggregate(node, itSlot, shape) {
 /**
  * How the groups come out: the engine's order of FIRST APPEARANCE
  * (§6.5) when nothing declares otherwise, else the group-key ordering
- * an `$orderby` asked for. `null` when the ordering names anything but
- * group keys — after a grouping there is nothing else a row can order
- * by that the plan could reproduce.
+ * an `$orderby` asked for. Proven aggregate expressions share the
+ * return's aggregate slots, including aggregates used only to order.
  * @param {any} orderby
  * @param {Map<number, number>} keySlot
- * @returns {'first-seen' | { index: number, desc: boolean, nullsFirst: boolean }[] | null}
+ * @param {(node: any) => any} build
+ * @returns {any}
  */
-function groupOrder(orderby, keySlot) {
+function groupOrder(orderby, keySlot, build) {
   if (orderby === null) return 'first-seen';
   const terms = [];
   for (const spec of orderby.specs) {
     if (spec.collation !== null || spec.collationName !== null) return null;
     const key = spec.key;
-    if (key.kind !== 'var' || key.external === true || !keySlot.has(key.slot)) return null;
-    terms.push({ index: keySlot.get(key.slot), desc: spec.desc === true,
+    let target;
+    if (key.kind === 'var' && key.external !== true && keySlot.has(key.slot))
+      target = { index: keySlot.get(key.slot) };
+    else {
+      // SQL SUM/AVG may accumulate in a different order (SQLite also
+      // compensates rounding). Such a value cannot safely decide group
+      // order. Counts and extrema have no accumulation-order ambiguity.
+      if (!['$count', '$min', '$max'].includes(key.name)) return null;
+      const aggregate = key.kind === 'op' ? build(key) : null;
+      if (aggregate?.p !== 'agg') return null;
+      target = { aggregate: aggregate.index };
+    }
+    terms.push({ ...target, desc: spec.desc === true,
       nullsFirst: (spec.emptyGreatest === true) === (spec.desc === true) });
   }
   return terms;
@@ -2187,16 +2245,14 @@ function groupOrder(orderby, keySlot) {
 /**
  * The projection TREE one `$return` compiles to, or `null` when the
  * shape is not one the plan can rebuild. Object and array constructors,
- * literals and singular member paths compose; anything else — a
- * function call, a conditional, a dynamic member, a reference to the
- * binding itself — refuses the WHOLE projection, because a projection
+ * literals, whole collection bindings and singular member paths compose;
+ * a function call, conditional or dynamic member refuses the WHOLE projection, because a projection
  * that dropped part of what the caller asked for would be a wrong
  * answer, not a partial one.
  *
  * Distinct paths are collected once: a path named twice is one fetched
- * column and two leaves pointing at it. A projection with NO path is
- * refused too — a statement needs a column to select, and a projection
- * of pure literals has nothing the database could contribute.
+ * column and two leaves pointing at it. A constant tree needs only a
+ * row marker from SQL: the selection contributes its cardinality.
  * @param {any} node - the `$return` AST node
  * @param {number} itSlot
  * @param {any} shape
@@ -2210,8 +2266,9 @@ function projectionTree(node, itSlot, shape) {
   const build = (child) => {
     assertDecidedKind(child);
     if (child.kind === 'literal') return { p: 'lit', value: child.value };
-    if (child.kind === 'path') {
-      const ref = pathRef(child, itSlot, shape);
+    if (child.kind === 'path' || isItVar(child, itSlot)) {
+      const ref = isItVar(child, itSlot)
+        ? { segments: [], type: 'unknown', column: null } : pathRef(child, itSlot, shape);
       if (ref === null) return null;
       const canonical = canonicalOf(ref.segments);
       let index = byCanonical.get(canonical);
@@ -2243,7 +2300,7 @@ function projectionTree(node, itSlot, shape) {
     return null;
   };
   const tree = build(node);
-  return tree === null || leaves.length === 0 ? null : { tree, leaves };
+  return tree === null ? null : { tree, leaves };
 }
 
 /**
@@ -2581,10 +2638,16 @@ function planCollectionCore(document, shape, options = undefined) {
   // pushable at all — a SQL fold over zero rows never sees a second
   // operand, and `aggregateSpec` refuses the declaration at open — so
   // the recognizer here reads the phrase and nothing else
+  const distinct = root.kind === 'op' && root.name === '$distinct';
+  if (distinct) {
+    root = root.args[0];
+    rawInner = rawInner?.$distinct;
+    assertDecidedKind(root);
+  }
   let aggregate = null;
   const registeredAggregate = root.kind === 'op' && !AGGREGATES.has(root.name)
     ? (options?.aggregate?.(root.name) ?? null) : null;
-  if (root.kind === 'op' && (AGGREGATES.has(root.name) || registeredAggregate !== null)) {
+  if (!distinct && root.kind === 'op' && (AGGREGATES.has(root.name) || registeredAggregate !== null)) {
     if (windows.length > 0) {
       return {
         analysis, plan: null, mode: 'set',
@@ -2603,7 +2666,7 @@ function planCollectionCore(document, shape, options = undefined) {
   // a document that IS a series operator over the collection: the
   // operand's own conjuncts (and what the frozen spec implies) narrow
   // through the index, and the kernel decides over what comes back
-  if (aggregate === null && root.kind === 'op' && SERIES_ROOT_OPS.includes(root.name)) {
+  if (!distinct && aggregate === null && root.kind === 'op' && SERIES_ROOT_OPS.includes(root.name)) {
     const temporal = planSeriesOperator(root, shape);
     if (temporal !== null) {
       // a peeled `$subsequence` composes as it does everywhere — over a
@@ -2637,6 +2700,29 @@ function planCollectionCore(document, shape, options = undefined) {
   const { plan } = flwor;
   const fullyPushed = flwor.whereFullyPushed && flwor.orderPushed;
 
+  if (distinct) {
+    const ref = flwor.projectedPath;
+    const sameOrder = root.orderby === null || (ref !== null && plan.order !== null
+      && plan.order.every((term) => canonicalOf(term.ref.segments) === canonicalOf(ref.segments)));
+    if (fullyPushed && sameOrder && ref !== null && ref.type !== 'unknown'
+      && flwor.group === null && flwor.bucket === null) {
+      // DISTINCT and GROUP BY share the query language's key relation.
+      // An absent path contributes no item, including under a window.
+      plan.filter = conjoin(plan.filter, { p: 'typeIs', ref, types: [], positive: true });
+      plan.group = { keys: [{ as: 'distinct', ref }], aggregates: [],
+        tree: { p: 'key', index: 0 }, order: root.orderby === null ? 'first-seen'
+          : plan.order.map((term) => ({ index: 0, desc: term.desc,
+            nullsFirst: (term.emptyGreatest === true) === (term.desc === true) })) };
+      plan.order = null;
+      plan.window = windows.length === 0 ? null : composeWindows(windows);
+      return { analysis, plan, mode: 'native', reasons: [], rowReturn: null,
+        udfs: flwor.udfs, prefilters: flwor.prefilters, series: null };
+    }
+    return { analysis, plan: null, mode: 'set',
+      reasons: [refusal('$distinct', PLAN_REASONS.distinctProjection)],
+      rowReturn: null, udfs: [], prefilters: [], series: null };
+  }
+
   if (flwor.knn !== null) {
     // the k-nearest mode: the recognized ordering under a window with
     // a finite limit, composed exactly as pushed windows are. The plan
@@ -2661,6 +2747,14 @@ function planCollectionCore(document, shape, options = undefined) {
         rowReturn: null, udfs: [], prefilters: flwor.prefilters, series: null };
     }
     if (flwor.bucket !== null || flwor.group !== null || flwor.bucketRefusal != null) {
+      if (aggregate.fn === 'count' && flwor.group !== null
+        && ['object', 'array', 'lit'].includes(flwor.group.tree.p)
+        && flwor.group.aggregates.every((entry) => entry.fn === 'rows')) {
+        plan.group = flwor.group;
+        plan.aggregate = { fn: 'count', ref: null };
+        return { analysis, plan, mode: 'native', reasons: [], rowReturn: null,
+          udfs: flwor.udfs, prefilters: flwor.prefilters, series: null };
+      }
       // the phrase's items are its GROUPS; a COUNT(*) over the rows
       // answered the row count for a `$count` of the groups
       return {
@@ -2722,7 +2816,10 @@ function planCollectionCore(document, shape, options = undefined) {
   // the temporal bucket: only over a WHOLE pushed selection, because a
   // conjunct the residual would still apply would arrive after the rows
   // were already summed
-  if (flwor.group !== null && fullyPushed && windows.length === 0 && aggregate === null) {
+  if (flwor.group !== null && fullyPushed && aggregate === null
+    && (windows.length === 0 || ['object', 'array', 'lit'].includes(flwor.group.tree.p)
+      || (flwor.group.tree.p === 'agg'
+        && flwor.group.aggregates[flwor.group.tree.index].empty === 'zero'))) {
     plan.group = flwor.group;
     return { analysis, plan, mode: 'native', reasons: [], rowReturn: null,
       udfs: flwor.udfs, prefilters: flwor.prefilters, series: null };
@@ -2759,7 +2856,13 @@ function planCollectionCore(document, shape, options = undefined) {
 
   if (!groupedResidual && fullyPushed && flwor.projectionNative
     && (windows.length === 0 || plan.window !== null)) {
-    if (flwor.projectedPath !== null) plan.project = { path: flwor.projectedPath };
+    if (flwor.projectedPath !== null) {
+      plan.project = { path: flwor.projectedPath };
+      // A path yields no item for an absent member. Windows count the
+      // projected items, so discard those rows before applying LIMIT.
+      if (windows.length > 0) plan.filter = conjoin(plan.filter,
+        { p: 'typeIs', ref: flwor.projectedPath, types: [], positive: true });
+    }
     else if (flwor.projectedTree !== null) plan.project = flwor.projectedTree;
     return { analysis, plan, mode: 'native', reasons: [], rowReturn: null,
       udfs: flwor.udfs, prefilters: flwor.prefilters,
@@ -2768,7 +2871,7 @@ function planCollectionCore(document, shape, options = undefined) {
 
   // the row residual: everything but the projection pushed
   if (!groupedResidual && fullyPushed && !flwor.projectionNative
-    && (windows.length === 0 || plan.window !== null)) {
+    && windows.length === 0) {
     const rawFlwor = rawInner;
     const name = flwor.itName ?? 'it';
     return {
@@ -2879,10 +2982,14 @@ export function entityShape(entity, entityMapping) {
   for (const column of entityMapping.columns) {
     const epoch = column.source === 'epoch(document)';
     flavors.set(canonicalOf([{ name: column.name }]), {
-      column: column.name,
+      column: column.physical ?? column.name,
       flavor: epoch ? 'entity-epoch' : 'entity-column',
       storage: column.storage,
+      codec: column.codec,
+      codecColumn: column.codec === undefined ? undefined : column,
+      nullPolicy: column.null,
       format: epoch ? entity.properties.get(column.name)?.format : undefined,
+      unsafe: column.codec !== undefined && !['text', 'integer', 'number', 'boolean', 'date', 'datetime'].includes(column.codec),
     });
   }
   for (const fk of entityMapping.foreignKeys) {
@@ -2895,6 +3002,7 @@ export function entityShape(entity, entityMapping) {
     schema: entity.schema,
     columnByCanonical: new Map(),
     entityFlavors: flavors,
+    columnOnly: entityMapping.document === false,
   };
 }
 
@@ -2911,6 +3019,7 @@ export function entityPathRef(node, slot, shape) {
   if (ref === null) return null;
   const canonical = canonicalOf(ref.segments);
   const flavored = shape.entityFlavors.get(canonical);
+  if (flavored?.unsafe || (shape.columnOnly && flavored === undefined)) return null;
   if (flavored !== undefined) {
     return {
       ...ref,
@@ -2918,6 +3027,9 @@ export function entityPathRef(node, slot, shape) {
       flavor: flavored.flavor,
       storage: flavored.storage,
       format: flavored.format,
+      nullPolicy: flavored.nullPolicy,
+      codec: flavored.codec,
+      codecColumn: flavored.codecColumn,
     };
   }
   // a nested path rides the JSONB document with the phase-A guards;
@@ -2930,20 +3042,54 @@ export function entityPathRef(node, slot, shape) {
  * collection projection composes — objects, arrays, literals and
  * singular member paths — with each leaf carrying the BINDING it reads
  * from, so the statement extracts it from that binding's alias. `null`
- * when the shape is not one the plan can rebuild, and a projection with
- * no path at all is refused for the same reason a collection's is: a
- * statement needs a column to select.
+ * when the shape is not one the plan can rebuild. A constant tree
+ * fetches a row marker, preserving the selection's cardinality.
  * @param {any} node - the `$return` AST node
  * @param {Map<number, any>} byName - binding slot → binding
  * @returns {{ tree: any, leaves: { binding: string, ref: any }[] } | null}
  */
-function entityProjectionTree(node, byName) {
+function entityProjectionTree(node, byName, entities, mapping) {
   const leaves = [];
   /** @type {Map<string, number>} */
   const byCanonical = new Map();
   const build = (child) => {
     assertDecidedKind(child);
     if (child.kind === 'literal') return { p: 'lit', value: child.value };
+    if (child.kind === 'op' && child.name === '$count') {
+      const inner = child.args[0];
+      if (inner?.kind !== 'flwor' || inner.forBindings.length !== 1 || inner.groupby !== null
+        || inner.fold !== null || inner.orderby !== null || inner.letBindings.length
+        || inner.asChecks !== null || inner.count !== null) return null;
+      const binding = inner.forBindings[0];
+      const entity = bindingEntity(binding, entities);
+      if (entity === null || binding.window !== null || binding.atSlot !== -1 || binding.allowingEmpty
+        || inner.ret.kind !== 'var' || inner.ret.slot !== binding.slot) return null;
+      const shape = entityShape(entities.get(entity), mapping.entities[entity]);
+      const innerBinding = { name: binding.name, slot: binding.slot, entity, shape };
+      const scope = new Map([...byName, [binding.slot, innerBinding]]);
+      const conjuncts = inner.where?.kind === 'op' && inner.where.name === '$and' ? inner.where.args : [inner.where];
+      const edges = [];
+      let filter = null;
+      for (const conjunct of conjuncts) {
+        if (conjunct === null) continue;
+        const cross = conjunct.kind === 'op' && conjunct.name === '$eq'
+          ? crossBindingComparison(conjunct, scope) : null;
+        if (cross !== null && (cross.left.binding === innerBinding || cross.right.binding === innerBinding)) {
+          edges.push({ left: { binding: cross.left.binding.name, ref: cross.left.ref },
+            right: { binding: cross.right.binding.name, ref: cross.right.ref } });
+          continue;
+        }
+        const slots = new Set(); collectBindingSlots(conjunct, scope, slots);
+        if (slots.size !== 1 || !slots.has(binding.slot)) return null;
+        const planned = planEntityPredicate(conjunct, binding.slot, shape);
+        if ('refusal' in planned) return null;
+        filter = conjoin(filter, planned.pred);
+      }
+      if (!edges.length) return null;
+      const index = leaves.length;
+      leaves.push({ count: { entity, binding: binding.name, edges, filter } });
+      return { p: 'leaf', index };
+    }
     if (child.kind === 'path') {
       const binding = child.external === true ? undefined : byName.get(child.rootSlot);
       if (binding === undefined) return null;
@@ -2979,7 +3125,7 @@ function entityProjectionTree(node, byName) {
     return null;
   };
   const tree = build(node);
-  return tree === null || leaves.length === 0 ? null : { tree, leaves };
+  return tree === null ? null : { tree, leaves };
 }
 
 /**
@@ -3037,9 +3183,17 @@ export function planEntityPredicate(node, slot, shape) {
     if (pred.p === 'and' || pred.p === 'or')
       return { ...pred, items: pred.items.map(reflavor) };
     if (pred.p === 'not') return { ...pred, item: reflavor(pred.item) };
+    if (pred.p === 'refCmp') {
+      const flavor = (ref) => reflavor({ p: 'typeIs', ref, types: [], positive: true }).ref;
+      return { ...pred, left: flavor(pred.left), right: flavor(pred.right) };
+    }
     if (!('ref' in pred) || pred.ref === null) return pred;
     const canonical = canonicalOf(pred.ref.segments);
     const flavored = shape.entityFlavors.get(canonical);
+    if (shape.columnOnly && (flavored === undefined || flavored.unsafe || flavored.nullPolicy === 'null')) {
+      blocked = refusal('physical', PREDICATE_REASONS.physicalCodec);
+      return pred;
+    }
     if (flavored === undefined) {
       // externals against DOC paths are not translated here (the
       // phase-A external forms assume the collection layout)
@@ -3049,7 +3203,8 @@ export function planEntityPredicate(node, slot, shape) {
       return { ...pred, ref: { ...pred.ref, flavor: 'entity-doc' } };
     }
     const ref = { ...pred.ref, column: flavored.column,
-      flavor: flavored.flavor, storage: flavored.storage, format: flavored.format };
+      flavor: flavored.flavor, storage: flavored.storage, format: flavored.format,
+      codec: flavored.codec, codecColumn: flavored.codecColumn, nullPolicy: flavored.nullPolicy };
     if (flavored.flavor === 'entity-epoch' && pred.p === 'cmp') {
       if ('ext' in pred.operand) {
         blocked = refusal(pred.op, ENTITY_REASONS.external);
@@ -3122,7 +3277,7 @@ function planEntityQueryCore(document, entities, mapping, operators) {
   if (root.kind !== 'flwor')
     return residual(root.kind, ENTITY_REASONS.notFlwor);
   if (root.fold !== null || root.letBindings.length > 0 || root.asChecks !== null
-    || root.groupby !== null || root.count !== null)
+    || root.count !== null)
     return residual('$let', KIND_REASONS.let);
 
   // bindings must each range over one entity's array
@@ -3140,6 +3295,19 @@ function planEntityQueryCore(document, entities, mapping, operators) {
     });
   }
   const byName = new Map(bindings.map((binding) => [binding.slot, binding]));
+  const group = root.groupby === null ? null : bindings.length === 1 && aggregate === null && !windows.length
+    ? planGeneralGrouping(root, bindings[0].slot, bindings[0].shape, entityPathRef) : null;
+  if (root.groupby !== null && group === null) return residual('$groupby', ENTITY_REASONS.projection);
+  if (group && [...group.keys, ...group.aggregates].some((entry) => entry.ref && entry.ref.flavor !== 'entity-column'))
+    return residual('$groupby', ENTITY_REASONS.projection);
+  // Integer accumulations carry a runtime exactness proof. Floating
+  // accumulations and date-codec validation need the decoded evaluator.
+  if (group && [...group.keys, ...group.aggregates].some((entry) => entry.ref
+    && (['date', 'datetime'].includes(entry.ref.codec)
+      || (['sum', 'avg'].includes(entry.fn) && entry.ref.type !== 'integer'))))
+    return residual('$groupby', ENTITY_REASONS.projection);
+  if (group?.order === 'first-seen' && entities.get(bindings[0].entity).physical?.keys.length > 1)
+    return residual('$groupby', ENTITY_REASONS.groupOrder);
   const conjuncts = root.where === null
     ? []
     : root.where.kind === 'op' && root.where.name === '$and'
@@ -3155,6 +3323,29 @@ function planEntityQueryCore(document, entities, mapping, operators) {
    * that turns a product into a join. */
   const refinements = [];
   const filters = new Map(bindings.map((binding) => [binding.slot, null]));
+  const scopedFilters = [];
+  // Boolean composition may span bindings once mandatory equijoin
+  // edges establish the candidate tuples. Each leaf still has exactly
+  // one owner and uses that binding's existing total predicate forms.
+  const scopedPredicate = (node) => {
+    if (node.kind === 'op' && (node.name === '$and' || node.name === '$or')) {
+      const items = node.args.map(scopedPredicate);
+      return items.some((item) => item === null) ? null
+        : { p: node.name === '$and' ? 'and' : 'or', items };
+    }
+    if (node.kind === 'op' && node.name === '$not') {
+      const item = scopedPredicate(node.args[0]);
+      return item === null ? null : { p: 'not', item };
+    }
+    const slots = new Set();
+    collectBindingSlots(node, byName, slots);
+    if (slots.size !== 1) return null;
+    const slot = [...slots][0];
+    const binding = byName.get(slot);
+    const outcome = planEntityPredicate(node, slot, binding.shape);
+    return 'refusal' in outcome ? null
+      : { p: 'binding', binding: binding.name, filter: outcome.pred };
+  };
   const reasons = [];
   let whereFullyPushed = true;
   for (const conjunct of conjuncts) {
@@ -3190,6 +3381,8 @@ function planEntityQueryCore(document, entities, mapping, operators) {
     const slots = new Set();
     collectBindingSlots(conjunct, byName, slots);
     if (slots.size !== 1) {
+      const scoped = scopedPredicate(conjunct);
+      if (scoped !== null) { scopedFilters.push(scoped); continue; }
       reasons.push(refusal('$where', ENTITY_REASONS.conjunctBinding));
       whereFullyPushed = false;
       continue;
@@ -3255,16 +3448,22 @@ function planEntityQueryCore(document, entities, mapping, operators) {
   // SHAPE the projection tree rebuilds from the bindings' members
   const retBinding = root.ret.kind === 'var' && root.ret.external !== true
     ? byName.get(root.ret.slot) : undefined;
-  const projection = retBinding === undefined && aggregate === null
-    ? entityProjectionTree(root.ret, byName) : null;
-  if (retBinding === undefined && projection === null) {
+  const projection = retBinding === undefined && group === null
+    ? entityProjectionTree(root.ret, byName, entities, mapping) : null;
+  if (retBinding === undefined && projection === null && group === null) {
     reasons.push(refusal('$return', ENTITY_REASONS.projection));
   }
+  if (projection?.tree.p === 'leaf' && !projection.leaves[projection.tree.index].count && (aggregate === 'count' || windows.length > 0)) {
+    const leaf = projection.leaves[projection.tree.index];
+    const binding = bindings.find((entry) => entry.name === leaf.binding);
+    filters.set(binding.slot, conjoin(filters.get(binding.slot),
+      { p: 'typeIs', ref: leaf.ref, types: [], positive: true }));
+  }
 
   // ordering over flavored refs of either binding
   let order = null;
   let orderPushed = true;
-  if (root.orderby !== null) {
+  if (root.orderby !== null && group === null) {
     const terms = [];
     for (const spec of root.orderby.specs) {
       const slot = spec.key.kind === 'path' ? spec.key.rootSlot : -1;
@@ -3276,6 +3475,7 @@ function planEntityQueryCore(document, entities, mapping, operators) {
       // (a COLUMN stores it absent, §9.3, so a nullable column pushes)
       if (ref === null || (ref.flavor === 'entity-doc' && ref.type === 'unknown')
         || ref.type === 'boolean'
+        || ref.nullPolicy === 'null'
         || (ref.flavor === 'entity-doc' && admitsNull(binding.shape.schema, ref.segments))
         || spec.collation !== null || spec.collationName !== null) {
         orderPushed = false;
@@ -3288,10 +3488,14 @@ function planEntityQueryCore(document, entities, mapping, operators) {
   }
 
   const fullyPushed = whereFullyPushed && orderPushed
-    && (retBinding !== undefined || projection !== null);
+    && (retBinding !== undefined || projection !== null || group !== null);
   if (!fullyPushed) {
     return { analysis, mode: 'set', plan: null, referenced, reasons };
   }
+  for (const filter of scopedFilters) {
+    const slot = bindings[0].slot;
+    filters.set(slot, conjoin(filters.get(slot), filter));
+  }
 
   let window = null;
   if (windows.length > 0) {
@@ -3314,7 +3518,11 @@ function planEntityQueryCore(document, entities, mapping, operators) {
     plan: {
       planVersion: PLAN_VERSION,
       alg: bindings.length > 1 ? 'entity-join' : 'entity-select',
-      bindings: bindings.map((binding) => ({ name: binding.name, entity: binding.entity })),
+      bindings: bindings.map((binding) => ({ name: binding.name, entity: binding.entity,
+        ...(entities.get(binding.entity).physical == null ? {} : {
+          keys: mapping.entities[binding.entity].keys.map((key) =>
+            mapping.entities[binding.entity].columns.find((c) => c.name === key).physical),
+        }) })),
       // the FROM order and each binding's join conditions; `bindings`
       // stays in the DOCUMENT's order, which is the nested-loop order
       // the ORDER BY reproduces
@@ -3322,8 +3530,10 @@ function planEntityQueryCore(document, entities, mapping, operators) {
         binding: join.binding,
         on: join.on.map((edge) => ({
           op: edge.op ?? 'eq',
-          left: { binding: edge.left.binding.name, column: edge.left.ref.column },
-          right: { binding: edge.right.binding.name, column: edge.right.ref.column },
+          left: { binding: edge.left.binding.name, column: edge.left.ref.column,
+            ...(edge.left.ref.codec === undefined ? {} : { codec: edge.left.ref.codec }) },
+          right: { binding: edge.right.binding.name, column: edge.right.ref.column,
+            ...(edge.right.ref.codec === undefined ? {} : { codec: edge.right.ref.codec }) },
         })),
       })),
       filters: bindings.map((binding) => ({
@@ -3336,12 +3546,13 @@ function planEntityQueryCore(document, entities, mapping, operators) {
       })),
       window,
       aggregate,
+      group,
       ret: retBinding === undefined ? null : retBinding.name,
       // the projected shape, when the return is one: leaves that name
       // the binding they read from, and the tree the decoder rebuilds
       project: projection === null ? null : {
         tree: projection.tree,
-        leaves: projection.leaves.map((leaf) => ({ binding: leaf.binding, ref: leaf.ref })),
+        leaves: projection.leaves,
       },
     },
   };