@omgbase/oqx 0.24.0 → 0.26.0

package/src/engine.ts

@@ -139,6 +139,8 @@ const HARD_DEPTH_CAP = 8;
 type Projection = Pick<Subquery, "select" | "values">;
 // The grouping clauses of a body (0.24). Both `Query` and `Subquery` carry them.
 type Grouping = Pick<Subquery, "groupBy" | "having">;
+// The `flatten` clauses of a body (0.25). Both `Query` and `Subquery` carry them.
+type Flattening = Pick<Subquery, "flatten">;
 
 // The `$key: expr` item of a projection, if any — the identity the rows it
 // yields carry (SEMANTICS §16, since 0.20).
@@ -182,18 +184,19 @@ export class InMemoryEngine implements Engine {
     const src = this.coerceRows(this.evalRaw(query.source, root), query.source);
     const rows = this.reproject(src.rows, query.from, root);
     // One object as the source yields one record (SEMANTICS §3) — unless a
-    // `from` re-projection or a walk fans it out.
-    const single = src.single && query.from.length === 0 && !query.follow;
+    // `from` re-projection, a walk or a `flatten` fans it out.
+    const single = src.single && query.from.length === 0 && !query.follow && query.flatten.length === 0;
 
     const bound = this.boundOf(query, root);
     if (query.follow) return this.runFollow(query, rows, root, bound);
 
     // Consumer-directed short-circuits (skipped under `distinct`, which must
-    // materialize + dedup by projection before reducing, and under `group by`,
-    // whose consumer counts groups). Counting never materializes rows;
-    // exists/none stop as soon as the bound is known to be non-empty (the
-    // (offset+1)th match — or the first, when unbounded).
-    if (!query.distinct && !query.groupBy && (query.consumer === "exists" || query.consumer === "none" || query.consumer === "count")) {
+    // materialize + dedup by projection before reducing, under `group by`,
+    // whose consumer counts groups, and under `flatten`, whose consumer counts
+    // flattened rows). Counting never materializes rows; exists/none stop as
+    // soon as the bound is known to be non-empty (the (offset+1)th match — or
+    // the first, when unbounded).
+    if (!query.distinct && !query.groupBy && query.flatten.length === 0 && (query.consumer === "exists" || query.consumer === "none" || query.consumer === "count")) {
       const need = query.consumer === "count" ? Infinity : bound.offset + 1;
       let n = 0;
       for (const r of rows) {
@@ -208,7 +211,7 @@ export class InMemoryEngine implements Engine {
     // first over an unordered, non-distinct set needs only the rows up to the
     // bound (offset + 1). single materializes everything so its error can
     // report how many rows actually matched.
-    const cap = query.consumer === "first" && !query.orderBy && !query.distinct && !query.groupBy
+    const cap = query.consumer === "first" && !query.orderBy && !query.distinct && !query.groupBy && query.flatten.length === 0
       ? bound.offset + Math.min(1, bound.limit ?? 1)
       : Infinity;
     let kept: Scope[] = [];
@@ -219,6 +222,7 @@ export class InMemoryEngine implements Engine {
         if (kept.length >= cap) break;
       }
     }
+    if (query.flatten.length > 0) kept = this.flattenScopes(kept, query);
     if (query.groupBy) kept = this.groupScopes(kept, query, root);
     this.applyKey(kept, query);
     this.sortScopes(kept, query.orderBy);
@@ -329,6 +333,44 @@ export class InMemoryEngine implements Engine {
     return s;
   }
 
+  // ---- flatten (SEMANTICS §25) -----------------------------------------------
+
+  // Each `flatten` clause, in writing order, fans the scopes out: the expression
+  // is read in the row's scope (as `where` is) and its value split — an array
+  // (or a host collection) → one scope per element in collection order, so an
+  // empty collection yields none; absent → one scope with the name absent;
+  // anything else (an object, a scalar, a range) → one scope with the value.
+  // A flattened scope IS the parent scope — the same row, parent, lifts and
+  // intrinsics, so `$it`, `^` and `$depth` are unchanged — with the name bound
+  // beside the lifts (step 4 of §2: it shadows a same-named row property) and
+  // `$key` the pair `[parent identity, element identity]` (§16): an element's
+  // identity is the host's (an entry's key), else its structural value.
+  private flattenScopes(scopes: Scope[], body: Flattening): Scope[] {
+    for (const f of body.flatten) {
+      const out: Scope[] = [];
+      for (const s of scopes) {
+        const parentKey = this.scopeIdentity(s);
+        for (const elem of this.flattenElements(this.evalRaw(f.expr, s))) {
+          const key = [parentKey, elem == null ? null : this.hostIdentity(elem)];
+          out.push({ ...s, lifts: { ...(s.lifts ?? {}), [f.name]: elem }, meta: { ...(s.meta ?? {}), [KEY]: key } });
+        }
+      }
+      scopes = out;
+    }
+    return scopes;
+  }
+
+  // The elements a flattened value splits into (§25): §3's coercion with one
+  // softening — a scalar is one element, not an error, since no rows are
+  // sourced from it — and absent as one absent element rather than none.
+  private flattenElements(v: unknown): unknown[] {
+    if (v == null) return [undefined];
+    if (typeof v !== "object" || isRange(v) || isEntry(v)) return [v];
+    if (Array.isArray(v)) return v;
+    const single = this.ctx.isSingle ? this.ctx.isSingle(v) : !(Symbol.iterator in v);
+    return single ? [v] : Array.from(this.ctx.toRows(v));
+  }
+
   // ---- grouping (SEMANTICS §24) ----------------------------------------------
 
   // The kept scopes of a body, one GROUP scope per distinct key tuple, in order
@@ -425,6 +467,7 @@ export class InMemoryEngine implements Engine {
     const occ = this.followWalk(this.keyed(seeds, query, root), query.follow!, query.frontier, root);
     let scopes: Scope[] = occ.map((o) => this.enter(o.row, root, { lifts: {}, meta: o.meta }));
     if (post) scopes = scopes.filter((s) => this.evalWhere(post, s));
+    if (query.flatten.length > 0) scopes = this.flattenScopes(scopes, query);
     if (query.groupBy) scopes = this.groupScopes(scopes, query, root);
     this.sortScopes(scopes, query.orderBy);
     if (query.distinct) scopes = this.dedupByProjection(scopes, query);
@@ -633,7 +676,6 @@ export class InMemoryEngine implements Engine {
   }
 
   private evalWhereOp(op: OpNode, scope: Scope): boolean {
-    if (op.sub.follow) throw new OqxError("`follow` is only valid on a select-position collect { … }, not a where op", "eval");
     const plan = this.planOf(op, scope);
     if (!plan.invariant) return this.evalWhereOpPlanned(op, plan, scope);
     return this.memoized(op, scope, () => this.evalWhereOpPlanned(op, plan, scope));
@@ -644,15 +686,21 @@ export class InMemoryEngine implements Engine {
     if (op.op === "exists" || op.op === "none" || op.op === "count") {
       const acc = this.access(op, plan, scope);
       let n: number;
-      if (plan.fromCardinality && acc.where === null) {
+      // A block with a `follow` (legal in every body since 0.26) yields its
+      // walk's occurrences: no cardinality answer, no first-match stop — the
+      // walk is run and counted like any other rows.
+      if (op.sub.follow) {
+        n = this.opRows(op, scope, bound, acc).length;
+      } else if (plan.fromCardinality && acc.where === null) {
         // Nothing left to evaluate per row: the accessed rows ARE the matched
         // rows, and the consumer only needs their count after the bound.
         n = boundedCount(acc.rows.length, bound);
         scope.run.trace?.({ kind: "cardinality", count: n });
-      } else if (op.op !== "count" && bound === UNBOUNDED && !op.sub.groupBy) {
+      } else if (op.op !== "count" && bound === UNBOUNDED && !op.sub.groupBy && op.sub.flatten.length === 0) {
         // Unbounded exists/none: stop at the first match (dedup cannot change
-        // emptiness). Bounded, or grouped (`having` may empty the groups): the
-        // offset/limit decide emptiness, so materialize.
+        // emptiness). Bounded, grouped (`having` may empty the groups) or
+        // flattened (an empty collection yields no row): the offset/limit
+        // decide emptiness, so materialize.
         n = this.anyMatch(op, scope, acc) ? 1 : 0;
       } else n = this.opRows(op, scope, bound, acc).length;
       if (op.op === "count") return op.countCmp ? compareCount(n, op.countCmp) : n > 0;
@@ -677,13 +725,28 @@ export class InMemoryEngine implements Engine {
     return this.opRows(op, scope, bound).length > 0;
   }
 
-  // The rows a consumer op reduces: matched → grouped → ordered → distinct → bounded.
-  private opRows(op: OpNode, scope: Scope, bound: Bound, acc?: Access): Scope[] {
-    let scopes = this.matchRows(op, scope, acc);
-    if (op.sub.groupBy) scopes = this.groupScopes(scopes, op.sub, scope);
-    this.applyKey(scopes, op.sub);
-    this.sortScopes(scopes, op.sub.orderBy);
-    if (op.distinct) scopes = this.dedupByProjection(scopes, op.sub);
+  // The rows a consumer op reduces — in every position (a where-position test,
+  // a select-position directive, a destination block): matched (or, with a
+  // `follow`, the walk's occurrences over the matched seeds) → flattened →
+  // grouped → ordered → distinct → bounded. `destination` marks a follow
+  // destination block: its `frontier` is the destination's (read by the outer
+  // walk), never a seed frontier for a `follow` of its own.
+  private opRows(op: OpNode, scope: Scope, bound: Bound, acc?: Access, destination = false): Scope[] {
+    const sub = op.sub;
+    let scopes: Scope[];
+    if (sub.follow) {
+      // The seeds are the matched rows themselves (`followWalk` enters them); a
+      // follow block is never correlated, so `where` here is the whole predicate.
+      const { rows, where } = acc ?? this.access(op, this.planOf(op, scope), scope);
+      const seeds = where ? rows.filter((r) => this.evalWhere(where, this.enter(r, scope, { lifts: {} }))) : rows;
+      scopes = this.followWalk(this.keyed(seeds, sub, scope), sub.follow, destination ? null : sub.frontier, scope).map((o) => this.enter(o.row, scope, { meta: o.meta }));
+    } else scopes = this.matchRows(op, scope, acc);
+    if (sub.flatten.length > 0) scopes = this.flattenScopes(scopes, sub);
+    if (sub.groupBy) scopes = this.groupScopes(scopes, sub, scope);
+    // An occurrence's `$key` is the walk's identity (`keyed` gave the seeds theirs).
+    if (!sub.follow) this.applyKey(scopes, sub);
+    this.sortScopes(scopes, sub.orderBy);
+    if (op.distinct) scopes = this.dedupByProjection(scopes, sub);
     return sliceBound(scopes, bound);
   }
 
@@ -765,9 +828,9 @@ export class InMemoryEngine implements Engine {
   }
 
   // The plain scan: the rows, re-projected by `from`, with the whole `where`.
-  // A `from` re-projection fans a single receiver out: the block is plural.
+  // A `from` re-projection or a `flatten` fans a single receiver out: the block is plural.
   private scan(src: Rows, op: OpNode, scope: Scope): Access {
-    return { rows: this.reproject(src.rows, op.sub.from, scope), where: op.sub.where, single: src.single && op.sub.from.length === 0 };
+    return { rows: this.reproject(src.rows, op.sub.from, scope), where: op.sub.where, single: src.single && op.sub.from.length === 0 && op.sub.flatten.length === 0 };
   }
 
   private reproject(rows: unknown[], from: readonly Expr[], scope: Scope): unknown[] {
@@ -883,23 +946,10 @@ export class InMemoryEngine implements Engine {
   private evalCollectScopes(op: OpNode, scope: Scope, plan: BlockPlan = this.planOf(op, scope), destination = false): { scopes: Scope[]; single: boolean } {
     const sub = op.sub;
     const bound = this.boundOf(sub, scope);
-    let scopes: Scope[];
-    let single = false;
-    if (sub.follow) {
-      // The seeds are the matched rows themselves (`followWalk` enters them); a
-      // follow block is never correlated, so `where` here is the whole predicate.
-      const { rows, where } = this.access(op, plan, scope);
-      const seeds = where ? rows.filter((r) => this.evalWhere(where, this.enter(r, scope, { lifts: {} }))) : rows;
-      scopes = this.followWalk(this.keyed(seeds, sub, scope), sub.follow, destination ? null : sub.frontier, scope).map((o) => this.enter(o.row, scope, { meta: o.meta }));
-      if (sub.groupBy) scopes = this.groupScopes(scopes, sub, scope);
-      this.sortScopes(scopes, sub.orderBy);
-      if (op.distinct) scopes = this.dedupByProjection(scopes, sub);
-      scopes = sliceBound(scopes, bound);
-    } else {
-      const acc = this.access(op, plan, scope);
-      single = acc.single;
-      scopes = this.opRows(op, scope, bound, acc);
-    }
+    const acc = this.access(op, plan, scope);
+    // A walk is always plural (SEMANTICS §20).
+    const single = sub.follow ? false : acc.single;
+    const scopes = this.opRows(op, scope, bound, acc, destination);
     switch (op.op) {
       case "collect": case "count": return { scopes, single };
       case "first": return { scopes: scopes.slice(0, 1), single };

package/src/optimize/analysis.ts

@@ -72,6 +72,8 @@ function subReadsScopeIn(sub: Subquery, at: number, upTo: number, destination: b
   if (sub.limit && exprReadsScopeIn(sub.limit, at, upTo)) return true;
   if (sub.offset && exprReadsScopeIn(sub.offset, at, upTo)) return true;
   if (sub.frontier && exprReadsScopeIn(sub.frontier, destination ? at - 1 : at, upTo)) return true;
+  // A `flatten` expression is read on each row (0.25); its block is an op read there.
+  if (sub.flatten.some((f) => exprReadsScopeIn(f.expr, at, upTo))) return true;
   // `group by` keys are read on the rows, `having` on the groups — both at `at` (0.24).
   if (sub.groupBy?.keys.some((k) => exprReadsScopeIn(k.expr, at, upTo))) return true;
   if (sub.having && whereReadsScopeIn(sub.having, at, upTo)) return true;
@@ -133,8 +135,9 @@ export function whereRaiseFree(w: Where, ctx: RuleContext): boolean {
 
 /** A where-position directive that can neither raise nor bind: `exists`,
  * `none` or `count` (never `single`, which raises on several rows; `collect`
- * binds lifts; `first`/`single` project), without `follow` (raises in where
- * position), without a bound (an invalid value raises), without `distinct`
+ * binds lifts; `first`/`single` project), without `follow` (a walk's
+ * destinations may raise, and a walk is never a cardinality answer), without a
+ * bound (an invalid value raises), without `distinct`
  * (dedup projects, and a projected range raises), whose receiver, `from`,
  * `where` and `order by` (evaluated by `count`) are raise-free and which lifts
  * nothing anywhere. */
@@ -148,6 +151,9 @@ export function opRaiseFree(op: OpNode, ctx: RuleContext): boolean {
   if (!sub.from.every((e) => exprRaiseFree(e, ctx))) return false;
   if (sub.where && !whereRaiseFree(sub.where, ctx)) return false;
   if (sub.orderBy && !sub.orderBy.every((o) => exprRaiseFree(o.expr, ctx))) return false;
+  // A `flatten` value is coerced without error (a scalar is one row, §25), so
+  // the clause raises only when its expression does.
+  if (!sub.flatten.every((f) => exprRaiseFree(f.expr, ctx))) return false;
   return !opHasLifts(op);
 }
 
@@ -163,6 +169,7 @@ function subHasLifts(sub: Subquery): boolean {
   if (sub.select.some((item) => (item.kind === "field" ? item.lift > 0 : opHasLifts(item.op)))) return true;
   if (sub.where && whereHasLifts(sub.where)) return true;
   if (sub.having && whereHasLifts(sub.having)) return true;
+  if (sub.flatten.some((f) => f.expr.kind === "op" && opHasLifts(f.expr))) return true;
   if (sub.follow?.destinations.some((d) => d.kind === "op" && opHasLifts(d))) return true;
   return false;
 }

package/src/optimize/ir.ts

@@ -10,7 +10,7 @@
 // cannot be built, a receiver that is not stable) always degrade to the plain
 // scan the plan started from.
 
-import type { Consumer, Expr, Follow, GroupBy, OpNode, OrderSpec, SelectItem, Where } from "../ast.ts";
+import type { Consumer, Expr, Flatten, Follow, GroupBy, OpNode, OrderSpec, SelectItem, Where } from "../ast.ts";
 
 /** One hoisted equality `local == outer` from the block's top-level `&&`
  * conjunction: `local` reads only the block's row (an identifier or member
@@ -45,6 +45,9 @@ export interface BlockPlan {
   readonly distinct: boolean;
   readonly bound: { readonly limit: Expr | null; readonly offset: Expr | null };
   readonly follow: Follow | null;
+  /** `flatten` clauses (0.25): the matched rows are fanned out per element
+   * before the consumer sees them, so a consumer counts FLATTENED rows. */
+  readonly flatten: readonly Flatten[];
   /** `group by` / `having` (0.24): the matched rows are grouped before the
    * consumer sees them, so a consumer counts GROUPS, never rows. */
   readonly groupBy: GroupBy | null;
@@ -94,6 +97,7 @@ export function logicalBlock(node: OpNode, depth: number): BlockPlan {
     distinct: node.distinct,
     bound: { limit: sub.limit, offset: sub.offset },
     follow: sub.follow,
+    flatten: sub.flatten,
     groupBy: sub.groupBy,
     having: sub.having,
     correlated: [],

package/src/optimize/rules.ts

@@ -113,7 +113,8 @@ export const invariantBlock: Rule = (plan) => {
 // Precondition. A where-position `exists`, `none` or `count` without `distinct`
 // (dedup projects every row), without `order by` (`count` evaluates the keys),
 // without `follow`, without `group by` (the consumer counts groups, not rows —
-// 0.24), and with nothing left to evaluate per row: either the block has no
+// 0.24), without `flatten` (the consumer counts flattened rows — 0.25), and
+// with nothing left to evaluate per row: either the block has no
 // `where`, or every conjunct is correlated (so the probe's bucket is the
 // matched set).
 //
@@ -128,7 +129,7 @@ export const invariantBlock: Rule = (plan) => {
 export const cardinalityOnly: Rule = (plan) => {
   if (plan.fromCardinality) return null;
   if (plan.consumer !== "exists" && plan.consumer !== "none" && plan.consumer !== "count") return null;
-  if (plan.distinct || plan.orderBy || plan.follow || plan.groupBy) return null;
+  if (plan.distinct || plan.orderBy || plan.follow || plan.groupBy || plan.flatten.length > 0) return null;
   if (plan.where !== null && plan.residual !== null) return null;
   return { ...plan, fromCardinality: true };
 };