@omgbase/oqx 0.14.0 → 0.14.1

package/src/optimize/driver.ts

@@ -0,0 +1,37 @@
+// The optimizer driver: applies a rule set to a block's logical plan until no
+// rule changes it (a fixpoint, bounded), and caches the result per AST node —
+// the analysis is syntactic, so a plan depends only on the node, its static
+// depth, the run's binding count and the rule set.
+
+import type { OpNode } from "../ast.ts";
+import type { BlockPlan, Rule, RuleContext } from "./ir.ts";
+import { logicalBlock } from "./ir.ts";
+
+const MAX_PASSES = 8;
+
+/** Rewrite `logicalBlock(node, depth)` to a fixpoint of `rules`. */
+export function optimizeBlock(node: OpNode, depth: number, ctx: RuleContext, rules: readonly Rule[]): BlockPlan {
+  let plan = logicalBlock(node, depth);
+  for (let pass = 0; pass < MAX_PASSES; pass++) {
+    let changed = false;
+    for (const rule of rules) {
+      const next = rule(plan, ctx);
+      if (next !== null && next !== plan) { plan = next; changed = true; }
+    }
+    if (!changed) break;
+  }
+  return plan;
+}
+
+interface Cached { depth: number; bindingCount: number; rules: readonly Rule[]; plan: BlockPlan }
+const cache = new WeakMap<OpNode, Cached>();
+
+/** `optimizeBlock`, memoized on the node (re-planned when depth, binding count
+ * or rule set differ from the cached entry). */
+export function planFor(node: OpNode, depth: number, ctx: RuleContext, rules: readonly Rule[]): BlockPlan {
+  const hit = cache.get(node);
+  if (hit && hit.depth === depth && hit.bindingCount === ctx.bindingCount && hit.rules === rules) return hit.plan;
+  const plan = optimizeBlock(node, depth, ctx, rules);
+  cache.set(node, { depth, bindingCount: ctx.bindingCount, rules, plan });
+  return plan;
+}

package/src/optimize/hash-index.ts

@@ -0,0 +1,91 @@
+// A hash index over the rows of one collection, keyed under OQX equality
+// (SEMANTICS §5) — the only key discipline that makes an index probe return
+// exactly the rows a `local == value` scan would keep:
+//
+//   • absent is one value: `undefined` and `null` share a bucket;
+//   • numbers are doubles: `-0` meets `0`; `NaN` equals nothing, so a row whose
+//     key is `NaN` is never indexed and a probe for `NaN` finds nothing;
+//   • strings, booleans and bigints key by value with a type tag, so `"5"`,
+//     `5` and `true` stay apart;
+//   • objects, arrays, functions and symbols key by REFERENCE — the reference
+//     engine's `===` — never structurally.
+//
+// Buckets hold row POSITIONS in ascending order, so a probe yields rows in the
+// receiver's order (§3 and §13 — a block's rows are the receiver's, in order).
+
+/** A pre-built equality index a `DataContext` may expose for a collection
+ * (see `DataContext.indexFor`): `lookup(value)` returns the ascending positions
+ * (into `toRows(collection)`) of the rows whose key equals `value` under §5.
+ *
+ * `lookupRows`, when present, answers the same probe with the ROWS themselves —
+ * exactly the rows `lookup(value)` would select, in receiver order — without
+ * the engine ever materializing `toRows(collection)`. A store-backed context
+ * implements it over its own indexes (one indexed statement per probe); the
+ * engine prefers it for a statically stable receiver, so the collection is
+ * never read whole. A throw is the query's error (as a `get` that throws is). */
+export interface RowIndex {
+  lookup(value: unknown): readonly number[];
+  lookupRows?(value: unknown): Iterable<unknown>;
+}
+
+const NONE: readonly number[] = Object.freeze([]);
+
+/** The engine's own `RowIndex`: `add` every row's key in row order, then probe. */
+export class HashIndex implements RowIndex {
+  private prim = new Map<string, number[]>();
+  private refs = new Map<unknown, number[]>();
+  private count = 0;
+
+  /** Record that the row at `position` has key `value`. Positions must be
+   * added in ascending order (row order). A `NaN` key is not indexed. */
+  add(value: unknown, position: number): void {
+    const k = primitiveKey(value);
+    if (k === null) return; // NaN: equals nothing
+    const map = k === undefined ? this.refs : this.prim;
+    const key = k === undefined ? value : k;
+    let bucket = map.get(key);
+    if (!bucket) map.set(key, (bucket = []));
+    bucket.push(position);
+    this.count++;
+  }
+
+  lookup(value: unknown): readonly number[] {
+    const k = primitiveKey(value);
+    if (k === null) return NONE;
+    return (k === undefined ? this.refs.get(value) : this.prim.get(k)) ?? NONE;
+  }
+
+  /** How many rows were indexed (rows with a `NaN` key are not). */
+  get size(): number {
+    return this.count;
+  }
+}
+
+/** The tagged bucket key of a primitive, `null` for `NaN` (no key: it equals
+ * nothing), `undefined` for a reference-keyed value (object, array, function,
+ * symbol — compared by identity). Mirrors `semantics.equals`: `===` after
+ * normalizing `undefined` to `null`. */
+export function primitiveKey(v: unknown): string | null | undefined {
+  switch (typeof v) {
+    case "undefined": return "∅"; // absent ≡ absent
+    case "object": return v === null ? "∅" : undefined;
+    case "number": return Number.isNaN(v) ? null : `n${v}`; // `${-0}` is "0": -0 meets 0
+    case "string": return `s${v}`;
+    case "boolean": return v ? "t" : "f";
+    case "bigint": return `i${v}`;
+    default: return undefined; // function, symbol: reference identity
+  }
+}
+
+/** Intersect two ascending position lists, ascending. */
+export function intersectPositions(a: readonly number[], b: readonly number[]): readonly number[] {
+  const out: number[] = [];
+  let i = 0, j = 0;
+  while (i < a.length && j < b.length) {
+    const x = a[i]!, y = b[j]!;
+    if (x === y) { out.push(x); i++; j++; }
+    else if (x < y) i++;
+    else j++;
+  }
+  return out;
+}

package/src/optimize/index.ts

@@ -0,0 +1,16 @@
+// The optimizer: a logical plan per nested block, rules that rewrite it, a
+// driver that applies them to a fixpoint, and the equality-faithful hash index
+// the hash-probe rule executes with. See `rules.ts` for the catalog and each
+// rule's semantics argument; README "Performance: relational patterns" for the
+// user-facing summary.
+
+export type { BlockPlan, Correlation, Rule, RuleContext } from "./ir.ts";
+export { logicalBlock, conjunction, conjuncts, residualWithout, lookupOrder } from "./ir.ts";
+export { optimizeBlock, planFor } from "./driver.ts";
+export { correlatedEqualityProbe, stableReceiver, invariantBlock, cardinalityOnly, DEFAULT_RULES } from "./rules.ts";
+export type { RowIndex } from "./hash-index.ts";
+export { HashIndex, primitiveKey, intersectPositions } from "./hash-index.ts";
+export {
+  exprReadsScopeIn, whereReadsScopeIn, opReadsScopeIn, exprReadsCurrentScope,
+  exprRaiseFree, whereRaiseFree, opRaiseFree, opHasLifts, whereHasLifts, localPath, SCOPE_INTRINSICS,
+} from "./analysis.ts";

package/src/optimize/ir.ts

@@ -0,0 +1,128 @@
+// The optimizer's logical representation of one nested block (an `OpNode`):
+// where its rows come from, which predicates correlate them with the enclosing
+// rows, what remains to be checked per row, how they are shaped, and how the
+// consumer reduces them. Rules rewrite a `BlockPlan`; the engine executes it.
+//
+// A plan starts as a faithful transcription of the AST (`logicalBlock`): every
+// predicate residual, nothing correlated, nothing memoized. Rules move work
+// from the per-row path into the per-run or per-probe path only when the
+// semantics argument in `rules.ts` holds; the engine's fallbacks (an index that
+// cannot be built, a receiver that is not stable) always degrade to the plain
+// scan the plan started from.
+
+import type { Consumer, Expr, Follow, OpNode, OrderSpec, SelectItem, Where } from "../ast.ts";
+
+/** One hoisted equality `local == outer` from the block's top-level `&&`
+ * conjunction: `local` reads only the block's row (an identifier or member
+ * chain; `path` is its property path, `[]` for `$value`), `outer` reads nothing
+ * from the block's row (outer references, bindings, literals, arithmetic over
+ * them) and cannot raise. */
+export interface Correlation {
+  readonly local: Expr;
+  readonly outer: Expr;
+  readonly path: readonly string[];
+  /** Position of the equality among `conjuncts(where)`, so a probe that
+   * answers ONE correlation (a store-backed `lookupRows`) can put the others
+   * back in their place as residual conjuncts. */
+  readonly index: number;
+}
+
+/** The plan for one block. `receiver`/`from`/`projection`/`orderBy`/`bound`/
+ * `consumer`/`follow`/`distinct` transcribe the AST; `correlated`/`residual`
+ * split its `where`; `receiverStable`, `invariant` and `fromCardinality` are the
+ * annotations rules add. */
+export interface BlockPlan {
+  readonly node: OpNode;
+  /** Scope depth of the ENCLOSING scope (root = 0): the receiver is read there;
+   * the block's rows are scopes at `depth + 1`. */
+  readonly depth: number;
+  readonly consumer: Consumer;
+  readonly receiver: Expr;
+  readonly from: readonly Expr[];
+  readonly where: Where | null;
+  readonly projection: { readonly select: readonly SelectItem[]; readonly values: boolean };
+  readonly orderBy: readonly OrderSpec[] | null;
+  readonly distinct: boolean;
+  readonly bound: { readonly limit: Expr | undefined; readonly offset: Expr | undefined };
+  readonly follow: Follow | null;
+  /** Equalities answered by an index probe on the receiver (in conjunct order). */
+  readonly correlated: readonly Correlation[];
+  /** The conjuncts still evaluated per row once the probe has selected its
+   * bucket; equals `where` when nothing is correlated. */
+  readonly residual: Where | null;
+  /** The receiver reads nothing from any enclosing row (only the root scope
+   * and bindings), so its rows are the same for every enclosing row and can be
+   * materialized and indexed once per run. */
+  readonly receiverStable: boolean;
+  /** The whole block reads nothing from any enclosing row and has no side
+   * effects on one (no lifts): its value is the same for every enclosing row
+   * and is computed once per run. */
+  readonly invariant: boolean;
+  /** `exists`/`none`/`count` may be answered from the cardinality of the
+   * accessed rows when no residual predicate remains (nothing per row to
+   * evaluate, order, dedup or project). */
+  readonly fromCardinality: boolean;
+}
+
+/** What a rule may consult besides the plan. */
+export interface RuleContext {
+  /** How many positional bindings this run has (a `${i}` with `i` at or past
+   * it raises when evaluated, so it is not raise-free). */
+  readonly bindingCount: number;
+}
+
+/** A rewrite: return an improved plan, or `null` when the rule does not apply
+ * (or has already been applied). Rules must be idempotent on their own output. */
+export type Rule = (plan: BlockPlan, ctx: RuleContext) => BlockPlan | null;
+
+/** The faithful transcription of a block: a scan with every predicate residual. */
+export function logicalBlock(node: OpNode, depth: number): BlockPlan {
+  const sub = node.sub;
+  return {
+    node,
+    depth,
+    consumer: node.op,
+    receiver: node.receiver,
+    from: sub.from,
+    where: sub.where,
+    projection: { select: sub.select, values: sub.values ?? false },
+    orderBy: sub.orderBy,
+    distinct: node.distinct ?? false,
+    bound: { limit: sub.limit, offset: sub.offset },
+    follow: sub.follow,
+    correlated: [],
+    residual: sub.where,
+    receiverStable: false,
+    invariant: false,
+    fromCardinality: false,
+  };
+}
+
+/** Rebuild a conjunction from its remaining parts. */
+export function conjunction(parts: readonly Where[]): Where | null {
+  return parts.length === 0 ? null : parts.length === 1 ? parts[0]! : { kind: "and", parts: [...parts] };
+}
+
+/** The top-level `&&` conjuncts of a where tree (a non-`and` tree is one). */
+export function conjuncts(where: Where): readonly Where[] {
+  return where.kind === "and" ? where.parts : [where];
+}
+
+/** The residual when exactly ONE correlation is answered by a probe (a context
+ * index's `lookupRows`): every other conjunct, the remaining equalities
+ * included, stays in its original place. A kept equality is true by
+ * construction for the rows the probe selects and, being two reads, can neither
+ * raise nor bind, so the scan's strict left-to-right order and outcome hold. */
+export function residualWithout(plan: BlockPlan, answered: Correlation): Where | null {
+  return plan.where ? conjunction(conjuncts(plan.where).filter((_, i) => i !== answered.index)) : null;
+}
+
+/** The order in which a single-correlation probe tries the plan's correlations:
+ * those whose outer side varies with the enclosing row first (a literal selects
+ * the same rows for every enclosing row and narrows nothing), each group in
+ * conjunct order. */
+export function lookupOrder(plan: BlockPlan): readonly Correlation[] {
+  const all = plan.correlated;
+  if (all.length < 2) return all;
+  return [...all.filter((c) => c.outer.kind !== "lit"), ...all.filter((c) => c.outer.kind === "lit")];
+}

package/src/optimize/rules.ts

@@ -0,0 +1,136 @@
+// The rule catalog. Each rule states its precondition and the argument that the
+// rewritten plan is indistinguishable from the scan it replaces — same rows in
+// the same order, same lifts, same error (same message) at the same point, or
+// no error — under SEMANTICS §13's strict left-to-right `&&`.
+
+import type { Expr, Where } from "../ast.ts";
+import {
+  exprRaiseFree, exprReadsCurrentScope, exprReadsScopeIn, localPath, opHasLifts, opReadsScopeIn,
+  whereHasLifts, whereRaiseFree,
+} from "./analysis.ts";
+import type { Correlation, Rule, RuleContext } from "./ir.ts";
+import { conjunction, conjuncts } from "./ir.ts";
+
+// ---- rule 1: correlated equality → hash probe ------------------------------------
+//
+// Precondition. The block has no `from` re-projection (its rows ARE the
+// receiver's rows) and no `follow`; its `where` has no lift anywhere (a lift
+// into an enclosing scope by one row could change what the outer side reads
+// for the next row); some top-level conjunct k is `local == outer` (either
+// side) with `local` an identifier/member chain on the block's row and `outer`
+// a raise-free expression reading nothing from the block's row; and every
+// conjunct left of k is raise-free (no call, no `single`, no lift, no bound, no
+// binding past the run's count).
+//
+// Argument. The scan evaluates, per receiver row r in order, c1(r) && … &&
+// ck(r) && …, short-circuiting. The probe selects exactly the rows with
+// local(r) == outer (the index reproduces §5 equality, `outer` is evaluated
+// once because nothing in the block can change what it reads) and evaluates the
+// residual conjunction over them, in receiver order. For a selected row every
+// conjunct is evaluated as the scan would (ck is dropped: it is true by
+// construction and, being two reads, cannot raise or bind). For a row the probe
+// rejects, the scan would have evaluated c1..c(k-1) — which cannot raise or
+// bind — then ck, false, and stopped: no result, no error, no lift. So the two
+// paths keep the same rows, raise the same first error, and bind the same
+// lifts. A probe that cannot be built at run time (a receiver that is not
+// stable, an index whose construction throws) falls back to the scan itself.
+
+export const correlatedEqualityProbe: Rule = (plan, ctx) => {
+  if (plan.correlated.length > 0 || plan.from.length > 0 || plan.follow || !plan.where) return null;
+  if (whereHasLifts(plan.where)) return null;
+  const correlated: Correlation[] = [];
+  const residual: Where[] = [];
+  let prefixRaiseFree = true;
+  conjuncts(plan.where).forEach((part, index) => {
+    const eq = prefixRaiseFree ? asCorrelation(part, index, ctx) : null;
+    if (eq) correlated.push(eq);
+    else {
+      residual.push(part);
+      prefixRaiseFree &&= whereRaiseFree(part, ctx);
+    }
+  });
+  if (correlated.length === 0) return null;
+  return { ...plan, correlated, residual: conjunction(residual) };
+};
+
+/** `local == outer` / `outer == local` as a correlation, else null. */
+function asCorrelation(w: Where, index: number, ctx: RuleContext): Correlation | null {
+  if (w.kind !== "scalar" || w.expr.kind !== "binary" || w.expr.op !== "==") return null;
+  const { left, right } = w.expr;
+  return pair(left, right, index, ctx) ?? pair(right, left, index, ctx);
+}
+
+function pair(local: Expr, outer: Expr, index: number, ctx: RuleContext): Correlation | null {
+  const path = localPath(local);
+  if (path === null) return null;
+  if (exprReadsCurrentScope(outer) || !exprRaiseFree(outer, ctx)) return null;
+  return { local, outer, path, index };
+}
+
+// ---- rule 1b: a receiver stable across enclosing rows ----------------------------
+//
+// Precondition. The receiver expression, read in the enclosing scope (depth
+// `plan.depth`), reads no scope at depth 1..depth: only the root (named roots,
+// `^…^name` reaching it, or past it) and bindings.
+//
+// Argument. Such an expression yields the same value for every enclosing row
+// (evaluation is pure; the root and the bindings do not change within a run),
+// so its rows can be materialized once per run and indexed once per path. A
+// receiver that does read an enclosing row is still indexable when it happens
+// to evaluate to the same object twice (the engine keys indexes by collection
+// identity as well); a receiver that yields a fresh value per row is scanned.
+
+export const stableReceiver: Rule = (plan) => {
+  if (plan.receiverStable) return null;
+  if (exprReadsScopeIn(plan.receiver, plan.depth, plan.depth)) return null;
+  return { ...plan, receiverStable: true };
+};
+
+// ---- rule 2: invariant nested block → evaluate once per run ----------------------
+//
+// Precondition. Nothing in the block — receiver, `from`, `where`, `select`,
+// `order by`, bound, `follow` — reads a scope at depth 1..depth (an enclosing
+// row), and the block lifts nothing (a lift is a side effect on an enclosing
+// scope that must recur per row).
+//
+// Argument. The block's value is a function of the root scope, the bindings
+// and its own rows only, all constant within a run, so every evaluation yields
+// the same value or raises the same error. The engine evaluates it the first
+// time the scan would (so an error surfaces at the same point) and returns the
+// memoized value afterwards; a run that raised never reaches a second
+// evaluation. The memoized value is shared by reference across the enclosing
+// rows that project it.
+
+export const invariantBlock: Rule = (plan) => {
+  if (plan.invariant) return null;
+  if (opHasLifts(plan.node)) return null;
+  if (opReadsScopeIn(plan.node, plan.depth, plan.depth)) return null;
+  return { ...plan, invariant: true };
+};
+
+// ---- rule 3: exists / none / count from cardinality -------------------------------
+//
+// Precondition. A where-position `exists`, `none` or `count` without `distinct`
+// (dedup projects every row), without `order by` (`count` evaluates the keys),
+// without `follow`, and with nothing left to evaluate per row: either the block
+// has no `where`, or every conjunct is correlated (so the probe's bucket is the
+// matched set).
+//
+// Argument. With no residual predicate the matched rows are exactly the
+// accessed rows (§13: the receiver's rows filtered by `where`), entering a row
+// as a scope cannot raise, and the consumers only need the count after the
+// bound (§18): `count` is `boundedCount(n)`, `exists` is `boundedCount(n) > 0`,
+// `none` its negation — what the scan computes after materializing. When the
+// probe is unavailable at run time the residual is the whole `where` and the
+// engine takes the per-row path.
+
+export const cardinalityOnly: Rule = (plan) => {
+  if (plan.fromCardinality) return null;
+  if (plan.consumer !== "exists" && plan.consumer !== "none" && plan.consumer !== "count") return null;
+  if (plan.distinct || plan.orderBy || plan.follow) return null;
+  if (plan.where !== null && plan.residual !== null) return null;
+  return { ...plan, fromCardinality: true };
+};
+
+/** The rules the engine applies by default, in order. */
+export const DEFAULT_RULES: readonly Rule[] = [correlatedEqualityProbe, stableReceiver, invariantBlock, cardinalityOnly];