@omgbase/oqx 0.14.0 → 0.15.0

package/dist/engine.d.ts

@@ -1,5 +1,6 @@
-import type { Query } from "./ast.js";
+import type { Query, OpNode } from "./ast.js";
 import type { DataContext } from "./context.js";
+import type { Rule } from "./optimize/index.js";
 /** The shaped result of a top-level query, discriminated by consumer. */
 export type OqxResult = {
     consumer: "collect";
@@ -24,9 +25,47 @@ export type OqxResult = {
 export interface Engine {
     run(query: Query, bindings: readonly unknown[]): OqxResult;
 }
+/** What the optimizer did at run time (for tests and diagnostics): an index
+ * built (or taken from the context) for a collection and path, a probe that
+ * selected a bucket, a context index that answered a probe with rows directly
+ * (`lookupRows`: the collection was never materialized), a memoized invariant
+ * block reused, a consumer answered from a cardinality, or a probe that fell
+ * back to the scan. */
+export type TraceEvent = {
+    kind: "index";
+    path: readonly string[];
+    rows: number;
+    source: "engine" | "context";
+} | {
+    kind: "probe";
+    paths: readonly (readonly string[])[];
+    candidates: number;
+} | {
+    kind: "lookup";
+    path: readonly string[];
+    candidates: number;
+} | {
+    kind: "memo";
+    op: OpNode;
+} | {
+    kind: "cardinality";
+    count: number;
+} | {
+    kind: "fallback";
+    reason: "unstable" | "index" | "probe-value" | "not-a-collection";
+};
+export interface InMemoryEngineOptions {
+    /** The optimizer rules to apply to nested blocks (default `DEFAULT_RULES`);
+     * `[]` runs the naive scan everywhere. */
+    rules?: readonly Rule[];
+    /** Receives a `TraceEvent` for each optimizer action during `run`. */
+    trace?: (event: TraceEvent) => void;
+}
 export declare class InMemoryEngine implements Engine {
     private ctx;
-    constructor(context?: DataContext);
+    private rules;
+    private trace;
+    constructor(context?: DataContext, options?: InMemoryEngineOptions);
     run(query: Query, bindings?: readonly unknown[]): OqxResult;
     private boundOf;
     private matches;
@@ -41,13 +80,27 @@ export declare class InMemoryEngine implements Engine {
     private itemValue;
     private evalWhere;
     private evalWhereOp;
+    private evalWhereOpPlanned;
     private opRows;
     private anyMatch;
     private iterMatchRows;
     private matchRows;
+    private planOf;
+    private memoized;
+    private access;
+    private scan;
+    private reproject;
+    private materialize;
+    private lookupRows;
+    private probe;
+    private outerValue;
+    private contextIndex;
+    private indexFor;
     private evalCollectValue;
+    private evalCollectValuePlanned;
     private sortScopes;
     private evalExpr;
+    private evalRaw;
     private resolveIn;
     private evalCall;
 }

package/dist/engine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EACV,KAAK,EACN,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOhD,yEAAyE;AACzE,MAAM,MAAM,SAAS,GACjB;IAAE,QAAQ,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,OAAO,EAAE,CAAA;CAAE,GACxC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACvC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GACnC;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAA;CAAE,GAC1C;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAA;CAAE,CAAC;AAEhD,6EAA6E;AAC7E,MAAM,WAAW,MAAM;IACrB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,SAAS,CAAC;CAC5D;AA4BD,qBAAa,cAAe,YAAW,MAAM;IAC3C,OAAO,CAAC,GAAG,CAAc;gBAEb,OAAO,GAAE,WAAkC;IAIvD,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,GAAE,SAAS,OAAO,EAAO,GAAG,SAAS;IAmD/D,OAAO,CAAC,OAAO;IAef,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,KAAK;IASb,OAAO,CAAC,KAAK;IAab,OAAO,CAAC,SAAS;IAiBjB,OAAO,CAAC,iBAAiB;IAuCzB,OAAO,CAAC,UAAU;IA4ElB,OAAO,CAAC,KAAK;IAgBb,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,WAAW;IAiCnB,OAAO,CAAC,MAAM;IAQd,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAE,aAAa;IAStB,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,gBAAgB;IA2BxB,OAAO,CAAC,UAAU;IAwBlB,OAAO,CAAC,QAAQ;IAsEhB,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,QAAQ;CAYjB;AAuGD,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,GAAG,SAAS,CAG9F"}
+{"version":3,"file":"engine.d.ts","sourceRoot":"","sources":["../src/engine.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EACV,KAAK,EAAe,MAAM,EAC3B,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAMhD,OAAO,KAAK,EAA0B,IAAI,EAAY,MAAM,qBAAqB,CAAC;AAGlF,yEAAyE;AACzE,MAAM,MAAM,SAAS,GACjB;IAAE,QAAQ,EAAE,SAAS,CAAC;IAAC,IAAI,EAAE,OAAO,EAAE,CAAA;CAAE,GACxC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,GACvC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GACnC;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACpC;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAA;CAAE,GAC1C;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAA;CAAE,CAAC;AAEhD,6EAA6E;AAC7E,MAAM,WAAW,MAAM;IACrB,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,SAAS,CAAC;CAC5D;AA2CD;;;;;uBAKuB;AACvB,MAAM,MAAM,UAAU,GAClB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,QAAQ,GAAG,SAAS,CAAA;CAAE,GACtF;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,KAAK,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC5E;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC/D;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC5B;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,UAAU,GAAG,OAAO,GAAG,aAAa,GAAG,kBAAkB,CAAA;CAAE,CAAC;AAE5F,MAAM,WAAW,qBAAqB;IACpC;8CAC0C;IAC1C,KAAK,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;IACxB,sEAAsE;IACtE,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;CACrC;AAcD,qBAAa,cAAe,YAAW,MAAM;IAC3C,OAAO,CAAC,GAAG,CAAc;IACzB,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,KAAK,CAA4C;gBAE7C,OAAO,GAAE,WAAkC,EAAE,OAAO,GAAE,qBAA0B;IAM5F,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,GAAE,SAAS,OAAO,EAAO,GAAG,SAAS;IAwD/D,OAAO,CAAC,OAAO;IAef,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,KAAK;IASb,OAAO,CAAC,KAAK;IAab,OAAO,CAAC,SAAS;IAiBjB,OAAO,CAAC,iBAAiB;IAuCzB,OAAO,CAAC,UAAU;IA4ElB,OAAO,CAAC,KAAK;IAgBb,OAAO,CAAC,UAAU;IASlB,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,SAAS;IAYjB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,kBAAkB;IAsC1B,OAAO,CAAC,MAAM;IASd,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAE,aAAa;IAQtB,OAAO,CAAC,SAAS;IAWjB,OAAO,CAAC,MAAM;IAMd,OAAO,CAAC,QAAQ;IAgBhB,OAAO,CAAC,MAAM;IA4Bd,OAAO,CAAC,IAAI;IAIZ,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,WAAW;IAOnB,OAAO,CAAC,UAAU;IAgBlB,OAAO,CAAC,KAAK;IAoBb,OAAO,CAAC,UAAU;IAOlB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAqBhB,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,uBAAuB;IA4B/B,OAAO,CAAC,UAAU;IA4BlB,OAAO,CAAC,QAAQ;IAQhB,OAAO,CAAC,OAAO;IAsEf,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,QAAQ;CAYjB;AA+HD,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,GAAG,SAAS,CAG9F"}

package/dist/engine.js

@@ -8,6 +8,12 @@
 // and `&&` evaluates strictly left to right — evaluation order is observable
 // through errors, so the engine never reorders conjuncts.)
 //
+// Nested blocks run through the optimizer (`./optimize`): each block gets a
+// `BlockPlan` the engine only EXECUTES (`access`), degrading to the plain scan
+// whenever the planned access path is unavailable at run time. The rules and
+// their semantics arguments live in `optimize/rules.ts`; `rules: []` turns the
+// optimizer off, which the conformance suite uses to compare both paths.
+//
 // Name resolution is strictly lexical and LOCAL: a bare identifier is read from
 // the current scope only, and an enclosing scope is reached solely through an
 // explicit `^name` (exactly one scope out per caret). There is no implicit
@@ -15,21 +21,32 @@
 import { DefaultContext } from "./context.js";
 import { OqxError } from "./errors.js";
 import { relate, arith, membership, truthy, compareForSort, compare, canonicalKey, makeRange, isEntry, isRange, } from "./semantics.js";
+import { DEFAULT_RULES, HashIndex, intersectPositions, lookupOrder, planFor, residualWithout } from "./optimize/index.js";
 const RECUR = new Set(["$depth", "$stop", "$leaf", "$frontier", "$ordinal"]);
 const KEY = "$key";
 const HARD_DEPTH_CAP = 8;
 const UNBOUNDED = { offset: 0, limit: null };
 export class InMemoryEngine {
     ctx;
-    constructor(context = new DefaultContext()) {
+    rules;
+    trace;
+    constructor(context = new DefaultContext(), options = {}) {
         this.ctx = context;
+        this.rules = options.rules ?? DEFAULT_RULES;
+        this.trace = options.trace;
     }
     run(query, bindings = []) {
-        const root = { row: null, parent: null, bindings };
-        let rows = this.rowsOf(this.evalExpr(query.source, root));
-        for (const proj of query.from) {
-            rows = rows.flatMap((r) => this.rowsOf(this.evalExpr(proj, this.child(r, root))));
-        }
+        const run = {
+            rules: this.rules,
+            bindingCount: bindings.length,
+            collections: new WeakMap(),
+            stableCollections: new Map(),
+            memo: new Map(),
+        };
+        if (this.trace)
+            run.trace = this.trace;
+        const root = { row: null, parent: null, bindings, run };
+        const rows = this.reproject(this.rowsOf(this.evalRaw(query.source, root)), query.from, root);
         const bound = this.boundOf(query, root);
         if (query.follow)
             return this.runFollow(query, rows, root, bound);
@@ -83,7 +100,7 @@ export class InMemoryEngine {
     boundOf(b, enclosing) {
         if (!b.limit && !b.offset)
             return UNBOUNDED;
-        const scope = enclosing.parent === null ? enclosing : { row: undefined, parent: enclosing, bindings: enclosing.bindings };
+        const scope = enclosing.parent === null ? enclosing : rowless(enclosing);
         const read = (e, word) => {
             if (!e)
                 return null;
@@ -108,12 +125,12 @@ export class InMemoryEngine {
         return this.enter(row, parent);
     }
     // Make the scope for a row. An `entries()` entry is unwrapped here: the scope's
-    // row is the property's VALUE (so `$value` and bare names read it) and the key
+    // row is the property's VALUE (so `$it` and bare names read it) and the key
     // becomes the `$key` intrinsic in `meta`. Every place a row becomes a scope
     // goes through this, so entries behave the same at the top level, in nested
     // blocks, as `from` re-projections, and as follow seeds.
     enter(row, parent, extra = {}) {
-        const s = { row, parent, bindings: parent.bindings };
+        const s = { row, parent, bindings: parent.bindings, run: parent.run };
         if (extra.lifts)
             s.lifts = extra.lifts;
         if (extra.meta)
@@ -190,7 +207,7 @@ export class InMemoryEngine {
             const rowScope = scopeFor(row);
             const raw = [];
             for (const dest of follow.destinations) {
-                const v = dest.kind === "op" ? this.evalCollectValue(dest, rowScope) : this.evalExpr(dest, rowScope);
+                const v = dest.kind === "op" ? this.evalCollectValue(dest, rowScope) : this.evalRaw(dest, rowScope);
                 raw.push(...this.rowsOf(v));
             }
             const kept = follow.where ? raw.filter((x) => truthy(this.evalExpr(follow.where, this.enter(x, rowScope)))) : raw;
@@ -303,12 +320,32 @@ export class InMemoryEngine {
     evalWhereOp(op, scope) {
         if (op.sub.follow)
             throw new OqxError("`follow` is only valid on a select-position collect { … }, not a where op", "eval");
+        const plan = this.planOf(op, scope);
+        if (!plan.invariant)
+            return this.evalWhereOpPlanned(op, plan, scope);
+        return this.memoized(op, scope, () => this.evalWhereOpPlanned(op, plan, scope));
+    }
+    evalWhereOpPlanned(op, plan, scope) {
         const bound = this.boundOf(op.sub, scope);
-        if (op.op === "exists" || op.op === "none") {
-            // Unbounded: stop at the first match (dedup cannot change emptiness).
-            // Bounded: the offset/limit decide emptiness, so materialize the set.
-            const any = bound === UNBOUNDED ? this.anyMatch(op, scope) : this.opRows(op, scope, bound).length > 0;
-            return op.op === "exists" ? any : !any;
+        if (op.op === "exists" || op.op === "none" || op.op === "count") {
+            const acc = this.access(op, plan, scope);
+            let n;
+            if (plan.fromCardinality && acc.where === null) {
+                // Nothing left to evaluate per row: the accessed rows ARE the matched
+                // rows, and the consumer only needs their count after the bound.
+                n = boundedCount(acc.rows.length, bound);
+                scope.run.trace?.({ kind: "cardinality", count: n });
+            }
+            else if (op.op !== "count" && bound === UNBOUNDED) {
+                // Unbounded exists/none: stop at the first match (dedup cannot change
+                // emptiness). Bounded: the offset/limit decide emptiness, so materialize.
+                n = this.anyMatch(op, scope, acc) ? 1 : 0;
+            }
+            else
+                n = this.opRows(op, scope, bound, acc).length;
+            if (op.op === "count")
+                return op.countCmp ? compareCount(n, op.countCmp) : n > 0;
+            return op.op === "exists" ? n > 0 : n === 0;
         }
         if (op.op === "collect") {
             const matched = this.opRows(op, scope, bound);
@@ -328,51 +365,218 @@ export class InMemoryEngine {
             }
             return matched.length > 0;
         }
-        if (op.op === "count") {
-            const n = this.opRows(op, scope, bound).length;
-            return op.countCmp ? compareCount(n, op.countCmp) : n > 0;
-        }
         return this.opRows(op, scope, bound).length > 0;
     }
     // The rows a consumer op reduces: matched → ordered → distinct → bounded.
-    opRows(op, scope, bound) {
-        let scopes = this.matchRows(op, scope);
+    opRows(op, scope, bound, acc) {
+        let scopes = this.matchRows(op, scope, acc);
         this.sortScopes(scopes, op.sub.orderBy);
         if (op.distinct)
             scopes = this.dedupByProjection(scopes, op.sub);
         return sliceBound(scopes, bound);
     }
-    // Short-circuiting existence check over a consumer receiver.
-    anyMatch(op, scope) {
-        for (const s of this.iterMatchRows(op, scope)) {
+    // Short-circuiting existence check: over a probe's bucket this stops at the
+    // first row that passes the residual, so `exists`/`none` never materialize it.
+    anyMatch(op, scope, acc) {
+        for (const s of this.iterMatchRows(op, scope, acc)) {
             void s;
             return true;
         }
         return false;
     }
-    *iterMatchRows(op, scope) {
-        let rows = this.rowsOf(this.evalExpr(op.receiver, scope));
-        for (const proj of op.sub.from)
-            rows = rows.flatMap((r) => this.rowsOf(this.evalExpr(proj, this.child(r, scope))));
+    // The block's accessed rows, entered as scopes under `scope`, filtered by the
+    // predicate the access left to evaluate.
+    *iterMatchRows(op, scope, acc) {
+        const { rows, where } = acc ?? this.access(op, this.planOf(op, scope), scope);
         for (const r of rows) {
             const s = this.enter(r, scope, { lifts: {} });
-            if (!op.sub.where || this.evalWhere(op.sub.where, s))
+            if (!where || this.evalWhere(where, s))
                 yield s;
         }
     }
-    matchRows(op, scope) {
-        return Array.from(this.iterMatchRows(op, scope));
+    matchRows(op, scope, acc) {
+        return Array.from(this.iterMatchRows(op, scope, acc));
+    }
+    // ---- planned access -------------------------------------------------------
+    //
+    // The engine EXECUTES a block's `BlockPlan`; the rules and the semantics
+    // argument for each live in `./optimize` (rules.ts). `access` is the one path
+    // every consumer takes to a block's rows, and whatever the plan asked for
+    // that cannot be served at run time degrades to the scan the plan started from.
+    planOf(op, scope) {
+        return planFor(op, scopeDepth(scope), { bindingCount: scope.run.bindingCount }, scope.run.rules);
+    }
+    // An invariant block (rule 2) is evaluated the first time it is reached and
+    // its value reused, by reference, for every other enclosing row.
+    memoized(op, scope, compute) {
+        const memo = scope.run.memo;
+        if (memo.has(op)) {
+            scope.run.trace?.({ kind: "memo", op });
+            return memo.get(op);
+        }
+        const value = compute();
+        memo.set(op, value);
+        return value;
+    }
+    // A block's rows and the predicate left to evaluate over each. A correlated
+    // plan is served, in order of preference, by a context index that yields rows
+    // (`lookupRows` — asked on every sight: it costs nothing and the collection
+    // is never read), by a probe on the engine's own index (built on the
+    // collection's second sight, or at once for a statically stable receiver: a
+    // collection seen once is cheaper to scan than to index), else by the scan.
+    // A statically stable receiver (rule 1b) is evaluated once per run; any other
+    // is keyed by its value's identity, so a fresh array per row is never indexed.
+    access(op, plan, scope) {
+        const run = scope.run;
+        let state = plan.receiverStable ? run.stableCollections.get(op) : undefined;
+        if (!state) {
+            const value = this.evalRaw(op.receiver, scope);
+            if (plan.receiverStable)
+                run.stableCollections.set(op, (state = newCollection(value)));
+            else if (plan.correlated.length === 0)
+                return this.scan(this.rowsOf(value), op, scope);
+            else if (value !== null && typeof value === "object") {
+                state = run.collections.get(value);
+                if (!state)
+                    run.collections.set(value, (state = newCollection(value)));
+            }
+            else {
+                run.trace?.({ kind: "fallback", reason: "not-a-collection" });
+                return this.scan(this.rowsOf(value), op, scope);
+            }
+        }
+        state.seen++;
+        if (plan.correlated.length > 0) {
+            const direct = this.lookupRows(plan, state, scope);
+            if (direct)
+                return direct;
+            if (state.seen >= 2 || plan.receiverStable) {
+                const probed = this.probe(plan, state, scope);
+                if (probed)
+                    return { rows: probed, where: plan.residual };
+            }
+            else
+                run.trace?.({ kind: "fallback", reason: "unstable" });
+        }
+        return this.scan(this.materialize(state), op, scope);
+    }
+    // The plain scan: the rows, re-projected by `from`, with the whole `where`.
+    scan(rows, op, scope) {
+        return { rows: this.reproject(rows, op.sub.from, scope), where: op.sub.where };
+    }
+    reproject(rows, from, scope) {
+        for (const proj of from)
+            rows = rows.flatMap((r) => this.rowsOf(this.evalRaw(proj, this.child(r, scope))));
+        return rows;
+    }
+    materialize(state) {
+        return (state.rows ??= this.rowsOf(state.value));
+    }
+    // A probe answered by a context index that yields rows directly, before the
+    // collection is materialized: one correlation is probed (`lookupOrder`) and
+    // every other conjunct stays in place per selected row (`residualWithout`).
+    lookupRows(plan, state, scope) {
+        for (const c of lookupOrder(plan)) {
+            const index = this.contextIndex(state, c);
+            if (!index?.lookupRows)
+                continue;
+            const outer = this.outerValue(c, scope);
+            if (!outer)
+                return null;
+            const rows = Array.from(index.lookupRows(outer.value));
+            scope.run.trace?.({ kind: "lookup", path: c.path, candidates: rows.length });
+            return { rows, where: residualWithout(plan, c) };
+        }
+        return null;
+    }
+    // The engine's probe: one bucket per correlation, intersected smallest-first;
+    // the candidates come back in receiver order. `null` (an index that could
+    // not be built, an outer value whose evaluation threw) leaves the scan in charge.
+    probe(plan, state, scope) {
+        const run = scope.run;
+        const rows = this.materialize(state);
+        const buckets = [];
+        for (const c of plan.correlated) {
+            const index = this.indexFor(state, c, scope);
+            if (!index) {
+                run.trace?.({ kind: "fallback", reason: "index" });
+                return null;
+            }
+            const outer = this.outerValue(c, scope);
+            if (!outer)
+                return null;
+            buckets.push(index.lookup(outer.value));
+        }
+        buckets.sort((a, b) => a.length - b.length);
+        let candidates = buckets[0];
+        for (let i = 1; i < buckets.length && candidates.length > 0; i++)
+            candidates = intersectPositions(candidates, buckets[i]);
+        run.trace?.({ kind: "probe", paths: plan.correlated.map((c) => c.path), candidates: candidates.length });
+        return candidates.map((i) => rows[i]);
+    }
+    // A correlation's outer side, read once in a row-less scope inside the block
+    // (as a bound is); `null` when it threw — the scan then raises it in place.
+    outerValue(c, scope) {
+        try {
+            return { value: this.evalExpr(c.outer, rowless(scope)) };
+        }
+        catch {
+            scope.run.trace?.({ kind: "fallback", reason: "probe-value" });
+            return null;
+        }
+    }
+    // The context's pre-built index on a correlation's local path, if it offers
+    // one (asked once per collection and path; a refusal is cached too).
+    contextIndex(state, c) {
+        const key = pathKey(c.path);
+        let index = state.contextIndexes.get(key);
+        if (index === undefined)
+            state.contextIndexes.set(key, (index = this.ctx.indexFor?.(state.value, c.path) ?? null));
+        return index;
+    }
+    // The index serving a correlation's local path: the context's, else one built
+    // here by evaluating the local expression in each row's scope — the same read
+    // the scan performs, so the keys are exactly what `==` would compare. A throw
+    // while building (a context that rejects a read) leaves the path un-indexed.
+    indexFor(state, c, scope) {
+        const key = pathKey(c.path);
+        const hit = state.indexes.get(key);
+        if (hit !== undefined)
+            return hit;
+        const rows = this.materialize(state);
+        let index = this.contextIndex(state, c);
+        let source = "context";
+        if (!index) {
+            source = "engine";
+            const built = new HashIndex();
+            try {
+                for (let i = 0; i < rows.length; i++)
+                    built.add(this.evalExpr(c.local, this.enter(rows[i], scope)), i);
+                index = built;
+            }
+            catch { /* un-indexed: the scan serves this path */ }
+        }
+        if (index)
+            scope.run.trace?.({ kind: "index", path: c.path, rows: rows.length, source });
+        state.indexes.set(key, index);
+        return index;
     }
     // A select-position collect/first/single, optionally recursive via `follow`.
     evalCollectValue(op, scope) {
+        const plan = this.planOf(op, scope);
+        if (!plan.invariant)
+            return this.evalCollectValuePlanned(op, plan, scope);
+        return this.memoized(op, scope, () => this.evalCollectValuePlanned(op, plan, scope));
+    }
+    evalCollectValuePlanned(op, plan, scope) {
         const sub = op.sub;
         const bound = this.boundOf(sub, scope);
         let scopes;
         if (sub.follow) {
-            let rows = this.rowsOf(this.evalExpr(op.receiver, scope));
-            for (const proj of sub.from)
-                rows = rows.flatMap((r) => this.rowsOf(this.evalExpr(proj, this.child(r, scope))));
-            const seeds = sub.where ? rows.filter((r) => this.evalWhere(sub.where, this.enter(r, scope, { lifts: {} }))) : rows;
+            // The seeds are the matched rows themselves (`followWalk` enters them); a
+            // follow block is never correlated, so `where` here is the whole predicate.
+            const { rows, where } = this.access(op, plan, scope);
+            const seeds = where ? rows.filter((r) => this.evalWhere(where, this.enter(r, scope, { lifts: {} }))) : rows;
             scopes = this.followWalk(seeds, sub.follow, scope).map((o) => this.enter(o.row, scope, { meta: o.meta }));
             this.sortScopes(scopes, sub.orderBy);
             if (op.distinct)
@@ -380,7 +584,7 @@ export class InMemoryEngine {
             scopes = sliceBound(scopes, bound);
         }
         else {
-            scopes = this.opRows(op, scope, bound);
+            scopes = this.opRows(op, scope, bound, this.access(op, plan, scope));
         }
         switch (op.op) {
             case "collect": return scopes.map((s) => this.projectRow(sub, s));
@@ -420,7 +624,18 @@ export class InMemoryEngine {
         });
     }
     // ---- scalar expression evaluation -----------------------------------------
+    // An expression as a VALUE: whatever `evalRaw` yields, with a context's lazy
+    // stand-in resolved (`DataContext.materialize`), so no operand, argument, key
+    // or projected item ever observes a value the context meant as a collection
+    // it had not read yet.
     evalExpr(e, scope) {
+        const v = this.evalRaw(e, scope);
+        return this.ctx.materialize ? this.ctx.materialize(v) : v;
+    }
+    // An expression as the context handed it out — for ROW POSITION only (the
+    // source, a receiver, a `from`, a `follow` destination), where the value goes
+    // straight to `toRows` / `indexFor`. Operands inside it are values (`evalExpr`).
+    evalRaw(e, scope) {
         switch (e.kind) {
             case "lit": return e.value;
             case "binding":
@@ -469,9 +684,9 @@ export class InMemoryEngine {
         }
     }
     // Resolve a name against ONE scope — never its ancestors. A scope provides,
-    // in order: `$value` (the scope's row itself — the current item, whatever its
+    // in order: `$it` (the scope's row itself — the current item, whatever its
     // type, so scalar collections are queryable; absent at the root, which has no
-    // row); `$key` (the property key) when it is an entry scope — see `enter`;
+    // row; `$value`, its pre-0.15 spelling, is an ordinary property name now); `$key` (the property key) when it is an entry scope — see `enter`;
     // the recursion intrinsics (`$depth`, …) when it is a follow occurrence;
     // values lifted into it by `^name:` items; then either the row's own property
     // or, for the root scope (no row), the context's named roots.
@@ -490,7 +705,7 @@ export class InMemoryEngine {
     // always spelled explicitly as `^name`. Present-but-falsy values (null, false,
     // 0, "") need no special case — there is no "absent, so look outward" rule.
     resolveIn(name, scope) {
-        if (name === "$value")
+        if (name === "$it")
             return scope.parent === null ? undefined : scope.row;
         if ((name === KEY || RECUR.has(name)) && scope.meta && Object.hasOwn(scope.meta, name))
             return scope.meta[name];
@@ -516,6 +731,24 @@ export class InMemoryEngine {
     }
 }
 // ---- free helpers -----------------------------------------------------------
+function newCollection(value) {
+    return { value, seen: 0, rows: null, indexes: new Map(), contextIndexes: new Map() };
+}
+function pathKey(path) {
+    return JSON.stringify(path);
+}
+// A row-less scope INSIDE a block: a bare name is absent, `^name` is the
+// enclosing row — how a block's bound and a probe's outer value are read.
+function rowless(enclosing) {
+    return { row: undefined, parent: enclosing, bindings: enclosing.bindings, run: enclosing.run };
+}
+// How many scopes out the root is (the root itself is 0).
+function scopeDepth(scope) {
+    let d = 0;
+    for (let s = scope; s.parent !== null; s = s.parent)
+        d++;
+    return d;
+}
 function isRelOp(op) {
     return op === "==" || op === "!=" || op === "<" || op === "<=" || op === ">" || op === ">=";
 }