@rocicorp/zero 0.25.0-canary.0 → 0.25.0-canary.2

package/out/zql/src/planner/planner-graph.js

@@ -0,0 +1,380 @@
+import { assert } from "../../../shared/src/asserts.js";
+import { PlannerSource } from "./planner-source.js";
+import { must } from "../../../shared/src/must.js";
+/**
+ * Maximum number of flippable joins to attempt exhaustive enumeration.
+ * With n flippable joins, we explore 2^n plans.
+ * 10 joins = 1024 plans (~100-200ms), 12 joins = 4096 plans (~400ms - 1 second)
+ */
+const MAX_FLIPPABLE_JOINS = 13;
+export class PlannerGraph {
+    // Sources indexed by table name
+    #sources = new Map();
+    // The final output node where constraint propagation starts
+    #terminus = undefined;
+    // Collections of nodes with mutable planning state
+    joins = [];
+    fanOuts = [];
+    fanIns = [];
+    connections = [];
+    /**
+     * Reset all planning state back to initial values for another planning pass.
+     * Resets only mutable planning state - graph structure is unchanged.
+     *
+     * This allows replanning the same query graph with different strategies.
+     */
+    resetPlanningState() {
+        for (const j of this.joins)
+            j.reset();
+        for (const fo of this.fanOuts)
+            fo.reset();
+        for (const fi of this.fanIns)
+            fi.reset();
+        for (const c of this.connections)
+            c.reset();
+    }
+    /**
+     * Create and register a source (table) in the graph.
+     */
+    addSource(name, model) {
+        assert(!this.#sources.has(name), `Source ${name} already exists in the graph`);
+        const source = new PlannerSource(name, model);
+        this.#sources.set(name, source);
+        return source;
+    }
+    /**
+     * Get a source by table name.
+     */
+    getSource(name) {
+        const source = this.#sources.get(name);
+        assert(source !== undefined, `Source ${name} not found in the graph`);
+        return source;
+    }
+    /**
+     * Check if a source exists by table name.
+     */
+    hasSource(name) {
+        return this.#sources.has(name);
+    }
+    /**
+     * Set the terminus (final output) node of the graph.
+     * Constraint propagation starts from this node.
+     */
+    setTerminus(terminus) {
+        this.#terminus = terminus;
+    }
+    /**
+     * Initiate constraint propagation from the terminus node.
+     * This sends constraints up through the graph to update
+     * connection cost estimates.
+     */
+    propagateConstraints(planDebugger) {
+        assert(this.#terminus !== undefined, 'Cannot propagate constraints without a terminus node');
+        this.#terminus.propagateConstraints(planDebugger);
+    }
+    /**
+     * Calculate total cost of the current plan.
+     * Total cost includes both startup cost (one-time, e.g., sorting) and running cost.
+     */
+    getTotalCost(planDebugger) {
+        const estimate = must(this.#terminus).estimateCost(planDebugger);
+        return estimate.cost + estimate.startupCost;
+    }
+    /**
+     * Capture a lightweight snapshot of the current planning state.
+     * Used for backtracking during multi-start greedy search.
+     *
+     * Captures mutable state including pinned flags, join types, and
+     * constraint maps to avoid needing repropagation on restore.
+     *
+     * @returns A snapshot that can be restored via restorePlanningSnapshot()
+     */
+    capturePlanningSnapshot() {
+        return {
+            connections: this.connections.map(c => ({
+                limit: c.limit,
+            })),
+            joins: this.joins.map(j => ({ type: j.type })),
+            fanOuts: this.fanOuts.map(fo => ({ type: fo.type })),
+            fanIns: this.fanIns.map(fi => ({ type: fi.type })),
+            connectionConstraints: this.connections.map(c => c.captureConstraints()),
+        };
+    }
+    /**
+     * Restore planning state from a previously captured snapshot.
+     * Used for backtracking when a planning attempt fails.
+     *
+     * Restores pinned flags, join types, and constraint maps, eliminating
+     * the need for repropagation.
+     *
+     * @param state - Snapshot created by capturePlanningSnapshot()
+     */
+    restorePlanningSnapshot(state) {
+        this.#validateSnapshotShape(state);
+        this.#restoreConnections(state);
+        this.#restoreJoins(state);
+        this.#restoreFanNodes(state);
+    }
+    /**
+     * Validate that snapshot shape matches current graph structure.
+     */
+    #validateSnapshotShape(state) {
+        assert(this.connections.length === state.connections.length, 'Plan state mismatch: connections');
+        assert(this.joins.length === state.joins.length, 'Plan state mismatch: joins');
+        assert(this.fanOuts.length === state.fanOuts.length, 'Plan state mismatch: fanOuts');
+        assert(this.fanIns.length === state.fanIns.length, 'Plan state mismatch: fanIns');
+        assert(this.connections.length === state.connectionConstraints.length, 'Plan state mismatch: connectionConstraints');
+    }
+    /**
+     * Restore connection pinned flags, limits, and constraint maps.
+     */
+    #restoreConnections(state) {
+        for (let i = 0; i < this.connections.length; i++) {
+            this.connections[i].limit = state.connections[i].limit;
+            this.connections[i].restoreConstraints(state.connectionConstraints[i]);
+        }
+    }
+    /**
+     * Restore join types and pinned flags.
+     */
+    #restoreJoins(state) {
+        for (let i = 0; i < this.joins.length; i++) {
+            const join = this.joins[i];
+            const targetState = state.joins[i];
+            // Reset to initial state first
+            join.reset();
+            // Apply target state
+            if (targetState.type === 'flipped') {
+                join.flip();
+            }
+        }
+    }
+    /**
+     * Restore FanOut and FanIn types.
+     */
+    #restoreFanNodes(state) {
+        for (let i = 0; i < this.fanOuts.length; i++) {
+            const fo = this.fanOuts[i];
+            const targetType = state.fanOuts[i].type;
+            if (targetType === 'UFO' && fo.type === 'FO') {
+                fo.convertToUFO();
+            }
+        }
+        for (let i = 0; i < this.fanIns.length; i++) {
+            const fi = this.fanIns[i];
+            const targetType = state.fanIns[i].type;
+            if (targetType === 'UFI' && fi.type === 'FI') {
+                fi.convertToUFI();
+            }
+        }
+    }
+    /**
+     * Main planning algorithm using exhaustive join flip enumeration.
+     *
+     * Enumerates all possible flip patterns for flippable joins (2^n for n flippable joins).
+     * Each pattern represents a different query execution plan. We evaluate the cost of each
+     * plan and select the one with the lowest cost.
+     *
+     * Connections are used only for cost estimation - the flip patterns determine the plan.
+     * FanOut/FanIn states (FO/UFO and FI/UFI) are automatically derived from join flip states.
+     *
+     * @param planDebugger - Optional debugger to receive structured events during planning
+     */
+    plan(planDebugger) {
+        // Get all flippable joins
+        const flippableJoins = this.joins.filter(j => j.isFlippable());
+        // Safety check: throw if too many flippable joins
+        if (flippableJoins.length > MAX_FLIPPABLE_JOINS) {
+            throw new Error(`Query has ${flippableJoins.length} EXISTS checks in a single RELATED call (or in the top level query), which would require ` +
+                `${2 ** flippableJoins.length} plan evaluations. This may be very slow. ` +
+                `Consider simplifying the query or increasing MAX_FLIPPABLE_JOINS (currently set to ${MAX_FLIPPABLE_JOINS}).`);
+        }
+        // Build FO→FI cache once to avoid redundant BFS traversals in each iteration
+        const fofiCache = buildFOFICache(this);
+        const numPatterns = 2 ** flippableJoins.length;
+        let bestCost = Infinity;
+        let bestPlan = undefined;
+        let bestAttemptNumber = -1;
+        // Enumerate all flip patterns
+        // try 7 and 32 (6 and 31)
+        const forcePattern = undefined; // 11 14
+        for (let pattern = 0; pattern < numPatterns; pattern++) {
+            if (forcePattern !== undefined && pattern !== forcePattern) {
+                continue;
+            }
+            // Reset to initial state
+            this.resetPlanningState();
+            if (planDebugger) {
+                planDebugger.log({
+                    type: 'attempt-start',
+                    attemptNumber: pattern,
+                    totalAttempts: numPatterns,
+                });
+            }
+            try {
+                // Apply flip pattern (treat pattern as bitmask)
+                // Bit i set to 1 means flip join i
+                for (let i = 0; i < flippableJoins.length; i++) {
+                    if (pattern & (1 << i)) {
+                        flippableJoins[i].flip();
+                    }
+                }
+                // Derive FO/UFO and FI/UFI states from join flip states
+                checkAndConvertFOFI(fofiCache);
+                // Propagate unlimiting for flipped joins
+                propagateUnlimitForFlippedJoins(this);
+                // Propagate constraints through the graph
+                this.propagateConstraints(planDebugger);
+                if (planDebugger) {
+                    planDebugger.log({
+                        type: 'constraints-propagated',
+                        attemptNumber: pattern,
+                        connectionConstraints: this.connections.map(c => ({
+                            connection: c.name,
+                            constraints: c.getConstraintsForDebug(),
+                            constraintCosts: c.getConstraintCostsForDebug(),
+                        })),
+                    });
+                }
+                // Evaluate this plan
+                const totalCost = this.getTotalCost(planDebugger);
+                if (planDebugger) {
+                    planDebugger.log({
+                        type: 'plan-complete',
+                        attemptNumber: pattern,
+                        totalCost,
+                        flipPattern: pattern, // Bitmask of which joins are flipped
+                        // TODO: we'll need a different way to collect these
+                        // nodeCosts: this.#collectNodeCosts(),
+                        joinStates: this.joins.map(j => {
+                            const info = j.getDebugInfo();
+                            return {
+                                join: info.name,
+                                type: info.type,
+                            };
+                        }),
+                    });
+                }
+                // Track best plan
+                if (totalCost < bestCost) {
+                    bestCost = totalCost;
+                    bestPlan = this.capturePlanningSnapshot();
+                    bestAttemptNumber = pattern;
+                }
+            }
+            catch (e) {
+                // This flip pattern is invalid (shouldn't happen with proper isFlippable() checks)
+                if (planDebugger) {
+                    planDebugger.log({
+                        type: 'plan-failed',
+                        attemptNumber: pattern,
+                        reason: `Flip pattern ${pattern.toString(2)} failed: ${e instanceof Error ? e.message : String(e)}`,
+                    });
+                }
+                continue;
+            }
+        }
+        // Restore best plan
+        if (bestPlan) {
+            this.restorePlanningSnapshot(bestPlan);
+            // Propagate constraints to ensure all derived state is consistent
+            this.propagateConstraints(planDebugger);
+            if (planDebugger) {
+                planDebugger.log({
+                    type: 'best-plan-selected',
+                    bestAttemptNumber,
+                    totalCost: bestCost,
+                    flipPattern: bestAttemptNumber, // The best attempt number is also the flip pattern
+                    joinStates: this.joins.map(j => ({
+                        join: j.getName(),
+                        type: j.type,
+                    })),
+                });
+            }
+        }
+        else {
+            // No valid plan found (all patterns failed)
+            throw new Error('No valid query plan found. This should not happen - check query structure.');
+        }
+    }
+}
+/**
+ * Build cache of FO→FI relationships and joins between them.
+ * Called once at the start of planning to avoid redundant BFS traversals.
+ */
+function buildFOFICache(graph) {
+    const cache = new Map();
+    for (const fo of graph.fanOuts) {
+        const info = findFIAndJoins(fo);
+        cache.set(fo, info);
+    }
+    return cache;
+}
+/**
+ * Check if any joins downstream of a FanOut (before reaching FanIn) are flipped.
+ * If so, convert the FO to UFO and the FI to UFI.
+ *
+ * This must be called after join flipping and before propagateConstraints.
+ */
+function checkAndConvertFOFI(fofiCache) {
+    for (const [fo, info] of fofiCache) {
+        const hasFlippedJoin = info.joinsBetween.some(j => j.type === 'flipped');
+        if (info.fi && hasFlippedJoin) {
+            fo.convertToUFO();
+            info.fi.convertToUFI();
+        }
+    }
+}
+/**
+ * Traverse from a FanOut through its outputs to find the corresponding FanIn
+ * and collect all joins along the way.
+ */
+function findFIAndJoins(fo) {
+    const joinsBetween = [];
+    let fi = undefined;
+    // BFS through FO outputs to find FI and collect joins
+    const queue = [...fo.outputs];
+    const visited = new Set();
+    while (queue.length > 0) {
+        const node = must(queue.shift());
+        if (visited.has(node))
+            continue;
+        visited.add(node);
+        switch (node.kind) {
+            case 'join':
+                joinsBetween.push(node);
+                queue.push(node.output);
+                break;
+            case 'fan-out':
+                // Nested FO - traverse its outputs
+                queue.push(...node.outputs);
+                break;
+            case 'fan-in':
+                // Found the FI - this is the boundary, don't traverse further
+                fi = node;
+                break;
+            case 'connection':
+                // Shouldn't happen in a well-formed graph
+                break;
+            case 'terminus':
+                // Reached the end without finding FI
+                break;
+        }
+    }
+    return { fi, joinsBetween };
+}
+/**
+ * Propagate unlimiting to all flipped joins in the graph.
+ * When a join is flipped, its child becomes the outer loop and should no longer
+ * be limited by EXISTS semantics.
+ *
+ * This must be called after join flipping and before propagateConstraints.
+ */
+function propagateUnlimitForFlippedJoins(graph) {
+    for (const join of graph.joins) {
+        if (join.type === 'flipped') {
+            join.propagateUnlimit();
+        }
+    }
+}
+//# sourceMappingURL=planner-graph.js.map

package/out/zql/src/planner/planner-graph.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-graph.js","sourceRoot":"","sources":["../../../../../zql/src/planner/planner-graph.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,gCAAgC,CAAC;AAOtD,OAAO,EAAC,aAAa,EAA2B,MAAM,qBAAqB,CAAC;AAE5E,OAAO,EAAC,IAAI,EAAC,MAAM,6BAA6B,CAAC;AAcjD;;;;GAIG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAW/B,MAAM,OAAO,YAAY;IACvB,gCAAgC;IACvB,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAC;IAErD,4DAA4D;IAC5D,SAAS,GAAgC,SAAS,CAAC;IAEnD,mDAAmD;IACnD,KAAK,GAAkB,EAAE,CAAC;IAC1B,OAAO,GAAoB,EAAE,CAAC;IAC9B,MAAM,GAAmB,EAAE,CAAC;IAC5B,WAAW,GAAwB,EAAE,CAAC;IAEtC;;;;;OAKG;IACH,kBAAkB;QAChB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK;YAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QACtC,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO;YAAE,EAAE,CAAC,KAAK,EAAE,CAAC;QAC1C,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,MAAM;YAAE,EAAE,CAAC,KAAK,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,WAAW;YAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,IAAY,EAAE,KAA0B;QAChD,MAAM,CACJ,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EACxB,UAAU,IAAI,8BAA8B,CAC7C,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC9C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAChC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,IAAY;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,UAAU,IAAI,yBAAyB,CAAC,CAAC;QACtE,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,WAAW,CAAC,QAAyB;QACnC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACH,oBAAoB,CAAC,YAA2B;QAC9C,MAAM,CACJ,IAAI,CAAC,SAAS,KAAK,SAAS,EAC5B,sDAAsD,CACvD,CAAC;QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;IACpD,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,YAA2B;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;QACjE,OAAO,QAAQ,CAAC,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC;IAC9C,CAAC;IAED;;;;;;;;OAQG;IACH,uBAAuB;QACrB,OAAO;YACL,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACtC,KAAK,EAAE,CAAC,CAAC,KAAK;aACf,CAAC,CAAC;YACH,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAC,CAAC,CAAC;YAC5C,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAC,CAAC,CAAC;YAClD,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAC,CAAC,CAAC;YAChD,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC;IAED;;;;;;;;OAQG;IACH,uBAAuB,CAAC,KAAgB;QACtC,IAAI,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACH,sBAAsB,CAAC,KAAgB;QACrC,MAAM,CACJ,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,KAAK,CAAC,WAAW,CAAC,MAAM,EACpD,kCAAkC,CACnC,CAAC;QACF,MAAM,CACJ,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM,EACxC,4BAA4B,CAC7B,CAAC;QACF,MAAM,CACJ,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO,CAAC,MAAM,EAC5C,8BAA8B,CAC/B,CAAC;QACF,MAAM,CACJ,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,MAAM,EAC1C,6BAA6B,CAC9B,CAAC;QACF,MAAM,CACJ,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,KAAK,CAAC,qBAAqB,CAAC,MAAM,EAC9D,4CAA4C,CAC7C,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,KAAgB;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YACvD,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAgB;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEnC,+BAA+B;YAC/B,IAAI,CAAC,KAAK,EAAE,CAAC;YAEb,qBAAqB;YACrB,IAAI,WAAW,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBACnC,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,KAAgB;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACzC,IAAI,UAAU,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBAC7C,EAAE,CAAC,YAAY,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACxC,IAAI,UAAU,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBAC7C,EAAE,CAAC,YAAY,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACH,IAAI,CAAC,YAA2B;QAC9B,0BAA0B;QAC1B,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAE/D,kDAAkD;QAClD,IAAI,cAAc,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CACb,aAAa,cAAc,CAAC,MAAM,2FAA2F;gBAC3H,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,4CAA4C;gBACzE,sFAAsF,mBAAmB,IAAI,CAChH,CAAC;QACJ,CAAC;QAED,6EAA6E;QAC7E,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QAEvC,MAAM,WAAW,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,CAAC;QAC/C,IAAI,QAAQ,GAAG,QAAQ,CAAC;QACxB,IAAI,QAAQ,GAA0B,SAAS,CAAC;QAChD,IAAI,iBAAiB,GAAG,CAAC,CAAC,CAAC;QAE3B,8BAA8B;QAC9B,0BAA0B;QAC1B,MAAM,YAAY,GAAG,SAAS,CAAC,CAAC,QAAQ;QACxC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,YAAY,KAAK,SAAS,IAAI,OAAO,KAAK,YAAY,EAAE,CAAC;gBAC3D,SAAS;YACX,CAAC;YACD,yBAAyB;YACzB,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,CAAC,GAAG,CAAC;oBACf,IAAI,EAAE,eAAe;oBACrB,aAAa,EAAE,OAAO;oBACtB,aAAa,EAAE,WAAW;iBAC3B,CAAC,CAAC;YACL,CAAC;YAED,IAAI,CAAC;gBACH,gDAAgD;gBAChD,mCAAmC;gBACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC/C,IAAI,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;wBACvB,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC3B,CAAC;gBACH,CAAC;gBAED,wDAAwD;gBACxD,mBAAmB,CAAC,SAAS,CAAC,CAAC;gBAE/B,yCAAyC;gBACzC,+BAA+B,CAAC,IAAI,CAAC,CAAC;gBAEtC,0CAA0C;gBAC1C,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;gBAExC,IAAI,YAAY,EAAE,CAAC;oBACjB,YAAY,CAAC,GAAG,CAAC;wBACf,IAAI,EAAE,wBAAwB;wBAC9B,aAAa,EAAE,OAAO;wBACtB,qBAAqB,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BAChD,UAAU,EAAE,CAAC,CAAC,IAAI;4BAClB,WAAW,EAAE,CAAC,CAAC,sBAAsB,EAAE;4BACvC,eAAe,EAAE,CAAC,CAAC,0BAA0B,EAAE;yBAChD,CAAC,CAAC;qBACJ,CAAC,CAAC;gBACL,CAAC;gBAED,qBAAqB;gBACrB,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;gBAElD,IAAI,YAAY,EAAE,CAAC;oBACjB,YAAY,CAAC,GAAG,CAAC;wBACf,IAAI,EAAE,eAAe;wBACrB,aAAa,EAAE,OAAO;wBACtB,SAAS;wBACT,WAAW,EAAE,OAAO,EAAE,qCAAqC;wBAC3D,oDAAoD;wBACpD,uCAAuC;wBACvC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;4BAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC;4BAC9B,OAAO;gCACL,IAAI,EAAE,IAAI,CAAC,IAAI;gCACf,IAAI,EAAE,IAAI,CAAC,IAAI;6BAChB,CAAC;wBACJ,CAAC,CAAC;qBACH,CAAC,CAAC;gBACL,CAAC;gBAED,kBAAkB;gBAClB,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;oBACzB,QAAQ,GAAG,SAAS,CAAC;oBACrB,QAAQ,GAAG,IAAI,CAAC,uBAAuB,EAAE,CAAC;oBAC1C,iBAAiB,GAAG,OAAO,CAAC;gBAC9B,CAAC;YACH,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,mFAAmF;gBACnF,IAAI,YAAY,EAAE,CAAC;oBACjB,YAAY,CAAC,GAAG,CAAC;wBACf,IAAI,EAAE,aAAa;wBACnB,aAAa,EAAE,OAAO;wBACtB,MAAM,EAAE,gBAAgB,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;qBACpG,CAAC,CAAC;gBACL,CAAC;gBACD,SAAS;YACX,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,CAAC;YACvC,kEAAkE;YAClE,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;YAExC,IAAI,YAAY,EAAE,CAAC;gBACjB,YAAY,CAAC,GAAG,CAAC;oBACf,IAAI,EAAE,oBAAoB;oBAC1B,iBAAiB;oBACjB,SAAS,EAAE,QAAQ;oBACnB,WAAW,EAAE,iBAAiB,EAAE,mDAAmD;oBACnF,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;wBAC/B,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE;wBACjB,IAAI,EAAE,CAAC,CAAC,IAAI;qBACb,CAAC,CAAC;iBACJ,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,CAAC;YACN,4CAA4C;YAC5C,MAAM,IAAI,KAAK,CACb,4EAA4E,CAC7E,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,KAAmB;IACzC,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IAEjD,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;QAChC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,SAAS,mBAAmB,CAAC,SAAuC;IAClE,KAAK,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;QACnC,MAAM,cAAc,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACzE,IAAI,IAAI,CAAC,EAAE,IAAI,cAAc,EAAE,CAAC;YAC9B,EAAE,CAAC,YAAY,EAAE,CAAC;YAClB,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,EAAiB;IACvC,MAAM,YAAY,GAAkB,EAAE,CAAC;IACvC,IAAI,EAAE,GAA6B,SAAS,CAAC;IAE7C,sDAAsD;IACtD,MAAM,KAAK,GAAkB,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAe,CAAC;IAEvC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACjC,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAChC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAElB,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,KAAK,MAAM;gBACT,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACxB,MAAM;YACR,KAAK,SAAS;gBACZ,mCAAmC;gBACnC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC5B,MAAM;YACR,KAAK,QAAQ;gBACX,8DAA8D;gBAC9D,EAAE,GAAG,IAAI,CAAC;gBACV,MAAM;YACR,KAAK,YAAY;gBACf,0CAA0C;gBAC1C,MAAM;YACR,KAAK,UAAU;gBACb,qCAAqC;gBACrC,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO,EAAC,EAAE,EAAE,YAAY,EAAC,CAAC;AAC5B,CAAC;AAED;;;;;;GAMG;AACH,SAAS,+BAA+B,CAAC,KAAmB;IAC1D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC5B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC"}

package/out/zql/src/planner/planner-join.d.ts

@@ -0,0 +1,136 @@
+import { type PlannerConstraint } from './planner-constraint.ts';
+import type { PlanDebugger } from './planner-debug.ts';
+import type { CostEstimate, JoinOrConnection, PlannerNode } from './planner-node.ts';
+import type { PlannerTerminus } from './planner-terminus.ts';
+/**
+ * Semi-join overhead multiplier.
+ *
+ * Semi-joins represent correlated subqueries (EXISTS checks) which have
+ * execution overhead compared to flipped joins, even when logical row counts
+ * are identical. This overhead comes from:
+ * - Need to execute a separate correlation check for each parent row
+ * - Cannot leverage combined constraint checking as effectively as flipped joins
+ *
+ * A multiplier of 1.5 means semi-joins are estimated to be ~50% more expensive
+ * than equivalent flipped joins, which empirically matches observed performance
+ * differences in production workloads (e.g., 1.7x in zbugs benchmarks).
+ *
+ * Flipped joins have a different overhead in that they become unlimited. This
+ * is accounted for when propagating unlimits rather than here.
+ */
+/**
+ * Represents a join between two data streams (parent and child).
+ *
+ * # Dual-State Pattern
+ * Like all planner nodes, PlannerJoin separates:
+ * 1. IMMUTABLE STRUCTURE: Parent/child nodes, constraints, flippability
+ * 2. MUTABLE STATE: Join type (semi/flipped), pinned status
+ *
+ * # Join Flipping
+ * A join can be in two states:
+ * - 'semi': Parent is outer loop, child is inner (semi-join for EXISTS)
+ * - 'flipped': Child is outer loop, parent is inner
+ *
+ * Flipping is the key optimization: choosing which table scans first.
+ * NOT EXISTS joins cannot be flipped (#flippable = false).
+ *
+ * # Constraint Propagation
+ * - Semi-join: Sends childConstraint to child, forwards received constraints to parent
+ * - Flipped join: Sends undefined to child, merges parentConstraint with received to parent
+ * - Unpinned join: Only forwards constraints to parent (doesn't constrain child yet)
+ *
+ * # Lifecycle
+ * 1. Construct with immutable structure (parent, child, constraints, flippability)
+ * 2. Wire to output node during graph construction
+ * 3. Planning calls flipIfNeeded() based on connection selection order
+ * 4. pin() locks the join type once chosen
+ * 5. reset() clears mutable state (type → 'semi', pinned → false)
+ */
+export declare class PlannerJoin {
+    #private;
+    readonly kind: "join";
+    readonly planId: number;
+    constructor(parent: Exclude<PlannerNode, PlannerTerminus>, child: Exclude<PlannerNode, PlannerTerminus>, parentConstraint: PlannerConstraint, childConstraint: PlannerConstraint, flippable: boolean, planId: number);
+    setOutput(node: PlannerNode): void;
+    get output(): PlannerNode;
+    closestJoinOrSource(): JoinOrConnection;
+    flipIfNeeded(input: PlannerNode): void;
+    flip(): void;
+    get type(): 'semi' | 'flipped';
+    isFlippable(): boolean;
+    /**
+     * Propagate unlimiting when this join is flipped.
+     * When a join is flipped:
+     * 1. Child becomes outer loop → produces all rows (unlimited)
+     * 2. Parent is fetched once per child row → effectively unlimited
+     *
+     * Example: If child produces 896 rows, parent is fetched 896 times.
+     * Even if each fetch returns 1 row, parent produces 896 total rows.
+     *
+     * Propagation rules:
+     * - Connection: call unlimit()
+     * - Semi-join: continue to parent (outer loop)
+     * - Flipped join: stop (already unlimited when it was flipped)
+     * - Fan-out/Fan-in: propagate to all inputs
+     */
+    propagateUnlimit(): void;
+    /**
+     * Called when a parent join is flipped and this join is part of its child subgraph.
+     * - Semi-join: continue propagation to parent (the outer loop)
+     * - Flipped join: stop propagation (already unlimited when it was flipped)
+     */
+    propagateUnlimitFromFlippedJoin(): void;
+    propagateConstraints(branchPattern: number[], constraint: PlannerConstraint | undefined, from?: PlannerNode, planDebugger?: PlanDebugger): void;
+    reset(): void;
+    estimateCost(
+    /**
+     * This argument is to deal with consecutive `andExists` statements.
+     * Each one will constrain how often a parent row passes all constraints.
+     * This means that we have to scan more and more parent rows the more
+     * constraints we add.
+     */
+    downstreamChildSelectivity: number, 
+    /**
+     * branchPattern uniquely identifies OR branches in the graph.
+     * Each path through an OR will have unique constraints to apply to the source
+     * connection.
+     * branchPattern allows us to correlate a path through the graph
+     * to the constraints that should be applied for that path.
+     *
+     * Example graph:
+     *  UFO
+     * /  \
+     * J1   J2
+     * \  /
+     *  UFI
+     *
+     * J1 and J2 are joins inside an OR (FO).
+     * branchPattern [0] = path through J1
+     * branchPattern [1] = path through J2
+     *
+     * If many ORs are nested, branchPattern will have multiple elements
+     * representing each level of OR.
+     *
+     * If no joins are flipped within the `OR`, then only a single
+     * branchPattern element will be needed, as FO represents all sub-joins
+     * as a single path.
+     */
+    branchPattern: number[], planDebugger?: PlanDebugger): CostEstimate;
+    /**
+     * Get a human-readable name for this join for debugging.
+     * Format: "parentName ⋈ childName"
+     */
+    getName(): string;
+    /**
+     * Get debug information about this join's state.
+     */
+    getDebugInfo(): {
+        name: string;
+        type: 'semi' | 'flipped';
+        planId: number;
+    };
+}
+export declare class UnflippableJoinError extends Error {
+    constructor(message: string);
+}
+//# sourceMappingURL=planner-join.d.ts.map

package/out/zql/src/planner/planner-join.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"planner-join.d.ts","sourceRoot":"","sources":["../../../../../zql/src/planner/planner-join.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EACV,YAAY,EACZ,gBAAgB,EAChB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAoC3D;;;;;;;;;;;;;;;GAeG;AAGH;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,WAAW;;IACtB,QAAQ,CAAC,IAAI,EAAG,MAAM,CAAU;IAOhC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;gBAOtB,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE,eAAe,CAAC,EAC7C,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,eAAe,CAAC,EAC5C,gBAAgB,EAAE,iBAAiB,EACnC,eAAe,EAAE,iBAAiB,EAClC,SAAS,EAAE,OAAO,EAClB,MAAM,EAAE,MAAM;IAWhB,SAAS,CAAC,IAAI,EAAE,WAAW,GAAG,IAAI;IAIlC,IAAI,MAAM,IAAI,WAAW,CAGxB;IAED,mBAAmB,IAAI,gBAAgB;IAIvC,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAWtC,IAAI,IAAI,IAAI;IAUZ,IAAI,IAAI,IAAI,MAAM,GAAG,SAAS,CAE7B;IACD,WAAW,IAAI,OAAO;IAItB;;;;;;;;;;;;;;OAcG;IACH,gBAAgB,IAAI,IAAI;IAMxB;;;;OAIG;IACH,+BAA+B,IAAI,IAAI;IAOvC,oBAAoB,CAClB,aAAa,EAAE,MAAM,EAAE,EACvB,UAAU,EAAE,iBAAiB,GAAG,SAAS,EACzC,IAAI,CAAC,EAAE,WAAW,EAClB,YAAY,CAAC,EAAE,YAAY,GAC1B,IAAI;IAuDP,KAAK,IAAI,IAAI;IAIb,YAAY;IACV;;;;;OAKG;IACH,0BAA0B,EAAE,MAAM;IAClC;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACH,aAAa,EAAE,MAAM,EAAE,EACvB,YAAY,CAAC,EAAE,YAAY,GAC1B,YAAY;IAgFf;;;OAGG;IACH,OAAO,IAAI,MAAM;IAMjB;;OAEG;IACH,YAAY,IAAI;QACd,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;QACzB,MAAM,EAAE,MAAM,CAAC;KAChB;CAOF;AAED,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B"}