@minnowdb/core 0.2.0 → 0.3.0

package/dist/engine/vector.js

@@ -1,9 +1,11 @@
-import { cachedListMembership, distinctFromComparison, explicitNullOrder, isScalarFunctionName, likeMatches, orderOutputName, parseQuantified, quantifiedComparison, scalarFunctionValue, } from "./query.js";
+import { cachedListMembership, childExpressions, distinctFromComparison, nullOrder, isScalarFunctionName, likeMatches, orderOutputName, parseQuantified, quantifiedComparison, scalarFunctionValue, } from "./query.js";
+import { jsonValueOf } from "./sql-json.js";
 import { bm25DocumentScore, cachedQueryTerms, FtsStatsAccumulator, fullTermsMask, renderDocumentValue, termFrequencies, termsMask, tokenize, } from "./fts.js";
 import { ByteGroupIndex } from "./group-index.js";
 import { ByteJoinIndex } from "./join-index.js";
 import { QueryMemoryBudgetError, QueryMemoryContext, } from "./memory.js";
-import { compareSqlStrings, defineSqlResultProperty } from "./sql-semantics.js";
+import { compareSqlStrings, compileSimilarPattern, defineSqlResultProperty, } from "./sql-semantics.js";
+import { exactNumericBinary, exactNumericCompare, collatedDomainCompare, enumDomainCompare, externalSqlDomainValue, isExactNumeric, protectedSqlTextValue, } from "./sql-domains.js";
 import { buildSortKeyColumn, sortKeyIndexes } from "./sort-keys.js";
 const DEFAULT_BATCH_ROWS = 2_048;
 /** Above this, locating each IN member separately costs more than scanning between them. */
@@ -59,7 +61,7 @@ export function createColumnarTable(name, columns, uniqueKey) {
         ...(uniqueKey === undefined ? {} : { uniqueKey }),
     };
 }
-export function columnarTableFromRows(name, rows, projectedColumnNames) {
+export function columnarTableFromRows(name, rows, projectedColumnNames, protectText = true) {
     const columnNameSet = new Set(projectedColumnNames);
     if (projectedColumnNames === undefined) {
         for (const row of rows) {
@@ -72,7 +74,10 @@ export function columnarTableFromRows(name, rows, projectedColumnNames) {
         return { name, rowCount: rows.length, columns: new Map() };
     const columns = new Map();
     for (const columnName of columnNames) {
-        const values = rows.map((row) => row[columnName] ?? null);
+        const values = rows.map((row) => {
+            const value = row[columnName] ?? null;
+            return protectText && typeof value === "string" ? protectedSqlTextValue(value) : value;
+        });
         columns.set(columnName, { type: inferVectorType(values), values });
     }
     return createColumnarTable(name, columns);
@@ -105,7 +110,7 @@ export function prepareVectorQuery(plan, inputTables, options = {}) {
             async executeAsync(executionOptions = {}) {
                 if (closed)
                     throw new Error("Prepared vector query is closed");
-                const canSpillSort = bound.orderBy.length > 0 && !bound.grouped;
+                const canSpillSort = bound.orderBy.length > 0 && !bound.grouped && !bound.sourceOrdered;
                 // An unordered grouped plan spills too: the empty ordering makes the pairwise merge a
                 // stable concatenation, and partition-wise accumulation bounds peak group state.
                 const canSpillHash = bound.grouped && bound.groupBy.length > 0;
@@ -255,7 +260,7 @@ export function vectorValue(vector, rowIndex) {
     return code === NULL_STRING_CODE ? null : (vector.dictionary[code] ?? null);
 }
 function columnarTablePayloadBytes(table) {
-    let total = 0;
+    let total = table.scanOrder?.byteLength ?? 0;
     for (const vector of table.columns.values()) {
         total = safeMemorySum(total, vector.validity.byteLength, "Column vector payload");
         if (vector.kind === "string") {
@@ -315,7 +320,178 @@ function disjunctiveNormalForm(predicate) {
     };
     return visit(predicate.left) && branches.length > 1 ? branches : undefined;
 }
+/**
+ * Gives a pure comma/CROSS-join group a connected physical order once schemas can resolve bare
+ * column names. SQLLogicTest deliberately permutes FROM lists; preserving that written order can
+ * otherwise materialize several disconnected Cartesian products before their WHERE equalities
+ * become applicable. The streamed base stays fixed; explicit/outer joins and wildcard projection
+ * keep their complete written order.
+ *
+ * One equality that connects each new source becomes the inner hash-join key. The remaining
+ * conditions stay ordinary predicates, so null and residual-filter semantics do not change.
+ */
+function orderCartesianJoins(plan, tables) {
+    if (plan.joins.length < 2 ||
+        plan.select.some((item) => item.expression.kind === "wildcard") ||
+        plan.joins.some((join) => !isCartesianJoin(join))) {
+        return plan;
+    }
+    const entries = [
+        { source: plan.base, originalIndex: 0 },
+        ...plan.joins.map((join, index) => ({
+            source: tableSourceOf(join),
+            originalIndex: index + 1,
+        })),
+    ];
+    const sourceTables = entries.map(({ source }) => tables.get(source.table));
+    if (sourceTables.some((table) => table === undefined))
+        return plan;
+    const predicateSources = plan.predicates.map((predicate) => ({
+        predicate,
+        sources: rawPredicateSources(predicate, entries, sourceTables),
+    }));
+    // The preparation layer may stream the written base table through a resident window. Keep
+    // that source fixed and order only the fully materialized join sides around it.
+    const remaining = new Set(entries.slice(1).map(({ originalIndex }) => originalIndex));
+    const ordered = [0];
+    while (remaining.size > 0) {
+        let best;
+        let bestScore = Number.NEGATIVE_INFINITY;
+        for (const candidate of remaining) {
+            const score = cartesianSourceScore(candidate, ordered, predicateSources);
+            if (score > bestScore || (score === bestScore && (best === undefined || candidate < best))) {
+                best = candidate;
+                bestScore = score;
+            }
+        }
+        if (best === undefined)
+            return plan;
+        ordered.push(best);
+        remaining.delete(best);
+    }
+    const predicates = [...plan.predicates];
+    const available = new Set([ordered[0] ?? 0]);
+    const joins = [];
+    for (const sourceIndex of ordered.slice(1)) {
+        const source = entries[sourceIndex]?.source;
+        if (source === undefined)
+            return plan;
+        const keyIndex = predicates.findIndex((predicate) => predicateJoinsSource(predicate, sourceIndex, available, entries, sourceTables));
+        const key = keyIndex < 0 ? undefined : predicates.splice(keyIndex, 1)[0];
+        joins.push(key === undefined ? cartesianJoin(source) : keyedInnerJoin(source, key));
+        available.add(sourceIndex);
+    }
+    return { ...plan, base: plan.base, joins, predicates };
+}
+function isCartesianJoin(join) {
+    const condition = join.on;
+    return (join.kind === "inner" &&
+        condition?.kind === "condition" &&
+        condition.operator === "=" &&
+        condition.left.kind === "literal" &&
+        condition.left.value === 1 &&
+        condition.right.kind === "literal" &&
+        condition.right.value === 1);
+}
+function tableSourceOf(join) {
+    const { kind, left, right, on, full, natural, ...source } = join;
+    void kind;
+    void left;
+    void right;
+    void on;
+    void full;
+    void natural;
+    return source;
+}
+function cartesianJoin(source) {
+    return {
+        ...source,
+        kind: "inner",
+        left: { kind: "literal", value: null },
+        right: { kind: "literal", value: null },
+        on: {
+            kind: "condition",
+            operator: "=",
+            left: { kind: "literal", value: 1 },
+            right: { kind: "literal", value: 1 },
+        },
+    };
+}
+function keyedInnerJoin(source, predicate) {
+    return {
+        ...source,
+        kind: "inner",
+        left: predicate.left,
+        right: predicate.right,
+    };
+}
+function cartesianSourceScore(candidate, ordered, predicates) {
+    const available = new Set(ordered);
+    let local = 0;
+    let connected = 0;
+    let equality = 0;
+    for (const entry of predicates) {
+        const sources = entry.sources;
+        if (sources?.has(candidate) !== true)
+            continue;
+        if (sources.size === 1) {
+            local++;
+            continue;
+        }
+        if ([...sources].every((source) => source === candidate || available.has(source))) {
+            connected++;
+            if (entry.predicate.operator === "=")
+                equality++;
+        }
+    }
+    // A usable equality dominates a general connection, which dominates a local filter; original
+    // order breaks ties. The empty-order branch keeps the helper correct if base selection is ever
+    // made schema-aware too, though the streaming executor currently fixes that source.
+    return ordered.length === 0 ? local : equality * 100 + connected * 10 + local;
+}
+function rawPredicateSources(predicate, entries, tables) {
+    const sources = new Set();
+    return rawExpressionSources(predicate.left, entries, tables, sources) &&
+        rawExpressionSources(predicate.right, entries, tables, sources)
+        ? sources
+        : undefined;
+}
+function rawExpressionSources(expression, entries, tables, into) {
+    if (expression.kind === "subquery" || expression.kind === "exists")
+        return false;
+    if (expression.kind === "column") {
+        const parts = expression.reference.split(".");
+        let matches;
+        if (parts.length === 2) {
+            const match = entries.findIndex(({ source }) => source.alias === parts[0]);
+            matches = match < 0 || tables[match]?.columns.has(parts[1] ?? "") !== true ? [] : [match];
+        }
+        else {
+            const column = parts[0] ?? "";
+            matches = tables.flatMap((table, index) => table?.columns.has(column) === true ? [index] : []);
+        }
+        if (matches.length !== 1)
+            return false;
+        into.add(matches[0] ?? -1);
+        return true;
+    }
+    return childExpressions(expression).every((child) => rawExpressionSources(child, entries, tables, into));
+}
+function predicateJoinsSource(predicate, candidate, available, entries, tables) {
+    if (predicate.operator !== "=")
+        return false;
+    const left = new Set();
+    const right = new Set();
+    if (!rawExpressionSources(predicate.left, entries, tables, left) ||
+        !rawExpressionSources(predicate.right, entries, tables, right)) {
+        return false;
+    }
+    const candidateOnly = (sources) => sources.size === 1 && sources.has(candidate);
+    const availableOnly = (sources) => sources.size > 0 && [...sources].every((source) => available.has(source));
+    return ((candidateOnly(left) && availableOnly(right)) || (candidateOnly(right) && availableOnly(left)));
+}
 function bindPlan(plan, tables, memory, ftsStats) {
+    plan = orderCartesianJoins(plan, tables);
     const sources = [plan.base, ...plan.joins];
     const sourceTables = sources.map((source) => {
         const table = tables.get(source.table);
@@ -422,6 +598,7 @@ function bindPlan(plan, tables, memory, ftsStats) {
         ...(nulls === undefined ? {} : { nulls }),
     }));
     const grouped = groupBy.length > 0 || aggregateSpecs.length > 0;
+    const sourceOrdered = sourceTables[0]?.orderedForPlan === true && standardJoins.length === 0 && !grouped;
     // A single bare string-column GROUP BY can group on dictionary codes: identical values share a
     // code within one vector, so a code-indexed slot table replaces per-row hashing. Streamed
     // vectors swap dictionaries per window; the accumulator remaps its slot table by value on
@@ -440,9 +617,11 @@ function bindPlan(plan, tables, memory, ftsStats) {
         groupBy,
         groupIndexBySignature,
         aggregates: aggregateSpecs,
+        hasListAggregate: aggregateSpecs.some((aggregate) => aggregate.name === "JSON_ARRAYAGG" || aggregate.name === "STRING_AGG"),
         select,
         orderBy,
         grouped,
+        sourceOrdered,
         ...(codeGrouping === undefined ? {} : { codeGrouping }),
         wildcard: plan.select[0]?.expression.kind === "wildcard",
         ...(plan.limit === undefined ? {} : { limit: plan.limit }),
@@ -513,7 +692,11 @@ function bindExpression(expression, sources, aggregateSpecs, aggregateIndexes, m
         };
     }
     if (expression.kind === "literal" || expression.kind === "wildcard") {
-        return { ...expression, signature };
+        return expression.kind === "literal" &&
+            typeof expression.value === "string" &&
+            expression.internalSqlValue !== true
+            ? { ...expression, value: protectedSqlTextValue(expression.value), signature }
+            : { ...expression, signature };
     }
     if (expression.kind === "column") {
         const parts = expression.reference.split(".");
@@ -599,12 +782,26 @@ function bindExpression(expression, sources, aggregateSpecs, aggregateIndexes, m
             argument.vector.kind === "datetime"
             ? { source: argument.source, vector: argument.vector }
             : undefined;
-        const rawNumber = argument.kind === "column" && argument.vector.kind === "number"
+        const rawNumber = expression.name !== "JSON_ARRAYAGG" &&
+            expression.name !== "STRING_AGG" &&
+            argument.kind === "column" &&
+            argument.vector.kind === "number"
             ? { source: argument.source, vector: argument.vector }
             : undefined;
         aggregateSpecs.push({
             name: expression.name,
             argument,
+            ...(expression.name === "STRING_AGG"
+                ? { delimiter: required(arguments_[1], "STRING_AGG delimiter is missing") }
+                : {}),
+            ...(expression.aggregateOrderBy === undefined
+                ? {}
+                : {
+                    orderBy: expression.aggregateOrderBy.map((order) => ({
+                        ...order,
+                        expression: bindExpression(order.expression, sources, aggregateSpecs, aggregateIndexes, memory, ftsBySignature, ftsStats),
+                    })),
+                }),
             // The signature this slot was keyed by is the JSON of the compiled call, which carries the
             // flag — so COUNT(x) and COUNT(DISTINCT x) in one select land in separate slots.
             ...(expression.distinct === true ? { distinct: true } : {}),
@@ -674,7 +871,8 @@ function ensureFtsDictionaryCache(expression, columnIndex, vector, scoring) {
         const tokenCount = withScores ? new Uint32Array(vector.dictionary.length) : undefined;
         const termTf = withScores ? new Uint32Array(vector.dictionary.length * termCount) : undefined;
         for (let code = 0; code < vector.dictionary.length; code += 1) {
-            const tokens = tokenize(vector.dictionary[code] ?? "");
+            const rendered = externalSqlDomainValue(vector.dictionary[code] ?? "");
+            const tokens = tokenize(typeof rendered === "string" ? rendered : "");
             masks[code] = termsMask(tokens, expression.terms);
             if (tokenCount !== undefined)
                 tokenCount[code] = tokens.length;
@@ -922,8 +1120,10 @@ function runScanBatch(plan, start, length, groups, output, memory) {
         const scan = sourceRows[plan.scanSource];
         if (scan === undefined)
             return false;
-        for (let index = 0; index < length; index += 1)
-            scan[index] = start + index;
+        const order = plan.sourceTables[plan.scanSource]?.scanOrder;
+        for (let index = 0; index < length; index += 1) {
+            scan[index] = order?.[start + index] ?? start + index;
+        }
         const batch = { length, rowsBySource: sourceRows, memory: batchMemory };
         return consumeJoinedBatches(plan, batch, 0, groups, output, memory);
     }
@@ -938,7 +1138,7 @@ function executeBoundPlan(plan, memory) {
     const groups = new GroupAccumulator(plan, memory);
     const output = new ResultSink(plan, memory, true);
     const scanRows = plan.sourceTables[plan.scanSource]?.rowCount ?? 0;
-    const narrowed = ascendingScanRange(plan, 0, scanRows);
+    const narrowed = plan.sourceOrdered ? undefined : ascendingScanRange(plan, 0, scanRows);
     const ranges = narrowed?.ranges ?? [
         { begin: narrowed?.begin ?? 0, end: narrowed?.end ?? scanRows },
     ];
@@ -972,7 +1172,7 @@ async function executeBoundPlanAsync(plan, memory, options) {
         // window cannot answer instead of stepping through them. `windowEnd` is where the loader
         // stopped, which is the block boundary — exactly the span the ordering check covers.
         const windowEnd = typeof residentEnd === "number" && residentEnd > start ? residentEnd : start + length;
-        const narrowed = ascendingScanRange(plan, start, windowEnd);
+        const narrowed = plan.sourceOrdered ? undefined : ascendingScanRange(plan, start, windowEnd);
         if (narrowed === undefined) {
             if (runScanBatch(plan, start, length, groups, output, memory))
                 break;
@@ -1161,8 +1361,18 @@ async function executeBoundPlanWithHashSpill(plan, memory, options) {
                             const raw = spec.argument.kind === "wildcard"
                                 ? 1
                                 : evaluateBatchExpression(plan, spec.argument, batch, row);
-                            spillRow[`a${String(index)}`] =
-                                raw === null || raw === undefined ? null : asQueryValue(raw);
+                            if (spec.name === "STRING_AGG" && raw !== null && raw !== undefined) {
+                                const delimiter = required(spec.delimiter, "STRING_AGG delimiter is missing");
+                                spillRow[`a${String(index)}`] = JSON.stringify([
+                                    raw,
+                                    evaluateBatchExpression(plan, delimiter, batch, row),
+                                    (spec.orderBy ?? []).map((order) => encodeSpillValue(asQueryValue(evaluateBatchExpression(plan, order.expression, batch, row)))),
+                                ]);
+                            }
+                            else {
+                                spillRow[`a${String(index)}`] =
+                                    raw === null || raw === undefined ? null : asQueryValue(raw);
+                            }
                         }
                         batchMemory.tally(queryRowPayloadBytes(spillRow), "Hash spill value row");
                         const partition = hashQueryValues(groupValues) & (partitionCount - 1);
@@ -1446,7 +1656,7 @@ function createSpillRunReader(store, ownerId, run, columns, memory) {
 }
 function compareOrderedRows(left, right, orderBy) {
     for (const order of orderBy) {
-        const placed = explicitNullOrder(left[order.outputName], right[order.outputName], order.nulls);
+        const placed = nullOrder(left[order.outputName], right[order.outputName], order.nulls, order.direction);
         if (placed !== undefined && placed !== 0)
             return placed;
         const comparison = compareValues(left[order.outputName], right[order.outputName]);
@@ -1533,7 +1743,7 @@ function executeMetadataCount(plan, memory) {
 }
 function finishResult(plan, inputRows, memory) {
     const rows = inputRows;
-    if (plan.orderBy.length > 0) {
+    if (plan.orderBy.length > 0 && !plan.sourceOrdered) {
         const ordering = memory.reserve(safeMemoryProduct(rows.length, Uint32Array.BYTES_PER_ELEMENT * 2 +
             Uint8Array.BYTES_PER_ELEMENT +
             // Per extracted sort key: one slot, plus the null mask a numeric column adds beside
@@ -2110,7 +2320,7 @@ function consumeJoinedBatches(plan, batch, joinIndex, groups, output, memory, pr
     let working = batch;
     let complete = prefiltered;
     let owned;
-    if (joinIndex === 0 && !prefiltered) {
+    if (!prefiltered) {
         const filtered = prefilterBatch(plan, batch, join);
         if (filtered !== undefined) {
             complete = filtered.complete;
@@ -2144,7 +2354,7 @@ function consumeJoinedBatches(plan, batch, joinIndex, groups, output, memory, pr
 function reachedEarlyLimit(plan, outputRows) {
     // Early termination must still produce the rows the trailing OFFSET will discard.
     return (!plan.grouped &&
-        plan.orderBy.length === 0 &&
+        (plan.orderBy.length === 0 || plan.sourceOrdered) &&
         plan.limit !== undefined &&
         outputRows >= plan.limit + (plan.offset ?? 0));
 }
@@ -2186,11 +2396,18 @@ function* joinBatches(plan, input, join, memory) {
             const probeKey = evaluateBatchExpression(plan, join.probe, input, row);
             let buildRow = probeKey === null ? -1 : join.lookup.firstRow(probeKey);
             if (buildRow < 0) {
-                if (join.kind === "left") {
+                if (join.kind === "left" || join.kind === "anti") {
                     appendJoinedRow(ensureOutput(), outputLength, input, row, join.buildSource, -1);
                     outputLength += 1;
                 }
             }
+            else if (join.kind === "semi") {
+                appendJoinedRow(ensureOutput(), outputLength, input, row, join.buildSource, buildRow);
+                outputLength += 1;
+            }
+            else if (join.kind === "anti") {
+                // A match excludes the probe row.
+            }
             else {
                 while (buildRow >= 0) {
                     appendJoinedRow(ensureOutput(), outputLength, input, row, join.buildSource, buildRow);
@@ -2254,12 +2471,16 @@ function* loopJoinBatches(plan, input, join, loop, memory) {
                 if (!holds)
                     continue;
                 matched = true;
-                appendJoinedRow(ensureOutput(), outputLength, input, row, join.buildSource, buildRow);
-                outputLength += 1;
-                if (outputLength === DEFAULT_BATCH_ROWS)
-                    yield* emit();
+                if (join.kind !== "anti") {
+                    appendJoinedRow(ensureOutput(), outputLength, input, row, join.buildSource, buildRow);
+                    outputLength += 1;
+                    if (outputLength === DEFAULT_BATCH_ROWS)
+                        yield* emit();
+                }
+                if (join.kind === "semi" || join.kind === "anti")
+                    break;
             }
-            if (!matched && join.kind === "left") {
+            if (!matched && (join.kind === "left" || join.kind === "anti")) {
                 appendJoinedRow(ensureOutput(), outputLength, input, row, join.buildSource, -1);
                 outputLength += 1;
                 if (outputLength === DEFAULT_BATCH_ROWS)
@@ -2309,10 +2530,12 @@ function joinUniqueBatch(plan, input, join, memory) {
                         cache[code] = buildRow;
                     }
                 }
-                if (buildRow < 0 && join.kind === "inner")
+                if ((buildRow < 0 && (join.kind === "inner" || join.kind === "semi")) ||
+                    (buildRow >= 0 && join.kind === "anti")) {
                     continue;
+                }
                 selectedRows[outputLength] = row;
-                buildRows[outputLength] = buildRow;
+                buildRows[outputLength] = join.kind === "anti" ? -1 : buildRow;
                 outputLength += 1;
             }
         }
@@ -2320,10 +2543,12 @@ function joinUniqueBatch(plan, input, join, memory) {
             for (let row = 0; row < input.length; row += 1) {
                 const probeKey = evaluateBatchExpression(plan, join.probe, input, row);
                 const buildRow = probeKey === null ? -1 : join.lookup.firstRow(probeKey);
-                if (buildRow < 0 && join.kind === "inner")
+                if ((buildRow < 0 && (join.kind === "inner" || join.kind === "semi")) ||
+                    (buildRow >= 0 && join.kind === "anti")) {
                     continue;
+                }
                 selectedRows[outputLength] = row;
-                buildRows[outputLength] = buildRow;
+                buildRows[outputLength] = join.kind === "anti" ? -1 : buildRow;
                 outputLength += 1;
             }
         }
@@ -2415,6 +2640,7 @@ class ResultSink {
         const capacity = (plan.limit ?? 0) + (plan.offset ?? 0);
         const scanRows = plan.sourceTables[plan.scanSource]?.rowCount ?? 0;
         const bounded = !plan.grouped &&
+            !plan.sourceOrdered &&
             plan.orderBy.length > 0 &&
             plan.limit !== undefined &&
             (plan.joins.length > 0 || capacity < scanRows * BOUNDED_SINK_MAX_SHARE);
@@ -2692,7 +2918,7 @@ class ResultSink {
         const orderBy = this.#plan.orderBy;
         for (let index = 0; index < orderBy.length; index += 1) {
             const order = required(orderBy[index], "Order term is missing");
-            const placed = explicitNullOrder(left[index], right[index], order.nulls);
+            const placed = nullOrder(left[index], right[index], order.nulls, order.direction);
             if (placed !== undefined && placed !== 0)
                 return placed;
             const comparison = compareValues(left[index], right[index]);
@@ -2820,7 +3046,7 @@ class GroupAccumulator {
      * COUNT/SUM/AVG aggregates: one pass with unboxed reads and no per-row dispatch. Returns false
      * when the plan shape needs the generic per-row path.
      */
-    consumeFast(batch, passes, hasPredicates) {
+    consumeFast(batch) {
         const plan = this.#plan;
         const codeGrouping = plan.codeGrouping;
         const globalGroup = plan.groupBy.length === 0;
@@ -2831,7 +3057,7 @@ class GroupAccumulator {
         if (specs === undefined)
             return false;
         // The purest shape — global COUNT(*) with no predicates — needs no row loop at all.
-        if (globalGroup && !hasPredicates && specs.every((spec) => spec.kind === "star")) {
+        if (globalGroup && specs.every((spec) => spec.kind === "star")) {
             const state = required(this.#index.getEmpty(), "Grouped query state is missing");
             for (let index = 0; index < specs.length; index += 1) {
                 state.counts[index] = (state.counts[index] ?? 0) + batch.length;
@@ -2881,8 +3107,6 @@ class GroupAccumulator {
                 slots: groupVector.codes.length,
             };
         for (let row = 0; row < batch.length; row += 1) {
-            if (hasPredicates && !passes(row))
-                continue;
             let state = globalState;
             if (state === undefined && grouping !== undefined && states !== undefined) {
                 const sourceRow = groupRows?.[row] ?? -1;
@@ -3101,7 +3325,7 @@ function consumeBatch(plan, batch, groups, output, memory, prefiltered = false)
         }
         return;
     }
-    if (plan.grouped && groups.consumeFast(batch, () => true, false))
+    if (plan.grouped && groups.consumeFast(batch))
         return;
     if (!plan.grouped && output.tryAddBatch(batch))
         return;
@@ -3133,6 +3357,9 @@ function createGroupState(groupValues, plan, memory) {
         payloadBytes = safeMemorySum(payloadBytes, queryValuePayloadBytes(value), "Group state");
     }
     payloadBytes = safeMemorySum(payloadBytes, safeMemoryProduct(plan.aggregates.length, AGGREGATE_ACCUMULATOR_BYTES, "Aggregate accumulator state"), "Group state");
+    if (plan.hasListAggregate) {
+        payloadBytes = safeMemorySum(payloadBytes, safeMemoryProduct(plan.aggregates.length, QUERY_REFERENCE_BYTES, "JSON aggregate list slots"), "Group state");
+    }
     // tally, not reserve: a group state lives until the context closes and is never released
     // on its own, so a per-group QueryMemoryReservation object — retained in the context's Set
     // for the whole query — is pure overhead at one per distinct group.
@@ -3162,6 +3389,9 @@ function createGroupState(groupValues, plan, memory) {
         counts: new Float64Array(plan.aggregates.length),
         sums: new Float64Array(plan.aggregates.length),
         values: new Array(plan.aggregates.length),
+        ...(plan.hasListAggregate
+            ? { lists: new Array(plan.aggregates.length) }
+            : {}),
         valueReservations: new Array(plan.aggregates.length),
         valueReservationBytes: new Float64Array(plan.aggregates.length),
         distincts,
@@ -3236,7 +3466,11 @@ function updateAggregates(plan, state, batch, row, memory) {
         const value = spec.argument.kind === "wildcard"
             ? 1
             : evaluateBatchExpression(plan, spec.argument, batch, row);
-        applyAggregateValue(spec, state, index, value, memory);
+        const delimiter = spec.delimiter === undefined
+            ? undefined
+            : evaluateBatchExpression(plan, spec.delimiter, batch, row);
+        const orderValues = (spec.orderBy ?? []).map((order) => asQueryValue(evaluateBatchExpression(plan, order.expression, batch, row)));
+        applyAggregateValue(spec, state, index, value, memory, delimiter, orderValues);
     }
 }
 /** Reads a float64 slot (number value or raw epoch milliseconds) unboxed, or null when invalid. */
@@ -3258,10 +3492,63 @@ function rawFloat64Value(vector, rowIndex) {
 function updateAggregatesFromValues(plan, state, values, memory) {
     for (let index = 0; index < plan.aggregates.length; index += 1) {
         const spec = required(plan.aggregates[index], "Aggregate specification is missing");
-        applyAggregateValue(spec, state, index, values[index], memory);
+        const value = values[index];
+        if (spec.name === "STRING_AGG" && typeof value === "string") {
+            const decoded = JSON.parse(value);
+            if (!Array.isArray(decoded) || decoded.length !== 3) {
+                throw new Error("Spilled STRING_AGG input is invalid");
+            }
+            const encodedOrder = decoded[2];
+            if (!Array.isArray(encodedOrder))
+                throw new Error("Spilled STRING_AGG order is invalid");
+            applyAggregateValue(spec, state, index, decoded[0], memory, decoded[1], encodedOrder.map(decodeSpillValue));
+        }
+        else {
+            applyAggregateValue(spec, state, index, value, memory);
+        }
     }
 }
-function applyAggregateValue(spec, state, index, value, memory) {
+function applyAggregateValue(spec, state, index, value, memory, delimiter, orderValues = []) {
+    if (spec.name === "JSON_ARRAYAGG") {
+        const member = asQueryValue(value ?? null);
+        if (spec.distinct === true && !firstOfItsKind(state, index, member, memory))
+            return;
+        state.counts[index] = (state.counts[index] ?? 0) + 1;
+        const lists = required(state.lists, "JSON aggregate list state is missing");
+        let list = lists[index];
+        if (list === undefined) {
+            list = [];
+            lists[index] = list;
+        }
+        list.push(member);
+        memory.tally(safeMemorySum(QUERY_REFERENCE_BYTES, queryValuePayloadBytes(member), "JSON_ARRAYAGG member"), "JSON_ARRAYAGG member");
+        return;
+    }
+    if (spec.name === "STRING_AGG") {
+        if (value === null || value === undefined)
+            return;
+        value = externalSqlDomainValue(value);
+        delimiter = externalSqlDomainValue(delimiter);
+        if (typeof value !== "string")
+            throw new TypeError("STRING_AGG value must be a string");
+        if (delimiter !== null && delimiter !== undefined && typeof delimiter !== "string") {
+            throw new TypeError("STRING_AGG delimiter must be a string");
+        }
+        const distinctMember = JSON.stringify([value, delimiter ?? ""]);
+        if (spec.distinct === true && !firstOfItsKind(state, index, distinctMember, memory))
+            return;
+        const member = JSON.stringify([value, delimiter ?? "", orderValues.map(encodeSpillValue)]);
+        state.counts[index] = (state.counts[index] ?? 0) + 1;
+        const lists = required(state.lists, "STRING_AGG list state is missing");
+        let list = lists[index];
+        if (list === undefined) {
+            list = [];
+            lists[index] = list;
+        }
+        list.push(member);
+        memory.tally(safeMemorySum(QUERY_REFERENCE_BYTES, queryValuePayloadBytes(member), "STRING_AGG member"), "STRING_AGG member");
+        return;
+    }
     if (value === null || value === undefined)
         return;
     // One gate for every path that accumulates: the generic per-row path, the raw datetime path,
@@ -3270,7 +3557,16 @@ function applyAggregateValue(spec, state, index, value, memory) {
         return;
     state.counts[index] = (state.counts[index] ?? 0) + 1;
     if (spec.name === "SUM" || spec.name === "AVG") {
-        state.sums[index] = (state.sums[index] ?? 0) + numeric(value);
+        if (isExactNumeric(value)) {
+            const current = state.values[index];
+            const sum = current === undefined ? value : exactNumericBinary("+", current, value);
+            if (sum === null || sum === undefined)
+                throw new Error("Exact NUMERIC sum disappeared");
+            replaceAggregateValue(state, index, sum, `${spec.name} aggregate value`, memory);
+        }
+        else {
+            state.sums[index] = (state.sums[index] ?? 0) + numeric(value);
+        }
     }
     else if (spec.name === "MIN" &&
         (state.values[index] === undefined || compareValues(value, state.values[index]) < 0)) {
@@ -3364,12 +3660,55 @@ function evaluateFinalExpression(plan, expression, group) {
         return count;
     if (count === 0)
         return null;
-    const sum = group.sums[aggregateIndex] ?? 0;
-    if (expression.name === "SUM")
-        return sum;
-    if (expression.name === "AVG")
-        return sum / count;
+    if (expression.name === "JSON_ARRAYAGG") {
+        return JSON.stringify((required(group.lists, "JSON aggregate list state is missing")[aggregateIndex] ?? []).map(jsonValueOf));
+    }
+    if (expression.name === "STRING_AGG") {
+        const members = required(group.lists, "STRING_AGG list state is missing")[aggregateIndex] ?? [];
+        const decoded = members.map((encoded) => {
+            if (typeof encoded !== "string")
+                throw new Error("STRING_AGG member is invalid");
+            const pair = JSON.parse(encoded);
+            if (!Array.isArray(pair) ||
+                typeof pair[0] !== "string" ||
+                typeof pair[1] !== "string" ||
+                !Array.isArray(pair[2])) {
+                throw new Error("STRING_AGG member is invalid");
+            }
+            return {
+                value: pair[0],
+                delimiter: pair[1],
+                order: pair[2].map(decodeSpillValue),
+            };
+        });
+        const orderBy = plan.aggregates[aggregateIndex]?.orderBy ?? [];
+        if (orderBy.length > 0) {
+            decoded.sort((left, right) => {
+                for (const [index, order] of orderBy.entries()) {
+                    const a = left.order[index];
+                    const b = right.order[index];
+                    const placed = nullOrder(a, b, order.nulls, order.direction);
+                    if (placed !== undefined && placed !== 0)
+                        return placed;
+                    const compared = compareValues(a, b);
+                    if (compared !== 0)
+                        return order.direction === "desc" ? -compared : compared;
+                }
+                return 0;
+            });
+        }
+        return protectedSqlTextValue(decoded
+            .map((member, index) => (index === 0 ? member.value : member.delimiter + member.value))
+            .join(""));
+    }
     const value = group.values[aggregateIndex] ?? null;
+    if (expression.name === "SUM")
+        return isExactNumeric(value) ? value : (group.sums[aggregateIndex] ?? 0);
+    if (expression.name === "AVG") {
+        if (isExactNumeric(value))
+            return exactNumericBinary("/", value, count);
+        return (group.sums[aggregateIndex] ?? 0) / count;
+    }
     // Raw-millisecond datetime extremes re-box into a Date only here, once per surviving group.
     if (typeof value === "number" && plan.aggregates[aggregateIndex]?.rawDatetime !== undefined) {
         return new Date(value);
@@ -3598,19 +3937,24 @@ function detectDictionaryLike(predicate) {
  */
 const dictionaryLikeCache = new WeakMap();
 function dictionaryLikeMatches(dictionary, pattern, caseInsensitive, escape) {
+    const externalPattern = externalSqlDomainValue(pattern);
+    if (typeof externalPattern !== "string")
+        throw new TypeError("LIKE requires string operands");
     let patterns = dictionaryLikeCache.get(dictionary);
     if (patterns === undefined) {
         patterns = new Map();
         dictionaryLikeCache.set(dictionary, patterns);
     }
-    const key = `${caseInsensitive ? "i" : "s"}${escape ?? ""} ${pattern}`;
+    const key = `${caseInsensitive ? "i" : "s"}${escape ?? ""} ${externalPattern}`;
     let matches = patterns.get(key);
     if (matches === undefined) {
         matches = new Uint8Array(dictionary.length);
         for (let index = 0; index < dictionary.length; index += 1) {
-            matches[index] = likeMatches(pattern, dictionary[index] ?? "", caseInsensitive, escape)
-                ? 1
-                : 0;
+            const value = externalSqlDomainValue(dictionary[index] ?? "");
+            matches[index] =
+                typeof value === "string" && likeMatches(externalPattern, value, caseInsensitive, escape)
+                    ? 1
+                    : 0;
         }
         if (patterns.size >= 32)
             patterns.clear();
@@ -3709,17 +4053,24 @@ function booleanTruth(expression, evaluateValue) {
         if (operator === "LIKE" ||
             operator === "NOT LIKE" ||
             operator === "ILIKE" ||
-            operator === "NOT ILIKE") {
-            const value = evaluateValue(expression.left);
-            const pattern = evaluateValue(expression.right);
+            operator === "NOT ILIKE" ||
+            operator === "SIMILAR TO" ||
+            operator === "NOT SIMILAR TO") {
+            const value = externalSqlDomainValue(evaluateValue(expression.left));
+            const pattern = externalSqlDomainValue(evaluateValue(expression.right));
             if (value === null || value === undefined || pattern === null || pattern === undefined) {
                 return null;
             }
             if (typeof value !== "string" || typeof pattern !== "string") {
                 throw new TypeError("LIKE requires string operands");
             }
-            const matched = likeMatches(pattern, value, operator === "ILIKE" || operator === "NOT ILIKE", expression.escape);
-            return operator === "LIKE" || operator === "ILIKE" ? matched : !matched;
+            const similar = operator === "SIMILAR TO" || operator === "NOT SIMILAR TO";
+            const matched = similar
+                ? compileSimilarPattern(pattern, expression.escape ?? "\\").test(value)
+                : likeMatches(pattern, value, operator === "ILIKE" || operator === "NOT ILIKE", expression.escape);
+            return operator === "LIKE" || operator === "ILIKE" || operator === "SIMILAR TO"
+                ? matched
+                : !matched;
         }
         if (operator === "IS DISTINCT FROM" || operator === "IS NOT DISTINCT FROM") {
             const distinct = distinctFromComparison(evaluateValue(expression.left), evaluateValue(expression.right));
@@ -3732,9 +4083,9 @@ function booleanTruth(expression, evaluateValue) {
         const a = comparable(left);
         const b = comparable(right);
         if (operator === "=")
-            return a === b;
+            return compareValues(a, b) === 0;
         if (operator === "!=" || operator === "<>")
-            return a !== b;
+            return compareValues(a, b) !== 0;
         const comparison = compareValues(a, b);
         if (operator === ">")
             return comparison > 0;
@@ -3933,8 +4284,11 @@ function binaryValue(operator, left, right) {
         if (typeof left !== "string" || typeof right !== "string") {
             throw new TypeError("|| requires string operands");
         }
-        return left + right;
+        return protectedSqlTextValue(String(externalSqlDomainValue(left)) + String(externalSqlDomainValue(right)));
     }
+    const exact = exactNumericBinary(operator, left, right);
+    if (exact !== undefined)
+        return exact;
     const a = numeric(left);
     const b = numeric(right);
     if (operator === "+")
@@ -3963,7 +4317,11 @@ function comparisonValue(operator, leftValue, rightValue) {
     if (operator === "LIKE" ||
         operator === "NOT LIKE" ||
         operator === "ILIKE" ||
-        operator === "NOT ILIKE") {
+        operator === "NOT ILIKE" ||
+        operator === "SIMILAR TO" ||
+        operator === "NOT SIMILAR TO") {
+        leftValue = externalSqlDomainValue(leftValue);
+        rightValue = externalSqlDomainValue(rightValue);
         if (leftValue === null ||
             leftValue === undefined ||
             rightValue === null ||
@@ -3973,8 +4331,13 @@ function comparisonValue(operator, leftValue, rightValue) {
         if (typeof leftValue !== "string" || typeof rightValue !== "string") {
             throw new TypeError("LIKE requires string operands");
         }
-        const matched = likeMatches(rightValue, leftValue, operator === "ILIKE" || operator === "NOT ILIKE");
-        return operator === "LIKE" || operator === "ILIKE" ? matched : !matched;
+        const similar = operator === "SIMILAR TO" || operator === "NOT SIMILAR TO";
+        const matched = similar
+            ? compileSimilarPattern(rightValue).test(leftValue)
+            : likeMatches(rightValue, leftValue, operator === "ILIKE" || operator === "NOT ILIKE");
+        return operator === "LIKE" || operator === "ILIKE" || operator === "SIMILAR TO"
+            ? matched
+            : !matched;
     }
     if (leftValue === null ||
         leftValue === undefined ||
@@ -3985,9 +4348,9 @@ function comparisonValue(operator, leftValue, rightValue) {
     const left = comparable(leftValue);
     const right = comparable(rightValue);
     if (operator === "=")
-        return left === right;
+        return compareValues(left, right) === 0;
     if (operator === "!=" || operator === "<>")
-        return left !== right;
+        return compareValues(left, right) !== 0;
     const comparison = compareValues(left, right);
     if (operator === ">")
         return comparison > 0;
@@ -4039,6 +4402,15 @@ function stableSortRows(rows, orderBy) {
     }
 }
 function compareValues(left, right) {
+    const collated = collatedDomainCompare(left, right);
+    if (collated !== undefined)
+        return collated;
+    const enumOrder = enumDomainCompare(left, right);
+    if (enumOrder !== undefined)
+        return enumOrder;
+    const exact = exactNumericCompare(left, right);
+    if (exact !== undefined)
+        return exact;
     const a = left instanceof Date ? left.getTime() : left;
     const b = right instanceof Date ? right.getTime() : right;
     if (a === b)