@rocicorp/zero 1.12.0-canary.1 → 1.12.0-canary.11

package/out/zql/src/builder/builder.js

@@ -1,5 +1,10 @@
 import { assert, unreachable } from "../../../shared/src/asserts.js";
+import { parse } from "../../../shared/src/valita.js";
 import { must } from "../../../shared/src/must.js";
+import { formatJsonPathReference, simpleConditionSchema } from "../../../zero-protocol/src/ast.js";
+import { assignProperty } from "../../../shared/src/objects.js";
+import { AGGREGATE_KEY_COLUMN, Aggregate, Reducer, aggregateSourceSchema, assertNotSyntheticColumn, isInvertible } from "../ivm/aggregate.js";
+import { AGGREGATE_TABLE_PREFIX } from "../../../zero-protocol/src/aggregate-table.js";
 import { completeOrdering } from "../query/complete-ordering.js";
 import { Take } from "../ivm/take.js";
 import { Cap } from "../ivm/cap.js";
@@ -12,8 +17,10 @@ import { JoinIndex } from "../ivm/join-index.js";
 import { createPredicate, transformFilters } from "./filter.js";
 import { TakeGate } from "../ivm/take-gate.js";
 import { FlippedJoin } from "../ivm/flipped-join.js";
+import { GroupBy, GroupProjection, groupOrdering, groupSourceSchema } from "../ivm/group-by.js";
 import { JoinIndexTap } from "../ivm/join-index-tap.js";
 import { Join } from "../ivm/join.js";
+import { LIFTED_FIELD_COLUMN, LiftField } from "../ivm/lift-field.js";
 import { Skip } from "../ivm/skip.js";
 import { UnionFanIn } from "../ivm/union-fan-in.js";
 import { UnionFanOut } from "../ivm/union-fan-out.js";
@@ -46,6 +53,12 @@ import { pushDownCorrelatedPredicates } from "./correlated-predicate-pushdown.js
 */
 function buildPipeline(ast, delegate, queryID, costModel, lc, planDebugger) {
 	ast = delegate.mapAst ? delegate.mapAst(ast) : ast;
+	assert(ast.groupBy === void 0 || ast.aggregate?.distinct === void 0, "a count of distinct values per group is not supported yet");
+	if ((ast.aggregate || ast.groupBy) && delegate.aggregatesFromSource) return buildPipelineInternal(ast, delegate, queryID, "");
+	if (ast.groupBy || ast.aggregate?.distinct) ast = {
+		...ast,
+		orderBy: groupOrdering(ast)
+	};
 	ast = completeOrdering(ast, (tableName) => must(delegate.getSource(tableName)).tableSchema.primaryKey);
 	const columnsOf = (tableName) => must(delegate.getSource(tableName)).tableSchema.columns;
 	const { planWarnings } = delegate;
@@ -71,11 +84,15 @@ function bindStaticParameters(ast, staticQueryParameters) {
 		}))
 	});
 	function bindCondition(condition) {
-		if (condition.type === "simple") return {
-			...condition,
-			left: bindValue(condition.left),
-			right: bindValue(condition.right)
-		};
+		if (condition.type === "simple") {
+			const bound = {
+				...condition,
+				left: bindValue(condition.left),
+				right: bindValue(condition.right)
+			};
+			if (bound.left.type === "json" && condition.right.type === "static") parse(bound, simpleConditionSchema);
+			return bound;
+		}
 		if (condition.type === "correlatedSubquery") return {
 			...condition,
 			related: {
@@ -129,6 +146,15 @@ function assertNoNotExists(condition) {
 	}
 }
 function buildPipelineInternal(ast, delegate, queryID, name, partitionKey, isNonFlippedExistsChild) {
+	assert(ast.groupBy === void 0 || name === "", "groupBy is only supported at the top level");
+	if (ast.aggregate && !ast.groupBy && delegate.aggregatesFromSource) {
+		const aggTable = topLevelAggregateTableName(queryID);
+		const aggSource = delegate.getSource(aggTable);
+		if (!aggSource) throw new Error(`Aggregate source not found: ${aggTable}`);
+		const conn = aggSource.connect([[AGGREGATE_KEY_COLUMN, "asc"]]);
+		return delegate.decorateInput(delegate.decorateSourceInput(conn, queryID), `${name}:aggregateSource`);
+	}
+	if (ast.groupBy && delegate.aggregatesFromSource) return groupFromSource(ast, delegate, queryID, name);
 	const source = delegate.getSource(ast.table);
 	if (!source) throw new Error(`Source not found: ${ast.table}`);
 	ast = uniquifyCorrelatedSubqueryConditionAliases(ast);
@@ -178,18 +204,20 @@ function buildPipelineInternal(ast, delegate, queryID, name, partitionKey, isNon
 		});
 	}
 	if (ast.where && (!fullyAppliedFilters || delegate.applyFiltersAnyway)) end = applyWhere(end, ast.where, delegate, name, partitionKey, takeGate, existsParentIndexes);
-	if (ast.limit !== void 0) if (useCap) {
-		const capName = `${name}:cap`;
-		const cap = new Cap(end, delegate.createStorage(capName), ast.limit, partitionKey);
-		delegate.addEdge(end, cap);
-		end = delegate.decorateInput(cap, capName);
-	} else {
-		const takeName = `${name}:take`;
-		const take = new Take(end, delegate.createStorage(takeName), ast.limit, partitionKey);
-		delegate.addEdge(end, take);
-		end = delegate.decorateInput(take, takeName);
-		takeGate?.setBoundProvider(take);
-		if (takeGate) take.setTakeGate(takeGate);
+	if (ast.limit !== void 0) {
+		if (useCap) {
+			const capName = `${name}:cap`;
+			const cap = new Cap(end, delegate.createStorage(capName), ast.limit, partitionKey);
+			delegate.addEdge(end, cap);
+			end = delegate.decorateInput(cap, capName);
+		} else {
+			const takeName = `${name}:take`;
+			const take = new Take(end, delegate.createStorage(takeName), ast.limit, partitionKey);
+			delegate.addEdge(end, take);
+			end = delegate.decorateInput(take, takeName);
+			takeGate?.setBoundProvider(take);
+			if (takeGate) take.setTakeGate(takeGate);
+		}
 	}
 	if (existsParentIndexes.length > 0) {
 		const tapName = `${name}:join-index`;
@@ -197,6 +225,30 @@ function buildPipelineInternal(ast, delegate, queryID, name, partitionKey, isNon
 		delegate.addEdge(end, tap);
 		end = delegate.decorateInput(tap, tapName);
 	}
+	if (ast.groupBy) {
+		assert(ast.start === void 0 && ast.limit === void 0 && ast.related === void 0, "groupBy does not support start, limit or related");
+		const sourceAppliesAll = csqConditions.length === 0 && (!ast.where || fullyAppliedFilters && !delegate.applyFiltersAnyway);
+		const groupName = `${name}:groupBy`;
+		const groupBy = new GroupBy(end, delegate.createStorage(groupName), groupOrdering(ast), groupTableName(queryID), reducerOf(ast), sourceAppliesAll ? conn : void 0);
+		delegate.addEdge(end, groupBy);
+		return delegate.decorateInput(groupBy, groupName);
+	}
+	if (ast.aggregate) {
+		const sourceAppliesAll = ast.start === void 0 && ast.limit === void 0 && csqConditions.length === 0 && (!ast.where || fullyAppliedFilters && !delegate.applyFiltersAnyway);
+		let countSource = conn;
+		if (ast.aggregate.distinct) {
+			assert(ast.start === void 0 && ast.limit === void 0, "a count of distinct values does not support start or limit");
+			const groupName = `${name}:groupBy`;
+			const groupBy = new GroupBy(end, delegate.createStorage(groupName), groupOrdering(ast), groupTableName(queryID), new Reducer("count", void 0, void 0), sourceAppliesAll ? conn : void 0);
+			delegate.addEdge(end, groupBy);
+			end = delegate.decorateInput(groupBy, groupName);
+			countSource = void 0;
+		}
+		const aggName = `${name}:aggregate`;
+		const aggregate = new Aggregate(end, delegate.createStorage(aggName), [], ast.aggregate.fn, ast.aggregate.field, topLevelAggregateTableName(queryID), sourceAppliesAll ? countSource : void 0, ast.aggregate.path);
+		delegate.addEdge(end, aggregate);
+		return delegate.decorateInput(aggregate, aggName);
+	}
 	if (ast.related) {
 		const byAlias = /* @__PURE__ */ new Map();
 		for (const csq of ast.related) byAlias.set(csq.subquery.alias ?? "", csq);
@@ -247,7 +299,7 @@ function applyFilterWithFlips(input, condition, delegate, name, parentPartitionK
 		}
 		case "correlatedSubquery": {
 			const sq = condition.related;
-			const child = buildPipelineInternal(sq.subquery, delegate, "", `${name}.${sq.subquery.alias}`, sq.correlation.childField, false);
+			const child = buildPipelineInternal(sq.subquery, delegate, "", childPipelineName(name, sq), sq.correlation.childField, false);
 			let parentIndex;
 			if (existsParentIndexes) {
 				parentIndex = new JoinIndex(sq.correlation.parentField, end.getSchema().primaryKey, parentPartitionKey);
@@ -334,18 +386,86 @@ function valuePosName(left) {
 		case "static": return left.field;
 		case "literal": return left.value;
 		case "column": return left.name;
+		case "json": return formatJsonPathReference(left);
+		default: unreachable(left);
 	}
 }
 function applyCorrelatedSubQuery(sq, delegate, queryID, end, name, fromCondition, parentPartitionKey, boundProvider, exists) {
 	if (sq.subquery.limit === 0 && fromCondition) return end;
 	assert(sq.subquery.alias, "Subquery must have an alias");
-	const child = buildPipelineInternal(sq.subquery, delegate, queryID, `${name}.${sq.subquery.alias}`, sq.correlation.childField, fromCondition);
+	let childInput;
+	let childKey = sq.correlation.childField;
+	if (sq.aggregate) {
+		assert(sq.aggregate.distinct === void 0, "a count of distinct values is not supported in a relationship yet");
+		const aggTable = aggregateTableName(queryID, sq, name);
+		if (delegate.aggregatesFromSource) {
+			childKey = delegate.aggregateKeyColumns?.(sq.subquery.table, sq.correlation.childField) ?? sq.correlation.childField;
+			for (const col of childKey) assertNotSyntheticColumn(col);
+			let source;
+			if (delegate.getAggregateSource) {
+				const childSource = delegate.getSource(sq.subquery.table);
+				if (!childSource) throw new Error(`Source not found: ${sq.subquery.table}`);
+				let inputColumns = childSource.tableSchema.columns;
+				const isJunction = !!(sq.subquery.related && sq.subquery.related.length);
+				if (isJunction) {
+					const destTable = must(sq.subquery.related)[0].subquery.table;
+					const destSource = delegate.getSource(destTable);
+					if (!destSource) throw new Error(`Source not found: ${destTable}`);
+					inputColumns = {
+						...childSource.tableSchema.columns,
+						...destSource.tableSchema.columns
+					};
+				}
+				const { columns: clientColumns } = aggregateSourceSchema(inputColumns, sq.correlation.childField, sq.aggregate.fn, sq.aggregate.field, sq.aggregate.path);
+				const columns = renameColumns(clientColumns, sq.correlation.childField, childKey);
+				const { fn } = sq.aggregate;
+				const invertible = isInvertible(fn);
+				let whereEvaluable = true;
+				let wherePredicate;
+				if (!isJunction && invertible && sq.subquery.where !== void 0) {
+					const { filters, conditionsRemoved } = transformFilters(sq.subquery.where);
+					if (conditionsRemoved || filters === void 0) whereEvaluable = false;
+					else wherePredicate = createPredicate(filters);
+				}
+				source = delegate.getAggregateSource(aggTable, columns, childKey, !isJunction && invertible && whereEvaluable && sq.subquery.limit === void 0 && sq.subquery.start === void 0 ? {
+					table: sq.subquery.table,
+					childField: sq.correlation.childField,
+					keyField: childKey,
+					fn,
+					field: sq.aggregate.field,
+					path: sq.aggregate.path,
+					predicate: wherePredicate
+				} : void 0);
+			} else source = delegate.getSource(aggTable);
+			if (!source) throw new Error(`Aggregate source not found: ${aggTable}`);
+			const conn = source.connect(childKey.map((f) => [f, "asc"]));
+			childInput = delegate.decorateSourceInput(conn, queryID);
+			childInput = delegate.decorateInput(childInput, `${name}:aggregateSource(${sq.subquery.alias})`);
+		} else {
+			const child = buildPipelineInternal(sq.subquery, delegate, queryID, childPipelineName(name, sq), sq.correlation.childField, fromCondition);
+			const aggName = `${name}:aggregate(${sq.subquery.alias})`;
+			let aggInput = child;
+			let aggField = sq.aggregate.field;
+			let aggPath = sq.aggregate.path;
+			if (sq.subquery.related && sq.subquery.related.length > 0) {
+				const destAlias = must(sq.subquery.related[0].subquery.alias);
+				const lift = new LiftField(child, destAlias, must(sq.aggregate.field), aggPath === void 0 ? void 0 : sq.aggregate);
+				delegate.addEdge(child, lift);
+				aggInput = lift;
+				aggField = LIFTED_FIELD_COLUMN;
+				aggPath = void 0;
+			}
+			const aggregate = new Aggregate(aggInput, delegate.createStorage(aggName), sq.correlation.childField, sq.aggregate.fn, aggField, aggTable, void 0, aggPath);
+			delegate.addEdge(aggInput, aggregate);
+			childInput = delegate.decorateInput(aggregate, aggName);
+		}
+	} else childInput = buildPipelineInternal(sq.subquery, delegate, queryID, childPipelineName(name, sq), sq.correlation.childField, fromCondition);
 	const joinName = `${name}:join(${sq.subquery.alias})`;
 	const join = new Join({
 		parent: end,
-		child,
+		child: childInput,
 		parentKey: sq.correlation.parentField,
-		childKey: sq.correlation.childField,
+		childKey,
 		relationshipName: sq.subquery.alias,
 		hidden: sq.hidden ?? false,
 		system: sq.system ?? "client",
@@ -354,7 +474,7 @@ function applyCorrelatedSubQuery(sq, delegate, queryID, end, name, fromCondition
 		exists
 	});
 	delegate.addEdge(end, join);
-	delegate.addEdge(child, join);
+	delegate.addEdge(childInput, join);
 	return delegate.decorateInput(join, joinName);
 }
 function applyCorrelatedSubqueryCondition(input, condition, delegate, name) {
@@ -389,6 +509,158 @@ function gatherCorrelatedSubqueryQueryConditions(condition) {
 	if (condition) gather(condition);
 	return csqs;
 }
+/** `columns` with each of `from` renamed to the name at its index in `to`. */
+function renameColumns(columns, from, to) {
+	const renamed = {};
+	for (const [name, schema] of Object.entries(columns)) {
+		const i = from.indexOf(name);
+		assignProperty(renamed, i === -1 ? name : to[i], schema);
+	}
+	return renamed;
+}
+/**
+* Deterministic synthetic table name for an `aggregate` relationship. Must
+* agree between the server (which produces the rows) and the synced client
+* (which reads them). It folds in the `queryID` — the client query hash, which
+* both sides share and which already distinguishes differently-filtered queries
+* — and the relationship's alias path from the root, so two aggregates never
+* collide, even ones with the same alias at different depths.
+*
+* @param parentName The pipeline name of the parent, which the builder forms
+*        from the aliases of the relationships leading to it (`''` at the
+*        root, `'.comments'` one level down). It is the same on both sides.
+*/
+function aggregateTableName(queryID, sq, parentName = "") {
+	const path = childPipelineName(parentName, sq).slice(1);
+	return `${AGGREGATE_TABLE_PREFIX}${queryID}:${path}`;
+}
+/**
+* The pipeline name of the child `sq` under the pipeline `name`: the parent's
+* name, a `.`, and the escaped alias. A relationship name can be any string, so
+* the alias is escaped to keep the `.`-joined path unambiguous and free of the
+* `/` and `:` that {@link aggregateTableName} must not contain.
+*/
+function childPipelineName(name, sq) {
+	return `${name}.${encodeURIComponent(sq.subquery.alias ?? "").replaceAll(".", "%2E")}`;
+}
+/**
+* Deterministic synthetic table name for a *top-level* (ungrouped) aggregate
+* query (e.g. `z.query.issue.count()`). There is no relationship alias, so the
+* name is just the prefix + `queryID` (`aggregate:<queryID>`). Distinct from
+* relationship-aggregate names (`aggregate:<queryID>:<alias>`), which always
+* carry an alias segment — which is how the client tells the two apart by name.
+*/
+function topLevelAggregateTableName(queryID) {
+	return `${AGGREGATE_TABLE_PREFIX}${queryID}`;
+}
+/**
+* The synthetic table of a grouped query's rows (a `distinct()` or a
+* `groupBy()`): `aggregate:<queryID>:` and a `#`, which an escaped
+* relationship alias never contains, so it can't be a relationship
+* aggregate's, and isn't a top-level aggregate's either, whose shape the
+* client derives from the name.
+*/
+function groupTableName(queryID) {
+	return `${AGGREGATE_TABLE_PREFIX}${queryID}${GROUP_TABLE_SUFFIX}`;
+}
+var GROUP_TABLE_SUFFIX = `:#group`;
+/** What a grouped query reduces each group to: its aggregate, or a count. */
+function reducerOf(ast) {
+	const { aggregate } = ast;
+	return aggregate ? new Reducer(aggregate.fn, aggregate.field, aggregate.path) : new Reducer("count", void 0, void 0);
+}
+/**
+* The names a grouped query's input rows have `columns` under: the query's
+* own, or, for a synced client reading them from the synthetic source, the
+* server's.
+*/
+function inputNames(ast, delegate, columns) {
+	return delegate.aggregatesFromSource && delegate.aggregateKeyColumns ? delegate.aggregateKeyColumns(ast.table, columns) : columns;
+}
+/**
+* A synced client reads a grouped query's rows from the synthetic source the
+* server streams them to; the table's rows are not synced for it.
+*/
+function groupFromSource(ast, delegate, queryID, name) {
+	const tableSource = must(delegate.getSource(ast.table), `Source not found: ${ast.table}`);
+	const reducer = reducerOf(ast);
+	const columns = [...must(ast.groupBy)];
+	if (reducer.field !== void 0 && !columns.includes(reducer.field)) columns.push(reducer.field);
+	const names = inputNames(ast, delegate, columns);
+	const inputName = (col) => names[columns.indexOf(col)];
+	const inputColumns = {};
+	for (const col of columns) assignProperty(inputColumns, inputName(col), tableSource.tableSchema.columns[col]);
+	const { columns: sourceColumns, primaryKey, sort } = groupSourceSchema(inputColumns, groupOrdering(ast).map(([col, dir]) => [inputName(col), dir]), {
+		fn: reducer.fn,
+		field: reducer.field === void 0 ? void 0 : inputName(reducer.field),
+		path: reducer.path
+	});
+	const table = groupTableName(queryID);
+	const source = delegate.getAggregateSource ? delegate.getAggregateSource(table, sourceColumns, primaryKey, groupOptimisticDelta(ast, reducer, inputName)) : delegate.getSource(table);
+	if (!source) throw new Error(`Group source not found: ${table}`);
+	return delegate.decorateInput(delegate.decorateSourceInput(source.connect(sort), queryID), `${name}:groupSource`);
+}
+/**
+* What the synced client needs to move a grouped query's rows when it mutates
+* a row of its table, so that a mutator reads its own writes and a view shows
+* them before the server answers. As for a relationship aggregate, only for
+* the invertible functions, and only when the `where` can be judged from the
+* row alone: otherwise the rows are server-authoritative.
+*/
+function groupOptimisticDelta(ast, reducer, inputName) {
+	if (!isInvertible(reducer.fn)) return;
+	let predicate;
+	if (ast.where !== void 0) {
+		const { filters, conditionsRemoved } = transformFilters(ast.where);
+		if (conditionsRemoved || filters === void 0) return;
+		predicate = createPredicate(filters);
+	}
+	const columns = groupOrdering(ast).map(([col]) => col);
+	return {
+		table: ast.table,
+		childField: columns,
+		keyField: columns.map(inputName),
+		fn: reducer.fn,
+		field: reducer.field,
+		path: ast.aggregate?.path,
+		predicate,
+		grouped: true
+	};
+}
+/**
+* The input of a view of a grouped query: its rows reduced to the group
+* columns, and for a `groupBy()` its value under the aggregate function's
+* name, under the names the view shows a row's columns by, those of the
+* pipeline's AST (see {@link GroupProjection}).
+*/
+function groupViewInput(input, ast, delegate) {
+	const viewNames = must((delegate.mapAst ? delegate.mapAst(ast) : ast).groupBy);
+	const names = delegate.aggregatesFromSource ? inputNames(ast, delegate, must(ast.groupBy)) : viewNames;
+	return new GroupProjection(input, viewNames.map((col, i) => [col, names[i]]), ast.aggregate?.fn);
+}
+/** True if `name` is a synthetic aggregate table (any kind). */
+function isAggregateTableName(name) {
+	return name.startsWith(AGGREGATE_TABLE_PREFIX);
+}
+/**
+* True if `name` is a *top-level* aggregate table (`aggregate:<queryID>`), as
+* opposed to a relationship one (`aggregate:<queryID>:<alias>`). Top-level names
+* have no further separator after the prefix. The client uses this to recognize
+* a synthetic source it can provision from the name alone (fixed key + value
+* shape), which relationship aggregates — needing the correlation key — cannot.
+*/
+function isTopLevelAggregateTableName(name) {
+	return name.startsWith("aggregate:") && !name.slice(10).includes(":");
+}
+/**
+* True if the rows of the synthetic table `name` are keyed by `_0_key`
+* alone: a top-level aggregate's ({@link topLevelAggregateTableName}) or a
+* grouped query's ({@link groupTableName}). A relationship aggregate's rows
+* are keyed by their correlation columns, whatever they are named.
+*/
+function isKeyedAggregateTableName(name) {
+	return isTopLevelAggregateTableName(name) || name.startsWith("aggregate:") && name.endsWith(GROUP_TABLE_SUFFIX);
+}
 function uniquifyCorrelatedSubqueryConditionAliases(ast) {
 	if (!ast.where) return ast;
 	const { where } = ast;
@@ -432,6 +704,6 @@ function partitionBranches(conditions, predicate) {
 	return [matched, notMatched];
 }
 //#endregion
-export { bindStaticParameters, buildPipeline };
+export { aggregateTableName, bindStaticParameters, buildPipeline, conditionIncludesFlippedSubqueryAtAnyLevel, groupSubqueryConditions, groupTableName, groupViewInput, isAggregateTableName, isKeyedAggregateTableName, isTopLevelAggregateTableName, partitionBranches, topLevelAggregateTableName };
 
 //# sourceMappingURL=builder.js.map