duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/planner/expression_binder/select_binder.cpp

@@ -1,145 +1,14 @@
 #include "duckdb/planner/expression_binder/select_binder.hpp"
 
-#include "duckdb/parser/expression/columnref_expression.hpp"
-#include "duckdb/parser/expression/window_expression.hpp"
-#include "duckdb/parser/parsed_expression_iterator.hpp"
-#include "duckdb/planner/expression/bound_columnref_expression.hpp"
-#include "duckdb/planner/expression/bound_window_expression.hpp"
-#include "duckdb/planner/expression_binder/aggregate_binder.hpp"
-#include "duckdb/planner/query_node/bound_select_node.hpp"
-#include "duckdb/parser/expression/operator_expression.hpp"
-#include "duckdb/common/string_util.hpp"
-#include "duckdb/planner/binder.hpp"
-
 namespace duckdb {
 
 SelectBinder::SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
                            case_insensitive_map_t<idx_t> alias_map)
-    : ExpressionBinder(binder, context), inside_window(false), node(node), info(info), alias_map(std::move(alias_map)) {
+    : BaseSelectBinder(binder, context, node, info, std::move(alias_map)) {
 }
 
 SelectBinder::SelectBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info)
     : SelectBinder(binder, context, node, info, case_insensitive_map_t<idx_t>()) {
 }
 
-BindResult SelectBinder::BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth, bool root_expression) {
-	auto &expr = **expr_ptr;
-	// check if the expression binds to one of the groups
-	auto group_index = TryBindGroup(expr, depth);
-	if (group_index != DConstants::INVALID_INDEX) {
-		return BindGroup(expr, depth, group_index);
-	}
-	switch (expr.expression_class) {
-	case ExpressionClass::COLUMN_REF:
-		return BindColumnRef(expr_ptr, depth);
-	case ExpressionClass::DEFAULT:
-		return BindResult("SELECT clause cannot contain DEFAULT clause");
-	case ExpressionClass::WINDOW:
-		return BindWindow((WindowExpression &)expr, depth);
-	default:
-		return ExpressionBinder::BindExpression(expr_ptr, depth);
-	}
-}
-
-idx_t SelectBinder::TryBindGroup(ParsedExpression &expr, idx_t depth) {
-	// first check the group alias map, if expr is a ColumnRefExpression
-	if (expr.type == ExpressionType::COLUMN_REF) {
-		auto &colref = (ColumnRefExpression &)expr;
-		if (!colref.IsQualified()) {
-			auto alias_entry = info.alias_map.find(colref.column_names[0]);
-			if (alias_entry != info.alias_map.end()) {
-				// found entry!
-				return alias_entry->second;
-			}
-		}
-	}
-	// no alias reference found
-	// check the list of group columns for a match
-	auto entry = info.map.find(&expr);
-	if (entry != info.map.end()) {
-		return entry->second;
-	}
-#ifdef DEBUG
-	for (auto entry : info.map) {
-		D_ASSERT(!entry.first->Equals(&expr));
-		D_ASSERT(!expr.Equals(entry.first));
-	}
-#endif
-	return DConstants::INVALID_INDEX;
-}
-
-BindResult SelectBinder::BindColumnRef(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth) {
-	// first try to bind the column reference regularly
-	auto result = ExpressionBinder::BindExpression(expr_ptr, depth);
-	if (!result.HasError()) {
-		return result;
-	}
-	// binding failed
-	// check in the alias map
-	auto &colref = (ColumnRefExpression &)**expr_ptr;
-	if (!colref.IsQualified()) {
-		auto alias_entry = alias_map.find(colref.column_names[0]);
-		if (alias_entry != alias_map.end()) {
-			// found entry!
-			auto index = alias_entry->second;
-			if (index >= node.select_list.size()) {
-				throw BinderException("Column \"%s\" referenced that exists in the SELECT clause - but this column "
-				                      "cannot be referenced before it is defined",
-				                      colref.column_names[0]);
-			}
-			if (node.select_list[index]->HasSideEffects()) {
-				throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has side "
-				                      "effects. This is not yet supported.",
-				                      colref.column_names[0]);
-			}
-			if (node.select_list[index]->HasSubquery()) {
-				throw BinderException("Alias \"%s\" referenced in a SELECT clause - but the expression has a subquery."
-				                      " This is not yet supported.",
-				                      colref.column_names[0]);
-			}
-			auto result = BindResult(node.select_list[index]->Copy());
-			if (result.expression->type == ExpressionType::BOUND_COLUMN_REF) {
-				auto &result_expr = (BoundColumnRefExpression &)*result.expression;
-				result_expr.depth = depth;
-			}
-			return result;
-		}
-	}
-	// entry was not found in the alias map: return the original error
-	return result;
-}
-
-BindResult SelectBinder::BindGroupingFunction(OperatorExpression &op, idx_t depth) {
-	if (op.children.empty()) {
-		throw InternalException("GROUPING requires at least one child");
-	}
-	if (node.groups.group_expressions.empty()) {
-		return BindResult(binder.FormatError(op, "GROUPING statement cannot be used without groups"));
-	}
-	if (op.children.size() >= 64) {
-		return BindResult(binder.FormatError(op, "GROUPING statement cannot have more than 64 groups"));
-	}
-	vector<idx_t> group_indexes;
-	group_indexes.reserve(op.children.size());
-	for (auto &child : op.children) {
-		ExpressionBinder::QualifyColumnNames(binder, child);
-		auto idx = TryBindGroup(*child, depth);
-		if (idx == DConstants::INVALID_INDEX) {
-			return BindResult(binder.FormatError(
-			    op, StringUtil::Format("GROUPING child \"%s\" must be a grouping column", child->GetName())));
-		}
-		group_indexes.push_back(idx);
-	}
-	auto col_idx = node.grouping_functions.size();
-	node.grouping_functions.push_back(std::move(group_indexes));
-	return BindResult(make_unique<BoundColumnRefExpression>(op.GetName(), LogicalType::BIGINT,
-	                                                        ColumnBinding(node.groupings_index, col_idx), depth));
-}
-
-BindResult SelectBinder::BindGroup(ParsedExpression &expr, idx_t depth, idx_t group_index) {
-	auto &group = node.groups.group_expressions[group_index];
-	return BindResult(make_unique<BoundColumnRefExpression>(expr.GetName(), group->return_type,
-	                                                        ColumnBinding(node.group_index, group_index), depth));
-}
-
 } // namespace duckdb

package/src/duckdb/src/planner/expression_binder.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/planner/expression_binder.hpp"
 
 #include "duckdb/parser/expression/columnref_expression.hpp"
+#include "duckdb/parser/expression/function_expression.hpp"
 #include "duckdb/parser/expression/positional_reference_expression.hpp"
 #include "duckdb/parser/expression/subquery_expression.hpp"
 #include "duckdb/parser/parsed_expression_iterator.hpp"
@@ -52,9 +53,15 @@ BindResult ExpressionBinder::BindExpression(unique_ptr<ParsedExpression> *expr,
 		return BindExpression((ConjunctionExpression &)expr_ref, depth);
 	case ExpressionClass::CONSTANT:
 		return BindExpression((ConstantExpression &)expr_ref, depth);
-	case ExpressionClass::FUNCTION:
+	case ExpressionClass::FUNCTION: {
+		auto &function = (FunctionExpression &)expr_ref;
+		if (function.function_name == "unnest" || function.function_name == "unlist") {
+			// special case, not in catalog
+			return BindUnnest(function, depth, root_expression);
+		}
 		// binding function expression has extra parameter needed for macro's
-		return BindExpression((FunctionExpression &)expr_ref, depth, expr);
+		return BindExpression(function, depth, expr);
+	}
 	case ExpressionClass::LAMBDA:
 		return BindExpression((LambdaExpression &)expr_ref, depth, false, LogicalTypeId::INVALID);
 	case ExpressionClass::OPERATOR:
@@ -157,7 +164,7 @@ LogicalType ExpressionBinder::ExchangeType(const LogicalType &type, LogicalTypeI
 		for (auto &child_type : child_types) {
 			child_type.second = ExchangeType(child_type.second, target, new_type);
 		}
-		return LogicalType::STRUCT(std::move(child_types));
+		return LogicalType::STRUCT(child_types);
 	}
 	case LogicalTypeId::UNION: {
 		auto member_types = UnionType::CopyMemberTypes(type);

package/src/duckdb/src/planner/expression_iterator.cpp

@@ -29,6 +29,11 @@ void ExpressionIterator::EnumerateChildren(Expression &expr,
 		if (aggr_expr.filter) {
 			callback(aggr_expr.filter);
 		}
+		if (aggr_expr.order_bys) {
+			for (auto &order : aggr_expr.order_bys->orders) {
+				callback(order.expression);
+			}
+		}
 		break;
 	}
 	case ExpressionClass::BOUND_BETWEEN: {
@@ -195,22 +200,24 @@ void ExpressionIterator::EnumerateQueryNodeChildren(BoundQueryNode &node,
 	}
 	case QueryNodeType::SELECT_NODE: {
 		auto &bound_select = (BoundSelectNode &)node;
-		for (idx_t i = 0; i < bound_select.select_list.size(); i++) {
-			EnumerateExpression(bound_select.select_list[i], callback);
+		for (auto &expr : bound_select.select_list) {
+			EnumerateExpression(expr, callback);
 		}
 		EnumerateExpression(bound_select.where_clause, callback);
-		for (idx_t i = 0; i < bound_select.groups.group_expressions.size(); i++) {
-			EnumerateExpression(bound_select.groups.group_expressions[i], callback);
+		for (auto &expr : bound_select.groups.group_expressions) {
+			EnumerateExpression(expr, callback);
 		}
 		EnumerateExpression(bound_select.having, callback);
-		for (idx_t i = 0; i < bound_select.aggregates.size(); i++) {
-			EnumerateExpression(bound_select.aggregates[i], callback);
+		for (auto &expr : bound_select.aggregates) {
+			EnumerateExpression(expr, callback);
 		}
-		for (idx_t i = 0; i < bound_select.unnests.size(); i++) {
-			EnumerateExpression(bound_select.unnests[i], callback);
+		for (auto &entry : bound_select.unnests) {
+			for (auto &expr : entry.second.expressions) {
+				EnumerateExpression(expr, callback);
+			}
 		}
-		for (idx_t i = 0; i < bound_select.windows.size(); i++) {
-			EnumerateExpression(bound_select.windows[i], callback);
+		for (auto &expr : bound_select.windows) {
+			EnumerateExpression(expr, callback);
 		}
 		if (bound_select.from_table) {
 			EnumerateTableRefChildren(*bound_select.from_table, callback);

package/src/duckdb/src/planner/filter/constant_filter.cpp

@@ -1,8 +1,6 @@
 #include "duckdb/planner/filter/constant_filter.hpp"
-
+#include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/common/field_writer.hpp"
-#include "duckdb/storage/statistics/numeric_statistics.hpp"
-#include "duckdb/storage/statistics/string_statistics.hpp"
 
 namespace duckdb {
 
@@ -12,7 +10,7 @@ ConstantFilter::ConstantFilter(ExpressionType comparison_type_p, Value constant_
 }
 
 FilterPropagateResult ConstantFilter::CheckStatistics(BaseStatistics &stats) {
-	D_ASSERT(constant.type().id() == stats.type.id());
+	D_ASSERT(constant.type().id() == stats.GetType().id());
 	switch (constant.type().InternalType()) {
 	case PhysicalType::UINT8:
 	case PhysicalType::UINT16:
@@ -25,9 +23,9 @@ FilterPropagateResult ConstantFilter::CheckStatistics(BaseStatistics &stats) {
 	case PhysicalType::INT128:
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
-		return ((NumericStatistics &)stats).CheckZonemap(comparison_type, constant);
+		return NumericStats::CheckZonemap(stats, comparison_type, constant);
 	case PhysicalType::VARCHAR:
-		return ((StringStatistics &)stats).CheckZonemap(comparison_type, StringValue::Get(constant));
+		return StringStats::CheckZonemap(stats, comparison_type, StringValue::Get(constant));
 	default:
 		return FilterPropagateResult::NO_PRUNING_POSSIBLE;
 	}

package/src/duckdb/src/planner/logical_operator.cpp

@@ -128,6 +128,8 @@ void LogicalOperator::Verify(ClientContext &context) {
 			continue;
 		}
 		BufferedSerializer serializer;
+		// We are serializing a query plan
+		serializer.is_query_plan = true;
 		try {
 			expressions[expr_idx]->Serialize(serializer);
 		} catch (NotImplementedException &ex) {
@@ -136,7 +138,7 @@ void LogicalOperator::Verify(ClientContext &context) {
 		}
 
 		auto data = serializer.GetData();
-		auto deserializer = BufferedDeserializer(data.data.get(), data.size);
+		auto deserializer = BufferedContextDeserializer(context, data.data.get(), data.size);
 
 		PlanDeserializationState state(context);
 		auto deserialized_expression = Expression::Deserialize(deserializer, state);
@@ -255,6 +257,9 @@ unique_ptr<LogicalOperator> LogicalOperator::Deserialize(Deserializer &deseriali
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 		result = LogicalDelimJoin::Deserialize(state, reader);
 		break;
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
+		result = LogicalAsOfJoin::Deserialize(state, reader);
+		break;
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		result = LogicalComparisonJoin::Deserialize(state, reader);
 		break;
@@ -371,7 +376,7 @@ unique_ptr<LogicalOperator> LogicalOperator::Copy(ClientContext &context) const
 		                              std::string(ex.what()));
 	}
 	auto data = logical_op_serializer.GetData();
-	auto logical_op_deserializer = BufferedDeserializer(data.data.get(), data.size);
+	auto logical_op_deserializer = BufferedContextDeserializer(context, data.data.get(), data.size);
 	PlanDeserializationState state(context);
 	auto op_copy = LogicalOperator::Deserialize(logical_op_deserializer, state);
 	return op_copy;

package/src/duckdb/src/planner/logical_operator_visitor.cpp

@@ -48,6 +48,11 @@ void LogicalOperatorVisitor::EnumerateExpressions(LogicalOperator &op,
 		for (auto &target : distinct.distinct_targets) {
 			callback(&target);
 		}
+		if (distinct.order_by) {
+			for (auto &order : distinct.order_by->orders) {
+				callback(&order.expression);
+			}
+		}
 		break;
 	}
 	case LogicalOperatorType::LOGICAL_INSERT: {
@@ -60,6 +65,7 @@ void LogicalOperatorVisitor::EnumerateExpressions(LogicalOperator &op,
 		}
 		break;
 	}
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {

package/src/duckdb/src/planner/operator/logical_asof_join.cpp

@@ -0,0 +1,8 @@
+#include "duckdb/planner/operator/logical_asof_join.hpp"
+
+namespace duckdb {
+
+LogicalAsOfJoin::LogicalAsOfJoin(JoinType type) : LogicalComparisonJoin(type, LogicalOperatorType::LOGICAL_ASOF_JOIN) {
+}
+
+} // namespace duckdb

package/src/duckdb/src/planner/operator/logical_distinct.cpp

@@ -15,6 +15,9 @@ string LogicalDistinct::ParamsToString() const {
 }
 void LogicalDistinct::Serialize(FieldWriter &writer) const {
 	writer.WriteSerializableList(distinct_targets);
+	if (order_by) {
+		throw NotImplementedException("Serializing ORDER BY not yet supported");
+	}
 }
 
 unique_ptr<LogicalOperator> LogicalDistinct::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {

package/src/duckdb/src/planner/planner.cpp

@@ -184,6 +184,7 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 	}
 
 	BufferedSerializer serializer;
+	serializer.is_query_plan = true;
 	try {
 		op->Serialize(serializer);
 	} catch (NotImplementedException &ex) {
@@ -191,7 +192,7 @@ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op
 		return;
 	}
 	auto data = serializer.GetData();
-	auto deserializer = BufferedDeserializer(data.data.get(), data.size);
+	auto deserializer = BufferedContextDeserializer(context, data.data.get(), data.size);
 
 	PlanDeserializationState state(context);
 	auto new_plan = LogicalOperator::Deserialize(deserializer, state);

package/src/duckdb/src/planner/pragma_handler.cpp

@@ -4,7 +4,7 @@
 
 #include "duckdb/catalog/catalog.hpp"
 #include "duckdb/catalog/catalog_entry/pragma_function_catalog_entry.hpp"
-
+#include "duckdb/parser/statement/multi_statement.hpp"
 #include "duckdb/parser/parsed_data/pragma_info.hpp"
 #include "duckdb/function/function.hpp"
 
@@ -22,6 +22,13 @@ PragmaHandler::PragmaHandler(ClientContext &context) : context(context) {
 void PragmaHandler::HandlePragmaStatementsInternal(vector<unique_ptr<SQLStatement>> &statements) {
 	vector<unique_ptr<SQLStatement>> new_statements;
 	for (idx_t i = 0; i < statements.size(); i++) {
+		if (statements[i]->type == StatementType::MULTI_STATEMENT) {
+			auto &multi_statement = (MultiStatement &)*statements[i];
+			for (auto &stmt : multi_statement.statements) {
+				statements.push_back(std::move(stmt));
+			}
+			continue;
+		}
 		if (statements[i]->type == StatementType::PRAGMA_STATEMENT) {
 			// PRAGMA statement: check if we need to replace it by a new set of statements
 			PragmaHandler handler(context);
@@ -47,7 +54,8 @@ void PragmaHandler::HandlePragmaStatements(ClientContextLock &lock, vector<uniqu
 	// first check if there are any pragma statements
 	bool found_pragma = false;
 	for (idx_t i = 0; i < statements.size(); i++) {
-		if (statements[i]->type == StatementType::PRAGMA_STATEMENT) {
+		if (statements[i]->type == StatementType::PRAGMA_STATEMENT ||
+		    statements[i]->type == StatementType::MULTI_STATEMENT) {
 			found_pragma = true;
 			break;
 		}

package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp

@@ -262,6 +262,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		return std::move(join);
 	}
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
+	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		auto &join = (LogicalJoin &)*plan;
 		D_ASSERT(plan->children.size() == 2);
@@ -334,7 +335,8 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 			auto right = make_unique<BoundColumnRefExpression>(
 			    correlated_columns[i].type, ColumnBinding(right_binding.table_index, right_binding.column_index + i));
 
-			if (join.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+			if (join.type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
+			    join.type == LogicalOperatorType::LOGICAL_ASOF_JOIN) {
 				JoinCondition cond;
 				cond.left = std::move(left);
 				cond.right = std::move(right);