duckdb 0.10.1 → 0.10.2-dev3.0

package/src/duckdb/src/optimizer/statistics/expression/propagate_operator.cpp

@@ -5,7 +5,7 @@
 namespace duckdb {
 
 unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(BoundOperatorExpression &expr,
-                                                                     unique_ptr<Expression> *expr_ptr) {
+                                                                     unique_ptr<Expression> &expr_ptr) {
 	bool all_have_stats = true;
 	vector<unique_ptr<BaseStatistics>> child_stats;
 	child_stats.reserve(expr.children.size());
@@ -47,7 +47,7 @@ unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(BoundOperat
 		D_ASSERT(expr.children.size() == child_stats.size());
 		if (expr.children.size() == 1) {
 			// coalesce of one entry: simply return that entry
-			*expr_ptr = std::move(expr.children[0]);
+			expr_ptr = std::move(expr.children[0]);
 		} else {
 			// coalesce of multiple entries
 			// merge the stats
@@ -59,25 +59,25 @@ unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(BoundOperat
 	case ExpressionType::OPERATOR_IS_NULL:
 		if (!child_stats[0]->CanHaveNull()) {
 			// child has no null values: x IS NULL will always be false
-			*expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(false));
-			return PropagateExpression(*expr_ptr);
+			expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(false));
+			return PropagateExpression(expr_ptr);
 		}
 		if (!child_stats[0]->CanHaveNoNull()) {
 			// child has no valid values: x IS NULL will always be true
-			*expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(true));
-			return PropagateExpression(*expr_ptr);
+			expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(true));
+			return PropagateExpression(expr_ptr);
 		}
 		return nullptr;
 	case ExpressionType::OPERATOR_IS_NOT_NULL:
 		if (!child_stats[0]->CanHaveNull()) {
 			// child has no null values: x IS NOT NULL will always be true
-			*expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(true));
-			return PropagateExpression(*expr_ptr);
+			expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(true));
+			return PropagateExpression(expr_ptr);
 		}
 		if (!child_stats[0]->CanHaveNoNull()) {
 			// child has no valid values: x IS NOT NULL will always be false
-			*expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(false));
-			return PropagateExpression(*expr_ptr);
+			expr_ptr = make_uniq<BoundConstantExpression>(Value::BOOLEAN(false));
+			return PropagateExpression(expr_ptr);
 		}
 		return nullptr;
 	default:

package/src/duckdb/src/optimizer/statistics/operator/propagate_aggregate.cpp

@@ -4,7 +4,7 @@
 namespace duckdb {
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalAggregate &aggr,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate statistics in the child node
 	node_stats = PropagateStatistics(aggr.children[0]);
 

package/src/duckdb/src/optimizer/statistics/operator/propagate_cross_product.cpp

@@ -4,7 +4,7 @@
 namespace duckdb {
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalCrossProduct &cp,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate statistics in the child node
 	auto left_stats = PropagateStatistics(cp.children[0]);
 	auto right_stats = PropagateStatistics(cp.children[1]);

package/src/duckdb/src/optimizer/statistics/operator/propagate_filter.cpp

@@ -1,3 +1,4 @@
+#include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/function/scalar/generic_functions.hpp"
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/expression/bound_between_expression.hpp"
@@ -15,12 +16,18 @@ static bool IsCompareDistinct(ExpressionType type) {
 }
 
 bool StatisticsPropagator::ExpressionIsConstant(Expression &expr, const Value &val) {
-	if (expr.GetExpressionClass() != ExpressionClass::BOUND_CONSTANT) {
+	Value expr_value;
+	if (expr.GetExpressionClass() == ExpressionClass::BOUND_CONSTANT) {
+		expr_value = expr.Cast<BoundConstantExpression>().value;
+	} else if (expr.IsFoldable()) {
+		if (!ExpressionExecutor::TryEvaluateScalar(context, expr, expr_value)) {
+			return false;
+		}
+	} else {
 		return false;
 	}
-	auto &bound_constant = expr.Cast<BoundConstantExpression>();
-	D_ASSERT(bound_constant.value.type() == val.type());
-	return Value::NotDistinctFrom(bound_constant.value, val);
+	D_ASSERT(expr_value.type() == val.type());
+	return Value::NotDistinctFrom(expr_value, val);
 }
 
 bool StatisticsPropagator::ExpressionIsConstantOrNull(Expression &expr, const Value &val) {
@@ -215,11 +222,11 @@ void StatisticsPropagator::UpdateFilterStatistics(Expression &condition) {
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalFilter &filter,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate to the child
 	node_stats = PropagateStatistics(filter.children[0]);
 	if (filter.children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) {
-		ReplaceWithEmptyResult(*node_ptr);
+		ReplaceWithEmptyResult(node_ptr);
 		return make_uniq<NodeStatistics>(0, 0);
 	}
 
@@ -240,7 +247,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalFilt
 		} else if (ExpressionIsConstant(*condition, Value::BOOLEAN(false)) ||
 		           ExpressionIsConstantOrNull(*condition, Value::BOOLEAN(false))) {
 			// filter is always false or null; this entire filter should be replaced by an empty result block
-			ReplaceWithEmptyResult(*node_ptr);
+			ReplaceWithEmptyResult(node_ptr);
 			return make_uniq<NodeStatistics>(0, 0);
 		} else {
 			// cannot prune this filter: propagate statistics from the filter

package/src/duckdb/src/optimizer/statistics/operator/propagate_get.cpp

@@ -31,7 +31,7 @@ void StatisticsPropagator::UpdateFilterStatistics(BaseStatistics &input, TableFi
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalGet &get,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	if (get.function.cardinality) {
 		node_stats = get.function.cardinality(context, get.bind_data.get());
 	}
@@ -85,7 +85,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalGet
 		case FilterPropagateResult::FILTER_FALSE_OR_NULL:
 		case FilterPropagateResult::FILTER_ALWAYS_FALSE:
 			// filter is always false; this entire filter should be replaced by an empty result block
-			ReplaceWithEmptyResult(*node_ptr);
+			ReplaceWithEmptyResult(node_ptr);
 			return make_uniq<NodeStatistics>(0, 0);
 		default:
 			// general case: filter can be true or false, update this columns' statistics

package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp

@@ -14,7 +14,7 @@
 
 namespace duckdb {
 
-void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, unique_ptr<LogicalOperator> *node_ptr) {
+void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, unique_ptr<LogicalOperator> &node_ptr) {
 	for (idx_t i = 0; i < join.conditions.size(); i++) {
 		auto &condition = join.conditions[i];
 		const auto stats_left = PropagateExpression(condition.left);
@@ -40,7 +40,7 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 				case JoinType::SEMI:
 				case JoinType::INNER:
 					// semi or inner join on false; entire node can be pruned
-					ReplaceWithEmptyResult(*node_ptr);
+					ReplaceWithEmptyResult(node_ptr);
 					return;
 				case JoinType::RIGHT_ANTI:
 				case JoinType::ANTI: {
@@ -48,7 +48,7 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 						std::swap(join.children[0], join.children[1]);
 					}
 					// If the filter is always false or Null, just return the left child.
-					*node_ptr = std::move(join.children[0]);
+					node_ptr = std::move(join.children[0]);
 					return;
 				}
 				case JoinType::LEFT:
@@ -106,19 +106,19 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 						auto limit = make_uniq<LogicalLimit>(BoundLimitNode::ConstantValue(1), BoundLimitNode());
 						limit->AddChild(std::move(join.children[1]));
 						auto cross_product = LogicalCrossProduct::Create(std::move(join.children[0]), std::move(limit));
-						*node_ptr = std::move(cross_product);
+						node_ptr = std::move(cross_product);
 						return;
 					}
 					case JoinType::INNER: {
 						// inner, replace with cross product
 						auto cross_product =
 						    LogicalCrossProduct::Create(std::move(join.children[0]), std::move(join.children[1]));
-						*node_ptr = std::move(cross_product);
+						node_ptr = std::move(cross_product);
 						return;
 					}
 					case JoinType::ANTI:
 					case JoinType::RIGHT_ANTI: {
-						ReplaceWithEmptyResult(*node_ptr);
+						ReplaceWithEmptyResult(node_ptr);
 						return;
 					}
 					default:
@@ -175,7 +175,7 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 	}
 }
 
-void StatisticsPropagator::PropagateStatistics(LogicalAnyJoin &join, unique_ptr<LogicalOperator> *node_ptr) {
+void StatisticsPropagator::PropagateStatistics(LogicalAnyJoin &join, unique_ptr<LogicalOperator> &node_ptr) {
 	// propagate the expression into the join condition
 	PropagateExpression(join.condition);
 }
@@ -201,7 +201,7 @@ void StatisticsPropagator::MultiplyCardinalities(unique_ptr<NodeStatistics> &sta
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalJoin &join,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate through the children of the join
 	node_stats = PropagateStatistics(join.children[0]);
 	for (idx_t child_idx = 1; child_idx < join.children.size(); child_idx++) {
@@ -274,7 +274,7 @@ static void MaxCardinalities(unique_ptr<NodeStatistics> &stats, NodeStatistics &
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalPositionalJoin &join,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	D_ASSERT(join.type == LogicalOperatorType::LOGICAL_POSITIONAL_JOIN);
 
 	// first propagate through the children of the join

package/src/duckdb/src/optimizer/statistics/operator/propagate_limit.cpp

@@ -4,7 +4,7 @@
 namespace duckdb {
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalLimit &limit,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// propagate statistics in the child node
 	PropagateStatistics(limit.children[0]);
 	// return the node stats, with as expected cardinality the amount specified in the limit

package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp

@@ -5,7 +5,7 @@
 namespace duckdb {
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalOrder &order,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate to the child
 	node_stats = PropagateStatistics(order.children[0]);
 

package/src/duckdb/src/optimizer/statistics/operator/propagate_projection.cpp

@@ -4,11 +4,11 @@
 namespace duckdb {
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalProjection &proj,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate to the child
 	node_stats = PropagateStatistics(proj.children[0]);
 	if (proj.children[0]->type == LogicalOperatorType::LOGICAL_EMPTY_RESULT) {
-		ReplaceWithEmptyResult(*node_ptr);
+		ReplaceWithEmptyResult(node_ptr);
 		return std::move(node_stats);
 	}
 	// then propagate to each of the expressions

package/src/duckdb/src/optimizer/statistics/operator/propagate_set_operation.cpp

@@ -24,7 +24,7 @@ void StatisticsPropagator::AddCardinalities(unique_ptr<NodeStatistics> &stats, N
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalSetOperation &setop,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate statistics in the child nodes
 	auto left_stats = PropagateStatistics(setop.children[0]);
 	auto right_stats = PropagateStatistics(setop.children[1]);

package/src/duckdb/src/optimizer/statistics/operator/propagate_window.cpp

@@ -5,7 +5,7 @@
 namespace duckdb {
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalWindow &window,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
 	// first propagate to the child
 	node_stats = PropagateStatistics(window.children[0]);
 

package/src/duckdb/src/optimizer/statistics_propagator.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/optimizer/statistics_propagator.hpp"
 
 #include "duckdb/main/client_context.hpp"
+#include "duckdb/optimizer/compressed_materialization.hpp"
 #include "duckdb/optimizer/optimizer.hpp"
 #include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
@@ -19,8 +20,9 @@
 
 namespace duckdb {
 
-StatisticsPropagator::StatisticsPropagator(Optimizer &optimizer_p)
-    : optimizer(optimizer_p), context(optimizer.context) {
+StatisticsPropagator::StatisticsPropagator(Optimizer &optimizer_p, LogicalOperator &root_p)
+    : optimizer(optimizer_p), context(optimizer.context), root(&root_p) {
+	root->ResolveOperatorTypes();
 }
 
 void StatisticsPropagator::ReplaceWithEmptyResult(unique_ptr<LogicalOperator> &node) {
@@ -28,7 +30,7 @@ void StatisticsPropagator::ReplaceWithEmptyResult(unique_ptr<LogicalOperator> &n
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateChildren(LogicalOperator &node,
-                                                                   unique_ptr<LogicalOperator> *node_ptr) {
+                                                                   unique_ptr<LogicalOperator> &node_ptr) {
 	for (idx_t child_idx = 0; child_idx < node.children.size(); child_idx++) {
 		PropagateStatistics(node.children[child_idx]);
 	}
@@ -36,45 +38,64 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateChildren(LogicalOperat
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalOperator &node,
-                                                                     unique_ptr<LogicalOperator> *node_ptr) {
+                                                                     unique_ptr<LogicalOperator> &node_ptr) {
+	unique_ptr<NodeStatistics> result;
 	switch (node.type) {
 	case LogicalOperatorType::LOGICAL_AGGREGATE_AND_GROUP_BY:
-		return PropagateStatistics(node.Cast<LogicalAggregate>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalAggregate>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_CROSS_PRODUCT:
-		return PropagateStatistics(node.Cast<LogicalCrossProduct>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalCrossProduct>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_FILTER:
-		return PropagateStatistics(node.Cast<LogicalFilter>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalFilter>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_GET:
-		return PropagateStatistics(node.Cast<LogicalGet>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalGet>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_PROJECTION:
-		return PropagateStatistics(node.Cast<LogicalProjection>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalProjection>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_ANY_JOIN:
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 	case LogicalOperatorType::LOGICAL_JOIN:
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
-		return PropagateStatistics(node.Cast<LogicalJoin>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalJoin>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_POSITIONAL_JOIN:
-		return PropagateStatistics(node.Cast<LogicalPositionalJoin>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalPositionalJoin>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_UNION:
 	case LogicalOperatorType::LOGICAL_EXCEPT:
 	case LogicalOperatorType::LOGICAL_INTERSECT:
-		return PropagateStatistics(node.Cast<LogicalSetOperation>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalSetOperation>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_ORDER_BY:
-		return PropagateStatistics(node.Cast<LogicalOrder>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalOrder>(), node_ptr);
+		break;
 	case LogicalOperatorType::LOGICAL_WINDOW:
-		return PropagateStatistics(node.Cast<LogicalWindow>(), node_ptr);
+		result = PropagateStatistics(node.Cast<LogicalWindow>(), node_ptr);
+		break;
 	default:
-		return PropagateChildren(node, node_ptr);
+		result = PropagateChildren(node, node_ptr);
 	}
+
+	if (!optimizer.OptimizerDisabled(OptimizerType::COMPRESSED_MATERIALIZATION)) {
+		// compress data based on statistics for materializing operators
+		CompressedMaterialization compressed_materialization(optimizer, *root, statistics_map);
+		compressed_materialization.Compress(node_ptr);
+	}
+
+	return result;
 }
 
 unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(unique_ptr<LogicalOperator> &node_ptr) {
-	return PropagateStatistics(*node_ptr, &node_ptr);
+	return PropagateStatistics(*node_ptr, node_ptr);
 }
 
 unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(Expression &expr,
-                                                                     unique_ptr<Expression> *expr_ptr) {
+                                                                     unique_ptr<Expression> &expr_ptr) {
 	switch (expr.GetExpressionClass()) {
 	case ExpressionClass::BOUND_AGGREGATE:
 		return PropagateExpression(expr.Cast<BoundAggregateExpression>(), expr_ptr);
@@ -104,7 +125,7 @@ unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(Expression
 }
 
 unique_ptr<BaseStatistics> StatisticsPropagator::PropagateExpression(unique_ptr<Expression> &expr) {
-	auto stats = PropagateExpression(*expr, &expr);
+	auto stats = PropagateExpression(*expr, expr);
 	if (ClientConfig::GetConfig(context).query_verification_enabled && stats) {
 		expr->verification_stats = stats->ToUnique();
 	}

package/src/duckdb/src/parallel/pipeline_finish_event.cpp

@@ -51,7 +51,7 @@ private:
 	//! Debugging state: number of times blocked
 	int debug_blocked_count = 0;
 	//! Number of times the Finalize will block before actually returning data
-	int debug_blocked_target_count = 10;
+	int debug_blocked_target_count = 1;
 #endif
 };
 

package/src/duckdb/src/parallel/task_scheduler.cpp

@@ -304,7 +304,14 @@ void TaskScheduler::RelaunchThreadsInternal(int32_t n) {
 		for (idx_t i = 0; i < create_new_threads; i++) {
 			// launch a thread and assign it a cancellation marker
 			auto marker = unique_ptr<atomic<bool>>(new atomic<bool>(true));
-			auto worker_thread = make_uniq<thread>(ThreadExecuteTasks, this, marker.get());
+			unique_ptr<thread> worker_thread;
+			try {
+				worker_thread = make_uniq<thread>(ThreadExecuteTasks, this, marker.get());
+			} catch (std::exception &ex) {
+				// thread constructor failed - this can happen when the system has too many threads allocated
+				// in this case we cannot allocate more threads - stop launching them
+				break;
+			}
 			auto thread_wrapper = make_uniq<SchedulerThread>(std::move(worker_thread));
 
 			threads.push_back(std::move(thread_wrapper));

package/src/duckdb/src/parser/constraints/unique_constraint.cpp

@@ -28,11 +28,13 @@ string UniqueConstraint::ToString() const {
 }
 
 unique_ptr<Constraint> UniqueConstraint::Copy() const {
-	if (index.index == DConstants::INVALID_INDEX) {
+	if (!HasIndex()) {
 		return make_uniq<UniqueConstraint>(columns, is_primary_key);
 	} else {
 		auto result = make_uniq<UniqueConstraint>(index, is_primary_key);
-		result->columns = columns;
+		if (!columns.empty()) {
+			result->columns.push_back(columns[0]);
+		}
 		return std::move(result);
 	}
 }

package/src/duckdb/src/parser/parsed_data/create_index_info.cpp

@@ -1,4 +1,6 @@
 #include "duckdb/parser/parsed_data/create_index_info.hpp"
+#include "duckdb/parser/parsed_expression_iterator.hpp"
+#include "duckdb/parser/expression/columnref_expression.hpp"
 
 namespace duckdb {
 
@@ -11,6 +13,68 @@ CreateIndexInfo::CreateIndexInfo(const duckdb::CreateIndexInfo &info)
       scan_types(info.scan_types), names(info.names) {
 }
 
+static void RemoveTableQualificationRecursive(unique_ptr<ParsedExpression> &expr, const string &table_name) {
+	if (expr->GetExpressionType() == ExpressionType::COLUMN_REF) {
+		auto &col_ref = expr->Cast<ColumnRefExpression>();
+		auto &col_names = col_ref.column_names;
+		if (col_ref.IsQualified() && col_ref.GetTableName() == table_name) {
+			col_names.erase(col_names.begin());
+		}
+	} else {
+		ParsedExpressionIterator::EnumerateChildren(*expr, [&table_name](unique_ptr<ParsedExpression> &child) {
+			RemoveTableQualificationRecursive(child, table_name);
+		});
+	}
+}
+
+string CreateIndexInfo::ToString() const {
+	string result;
+
+	result += "CREATE";
+	D_ASSERT(constraint_type == IndexConstraintType::UNIQUE || constraint_type == IndexConstraintType::NONE);
+	if (constraint_type == IndexConstraintType::UNIQUE) {
+		result += " UNIQUE";
+	}
+	result += " INDEX ";
+	if (on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) {
+		result += "IF NOT EXISTS ";
+	}
+	result += KeywordHelper::WriteOptionallyQuoted(index_name);
+	result += " ON ";
+	result += KeywordHelper::WriteOptionallyQuoted(table);
+	if (index_type != "ART") {
+		result += " USING ";
+		result += KeywordHelper::WriteOptionallyQuoted(index_type);
+		result += " ";
+	}
+	result += "(";
+	for (idx_t i = 0; i < parsed_expressions.size(); i++) {
+		auto &expr = parsed_expressions[i];
+		auto copy = expr->Copy();
+		if (i > 0) {
+			result += ", ";
+		}
+		// column ref expressions are qualified with the table name
+		// we need to remove them to reproduce the original query
+		RemoveTableQualificationRecursive(copy, table);
+		result += copy->ToString();
+	}
+	result += ")";
+	if (!options.empty()) {
+		result += " WITH (";
+		idx_t i = 0;
+		for (auto &opt : options) {
+			result += StringUtil::Format("%s = %s", opt.first, opt.second.ToString());
+			if (i > 0) {
+				result += ", ";
+			}
+			i++;
+		}
+		result += " )";
+	}
+	return result;
+}
+
 unique_ptr<CreateInfo> CreateIndexInfo::Copy() const {
 
 	auto result = make_uniq<CreateIndexInfo>(*this);

package/src/duckdb/src/parser/parsed_data/vacuum_info.cpp

@@ -11,6 +11,7 @@ unique_ptr<VacuumInfo> VacuumInfo::Copy() {
 	if (has_table) {
 		result->ref = ref->Copy();
 	}
+	result->columns = columns;
 	return result;
 }
 

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -163,12 +163,13 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			throw ParserException("EXPORT_STATE is not supported for window functions!");
 		}
 
-		if (win_fun_type == ExpressionType::WINDOW_AGGREGATE && root.agg_ignore_nulls) {
-			throw ParserException("IGNORE NULLS is not supported for windowed aggregates");
+		if (win_fun_type == ExpressionType::WINDOW_AGGREGATE &&
+		    root.agg_ignore_nulls != duckdb_libpgquery::PG_DEFAULT_NULLS) {
+			throw ParserException("RESPECT/IGNORE NULLS is not supported for windowed aggregates");
 		}
 
 		auto expr = make_uniq<WindowExpression>(win_fun_type, std::move(catalog), std::move(schema), lowercase_name);
-		expr->ignore_nulls = root.agg_ignore_nulls;
+		expr->ignore_nulls = (root.agg_ignore_nulls == duckdb_libpgquery::PG_IGNORE_NULLS);
 		expr->distinct = root.agg_distinct;
 
 		if (root.agg_filter) {
@@ -238,8 +239,8 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		return std::move(expr);
 	}
 
-	if (root.agg_ignore_nulls) {
-		throw ParserException("IGNORE NULLS is not supported for non-window functions");
+	if (root.agg_ignore_nulls != duckdb_libpgquery::PG_DEFAULT_NULLS) {
+		throw ParserException("RESPECT/IGNORE NULLS is not supported for non-window functions");
 	}
 
 	unique_ptr<ParsedExpression> filter_expr;
@@ -298,6 +299,13 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		auto construct_array = make_uniq<OperatorExpression>(ExpressionType::ARRAY_CONSTRUCTOR);
 		construct_array->children = std::move(children);
 		return std::move(construct_array);
+	} else if (lowercase_name == "__internal_position_operator") {
+		if (children.size() != 2) {
+			throw ParserException("Wrong number of arguments to __internal_position_operator.");
+		}
+		// swap arguments for POSITION(x IN y)
+		std::swap(children[0], children[1]);
+		lowercase_name = "position";
 	} else if (lowercase_name == "ifnull") {
 		if (children.size() != 2) {
 			throw ParserException("Wrong number of arguments to IFNULL.");

package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp

@@ -2,6 +2,8 @@
 #include "duckdb/parser/transformer.hpp"
 #include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/tableref/subqueryref.hpp"
+#include "duckdb/parser/expression/constant_expression.hpp"
+#include "duckdb/parser/expression/positional_reference_expression.hpp"
 
 namespace duckdb {
 
@@ -69,6 +71,28 @@ unique_ptr<ParsedExpression> Transformer::TransformSubquery(duckdb_libpgquery::P
 		vector<unique_ptr<ParsedExpression>> children;
 		children.push_back(std::move(columns_star));
 		auto aggr = make_uniq<FunctionExpression>("array_agg", std::move(children));
+		// push ORDER BY modifiers into the array_agg
+		for (auto &modifier : subquery_expr->subquery->node->modifiers) {
+			if (modifier->type == ResultModifierType::ORDER_MODIFIER) {
+				aggr->order_bys = unique_ptr_cast<ResultModifier, OrderModifier>(modifier->Copy());
+				break;
+			}
+		}
+		// transform constants (e.g. ORDER BY 1) into positional references (ORDER BY #1)
+		if (aggr->order_bys) {
+			for (auto &order : aggr->order_bys->orders) {
+				if (order.expression->type == ExpressionType::VALUE_CONSTANT) {
+					auto &constant_expr = order.expression->Cast<ConstantExpression>();
+					Value bigint_value;
+					string error;
+					if (constant_expr.value.DefaultTryCastAs(LogicalType::BIGINT, bigint_value, &error)) {
+						int64_t order_index = BigIntValue::Get(bigint_value);
+						idx_t positional_index = order_index < 0 ? NumericLimits<idx_t>::Maximum() : idx_t(order_index);
+						order.expression = make_uniq<PositionalReferenceExpression>(positional_index);
+					}
+				}
+			}
+		}
 		// ARRAY_AGG(COLUMNS(*)) IS NULL
 		auto agg_is_null = make_uniq<OperatorExpression>(ExpressionType::OPERATOR_IS_NULL, aggr->Copy());
 		// empty list

package/src/duckdb/src/parser/transform/helpers/transform_cte.cpp

@@ -31,8 +31,7 @@ void Transformer::ExtractCTEsRecursive(CommonTableExpressionMap &cte_map) {
 	}
 }
 
-void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause, CommonTableExpressionMap &cte_map,
-                               vector<unique_ptr<CTENode>> &materialized_ctes) {
+void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause, CommonTableExpressionMap &cte_map) {
 	stored_cte_map.push_back(&cte_map);
 
 	// TODO: might need to update in case of future lawsuit
@@ -88,12 +87,6 @@ void Transformer::TransformCTE(duckdb_libpgquery::PGWithClause &de_with_clause,
 #else
 		if (cte.ctematerialized == duckdb_libpgquery::PGCTEMaterializeAlways) {
 #endif
-			auto materialize = make_uniq<CTENode>();
-			materialize->query = info->query->node->Copy();
-			materialize->ctename = cte_name;
-			materialize->aliases = info->aliases;
-			materialized_ctes.push_back(std::move(materialize));
-
 			info->materialized = CTEMaterialize::CTE_MATERIALIZE_ALWAYS;
 		}
 

package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp

@@ -40,10 +40,11 @@ unique_ptr<CreateStatement> Transformer::TransformCreateFunction(duckdb_libpgque
 		break;
 	case duckdb_libpgquery::PG_RELPERSISTENCE_UNLOGGED:
 		throw ParserException("Unlogged flag not supported for macros: '%s'", qname.name);
-		break;
 	case duckdb_libpgquery::RELPERSISTENCE_PERMANENT:
 		info->temporary = false;
 		break;
+	default:
+		throw ParserException("Unsupported persistence flag for table '%s'", qname.name);
 	}
 
 	// what to do on conflict

package/src/duckdb/src/parser/transform/statement/transform_create_index.cpp

@@ -77,6 +77,9 @@ unique_ptr<CreateStatement> Transformer::TransformCreateIndex(duckdb_libpgquery:
 	for (auto &expr : info->expressions) {
 		info->parsed_expressions.emplace_back(expr->Copy());
 	}
+	if (stmt.whereClause) {
+		throw NotImplementedException("Creating partial indexes is not supported currently");
+	}
 	result->info = std::move(info);
 	return result;
 }