duckdb 0.8.2-dev1.0 → 0.8.2-dev1182.0

package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp

@@ -3,6 +3,8 @@
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
+#include "duckdb/function/scalar/string_functions.hpp"
+#include "duckdb/function/scalar/regexp.hpp"
 
 #include "re2/re2.h"
 #include "re2/regexp.h"
@@ -12,17 +14,138 @@ namespace duckdb {
 RegexOptimizationRule::RegexOptimizationRule(ExpressionRewriter &rewriter) : Rule(rewriter) {
 	auto func = make_uniq<FunctionExpressionMatcher>();
 	func->function = make_uniq<SpecificFunctionMatcher>("regexp_matches");
-	func->policy = SetMatcher::Policy::ORDERED;
+	func->policy = SetMatcher::Policy::SOME_ORDERED;
 	func->matchers.push_back(make_uniq<ExpressionMatcher>());
 	func->matchers.push_back(make_uniq<ConstantExpressionMatcher>());
+
 	root = std::move(func);
 }
 
+struct LikeString {
+	bool exists = true;
+	bool escaped = false;
+	string like_string = "";
+};
+
+static void AddCharacter(char chr, LikeString &ret, bool contains) {
+	// if we are not converting into a contains, and the string has LIKE special characters
+	// then don't return a possible LIKE match
+	// same if the character is a control character
+	if (iscntrl(chr) || (!contains && (chr == '%' || chr == '_'))) {
+		ret.exists = false;
+		return;
+	}
+	auto run_as_str {chr};
+	ret.like_string += run_as_str;
+}
+
+static LikeString GetLikeStringEscaped(duckdb_re2::Regexp *regexp, bool contains = false) {
+	D_ASSERT(regexp->op() == duckdb_re2::kRegexpLiteralString || regexp->op() == duckdb_re2::kRegexpLiteral);
+	LikeString ret;
+
+	if (regexp->parse_flags() & duckdb_re2::Regexp::FoldCase ||
+	    !(regexp->parse_flags() & duckdb_re2::Regexp::OneLine)) {
+		// parse flags can turn on and off within a regex match, return no optimization
+		// For now, we just don't optimize if these every turn on.
+		// TODO: logic to attempt the optimization, then if the parse flags change, then abort
+		ret.exists = false;
+		return ret;
+	}
+
+	// case insensitivity may be on now, but it can also turn off.
+	if (regexp->op() == duckdb_re2::kRegexpLiteralString) {
+		auto nrunes = (idx_t)regexp->nrunes();
+		auto runes = regexp->runes();
+		for (idx_t i = 0; i < nrunes; i++) {
+			char chr = toascii(runes[i]);
+			AddCharacter(chr, ret, contains);
+			if (!ret.exists) {
+				return ret;
+			}
+		}
+	} else {
+		auto rune = regexp->rune();
+		char chr = toascii(rune);
+		AddCharacter(chr, ret, contains);
+	}
+	D_ASSERT(ret.like_string.size() >= 1 || !ret.exists);
+	return ret;
+}
+
+static LikeString LikeMatchFromRegex(duckdb_re2::RE2 &pattern) {
+	LikeString ret = LikeString();
+	auto num_subs = pattern.Regexp()->nsub();
+	auto subs = pattern.Regexp()->sub();
+	auto cur_sub_index = 0;
+	while (cur_sub_index < num_subs) {
+		switch (subs[cur_sub_index]->op()) {
+		case duckdb_re2::kRegexpAnyChar:
+			if (cur_sub_index == 0) {
+				ret.like_string += "%";
+			}
+			ret.like_string += "_";
+			if (cur_sub_index + 1 == num_subs) {
+				ret.like_string += "%";
+			}
+			break;
+		case duckdb_re2::kRegexpStar:
+			// .* is a Star operator is a anyChar operator as a child.
+			// any other child operator would represent a pattern LIKE cannot match.
+			if (subs[cur_sub_index]->nsub() == 1 && subs[cur_sub_index]->sub()[0]->op() == duckdb_re2::kRegexpAnyChar) {
+				ret.like_string += "%";
+				break;
+			}
+			ret.exists = false;
+			return ret;
+		case duckdb_re2::kRegexpLiteralString:
+		case duckdb_re2::kRegexpLiteral: {
+			// if this is the only matching op, we should have directly called
+			// GetEscapedLikeString
+			D_ASSERT(!(cur_sub_index == 0 && cur_sub_index + 1 == num_subs));
+			if (cur_sub_index == 0) {
+				ret.like_string += "%";
+			}
+			// if the kRegexpLiteral or kRegexpLiteralString is the only op to match
+			// the string can directly be converted into a contains
+			LikeString escaped_like_string = GetLikeStringEscaped(subs[cur_sub_index], false);
+			if (!escaped_like_string.exists) {
+				return escaped_like_string;
+			}
+			ret.like_string += escaped_like_string.like_string;
+			ret.escaped = escaped_like_string.escaped;
+			if (cur_sub_index + 1 == num_subs) {
+				ret.like_string += "%";
+			}
+			break;
+		}
+		case duckdb_re2::kRegexpEndText:
+		case duckdb_re2::kRegexpEmptyMatch:
+		case duckdb_re2::kRegexpBeginText: {
+			break;
+		}
+		default:
+			// some other regexp op that doesn't have an equivalent to a like string
+			// return false;
+			ret.exists = false;
+			return ret;
+		}
+		cur_sub_index += 1;
+	}
+	return ret;
+}
+
 unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<reference<Expression>> &bindings,
                                                     bool &changes_made, bool is_root) {
 	auto &root = bindings[0].get().Cast<BoundFunctionExpression>();
 	auto &constant_expr = bindings[2].get().Cast<BoundConstantExpression>();
-	D_ASSERT(root.children.size() == 2);
+	D_ASSERT(root.children.size() == 2 || root.children.size() == 3);
+	auto regexp_bind_data = root.bind_info.get()->Cast<RegexpMatchesBindData>();
+
+	auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), constant_expr);
+	D_ASSERT(constant_value.type() == constant_expr.return_type);
+	auto patt_str = StringValue::Get(constant_value);
+
+	duckdb_re2::RE2::Options parsed_options = regexp_bind_data.options;
 
 	if (constant_expr.value.IsNull()) {
 		return make_uniq<BoundConstantExpression>(Value(root.return_type));
@@ -31,34 +154,50 @@ unique_ptr<Expression> RegexOptimizationRule::Apply(LogicalOperator &op, vector<
 	// the constant_expr is a scalar expression that we have to fold
 	if (!constant_expr.IsFoldable()) {
 		return nullptr;
-	}
+	};
 
-	auto constant_value = ExpressionExecutor::EvaluateScalar(GetContext(), constant_expr);
-	D_ASSERT(constant_value.type() == constant_expr.return_type);
-	auto patt_str = StringValue::Get(constant_value);
-
-	duckdb_re2::RE2 pattern(patt_str);
+	duckdb_re2::RE2 pattern(patt_str, parsed_options);
 	if (!pattern.ok()) {
 		return nullptr; // this should fail somewhere else
 	}
 
+	LikeString like_string;
+	// check for a like string. If we can convert it to a like string, the like string
+	// optimizer will further optimize suffix and prefix things.
 	if (pattern.Regexp()->op() == duckdb_re2::kRegexpLiteralString ||
 	    pattern.Regexp()->op() == duckdb_re2::kRegexpLiteral) {
-
-		string min;
-		string max;
-		pattern.PossibleMatchRange(&min, &max, patt_str.size() + 1);
-		if (min != max) {
+		// convert to contains.
+		LikeString escaped_like_string = GetLikeStringEscaped(pattern.Regexp(), true);
+		if (!escaped_like_string.exists) {
 			return nullptr;
 		}
-		auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(min)));
+		auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(escaped_like_string.like_string)));
 		auto contains = make_uniq<BoundFunctionExpression>(root.return_type, ContainsFun::GetFunction(),
 		                                                   std::move(root.children), nullptr);
 		contains->children[1] = std::move(parameter);
 
 		return std::move(contains);
+	} else if (pattern.Regexp()->op() == duckdb_re2::kRegexpConcat) {
+		like_string = LikeMatchFromRegex(pattern);
+	} else {
+		like_string.exists = false;
+	}
+
+	if (!like_string.exists) {
+		return nullptr;
+	}
+
+	// if regexp had options, remove them so the new Like Expression can be matched for other optimizers.
+	if (root.children.size() == 3) {
+		root.children.pop_back();
+		D_ASSERT(root.children.size() == 2);
 	}
-	return nullptr;
+
+	auto like_expression = make_uniq<BoundFunctionExpression>(root.return_type, LikeFun::GetLikeFunction(),
+	                                                          std::move(root.children), nullptr);
+	auto parameter = make_uniq<BoundConstantExpression>(Value(std::move(like_string.like_string)));
+	like_expression->children[1] = std::move(parameter);
+	return std::move(like_expression);
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp

@@ -1,9 +1,13 @@
 #include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/optimizer/filter_pushdown.hpp"
 #include "duckdb/optimizer/statistics_propagator.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_comparison_expression.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/operator/logical_any_join.hpp"
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_cross_product.hpp"
+#include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_join.hpp"
 #include "duckdb/planner/operator/logical_limit.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
@@ -13,8 +17,8 @@ namespace duckdb {
 void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, unique_ptr<LogicalOperator> *node_ptr) {
 	for (idx_t i = 0; i < join.conditions.size(); i++) {
 		auto &condition = join.conditions[i];
-		auto stats_left = PropagateExpression(condition.left);
-		auto stats_right = PropagateExpression(condition.right);
+		const auto stats_left = PropagateExpression(condition.left);
+		const auto stats_right = PropagateExpression(condition.right);
 		if (stats_left && stats_right) {
 			if ((condition.comparison == ExpressionType::COMPARE_DISTINCT_FROM ||
 			     condition.comparison == ExpressionType::COMPARE_NOT_DISTINCT_FROM) &&
@@ -25,8 +29,8 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 			}
 			auto prune_result = PropagateComparison(*stats_left, *stats_right, condition.comparison);
 			// Add stats to logical_join for perfect hash join
-			join.join_stats.push_back(std::move(stats_left));
-			join.join_stats.push_back(std::move(stats_right));
+			join.join_stats.push_back(stats_left->ToUnique());
+			join.join_stats.push_back(stats_right->ToUnique());
 			switch (prune_result) {
 			case FilterPropagateResult::FILTER_FALSE_OR_NULL:
 			case FilterPropagateResult::FILTER_ALWAYS_FALSE:
@@ -123,12 +127,21 @@ void StatisticsPropagator::PropagateStatistics(LogicalComparisonJoin &join, uniq
 		case JoinType::INNER:
 		case JoinType::SEMI: {
 			UpdateFilterStatistics(*condition.left, *condition.right, condition.comparison);
-			auto stats_left = PropagateExpression(condition.left);
-			auto stats_right = PropagateExpression(condition.right);
+			auto updated_stats_left = PropagateExpression(condition.left);
+			auto updated_stats_right = PropagateExpression(condition.right);
+
+			// Try to push lhs stats down rhs and vice versa
+			if (!context.config.force_index_join && stats_left && stats_right && updated_stats_left &&
+			    updated_stats_right && condition.left->type == ExpressionType::BOUND_COLUMN_REF &&
+			    condition.right->type == ExpressionType::BOUND_COLUMN_REF) {
+				CreateFilterFromJoinStats(join.children[0], condition.left, *stats_left, *updated_stats_left);
+				CreateFilterFromJoinStats(join.children[1], condition.right, *stats_right, *updated_stats_right);
+			}
+
 			// Update join_stats when is already part of the join
 			if (join.join_stats.size() == 2) {
-				join.join_stats[0] = std::move(stats_left);
-				join.join_stats[1] = std::move(stats_right);
+				join.join_stats[0] = std::move(updated_stats_left);
+				join.join_stats[1] = std::move(updated_stats_right);
 			}
 			break;
 		}
@@ -281,4 +294,48 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalPosi
 	return std::move(node_stats);
 }
 
+void StatisticsPropagator::CreateFilterFromJoinStats(unique_ptr<LogicalOperator> &child, unique_ptr<Expression> &expr,
+                                                     const BaseStatistics &stats_before,
+                                                     const BaseStatistics &stats_after) {
+	// Only do this for integral colref's that have stats
+	if (expr->type != ExpressionType::BOUND_COLUMN_REF || !expr->return_type.IsIntegral() ||
+	    !NumericStats::HasMinMax(stats_before) || !NumericStats::HasMinMax(stats_after)) {
+		return;
+	}
+
+	// Retrieve min/max
+	auto min_before = NumericStats::Min(stats_before);
+	auto max_before = NumericStats::Max(stats_before);
+	auto min_after = NumericStats::Min(stats_after);
+	auto max_after = NumericStats::Max(stats_after);
+
+	vector<unique_ptr<Expression>> filter_exprs;
+	if (min_after > min_before) {
+		filter_exprs.emplace_back(
+		    make_uniq<BoundComparisonExpression>(ExpressionType::COMPARE_GREATERTHANOREQUALTO, expr->Copy(),
+		                                         make_uniq<BoundConstantExpression>(std::move(min_after))));
+	}
+	if (max_after < max_before) {
+		filter_exprs.emplace_back(
+		    make_uniq<BoundComparisonExpression>(ExpressionType::COMPARE_LESSTHANOREQUALTO, expr->Copy(),
+		                                         make_uniq<BoundConstantExpression>(std::move(max_after))));
+	}
+
+	if (filter_exprs.empty()) {
+		return;
+	}
+
+	auto filter = make_uniq<LogicalFilter>();
+	filter->children.emplace_back(std::move(child));
+	child = std::move(filter);
+
+	for (auto &filter_expr : filter_exprs) {
+		child->expressions.emplace_back(std::move(filter_expr));
+	}
+
+	FilterPushdown filter_pushdown(optimizer);
+	child = filter_pushdown.Rewrite(std::move(child));
+	PropagateExpression(expr);
+}
+
 } // namespace duckdb

package/src/duckdb/src/optimizer/statistics/operator/propagate_order.cpp

@@ -11,7 +11,7 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalOrde
 
 	// then propagate to each of the order expressions
 	for (auto &bound_order : order.orders) {
-		PropagateAndCompress(bound_order.expression, bound_order.stats);
+		bound_order.stats = PropagateExpression(bound_order.expression);
 	}
 	return std::move(node_stats);
 }

package/src/duckdb/src/optimizer/statistics_propagator.cpp

@@ -1,24 +1,26 @@
 #include "duckdb/optimizer/statistics_propagator.hpp"
 
 #include "duckdb/main/client_context.hpp"
+#include "duckdb/optimizer/optimizer.hpp"
+#include "duckdb/planner/expression/list.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/logical_operator.hpp"
 #include "duckdb/planner/operator/logical_aggregate.hpp"
-#include "duckdb/planner/operator/logical_empty_result.hpp"
 #include "duckdb/planner/operator/logical_cross_product.hpp"
+#include "duckdb/planner/operator/logical_empty_result.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/operator/logical_join.hpp"
-#include "duckdb/planner/operator/logical_projection.hpp"
+#include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_positional_join.hpp"
+#include "duckdb/planner/operator/logical_projection.hpp"
 #include "duckdb/planner/operator/logical_set_operation.hpp"
-#include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_window.hpp"
-#include "duckdb/planner/expression/list.hpp"
 
 namespace duckdb {
 
-StatisticsPropagator::StatisticsPropagator(ClientContext &context) : context(context) {
+StatisticsPropagator::StatisticsPropagator(Optimizer &optimizer_p)
+    : optimizer(optimizer_p), context(optimizer.context) {
 }
 
 void StatisticsPropagator::ReplaceWithEmptyResult(unique_ptr<LogicalOperator> &node) {

package/src/duckdb/src/optimizer/topn_optimizer.cpp

@@ -1,24 +1,34 @@
 #include "duckdb/optimizer/topn_optimizer.hpp"
-#include "duckdb/planner/operator/logical_order.hpp"
+
+#include "duckdb/common/limits.hpp"
 #include "duckdb/planner/operator/logical_limit.hpp"
+#include "duckdb/planner/operator/logical_order.hpp"
 #include "duckdb/planner/operator/logical_top_n.hpp"
-#include "duckdb/common/limits.hpp"
 
 namespace duckdb {
 
-unique_ptr<LogicalOperator> TopN::Optimize(unique_ptr<LogicalOperator> op) {
-	if (op->type == LogicalOperatorType::LOGICAL_LIMIT &&
-	    op->children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
-		auto &limit = op->Cast<LogicalLimit>();
-		auto &order_by = (op->children[0])->Cast<LogicalOrder>();
+bool TopN::CanOptimize(LogicalOperator &op) {
+	if (op.type == LogicalOperatorType::LOGICAL_LIMIT &&
+	    op.children[0]->type == LogicalOperatorType::LOGICAL_ORDER_BY) {
+		auto &limit = op.Cast<LogicalLimit>();
 
 		// This optimization doesn't apply when OFFSET is present without LIMIT
 		// Or if offset is not constant
 		if (limit.limit_val != NumericLimits<int64_t>::Maximum() || limit.offset) {
-			auto topn = make_uniq<LogicalTopN>(std::move(order_by.orders), limit.limit_val, limit.offset_val);
-			topn->AddChild(std::move(order_by.children[0]));
-			op = std::move(topn);
+			return true;
 		}
+	}
+	return false;
+}
+
+unique_ptr<LogicalOperator> TopN::Optimize(unique_ptr<LogicalOperator> op) {
+	if (CanOptimize(*op)) {
+		auto &limit = op->Cast<LogicalLimit>();
+		auto &order_by = (op->children[0])->Cast<LogicalOrder>();
+
+		auto topn = make_uniq<LogicalTopN>(std::move(order_by.orders), limit.limit_val, limit.offset_val);
+		topn->AddChild(std::move(order_by.children[0]));
+		op = std::move(topn);
 	} else {
 		for (auto &child : op->children) {
 			child = Optimize(std::move(child));

package/src/duckdb/src/parallel/executor.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/execution/operator/helper/physical_result_collector.hpp"
 #include "duckdb/execution/operator/set/physical_recursive_cte.hpp"
+#include "duckdb/execution/operator/set/physical_cte.hpp"
 #include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
@@ -267,6 +268,10 @@ void Executor::AddRecursiveCTE(PhysicalOperator &rec_cte) {
 	recursive_ctes.push_back(rec_cte);
 }
 
+void Executor::AddMaterializedCTE(PhysicalOperator &mat_cte) {
+	materialized_ctes.push_back(mat_cte);
+}
+
 void Executor::ReschedulePipelines(const vector<shared_ptr<MetaPipeline>> &pipelines_p,
                                    vector<shared_ptr<Event>> &events_p) {
 	ScheduleEventData event_data(pipelines_p, events_p, false);
@@ -344,6 +349,12 @@ void Executor::InitializeInternal(PhysicalOperator &plan) {
 			rec_cte.recursive_meta_pipeline->Ready();
 		}
 
+		// ready materialized cte pipelines too
+		for (auto &mat_cte_ref : materialized_ctes) {
+			auto &mat_cte = mat_cte_ref.get().Cast<PhysicalCTE>();
+			mat_cte.recursive_meta_pipeline->Ready();
+		}
+
 		// set root pipelines, i.e., all pipelines that end in the final sink
 		root_pipeline->GetPipelines(root_pipelines, false);
 		root_pipeline_idx = 0;
@@ -381,6 +392,10 @@ void Executor::CancelTasks() {
 			auto &rec_cte = rec_cte_ref.get().Cast<PhysicalRecursiveCTE>();
 			rec_cte.recursive_meta_pipeline.reset();
 		}
+		for (auto &mat_cte_ref : materialized_ctes) {
+			auto &mat_cte = mat_cte_ref.get().Cast<PhysicalCTE>();
+			mat_cte.recursive_meta_pipeline.reset();
+		}
 		pipelines.clear();
 		root_pipelines.clear();
 		to_be_rescheduled_tasks.clear();

package/src/duckdb/src/parallel/pipeline_executor.cpp

@@ -17,11 +17,10 @@ PipelineExecutor::PipelineExecutor(ClientContext &context_p, Pipeline &pipeline_
 		requires_batch_index = pipeline.sink->RequiresBatchIndex() && pipeline.source->SupportsBatchIndex();
 		if (requires_batch_index) {
 			auto &partition_info = local_sink_state->partition_info;
-			if (!partition_info.batch_index.IsValid()) {
-				// batch index is not set yet - initialize before fetching anything
-				partition_info.batch_index = pipeline.RegisterNewBatchIndex();
-				partition_info.min_batch_index = partition_info.batch_index;
-			}
+			D_ASSERT(!partition_info.batch_index.IsValid());
+			// batch index is not set yet - initialize before fetching anything
+			partition_info.batch_index = pipeline.RegisterNewBatchIndex();
+			partition_info.min_batch_index = partition_info.batch_index;
 		}
 	}
 	local_source_state = pipeline.source->GetLocalSourceState(context, *pipeline.source_state);
@@ -79,6 +78,7 @@ bool PipelineExecutor::TryFlushCachingOperators() {
 		OperatorResultType push_result;
 
 		if (in_process_operators.empty()) {
+			curr_chunk.Reset();
 			StartOperator(current_operator);
 			finalize_result = current_operator.FinalExecute(context, curr_chunk, *current_operator.op_state,
 			                                                *intermediate_states[flushing_idx]);
@@ -477,7 +477,8 @@ SourceResultType PipelineExecutor::FetchFromSource(DataChunk &result) {
 		} else {
 			next_batch_index =
 			    pipeline.source->GetBatchIndex(context, result, *pipeline.source_state, *local_source_state);
-			next_batch_index += pipeline.base_batch_index;
+			// we start with the base_batch_index as a valid starting value. Make sure that next batch is called below
+			next_batch_index += pipeline.base_batch_index + 1;
 		}
 		auto &partition_info = local_sink_state->partition_info;
 		if (next_batch_index != partition_info.batch_index.GetIndex()) {

package/src/duckdb/src/parallel/task_scheduler.cpp

@@ -1,13 +1,14 @@
 #include "duckdb/parallel/task_scheduler.hpp"
 
+#include "duckdb/common/chrono.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "concurrentqueue.h"
-#include "lightweightsemaphore.h"
 #include "duckdb/common/thread.hpp"
+#include "lightweightsemaphore.h"
 #else
 #include <queue>
 #endif
@@ -93,7 +94,9 @@ ProducerToken::ProducerToken(TaskScheduler &scheduler, unique_ptr<QueueProducerT
 ProducerToken::~ProducerToken() {
 }
 
-TaskScheduler::TaskScheduler(DatabaseInstance &db) : db(db), queue(make_uniq<ConcurrentQueue>()) {
+TaskScheduler::TaskScheduler(DatabaseInstance &db)
+    : db(db), queue(make_uniq<ConcurrentQueue>()),
+      allocator_flush_threshold(db.config.options.allocator_flush_threshold) {
 }
 
 TaskScheduler::~TaskScheduler() {
@@ -146,6 +149,9 @@ void TaskScheduler::ExecuteForever(atomic<bool> *marker) {
 				task.reset();
 				break;
 			}
+
+			// Flushes the outstanding allocator's outstanding allocations
+			Allocator::ThreadFlush(allocator_flush_threshold);
 		}
 	}
 #else
@@ -241,6 +247,9 @@ void TaskScheduler::SetThreads(int32_t n) {
 #endif
 }
 
+void TaskScheduler::SetAllocatorFlushTreshold(idx_t threshold) {
+}
+
 void TaskScheduler::Signal(idx_t n) {
 #ifndef DUCKDB_NO_THREADS
 	queue->semaphore.signal(n);

package/src/duckdb/src/parser/common_table_expression_info.cpp

@@ -7,12 +7,14 @@ namespace duckdb {
 void CommonTableExpressionInfo::FormatSerialize(FormatSerializer &serializer) const {
 	serializer.WriteProperty("aliases", aliases);
 	serializer.WriteProperty("query", query);
+	serializer.WriteProperty("materialized", materialized);
 }
 
 unique_ptr<CommonTableExpressionInfo> CommonTableExpressionInfo::FormatDeserialize(FormatDeserializer &deserializer) {
 	auto result = make_uniq<CommonTableExpressionInfo>();
 	result->aliases = deserializer.ReadProperty<vector<string>>("aliases");
 	result->query = deserializer.ReadProperty<unique_ptr<SelectStatement>>("query");
+	result->materialized = deserializer.ReadProperty<CTEMaterialize>("materialized");
 	return result;
 }
 

package/src/duckdb/src/parser/expression/lambda_expression.cpp

@@ -13,7 +13,7 @@ LambdaExpression::LambdaExpression(unique_ptr<ParsedExpression> lhs, unique_ptr<
 }
 
 string LambdaExpression::ToString() const {
-	return lhs->ToString() + " -> " + expr->ToString();
+	return "(" + lhs->ToString() + " -> " + expr->ToString() + ")";
 }
 
 bool LambdaExpression::Equal(const LambdaExpression &a, const LambdaExpression &b) {

package/src/duckdb/src/parser/parsed_expression_iterator.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/parser/expression/list.hpp"
 #include "duckdb/parser/query_node.hpp"
 #include "duckdb/parser/query_node/recursive_cte_node.hpp"
+#include "duckdb/parser/query_node/cte_node.hpp"
 #include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/query_node/set_operation_node.hpp"
 #include "duckdb/parser/tableref/list.hpp"
@@ -256,6 +257,12 @@ void ParsedExpressionIterator::EnumerateQueryNodeChildren(
 		EnumerateQueryNodeChildren(*rcte_node.right, callback);
 		break;
 	}
+	case QueryNodeType::CTE_NODE: {
+		auto &cte_node = node.Cast<CTENode>();
+		EnumerateQueryNodeChildren(*cte_node.query, callback);
+		EnumerateQueryNodeChildren(*cte_node.child, callback);
+		break;
+	}
 	case QueryNodeType::SELECT_NODE: {
 		auto &sel_node = node.Cast<SelectNode>();
 		for (idx_t i = 0; i < sel_node.select_list.size(); i++) {

package/src/duckdb/src/parser/query_node/cte_node.cpp

@@ -0,0 +1,75 @@
+#include "duckdb/parser/query_node/cte_node.hpp"
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
+namespace duckdb {
+
+string CTENode::ToString() const {
+	string result;
+	result += child->ToString();
+	return result;
+}
+
+bool CTENode::Equals(const QueryNode *other_p) const {
+	if (!QueryNode::Equals(other_p)) {
+		return false;
+	}
+	if (this == other_p) {
+		return true;
+	}
+	auto &other = other_p->Cast<CTENode>();
+
+	if (!query->Equals(other.query.get())) {
+		return false;
+	}
+	if (!child->Equals(other.child.get())) {
+		return false;
+	}
+	return true;
+}
+
+unique_ptr<QueryNode> CTENode::Copy() const {
+	auto result = make_uniq<CTENode>();
+	result->ctename = ctename;
+	result->query = query->Copy();
+	result->child = child->Copy();
+	result->aliases = aliases;
+	this->CopyProperties(*result);
+	return std::move(result);
+}
+
+void CTENode::Serialize(FieldWriter &writer) const {
+	writer.WriteString(ctename);
+	writer.WriteSerializable(*query);
+	writer.WriteSerializable(*child);
+	writer.WriteList<string>(aliases);
+}
+
+unique_ptr<QueryNode> CTENode::Deserialize(FieldReader &reader) {
+	auto result = make_uniq<CTENode>();
+	result->ctename = reader.ReadRequired<string>();
+	result->query = reader.ReadRequiredSerializable<QueryNode>();
+	result->child = reader.ReadRequiredSerializable<QueryNode>();
+	result->aliases = reader.ReadRequiredList<string>();
+	return std::move(result);
+}
+
+void CTENode::FormatSerialize(FormatSerializer &serializer) const {
+	QueryNode::FormatSerialize(serializer);
+	serializer.WriteProperty("cte_name", ctename);
+	serializer.WriteProperty("query", *query);
+	serializer.WriteProperty("child", *child);
+	serializer.WriteProperty("aliases", aliases);
+}
+
+unique_ptr<QueryNode> CTENode::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto result = make_uniq<CTENode>();
+	deserializer.ReadProperty("cte_name", result->ctename);
+	deserializer.ReadProperty("query", result->query);
+	deserializer.ReadProperty("child", result->child);
+	deserializer.ReadProperty("aliases", result->aliases);
+	return std::move(result);
+}
+
+} // namespace duckdb