duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/execution/operator/join/physical_index_join.cpp

@@ -12,6 +12,7 @@
 #include "duckdb/storage/table/append_state.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 
@@ -64,6 +65,8 @@ PhysicalIndexJoin::PhysicalIndexJoin(LogicalOperator &op, unique_ptr<PhysicalOpe
     : CachingPhysicalOperator(PhysicalOperatorType::INDEX_JOIN, std::move(op.types), estimated_cardinality),
       left_projection_map(left_projection_map_p), right_projection_map(std::move(right_projection_map_p)),
       index(index_p), conditions(std::move(cond)), join_type(join_type), lhs_first(lhs_first) {
+	D_ASSERT(right->type == PhysicalOperatorType::TABLE_SCAN);
+	auto &tbl_scan = (PhysicalTableScan &)*right;
 	column_ids = std::move(column_ids_p);
 	children.push_back(std::move(left));
 	children.push_back(std::move(right));
@@ -74,11 +77,17 @@ PhysicalIndexJoin::PhysicalIndexJoin(LogicalOperator &op, unique_ptr<PhysicalOpe
 	for (auto &index_id : index->column_ids) {
 		index_ids.insert(index_id);
 	}
-	for (idx_t column_id = 0; column_id < column_ids.size(); column_id++) {
-		auto it = index_ids.find(column_ids[column_id]);
+
+	for (idx_t i = 0; i < column_ids.size(); i++) {
+		auto column_id = column_ids[i];
+		auto it = index_ids.find(column_id);
 		if (it == index_ids.end()) {
-			fetch_ids.push_back(column_ids[column_id]);
-			fetch_types.push_back(children[1]->types[column_id]);
+			fetch_ids.push_back(column_id);
+			if (column_id == COLUMN_IDENTIFIER_ROW_ID) {
+				fetch_types.emplace_back(LogicalType::ROW_TYPE);
+			} else {
+				fetch_types.push_back(tbl_scan.returned_types[column_id]);
+			}
 		}
 	}
 	if (right_projection_map.empty()) {

package/src/duckdb/src/execution/operator/join/physical_join.cpp

@@ -61,9 +61,6 @@ void PhysicalJoin::BuildJoinPipelines(Pipeline &current, MetaPipeline &meta_pipe
 	bool add_child_pipeline = false;
 	auto &join_op = (PhysicalJoin &)op;
 	if (IsRightOuterJoin(join_op.join_type)) {
-		if (meta_pipeline.HasRecursiveCTE()) {
-			throw NotImplementedException("FULL and RIGHT outer joins are not supported in recursive CTEs yet");
-		}
 		add_child_pipeline = true;
 	}
 

package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp

@@ -30,25 +30,20 @@ PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, uniq
 		switch (cond.comparison) {
 		case ExpressionType::COMPARE_LESSTHAN:
 		case ExpressionType::COMPARE_LESSTHANOREQUALTO:
-			lhs_orders.emplace_back(
-			    BoundOrderByNode(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(left)));
-			rhs_orders.emplace_back(
-			    BoundOrderByNode(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(right)));
+			lhs_orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(left));
+			rhs_orders.emplace_back(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, std::move(right));
 			break;
 		case ExpressionType::COMPARE_GREATERTHAN:
 		case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
-			lhs_orders.emplace_back(
-			    BoundOrderByNode(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, std::move(left)));
-			rhs_orders.emplace_back(
-			    BoundOrderByNode(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, std::move(right)));
+			lhs_orders.emplace_back(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, std::move(left));
+			rhs_orders.emplace_back(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, std::move(right));
 			break;
 		case ExpressionType::COMPARE_NOTEQUAL:
 		case ExpressionType::COMPARE_DISTINCT_FROM:
 			// Allowed in multi-predicate joins, but can't be first/sort.
 			D_ASSERT(!lhs_orders.empty());
-			lhs_orders.emplace_back(BoundOrderByNode(OrderType::INVALID, OrderByNullType::NULLS_LAST, std::move(left)));
-			rhs_orders.emplace_back(
-			    BoundOrderByNode(OrderType::INVALID, OrderByNullType::NULLS_LAST, std::move(right)));
+			lhs_orders.emplace_back(OrderType::INVALID, OrderByNullType::NULLS_LAST, std::move(left));
+			rhs_orders.emplace_back(OrderType::INVALID, OrderByNullType::NULLS_LAST, std::move(right));
 			break;
 
 		default:

package/src/duckdb/src/execution/operator/join/physical_range_join.cpp

@@ -46,7 +46,7 @@ void PhysicalRangeJoin::LocalSortedTable::Sink(DataChunk &input, GlobalSortState
 
 	//	Only sort the primary key
 	DataChunk join_head;
-	join_head.data.emplace_back(Vector(keys.data[0]));
+	join_head.data.emplace_back(keys.data[0]);
 	join_head.SetCardinality(keys.size());
 
 	// Sink the data into the local sort state
@@ -334,7 +334,9 @@ idx_t PhysicalRangeJoin::SelectJoinTail(const ExpressionType &condition, Vector
 	case ExpressionType::COMPARE_DISTINCT_FROM:
 		return VectorOperations::DistinctFrom(left, right, sel, count, true_sel, nullptr);
 	case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
+		return VectorOperations::NotDistinctFrom(left, right, sel, count, true_sel, nullptr);
 	case ExpressionType::COMPARE_EQUAL:
+		return VectorOperations::Equals(left, right, sel, count, true_sel, nullptr);
 	default:
 		throw InternalException("Unsupported comparison type for PhysicalRangeJoin");
 	}

package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp

@@ -121,6 +121,9 @@ bool TryCastFloatingValueCommaSeparated(const string_t &value_str, const Logical
 }
 
 bool BaseCSVReader::TryCastValue(const Value &value, const LogicalType &sql_type) {
+	if (value.IsNull()) {
+		return true;
+	}
 	if (options.has_format[LogicalTypeId::DATE] && sql_type.id() == LogicalTypeId::DATE) {
 		date_t result;
 		string error_message;
@@ -317,7 +320,11 @@ bool BaseCSVReader::AddRow(DataChunk &insert_chunk, idx_t &column, string &error
 	}
 
 	if (column < return_types.size() && mode != ParserMode::SNIFFING_DIALECT) {
-		if (options.ignore_errors) {
+		if (options.null_padding) {
+			for (; column < return_types.size(); column++) {
+				FlatVector::SetNull(parse_chunk.data[column], parse_chunk.size(), true);
+			}
+		} else if (options.ignore_errors) {
 			column = 0;
 			return false;
 		} else {
@@ -495,12 +502,12 @@ bool BaseCSVReader::Flush(DataChunk &insert_chunk, bool try_add_line) {
 			}
 
 			// figure out the exact line number
+			UnifiedVectorFormat inserted_column_data;
+			insert_chunk.data[col_idx].ToUnifiedFormat(parse_chunk.size(), inserted_column_data);
 			idx_t row_idx;
 			for (row_idx = 0; row_idx < parse_chunk.size(); row_idx++) {
-				auto &inserted_column = insert_chunk.data[col_idx];
 				auto &parsed_column = parse_chunk.data[col_idx];
-
-				if (FlatVector::IsNull(inserted_column, row_idx) && !FlatVector::IsNull(parsed_column, row_idx)) {
+				if (!inserted_column_data.validity.RowIsValid(row_idx) && !FlatVector::IsNull(parsed_column, row_idx)) {
 					break;
 				}
 			}

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -419,6 +419,7 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 	}
 
 	idx_t best_consistent_rows = 0;
+	idx_t prev_padding_count = 0;
 	for (auto quoterule : quoterule_candidates) {
 		const auto &quote_candidates = quote_candidates_map[static_cast<uint8_t>(quoterule)];
 		for (const auto &quote : quote_candidates) {
@@ -441,20 +442,29 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 
 					idx_t start_row = original_options.skip_rows;
 					idx_t consistent_rows = 0;
-					idx_t num_cols = 0;
-
+					idx_t num_cols = sniffed_column_counts.empty() ? 0 : sniffed_column_counts[0];
+					idx_t padding_count = 0;
+					bool allow_padding = original_options.null_padding;
 					for (idx_t row = 0; row < sniffed_column_counts.size(); row++) {
 						if (sniffed_column_counts[row] == num_cols) {
 							consistent_rows++;
-						} else {
+						} else if (num_cols < sniffed_column_counts[row] && !original_options.skip_rows_set) {
+							// we use the maximum amount of num_cols that we find
 							num_cols = sniffed_column_counts[row];
 							start_row = row + original_options.skip_rows;
 							consistent_rows = 1;
+							padding_count = 0;
+						} else if (num_cols >= sniffed_column_counts[row] && allow_padding) {
+							// we are missing some columns, we can parse this as long as we add padding
+							padding_count++;
 						}
 					}
 
 					// some logic
+					consistent_rows += padding_count;
 					bool more_values = (consistent_rows > best_consistent_rows && num_cols >= best_num_cols);
+					bool require_more_padding = padding_count > prev_padding_count;
+					bool require_less_padding = padding_count < prev_padding_count;
 					bool single_column_before = best_num_cols < 2 && num_cols > best_num_cols;
 					bool rows_consistent =
 					    start_row + consistent_rows - original_options.skip_rows == sniffed_column_counts.size();
@@ -464,16 +474,19 @@ void BufferedCSVReader::DetectDialect(const vector<LogicalType> &requested_types
 
 					if (!requested_types.empty() && requested_types.size() != num_cols) {
 						continue;
-					} else if ((more_values || single_column_before) && rows_consistent) {
+					} else if (rows_consistent && (single_column_before || (more_values && !require_more_padding) ||
+					                               (more_than_one_column && require_less_padding))) {
 						sniff_info.skip_rows = start_row;
 						sniff_info.num_cols = num_cols;
 						sniff_info.new_line = options.new_line;
 						best_consistent_rows = consistent_rows;
 						best_num_cols = num_cols;
+						prev_padding_count = padding_count;
 
 						info_candidates.clear();
 						info_candidates.push_back(sniff_info);
-					} else if (more_than_one_row && more_than_one_column && start_good && rows_consistent) {
+					} else if (more_than_one_row && more_than_one_column && start_good && rows_consistent &&
+					           !require_more_padding) {
 						bool same_quote_is_candidate = false;
 						for (auto &info_candidate : info_candidates) {
 							if (quote.compare(info_candidate.quote) == 0) {
@@ -555,7 +568,7 @@ void BufferedCSVReader::DetectCandidateTypes(const vector<LogicalType> &type_can
 					// try formatting for date types if the user did not specify one and it starts with numeric values.
 					string separator;
 					if (has_format_candidates.count(sql_type.id()) && !original_options.has_format[sql_type.id()] &&
-					    StartsWithNumericDate(separator, StringValue::Get(dummy_val))) {
+					    !dummy_val.IsNull() && StartsWithNumericDate(separator, StringValue::Get(dummy_val))) {
 						// generate date format candidates the first time through
 						auto &type_format_candidates = format_candidates[sql_type.id()];
 						const auto had_format_candidates = has_format_candidates[sql_type.id()];
@@ -870,16 +883,7 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	// #######
 	// ### type detection (initial)
 	// #######
-	// type candidates, ordered by descending specificity (~ from high to low)
-	vector<LogicalType> type_candidates = {
-	    LogicalType::VARCHAR,
-	    LogicalType::TIMESTAMP,
-	    LogicalType::DATE,
-	    LogicalType::TIME,
-	    LogicalType::DOUBLE,
-	    /* LogicalType::FLOAT,*/ LogicalType::BIGINT,
-	    /*LogicalType::INTEGER,*/ /*LogicalType::SMALLINT, LogicalType::TINYINT,*/ LogicalType::BOOLEAN,
-	    LogicalType::SQLNULL};
+
 	// format template candidates, ordered by descending specificity (~ from high to low)
 	std::map<LogicalTypeId, vector<const char *>> format_template_candidates = {
 	    {LogicalTypeId::DATE, {"%m-%d-%Y", "%m-%d-%y", "%d-%m-%Y", "%d-%m-%y", "%Y-%m-%d", "%y-%m-%d"}},
@@ -890,8 +894,8 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	vector<vector<LogicalType>> best_sql_types_candidates;
 	map<LogicalTypeId, vector<string>> best_format_candidates;
 	DataChunk best_header_row;
-	DetectCandidateTypes(type_candidates, format_template_candidates, info_candidates, original_options, best_num_cols,
-	                     best_sql_types_candidates, best_format_candidates, best_header_row);
+	DetectCandidateTypes(options.auto_type_candidates, format_template_candidates, info_candidates, original_options,
+	                     best_num_cols, best_sql_types_candidates, best_format_candidates, best_header_row);
 
 	if (best_format_candidates.empty() || best_header_row.size() == 0) {
 		throw InvalidInputException(
@@ -939,7 +943,8 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	// #######
 	// ### type detection (refining)
 	// #######
-	return RefineTypeDetection(type_candidates, requested_types, best_sql_types_candidates, best_format_candidates);
+	return RefineTypeDetection(options.auto_type_candidates, requested_types, best_sql_types_candidates,
+	                           best_format_candidates);
 }
 
 bool BufferedCSVReader::TryParseComplexCSV(DataChunk &insert_chunk, string &error_message) {

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -145,6 +145,7 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		}
 	} else if (loption == "skip") {
 		skip_rows = ParseInteger(value, loption);
+		skip_rows_set = true;
 	} else if (loption == "max_line_size" || loption == "maximum_line_size") {
 		maximum_line_size = ParseInteger(value, loption);
 	} else if (loption == "sample_chunk_size") {
@@ -183,6 +184,8 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 		if (decimal_separator != "." && decimal_separator != ",") {
 			throw BinderException("Unsupported parameter for DECIMAL_SEPARATOR: should be '.' or ','");
 		}
+	} else if (loption == "null_padding") {
+		null_padding = ParseBoolean(value, loption);
 	} else {
 		throw BinderException("Unrecognized option for CSV reader \"%s\"", loption);
 	}

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -1,12 +1,13 @@
 #include "duckdb/execution/operator/persistent/physical_batch_insert.hpp"
 
 #include "duckdb/parallel/thread_context.hpp"
-#include "duckdb/parser/parsed_data/create_table_info.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/storage/table/row_group_collection.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/transaction/local_storage.hpp"
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp

@@ -85,8 +85,8 @@ static string CreateDirRecursive(const vector<idx_t> &cols, const vector<string>
 	CreateDir(path, fs);
 
 	for (idx_t i = 0; i < cols.size(); i++) {
-		auto partition_col_name = names[cols[i]];
-		auto partition_value = values[i];
+		const auto &partition_col_name = names[cols[i]];
+		const auto &partition_value = values[i];
 		string p_dir = partition_col_name + "=" + partition_value.ToString();
 		path = fs.JoinPath(path, p_dir);
 		CreateDir(path, fs);

package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp

@@ -2,11 +2,9 @@
 
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/data_table.hpp"
-#include "duckdb/transaction/transaction.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/common/types/column_data_collection.hpp"
-
-#include "duckdb/common/atomic.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp

@@ -16,6 +16,7 @@
 #include "duckdb/common/types/conflict_manager.hpp"
 #include "duckdb/execution/index/art/art.hpp"
 #include "duckdb/transaction/duck_transaction.hpp"
+#include "duckdb/storage/table/append_state.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/operator/projection/physical_projection.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/execution/operator/projection/physical_projection.hpp"
 #include "duckdb/parallel/thread_context.hpp"
 #include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 
 namespace duckdb {
 
@@ -35,6 +36,39 @@ unique_ptr<OperatorState> PhysicalProjection::GetOperatorState(ExecutionContext
 	return make_unique<ProjectionState>(context, select_list);
 }
 
+unique_ptr<PhysicalOperator>
+PhysicalProjection::CreateJoinProjection(vector<LogicalType> proj_types, const vector<LogicalType> &lhs_types,
+                                         const vector<LogicalType> &rhs_types, const vector<idx_t> &left_projection_map,
+                                         const vector<idx_t> &right_projection_map, const idx_t estimated_cardinality) {
+
+	vector<unique_ptr<Expression>> proj_selects;
+	proj_selects.reserve(proj_types.size());
+
+	if (left_projection_map.empty()) {
+		for (storage_t i = 0; i < lhs_types.size(); ++i) {
+			proj_selects.emplace_back(make_unique<BoundReferenceExpression>(lhs_types[i], i));
+		}
+	} else {
+		for (auto i : left_projection_map) {
+			proj_selects.emplace_back(make_unique<BoundReferenceExpression>(lhs_types[i], i));
+		}
+	}
+	const auto left_cols = lhs_types.size();
+
+	if (right_projection_map.empty()) {
+		for (storage_t i = 0; i < rhs_types.size(); ++i) {
+			proj_selects.emplace_back(make_unique<BoundReferenceExpression>(rhs_types[i], left_cols + i));
+		}
+
+	} else {
+		for (auto i : right_projection_map) {
+			proj_selects.emplace_back(make_unique<BoundReferenceExpression>(rhs_types[i], left_cols + i));
+		}
+	}
+
+	return make_unique<PhysicalProjection>(std::move(proj_types), std::move(proj_selects), estimated_cardinality);
+}
+
 string PhysicalProjection::ParamsToString() const {
 	string extra_info;
 	for (auto &expr : select_list) {

package/src/duckdb/src/execution/operator/scan/physical_positional_scan.cpp

@@ -12,13 +12,28 @@ namespace duckdb {
 PhysicalPositionalScan::PhysicalPositionalScan(vector<LogicalType> types, unique_ptr<PhysicalOperator> left,
                                                unique_ptr<PhysicalOperator> right)
     : PhysicalOperator(PhysicalOperatorType::POSITIONAL_SCAN, std::move(types),
-                       MinValue(left->estimated_cardinality, right->estimated_cardinality)) {
+                       MaxValue(left->estimated_cardinality, right->estimated_cardinality)) {
 
 	// Manage the children ourselves
-	D_ASSERT(left->type == PhysicalOperatorType::TABLE_SCAN);
-	D_ASSERT(right->type == PhysicalOperatorType::TABLE_SCAN);
-	child_tables.emplace_back(std::move(left));
-	child_tables.emplace_back(std::move(right));
+	if (left->type == PhysicalOperatorType::TABLE_SCAN) {
+		child_tables.emplace_back(std::move(left));
+	} else if (left->type == PhysicalOperatorType::POSITIONAL_SCAN) {
+		auto &left_scan = (PhysicalPositionalScan &)*left;
+		child_tables = std::move(left_scan.child_tables);
+	} else {
+		throw InternalException("Invalid left input for PhysicalPositionalScan");
+	}
+
+	if (right->type == PhysicalOperatorType::TABLE_SCAN) {
+		child_tables.emplace_back(std::move(right));
+	} else if (right->type == PhysicalOperatorType::POSITIONAL_SCAN) {
+		auto &right_scan = (PhysicalPositionalScan &)*right;
+		auto &right_tables = right_scan.child_tables;
+		child_tables.reserve(child_tables.size() + right_tables.size());
+		std::move(right_tables.begin(), right_tables.end(), std::back_inserter(child_tables));
+	} else {
+		throw InternalException("Invalid right input for PhysicalPositionalScan");
+	}
 }
 
 class PositionalScanGlobalSourceState : public GlobalSourceState {

package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp

@@ -15,10 +15,11 @@ PhysicalCreateType::PhysicalCreateType(unique_ptr<CreateTypeInfo> info, idx_t es
 //===--------------------------------------------------------------------===//
 class CreateTypeGlobalState : public GlobalSinkState {
 public:
-	explicit CreateTypeGlobalState(ClientContext &context) : collection(context, {LogicalType::VARCHAR}) {
+	explicit CreateTypeGlobalState(ClientContext &context) : result(LogicalType::VARCHAR) {
 	}
-
-	ColumnDataCollection collection;
+	Vector result;
+	idx_t size = 0;
+	idx_t capacity = STANDARD_VECTOR_SIZE;
 };
 
 unique_ptr<GlobalSinkState> PhysicalCreateType::GetGlobalSinkState(ClientContext &context) const {
@@ -28,7 +29,7 @@ unique_ptr<GlobalSinkState> PhysicalCreateType::GetGlobalSinkState(ClientContext
 SinkResultType PhysicalCreateType::Sink(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p,
                                         DataChunk &input) const {
 	auto &gstate = (CreateTypeGlobalState &)gstate_p;
-	idx_t total_row_count = gstate.collection.Count() + input.size();
+	idx_t total_row_count = gstate.size + input.size();
 	if (total_row_count > NumericLimits<uint32_t>::Maximum()) {
 		throw InvalidInputException("Attempted to create ENUM of size %llu, which exceeds the maximum size of %llu",
 		                            total_row_count, NumericLimits<uint32_t>::Maximum());
@@ -36,15 +37,23 @@ SinkResultType PhysicalCreateType::Sink(ExecutionContext &context, GlobalSinkSta
 	UnifiedVectorFormat sdata;
 	input.data[0].ToUnifiedFormat(input.size(), sdata);
 
+	if (total_row_count > gstate.capacity) {
+		// We must resize our result vector
+		gstate.result.Resize(gstate.capacity, gstate.capacity * 2);
+		gstate.capacity *= 2;
+	}
+
+	auto src_ptr = (string_t *)sdata.data;
+	auto result_ptr = FlatVector::GetData<string_t>(gstate.result);
 	// Input vector has NULL value, we just throw an exception
 	for (idx_t i = 0; i < input.size(); i++) {
 		idx_t idx = sdata.sel->get_index(i);
 		if (!sdata.validity.RowIsValid(idx)) {
 			throw InvalidInputException("Attempted to create ENUM type with NULL value!");
 		}
+		result_ptr[gstate.size++] =
+		    StringVector::AddStringOrBlob(gstate.result, src_ptr[idx].GetDataUnsafe(), src_ptr[idx].GetSize());
 	}
-
-	gstate.collection.Append(input);
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
@@ -72,44 +81,15 @@ void PhysicalCreateType::GetData(ExecutionContext &context, DataChunk &chunk, Gl
 
 	if (IsSink()) {
 		D_ASSERT(info->type == LogicalType::INVALID);
-
 		auto &g_sink_state = (CreateTypeGlobalState &)*sink_state;
-		auto &collection = g_sink_state.collection;
-
-		idx_t total_row_count = collection.Count();
-
-		ColumnDataScanState scan_state;
-		collection.InitializeScan(scan_state);
-
-		DataChunk scan_chunk;
-		collection.InitializeScanChunk(scan_chunk);
-
-		Vector result(LogicalType::VARCHAR, total_row_count);
-		auto result_ptr = FlatVector::GetData<string_t>(result);
-
-		idx_t offset = 0;
-		while (collection.Scan(scan_state, scan_chunk)) {
-			idx_t src_row_count = scan_chunk.size();
-			auto &src_vec = scan_chunk.data[0];
-			D_ASSERT(src_vec.GetVectorType() == VectorType::FLAT_VECTOR);
-			D_ASSERT(src_vec.GetType().id() == LogicalType::VARCHAR);
-
-			auto src_ptr = FlatVector::GetData<string_t>(src_vec);
-
-			for (idx_t i = 0; i < src_row_count; i++) {
-				idx_t target_index = offset + i;
-				result_ptr[target_index] =
-				    StringVector::AddStringOrBlob(result, src_ptr[i].GetDataUnsafe(), src_ptr[i].GetSize());
-			}
-
-			offset += src_row_count;
-		}
-
-		info->type = LogicalType::ENUM(info->name, result, total_row_count);
+		info->type = LogicalType::ENUM(info->name, g_sink_state.result, g_sink_state.size);
 	}
 
 	auto &catalog = Catalog::GetCatalog(context.client, info->catalog);
-	catalog.CreateType(context.client, info.get());
+	auto catalog_entry = catalog.CreateType(context.client, info.get());
+	D_ASSERT(catalog_entry->type == CatalogType::TYPE_ENTRY);
+	auto catalog_type = (TypeCatalogEntry *)catalog_entry;
+	LogicalType::SetCatalog(info->type, catalog_type);
 	state.finished = true;
 }
 

package/src/duckdb/src/execution/operator/set/physical_recursive_cte.cpp

@@ -42,6 +42,7 @@ public:
 	bool initialized = false;
 	bool finished_scan = false;
 	SelectionVector new_groups;
+	AggregateHTAppendState append_state;
 };
 
 unique_ptr<GlobalSinkState> PhysicalRecursiveCTE::GetGlobalSinkState(ClientContext &context) const {
@@ -52,7 +53,7 @@ idx_t PhysicalRecursiveCTE::ProbeHT(DataChunk &chunk, RecursiveCTEState &state)
 	Vector dummy_addresses(LogicalType::POINTER);
 
 	// Use the HT to eliminate duplicate rows
-	idx_t new_group_count = state.ht->FindOrCreateGroups(chunk, dummy_addresses, state.new_groups);
+	idx_t new_group_count = state.ht->FindOrCreateGroups(state.append_state, chunk, dummy_addresses, state.new_groups);
 
 	// we only return entries we have not seen before (i.e. new groups)
 	chunk.Slice(state.new_groups, new_group_count);
@@ -180,10 +181,6 @@ void PhysicalRecursiveCTE::BuildPipelines(Pipeline &current, MetaPipeline &meta_
 	auto &executor = meta_pipeline.GetExecutor();
 	executor.AddRecursiveCTE(this);
 
-	if (meta_pipeline.HasRecursiveCTE()) {
-		throw InternalException("Recursive CTE detected WITHIN a recursive CTE node");
-	}
-
 	// the LHS of the recursive CTE is our initial state
 	auto initial_state_pipeline = meta_pipeline.CreateChildMetaPipeline(current, this);
 	initial_state_pipeline->Build(children[0].get());

package/src/duckdb/src/execution/partitionable_hashtable.cpp

@@ -62,21 +62,35 @@ PartitionableHashTable::PartitionableHashTable(ClientContext &context, Allocator
 	for (hash_t r = 0; r < partition_info.n_partitions; r++) {
 		sel_vectors[r].Initialize();
 	}
+
+	RowLayout layout;
+	layout.Initialize(group_types, AggregateObject::CreateAggregateObjects(bindings));
+	tuple_size = layout.GetRowWidth();
+}
+
+HtEntryType PartitionableHashTable::GetHTEntrySize() {
+	// we need at least STANDARD_VECTOR_SIZE entries to fit in the hash table
+	if (GroupedAggregateHashTable::GetMaxCapacity(HtEntryType::HT_WIDTH_32, tuple_size) < STANDARD_VECTOR_SIZE) {
+		return HtEntryType::HT_WIDTH_64;
+	}
+	return HtEntryType::HT_WIDTH_32;
 }
 
 idx_t PartitionableHashTable::ListAddChunk(HashTableList &list, DataChunk &groups, Vector &group_hashes,
                                            DataChunk &payload, const vector<idx_t> &filter) {
 	// If this is false, a single AddChunk would overflow the max capacity
 	D_ASSERT(list.empty() || groups.size() <= list.back()->MaxCapacity());
-	if (list.empty() || list.back()->Size() + groups.size() > list.back()->MaxCapacity()) {
+	if (list.empty() || list.back()->Size() + groups.size() >= list.back()->MaxCapacity()) {
+		idx_t new_capacity = GroupedAggregateHashTable::InitialCapacity();
 		if (!list.empty()) {
+			new_capacity = list.back()->Capacity();
 			// early release first part of ht and prevent adding of more data
 			list.back()->Finalize();
 		}
 		list.push_back(make_unique<GroupedAggregateHashTable>(context, allocator, group_types, payload_types, bindings,
-		                                                      HtEntryType::HT_WIDTH_32));
+		                                                      GetHTEntrySize(), new_capacity));
 	}
-	return list.back()->AddChunk(groups, group_hashes, payload, filter);
+	return list.back()->AddChunk(append_state, groups, group_hashes, payload, filter);
 }
 
 idx_t PartitionableHashTable::AddChunk(DataChunk &groups, DataChunk &payload, bool do_partition,
@@ -138,10 +152,11 @@ void PartitionableHashTable::Partition() {
 	D_ASSERT(partition_info.n_partitions > 1);
 
 	vector<GroupedAggregateHashTable *> partition_hts(partition_info.n_partitions);
+	radix_partitioned_hts.resize(partition_info.n_partitions);
 	for (auto &unpartitioned_ht : unpartitioned_hts) {
 		for (idx_t r = 0; r < partition_info.n_partitions; r++) {
 			radix_partitioned_hts[r].push_back(make_unique<GroupedAggregateHashTable>(
-			    context, allocator, group_types, payload_types, bindings, HtEntryType::HT_WIDTH_32));
+			    context, allocator, group_types, payload_types, bindings, GetHTEntrySize()));
 			partition_hts[r] = radix_partitioned_hts[r].back().get();
 		}
 		unpartitioned_ht->Partition(partition_hts, partition_info.radix_mask, partition_info.RADIX_SHIFT);
@@ -169,7 +184,7 @@ HashTableList PartitionableHashTable::GetUnpartitioned() {
 void PartitionableHashTable::Finalize() {
 	if (IsPartitioned()) {
 		for (auto &ht_list : radix_partitioned_hts) {
-			for (auto &ht : ht_list.second) {
+			for (auto &ht : ht_list) {
 				D_ASSERT(ht);
 				ht->Finalize();
 			}