duckdb 0.7.2-dev1898.0 → 0.7.2-dev2144.0

package/binding.gyp

@@ -22,6 +22,8 @@
                 "src/duckdb/ub_src_common_serializer.cpp", 
                 "src/duckdb/ub_src_common_sort.cpp", 
                 "src/duckdb/ub_src_common_types.cpp", 
+                "src/duckdb/ub_src_common_types_column.cpp", 
+                "src/duckdb/ub_src_common_types_row.cpp", 
                 "src/duckdb/ub_src_common_value_operations.cpp", 
                 "src/duckdb/src/common/vector_operations/boolean_operators.cpp", 
                 "src/duckdb/src/common/vector_operations/comparison_operators.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.7.2-dev1898.0",
+  "version": "0.7.2-dev2144.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/data_chunk.cpp

@@ -145,7 +145,19 @@ Napi::Array EncodeDataChunk(Napi::Env env, duckdb::DataChunk &chunk, bool with_t
 				}
 				break;
 			}
-			case duckdb::LogicalTypeId::BLOB:
+			case duckdb::LogicalTypeId::BLOB: {
+				if (with_data) {
+					auto array = Napi::Array::New(env, chunk.size());
+					auto data = duckdb::FlatVector::GetData<duckdb::string_t>(*vec);
+
+					for (size_t i = 0; i < chunk.size(); ++i) {
+						auto buf = Napi::Buffer<char>::Copy(env, data[i].GetDataUnsafe(), data[i].GetSize());
+						array.Set(i, buf);
+					}
+					desc.Set("data", array);
+				}
+				break;
+			}
 			case duckdb::LogicalTypeId::VARCHAR: {
 				if (with_data) {
 					auto array = Napi::Array::New(env, chunk.size());

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -14,7 +14,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #endif
 
 #include "parquet_types.h"

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -1,12 +1,14 @@
 #include "parquet_metadata.hpp"
+
 #include "parquet_statistics.hpp"
+
 #include <sstream>
 
 #ifndef DUCKDB_AMALGAMATION
+#include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/types/blob.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/main/config.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
-#include "duckdb/common/multi_file_reader.hpp"
 #endif
 
 namespace duckdb {

package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp

@@ -13,7 +13,7 @@ DuckIndexEntry::~DuckIndexEntry() {
 	if (!info || !index) {
 		return;
 	}
-	info->indexes.RemoveIndex(index);
+	info->indexes.RemoveIndex(*index);
 }
 
 string DuckIndexEntry::GetSchemaName() const {

package/src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -12,7 +12,7 @@ namespace duckdb {
 // Arrow append data
 //===--------------------------------------------------------------------===//
 typedef void (*initialize_t)(ArrowAppendData &result, const LogicalType &type, idx_t capacity);
-typedef void (*append_vector_t)(ArrowAppendData &append_data, Vector &input, idx_t size);
+typedef void (*append_vector_t)(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size);
 typedef void (*finalize_t)(ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result);
 
 struct ArrowAppendData {
@@ -84,8 +84,9 @@ static void SetNull(ArrowAppendData &append_data, uint8_t *validity_data, idx_t
 	append_data.null_count++;
 }
 
-static void AppendValidity(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t size) {
+static void AppendValidity(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t from, idx_t to) {
 	// resize the buffer, filling the validity buffer with all valid values
+	idx_t size = to - from;
 	ResizeValidity(append_data.validity, append_data.row_count + size);
 	if (format.validity.AllValid()) {
 		// if all values are valid we don't need to do anything else
@@ -97,7 +98,7 @@ static void AppendValidity(ArrowAppendData &append_data, UnifiedVectorFormat &fo
 	uint8_t current_bit;
 	idx_t current_byte;
 	GetBitPosition(append_data.row_count, current_byte, current_bit);
-	for (idx_t i = 0; i < size; i++) {
+	for (idx_t i = from; i < to; i++) {
 		auto source_idx = format.sel->get_index(i);
 		// append the validity mask
 		if (!format.validity.RowIsValid(source_idx)) {
@@ -146,21 +147,22 @@ struct ArrowIntervalConverter {
 
 template <class TGT, class SRC = TGT, class OP = ArrowScalarConverter>
 struct ArrowScalarBaseData {
-	static void Append(ArrowAppendData &append_data, Vector &input, idx_t size) {
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		idx_t size = to - from;
 		UnifiedVectorFormat format;
-		input.ToUnifiedFormat(size, format);
+		input.ToUnifiedFormat(input_size, format);
 
 		// append the validity mask
-		AppendValidity(append_data, format, size);
+		AppendValidity(append_data, format, from, to);
 
 		// append the main data
 		append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(TGT) * size);
 		auto data = (SRC *)format.data;
 		auto result_data = (TGT *)append_data.main_buffer.data();
 
-		for (idx_t i = 0; i < size; i++) {
+		for (idx_t i = from; i < to; i++) {
 			auto source_idx = format.sel->get_index(i);
-			auto result_idx = append_data.row_count + i;
+			auto result_idx = append_data.row_count + i - from;
 
 			if (OP::SkipNulls() && !format.validity.RowIsValid(source_idx)) {
 				OP::template SetNull<TGT>(result_data[result_idx]);
@@ -254,9 +256,10 @@ struct ArrowBoolData {
 		result.main_buffer.reserve(byte_count);
 	}
 
-	static void Append(ArrowAppendData &append_data, Vector &input, idx_t size) {
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		idx_t size = to - from;
 		UnifiedVectorFormat format;
-		input.ToUnifiedFormat(size, format);
+		input.ToUnifiedFormat(input_size, format);
 
 		// we initialize both the validity and the bit set to 1's
 		ResizeValidity(append_data.validity, append_data.row_count + size);
@@ -268,7 +271,7 @@ struct ArrowBoolData {
 		uint8_t current_bit;
 		idx_t current_byte;
 		GetBitPosition(append_data.row_count, current_byte, current_bit);
-		for (idx_t i = 0; i < size; i++) {
+		for (idx_t i = from; i < to; i++) {
 			auto source_idx = format.sel->get_index(i);
 			// append the validity mask
 			if (!format.validity.RowIsValid(source_idx)) {
@@ -321,9 +324,10 @@ struct ArrowVarcharData {
 		result.aux_buffer.reserve(capacity);
 	}
 
-	static void Append(ArrowAppendData &append_data, Vector &input, idx_t size) {
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
+		idx_t size = to - from;
 		UnifiedVectorFormat format;
-		input.ToUnifiedFormat(size, format);
+		input.ToUnifiedFormat(input_size, format);
 
 		// resize the validity mask and set up the validity buffer for iteration
 		ResizeValidity(append_data.validity, append_data.row_count + size);
@@ -340,14 +344,14 @@ struct ArrowVarcharData {
 		// now append the string data to the auxiliary buffer
 		// the auxiliary buffer's length depends on the string lengths, so we resize as required
 		auto last_offset = offset_data[append_data.row_count];
-		for (idx_t i = 0; i < size; i++) {
+		for (idx_t i = from; i < to; i++) {
 			auto source_idx = format.sel->get_index(i);
-			auto offset_idx = append_data.row_count + i + 1;
+			auto offset_idx = append_data.row_count + i + 1 - from;
 
 			if (!format.validity.RowIsValid(source_idx)) {
 				uint8_t current_bit;
 				idx_t current_byte;
-				GetBitPosition(append_data.row_count + i, current_byte, current_bit);
+				GetBitPosition(append_data.row_count + i - from, current_byte, current_bit);
 				SetNull(append_data, validity_data, current_byte, current_bit);
 				offset_data[offset_idx] = last_offset;
 				continue;
@@ -387,17 +391,17 @@ struct ArrowStructData {
 		}
 	}
 
-	static void Append(ArrowAppendData &append_data, Vector &input, idx_t size) {
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
 		UnifiedVectorFormat format;
-		input.ToUnifiedFormat(size, format);
-
-		AppendValidity(append_data, format, size);
+		input.ToUnifiedFormat(input_size, format);
+		idx_t size = to - from;
+		AppendValidity(append_data, format, from, to);
 		// append the children of the struct
 		auto &children = StructVector::GetEntries(input);
 		for (idx_t child_idx = 0; child_idx < children.size(); child_idx++) {
 			auto &child = children[child_idx];
 			auto &child_data = *append_data.child_data[child_idx];
-			child_data.append_vector(child_data, *child, size);
+			child_data.append_vector(child_data, *child, from, to, size);
 		}
 		append_data.row_count += size;
 	}
@@ -419,9 +423,10 @@ struct ArrowStructData {
 //===--------------------------------------------------------------------===//
 // Lists
 //===--------------------------------------------------------------------===//
-void AppendListOffsets(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t size,
+void AppendListOffsets(ArrowAppendData &append_data, UnifiedVectorFormat &format, idx_t from, idx_t to,
                        vector<sel_t> &child_sel) {
 	// resize the offset buffer - the offset buffer holds the offsets into the child array
+	idx_t size = to - from;
 	append_data.main_buffer.resize(append_data.main_buffer.size() + sizeof(uint32_t) * (size + 1));
 	auto data = (list_entry_t *)format.data;
 	auto offset_data = (uint32_t *)append_data.main_buffer.data();
@@ -431,9 +436,9 @@ void AppendListOffsets(ArrowAppendData &append_data, UnifiedVectorFormat &format
 	}
 	// set up the offsets using the list entries
 	auto last_offset = offset_data[append_data.row_count];
-	for (idx_t i = 0; i < size; i++) {
+	for (idx_t i = from; i < to; i++) {
 		auto source_idx = format.sel->get_index(i);
-		auto offset_idx = append_data.row_count + i + 1;
+		auto offset_idx = append_data.row_count + i + 1 - from;
 
 		if (!format.validity.RowIsValid(source_idx)) {
 			offset_data[offset_idx] = last_offset;
@@ -459,21 +464,28 @@ struct ArrowListData {
 		result.child_data.push_back(std::move(child_buffer));
 	}
 
-	static void Append(ArrowAppendData &append_data, Vector &input, idx_t size) {
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
 		UnifiedVectorFormat format;
-		input.ToUnifiedFormat(size, format);
-
+		input.ToUnifiedFormat(input_size, format);
+		idx_t size = to - from;
 		vector<sel_t> child_indices;
-		AppendValidity(append_data, format, size);
-		AppendListOffsets(append_data, format, size, child_indices);
+		AppendValidity(append_data, format, from, to);
+		AppendListOffsets(append_data, format, from, to, child_indices);
 
 		// append the child vector of the list
 		SelectionVector child_sel(child_indices.data());
 		auto &child = ListVector::GetEntry(input);
 		auto child_size = child_indices.size();
-		child.Slice(child_sel, child_size);
-
-		append_data.child_data[0]->append_vector(*append_data.child_data[0], child, child_size);
+		if (size != input_size) {
+			// Let's avoid doing this
+			Vector child_copy(child.GetType());
+			child_copy.Slice(child, child_sel, child_size);
+			append_data.child_data[0]->append_vector(*append_data.child_data[0], child_copy, 0, child_size, child_size);
+		} else {
+			// We don't care about the vector, slice it
+			child.Slice(child_sel, child_size);
+			append_data.child_data[0]->append_vector(*append_data.child_data[0], child, 0, child_size, child_size);
+		}
 		append_data.row_count += size;
 	}
 
@@ -508,26 +520,39 @@ struct ArrowMapData {
 		result.child_data.push_back(std::move(internal_struct));
 	}
 
-	static void Append(ArrowAppendData &append_data, Vector &input, idx_t size) {
+	static void Append(ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size) {
 		UnifiedVectorFormat format;
-		input.ToUnifiedFormat(size, format);
-
-		AppendValidity(append_data, format, size);
+		input.ToUnifiedFormat(input_size, format);
+		idx_t size = to - from;
+		AppendValidity(append_data, format, from, to);
 		vector<sel_t> child_indices;
-		AppendListOffsets(append_data, format, size, child_indices);
+		AppendListOffsets(append_data, format, from, to, child_indices);
 
 		SelectionVector child_sel(child_indices.data());
 		auto &key_vector = MapVector::GetKeys(input);
 		auto &value_vector = MapVector::GetValues(input);
 		auto list_size = child_indices.size();
-		key_vector.Slice(child_sel, list_size);
-		value_vector.Slice(child_sel, list_size);
 
 		auto &struct_data = *append_data.child_data[0];
 		auto &key_data = *struct_data.child_data[0];
 		auto &value_data = *struct_data.child_data[1];
-		key_data.append_vector(key_data, key_vector, list_size);
-		value_data.append_vector(value_data, value_vector, list_size);
+
+		if (size != input_size) {
+			// Let's avoid doing this
+			Vector key_vector_copy(key_vector.GetType());
+			key_vector_copy.Slice(key_vector, child_sel, list_size);
+			Vector value_vector_copy(value_vector.GetType());
+			value_vector_copy.Slice(value_vector, child_sel, list_size);
+			key_data.append_vector(key_data, key_vector_copy, 0, list_size, list_size);
+			value_data.append_vector(value_data, value_vector_copy, 0, list_size, list_size);
+		} else {
+			// We don't care about the vector, slice it
+			key_vector.Slice(child_sel, list_size);
+			value_vector.Slice(child_sel, list_size);
+			key_data.append_vector(key_data, key_vector, 0, list_size, list_size);
+			value_data.append_vector(value_data, value_vector, 0, list_size, list_size);
+		}
+
 		append_data.row_count += size;
 		struct_data.row_count += size;
 	}
@@ -567,12 +592,12 @@ struct ArrowMapData {
 };
 
 //! Append a data chunk to the underlying arrow array
-void ArrowAppender::Append(DataChunk &input) {
+void ArrowAppender::Append(DataChunk &input, idx_t from, idx_t to, idx_t input_size) {
 	D_ASSERT(types == input.GetTypes());
 	for (idx_t i = 0; i < input.ColumnCount(); i++) {
-		root_data[i]->append_vector(*root_data[i], input.data[i], input.size());
+		root_data[i]->append_vector(*root_data[i], input.data[i], from, to, input_size);
 	}
-	row_count += input.size();
+	row_count += to - from;
 }
 //===--------------------------------------------------------------------===//
 // Initialize Arrow Child

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -17,7 +17,7 @@ namespace duckdb {
 
 void ArrowConverter::ToArrowArray(DataChunk &input, ArrowArray *out_array) {
 	ArrowAppender appender(input.GetTypes(), input.size());
-	appender.Append(input);
+	appender.Append(input, 0, input.size(), input.size());
 	*out_array = appender.Finalize();
 }
 

package/src/duckdb/src/common/arrow/arrow_wrapper.cpp

@@ -187,6 +187,15 @@ bool ArrowUtil::TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray
                               PreservedError &error) {
 	count = 0;
 	ArrowAppender appender(result->types, chunk_size);
+	auto &current_chunk = result->current_chunk;
+	if (current_chunk.Valid()) {
+		// We start by scanning the non-finished current chunk
+		idx_t cur_consumption = current_chunk.RemainingSize() > chunk_size ? chunk_size : current_chunk.RemainingSize();
+		count += cur_consumption;
+		appender.Append(*current_chunk.data_chunk, current_chunk.position, current_chunk.position + cur_consumption,
+		                current_chunk.data_chunk->size());
+		current_chunk.position += cur_consumption;
+	}
 	while (count < chunk_size) {
 		unique_ptr<DataChunk> data_chunk;
 		if (!TryFetchNext(*result, data_chunk, error)) {
@@ -198,8 +207,17 @@ bool ArrowUtil::TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray
 		if (!data_chunk || data_chunk->size() == 0) {
 			break;
 		}
-		count += data_chunk->size();
-		appender.Append(*data_chunk);
+		if (count + data_chunk->size() > chunk_size) {
+			// We have to split the chunk between this and the next batch
+			idx_t available_space = chunk_size - count;
+			appender.Append(*data_chunk, 0, available_space, data_chunk->size());
+			count += available_space;
+			current_chunk.data_chunk = std::move(data_chunk);
+			current_chunk.position = available_space;
+		} else {
+			count += data_chunk->size();
+			appender.Append(*data_chunk, 0, data_chunk->size(), data_chunk->size());
+		}
 	}
 	if (count > 0) {
 		*out = appender.Finalize();

package/src/duckdb/src/common/box_renderer.cpp

@@ -1,8 +1,10 @@
 #include "duckdb/common/box_renderer.hpp"
-#include "duckdb/common/types/column_data_collection.hpp"
+
 #include "duckdb/common/printer.hpp"
-#include "utf8proc_wrapper.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "utf8proc_wrapper.hpp"
+
 #include <sstream>
 
 namespace duckdb {

package/src/duckdb/src/common/constants.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/common/constants.hpp"
-#include "duckdb/common/vector_size.hpp"
+
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/vector_size.hpp"
 
 namespace duckdb {
 
@@ -15,6 +16,10 @@ const transaction_t MAX_TRANSACTION_ID = NumericLimits<transaction_t>::Maximum()
 const transaction_t NOT_DELETED_ID = NumericLimits<transaction_t>::Maximum() - 1; // 2^64 - 1
 const transaction_t MAXIMUM_QUERY_ID = NumericLimits<transaction_t>::Maximum();   // 2^64
 
+bool IsPowerOfTwo(uint64_t v) {
+	return (v & (v - 1)) == 0;
+}
+
 uint64_t NextPowerOfTwo(uint64_t v) {
 	v--;
 	v |= v >> 1;
@@ -27,6 +32,10 @@ uint64_t NextPowerOfTwo(uint64_t v) {
 	return v;
 }
 
+uint64_t PreviousPowerOfTwo(uint64_t v) {
+	return NextPowerOfTwo((v / 2) + 1);
+}
+
 bool IsInvalidSchema(const string &str) {
 	return str.empty();
 }

package/src/duckdb/src/common/filename_pattern.cpp

@@ -0,0 +1,41 @@
+#include "duckdb/common/filename_pattern.hpp"
+#include "duckdb/common/string_util.hpp"
+
+namespace duckdb {
+
+void FilenamePattern::SetFilenamePattern(const string &pattern) {
+	const string id_format {"{i}"};
+	const string uuid_format {"{uuid}"};
+
+	_base = pattern;
+
+	_pos = _base.find(id_format);
+	if (_pos != string::npos) {
+		_base = StringUtil::Replace(_base, id_format, "");
+		_uuid = false;
+	}
+
+	_pos = _base.find(uuid_format);
+	if (_pos != string::npos) {
+		_base = StringUtil::Replace(_base, uuid_format, "");
+		_uuid = true;
+	}
+
+	_pos = std::min(_pos, (idx_t)_base.length());
+}
+
+string FilenamePattern::CreateFilename(const FileSystem &fs, const string &path, const string &extension,
+                                       idx_t offset) const {
+	string result(_base);
+	string replacement;
+
+	if (_uuid) {
+		replacement = UUID::ToString(UUID::GenerateRandomUUID());
+	} else {
+		replacement = std::to_string(offset);
+	}
+	result.insert(_pos, replacement);
+	return fs.JoinPath(path, result + "." + extension);
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -1,11 +1,12 @@
 #include "duckdb/common/hive_partitioning.hpp"
-#include "duckdb/planner/table_filter.hpp"
+
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/optimizer/filter_combiner.hpp"
-#include "duckdb/planner/expression_iterator.hpp"
-#include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
+#include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/table_filter.hpp"
 #include "re2/re2.h"
 
 namespace duckdb {
@@ -140,29 +141,157 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
 }
 
 HivePartitionedColumnData::HivePartitionedColumnData(const HivePartitionedColumnData &other)
-    : PartitionedColumnData(other) {
+    : PartitionedColumnData(other), hashes_v(LogicalType::HASH) {
 	// Synchronize to ensure consistency of shared partition map
 	if (other.global_state) {
 		global_state = other.global_state;
 		unique_lock<mutex> lck(global_state->lock);
 		SynchronizeLocalMap();
 	}
+	InitializeKeys();
 }
 
-void HivePartitionedColumnData::ComputePartitionIndices(PartitionedColumnDataAppendState &state, DataChunk &input) {
-	Vector hashes(LogicalType::HASH, input.size());
-	input.Hash(group_by_columns, hashes);
-	hashes.Flatten(input.size());
-
-	for (idx_t i = 0; i < input.size(); i++) {
-		HivePartitionKey key;
-		key.hash = FlatVector::GetData<hash_t>(hashes)[i];
-		for (auto &col : group_by_columns) {
-			key.values.emplace_back(input.GetValue(col, i));
+void HivePartitionedColumnData::InitializeKeys() {
+	keys.resize(STANDARD_VECTOR_SIZE);
+	for (idx_t i = 0; i < STANDARD_VECTOR_SIZE; i++) {
+		keys[i].values.resize(group_by_columns.size());
+	}
+}
+
+template <class T>
+static inline Value GetHiveKeyValue(const T &val) {
+	return Value::CreateValue<T>(val);
+}
+
+template <class T>
+static inline Value GetHiveKeyValue(const T &val, const LogicalType &type) {
+	auto result = GetHiveKeyValue(val);
+	result.Reinterpret(type);
+	return result;
+}
+
+static inline Value GetHiveKeyNullValue(const LogicalType &type) {
+	Value result;
+	result.Reinterpret(type);
+	return result;
+}
+
+template <class T>
+static void TemplatedGetHivePartitionValues(Vector &input, vector<HivePartitionKey> &keys, const idx_t col_idx,
+                                            const idx_t count) {
+	UnifiedVectorFormat format;
+	input.ToUnifiedFormat(count, format);
+
+	const auto &sel = *format.sel;
+	const auto data = (T *)format.data;
+	const auto &validity = format.validity;
+
+	const auto &type = input.GetType();
+
+	const auto reinterpret = Value::CreateValue<T>(data[0]).GetTypeMutable() != type;
+	if (reinterpret) {
+		for (idx_t i = 0; i < count; i++) {
+			auto &key = keys[i];
+			const auto idx = sel.get_index(i);
+			if (validity.RowIsValid(idx)) {
+				key.values[col_idx] = GetHiveKeyValue(data[idx], type);
+			} else {
+				key.values[col_idx] = GetHiveKeyNullValue(type);
+			}
+		}
+	} else {
+		for (idx_t i = 0; i < count; i++) {
+			auto &key = keys[i];
+			const auto idx = sel.get_index(i);
+			if (validity.RowIsValid(idx)) {
+				key.values[col_idx] = GetHiveKeyValue(data[idx]);
+			} else {
+				key.values[col_idx] = GetHiveKeyNullValue(type);
+			}
 		}
+	}
+}
+
+static void GetNestedHivePartitionValues(Vector &input, vector<HivePartitionKey> &keys, const idx_t col_idx,
+                                         const idx_t count) {
+	for (idx_t i = 0; i < count; i++) {
+		auto &key = keys[i];
+		key.values[col_idx] = input.GetValue(i);
+	}
+}
+
+static void GetHivePartitionValuesTypeSwitch(Vector &input, vector<HivePartitionKey> &keys, const idx_t col_idx,
+                                             const idx_t count) {
+	const auto &type = input.GetType();
+	switch (type.InternalType()) {
+	case PhysicalType::BOOL:
+		TemplatedGetHivePartitionValues<bool>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::INT8:
+		TemplatedGetHivePartitionValues<int8_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::INT16:
+		TemplatedGetHivePartitionValues<int16_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::INT32:
+		TemplatedGetHivePartitionValues<int32_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::INT64:
+		TemplatedGetHivePartitionValues<int64_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::INT128:
+		TemplatedGetHivePartitionValues<hugeint_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::UINT8:
+		TemplatedGetHivePartitionValues<uint8_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::UINT16:
+		TemplatedGetHivePartitionValues<uint16_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::UINT32:
+		TemplatedGetHivePartitionValues<uint32_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::UINT64:
+		TemplatedGetHivePartitionValues<uint64_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::FLOAT:
+		TemplatedGetHivePartitionValues<float>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::DOUBLE:
+		TemplatedGetHivePartitionValues<double>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::INTERVAL:
+		TemplatedGetHivePartitionValues<interval_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::VARCHAR:
+		TemplatedGetHivePartitionValues<string_t>(input, keys, col_idx, count);
+		break;
+	case PhysicalType::STRUCT:
+	case PhysicalType::LIST:
+		GetNestedHivePartitionValues(input, keys, col_idx, count);
+		break;
+	default:
+		throw InternalException("Unsupported type for HivePartitionedColumnData::ComputePartitionIndices");
+	}
+}
+
+void HivePartitionedColumnData::ComputePartitionIndices(PartitionedColumnDataAppendState &state, DataChunk &input) {
+	const auto count = input.size();
+
+	input.Hash(group_by_columns, hashes_v);
+	hashes_v.Flatten(count);
+
+	for (idx_t col_idx = 0; col_idx < group_by_columns.size(); col_idx++) {
+		auto &group_by_col = input.data[group_by_columns[col_idx]];
+		GetHivePartitionValuesTypeSwitch(group_by_col, keys, col_idx, count);
+	}
 
+	const auto hashes = FlatVector::GetData<hash_t>(hashes_v);
+	const auto partition_indices = FlatVector::GetData<idx_t>(state.partition_indices);
+	for (idx_t i = 0; i < count; i++) {
+		auto &key = keys[i];
+		key.hash = hashes[i];
 		auto lookup = local_partition_map.find(key);
-		const auto partition_indices = FlatVector::GetData<idx_t>(state.partition_indices);
 		if (lookup == local_partition_map.end()) {
 			idx_t new_partition_id = RegisterNewPartition(key, state);
 			partition_indices[i] = new_partition_id;