duckdb 0.8.2-dev33.0 → 0.8.2-dev3300.0

package/src/duckdb/src/include/duckdb/common/operator/subtract.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/type_util.hpp"
+#include "duckdb/common/types/cast_helpers.hpp"
 
 namespace duckdb {
 
@@ -75,8 +76,8 @@ struct SubtractOperatorOverflowCheck {
 	static inline TR Operation(TA left, TB right) {
 		TR result;
 		if (!TrySubtractOperator::Operation(left, right, result)) {
-			throw OutOfRangeException("Overflow in subtraction of %s (%d - %d)!", TypeIdToString(GetTypeId<TA>()), left,
-			                          right);
+			throw OutOfRangeException("Overflow in subtraction of %s (%s - %s)!", TypeIdToString(GetTypeId<TA>()),
+			                          NumericHelper::ToString(left), NumericHelper::ToString(right));
 		}
 		return result;
 	}

package/src/duckdb/src/include/duckdb/common/radix.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "duckdb/common/bswap.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types.hpp"
@@ -20,18 +21,6 @@
 
 namespace duckdb {
 
-#define BSWAP16(x) ((uint16_t)((((uint16_t)(x)&0xff00) >> 8) | (((uint16_t)(x)&0x00ff) << 8)))
-
-#define BSWAP32(x)                                                                                                     \
-	((uint32_t)((((uint32_t)(x)&0xff000000) >> 24) | (((uint32_t)(x)&0x00ff0000) >> 8) |                               \
-	            (((uint32_t)(x)&0x0000ff00) << 8) | (((uint32_t)(x)&0x000000ff) << 24)))
-
-#define BSWAP64(x)                                                                                                     \
-	((uint64_t)((((uint64_t)(x)&0xff00000000000000ull) >> 56) | (((uint64_t)(x)&0x00ff000000000000ull) >> 40) |        \
-	            (((uint64_t)(x)&0x0000ff0000000000ull) >> 24) | (((uint64_t)(x)&0x000000ff00000000ull) >> 8) |         \
-	            (((uint64_t)(x)&0x00000000ff000000ull) << 8) | (((uint64_t)(x)&0x0000000000ff0000ull) << 24) |         \
-	            (((uint64_t)(x)&0x000000000000ff00ull) << 40) | (((uint64_t)(x)&0x00000000000000ffull) << 56)))
-
 struct Radix {
 public:
 	static inline bool IsLittleEndian() {
@@ -134,19 +123,19 @@ inline void Radix::EncodeData(data_ptr_t dataptr, int8_t value) {
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, int16_t value) {
-	Store<uint16_t>(BSWAP16(value), dataptr);
+	Store<uint16_t>(BSwap<uint16_t>(value), dataptr);
 	dataptr[0] = FlipSign(dataptr[0]);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, int32_t value) {
-	Store<uint32_t>(BSWAP32(value), dataptr);
+	Store<uint32_t>(BSwap<uint32_t>(value), dataptr);
 	dataptr[0] = FlipSign(dataptr[0]);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, int64_t value) {
-	Store<uint64_t>(BSWAP64(value), dataptr);
+	Store<uint64_t>(BSwap<uint64_t>(value), dataptr);
 	dataptr[0] = FlipSign(dataptr[0]);
 }
 
@@ -157,17 +146,17 @@ inline void Radix::EncodeData(data_ptr_t dataptr, uint8_t value) {
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, uint16_t value) {
-	Store<uint16_t>(BSWAP16(value), dataptr);
+	Store<uint16_t>(BSwap<uint16_t>(value), dataptr);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, uint32_t value) {
-	Store<uint32_t>(BSWAP32(value), dataptr);
+	Store<uint32_t>(BSwap<uint32_t>(value), dataptr);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, uint64_t value) {
-	Store<uint64_t>(BSWAP64(value), dataptr);
+	Store<uint64_t>(BSwap<uint64_t>(value), dataptr);
 }
 
 template <>
@@ -179,13 +168,13 @@ inline void Radix::EncodeData(data_ptr_t dataptr, hugeint_t value) {
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, float value) {
 	uint32_t converted_value = EncodeFloat(value);
-	Store<uint32_t>(BSWAP32(converted_value), dataptr);
+	Store<uint32_t>(BSwap<uint32_t>(converted_value), dataptr);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, double value) {
 	uint64_t converted_value = EncodeDouble(value);
-	Store<uint64_t>(BSWAP64(converted_value), dataptr);
+	Store<uint64_t>(BSwap<uint64_t>(converted_value), dataptr);
 }
 
 template <>

package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp

@@ -22,6 +22,9 @@ struct SelectionVector;
 //! Generic radix partitioning functions
 struct RadixPartitioning {
 public:
+	//! 4096 partitions ought to be enough to go out-of-core properly
+	static constexpr const idx_t MAX_RADIX_BITS = 12;
+
 	//! The number of partitions for a given number of radix bits
 	static inline constexpr idx_t NumberOfPartitions(idx_t radix_bits) {
 		return idx_t(1) << radix_bits;
@@ -38,10 +41,12 @@ public:
 		throw InternalException("RadixPartitioning::RadixBits unable to find partition count!");
 	}
 
+	//! Radix bits begin after uint16_t because these bits are used as salt in the aggregate HT
 	static inline constexpr idx_t Shift(idx_t radix_bits) {
-		return 48 - radix_bits;
+		return (sizeof(hash_t) - sizeof(uint16_t)) * 8 - radix_bits;
 	}
 
+	//! Mask of the radix bits of the hash
 	static inline constexpr hash_t Mask(idx_t radix_bits) {
 		return (hash_t(1 << radix_bits) - 1) << Shift(radix_bits);
 	}
@@ -49,26 +54,6 @@ public:
 	//! Select using a cutoff on the radix bits of the hash
 	static idx_t Select(Vector &hashes, const SelectionVector *sel, idx_t count, idx_t radix_bits, idx_t cutoff,
 	                    SelectionVector *true_sel, SelectionVector *false_sel);
-
-	//! Convert hashes to bins
-	static void HashesToBins(Vector &hashes, idx_t radix_bits, Vector &bins, idx_t count);
-};
-
-//! Templated radix partitioning constants, can be templated to the number of radix bits
-template <idx_t radix_bits>
-struct RadixPartitioningConstants {
-public:
-	//! Bitmask of the upper bits of the 5th byte
-	static constexpr const idx_t NUM_PARTITIONS = RadixPartitioning::NumberOfPartitions(radix_bits);
-	static constexpr const idx_t SHIFT = RadixPartitioning::Shift(radix_bits);
-	static constexpr const hash_t MASK = RadixPartitioning::Mask(radix_bits);
-
-public:
-	//! Apply bitmask and right shift to get a number between 0 and NUM_PARTITIONS
-	static inline hash_t ApplyMask(hash_t hash) {
-		D_ASSERT((hash & MASK) >> SHIFT < NUM_PARTITIONS);
-		return (hash & MASK) >> SHIFT;
-	}
 };
 
 //! RadixPartitionedColumnData is a PartitionedColumnData that partitions input based on the radix of a hash

package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp

@@ -13,7 +13,7 @@
 
 namespace duckdb {
 
-class Allocator;
+class ArenaAllocator;
 struct AggregateObject;
 struct AggregateFilterData;
 class DataChunk;
@@ -26,10 +26,10 @@ class Vector;
 struct UnifiedVectorFormat;
 
 struct RowOperationsState {
-	RowOperationsState(Allocator &allocator) : allocator(allocator) {
+	explicit RowOperationsState(ArenaAllocator &allocator) : allocator(allocator) {
 	}
 
-	Allocator &allocator;
+	ArenaAllocator &allocator;
 };
 
 // RowOperations contains a set of operations that operate on data using a RowLayout

package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp

@@ -1,17 +1,41 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/binary_deserializer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
+
 #include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
+class ClientContext;
 
 class BinaryDeserializer : public FormatDeserializer {
 public:
+	template <class T>
+	unique_ptr<T> Deserialize() {
+		OnObjectBegin();
+		auto result = T::FormatDeserialize(*this);
+		OnObjectEnd();
+		return result;
+	}
+
 	template <class T>
 	static unique_ptr<T> Deserialize(data_ptr_t ptr, idx_t length) {
 		BinaryDeserializer deserializer(ptr, length);
-		deserializer.OnObjectBegin();
-		auto result = T::FormatDeserialize(deserializer);
-		deserializer.OnObjectEnd();
-		return result;
+		return deserializer.template Deserialize<T>();
+	}
+
+	template <class T>
+	static unique_ptr<T> Deserialize(ClientContext &context, bound_parameter_map_t &parameters, data_ptr_t ptr,
+	                                 idx_t length) {
+		BinaryDeserializer deserializer(ptr, length);
+		deserializer.Set<ClientContext &>(context);
+		deserializer.Set<bound_parameter_map_t &>(parameters);
+		return deserializer.template Deserialize<T>();
 	}
 
 private:
@@ -21,13 +45,17 @@ private:
 	struct State {
 		uint32_t expected_field_count;
 		idx_t expected_size;
+		field_id_t expected_field_id;
 		uint32_t read_field_count;
-		State(uint32_t expected_field_count, idx_t expected_size)
-		    : expected_field_count(expected_field_count), expected_size(expected_size), read_field_count(0) {
+
+		State(uint32_t expected_field_count, idx_t expected_size, field_id_t expected_field_id)
+		    : expected_field_count(expected_field_count), expected_size(expected_size),
+		      expected_field_id(expected_field_id), read_field_count(0) {
 		}
 	};
 
 	const char *current_tag = nullptr;
+	field_id_t current_field_id = 0;
 	data_ptr_t ptr;
 	data_ptr_t end_ptr;
 	vector<State> stack;
@@ -48,7 +76,7 @@ private:
 	}
 
 	// Set the 'tag' of the property to read
-	void SetTag(const char *tag) final;
+	void SetTag(const field_id_t field_id, const char *tag) final;
 
 	//===--------------------------------------------------------------------===//
 	// Nested Types Hooks

package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp

@@ -1,3 +1,11 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/binary_serializer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
 #include "duckdb/common/serializer/format_serializer.hpp"
@@ -5,7 +13,6 @@
 namespace duckdb {
 
 struct BinarySerializer : public FormatSerializer {
-
 private:
 	struct State {
 		// how many fields are present in the object
@@ -17,6 +24,7 @@ private:
 	};
 
 	const char *current_tag;
+	field_id_t current_field_id = 0;
 
 	vector<data_t> data;
 	vector<State> stack;
@@ -24,14 +32,14 @@ private:
 	template <class T>
 	void Write(T element) {
 		static_assert(std::is_trivially_destructible<T>(), "Write element must be trivially destructible");
-		WriteData(const_data_ptr_cast(&element), sizeof(T));
+		WriteDataInternal(const_data_ptr_cast(&element), sizeof(T));
 	}
-	void WriteData(const_data_ptr_t buffer, idx_t write_size) {
+	void WriteDataInternal(const_data_ptr_t buffer, idx_t write_size) {
 		data.insert(data.end(), buffer, buffer + write_size);
 		stack.back().size += write_size;
 	}
-	void WriteData(const char *ptr, idx_t write_size) {
-		WriteData(const_data_ptr_cast(ptr), write_size);
+	void WriteDataInternal(const char *ptr, idx_t write_size) {
+		WriteDataInternal(const_data_ptr_cast(ptr), write_size);
 	}
 
 	explicit BinarySerializer() {
@@ -48,7 +56,7 @@ public:
 		return std::move(serializer.data);
 	}
 
-	void SetTag(const char *tag) final;
+	void SetTag(const field_id_t field_id, const char *tag) final;
 
 	//===--------------------------------------------------------------------===//
 	// Nested Types Hooks

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp

@@ -37,14 +37,10 @@ public:
 
 	ClientContext &GetContext() override;
 
-	optional_ptr<Catalog> GetCatalog() override;
-	void SetCatalog(Catalog &catalog);
-
 private:
 	idx_t file_size;
 	idx_t total_read;
 	optional_ptr<ClientContext> context;
-	optional_ptr<Catalog> catalog;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp

@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/deserialization_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/stack.hpp"
+#include "duckdb/planner/bound_parameter_map.hpp"
+
+namespace duckdb {
+class ClientContext;
+class Catalog;
+enum class ExpressionType : uint8_t;
+
+struct DeserializationData {
+	stack<reference<ClientContext>> contexts;
+	stack<idx_t> enums;
+	stack<reference<bound_parameter_map_t>> parameter_data;
+
+	template <class T>
+	void Set(T entry) = delete;
+
+	template <class T>
+	T Get() = delete;
+
+	template <class T>
+	void Unset() = delete;
+
+	template <class T>
+	inline void AssertNotEmpty(const stack<T> &e) {
+		if (e.empty()) {
+			throw InternalException("DeserializationData - unexpected empty stack");
+		}
+	}
+};
+
+template <>
+inline void DeserializationData::Set(ExpressionType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline ExpressionType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return ExpressionType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<ExpressionType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(LogicalOperatorType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline LogicalOperatorType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return LogicalOperatorType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<LogicalOperatorType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(ClientContext &context) {
+	contexts.push(context);
+}
+
+template <>
+inline ClientContext &DeserializationData::Get() {
+	AssertNotEmpty(contexts);
+	return contexts.top();
+}
+
+template <>
+inline void DeserializationData::Unset<ClientContext>() {
+	AssertNotEmpty(contexts);
+	contexts.pop();
+}
+
+template <>
+inline void DeserializationData::Set(bound_parameter_map_t &context) {
+	parameter_data.push(context);
+}
+
+template <>
+inline bound_parameter_map_t &DeserializationData::Get() {
+	AssertNotEmpty(parameter_data);
+	return parameter_data.top();
+}
+
+template <>
+inline void DeserializationData::Unset<bound_parameter_map_t>() {
+	AssertNotEmpty(parameter_data);
+	parameter_data.pop();
+}
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/serializer.hpp"
 #include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/serializer/serialization_traits.hpp"
+#include "duckdb/common/serializer/deserialization_data.hpp"
 #include "duckdb/common/types/interval.hpp"
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/common/unordered_map.hpp"
@@ -24,26 +25,27 @@ class FormatDeserializer {
 
 protected:
 	bool deserialize_enum_from_string = false;
+	DeserializationData data;
 
 public:
 	// Read into an existing value
 	template <typename T>
-	inline void ReadProperty(const char *tag, T &ret) {
-		SetTag(tag);
+	inline void ReadProperty(const field_id_t field_id, const char *tag, T &ret) {
+		SetTag(field_id, tag);
 		ret = Read<T>();
 	}
 
 	// Read and return a value
 	template <typename T>
-	inline T ReadProperty(const char *tag) {
-		SetTag(tag);
+	inline T ReadProperty(const field_id_t field_id, const char *tag) {
+		SetTag(field_id, tag);
 		return Read<T>();
 	}
 
 	// Read optional property and return a value, or forward a default value
 	template <typename T>
-	inline T ReadOptionalPropertyOrDefault(const char *tag, T &&default_value) {
-		SetTag(tag);
+	inline T ReadOptionalPropertyOrDefault(const field_id_t field_id, const char *tag, T &&default_value) {
+		SetTag(field_id, tag);
 		auto present = OnOptionalBegin();
 		if (present) {
 			auto item = Read<T>();
@@ -57,8 +59,8 @@ public:
 
 	// Read optional property into an existing value, or use a default value
 	template <typename T>
-	inline void ReadOptionalPropertyOrDefault(const char *tag, T &ret, T &&default_value) {
-		SetTag(tag);
+	inline void ReadOptionalPropertyOrDefault(const field_id_t field_id, const char *tag, T &ret, T &&default_value) {
+		SetTag(field_id, tag);
 		auto present = OnOptionalBegin();
 		if (present) {
 			ret = Read<T>();
@@ -72,8 +74,8 @@ public:
 	// Read optional property and return a value, or default construct it
 	template <typename T>
 	inline typename std::enable_if<std::is_default_constructible<T>::value, T>::type
-	ReadOptionalProperty(const char *tag) {
-		SetTag(tag);
+	ReadOptionalProperty(const field_id_t field_id, const char *tag) {
+		SetTag(field_id, tag);
 		auto present = OnOptionalBegin();
 		if (present) {
 			auto item = Read<T>();
@@ -88,8 +90,8 @@ public:
 	// Read optional property into an existing value, or default construct it
 	template <typename T>
 	inline typename std::enable_if<std::is_default_constructible<T>::value, void>::type
-	ReadOptionalProperty(const char *tag, T &ret) {
-		SetTag(tag);
+	ReadOptionalProperty(const field_id_t field_id, const char *tag, T &ret) {
+		SetTag(field_id, tag);
 		auto present = OnOptionalBegin();
 		if (present) {
 			ret = Read<T>();
@@ -102,11 +104,39 @@ public:
 
 	// Special case:
 	// Read into an existing data_ptr_t
-	inline void ReadProperty(const char *tag, data_ptr_t ret, idx_t count) {
-		SetTag(tag);
+	inline void ReadProperty(const field_id_t field_id, const char *tag, data_ptr_t ret, idx_t count) {
+		SetTag(field_id, tag);
 		ReadDataPtr(ret, count);
 	}
 
+	//! Set a serialization property
+	template <class T>
+	void Set(T entry) {
+		return data.Set<T>(entry);
+	}
+
+	//! Retrieve the last set serialization property of this type
+	template <class T>
+	T Get() {
+		return data.Get<T>();
+	}
+
+	//! Unset a serialization property
+	template <class T>
+	void Unset() {
+		return data.Unset<T>();
+	}
+
+	// Manually begin an object - should be followed by EndObject
+	void BeginObject(const field_id_t field_id, const char *tag) {
+		SetTag(field_id, tag);
+		OnObjectBegin();
+	}
+
+	void EndObject() {
+		OnObjectEnd();
+	}
+
 private:
 	// Deserialize anything implementing a FormatDeserialize method
 	template <typename T = void>
@@ -152,6 +182,19 @@ private:
 		return vec;
 	}
 
+	template <typename T = void>
+	inline typename std::enable_if<is_unsafe_vector<T>::value, T>::type Read() {
+		using ELEMENT_TYPE = typename is_unsafe_vector<T>::ELEMENT_TYPE;
+		T vec;
+		auto size = OnListBegin();
+		for (idx_t i = 0; i < size; i++) {
+			vec.push_back(Read<ELEMENT_TYPE>());
+		}
+		OnListEnd();
+
+		return vec;
+	}
+
 	// Deserialize a map
 	template <typename T = void>
 	inline typename std::enable_if<is_unordered_map<T>::value, T>::type Read() {
@@ -175,6 +218,28 @@ private:
 		return map;
 	}
 
+	template <typename T = void>
+	inline typename std::enable_if<is_map<T>::value, T>::type Read() {
+		using KEY_TYPE = typename is_map<T>::KEY_TYPE;
+		using VALUE_TYPE = typename is_map<T>::VALUE_TYPE;
+
+		T map;
+		auto size = OnMapBegin();
+		for (idx_t i = 0; i < size; i++) {
+			OnMapEntryBegin();
+			OnMapKeyBegin();
+			auto key = Read<KEY_TYPE>();
+			OnMapKeyEnd();
+			OnMapValueBegin();
+			auto value = Read<VALUE_TYPE>();
+			OnMapValueEnd();
+			OnMapEntryEnd();
+			map[std::move(key)] = std::move(value);
+		}
+		OnMapEnd();
+		return map;
+	}
+
 	// Deserialize an unordered set
 	template <typename T = void>
 	inline typename std::enable_if<is_unordered_set<T>::value, T>::type Read() {
@@ -308,14 +373,27 @@ private:
 		return ReadInterval();
 	}
 
-	// Deserialize a interval_t
+	// Deserialize a hugeint_t
 	template <typename T = void>
 	inline typename std::enable_if<std::is_same<T, hugeint_t>::value, T>::type Read() {
 		return ReadHugeInt();
 	}
 
+	// Deserialize a LogicalIndex
+	template <typename T = void>
+	inline typename std::enable_if<std::is_same<T, LogicalIndex>::value, T>::type Read() {
+		return LogicalIndex(ReadUnsignedInt64());
+	}
+
+	// Deserialize a PhysicalIndex
+	template <typename T = void>
+	inline typename std::enable_if<std::is_same<T, PhysicalIndex>::value, T>::type Read() {
+		return PhysicalIndex(ReadUnsignedInt64());
+	}
+
 protected:
-	virtual void SetTag(const char *tag) {
+	virtual void SetTag(const field_id_t field_id, const char *tag) {
+		(void)field_id;
 		(void)tag;
 	}