duckdb 0.8.2-dev1862.0 → 0.8.2-dev2044.0

package/src/duckdb/src/include/duckdb/common/bitpacking.hpp

@@ -13,11 +13,17 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 
 namespace duckdb {
 
 using bitpacking_width_t = uint8_t;
 
+struct HugeIntPacker {
+	static void Pack(const hugeint_t *__restrict in, uint32_t *__restrict out, bitpacking_width_t width);
+	static void Unpack(const uint32_t *__restrict in, hugeint_t *__restrict out, bitpacking_width_t width);
+};
+
 class BitpackingPrimitives {
 
 public:
@@ -37,9 +43,7 @@ public:
 			idx_t misaligned_count = count % BITPACKING_ALGORITHM_GROUP_SIZE;
 			T tmp_buffer[BITPACKING_ALGORITHM_GROUP_SIZE]; // TODO maybe faster on the heap?
 
-			if (misaligned_count) {
-				count -= misaligned_count;
-			}
+			count -= misaligned_count;
 
 			for (idx_t i = 0; i < count; i += BITPACKING_ALGORITHM_GROUP_SIZE) {
 				PackGroup<T>(dst + (i * width) / 8, src + i, width);
@@ -78,22 +82,22 @@ public:
 	}
 
 	// Calculates the minimum required number of bits per value that can store all values
-	template <class T>
+	template <class T, bool is_signed = NumericLimits<T>::IsSigned()>
 	inline static bitpacking_width_t MinimumBitWidth(T value) {
-		return FindMinimumBitWidth<T, BYTE_ALIGNED>(value, value);
+		return FindMinimumBitWidth<T, is_signed, BYTE_ALIGNED>(value, value);
 	}
 
 	// Calculates the minimum required number of bits per value that can store all values
-	template <class T>
+	template <class T, bool is_signed = NumericLimits<T>::IsSigned()>
 	inline static bitpacking_width_t MinimumBitWidth(T *values, idx_t count) {
-		return FindMinimumBitWidth<T, BYTE_ALIGNED>(values, count);
+		return FindMinimumBitWidth<T, is_signed, BYTE_ALIGNED>(values, count);
 	}
 
 	// Calculates the minimum required number of bits per value that can store all values,
 	// given a predetermined minimum and maximum value of the buffer
-	template <class T>
+	template <class T, bool is_signed = NumericLimits<T>::IsSigned()>
 	inline static bitpacking_width_t MinimumBitWidth(T minimum, T maximum) {
-		return FindMinimumBitWidth<T, BYTE_ALIGNED>(minimum, maximum);
+		return FindMinimumBitWidth<T, is_signed, BYTE_ALIGNED>(minimum, maximum);
 	}
 
 	inline static idx_t GetRequiredSize(idx_t count, bitpacking_width_t width) {
@@ -112,7 +116,7 @@ public:
 	}
 
 private:
-	template <class T, bool round_to_next_byte = false>
+	template <class T, bool is_signed, bool round_to_next_byte = false>
 	static bitpacking_width_t FindMinimumBitWidth(T *values, idx_t count) {
 		T min_value = values[0];
 		T max_value = values[0];
@@ -122,7 +126,7 @@ private:
 				max_value = values[i];
 			}
 
-			if (std::is_signed<T>::value) {
+			if (is_signed) {
 				if (values[i] < min_value) {
 					min_value = values[i];
 				}
@@ -132,12 +136,12 @@ private:
 		return FindMinimumBitWidth<T, round_to_next_byte>(min_value, max_value);
 	}
 
-	template <class T, bool round_to_next_byte = false>
+	template <class T, bool is_signed, bool round_to_next_byte = false>
 	static bitpacking_width_t FindMinimumBitWidth(T min_value, T max_value) {
 		bitpacking_width_t bitwidth;
 		T value;
 
-		if (std::is_signed<T>::value) {
+		if (is_signed) {
 			if (min_value == NumericLimits<T>::Minimum()) {
 				// handle special case of the minimal value, as it cannot be negated like all other values.
 				return sizeof(T) * 8;
@@ -152,7 +156,7 @@ private:
 			return 0;
 		}
 
-		if (std::is_signed<T>::value) {
+		if (is_signed) {
 			bitwidth = 1;
 		} else {
 			bitwidth = 0;
@@ -168,58 +172,37 @@ private:
 		// Assert results are correct
 #ifdef DEBUG
 		if (bitwidth < sizeof(T) * 8 && bitwidth != 0) {
-			if (std::is_signed<T>::value) {
-				D_ASSERT((int64_t)max_value <= (int64_t)(1L << (bitwidth - 1)) - 1);
-				D_ASSERT((int64_t)min_value >= (int64_t)(-1 * ((1L << (bitwidth - 1)) - 1) - 1));
+			if (is_signed) {
+				D_ASSERT(max_value <= (T(1) << (bitwidth - 1)) - 1);
+				D_ASSERT(min_value >= (T(-1) * ((T(1) << (bitwidth - 1)) - 1) - 1));
 			} else {
-				D_ASSERT((uint64_t)max_value <= (uint64_t)(1L << (bitwidth)) - 1);
+				D_ASSERT(max_value <= (T(1) << (bitwidth)) - 1);
 			}
 		}
 #endif
 		if (round_to_next_byte) {
 			return (bitwidth / 8 + (bitwidth % 8 != 0)) * 8;
-		} else {
-			return bitwidth;
 		}
+		return bitwidth;
 	}
 
 	// Sign bit extension
-	template <class T, class T_U = typename std::make_unsigned<T>::type>
+	template <class T, class T_U = typename MakeUnsigned<T>::type>
 	static void SignExtend(data_ptr_t dst, bitpacking_width_t width) {
-		T const mask = ((T_U)1) << (width - 1);
+		T const mask = T_U(1) << (width - 1);
 		for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) {
 			T value = Load<T>(dst + i * sizeof(T));
-			value = value & ((((T_U)1) << width) - ((T_U)1));
+			value = value & ((T_U(1) << width) - T_U(1));
 			T result = (value ^ mask) - mask;
 			Store(result, dst + i * sizeof(T));
 		}
 	}
 
-	template <class T>
-	static void UnPackGroup(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
-	                        bool skip_sign_extension = false) {
-		if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint8_t *)src, (uint8_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint16_t *)src, (uint16_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint32_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint64_t *)dst, (uint32_t)width);
-		} else {
-			throw InternalException("Unsupported type found in bitpacking.");
-		}
-
-		if (NumericLimits<T>::IsSigned() && !skip_sign_extension && width > 0 && width < sizeof(T) * 8) {
-			SignExtend<T>(dst, width);
-		}
-	}
-
 	// Prevent compression at widths that are ineffective
 	template <class T>
 	static bitpacking_width_t GetEffectiveWidth(bitpacking_width_t width) {
-		auto bits_of_type = sizeof(T) * 8;
-		auto type_size = sizeof(T);
+		bitpacking_width_t bits_of_type = sizeof(T) * 8;
+		bitpacking_width_t type_size = sizeof(T);
 		if (width + type_size > bits_of_type) {
 			return bits_of_type;
 		}
@@ -227,17 +210,49 @@ private:
 	}
 
 	template <class T>
-	static void PackGroup(data_ptr_t dst, T *values, bitpacking_width_t width) {
-		if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint8_t *)values, (uint8_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint16_t *)values, (uint16_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint32_t *)values, (uint32_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint64_t *)values, (uint32_t *)dst, (uint32_t)width);
+	static inline void PackGroup(data_ptr_t dst, T *values, bitpacking_width_t width) {
+		if (std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint8_t *>(values), reinterpret_cast<uint8_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint16_t *>(values), reinterpret_cast<uint16_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int32_t>::value || std::is_same<T, uint32_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint32_t *>(values), reinterpret_cast<uint32_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int64_t>::value || std::is_same<T, uint64_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint64_t *>(values), reinterpret_cast<uint32_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, hugeint_t>::value) {
+			HugeIntPacker::Pack(reinterpret_cast<const hugeint_t *>(values), reinterpret_cast<uint32_t *>(dst), width);
 		} else {
-			throw InternalException("Unsupported type found in bitpacking.");
+			throw InternalException("Unsupported type for bitpacking");
+		}
+	}
+
+	template <class T>
+	static inline void UnPackGroup(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
+	                               bool skip_sign_extension = false) {
+		if (std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint8_t *>(src), reinterpret_cast<uint8_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint16_t *>(src), reinterpret_cast<uint16_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int32_t>::value || std::is_same<T, uint32_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint32_t *>(src), reinterpret_cast<uint32_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int64_t>::value || std::is_same<T, uint64_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint32_t *>(src), reinterpret_cast<uint64_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, hugeint_t>::value) {
+			HugeIntPacker::Unpack(reinterpret_cast<const uint32_t *>(src), reinterpret_cast<hugeint_t *>(dst), width);
+		} else {
+			throw InternalException("Unsupported type for bitpacking");
+		}
+
+		if (NumericLimits<T>::IsSigned() && !skip_sign_extension && width > 0 && width < sizeof(T) * 8) {
+			SignExtend<T>(dst, width);
 		}
 	}
 };

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -17,6 +17,8 @@
 namespace duckdb {
 class Serializer;
 class Deserializer;
+class FormatSerializer;
+class FormatDeserializer;
 
 //! inline std directives that we use frequently
 #ifndef DUCKDB_DEBUG_MOVE

package/src/duckdb/src/include/duckdb/common/enum_util.hpp

@@ -40,6 +40,8 @@ enum class AggregateOrderDependent : uint8_t;
 
 enum class AggregateType : uint8_t;
 
+enum class AlterForeignKeyType : uint8_t;
+
 enum class AlterScalarFunctionType : uint8_t;
 
 enum class AlterTableFunctionType : uint8_t;
@@ -128,6 +130,8 @@ enum class FunctionNullHandling : uint8_t;
 
 enum class FunctionSideEffects : uint8_t;
 
+enum class IndexConstraintType : uint8_t;
+
 enum class IndexType : uint8_t;
 
 enum class InsertColumnOrder : uint8_t;
@@ -140,6 +144,8 @@ enum class JoinType : uint8_t;
 
 enum class KeywordCategory : uint8_t;
 
+enum class LoadType : uint8_t;
+
 enum class LogicalOperatorType : uint8_t;
 
 enum class LogicalTypeId : uint8_t;
@@ -174,6 +180,8 @@ enum class OrderType : uint8_t;
 
 enum class OutputStream : uint8_t;
 
+enum class ParseInfoType : uint8_t;
+
 enum class ParserExtensionResultType : uint8_t;
 
 enum class ParserMode : uint8_t;
@@ -214,6 +222,8 @@ enum class SetType : uint8_t;
 
 enum class SimplifiedTokenType : uint8_t;
 
+enum class SinkCombineResultType : uint8_t;
+
 enum class SinkFinalizeType : uint8_t;
 
 enum class SinkResultType : uint8_t;
@@ -283,6 +293,9 @@ const char* EnumUtil::ToChars<AggregateOrderDependent>(AggregateOrderDependent v
 template<>
 const char* EnumUtil::ToChars<AggregateType>(AggregateType value);
 
+template<>
+const char* EnumUtil::ToChars<AlterForeignKeyType>(AlterForeignKeyType value);
+
 template<>
 const char* EnumUtil::ToChars<AlterScalarFunctionType>(AlterScalarFunctionType value);
 
@@ -415,6 +428,9 @@ const char* EnumUtil::ToChars<FunctionNullHandling>(FunctionNullHandling value);
 template<>
 const char* EnumUtil::ToChars<FunctionSideEffects>(FunctionSideEffects value);
 
+template<>
+const char* EnumUtil::ToChars<IndexConstraintType>(IndexConstraintType value);
+
 template<>
 const char* EnumUtil::ToChars<IndexType>(IndexType value);
 
@@ -433,6 +449,9 @@ const char* EnumUtil::ToChars<JoinType>(JoinType value);
 template<>
 const char* EnumUtil::ToChars<KeywordCategory>(KeywordCategory value);
 
+template<>
+const char* EnumUtil::ToChars<LoadType>(LoadType value);
+
 template<>
 const char* EnumUtil::ToChars<LogicalOperatorType>(LogicalOperatorType value);
 
@@ -484,6 +503,9 @@ const char* EnumUtil::ToChars<OrderType>(OrderType value);
 template<>
 const char* EnumUtil::ToChars<OutputStream>(OutputStream value);
 
+template<>
+const char* EnumUtil::ToChars<ParseInfoType>(ParseInfoType value);
+
 template<>
 const char* EnumUtil::ToChars<ParserExtensionResultType>(ParserExtensionResultType value);
 
@@ -544,6 +566,9 @@ const char* EnumUtil::ToChars<SetType>(SetType value);
 template<>
 const char* EnumUtil::ToChars<SimplifiedTokenType>(SimplifiedTokenType value);
 
+template<>
+const char* EnumUtil::ToChars<SinkCombineResultType>(SinkCombineResultType value);
+
 template<>
 const char* EnumUtil::ToChars<SinkFinalizeType>(SinkFinalizeType value);
 
@@ -641,6 +666,9 @@ AggregateOrderDependent EnumUtil::FromString<AggregateOrderDependent>(const char
 template<>
 AggregateType EnumUtil::FromString<AggregateType>(const char *value);
 
+template<>
+AlterForeignKeyType EnumUtil::FromString<AlterForeignKeyType>(const char *value);
+
 template<>
 AlterScalarFunctionType EnumUtil::FromString<AlterScalarFunctionType>(const char *value);
 
@@ -773,6 +801,9 @@ FunctionNullHandling EnumUtil::FromString<FunctionNullHandling>(const char *valu
 template<>
 FunctionSideEffects EnumUtil::FromString<FunctionSideEffects>(const char *value);
 
+template<>
+IndexConstraintType EnumUtil::FromString<IndexConstraintType>(const char *value);
+
 template<>
 IndexType EnumUtil::FromString<IndexType>(const char *value);
 
@@ -791,6 +822,9 @@ JoinType EnumUtil::FromString<JoinType>(const char *value);
 template<>
 KeywordCategory EnumUtil::FromString<KeywordCategory>(const char *value);
 
+template<>
+LoadType EnumUtil::FromString<LoadType>(const char *value);
+
 template<>
 LogicalOperatorType EnumUtil::FromString<LogicalOperatorType>(const char *value);
 
@@ -842,6 +876,9 @@ OrderType EnumUtil::FromString<OrderType>(const char *value);
 template<>
 OutputStream EnumUtil::FromString<OutputStream>(const char *value);
 
+template<>
+ParseInfoType EnumUtil::FromString<ParseInfoType>(const char *value);
+
 template<>
 ParserExtensionResultType EnumUtil::FromString<ParserExtensionResultType>(const char *value);
 
@@ -902,6 +939,9 @@ SetType EnumUtil::FromString<SetType>(const char *value);
 template<>
 SimplifiedTokenType EnumUtil::FromString<SimplifiedTokenType>(const char *value);
 
+template<>
+SinkCombineResultType EnumUtil::FromString<SinkCombineResultType>(const char *value);
+
 template<>
 SinkFinalizeType EnumUtil::FromString<SinkFinalizeType>(const char *value);
 

package/src/duckdb/src/include/duckdb/common/enums/index_type.hpp

@@ -23,7 +23,7 @@ enum class IndexType : uint8_t {
 //===--------------------------------------------------------------------===//
 // Index Constraint Types
 //===--------------------------------------------------------------------===//
-enum IndexConstraintType : uint8_t {
+enum class IndexConstraintType : uint8_t {
 	NONE = 0,    // index is an index don't built to any constraint
 	UNIQUE = 1,  // index is an index built to enforce a UNIQUE constraint
 	PRIMARY = 2, // index is an index built to enforce a PRIMARY KEY constraint

package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp

@@ -46,10 +46,14 @@ enum class SourceResultType : uint8_t { HAVE_MORE_OUTPUT, FINISHED, BLOCKED };
 //! BLOCKED means the sink is currently blocked, e.g. by some async I/O.
 enum class SinkResultType : uint8_t { NEED_MORE_INPUT, FINISHED, BLOCKED };
 
+// todo comment
+enum class SinkCombineResultType : uint8_t { FINISHED, BLOCKED };
+
 //! The SinkFinalizeType is used to indicate the result of a Finalize call on a sink
 //! There are two possible results:
 //! READY means the sink is ready for further processing
 //! NO_OUTPUT_POSSIBLE means the sink will never provide output, and any pipelines involving the sink can be skipped
-enum class SinkFinalizeType : uint8_t { READY, NO_OUTPUT_POSSIBLE };
+//! BLOCKED means the finalize call to the sink is currently blocked, e.g. by some async I/O.
+enum class SinkFinalizeType : uint8_t { READY, NO_OUTPUT_POSSIBLE, BLOCKED };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/hugeint.hpp

@@ -3,6 +3,7 @@
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/common/string.hpp"
 #include <stdint.h>
+#include "duckdb/common/typedefs.hpp"
 
 namespace duckdb {
 
@@ -58,6 +59,20 @@ public:
 	DUCKDB_API hugeint_t &operator&=(const hugeint_t &rhs);
 	DUCKDB_API hugeint_t &operator|=(const hugeint_t &rhs);
 	DUCKDB_API hugeint_t &operator^=(const hugeint_t &rhs);
+
+	// boolean operators
+	DUCKDB_API explicit operator bool() const;
+	DUCKDB_API bool operator!() const;
+
+	// cast operators
+	DUCKDB_API explicit operator uint8_t() const;
+	DUCKDB_API explicit operator uint16_t() const;
+	DUCKDB_API explicit operator uint32_t() const;
+	DUCKDB_API explicit operator uint64_t() const;
+	DUCKDB_API explicit operator int8_t() const;
+	DUCKDB_API explicit operator int16_t() const;
+	DUCKDB_API explicit operator int32_t() const;
+	DUCKDB_API explicit operator int64_t() const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/index_vector.hpp

@@ -10,6 +10,8 @@
 
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/vector.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -59,6 +61,16 @@ public:
 		return internal_vector.end();
 	}
 
+	void FormatSerialize(FormatSerializer &serializer) const {
+		serializer.WriteProperty("internal_vector", internal_vector);
+	}
+
+	static IndexVector<T, INDEX_TYPE> FormatDeserialize(FormatDeserializer &deserializer) {
+		IndexVector<T, INDEX_TYPE> result;
+		deserializer.ReadProperty("internal_vector", result.internal_vector);
+		return result;
+	}
+
 private:
 	vector<T> internal_vector;
 };