duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/include/duckdb/common/assert.hpp

@@ -10,7 +10,7 @@
 
 #pragma once
 
-#if (defined(DUCKDB_USE_STANDARD_ASSERT) || !defined(DEBUG)) && !defined(DUCKDB_FORCE_ASSERT)
+#if (defined(DUCKDB_USE_STANDARD_ASSERT) || !defined(DEBUG)) && !defined(DUCKDB_FORCE_ASSERT) && !defined(__MVS__)
 
 #include <assert.h>
 #define D_ASSERT assert

package/src/duckdb/src/include/duckdb/common/bitpacking.hpp

@@ -13,11 +13,17 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/limits.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 
 namespace duckdb {
 
 using bitpacking_width_t = uint8_t;
 
+struct HugeIntPacker {
+	static void Pack(const hugeint_t *__restrict in, uint32_t *__restrict out, bitpacking_width_t width);
+	static void Unpack(const uint32_t *__restrict in, hugeint_t *__restrict out, bitpacking_width_t width);
+};
+
 class BitpackingPrimitives {
 
 public:
@@ -37,9 +43,7 @@ public:
 			idx_t misaligned_count = count % BITPACKING_ALGORITHM_GROUP_SIZE;
 			T tmp_buffer[BITPACKING_ALGORITHM_GROUP_SIZE]; // TODO maybe faster on the heap?
 
-			if (misaligned_count) {
-				count -= misaligned_count;
-			}
+			count -= misaligned_count;
 
 			for (idx_t i = 0; i < count; i += BITPACKING_ALGORITHM_GROUP_SIZE) {
 				PackGroup<T>(dst + (i * width) / 8, src + i, width);
@@ -78,22 +82,22 @@ public:
 	}
 
 	// Calculates the minimum required number of bits per value that can store all values
-	template <class T>
+	template <class T, bool is_signed = NumericLimits<T>::IsSigned()>
 	inline static bitpacking_width_t MinimumBitWidth(T value) {
-		return FindMinimumBitWidth<T, BYTE_ALIGNED>(value, value);
+		return FindMinimumBitWidth<T, is_signed, BYTE_ALIGNED>(value, value);
 	}
 
 	// Calculates the minimum required number of bits per value that can store all values
-	template <class T>
+	template <class T, bool is_signed = NumericLimits<T>::IsSigned()>
 	inline static bitpacking_width_t MinimumBitWidth(T *values, idx_t count) {
-		return FindMinimumBitWidth<T, BYTE_ALIGNED>(values, count);
+		return FindMinimumBitWidth<T, is_signed, BYTE_ALIGNED>(values, count);
 	}
 
 	// Calculates the minimum required number of bits per value that can store all values,
 	// given a predetermined minimum and maximum value of the buffer
-	template <class T>
+	template <class T, bool is_signed = NumericLimits<T>::IsSigned()>
 	inline static bitpacking_width_t MinimumBitWidth(T minimum, T maximum) {
-		return FindMinimumBitWidth<T, BYTE_ALIGNED>(minimum, maximum);
+		return FindMinimumBitWidth<T, is_signed, BYTE_ALIGNED>(minimum, maximum);
 	}
 
 	inline static idx_t GetRequiredSize(idx_t count, bitpacking_width_t width) {
@@ -112,7 +116,7 @@ public:
 	}
 
 private:
-	template <class T, bool round_to_next_byte = false>
+	template <class T, bool is_signed, bool round_to_next_byte = false>
 	static bitpacking_width_t FindMinimumBitWidth(T *values, idx_t count) {
 		T min_value = values[0];
 		T max_value = values[0];
@@ -122,7 +126,7 @@ private:
 				max_value = values[i];
 			}
 
-			if (std::is_signed<T>::value) {
+			if (is_signed) {
 				if (values[i] < min_value) {
 					min_value = values[i];
 				}
@@ -132,12 +136,12 @@ private:
 		return FindMinimumBitWidth<T, round_to_next_byte>(min_value, max_value);
 	}
 
-	template <class T, bool round_to_next_byte = false>
+	template <class T, bool is_signed, bool round_to_next_byte = false>
 	static bitpacking_width_t FindMinimumBitWidth(T min_value, T max_value) {
 		bitpacking_width_t bitwidth;
 		T value;
 
-		if (std::is_signed<T>::value) {
+		if (is_signed) {
 			if (min_value == NumericLimits<T>::Minimum()) {
 				// handle special case of the minimal value, as it cannot be negated like all other values.
 				return sizeof(T) * 8;
@@ -152,7 +156,7 @@ private:
 			return 0;
 		}
 
-		if (std::is_signed<T>::value) {
+		if (is_signed) {
 			bitwidth = 1;
 		} else {
 			bitwidth = 0;
@@ -168,58 +172,37 @@ private:
 		// Assert results are correct
 #ifdef DEBUG
 		if (bitwidth < sizeof(T) * 8 && bitwidth != 0) {
-			if (std::is_signed<T>::value) {
-				D_ASSERT((int64_t)max_value <= (int64_t)(1L << (bitwidth - 1)) - 1);
-				D_ASSERT((int64_t)min_value >= (int64_t)(-1 * ((1L << (bitwidth - 1)) - 1) - 1));
+			if (is_signed) {
+				D_ASSERT(max_value <= (T(1) << (bitwidth - 1)) - 1);
+				D_ASSERT(min_value >= (T(-1) * ((T(1) << (bitwidth - 1)) - 1) - 1));
 			} else {
-				D_ASSERT((uint64_t)max_value <= (uint64_t)(1L << (bitwidth)) - 1);
+				D_ASSERT(max_value <= (T(1) << (bitwidth)) - 1);
 			}
 		}
 #endif
 		if (round_to_next_byte) {
 			return (bitwidth / 8 + (bitwidth % 8 != 0)) * 8;
-		} else {
-			return bitwidth;
 		}
+		return bitwidth;
 	}
 
 	// Sign bit extension
-	template <class T, class T_U = typename std::make_unsigned<T>::type>
+	template <class T, class T_U = typename MakeUnsigned<T>::type>
 	static void SignExtend(data_ptr_t dst, bitpacking_width_t width) {
-		T const mask = ((T_U)1) << (width - 1);
+		T const mask = T_U(1) << (width - 1);
 		for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) {
 			T value = Load<T>(dst + i * sizeof(T));
-			value = value & ((((T_U)1) << width) - ((T_U)1));
+			value = value & ((T_U(1) << width) - T_U(1));
 			T result = (value ^ mask) - mask;
 			Store(result, dst + i * sizeof(T));
 		}
 	}
 
-	template <class T>
-	static void UnPackGroup(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
-	                        bool skip_sign_extension = false) {
-		if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint8_t *)src, (uint8_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint16_t *)src, (uint16_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint32_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
-			duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint64_t *)dst, (uint32_t)width);
-		} else {
-			throw InternalException("Unsupported type found in bitpacking.");
-		}
-
-		if (NumericLimits<T>::IsSigned() && !skip_sign_extension && width > 0 && width < sizeof(T) * 8) {
-			SignExtend<T>(dst, width);
-		}
-	}
-
 	// Prevent compression at widths that are ineffective
 	template <class T>
 	static bitpacking_width_t GetEffectiveWidth(bitpacking_width_t width) {
-		auto bits_of_type = sizeof(T) * 8;
-		auto type_size = sizeof(T);
+		bitpacking_width_t bits_of_type = sizeof(T) * 8;
+		bitpacking_width_t type_size = sizeof(T);
 		if (width + type_size > bits_of_type) {
 			return bits_of_type;
 		}
@@ -227,17 +210,49 @@ private:
 	}
 
 	template <class T>
-	static void PackGroup(data_ptr_t dst, T *values, bitpacking_width_t width) {
-		if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint8_t *)values, (uint8_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint16_t *)values, (uint16_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint32_t *)values, (uint32_t *)dst, (uint32_t)width);
-		} else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
-			duckdb_fastpforlib::fastpack((const uint64_t *)values, (uint32_t *)dst, (uint32_t)width);
+	static inline void PackGroup(data_ptr_t dst, T *values, bitpacking_width_t width) {
+		if (std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint8_t *>(values), reinterpret_cast<uint8_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint16_t *>(values), reinterpret_cast<uint16_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int32_t>::value || std::is_same<T, uint32_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint32_t *>(values), reinterpret_cast<uint32_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int64_t>::value || std::is_same<T, uint64_t>::value) {
+			duckdb_fastpforlib::fastpack(reinterpret_cast<const uint64_t *>(values), reinterpret_cast<uint32_t *>(dst),
+			                             static_cast<uint32_t>(width));
+		} else if (std::is_same<T, hugeint_t>::value) {
+			HugeIntPacker::Pack(reinterpret_cast<const hugeint_t *>(values), reinterpret_cast<uint32_t *>(dst), width);
 		} else {
-			throw InternalException("Unsupported type found in bitpacking.");
+			throw InternalException("Unsupported type for bitpacking");
+		}
+	}
+
+	template <class T>
+	static inline void UnPackGroup(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
+	                               bool skip_sign_extension = false) {
+		if (std::is_same<T, int8_t>::value || std::is_same<T, uint8_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint8_t *>(src), reinterpret_cast<uint8_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int16_t>::value || std::is_same<T, uint16_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint16_t *>(src), reinterpret_cast<uint16_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int32_t>::value || std::is_same<T, uint32_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint32_t *>(src), reinterpret_cast<uint32_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, int64_t>::value || std::is_same<T, uint64_t>::value) {
+			duckdb_fastpforlib::fastunpack(reinterpret_cast<const uint32_t *>(src), reinterpret_cast<uint64_t *>(dst),
+			                               static_cast<uint32_t>(width));
+		} else if (std::is_same<T, hugeint_t>::value) {
+			HugeIntPacker::Unpack(reinterpret_cast<const uint32_t *>(src), reinterpret_cast<hugeint_t *>(dst), width);
+		} else {
+			throw InternalException("Unsupported type for bitpacking");
+		}
+
+		if (NumericLimits<T>::IsSigned() && !skip_sign_extension && width > 0 && width < sizeof(T) * 8) {
+			SignExtend<T>(dst, width);
 		}
 	}
 };

package/src/duckdb/src/include/duckdb/common/bswap.hpp

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/bswap.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+
+namespace duckdb {
+
+#define BSWAP16(x) ((uint16_t)((((uint16_t)(x)&0xff00) >> 8) | (((uint16_t)(x)&0x00ff) << 8)))
+
+#define BSWAP32(x)                                                                                                     \
+	((uint32_t)((((uint32_t)(x)&0xff000000) >> 24) | (((uint32_t)(x)&0x00ff0000) >> 8) |                               \
+	            (((uint32_t)(x)&0x0000ff00) << 8) | (((uint32_t)(x)&0x000000ff) << 24)))
+
+#define BSWAP64(x)                                                                                                     \
+	((uint64_t)((((uint64_t)(x)&0xff00000000000000ull) >> 56) | (((uint64_t)(x)&0x00ff000000000000ull) >> 40) |        \
+	            (((uint64_t)(x)&0x0000ff0000000000ull) >> 24) | (((uint64_t)(x)&0x000000ff00000000ull) >> 8) |         \
+	            (((uint64_t)(x)&0x00000000ff000000ull) << 8) | (((uint64_t)(x)&0x0000000000ff0000ull) << 24) |         \
+	            (((uint64_t)(x)&0x000000000000ff00ull) << 40) | (((uint64_t)(x)&0x00000000000000ffull) << 56)))
+
+template <class T>
+static inline T BSwap(const T &x) {
+	static_assert(sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8,
+	              "Size of type must be 1, 2, 4, or 8 for BSwap");
+	if (sizeof(T) == 1) {
+		return x;
+	} else if (sizeof(T) == 2) {
+		return BSWAP16(x);
+	} else if (sizeof(T) == 4) {
+		return BSWAP32(x);
+	} else {
+		return BSWAP64(x);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/case_insensitive_map.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/common/helper.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -17,6 +17,8 @@
 namespace duckdb {
 class Serializer;
 class Deserializer;
+class FormatSerializer;
+class FormatDeserializer;
 
 //! inline std directives that we use frequently
 #ifndef DUCKDB_DEBUG_MOVE
@@ -39,6 +41,8 @@ DUCKDB_API bool IsRowIdColumnId(column_t column_id);
 
 //! The maximum row identifier used in tables
 extern const row_t MAX_ROW_ID;
+//! Transaction-local row IDs start at MAX_ROW_ID
+extern const row_t MAX_ROW_ID_LOCAL;
 
 extern const transaction_t TRANSACTION_ID_START;
 extern const transaction_t MAX_TRANSACTION_ID;

package/src/duckdb/src/include/duckdb/common/dl.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/windows.hpp"
 #include "duckdb/common/local_file_system.hpp"
+#include "duckdb/common/windows_util.hpp"
 
 #ifndef _WIN32
 #include <dlfcn.h>
@@ -24,7 +25,8 @@ namespace duckdb {
 
 inline void *dlopen(const char *file, int mode) {
 	D_ASSERT(file);
-	return (void *)LoadLibrary(file);
+	auto fpath = WindowsUtil::UTF8ToUnicode(file);
+	return (void *)LoadLibraryW(fpath.c_str());
 }
 
 inline void *dlsym(void *handle, const char *name) {