duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/src/include/duckdb/common/bit_utils.hpp

@@ -10,68 +10,7 @@
 
 #include "duckdb/common/hugeint.hpp"
 #include "duckdb/common/uhugeint.hpp"
-
-#if defined(_MSC_VER) && !defined(__clang__)
-#define __restrict__
-#define __BYTE_ORDER__          __ORDER_LITTLE_ENDIAN__
-#define __ORDER_LITTLE_ENDIAN__ 2
-#include <intrin.h>
-static inline int __builtin_ctzll(unsigned long long x) {
-#ifdef _WIN64
-	unsigned long ret;
-	_BitScanForward64(&ret, x);
-	return (int)ret;
-#else
-	unsigned long low, high;
-	bool low_set = _BitScanForward(&low, (unsigned __int32)(x)) != 0;
-	_BitScanForward(&high, (unsigned __int32)(x >> 32));
-	high += 32;
-	return low_set ? low : high;
-#endif
-}
-static inline int __builtin_clzll(unsigned long long mask) {
-	unsigned long where;
-// BitScanReverse scans from MSB to LSB for first set bit.
-// Returns 0 if no set bit is found.
-#if defined(_WIN64)
-	if (_BitScanReverse64(&where, mask))
-		return static_cast<int>(63 - where);
-#elif defined(_WIN32)
-	// Scan the high 32 bits.
-	if (_BitScanReverse(&where, static_cast<unsigned long>(mask >> 32)))
-		return static_cast<int>(63 - (where + 32)); // Create a bit offset from the MSB.
-	// Scan the low 32 bits.
-	if (_BitScanReverse(&where, static_cast<unsigned long>(mask)))
-		return static_cast<int>(63 - where);
-#else
-#error "Implementation of __builtin_clzll required"
-#endif
-	return 64; // Undefined Behavior.
-}
-
-static inline int __builtin_ctz(unsigned int value) {
-	unsigned long trailing_zero = 0;
-
-	if (_BitScanForward(&trailing_zero, value)) {
-		return trailing_zero;
-	} else {
-		// This is undefined, I better choose 32 than 0
-		return 32;
-	}
-}
-
-static inline int __builtin_clz(unsigned int value) {
-	unsigned long leading_zero = 0;
-
-	if (_BitScanReverse(&leading_zero, value)) {
-		return 31 - leading_zero;
-	} else {
-		// Same remarks as above
-		return 32;
-	}
-}
-
-#endif
+#include "duckdb/common/numeric_utils.hpp"
 
 namespace duckdb {
 
@@ -79,60 +18,86 @@ template <class T>
 struct CountZeros {};
 
 template <>
-struct CountZeros<uint32_t> {
-	inline static int Leading(uint32_t value) {
-		if (!value) {
-			return 32;
+struct CountZeros<uint64_t> {
+	// see here: https://en.wikipedia.org/wiki/De_Bruijn_sequence
+	inline static idx_t Leading(const uint64_t value_in) {
+		if (!value_in) {
+			return 64;
 		}
-		return __builtin_clz(value);
+
+		uint64_t value = value_in;
+
+		constexpr uint64_t index64msb[] = {0,  47, 1,  56, 48, 27, 2,  60, 57, 49, 41, 37, 28, 16, 3,  61,
+		                                   54, 58, 35, 52, 50, 42, 21, 44, 38, 32, 29, 23, 17, 11, 4,  62,
+		                                   46, 55, 26, 59, 40, 36, 15, 53, 34, 51, 20, 43, 31, 22, 10, 45,
+		                                   25, 39, 14, 33, 19, 30, 9,  24, 13, 18, 8,  12, 7,  6,  5,  63};
+
+		constexpr uint64_t debruijn64msb = 0X03F79D71B4CB0A89;
+
+		value |= value >> 1;
+		value |= value >> 2;
+		value |= value >> 4;
+		value |= value >> 8;
+		value |= value >> 16;
+		value |= value >> 32;
+		auto result = 63 - index64msb[(value * debruijn64msb) >> 58];
+#ifdef __clang__
+		D_ASSERT(result == static_cast<uint64_t>(__builtin_clzl(value_in)));
+#endif
+		return result;
 	}
-	inline static int Trailing(uint32_t value) {
-		if (!value) {
-			return 32;
+	inline static idx_t Trailing(uint64_t value_in) {
+		if (!value_in) {
+			return 64;
 		}
-		return __builtin_ctz(value);
+		uint64_t value = value_in;
+
+		constexpr uint64_t index64lsb[] = {63, 0,  58, 1,  59, 47, 53, 2,  60, 39, 48, 27, 54, 33, 42, 3,
+		                                   61, 51, 37, 40, 49, 18, 28, 20, 55, 30, 34, 11, 43, 14, 22, 4,
+		                                   62, 57, 46, 52, 38, 26, 32, 41, 50, 36, 17, 19, 29, 10, 13, 21,
+		                                   56, 45, 25, 31, 35, 16, 9,  12, 44, 24, 15, 8,  23, 7,  6,  5};
+		constexpr uint64_t debruijn64lsb = 0x07EDD5E59A4E28C2ULL;
+		auto result = index64lsb[((value & -value) * debruijn64lsb) >> 58];
+#ifdef __clang__
+		D_ASSERT(result == static_cast<uint64_t>(__builtin_ctzl(value_in)));
+#endif
+		return result;
 	}
 };
 
 template <>
-struct CountZeros<uint64_t> {
-	inline static int Leading(uint64_t value) {
-		if (!value) {
-			return 64;
-		}
-		return __builtin_clzll(value);
+struct CountZeros<uint32_t> {
+	inline static idx_t Leading(uint32_t value) {
+		return CountZeros<uint64_t>::Leading(static_cast<uint64_t>(value)) - 32;
 	}
-	inline static int Trailing(uint64_t value) {
-		if (!value) {
-			return 64;
-		}
-		return __builtin_ctzll(value);
+	inline static idx_t Trailing(uint32_t value) {
+		return CountZeros<uint64_t>::Trailing(static_cast<uint64_t>(value));
 	}
 };
 
 template <>
 struct CountZeros<hugeint_t> {
-	inline static int Leading(hugeint_t value) {
-		const uint64_t upper = (uint64_t)value.upper;
+	inline static idx_t Leading(hugeint_t value) {
+		const uint64_t upper = static_cast<uint64_t>(value.upper);
 		const uint64_t lower = value.lower;
 
 		if (upper) {
-			return __builtin_clzll(upper);
+			return CountZeros<uint64_t>::Leading(upper);
 		} else if (lower) {
-			return 64 + __builtin_clzll(lower);
+			return 64 + CountZeros<uint64_t>::Leading(lower);
 		} else {
 			return 128;
 		}
 	}
 
-	inline static int Trailing(hugeint_t value) {
-		const uint64_t upper = (uint64_t)value.upper;
+	inline static idx_t Trailing(hugeint_t value) {
+		const uint64_t upper = static_cast<uint64_t>(value.upper);
 		const uint64_t lower = value.lower;
 
 		if (lower) {
-			return __builtin_ctzll(lower);
+			return CountZeros<uint64_t>::Trailing(lower);
 		} else if (upper) {
-			return 64 + __builtin_ctzll(upper);
+			return 64 + CountZeros<uint64_t>::Trailing(upper);
 		} else {
 			return 128;
 		}
@@ -141,27 +106,27 @@ struct CountZeros<hugeint_t> {
 
 template <>
 struct CountZeros<uhugeint_t> {
-	inline static int Leading(uhugeint_t value) {
-		const uint64_t upper = (uint64_t)value.upper;
+	inline static idx_t Leading(uhugeint_t value) {
+		const uint64_t upper = static_cast<uint64_t>(value.upper);
 		const uint64_t lower = value.lower;
 
 		if (upper) {
-			return __builtin_clzll(upper);
+			return CountZeros<uint64_t>::Leading(upper);
 		} else if (lower) {
-			return 64 + __builtin_clzll(lower);
+			return 64 + CountZeros<uint64_t>::Leading(lower);
 		} else {
 			return 128;
 		}
 	}
 
-	inline static int Trailing(uhugeint_t value) {
-		const uint64_t upper = (uint64_t)value.upper;
+	inline static idx_t Trailing(uhugeint_t value) {
+		const uint64_t upper = static_cast<uint64_t>(value.upper);
 		const uint64_t lower = value.lower;
 
 		if (lower) {
-			return __builtin_ctzll(lower);
+			return CountZeros<uint64_t>::Trailing(lower);
 		} else if (upper) {
-			return 64 + __builtin_ctzll(upper);
+			return 64 + CountZeros<uint64_t>::Trailing(upper);
 		} else {
 			return 128;
 		}

package/src/duckdb/src/include/duckdb/common/bitpacking.hpp

@@ -112,7 +112,7 @@ public:
 			return num_to_round;
 		}
 
-		return num_to_round + BITPACKING_ALGORITHM_GROUP_SIZE - remainder;
+		return num_to_round + BITPACKING_ALGORITHM_GROUP_SIZE - NumericCast<idx_t>(remainder);
 	}
 
 private:
@@ -133,7 +133,7 @@ private:
 			}
 		}
 
-		return FindMinimumBitWidth<T, round_to_next_byte>(min_value, max_value);
+		return FindMinimumBitWidth<T, is_signed, round_to_next_byte>(min_value, max_value);
 	}
 
 	template <class T, bool is_signed, bool round_to_next_byte = false>
@@ -174,7 +174,7 @@ private:
 		if (bitwidth < sizeof(T) * 8 && bitwidth != 0) {
 			if (is_signed) {
 				D_ASSERT(max_value <= (T(1) << (bitwidth - 1)) - 1);
-				D_ASSERT(min_value >= (T(-1) * ((T(1) << (bitwidth - 1)) - 1) - 1));
+				// D_ASSERT(min_value >= (T(-1) * ((T(1) << (bitwidth - 1)) - 1) - 1));
 			} else {
 				D_ASSERT(max_value <= (T(1) << (bitwidth)) - 1);
 			}
@@ -192,7 +192,7 @@ private:
 		T const mask = UnsafeNumericCast<T>(T_U(1) << (width - 1));
 		for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) {
 			T value = Load<T>(dst + i * sizeof(T));
-			value = UnsafeNumericCast<T>(value & ((T_U(1) << width) - T_U(1)));
+			value = UnsafeNumericCast<T>(T_U(value) & ((T_U(1) << width) - T_U(1)));
 			T result = (value ^ mask) - mask;
 			Store(result, dst + i * sizeof(T));
 		}

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -57,6 +57,8 @@ extern const double PI;
 struct DConstants {
 	//! The value used to signify an invalid index entry
 	static constexpr const idx_t INVALID_INDEX = idx_t(-1);
+	//! The total maximum vector size (128GB)
+	static constexpr const idx_t MAX_VECTOR_SIZE = 1ULL << 37ULL;
 };
 
 struct LogicalIndex {

package/src/duckdb/src/include/duckdb/common/enable_shared_from_this.ipp

@@ -0,0 +1,42 @@
+namespace duckdb {
+
+template <class T>
+class enable_shared_from_this { // NOLINT: invalid case style
+public:
+	template <class U, bool SAFE>
+	friend class shared_ptr;
+
+private:
+	mutable weak_ptr<T> __weak_this_; // NOLINT: __weak_this_ is reserved
+
+protected:
+	constexpr enable_shared_from_this() noexcept {
+	}
+	enable_shared_from_this(enable_shared_from_this const &) noexcept { // NOLINT: not marked as explicit
+	}
+	enable_shared_from_this &operator=(enable_shared_from_this const &) noexcept {
+		return *this;
+	}
+	~enable_shared_from_this() {
+	}
+
+public:
+	shared_ptr<T> shared_from_this() { // NOLINT: invalid case style
+		return shared_ptr<T>(__weak_this_);
+	}
+	shared_ptr<T const> shared_from_this() const { // NOLINT: invalid case style
+		return shared_ptr<const T>(__weak_this_);
+	}
+
+#if _LIBCPP_STD_VER >= 17
+	weak_ptr<T> weak_from_this() noexcept { // NOLINT: invalid case style
+		return __weak_this_;
+	}
+
+	weak_ptr<const T> weak_from_this() const noexcept { // NOLINT: invalid case style
+		return __weak_this_;
+	}
+#endif // _LIBCPP_STD_VER >= 17
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enum_util.hpp

@@ -62,6 +62,8 @@ enum class ArrowOffsetSize : uint8_t;
 
 enum class ArrowVariableSizeType : uint8_t;
 
+enum class BinderType : uint8_t;
+
 enum class BindingMode : uint8_t;
 
 enum class BitpackingMode : uint8_t;
@@ -120,13 +122,17 @@ enum class ExplainOutputType : uint8_t;
 
 enum class ExplainType : uint8_t;
 
+enum class ExponentType : uint8_t;
+
 enum class ExpressionClass : uint8_t;
 
 enum class ExpressionType : uint8_t;
 
+enum class ExtensionInstallMode : uint8_t;
+
 enum class ExtensionLoadResult : uint8_t;
 
-enum class ExternalDependenciesType : uint8_t;
+enum class ExtensionUpdateResultTag : uint8_t;
 
 enum class ExtraDropInfoType : uint8_t;
 
@@ -136,6 +142,8 @@ enum class FileBufferType : uint8_t;
 
 enum class FileCompressionType : uint8_t;
 
+enum class FileExpandResult : uint8_t;
+
 enum class FileGlobOptions : uint8_t;
 
 enum class FileLockType : uint8_t;
@@ -242,6 +250,8 @@ enum class ResultModifierType : uint8_t;
 
 enum class SampleMethod : uint8_t;
 
+enum class SampleType : uint8_t;
+
 enum class ScanType : uint8_t;
 
 enum class SecretDisplayType : uint8_t;
@@ -370,6 +380,9 @@ const char* EnumUtil::ToChars<ArrowOffsetSize>(ArrowOffsetSize value);
 template<>
 const char* EnumUtil::ToChars<ArrowVariableSizeType>(ArrowVariableSizeType value);
 
+template<>
+const char* EnumUtil::ToChars<BinderType>(BinderType value);
+
 template<>
 const char* EnumUtil::ToChars<BindingMode>(BindingMode value);
 
@@ -457,17 +470,23 @@ const char* EnumUtil::ToChars<ExplainOutputType>(ExplainOutputType value);
 template<>
 const char* EnumUtil::ToChars<ExplainType>(ExplainType value);
 
+template<>
+const char* EnumUtil::ToChars<ExponentType>(ExponentType value);
+
 template<>
 const char* EnumUtil::ToChars<ExpressionClass>(ExpressionClass value);
 
 template<>
 const char* EnumUtil::ToChars<ExpressionType>(ExpressionType value);
 
+template<>
+const char* EnumUtil::ToChars<ExtensionInstallMode>(ExtensionInstallMode value);
+
 template<>
 const char* EnumUtil::ToChars<ExtensionLoadResult>(ExtensionLoadResult value);
 
 template<>
-const char* EnumUtil::ToChars<ExternalDependenciesType>(ExternalDependenciesType value);
+const char* EnumUtil::ToChars<ExtensionUpdateResultTag>(ExtensionUpdateResultTag value);
 
 template<>
 const char* EnumUtil::ToChars<ExtraDropInfoType>(ExtraDropInfoType value);
@@ -481,6 +500,9 @@ const char* EnumUtil::ToChars<FileBufferType>(FileBufferType value);
 template<>
 const char* EnumUtil::ToChars<FileCompressionType>(FileCompressionType value);
 
+template<>
+const char* EnumUtil::ToChars<FileExpandResult>(FileExpandResult value);
+
 template<>
 const char* EnumUtil::ToChars<FileGlobOptions>(FileGlobOptions value);
 
@@ -640,6 +662,9 @@ const char* EnumUtil::ToChars<ResultModifierType>(ResultModifierType value);
 template<>
 const char* EnumUtil::ToChars<SampleMethod>(SampleMethod value);
 
+template<>
+const char* EnumUtil::ToChars<SampleType>(SampleType value);
+
 template<>
 const char* EnumUtil::ToChars<ScanType>(ScanType value);
 
@@ -809,6 +834,9 @@ ArrowOffsetSize EnumUtil::FromString<ArrowOffsetSize>(const char *value);
 template<>
 ArrowVariableSizeType EnumUtil::FromString<ArrowVariableSizeType>(const char *value);
 
+template<>
+BinderType EnumUtil::FromString<BinderType>(const char *value);
+
 template<>
 BindingMode EnumUtil::FromString<BindingMode>(const char *value);
 
@@ -896,17 +924,23 @@ ExplainOutputType EnumUtil::FromString<ExplainOutputType>(const char *value);
 template<>
 ExplainType EnumUtil::FromString<ExplainType>(const char *value);
 
+template<>
+ExponentType EnumUtil::FromString<ExponentType>(const char *value);
+
 template<>
 ExpressionClass EnumUtil::FromString<ExpressionClass>(const char *value);
 
 template<>
 ExpressionType EnumUtil::FromString<ExpressionType>(const char *value);
 
+template<>
+ExtensionInstallMode EnumUtil::FromString<ExtensionInstallMode>(const char *value);
+
 template<>
 ExtensionLoadResult EnumUtil::FromString<ExtensionLoadResult>(const char *value);
 
 template<>
-ExternalDependenciesType EnumUtil::FromString<ExternalDependenciesType>(const char *value);
+ExtensionUpdateResultTag EnumUtil::FromString<ExtensionUpdateResultTag>(const char *value);
 
 template<>
 ExtraDropInfoType EnumUtil::FromString<ExtraDropInfoType>(const char *value);
@@ -920,6 +954,9 @@ FileBufferType EnumUtil::FromString<FileBufferType>(const char *value);
 template<>
 FileCompressionType EnumUtil::FromString<FileCompressionType>(const char *value);
 
+template<>
+FileExpandResult EnumUtil::FromString<FileExpandResult>(const char *value);
+
 template<>
 FileGlobOptions EnumUtil::FromString<FileGlobOptions>(const char *value);
 
@@ -1079,6 +1116,9 @@ ResultModifierType EnumUtil::FromString<ResultModifierType>(const char *value);
 template<>
 SampleMethod EnumUtil::FromString<SampleMethod>(const char *value);
 
+template<>
+SampleType EnumUtil::FromString<SampleType>(const char *value);
+
 template<>
 ScanType EnumUtil::FromString<ScanType>(const char *value);
 

package/src/duckdb/src/include/duckdb/common/enums/checkpoint_type.hpp

@@ -0,0 +1,38 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/enums/checkpoint_type.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+
+namespace duckdb {
+
+enum class CheckpointWALAction {
+	//! Delete the WAL file after the checkpoint completes - generally done on shutdown
+	DELETE_WAL,
+	//! Leave the WAL file alone
+	DONT_DELETE_WAL
+};
+
+enum class CheckpointAction {
+	//! Checkpoint only if a checkpoint is required (i.e. the WAL has data in it that can be flushed)
+	CHECKPOINT_IF_REQUIRED,
+	//! Force a checkpoint regardless of whether or not there is data in the WAL to flush
+	FORCE_CHECKPOINT
+};
+
+enum class CheckpointType {
+	//! Full checkpoints involve vacuuming deleted rows and updates
+	//! They can only be run if no transaction need to read old data (that would be cleaned up/vacuumed)
+	FULL_CHECKPOINT,
+	//! Concurrent checkpoints write committed data to disk but do less clean-up
+	//! They can be run even when active transactions need to read old data
+	CONCURRENT_CHECKPOINT
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/logical_operator_type.hpp

@@ -100,6 +100,7 @@ enum class LogicalOperatorType : uint8_t {
 	LOGICAL_SET = 179,
 	LOGICAL_LOAD = 180,
 	LOGICAL_RESET = 181,
+	LOGICAL_UPDATE_EXTENSIONS = 182,
 
 	// -----------------------------
 	// Secrets

package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp

@@ -113,6 +113,7 @@ enum class PhysicalOperatorType : uint8_t {
 	RESET,
 	EXTENSION,
 	VERIFY_VECTOR,
+	UPDATE_EXTENSIONS,
 
 	// -----------------------------
 	// Secret

package/src/duckdb/src/include/duckdb/common/enums/relation_type.hpp

@@ -32,6 +32,7 @@ enum class RelationType : uint8_t {
 	CREATE_TABLE_RELATION,
 	INSERT_RELATION,
 	VALUE_LIST_RELATION,
+	MATERIALIZED_RELATION,
 	DELETE_RELATION,
 	UPDATE_RELATION,
 	WRITE_CSV_RELATION,

package/src/duckdb/src/include/duckdb/common/enums/scan_options.hpp

@@ -21,7 +21,9 @@ enum class TableScanType : uint8_t {
 	TABLE_SCAN_COMMITTED_ROWS_DISALLOW_UPDATES = 2,
 	//! Scan all rows, excluding any permanently deleted rows.
 	//! Permanently deleted rows are rows which no transaction will ever need again.
-	TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED = 3
+	TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED = 3,
+	//! Scan the latest committed rows
+	TABLE_SCAN_LATEST_COMMITTED_ROWS = 4
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/scan_vector_type.hpp

@@ -0,0 +1,17 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/enums/scan_vector_type.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+
+namespace duckdb {
+
+enum class ScanVectorType { SCAN_ENTIRE_VECTOR, SCAN_FLAT_VECTOR };
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/statement_type.hpp

@@ -46,6 +46,7 @@ enum class StatementType : uint8_t {
 	DETACH_STATEMENT,
 	MULTI_STATEMENT,
 	COPY_DATABASE_STATEMENT,
+	UPDATE_EXTENSIONS_STATEMENT,
 };
 
 DUCKDB_API string StatementTypeToString(StatementType type);

package/src/duckdb/src/include/duckdb/common/enums/tableref_type.hpp

@@ -25,7 +25,8 @@ enum class TableReferenceType : uint8_t {
 	CTE = 7,             // Recursive CTE
 	EMPTY_FROM = 8,      // placeholder for empty FROM
 	PIVOT = 9,           // pivot statement
-	SHOW_REF = 10        // SHOW statement
+	SHOW_REF = 10,       // SHOW statement
+	COLUMN_DATA = 11     // column data collection
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/enums/undo_flags.hpp

@@ -17,7 +17,8 @@ enum class UndoFlags : uint32_t { // far too big but aligned (TM)
 	CATALOG_ENTRY = 1,
 	INSERT_TUPLE = 2,
 	DELETE_TUPLE = 3,
-	UPDATE_TUPLE = 4
+	UPDATE_TUPLE = 4,
+	SEQUENCE_VALUE = 5
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/exception.hpp

@@ -10,7 +10,6 @@
 
 #include "duckdb/common/assert.hpp"
 #include "duckdb/common/exception_format_value.hpp"
-#include "duckdb/common/shared_ptr.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/typedefs.hpp"