duckdb 0.10.2 → 0.10.3-dev10.0

package/src/duckdb/src/include/duckdb/common/optionally_owned_ptr.hpp

@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/optionally_owned_ptr.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+
+namespace duckdb {
+
+template <class T>
+class optionally_owned_ptr { // NOLINT: mimic std casing
+public:
+	optionally_owned_ptr() {
+	}
+	optionally_owned_ptr(T *ptr_p) : ptr(ptr_p) { // NOLINT: allow implicit creation from pointer
+	}
+	optionally_owned_ptr(T &ref) : ptr(&ref) { // NOLINT: allow implicit creation from reference
+	}
+	optionally_owned_ptr(unique_ptr<T> &&owned_p) // NOLINT: allow implicit creation from moved unique_ptr
+	    : owned(std::move(owned_p)), ptr(owned) {
+	}
+	// Move constructor
+	optionally_owned_ptr(optionally_owned_ptr &&other) noexcept : owned(std::move(other.owned)), ptr(other.ptr) {
+		other.ptr = nullptr;
+	}
+	// Copy constructor
+	optionally_owned_ptr(const optionally_owned_ptr &other) = delete;
+
+	operator bool() const { // NOLINT: allow implicit conversion to bool
+		return ptr;
+	}
+	T &operator*() {
+		return *ptr;
+	}
+	const T &operator*() const {
+		return *ptr;
+	}
+	T *operator->() {
+		return ptr.get();
+	}
+	const T *operator->() const {
+		return ptr.get();
+	}
+	T *get() { // NOLINT: mimic std casing
+		return ptr.get();
+	}
+	const T *get() const { // NOLINT: mimic std casing
+		return ptr.get();
+	}
+	bool is_owned() const { // NOLINT: mimic std casing
+		return owned != nullptr;
+	}
+	// this looks dirty - but this is the default behavior of raw pointers
+	T *get_mutable() const { // NOLINT: mimic std casing
+		return ptr.get();
+	}
+
+	optionally_owned_ptr<T> &operator=(T &ref) {
+		owned = nullptr;
+		ptr = optional_ptr<T>(ref);
+		return *this;
+	}
+	optionally_owned_ptr<T> &operator=(T *ref) {
+		owned = nullptr;
+		ptr = optional_ptr<T>(ref);
+		return *this;
+	}
+
+	bool operator==(const optionally_owned_ptr<T> &rhs) const {
+		if (owned != rhs.owned) {
+			return false;
+		}
+		return ptr == rhs.ptr;
+	}
+
+	bool operator!=(const optionally_owned_ptr<T> &rhs) const {
+		return !(*this == rhs);
+	}
+
+private:
+	unique_ptr<T> owned;
+	optional_ptr<T> ptr;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/platform.h

@@ -1,5 +1,10 @@
 #include <string>
-#include "duckdb/common/string_util.hpp"
+
+// duplicated from string_util.h to avoid linking issues
+#ifndef DUCKDB_QUOTE_DEFINE
+#define DUCKDB_QUOTE_DEFINE_IMPL(x) #x
+#define DUCKDB_QUOTE_DEFINE(x)      DUCKDB_QUOTE_DEFINE_IMPL(x)
+#endif
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/common/radix.hpp

@@ -117,25 +117,33 @@ inline void Radix::EncodeData(data_ptr_t dataptr, bool value) {
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, int8_t value) {
-	Store<uint8_t>(value, dataptr);
+	uint8_t bytes; // dance around signedness conversion check
+	Store<int8_t>(value, data_ptr_cast(&bytes));
+	Store<uint8_t>(bytes, dataptr);
 	dataptr[0] = FlipSign(dataptr[0]);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, int16_t value) {
-	Store<uint16_t>(BSwap<uint16_t>(value), dataptr);
+	uint16_t bytes;
+	Store<int16_t>(value, data_ptr_cast(&bytes));
+	Store<uint16_t>(BSwap<uint16_t>(bytes), dataptr);
 	dataptr[0] = FlipSign(dataptr[0]);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, int32_t value) {
-	Store<uint32_t>(BSwap<uint32_t>(value), dataptr);
+	uint32_t bytes;
+	Store<int32_t>(value, data_ptr_cast(&bytes));
+	Store<uint32_t>(BSwap<uint32_t>(bytes), dataptr);
 	dataptr[0] = FlipSign(dataptr[0]);
 }
 
 template <>
 inline void Radix::EncodeData(data_ptr_t dataptr, int64_t value) {
-	Store<uint64_t>(BSwap<uint64_t>(value), dataptr);
+	uint64_t bytes;
+	Store<int64_t>(value, data_ptr_cast(&bytes));
+	Store<uint64_t>(BSwap<uint64_t>(bytes), dataptr);
 	dataptr[0] = FlipSign(dataptr[0]);
 }
 

package/src/duckdb/src/include/duckdb/common/re2_regex.hpp

@@ -4,7 +4,8 @@
 
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/common/vector.hpp"
-#include <string>
+#include "duckdb/common/shared_ptr.hpp"
+#include "duckdb/common/string.hpp"
 #include <stdexcept>
 
 namespace duckdb_re2 {
@@ -22,7 +23,7 @@ public:
 	}
 
 private:
-	std::shared_ptr<duckdb_re2::RE2> regex;
+	duckdb::shared_ptr<duckdb_re2::RE2> regex;
 };
 
 struct GroupMatch {

package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp

@@ -31,12 +31,14 @@ class NestedValidity {
 	data_ptr_t *struct_validity_locations;
 	idx_t entry_idx;
 	idx_t idx_in_entry;
+	idx_t list_validity_offset;
 
 public:
 	explicit NestedValidity(data_ptr_t validitymask_location);
 	NestedValidity(data_ptr_t *validitymask_locations, idx_t child_vector_index);
 	void SetInvalid(idx_t idx);
 	bool IsValid(idx_t idx);
+	void OffsetListBy(idx_t offset);
 };
 
 struct RowOperationsState {

package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp

@@ -89,6 +89,7 @@ private:
 	}
 
 	void ReadData(data_ptr_t buffer, idx_t read_size) {
+		D_ASSERT(!has_buffered_field);
 		stream.ReadData(buffer, read_size);
 	}
 
@@ -102,7 +103,7 @@ private:
 	template <class T>
 	T VarIntDecode() {
 		// FIXME: maybe we should pass a source to EncodingUtil instead
-		uint8_t buffer[16];
+		uint8_t buffer[16] = {};
 		idx_t varint_size;
 		for (varint_size = 0; varint_size < 16; varint_size++) {
 			ReadData(buffer + varint_size, 1);

package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp

@@ -17,9 +17,11 @@ namespace duckdb {
 
 class BinarySerializer : public Serializer {
 public:
-	explicit BinarySerializer(WriteStream &stream, bool serialize_default_values_p = false) : stream(stream) {
-		serialize_default_values = serialize_default_values_p;
-		serialize_enum_as_string = false;
+	explicit BinarySerializer(WriteStream &stream, SerializationOptions options_p = SerializationOptions())
+	    : stream(stream) {
+		options = std::move(options_p);
+		// Override the value set by the passed in SerializationOptions
+		options.serialize_enum_as_string = false;
 	}
 
 private:
@@ -44,7 +46,7 @@ private:
 
 	template <class T>
 	void VarIntEncode(T value) {
-		uint8_t buffer[16];
+		uint8_t buffer[16] = {};
 		auto write_size = EncodingUtil::EncodeLEB128<T>(buffer, value);
 		D_ASSERT(write_size <= sizeof(buffer));
 		WriteData(buffer, write_size);
@@ -52,8 +54,8 @@ private:
 
 public:
 	template <class T>
-	static void Serialize(const T &value, WriteStream &stream, bool serialize_default_values = false) {
-		BinarySerializer serializer(stream, serialize_default_values);
+	static void Serialize(const T &value, WriteStream &stream, SerializationOptions options = SerializationOptions()) {
+		BinarySerializer serializer(stream, std::move(options));
 		serializer.OnObjectBegin();
 		value.Serialize(serializer);
 		serializer.OnObjectEnd();

package/src/duckdb/src/include/duckdb/common/serializer/deserializer.hpp

@@ -196,6 +196,12 @@ private:
 		return val;
 	}
 
+	// Deserialize a optionally_owned_ptr
+	template <class T, typename ELEMENT_TYPE = typename is_optionally_owned_ptr<T>::ELEMENT_TYPE>
+	inline typename std::enable_if<is_optionally_owned_ptr<T>::value, T>::type Read() {
+		return optionally_owned_ptr<ELEMENT_TYPE>(Read<unique_ptr<ELEMENT_TYPE>>());
+	}
+
 	// Deserialize unique_ptr if the element type has a Deserialize method
 	template <class T, typename ELEMENT_TYPE = typename is_unique_ptr<T>::ELEMENT_TYPE>
 	inline typename std::enable_if<is_unique_ptr<T>::value && has_deserialize<ELEMENT_TYPE>::value, T>::type Read() {
@@ -302,6 +308,23 @@ private:
 		return map;
 	}
 
+	template <typename T = void>
+	inline typename std::enable_if<is_insertion_preserving_map<T>::value, T>::type Read() {
+		using VALUE_TYPE = typename is_insertion_preserving_map<T>::VALUE_TYPE;
+
+		T map;
+		auto size = OnListBegin();
+		for (idx_t i = 0; i < size; i++) {
+			OnObjectBegin();
+			auto key = ReadProperty<string>(0, "key");
+			auto value = ReadProperty<VALUE_TYPE>(1, "value");
+			OnObjectEnd();
+			map[key] = std::move(value);
+		}
+		OnListEnd();
+		return map;
+	}
+
 	// Deserialize an unordered set
 	template <typename T = void>
 	inline typename std::enable_if<is_unordered_set<T>::value, T>::type Read() {
@@ -340,6 +363,19 @@ private:
 		return std::make_pair(first, second);
 	}
 
+	// Deserialize a priority_queue
+	template <typename T = void>
+	inline typename std::enable_if<is_queue<T>::value, T>::type Read() {
+		using ELEMENT_TYPE = typename is_queue<T>::ELEMENT_TYPE;
+		T queue;
+		auto size = OnListBegin();
+		for (idx_t i = 0; i < size; i++) {
+			queue.emplace(Read<ELEMENT_TYPE>());
+		}
+		OnListEnd();
+		return queue;
+	}
+
 	// Primitive types
 	// Deserialize a bool
 	template <typename T = void>

package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp

@@ -10,8 +10,11 @@
 #include "duckdb/common/set.hpp"
 #include "duckdb/common/shared_ptr.hpp"
 #include "duckdb/common/unique_ptr.hpp"
+#include "duckdb/common/queue.hpp"
 #include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/common/optionally_owned_ptr.hpp"
 #include "duckdb/common/optional_idx.hpp"
+#include "duckdb/common/insertion_order_preserving_map.hpp"
 
 namespace duckdb {
 
@@ -50,6 +53,13 @@ struct has_deserialize<
     T, typename std::enable_if<std::is_same<decltype(T::Deserialize), shared_ptr<T>(Deserializer &)>::value, T>::type>
     : std::true_type {};
 
+// Accept `static shared_ptr<T> Deserialize(Deserializer& deserializer)`
+template <typename T>
+struct has_deserialize<
+    T,
+    typename std::enable_if<std::is_same<decltype(T::Deserialize), std::shared_ptr<T>(Deserializer &)>::value, T>::type>
+    : std::true_type {};
+
 // Accept `static T Deserialize(Deserializer& deserializer)`
 template <typename T>
 struct has_deserialize<
@@ -92,6 +102,20 @@ struct is_map<typename duckdb::map<Args...>> : std::true_type {
 	typedef typename std::tuple_element<3, std::tuple<Args...>>::type EQUAL_TYPE;
 };
 
+template <typename T>
+struct is_insertion_preserving_map : std::false_type {};
+template <typename... Args>
+struct is_insertion_preserving_map<typename duckdb::InsertionOrderPreservingMap<Args...>> : std::true_type {
+	typedef typename std::tuple_element<0, std::tuple<Args...>>::type VALUE_TYPE;
+};
+
+template <typename T>
+struct is_queue : std::false_type {};
+template <typename T>
+struct is_queue<typename std::priority_queue<T>> : std::true_type {
+	typedef T ELEMENT_TYPE;
+};
+
 template <typename T>
 struct is_unique_ptr : std::false_type {};
 template <typename T>
@@ -105,6 +129,10 @@ template <typename T>
 struct is_shared_ptr<shared_ptr<T>> : std::true_type {
 	typedef T ELEMENT_TYPE;
 };
+template <typename T>
+struct is_shared_ptr<std::shared_ptr<T>> : std::true_type {
+	typedef T ELEMENT_TYPE;
+};
 
 template <typename T>
 struct is_optional_ptr : std::false_type {};
@@ -113,6 +141,13 @@ struct is_optional_ptr<optional_ptr<T>> : std::true_type {
 	typedef T ELEMENT_TYPE;
 };
 
+template <typename T>
+struct is_optionally_owned_ptr : std::false_type {};
+template <typename T>
+struct is_optionally_owned_ptr<optionally_owned_ptr<T>> : std::true_type {
+	typedef T ELEMENT_TYPE;
+};
+
 template <typename T>
 struct is_pair : std::false_type {};
 template <typename T, typename U>
@@ -193,6 +228,16 @@ struct SerializationDefaultValue {
 		return !value;
 	}
 
+	template <typename T = void>
+	static inline typename std::enable_if<is_optionally_owned_ptr<T>::value, T>::type GetDefault() {
+		return T();
+	}
+
+	template <typename T = void>
+	static inline bool IsDefault(const typename std::enable_if<is_optionally_owned_ptr<T>::value, T>::type &value) {
+		return !value;
+	}
+
 	template <typename T = void>
 	static inline typename std::enable_if<is_shared_ptr<T>::value, T>::type GetDefault() {
 		return T();
@@ -213,6 +258,16 @@ struct SerializationDefaultValue {
 		return value.empty();
 	}
 
+	template <typename T = void>
+	static inline typename std::enable_if<is_queue<T>::value, T>::type GetDefault() {
+		return T();
+	}
+
+	template <typename T = void>
+	static inline bool IsDefault(const typename std::enable_if<is_queue<T>::value, T>::type &value) {
+		return value.empty();
+	}
+
 	template <typename T = void>
 	static inline typename std::enable_if<is_unsafe_vector<T>::value, T>::type GetDefault() {
 		return T();
@@ -253,6 +308,16 @@ struct SerializationDefaultValue {
 		return value.empty();
 	}
 
+	template <typename T = void>
+	static inline typename std::enable_if<is_insertion_preserving_map<T>::value, T>::type GetDefault() {
+		return T();
+	}
+
+	template <typename T = void>
+	static inline bool IsDefault(const typename std::enable_if<is_insertion_preserving_map<T>::value, T>::type &value) {
+		return value.empty();
+	}
+
 	template <typename T = void>
 	static inline typename std::enable_if<std::is_same<T, string>::value, T>::type GetDefault() {
 		return T();

package/src/duckdb/src/include/duckdb/common/serializer/serializer.hpp

@@ -16,20 +16,33 @@
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
 #include "duckdb/common/optional_idx.hpp"
+#include "duckdb/common/optionally_owned_ptr.hpp"
 #include "duckdb/common/value_operations/value_operations.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_option.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/common/insertion_order_preserving_map.hpp"
 
 namespace duckdb {
 
-class Serializer {
-protected:
+class SerializationOptions {
+public:
 	bool serialize_enum_as_string = false;
 	bool serialize_default_values = false;
+	SerializationCompatibility serialization_compatibility = SerializationCompatibility::Default();
+};
+
+class Serializer {
+protected:
+	SerializationOptions options;
 
 public:
 	virtual ~Serializer() {
 	}
 
+	bool ShouldSerialize(idx_t version_added) {
+		return options.serialization_compatibility.Compare(version_added);
+	}
+
 	class List {
 		friend Serializer;
 
@@ -64,7 +77,7 @@ public:
 	template <class T>
 	void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const T &value) {
 		// If current value is default, don't write it
-		if (!serialize_default_values && SerializationDefaultValue::IsDefault<T>(value)) {
+		if (!options.serialize_default_values && SerializationDefaultValue::IsDefault<T>(value)) {
 			OnOptionalPropertyBegin(field_id, tag, false);
 			OnOptionalPropertyEnd(false);
 			return;
@@ -77,7 +90,7 @@ public:
 	template <class T>
 	void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const T &value, const T &&default_value) {
 		// If current value is default, don't write it
-		if (!serialize_default_values && (value == default_value)) {
+		if (!options.serialize_default_values && (value == default_value)) {
 			OnOptionalPropertyBegin(field_id, tag, false);
 			OnOptionalPropertyEnd(false);
 			return;
@@ -92,7 +105,7 @@ public:
 	void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const CSVOption<T> &value,
 	                              const T &&default_value) {
 		// If current value is default, don't write it
-		if (!serialize_default_values && (value == default_value)) {
+		if (!options.serialize_default_values && (value == default_value)) {
 			OnOptionalPropertyBegin(field_id, tag, false);
 			OnOptionalPropertyEnd(false);
 			return;
@@ -134,7 +147,7 @@ public:
 protected:
 	template <typename T>
 	typename std::enable_if<std::is_enum<T>::value, void>::type WriteValue(const T value) {
-		if (serialize_enum_as_string) {
+		if (options.serialize_enum_as_string) {
 			// Use the enum serializer to lookup tostring function
 			auto str = EnumUtil::ToChars(value);
 			WriteValue(str);
@@ -144,6 +157,12 @@ protected:
 		}
 	}
 
+	// Optionally Owned Pointer Ref
+	template <typename T>
+	void WriteValue(const optionally_owned_ptr<T> &ptr) {
+		WriteValue(ptr.get());
+	}
+
 	// Unique Pointer Ref
 	template <typename T>
 	void WriteValue(const unique_ptr<T> &ptr) {
@@ -259,6 +278,33 @@ protected:
 		OnListEnd();
 	}
 
+	// Insertion Order Preserving Map
+	// serialized as a list of pairs
+	template <class V>
+	void WriteValue(const duckdb::InsertionOrderPreservingMap<V> &map) {
+		auto count = map.size();
+		OnListBegin(count);
+		for (auto &entry : map) {
+			OnObjectBegin();
+			WriteProperty(0, "key", entry.first);
+			WriteProperty(1, "value", entry.second);
+			OnObjectEnd();
+		}
+		OnListEnd();
+	}
+
+	// priority queue
+	template <typename T>
+	void WriteValue(const std::priority_queue<T> &queue) {
+		vector<T> placeholder;
+		auto queue_copy = std::priority_queue<T>(queue);
+		while (queue_copy.size() > 0) {
+			placeholder.emplace_back(queue_copy.top());
+			queue_copy.pop();
+		}
+		WriteValue(placeholder);
+	}
+
 	// class or struct implementing `Serialize(Serializer& Serializer)`;
 	template <typename T>
 	typename std::enable_if<has_serialize<T>::value>::type WriteValue(const T &value) {

package/src/duckdb/src/include/duckdb/common/shared_ptr.hpp

@@ -8,12 +8,45 @@
 
 #pragma once
 
+#include "duckdb/common/unique_ptr.hpp"
+#include "duckdb/common/likely.hpp"
+#include "duckdb/common/memory_safety.hpp"
+
 #include <memory>
+#include <type_traits>
 
 namespace duckdb {
 
-using std::make_shared;
-using std::shared_ptr;
-using std::weak_ptr;
+// This implementation is taken from the llvm-project, at this commit hash:
+// https://github.com/llvm/llvm-project/blob/08bb121835be432ac52372f92845950628ce9a4a/libcxx/include/__memory/shared_ptr.h#353
+// originally named '__compatible_with'
+
+#if _LIBCPP_STD_VER >= 17
+template <class U, class T>
+struct __bounded_convertible_to_unbounded : std::false_type {};
+
+template <class _Up, std::size_t _Np, class T>
+struct __bounded_convertible_to_unbounded<_Up[_Np], T> : std::is_same<std::remove_cv<T>, _Up[]> {};
+
+template <class U, class T>
+struct compatible_with_t : std::_Or<std::is_convertible<U *, T *>, __bounded_convertible_to_unbounded<U, T>> {};
+#else
+template <class U, class T>
+struct compatible_with_t : std::is_convertible<U *, T *> {}; // NOLINT: invalid case style
+#endif // _LIBCPP_STD_VER >= 17
+
+} // namespace duckdb
+
+#include "duckdb/common/shared_ptr.ipp"
+#include "duckdb/common/weak_ptr.ipp"
+#include "duckdb/common/enable_shared_from_this.ipp"
+
+namespace duckdb {
+
+template <typename T>
+using unsafe_shared_ptr = shared_ptr<T, false>;
+
+template <typename T>
+using unsafe_weak_ptr = weak_ptr<T, false>;
 
 } // namespace duckdb