duckdb 0.7.2-dev0.0 → 0.7.2-dev1138.0

package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp

@@ -0,0 +1,268 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/format_serializer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/enum_serializer.hpp"
+#include "duckdb/common/serializer/serialization_traits.hpp"
+#include "duckdb/common/types/interval.hpp"
+#include "duckdb/common/types/string_type.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+
+namespace duckdb {
+
+class FormatSerializer {
+protected:
+	bool serialize_enum_as_string = false;
+
+public:
+	// Serialize a value
+	template <class T>
+	typename std::enable_if<!std::is_enum<T>::value, void>::type WriteProperty(const char *tag, T &value) {
+		SetTag(tag);
+		WriteValue(value);
+	}
+
+	// Serialize a range
+	template <class T>
+	typename std::enable_if<std::is_pointer<T>::value, void>::type WriteProperty(const char *tag, const T start,
+	                                                                             idx_t count) {
+		SetTag(tag);
+		WriteValue(start, count);
+	}
+
+	// Serialize an enum
+	template <class T>
+	typename std::enable_if<std::is_enum<T>::value, void>::type WriteProperty(const char *tag, T value) {
+		SetTag(tag);
+		if (serialize_enum_as_string) {
+			// Use the enum serializer to lookup tostring function
+			auto str = EnumSerializer::EnumToString(value);
+			WriteValue(str);
+		} else {
+			// Use the underlying type
+			WriteValue(static_cast<typename std::underlying_type<T>::type>(value));
+		}
+	}
+
+	// Optional pointer
+	template <class T>
+	void WriteOptionalProperty(const char *tag, T *ptr) {
+		SetTag(tag);
+		if (ptr == nullptr) {
+			OnOptionalBegin(false);
+			OnOptionalEnd(false);
+		} else {
+			OnOptionalBegin(true);
+			WriteValue(*ptr);
+			OnOptionalEnd(true);
+		}
+	}
+
+	// Optional unique_ptr
+	template <class T>
+	void WriteOptionalProperty(const char *tag, const unique_ptr<T> &ptr) {
+		SetTag(tag);
+		if (ptr == nullptr) {
+			OnOptionalBegin(false);
+			OnOptionalEnd(false);
+		} else {
+			OnOptionalBegin(true);
+			WriteValue(*ptr);
+			OnOptionalEnd(true);
+		}
+	}
+
+protected:
+	// Unique Pointer Ref
+	template <typename T>
+	void WriteValue(const unique_ptr<T> &ptr) {
+		WriteValue(ptr.get());
+	}
+
+	// Pointer
+	template <typename T>
+	typename std::enable_if<std::is_pointer<T>::value, void>::type WriteValue(const T ptr) {
+		if (ptr == nullptr) {
+			WriteNull();
+		} else {
+			WriteValue(*ptr);
+		}
+	}
+
+	// data_ptr_t
+	void WriteValue(data_ptr_t ptr, idx_t count) {
+		OnListBegin(count);
+		auto end = ptr + count;
+		while (ptr != end) {
+			WriteValue(*ptr);
+			ptr++;
+		}
+		OnListEnd(count);
+	}
+
+	void WriteValue(const_data_ptr_t ptr, idx_t count) {
+		OnListBegin(count);
+		auto end = ptr + count;
+		while (ptr != end) {
+			WriteValue(*ptr);
+			ptr++;
+		}
+		OnListEnd(count);
+	}
+
+	// Pair
+	template <class K, class V>
+	void WriteValue(const std::pair<K, V> &pair) {
+		OnPairBegin();
+		OnPairKeyBegin();
+		WriteValue(pair.first);
+		OnPairKeyEnd();
+		OnPairValueBegin();
+		WriteValue(pair.second);
+		OnPairValueEnd();
+		OnPairEnd();
+	}
+
+	// Vector
+	template <class T>
+	void WriteValue(const vector<T> &vec) {
+		auto count = vec.size();
+		OnListBegin(count);
+		for (auto &item : vec) {
+			WriteValue(item);
+		}
+		OnListEnd(count);
+	}
+
+	// UnorderedSet
+	// Serialized the same way as a list/vector
+	template <class T, class HASH, class CMP>
+	void WriteValue(const unordered_set<T, HASH, CMP> &set) {
+		auto count = set.size();
+		OnListBegin(count);
+		for (auto &item : set) {
+			WriteValue(item);
+		}
+		OnListEnd(count);
+	}
+
+	// Set
+	// Serialized the same way as a list/vector
+	template <class T, class HASH, class CMP>
+	void WriteValue(const set<T, HASH, CMP> &set) {
+		auto count = set.size();
+		OnListBegin(count);
+		for (auto &item : set) {
+			WriteValue(item);
+		}
+		OnListEnd(count);
+	}
+
+	// Map
+	template <class K, class V, class HASH, class CMP>
+	void WriteValue(const std::unordered_map<K, V, HASH, CMP> &map) {
+		auto count = map.size();
+		OnMapBegin(count);
+		for (auto &item : map) {
+			OnMapEntryBegin();
+			OnMapKeyBegin();
+			WriteValue(item.first);
+			OnMapKeyEnd();
+			OnMapValueBegin();
+			WriteValue(item.second);
+			OnMapValueEnd();
+			OnMapEntryEnd();
+		}
+		OnMapEnd(count);
+	}
+
+	// class or struct implementing `FormatSerialize(FormatSerializer& FormatSerializer)`;
+	template <typename T>
+	typename std::enable_if<has_serialize_v<T>(), void>::type WriteValue(T &value) {
+		// Else, we defer to the .FormatSerialize method
+		OnObjectBegin();
+		value.FormatSerialize(*this);
+		OnObjectEnd();
+	}
+
+	// Handle setting a "tag" (optional)
+	virtual void SetTag(const char *tag) {
+		(void)tag;
+	}
+
+	// Hooks for subclasses to override to implement custom behavior
+	virtual void OnListBegin(idx_t count) {
+		(void)count;
+	}
+	virtual void OnListEnd(idx_t count) {
+		(void)count;
+	}
+	virtual void OnMapBegin(idx_t count) {
+		(void)count;
+	}
+	virtual void OnMapEnd(idx_t count) {
+		(void)count;
+	}
+	virtual void OnMapEntryBegin() {
+	}
+	virtual void OnMapEntryEnd() {
+	}
+	virtual void OnMapKeyBegin() {
+	}
+	virtual void OnMapKeyEnd() {
+	}
+	virtual void OnMapValueBegin() {
+	}
+	virtual void OnMapValueEnd() {
+	}
+	virtual void OnOptionalBegin(bool present) {
+	}
+	virtual void OnOptionalEnd(bool present) {
+	}
+	virtual void OnObjectBegin() {
+	}
+	virtual void OnObjectEnd() {
+	}
+	virtual void OnPairBegin() {
+	}
+	virtual void OnPairKeyBegin() {
+	}
+	virtual void OnPairKeyEnd() {
+	}
+	virtual void OnPairValueBegin() {
+	}
+	virtual void OnPairValueEnd() {
+	}
+	virtual void OnPairEnd() {
+	}
+
+	// Handle primitive types, a serializer needs to implement these.
+	virtual void WriteNull() = 0;
+	virtual void WriteValue(uint8_t value) = 0;
+	virtual void WriteValue(int8_t value) = 0;
+	virtual void WriteValue(uint16_t value) = 0;
+	virtual void WriteValue(int16_t value) = 0;
+	virtual void WriteValue(uint32_t value) = 0;
+	virtual void WriteValue(int32_t value) = 0;
+	virtual void WriteValue(uint64_t value) = 0;
+	virtual void WriteValue(int64_t value) = 0;
+	virtual void WriteValue(hugeint_t value) = 0;
+	virtual void WriteValue(float value) = 0;
+	virtual void WriteValue(double value) = 0;
+	virtual void WriteValue(interval_t value) = 0;
+	virtual void WriteValue(const string &value) = 0;
+	virtual void WriteValue(const string_t value) = 0;
+	virtual void WriteValue(const char *value) = 0;
+	virtual void WriteValue(bool value) = 0;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp

@@ -0,0 +1,126 @@
+#pragma once
+#include <type_traits>
+#include "duckdb/common/vector.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+
+namespace duckdb {
+
+class FormatSerializer;   // Forward declare
+class FormatDeserializer; // Forward declare
+
+// Backport to c++11
+template <class...>
+using void_t = void;
+
+// Check for anything implementing a `void FormatSerialize(FormatSerializer &FormatSerializer)` method
+template <typename T, typename = void_t<>>
+struct has_serialize : std::false_type {};
+
+template <typename T>
+struct has_serialize<T, void_t<decltype(std::declval<T>().FormatSerialize(std::declval<FormatSerializer &>()))>>
+    : std::true_type {};
+
+template <typename T>
+constexpr bool has_serialize_v() {
+	return has_serialize<T>::value;
+}
+
+// Check for anything implementing a static `T FormatDeserialize(FormatDeserializer&)` method
+/*
+template <typename T, typename = void_t<>>
+struct has_deserialize : std::false_type {};
+
+template <typename T>
+struct has_deserialize<T, void_t<decltype(T::FormatDeserialize)>> : std::true_type {};
+
+template <typename T>
+constexpr bool has_deserialize_v() {
+    return has_deserialize<T>::value;
+}*/
+
+template <typename T, typename = T>
+struct has_deserialize : std::false_type {};
+
+// Accept `static unique_ptr<T> FormatDeserialize(FormatDeserializer& deserializer)`
+template <typename T>
+struct has_deserialize<
+    T, typename std::enable_if<std::is_same<decltype(T::FormatDeserialize), unique_ptr<T>(FormatDeserializer &)>::value,
+                               T>::type> : std::true_type {};
+
+// Accept `static shared_ptr<T> FormatDeserialize(FormatDeserializer& deserializer)`
+template <typename T>
+struct has_deserialize<
+    T, typename std::enable_if<std::is_same<decltype(T::FormatDeserialize), shared_ptr<T>(FormatDeserializer &)>::value,
+                               T>::type> : std::true_type {};
+
+// Accept `static T FormatDeserialize(FormatDeserializer& deserializer)`
+template <typename T>
+struct has_deserialize<
+    T, typename std::enable_if<std::is_same<decltype(T::FormatDeserialize), T(FormatDeserializer &)>::value, T>::type>
+    : std::true_type {};
+
+// Check if T is a vector, and provide access to the inner type
+template <typename T>
+struct is_vector : std::false_type {};
+template <typename T>
+struct is_vector<typename std::vector<T>> : std::true_type {
+	typedef T ELEMENT_TYPE;
+};
+
+// Check if T is a unordered map, and provide access to the inner type
+template <typename T>
+struct is_unordered_map : std::false_type {};
+template <typename... Args>
+struct is_unordered_map<typename std::unordered_map<Args...>> : std::true_type {
+	typedef typename std::tuple_element<0, std::tuple<Args...>>::type KEY_TYPE;
+	typedef typename std::tuple_element<1, std::tuple<Args...>>::type VALUE_TYPE;
+	typedef typename std::tuple_element<2, std::tuple<Args...>>::type HASH_TYPE;
+	typedef typename std::tuple_element<3, std::tuple<Args...>>::type EQUAL_TYPE;
+};
+
+template <typename T>
+struct is_unique_ptr : std::false_type {};
+
+template <typename T, typename D>
+struct is_unique_ptr<unique_ptr<T, D>> : std::true_type {
+	typedef T ELEMENT_TYPE;
+	typedef D DELETER_TYPE;
+};
+
+template <typename T>
+struct is_shared_ptr : std::false_type {};
+
+template <typename T>
+struct is_shared_ptr<shared_ptr<T>> : std::true_type {
+	typedef T ELEMENT_TYPE;
+};
+
+template <typename T>
+struct is_pair : std::false_type {};
+
+template <typename T, typename U>
+struct is_pair<std::pair<T, U>> : std::true_type {
+	typedef T FIRST_TYPE;
+	typedef U SECOND_TYPE;
+};
+
+template <typename T>
+struct is_unordered_set : std::false_type {};
+template <typename... Args>
+struct is_unordered_set<std::unordered_set<Args...>> : std::true_type {
+	typedef typename std::tuple_element<0, std::tuple<Args...>>::type ELEMENT_TYPE;
+	typedef typename std::tuple_element<1, std::tuple<Args...>>::type HASH_TYPE;
+	typedef typename std::tuple_element<2, std::tuple<Args...>>::type EQUAL_TYPE;
+};
+
+template <typename T>
+struct is_set : std::false_type {};
+template <typename... Args>
+struct is_set<std::set<Args...>> : std::true_type {
+	typedef typename std::tuple_element<0, std::tuple<Args...>>::type ELEMENT_TYPE;
+	typedef typename std::tuple_element<1, std::tuple<Args...>>::type HASH_TYPE;
+	typedef typename std::tuple_element<2, std::tuple<Args...>>::type EQUAL_TYPE;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "duckdb/catalog/catalog.hpp"
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/vector.hpp"
@@ -21,6 +22,8 @@ private:
 	uint64_t version = 0L;
 
 public:
+	bool is_query_plan = false;
+
 	virtual ~Serializer() {
 	}
 
@@ -111,6 +114,16 @@ public:
 	//! Reads [read_size] bytes into the buffer
 	virtual void ReadData(data_ptr_t buffer, idx_t read_size) = 0;
 
+	//! Gets the context for the deserializer
+	virtual ClientContext &GetContext() {
+		throw InternalException("This deserializer does not have a client-context");
+	};
+
+	//! Gets the catalog for the deserializer
+	virtual Catalog *GetCatalog() {
+		return nullptr;
+	};
+
 	template <class T>
 	T Read() {
 		T value;

package/src/duckdb/src/include/duckdb/common/string_util.hpp

@@ -21,6 +21,19 @@ namespace duckdb {
  */
 class StringUtil {
 public:
+	static uint8_t GetHexValue(char c) {
+		if (c >= '0' && c <= '9') {
+			return c - '0';
+		}
+		if (c >= 'a' && c <= 'f') {
+			return c - 'a' + 10;
+		}
+		if (c >= 'A' && c <= 'F') {
+			return c - 'A' + 10;
+		}
+		throw InvalidInputException("Invalid input for hex digit: %s", string(c, 1));
+	}
+
 	DUCKDB_API static bool CharacterIsSpace(char c) {
 		return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r';
 	}
@@ -170,6 +183,18 @@ public:
 	//! Equivalent to calling TopNLevenshtein followed by CandidatesMessage
 	DUCKDB_API static string CandidatesErrorMessage(const vector<string> &strings, const string &target,
 	                                                const string &message_prefix, idx_t n = 5);
+
+	//! Returns true if two null-terminated strings are equal or point to the same address.
+	//! Returns false if only one of the strings is nullptr
+	DUCKDB_API static bool Equals(const char *s1, const char *s2) {
+		if (s1 == s2) {
+			return true;
+		}
+		if (s1 == nullptr || s2 == nullptr) {
+			return false;
+		}
+		return strcmp(s1, s2) == 0;
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/bit.hpp

@@ -25,7 +25,7 @@ public:
 	//! Extracts the nth bit from bit string; the first (leftmost) bit is indexed 0
 	DUCKDB_API static idx_t GetBit(string_t bit_string, idx_t n);
 	//! Sets the nth bit in bit string to newvalue; the first (leftmost) bit is indexed 0
-	DUCKDB_API static void SetBit(const string_t &bit_string, idx_t n, idx_t new_value, string_t &result);
+	DUCKDB_API static void SetBit(string_t &bit_string, idx_t n, idx_t new_value);
 	//! Returns first starting index of the specified substring within bits, or zero if it's not present.
 	DUCKDB_API static idx_t BitPosition(string_t substring, string_t bits);
 	//! Converts bits to a string, writing the output to the designated output string.
@@ -36,9 +36,13 @@ public:
 	DUCKDB_API static bool TryGetBitStringSize(string_t str, idx_t &result_size, string *error_message);
 	//! Convert a string to a bit. This function should ONLY be called after calling GetBitSize, since it does NOT
 	//! perform data validation.
-	DUCKDB_API static void ToBit(string_t str, data_ptr_t output);
+	DUCKDB_API static void ToBit(string_t str, string_t &output);
 	DUCKDB_API static string ToBit(string_t str);
+	//! Creates a new bitstring of determined length
+	DUCKDB_API static void BitString(const string_t &input, const idx_t &len, string_t &result);
 	DUCKDB_API static void SetEmptyBitString(string_t &target, string_t &input);
+	DUCKDB_API static void SetEmptyBitString(string_t &target, idx_t len);
+	DUCKDB_API static idx_t ComputeBitstringLen(idx_t len);
 
 	DUCKDB_API static void RightShift(const string_t &bit_string, const idx_t &shif, string_t &result);
 	DUCKDB_API static void LeftShift(const string_t &bit_string, const idx_t &shift, string_t &result);
@@ -47,11 +51,12 @@ public:
 	DUCKDB_API static void BitwiseXor(const string_t &rhs, const string_t &lhs, string_t &result);
 	DUCKDB_API static void BitwiseNot(const string_t &rhs, string_t &result);
 
+	DUCKDB_API static void Verify(const string_t &input);
+
 private:
-	//! Returns the amount of padded zeroes to fill up to a full byte. This information is stored in the first byte of
-	//! the bitstring.
-	DUCKDB_API static idx_t GetPadding(const string_t &bit_string);
-	DUCKDB_API static idx_t GetBitSize(string_t str);
-	DUCKDB_API static void SetBit(string_t &bit_string, idx_t n, idx_t new_value);
+	static void Finalize(string_t &str);
+	static idx_t GetBitInternal(string_t bit_string, idx_t n);
+	static void SetBitInternal(string_t &bit_string, idx_t n, idx_t new_value);
+	static idx_t GetBitIndex(idx_t n);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/interval.hpp

@@ -101,8 +101,44 @@ public:
 	static dtime_t Add(dtime_t left, interval_t right, date_t &date);
 
 	//! Comparison operators
-	static bool Equals(interval_t left, interval_t right);
-	static bool GreaterThan(interval_t left, interval_t right);
-	static bool GreaterThanEquals(interval_t left, interval_t right);
+	inline static bool Equals(const interval_t &left, const interval_t &right);
+	inline static bool GreaterThan(const interval_t &left, const interval_t &right);
 };
+static void NormalizeIntervalEntries(interval_t input, int64_t &months, int64_t &days, int64_t &micros) {
+	int64_t extra_months_d = input.days / Interval::DAYS_PER_MONTH;
+	int64_t extra_months_micros = input.micros / Interval::MICROS_PER_MONTH;
+	input.days -= extra_months_d * Interval::DAYS_PER_MONTH;
+	input.micros -= extra_months_micros * Interval::MICROS_PER_MONTH;
+
+	int64_t extra_days_micros = input.micros / Interval::MICROS_PER_DAY;
+	input.micros -= extra_days_micros * Interval::MICROS_PER_DAY;
+
+	months = input.months + extra_months_d + extra_months_micros;
+	days = input.days + extra_days_micros;
+	micros = input.micros;
+}
+
+bool Interval::Equals(const interval_t &left, const interval_t &right) {
+	return left.months == right.months && left.days == right.days && left.micros == right.micros;
+}
+
+bool Interval::GreaterThan(const interval_t &left, const interval_t &right) {
+	int64_t lmonths, ldays, lmicros;
+	int64_t rmonths, rdays, rmicros;
+	NormalizeIntervalEntries(left, lmonths, ldays, lmicros);
+	NormalizeIntervalEntries(right, rmonths, rdays, rmicros);
+
+	if (lmonths > rmonths) {
+		return true;
+	} else if (lmonths < rmonths) {
+		return false;
+	}
+	if (ldays > rdays) {
+		return true;
+	} else if (ldays < rdays) {
+		return false;
+	}
+	return lmicros > rmicros;
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp

@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/types/list_segment.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#include "duckdb/common/allocator.hpp"
+#include "duckdb/common/constants.hpp"
+#include "duckdb/common/types/vector.hpp"
+
+#pragma once
+
+namespace duckdb {
+
+struct ListSegment {
+	uint16_t count;
+	uint16_t capacity;
+	ListSegment *next;
+};
+struct LinkedList {
+	LinkedList() {};
+	LinkedList(idx_t total_capacity_p, ListSegment *first_segment_p, ListSegment *last_segment_p)
+	    : total_capacity(total_capacity_p), first_segment(first_segment_p), last_segment(last_segment_p) {
+	}
+
+	idx_t total_capacity = 0;
+	ListSegment *first_segment = nullptr;
+	ListSegment *last_segment = nullptr;
+};
+
+// forward declarations
+struct WriteDataToSegment;
+struct ReadDataFromSegment;
+struct CopyDataFromSegment;
+typedef ListSegment *(*create_segment_t)(WriteDataToSegment &write_data_to_segment, Allocator &allocator,
+                                         vector<AllocatedData> &owning_vector, const uint16_t &capacity);
+typedef void (*write_data_to_segment_t)(WriteDataToSegment &write_data_to_segment, Allocator &allocator,
+                                        vector<AllocatedData> &owning_vector, ListSegment *segment, Vector &input,
+                                        idx_t &entry_idx, idx_t &count);
+typedef void (*read_data_from_segment_t)(ReadDataFromSegment &read_data_from_segment, const ListSegment *segment,
+                                         Vector &result, idx_t &total_count);
+typedef ListSegment *(*copy_data_from_segment_t)(CopyDataFromSegment &copy_data_from_segment, const ListSegment *source,
+                                                 Allocator &allocator, vector<AllocatedData> &owning_vector);
+
+struct WriteDataToSegment {
+	create_segment_t create_segment;
+	write_data_to_segment_t segment_function;
+	vector<WriteDataToSegment> child_functions;
+	void AppendRow(Allocator &allocator, vector<AllocatedData> &owning_vector, LinkedList *linked_list, Vector &input,
+	               idx_t &entry_idx, idx_t &count);
+};
+
+struct ReadDataFromSegment {
+	read_data_from_segment_t segment_function;
+	vector<ReadDataFromSegment> child_functions;
+	void BuildListVector(LinkedList *linked_list, Vector &result, idx_t &initial_total_count);
+};
+
+struct CopyDataFromSegment {
+	copy_data_from_segment_t segment_function;
+	vector<CopyDataFromSegment> child_functions;
+	void CopyLinkedList(const LinkedList *source_list, LinkedList &target_list, Allocator &allocator,
+	                    vector<AllocatedData> &owning_vector);
+};
+
+void GetSegmentDataFunctions(WriteDataToSegment &write_data_to_segment, ReadDataFromSegment &read_data_from_segment,
+                             CopyDataFromSegment &copy_data_from_segment, const LogicalType &type);
+} // namespace duckdb