duckdb 0.8.2-dev3458.0 → 0.8.2-dev3949.0

package/src/duckdb/src/include/duckdb/common/fixed_size_map.hpp

@@ -0,0 +1,208 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/fixed_size_map.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/pair.hpp"
+#include "duckdb/common/types.hpp"
+#include "duckdb/common/types/validity_mask.hpp"
+
+namespace duckdb {
+
+template <typename T>
+struct fixed_size_map_iterator_t;
+
+template <typename T>
+struct const_fixed_size_map_iterator_t;
+
+template <typename T>
+class fixed_size_map_t {
+	friend struct fixed_size_map_iterator_t<T>;
+	friend struct const_fixed_size_map_iterator_t<T>;
+
+public:
+	using key_type = idx_t;
+	using mapped_type = T;
+
+public:
+	explicit fixed_size_map_t(idx_t capacity_p = 0) : capacity(capacity_p) {
+		resize(capacity);
+	}
+
+	idx_t size() const {
+		return count;
+	}
+
+	void resize(idx_t capacity_p) {
+		capacity = capacity_p;
+		occupied = ValidityMask(capacity);
+		values = make_unsafe_uniq_array<T>(capacity + 1);
+		clear();
+	}
+
+	void clear() {
+		count = 0;
+		occupied.SetAllInvalid(capacity);
+	}
+
+	T &operator[](const idx_t &key) {
+		D_ASSERT(key < capacity);
+		count += 1 - occupied.RowIsValid(key);
+		occupied.SetValidUnsafe(key);
+		return values[key];
+	}
+
+	const T &operator[](const idx_t &key) const {
+		D_ASSERT(key < capacity);
+		return values[key];
+	}
+
+	fixed_size_map_iterator_t<T> begin() {
+		return fixed_size_map_iterator_t<T>(begin_internal(), *this);
+	}
+
+	const_fixed_size_map_iterator_t<T> begin() const {
+		return const_fixed_size_map_iterator_t<T>(begin_internal(), *this);
+	}
+
+	fixed_size_map_iterator_t<T> end() {
+		return fixed_size_map_iterator_t<T>(capacity, *this);
+	}
+
+	const_fixed_size_map_iterator_t<T> end() const {
+		return const_fixed_size_map_iterator_t<T>(capacity, *this);
+	}
+
+	fixed_size_map_iterator_t<T> find(const idx_t &index) {
+		if (occupied.RowIsValid(index)) {
+			return fixed_size_map_iterator_t<T>(index, *this);
+		} else {
+			return end();
+		}
+	}
+
+	const_fixed_size_map_iterator_t<T> find(const idx_t &index) const {
+		if (occupied.RowIsValid(index)) {
+			return const_fixed_size_map_iterator_t<T>(index, *this);
+		} else {
+			return end();
+		}
+	}
+
+private:
+	idx_t begin_internal() const {
+		idx_t index;
+		for (index = 0; index < capacity; index++) {
+			if (occupied.RowIsValid(index)) {
+				break;
+			}
+		}
+		return index;
+	}
+
+private:
+	idx_t capacity;
+	idx_t count;
+
+	ValidityMask occupied;
+	unsafe_unique_array<T> values;
+};
+
+template <typename T>
+struct fixed_size_map_iterator_t {
+public:
+	fixed_size_map_iterator_t(idx_t index_p, fixed_size_map_t<T> &map_p) : map(map_p), current(index_p) {
+	}
+
+	fixed_size_map_iterator_t<T> &operator++() {
+		for (current++; current < map.capacity; current++) {
+			if (map.occupied.RowIsValidUnsafe(current)) {
+				break;
+			}
+		}
+		return *this;
+	}
+
+	fixed_size_map_iterator_t<T> operator++(int) {
+		fixed_size_map_iterator_t<T> tmp = *this;
+		++(*this);
+		return tmp;
+	}
+
+	idx_t &GetKey() {
+		return current;
+	}
+
+	const idx_t &GetKey() const {
+		return current;
+	}
+
+	T &GetValue() {
+		return map.values[current];
+	}
+
+	const T &GetValue() const {
+		return map.values[current];
+	}
+
+	friend bool operator==(const fixed_size_map_iterator_t<T> &a, const fixed_size_map_iterator_t<T> &b) {
+		return a.current == b.current;
+	}
+
+	friend bool operator!=(const fixed_size_map_iterator_t<T> &a, const fixed_size_map_iterator_t<T> &b) {
+		return !(a == b);
+	}
+
+private:
+	fixed_size_map_t<T> &map;
+	idx_t current;
+};
+
+template <typename T>
+struct const_fixed_size_map_iterator_t {
+public:
+	const_fixed_size_map_iterator_t(idx_t index_p, const fixed_size_map_t<T> &map_p) : map(map_p), current(index_p) {
+	}
+
+	const_fixed_size_map_iterator_t<T> &operator++() {
+		for (current++; current < map.capacity; current++) {
+			if (map.occupied.RowIsValidUnsafe(current)) {
+				break;
+			}
+		}
+		return *this;
+	}
+
+	const_fixed_size_map_iterator_t<T> operator++(int) {
+		const_fixed_size_map_iterator_t<T> tmp = *this;
+		++(*this);
+		return tmp;
+	}
+
+	const idx_t &GetKey() const {
+		return current;
+	}
+
+	const T &GetValue() const {
+		return map.values[current];
+	}
+
+	friend bool operator==(const const_fixed_size_map_iterator_t<T> &a, const const_fixed_size_map_iterator_t<T> &b) {
+		return a.current == b.current;
+	}
+
+	friend bool operator!=(const const_fixed_size_map_iterator_t<T> &a, const const_fixed_size_map_iterator_t<T> &b) {
+		return !(a == b);
+	}
+
+private:
+	const fixed_size_map_t<T> &map;
+	idx_t current;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/optional_idx.hpp

@@ -31,6 +31,9 @@ public:
 	bool IsValid() const {
 		return index != DConstants::INVALID_INDEX;
 	}
+	void Invalidate() {
+		index = INVALID_INDEX;
+	}
 	idx_t GetIndex() {
 		if (index == INVALID_INDEX) {
 			throw InternalException("Attempting to get the index of an optional_idx that is not set");

package/src/duckdb/src/include/duckdb/common/perfect_map_set.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/common/string_map_set.hpp
+// duckdb/common/perfect_map_set.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/types/validity_mask.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
 

package/src/duckdb/src/include/duckdb/common/printer.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/string_util.hpp"
 
 namespace duckdb {
 
@@ -21,6 +22,16 @@ public:
 	DUCKDB_API static void Print(OutputStream stream, const string &str);
 	//! Print the object to stderr
 	DUCKDB_API static void Print(const string &str);
+	//! Print the formatted object to the stream
+	template <typename... Args>
+	static void PrintF(OutputStream stream, const string &str, Args... params) {
+		Printer::Print(stream, StringUtil::Format(str, params...));
+	}
+	//! Print the formatted object to stderr
+	template <typename... Args>
+	static void PrintF(const string &str, Args... params) {
+		Printer::PrintF(OutputStream::STREAM_STDERR, str, std::forward<Args>(params)...);
+	}
 	//! Directly prints the string to stdout without a newline
 	DUCKDB_API static void RawPrint(OutputStream stream, const string &str);
 	//! Flush an output stream

package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/encoding_util.hpp"
 
 namespace duckdb {
 class ClientContext;
@@ -20,6 +21,7 @@ public:
 		OnObjectBegin();
 		auto result = T::FormatDeserialize(*this);
 		OnObjectEnd();
+		D_ASSERT(nesting_level == 0); // make sure we are at the root level
 		return result;
 	}
 
@@ -42,23 +44,35 @@ private:
 	explicit BinaryDeserializer(data_ptr_t ptr, idx_t length) : ptr(ptr), end_ptr(ptr + length) {
 		deserialize_enum_from_string = false;
 	}
-	struct State {
-		uint32_t expected_field_count;
-		idx_t expected_size;
-		field_id_t expected_field_id;
-		uint32_t read_field_count;
 
-		State(uint32_t expected_field_count, idx_t expected_size, field_id_t expected_field_id)
-		    : expected_field_count(expected_field_count), expected_size(expected_size),
-		      expected_field_id(expected_field_id), read_field_count(0) {
-		}
-	};
-
-	const char *current_tag = nullptr;
-	field_id_t current_field_id = 0;
 	data_ptr_t ptr;
 	data_ptr_t end_ptr;
-	vector<State> stack;
+	idx_t nesting_level = 0;
+
+	// Allow peeking 1 field ahead
+	bool has_buffered_field = false;
+	field_id_t buffered_field = 0;
+	field_id_t PeekField() {
+		if (!has_buffered_field) {
+			buffered_field = ReadPrimitive<field_id_t>();
+			has_buffered_field = true;
+		}
+		return buffered_field;
+	}
+	void ConsumeField() {
+		if (!has_buffered_field) {
+			buffered_field = ReadPrimitive<field_id_t>();
+		} else {
+			has_buffered_field = false;
+		}
+	}
+	field_id_t NextField() {
+		if (has_buffered_field) {
+			has_buffered_field = false;
+			return buffered_field;
+		}
+		return ReadPrimitive<field_id_t>();
+	}
 
 	template <class T>
 	T ReadPrimitive() {
@@ -69,39 +83,39 @@ private:
 
 	void ReadData(data_ptr_t buffer, idx_t read_size) {
 		if (ptr + read_size > end_ptr) {
-			throw SerializationException("Failed to deserialize: not enough data in buffer to fulfill read request");
+			throw InternalException("Failed to deserialize: not enough data in buffer to fulfill read request");
 		}
 		memcpy(buffer, ptr, read_size);
 		ptr += read_size;
 	}
 
-	// Set the 'tag' of the property to read
-	void SetTag(const field_id_t field_id, const char *tag) final;
+	template <class T>
+	T VarIntDecode() {
+		T value;
+		auto read_size = EncodingUtil::DecodeLEB128<T>(ptr, value);
+		ptr += read_size;
+		return value;
+	}
 
 	//===--------------------------------------------------------------------===//
 	// Nested Types Hooks
 	//===--------------------------------------------------------------------===//
+	void OnPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnPropertyEnd() final;
+	bool OnOptionalPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnOptionalPropertyEnd(bool present) final;
 	void OnObjectBegin() final;
 	void OnObjectEnd() final;
 	idx_t OnListBegin() final;
 	void OnListEnd() final;
-	idx_t OnMapBegin() final;
-	void OnMapEnd() final;
-	void OnMapEntryBegin() final;
-	void OnMapEntryEnd() final;
-	void OnMapKeyBegin() final;
-	void OnMapValueBegin() final;
-	bool OnOptionalBegin() final;
-
-	void OnPairBegin() final;
-	void OnPairKeyBegin() final;
-	void OnPairValueBegin() final;
-	void OnPairEnd() final;
+	bool OnNullableBegin() final;
+	void OnNullableEnd() final;
 
 	//===--------------------------------------------------------------------===//
 	// Primitive Types
 	//===--------------------------------------------------------------------===//
 	bool ReadBool() final;
+	char ReadChar() final;
 	int8_t ReadSignedInt8() final;
 	uint8_t ReadUnsignedInt8() final;
 	int16_t ReadSignedInt16() final;
@@ -113,7 +127,6 @@ private:
 	float ReadFloat() final;
 	double ReadDouble() final;
 	string ReadString() final;
-	interval_t ReadInterval() final;
 	hugeint_t ReadHugeInt() final;
 	void ReadDataPtr(data_ptr_t &ptr, idx_t count) final;
 };

package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp

@@ -9,25 +9,21 @@
 #pragma once
 
 #include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/pair.hpp"
+#include "duckdb/common/serializer/encoding_util.hpp"
 
 namespace duckdb {
 
 struct BinarySerializer : public FormatSerializer {
 private:
-	struct State {
-		// how many fields are present in the object
-		uint32_t field_count;
-		// the size of the object
-		uint64_t size;
-		// the offset of the object start in the buffer
-		uint64_t offset;
+	struct DebugState {
+		unordered_set<const char *> seen_field_tags;
+		unordered_set<field_id_t> seen_field_ids;
+		vector<pair<const char *, field_id_t>> seen_fields;
 	};
 
-	const char *current_tag;
-	field_id_t current_field_id = 0;
-
+	vector<DebugState> debug_stack;
 	vector<data_t> data;
-	vector<State> stack;
 
 	template <class T>
 	void Write(T element) {
@@ -36,51 +32,57 @@ private:
 	}
 	void WriteDataInternal(const_data_ptr_t buffer, idx_t write_size) {
 		data.insert(data.end(), buffer, buffer + write_size);
-		stack.back().size += write_size;
 	}
 	void WriteDataInternal(const char *ptr, idx_t write_size) {
 		WriteDataInternal(const_data_ptr_cast(ptr), write_size);
 	}
 
-	explicit BinarySerializer() {
+	template <class T>
+	void VarIntEncode(T value) {
+		uint8_t buffer[16];
+		auto write_size = EncodingUtil::EncodeLEB128<T>(buffer, value);
+		D_ASSERT(write_size <= sizeof(buffer));
+		WriteDataInternal(buffer, write_size);
+	}
+
+	explicit BinarySerializer(bool serialize_default_values_p) {
+		serialize_default_values = serialize_default_values_p;
 		serialize_enum_as_string = false;
 	}
 
 public:
+	//! Serializes the given object into a binary blob, optionally serializing default values if
+	//! serialize_default_values is set to true, otherwise properties set to their provided default value
+	//! will not be serialized
 	template <class T>
-	static vector<data_t> Serialize(T &obj) {
-		BinarySerializer serializer;
+	static vector<data_t> Serialize(T &obj, bool serialize_default_values) {
+		BinarySerializer serializer(serialize_default_values);
 		serializer.OnObjectBegin();
 		obj.FormatSerialize(serializer);
 		serializer.OnObjectEnd();
 		return std::move(serializer.data);
 	}
 
-	void SetTag(const field_id_t field_id, const char *tag) final;
-
-	//===--------------------------------------------------------------------===//
-	// Nested Types Hooks
-	//===--------------------------------------------------------------------===//
-	void OnOptionalBegin(bool present) final;
+	//-------------------------------------------------------------------------
+	// Nested Type Hooks
+	//-------------------------------------------------------------------------
+	// We serialize optional values as a message with a "present" flag, followed by the value.
+	void OnPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnPropertyEnd() final;
+	void OnOptionalPropertyBegin(const field_id_t field_id, const char *tag, bool present) final;
+	void OnOptionalPropertyEnd(bool present) final;
 	void OnListBegin(idx_t count) final;
-	void OnListEnd(idx_t count) final;
-	void OnMapBegin(idx_t count) final;
-	void OnMapEntryBegin() final;
-	void OnMapEntryEnd() final;
-	void OnMapKeyBegin() final;
-	void OnMapValueBegin() final;
-	void OnMapEnd(idx_t count) final;
+	void OnListEnd() final;
 	void OnObjectBegin() final;
 	void OnObjectEnd() final;
-	void OnPairBegin() final;
-	void OnPairKeyBegin() final;
-	void OnPairValueBegin() final;
-	void OnPairEnd() final;
+	void OnNullableBegin(bool present) final;
+	void OnNullableEnd() final;
 
-	//===--------------------------------------------------------------------===//
+	//-------------------------------------------------------------------------
 	// Primitive Types
-	//===--------------------------------------------------------------------===//
+	//-------------------------------------------------------------------------
 	void WriteNull() final;
+	void WriteValue(char value) final;
 	void WriteValue(uint8_t value) final;
 	void WriteValue(int8_t value) final;
 	void WriteValue(uint16_t value) final;
@@ -92,7 +94,6 @@ public:
 	void WriteValue(hugeint_t value) final;
 	void WriteValue(float value) final;
 	void WriteValue(double value) final;
-	void WriteValue(interval_t value) final;
 	void WriteValue(const string_t value) final;
 	void WriteValue(const string &value) final;
 	void WriteValue(const char *value) final;

package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp

@@ -21,6 +21,7 @@ struct DeserializationData {
 	stack<reference<ClientContext>> contexts;
 	stack<idx_t> enums;
 	stack<reference<bound_parameter_map_t>> parameter_data;
+	stack<reference<LogicalType>> types;
 
 	template <class T>
 	void Set(T entry) = delete;
@@ -107,4 +108,21 @@ inline void DeserializationData::Unset<bound_parameter_map_t>() {
 	parameter_data.pop();
 }
 
+template <>
+inline void DeserializationData::Set(LogicalType &type) {
+	types.emplace(type);
+}
+
+template <>
+inline LogicalType &DeserializationData::Get() {
+	AssertNotEmpty(types);
+	return types.top();
+}
+
+template <>
+inline void DeserializationData::Unset<LogicalType>() {
+	AssertNotEmpty(types);
+	types.pop();
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/encoding_util.hpp

@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/encoding_util.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/typedefs.hpp"
+#include <type_traits>
+
+namespace duckdb {
+
+struct EncodingUtil {
+
+	// Encode unsigned integer, returns the number of bytes written
+	template <class T>
+	static idx_t EncodeUnsignedLEB128(data_ptr_t target, T value) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_unsigned<T>::value, "Must be unsigned");
+		static_assert(sizeof(T) <= sizeof(uint64_t), "Must be uint64_t or smaller");
+
+		idx_t offset = 0;
+		do {
+			uint8_t byte = value & 0x7F;
+			value >>= 7;
+			if (value != 0) {
+				byte |= 0x80;
+			}
+			target[offset++] = byte;
+		} while (value != 0);
+		return offset;
+	}
+
+	// Decode unsigned integer, returns the number of bytes read
+	template <class T>
+	static idx_t DecodeUnsignedLEB128(const_data_ptr_t source, T &result) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_unsigned<T>::value, "Must be unsigned");
+		static_assert(sizeof(T) <= sizeof(uint64_t), "Must be uint64_t or smaller");
+
+		result = 0;
+		idx_t shift = 0;
+		idx_t offset = 0;
+		uint8_t byte;
+		do {
+			byte = source[offset++];
+			result |= static_cast<T>(byte & 0x7F) << shift;
+			shift += 7;
+		} while (byte & 0x80);
+
+		return offset;
+	}
+
+	// Encode signed integer, returns the number of bytes written
+	template <class T>
+	static idx_t EncodeSignedLEB128(data_ptr_t target, T value) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_signed<T>::value, "Must be signed");
+		static_assert(sizeof(T) <= sizeof(int64_t), "Must be int64_t or smaller");
+
+		idx_t offset = 0;
+		do {
+			uint8_t byte = value & 0x7F;
+			value >>= 7;
+
+			// Determine whether more bytes are needed
+			if ((value == 0 && (byte & 0x40) == 0) || (value == -1 && (byte & 0x40))) {
+				target[offset++] = byte;
+				break;
+			} else {
+				byte |= 0x80;
+				target[offset++] = byte;
+			}
+		} while (true);
+		return offset;
+	}
+
+	// Decode signed integer, returns the number of bytes read
+	template <class T>
+	static idx_t DecodeSignedLEB128(const_data_ptr_t source, T &result) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_signed<T>::value, "Must be signed");
+		static_assert(sizeof(T) <= sizeof(int64_t), "Must be int64_t or smaller");
+
+		// This is used to avoid undefined behavior when shifting into the sign bit
+		using unsigned_type = typename std::make_unsigned<T>::type;
+
+		result = 0;
+		idx_t shift = 0;
+		idx_t offset = 0;
+
+		uint8_t byte;
+		do {
+			byte = source[offset++];
+			result |= static_cast<unsigned_type>(byte & 0x7F) << shift;
+			shift += 7;
+		} while (byte & 0x80);
+
+		// Sign-extend if the most significant bit of the last byte is set
+		if (shift < sizeof(T) * 8 && (byte & 0x40)) {
+			result |= -(static_cast<unsigned_type>(1) << shift);
+		}
+		return offset;
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_signed<T>::value, idx_t>::type DecodeLEB128(const_data_ptr_t source,
+	                                                                                   T &result) {
+		return DecodeSignedLEB128(source, result);
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_unsigned<T>::value, idx_t>::type DecodeLEB128(const_data_ptr_t source,
+	                                                                                     T &result) {
+		return DecodeUnsignedLEB128(source, result);
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_signed<T>::value, idx_t>::type EncodeLEB128(data_ptr_t target, T value) {
+		return EncodeSignedLEB128(target, value);
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_unsigned<T>::value, idx_t>::type EncodeLEB128(data_ptr_t target, T value) {
+		return EncodeUnsignedLEB128(target, value);
+	}
+};
+
+} // namespace duckdb