duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp

@@ -1,36 +1,78 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/binary_deserializer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
+
 #include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/encoding_util.hpp"
 
 namespace duckdb {
+class ClientContext;
 
 class BinaryDeserializer : public FormatDeserializer {
 public:
+	template <class T>
+	unique_ptr<T> Deserialize() {
+		OnObjectBegin();
+		auto result = T::FormatDeserialize(*this);
+		OnObjectEnd();
+		D_ASSERT(nesting_level == 0); // make sure we are at the root level
+		return result;
+	}
+
 	template <class T>
 	static unique_ptr<T> Deserialize(data_ptr_t ptr, idx_t length) {
 		BinaryDeserializer deserializer(ptr, length);
-		deserializer.OnObjectBegin();
-		auto result = T::FormatDeserialize(deserializer);
-		deserializer.OnObjectEnd();
-		return result;
+		return deserializer.template Deserialize<T>();
+	}
+
+	template <class T>
+	static unique_ptr<T> Deserialize(ClientContext &context, bound_parameter_map_t &parameters, data_ptr_t ptr,
+	                                 idx_t length) {
+		BinaryDeserializer deserializer(ptr, length);
+		deserializer.Set<ClientContext &>(context);
+		deserializer.Set<bound_parameter_map_t &>(parameters);
+		return deserializer.template Deserialize<T>();
 	}
 
 private:
 	explicit BinaryDeserializer(data_ptr_t ptr, idx_t length) : ptr(ptr), end_ptr(ptr + length) {
 		deserialize_enum_from_string = false;
 	}
-	struct State {
-		uint32_t expected_field_count;
-		idx_t expected_size;
-		uint32_t read_field_count;
-		State(uint32_t expected_field_count, idx_t expected_size)
-		    : expected_field_count(expected_field_count), expected_size(expected_size), read_field_count(0) {
-		}
-	};
 
-	const char *current_tag = nullptr;
 	data_ptr_t ptr;
 	data_ptr_t end_ptr;
-	vector<State> stack;
+	idx_t nesting_level = 0;
+
+	// Allow peeking 1 field ahead
+	bool has_buffered_field = false;
+	field_id_t buffered_field = 0;
+	field_id_t PeekField() {
+		if (!has_buffered_field) {
+			buffered_field = ReadPrimitive<field_id_t>();
+			has_buffered_field = true;
+		}
+		return buffered_field;
+	}
+	void ConsumeField() {
+		if (!has_buffered_field) {
+			buffered_field = ReadPrimitive<field_id_t>();
+		} else {
+			has_buffered_field = false;
+		}
+	}
+	field_id_t NextField() {
+		if (has_buffered_field) {
+			has_buffered_field = false;
+			return buffered_field;
+		}
+		return ReadPrimitive<field_id_t>();
+	}
 
 	template <class T>
 	T ReadPrimitive() {
@@ -41,39 +83,39 @@ private:
 
 	void ReadData(data_ptr_t buffer, idx_t read_size) {
 		if (ptr + read_size > end_ptr) {
-			throw SerializationException("Failed to deserialize: not enough data in buffer to fulfill read request");
+			throw InternalException("Failed to deserialize: not enough data in buffer to fulfill read request");
 		}
 		memcpy(buffer, ptr, read_size);
 		ptr += read_size;
 	}
 
-	// Set the 'tag' of the property to read
-	void SetTag(const char *tag) final;
+	template <class T>
+	T VarIntDecode() {
+		T value;
+		auto read_size = EncodingUtil::DecodeLEB128<T>(ptr, value);
+		ptr += read_size;
+		return value;
+	}
 
 	//===--------------------------------------------------------------------===//
 	// Nested Types Hooks
 	//===--------------------------------------------------------------------===//
+	void OnPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnPropertyEnd() final;
+	bool OnOptionalPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnOptionalPropertyEnd(bool present) final;
 	void OnObjectBegin() final;
 	void OnObjectEnd() final;
 	idx_t OnListBegin() final;
 	void OnListEnd() final;
-	idx_t OnMapBegin() final;
-	void OnMapEnd() final;
-	void OnMapEntryBegin() final;
-	void OnMapEntryEnd() final;
-	void OnMapKeyBegin() final;
-	void OnMapValueBegin() final;
-	bool OnOptionalBegin() final;
-
-	void OnPairBegin() final;
-	void OnPairKeyBegin() final;
-	void OnPairValueBegin() final;
-	void OnPairEnd() final;
+	bool OnNullableBegin() final;
+	void OnNullableEnd() final;
 
 	//===--------------------------------------------------------------------===//
 	// Primitive Types
 	//===--------------------------------------------------------------------===//
 	bool ReadBool() final;
+	char ReadChar() final;
 	int8_t ReadSignedInt8() final;
 	uint8_t ReadUnsignedInt8() final;
 	int16_t ReadSignedInt16() final;
@@ -85,7 +127,6 @@ private:
 	float ReadFloat() final;
 	double ReadDouble() final;
 	string ReadString() final;
-	interval_t ReadInterval() final;
 	hugeint_t ReadHugeInt() final;
 	void ReadDataPtr(data_ptr_t &ptr, idx_t count) final;
 };

package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp

@@ -1,78 +1,88 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/binary_serializer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
 #include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/pair.hpp"
+#include "duckdb/common/serializer/encoding_util.hpp"
 
 namespace duckdb {
 
 struct BinarySerializer : public FormatSerializer {
-
 private:
-	struct State {
-		// how many fields are present in the object
-		uint32_t field_count;
-		// the size of the object
-		uint64_t size;
-		// the offset of the object start in the buffer
-		uint64_t offset;
+	struct DebugState {
+		unordered_set<const char *> seen_field_tags;
+		unordered_set<field_id_t> seen_field_ids;
+		vector<pair<const char *, field_id_t>> seen_fields;
 	};
 
-	const char *current_tag;
-
+	vector<DebugState> debug_stack;
 	vector<data_t> data;
-	vector<State> stack;
 
 	template <class T>
 	void Write(T element) {
 		static_assert(std::is_trivially_destructible<T>(), "Write element must be trivially destructible");
-		WriteData(const_data_ptr_cast(&element), sizeof(T));
+		WriteDataInternal(const_data_ptr_cast(&element), sizeof(T));
 	}
-	void WriteData(const_data_ptr_t buffer, idx_t write_size) {
+	void WriteDataInternal(const_data_ptr_t buffer, idx_t write_size) {
 		data.insert(data.end(), buffer, buffer + write_size);
-		stack.back().size += write_size;
 	}
-	void WriteData(const char *ptr, idx_t write_size) {
-		WriteData(const_data_ptr_cast(ptr), write_size);
+	void WriteDataInternal(const char *ptr, idx_t write_size) {
+		WriteDataInternal(const_data_ptr_cast(ptr), write_size);
 	}
 
-	explicit BinarySerializer() {
+	template <class T>
+	void VarIntEncode(T value) {
+		uint8_t buffer[16];
+		auto write_size = EncodingUtil::EncodeLEB128<T>(buffer, value);
+		D_ASSERT(write_size <= sizeof(buffer));
+		WriteDataInternal(buffer, write_size);
+	}
+
+	explicit BinarySerializer(bool serialize_default_values_p) {
+		serialize_default_values = serialize_default_values_p;
 		serialize_enum_as_string = false;
 	}
 
 public:
+	//! Serializes the given object into a binary blob, optionally serializing default values if
+	//! serialize_default_values is set to true, otherwise properties set to their provided default value
+	//! will not be serialized
 	template <class T>
-	static vector<data_t> Serialize(T &obj) {
-		BinarySerializer serializer;
+	static vector<data_t> Serialize(T &obj, bool serialize_default_values) {
+		BinarySerializer serializer(serialize_default_values);
 		serializer.OnObjectBegin();
 		obj.FormatSerialize(serializer);
 		serializer.OnObjectEnd();
 		return std::move(serializer.data);
 	}
 
-	void SetTag(const char *tag) final;
-
-	//===--------------------------------------------------------------------===//
-	// Nested Types Hooks
-	//===--------------------------------------------------------------------===//
-	void OnOptionalBegin(bool present) final;
+	//-------------------------------------------------------------------------
+	// Nested Type Hooks
+	//-------------------------------------------------------------------------
+	// We serialize optional values as a message with a "present" flag, followed by the value.
+	void OnPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnPropertyEnd() final;
+	void OnOptionalPropertyBegin(const field_id_t field_id, const char *tag, bool present) final;
+	void OnOptionalPropertyEnd(bool present) final;
 	void OnListBegin(idx_t count) final;
-	void OnListEnd(idx_t count) final;
-	void OnMapBegin(idx_t count) final;
-	void OnMapEntryBegin() final;
-	void OnMapEntryEnd() final;
-	void OnMapKeyBegin() final;
-	void OnMapValueBegin() final;
-	void OnMapEnd(idx_t count) final;
+	void OnListEnd() final;
 	void OnObjectBegin() final;
 	void OnObjectEnd() final;
-	void OnPairBegin() final;
-	void OnPairKeyBegin() final;
-	void OnPairValueBegin() final;
-	void OnPairEnd() final;
+	void OnNullableBegin(bool present) final;
+	void OnNullableEnd() final;
 
-	//===--------------------------------------------------------------------===//
+	//-------------------------------------------------------------------------
 	// Primitive Types
-	//===--------------------------------------------------------------------===//
+	//-------------------------------------------------------------------------
 	void WriteNull() final;
+	void WriteValue(char value) final;
 	void WriteValue(uint8_t value) final;
 	void WriteValue(int8_t value) final;
 	void WriteValue(uint16_t value) final;
@@ -84,7 +94,6 @@ public:
 	void WriteValue(hugeint_t value) final;
 	void WriteValue(float value) final;
 	void WriteValue(double value) final;
-	void WriteValue(interval_t value) final;
 	void WriteValue(const string_t value) final;
 	void WriteValue(const string &value) final;
 	void WriteValue(const char *value) final;

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp

@@ -37,14 +37,10 @@ public:
 
 	ClientContext &GetContext() override;
 
-	optional_ptr<Catalog> GetCatalog() override;
-	void SetCatalog(Catalog &catalog);
-
 private:
 	idx_t file_size;
 	idx_t total_read;
 	optional_ptr<ClientContext> context;
-	optional_ptr<Catalog> catalog;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp

@@ -0,0 +1,128 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/deserialization_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/stack.hpp"
+#include "duckdb/planner/bound_parameter_map.hpp"
+
+namespace duckdb {
+class ClientContext;
+class Catalog;
+enum class ExpressionType : uint8_t;
+
+struct DeserializationData {
+	stack<reference<ClientContext>> contexts;
+	stack<idx_t> enums;
+	stack<reference<bound_parameter_map_t>> parameter_data;
+	stack<reference<LogicalType>> types;
+
+	template <class T>
+	void Set(T entry) = delete;
+
+	template <class T>
+	T Get() = delete;
+
+	template <class T>
+	void Unset() = delete;
+
+	template <class T>
+	inline void AssertNotEmpty(const stack<T> &e) {
+		if (e.empty()) {
+			throw InternalException("DeserializationData - unexpected empty stack");
+		}
+	}
+};
+
+template <>
+inline void DeserializationData::Set(ExpressionType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline ExpressionType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return ExpressionType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<ExpressionType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(LogicalOperatorType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline LogicalOperatorType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return LogicalOperatorType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<LogicalOperatorType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(ClientContext &context) {
+	contexts.push(context);
+}
+
+template <>
+inline ClientContext &DeserializationData::Get() {
+	AssertNotEmpty(contexts);
+	return contexts.top();
+}
+
+template <>
+inline void DeserializationData::Unset<ClientContext>() {
+	AssertNotEmpty(contexts);
+	contexts.pop();
+}
+
+template <>
+inline void DeserializationData::Set(bound_parameter_map_t &context) {
+	parameter_data.push(context);
+}
+
+template <>
+inline bound_parameter_map_t &DeserializationData::Get() {
+	AssertNotEmpty(parameter_data);
+	return parameter_data.top();
+}
+
+template <>
+inline void DeserializationData::Unset<bound_parameter_map_t>() {
+	AssertNotEmpty(parameter_data);
+	parameter_data.pop();
+}
+
+template <>
+inline void DeserializationData::Set(LogicalType &type) {
+	types.emplace(type);
+}
+
+template <>
+inline LogicalType &DeserializationData::Get() {
+	AssertNotEmpty(types);
+	return types.top();
+}
+
+template <>
+inline void DeserializationData::Unset<LogicalType>() {
+	AssertNotEmpty(types);
+	types.pop();
+}
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/encoding_util.hpp

@@ -0,0 +1,132 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/encoding_util.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/typedefs.hpp"
+#include <type_traits>
+
+namespace duckdb {
+
+struct EncodingUtil {
+
+	// Encode unsigned integer, returns the number of bytes written
+	template <class T>
+	static idx_t EncodeUnsignedLEB128(data_ptr_t target, T value) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_unsigned<T>::value, "Must be unsigned");
+		static_assert(sizeof(T) <= sizeof(uint64_t), "Must be uint64_t or smaller");
+
+		idx_t offset = 0;
+		do {
+			uint8_t byte = value & 0x7F;
+			value >>= 7;
+			if (value != 0) {
+				byte |= 0x80;
+			}
+			target[offset++] = byte;
+		} while (value != 0);
+		return offset;
+	}
+
+	// Decode unsigned integer, returns the number of bytes read
+	template <class T>
+	static idx_t DecodeUnsignedLEB128(const_data_ptr_t source, T &result) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_unsigned<T>::value, "Must be unsigned");
+		static_assert(sizeof(T) <= sizeof(uint64_t), "Must be uint64_t or smaller");
+
+		result = 0;
+		idx_t shift = 0;
+		idx_t offset = 0;
+		uint8_t byte;
+		do {
+			byte = source[offset++];
+			result |= static_cast<T>(byte & 0x7F) << shift;
+			shift += 7;
+		} while (byte & 0x80);
+
+		return offset;
+	}
+
+	// Encode signed integer, returns the number of bytes written
+	template <class T>
+	static idx_t EncodeSignedLEB128(data_ptr_t target, T value) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_signed<T>::value, "Must be signed");
+		static_assert(sizeof(T) <= sizeof(int64_t), "Must be int64_t or smaller");
+
+		idx_t offset = 0;
+		do {
+			uint8_t byte = value & 0x7F;
+			value >>= 7;
+
+			// Determine whether more bytes are needed
+			if ((value == 0 && (byte & 0x40) == 0) || (value == -1 && (byte & 0x40))) {
+				target[offset++] = byte;
+				break;
+			} else {
+				byte |= 0x80;
+				target[offset++] = byte;
+			}
+		} while (true);
+		return offset;
+	}
+
+	// Decode signed integer, returns the number of bytes read
+	template <class T>
+	static idx_t DecodeSignedLEB128(const_data_ptr_t source, T &result) {
+		static_assert(std::is_integral<T>::value, "Must be integral");
+		static_assert(std::is_signed<T>::value, "Must be signed");
+		static_assert(sizeof(T) <= sizeof(int64_t), "Must be int64_t or smaller");
+
+		// This is used to avoid undefined behavior when shifting into the sign bit
+		using unsigned_type = typename std::make_unsigned<T>::type;
+
+		result = 0;
+		idx_t shift = 0;
+		idx_t offset = 0;
+
+		uint8_t byte;
+		do {
+			byte = source[offset++];
+			result |= static_cast<unsigned_type>(byte & 0x7F) << shift;
+			shift += 7;
+		} while (byte & 0x80);
+
+		// Sign-extend if the most significant bit of the last byte is set
+		if (shift < sizeof(T) * 8 && (byte & 0x40)) {
+			result |= -(static_cast<unsigned_type>(1) << shift);
+		}
+		return offset;
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_signed<T>::value, idx_t>::type DecodeLEB128(const_data_ptr_t source,
+	                                                                                   T &result) {
+		return DecodeSignedLEB128(source, result);
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_unsigned<T>::value, idx_t>::type DecodeLEB128(const_data_ptr_t source,
+	                                                                                     T &result) {
+		return DecodeUnsignedLEB128(source, result);
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_signed<T>::value, idx_t>::type EncodeLEB128(data_ptr_t target, T value) {
+		return EncodeSignedLEB128(target, value);
+	}
+
+	template <class T>
+	static typename std::enable_if<std::is_unsigned<T>::value, idx_t>::type EncodeLEB128(data_ptr_t target, T value) {
+		return EncodeUnsignedLEB128(target, value);
+	}
+};
+
+} // namespace duckdb