duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp

@@ -1,36 +1,96 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/binary_deserializer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
-#include "duckdb/common/serializer/format_deserializer.hpp"
+
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/encoding_util.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
 
 namespace duckdb {
+class ClientContext;
 
-class BinaryDeserializer : public FormatDeserializer {
+class BinaryDeserializer : public Deserializer {
 public:
+	explicit BinaryDeserializer(ReadStream &stream) : stream(stream) {
+		deserialize_enum_from_string = false;
+	}
+
 	template <class T>
-	static unique_ptr<T> Deserialize(data_ptr_t ptr, idx_t length) {
-		BinaryDeserializer deserializer(ptr, length);
-		deserializer.OnObjectBegin();
-		auto result = T::FormatDeserialize(deserializer);
-		deserializer.OnObjectEnd();
+	unique_ptr<T> Deserialize() {
+		OnObjectBegin();
+		auto result = T::Deserialize(*this);
+		OnObjectEnd();
+		D_ASSERT(nesting_level == 0); // make sure we are at the root level
 		return result;
 	}
 
+	template <class T>
+	static unique_ptr<T> Deserialize(ReadStream &stream) {
+		BinaryDeserializer deserializer(stream);
+		return deserializer.template Deserialize<T>();
+	}
+
+	template <class T>
+	static unique_ptr<T> Deserialize(ReadStream &stream, ClientContext &context, bound_parameter_map_t &parameters) {
+		BinaryDeserializer deserializer(stream);
+		deserializer.Set<ClientContext &>(context);
+		deserializer.Set<bound_parameter_map_t &>(parameters);
+		return deserializer.template Deserialize<T>();
+	}
+
+	void Begin() {
+		OnObjectBegin();
+	}
+
+	void End() {
+		OnObjectEnd();
+		D_ASSERT(nesting_level == 0); // make sure we are at the root level
+	}
+
+	ReadStream &GetStream() {
+		return stream;
+	}
+
 private:
-	explicit BinaryDeserializer(data_ptr_t ptr, idx_t length) : ptr(ptr), end_ptr(ptr + length) {
-		deserialize_enum_from_string = false;
+	ReadStream &stream;
+	idx_t nesting_level = 0;
+
+	// Allow peeking 1 field ahead
+	bool has_buffered_field = false;
+	field_id_t buffered_field = 0;
+
+private:
+	field_id_t PeekField() {
+		if (!has_buffered_field) {
+			buffered_field = ReadPrimitive<field_id_t>();
+			has_buffered_field = true;
+		}
+		return buffered_field;
+	}
+	void ConsumeField() {
+		if (!has_buffered_field) {
+			buffered_field = ReadPrimitive<field_id_t>();
+		} else {
+			has_buffered_field = false;
+		}
 	}
-	struct State {
-		uint32_t expected_field_count;
-		idx_t expected_size;
-		uint32_t read_field_count;
-		State(uint32_t expected_field_count, idx_t expected_size)
-		    : expected_field_count(expected_field_count), expected_size(expected_size), read_field_count(0) {
+	field_id_t NextField() {
+		if (has_buffered_field) {
+			has_buffered_field = false;
+			return buffered_field;
 		}
-	};
+		return ReadPrimitive<field_id_t>();
+	}
 
-	const char *current_tag = nullptr;
-	data_ptr_t ptr;
-	data_ptr_t end_ptr;
-	vector<State> stack;
+	void ReadData(data_ptr_t buffer, idx_t read_size) {
+		stream.ReadData(buffer, read_size);
+	}
 
 	template <class T>
 	T ReadPrimitive() {
@@ -39,41 +99,44 @@ private:
 		return value;
 	}
 
-	void ReadData(data_ptr_t buffer, idx_t read_size) {
-		if (ptr + read_size > end_ptr) {
-			throw SerializationException("Failed to deserialize: not enough data in buffer to fulfill read request");
+	template <class T>
+	T VarIntDecode() {
+		// FIXME: maybe we should pass a source to EncodingUtil instead
+		uint8_t buffer[16];
+		idx_t varint_size;
+		for (varint_size = 0; varint_size < 16; varint_size++) {
+			ReadData(buffer + varint_size, 1);
+			if (!(buffer[varint_size] & 0x80)) {
+				varint_size++;
+				break;
+			}
 		}
-		memcpy(buffer, ptr, read_size);
-		ptr += read_size;
+		T value;
+		auto read_size = EncodingUtil::DecodeLEB128<T>(buffer, value);
+		D_ASSERT(read_size == varint_size);
+		(void)read_size;
+		return value;
 	}
 
-	// Set the 'tag' of the property to read
-	void SetTag(const char *tag) final;
-
 	//===--------------------------------------------------------------------===//
 	// Nested Types Hooks
 	//===--------------------------------------------------------------------===//
+	void OnPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnPropertyEnd() final;
+	bool OnOptionalPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnOptionalPropertyEnd(bool present) final;
 	void OnObjectBegin() final;
 	void OnObjectEnd() final;
 	idx_t OnListBegin() final;
 	void OnListEnd() final;
-	idx_t OnMapBegin() final;
-	void OnMapEnd() final;
-	void OnMapEntryBegin() final;
-	void OnMapEntryEnd() final;
-	void OnMapKeyBegin() final;
-	void OnMapValueBegin() final;
-	bool OnOptionalBegin() final;
-
-	void OnPairBegin() final;
-	void OnPairKeyBegin() final;
-	void OnPairValueBegin() final;
-	void OnPairEnd() final;
+	bool OnNullableBegin() final;
+	void OnNullableEnd() final;
 
 	//===--------------------------------------------------------------------===//
 	// Primitive Types
 	//===--------------------------------------------------------------------===//
 	bool ReadBool() final;
+	char ReadChar() final;
 	int8_t ReadSignedInt8() final;
 	uint8_t ReadUnsignedInt8() final;
 	int16_t ReadSignedInt16() final;
@@ -85,7 +148,6 @@ private:
 	float ReadFloat() final;
 	double ReadDouble() final;
 	string ReadString() final;
-	interval_t ReadInterval() final;
 	hugeint_t ReadHugeInt() final;
 	void ReadDataPtr(data_ptr_t &ptr, idx_t count) final;
 };

package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp

@@ -1,78 +1,92 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/binary_serializer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
-#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
+#include "duckdb/common/pair.hpp"
+#include "duckdb/common/serializer/encoding_util.hpp"
 
 namespace duckdb {
 
-struct BinarySerializer : public FormatSerializer {
+class BinarySerializer : public Serializer {
+public:
+	explicit BinarySerializer(WriteStream &stream, bool serialize_default_values_p = false) : stream(stream) {
+		serialize_default_values = serialize_default_values_p;
+		serialize_enum_as_string = false;
+	}
 
 private:
-	struct State {
-		// how many fields are present in the object
-		uint32_t field_count;
-		// the size of the object
-		uint64_t size;
-		// the offset of the object start in the buffer
-		uint64_t offset;
+	struct DebugState {
+		unordered_set<const char *> seen_field_tags;
+		unordered_set<field_id_t> seen_field_ids;
+		vector<pair<const char *, field_id_t>> seen_fields;
 	};
 
-	const char *current_tag;
-
-	vector<data_t> data;
-	vector<State> stack;
+	void WriteData(const_data_ptr_t buffer, idx_t write_size) {
+		stream.WriteData(buffer, write_size);
+	}
 
 	template <class T>
 	void Write(T element) {
 		static_assert(std::is_trivially_destructible<T>(), "Write element must be trivially destructible");
 		WriteData(const_data_ptr_cast(&element), sizeof(T));
 	}
-	void WriteData(const_data_ptr_t buffer, idx_t write_size) {
-		data.insert(data.end(), buffer, buffer + write_size);
-		stack.back().size += write_size;
-	}
 	void WriteData(const char *ptr, idx_t write_size) {
 		WriteData(const_data_ptr_cast(ptr), write_size);
 	}
 
-	explicit BinarySerializer() {
-		serialize_enum_as_string = false;
+	template <class T>
+	void VarIntEncode(T value) {
+		uint8_t buffer[16];
+		auto write_size = EncodingUtil::EncodeLEB128<T>(buffer, value);
+		D_ASSERT(write_size <= sizeof(buffer));
+		WriteData(buffer, write_size);
 	}
 
 public:
 	template <class T>
-	static vector<data_t> Serialize(T &obj) {
-		BinarySerializer serializer;
+	static void Serialize(const T &value, WriteStream &stream, bool serialize_default_values = false) {
+		BinarySerializer serializer(stream, serialize_default_values);
 		serializer.OnObjectBegin();
-		obj.FormatSerialize(serializer);
+		value.Serialize(serializer);
 		serializer.OnObjectEnd();
-		return std::move(serializer.data);
 	}
 
-	void SetTag(const char *tag) final;
+	void Begin() {
+		OnObjectBegin();
+	}
+	void End() {
+		OnObjectEnd();
+	}
 
-	//===--------------------------------------------------------------------===//
-	// Nested Types Hooks
-	//===--------------------------------------------------------------------===//
-	void OnOptionalBegin(bool present) final;
+protected:
+	//-------------------------------------------------------------------------
+	// Nested Type Hooks
+	//-------------------------------------------------------------------------
+	// We serialize optional values as a message with a "present" flag, followed by the value.
+	void OnPropertyBegin(const field_id_t field_id, const char *tag) final;
+	void OnPropertyEnd() final;
+	void OnOptionalPropertyBegin(const field_id_t field_id, const char *tag, bool present) final;
+	void OnOptionalPropertyEnd(bool present) final;
 	void OnListBegin(idx_t count) final;
-	void OnListEnd(idx_t count) final;
-	void OnMapBegin(idx_t count) final;
-	void OnMapEntryBegin() final;
-	void OnMapEntryEnd() final;
-	void OnMapKeyBegin() final;
-	void OnMapValueBegin() final;
-	void OnMapEnd(idx_t count) final;
+	void OnListEnd() final;
 	void OnObjectBegin() final;
 	void OnObjectEnd() final;
-	void OnPairBegin() final;
-	void OnPairKeyBegin() final;
-	void OnPairValueBegin() final;
-	void OnPairEnd() final;
+	void OnNullableBegin(bool present) final;
+	void OnNullableEnd() final;
 
-	//===--------------------------------------------------------------------===//
+	//-------------------------------------------------------------------------
 	// Primitive Types
-	//===--------------------------------------------------------------------===//
+	//-------------------------------------------------------------------------
 	void WriteNull() final;
+	void WriteValue(char value) final;
 	void WriteValue(uint8_t value) final;
 	void WriteValue(int8_t value) final;
 	void WriteValue(uint16_t value) final;
@@ -84,12 +98,15 @@ public:
 	void WriteValue(hugeint_t value) final;
 	void WriteValue(float value) final;
 	void WriteValue(double value) final;
-	void WriteValue(interval_t value) final;
 	void WriteValue(const string_t value) final;
 	void WriteValue(const string &value) final;
 	void WriteValue(const char *value) final;
 	void WriteValue(bool value) final;
 	void WriteDataPtr(const_data_ptr_t ptr, idx_t count) final;
+
+private:
+	vector<DebugState> debug_stack;
+	WriteStream &stream;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp

@@ -9,13 +9,14 @@
 #pragma once
 
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
 
 namespace duckdb {
 
-class BufferedFileReader : public Deserializer {
+class BufferedFileReader : public ReadStream {
 public:
-	BufferedFileReader(FileSystem &fs, const char *path, optional_ptr<ClientContext> context,
-	                   FileLockType lock_type = FileLockType::READ_LOCK, optional_ptr<FileOpener> opener = nullptr);
+	BufferedFileReader(FileSystem &fs, const char *path, FileLockType lock_type = FileLockType::READ_LOCK,
+	                   optional_ptr<FileOpener> opener = nullptr);
 
 	FileSystem &fs;
 	unsafe_unique_array<data_t> data;
@@ -35,16 +36,9 @@ public:
 	void Seek(uint64_t location);
 	uint64_t CurrentOffset();
 
-	ClientContext &GetContext() override;
-
-	optional_ptr<Catalog> GetCatalog() override;
-	void SetCatalog(Catalog &catalog);
-
 private:
 	idx_t file_size;
 	idx_t total_read;
-	optional_ptr<ClientContext> context;
-	optional_ptr<Catalog> catalog;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp

@@ -8,14 +8,14 @@
 
 #pragma once
 
-#include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
 #include "duckdb/common/file_system.hpp"
 
 namespace duckdb {
 
 #define FILE_BUFFER_SIZE 4096
 
-class BufferedFileWriter : public Serializer {
+class BufferedFileWriter : public WriteStream {
 public:
 	static constexpr uint8_t DEFAULT_OPEN_FLAGS = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE;
 
@@ -31,7 +31,7 @@ public:
 	unique_ptr<FileHandle> handle;
 
 public:
-	DUCKDB_API void WriteData(const_data_ptr_t buffer, uint64_t write_size) override;
+	DUCKDB_API void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
 	//! Flush the buffer to disk and sync the file to ensure writing is completed
 	DUCKDB_API void Sync();
 	//! Flush the buffer to the file (without sync)

package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp

@@ -0,0 +1,181 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/deserialization_data.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/stack.hpp"
+#include "duckdb/planner/bound_parameter_map.hpp"
+
+namespace duckdb {
+class ClientContext;
+class Catalog;
+class DatabaseInstance;
+enum class ExpressionType : uint8_t;
+
+struct DeserializationData {
+	stack<reference<ClientContext>> contexts;
+	stack<reference<DatabaseInstance>> databases;
+	stack<idx_t> enums;
+	stack<reference<bound_parameter_map_t>> parameter_data;
+	stack<reference<LogicalType>> types;
+
+	template <class T>
+	void Set(T entry) = delete;
+
+	template <class T>
+	T Get() = delete;
+
+	template <class T>
+	void Unset() = delete;
+
+	template <class T>
+	inline void AssertNotEmpty(const stack<T> &e) {
+		if (e.empty()) {
+			throw InternalException("DeserializationData - unexpected empty stack");
+		}
+	}
+};
+
+template <>
+inline void DeserializationData::Set(ExpressionType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline ExpressionType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return ExpressionType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<ExpressionType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(LogicalOperatorType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline LogicalOperatorType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return LogicalOperatorType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<LogicalOperatorType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(CompressionType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline CompressionType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return CompressionType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<CompressionType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(CatalogType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline CatalogType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return CatalogType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<CatalogType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
+template <>
+inline void DeserializationData::Set(ClientContext &context) {
+	contexts.push(context);
+}
+
+template <>
+inline ClientContext &DeserializationData::Get() {
+	AssertNotEmpty(contexts);
+	return contexts.top();
+}
+
+template <>
+inline void DeserializationData::Unset<ClientContext>() {
+	AssertNotEmpty(contexts);
+	contexts.pop();
+}
+
+template <>
+inline void DeserializationData::Set(DatabaseInstance &db) {
+	databases.push(db);
+}
+
+template <>
+inline DatabaseInstance &DeserializationData::Get() {
+	AssertNotEmpty(databases);
+	return databases.top();
+}
+
+template <>
+inline void DeserializationData::Unset<DatabaseInstance>() {
+	AssertNotEmpty(databases);
+	databases.pop();
+}
+
+template <>
+inline void DeserializationData::Set(bound_parameter_map_t &context) {
+	parameter_data.push(context);
+}
+
+template <>
+inline bound_parameter_map_t &DeserializationData::Get() {
+	AssertNotEmpty(parameter_data);
+	return parameter_data.top();
+}
+
+template <>
+inline void DeserializationData::Unset<bound_parameter_map_t>() {
+	AssertNotEmpty(parameter_data);
+	parameter_data.pop();
+}
+
+template <>
+inline void DeserializationData::Set(LogicalType &type) {
+	types.emplace(type);
+}
+
+template <>
+inline LogicalType &DeserializationData::Get() {
+	AssertNotEmpty(types);
+	return types.top();
+}
+
+template <>
+inline void DeserializationData::Unset<LogicalType>() {
+	AssertNotEmpty(types);
+	types.pop();
+}
+
+} // namespace duckdb