duckdb 0.8.2-dev4126.0 → 0.8.2-dev4203.0

package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp

@@ -13,8 +13,6 @@
 
 namespace duckdb {
 
-class FieldReader;
-
 //! Extra Type Info Type
 enum class ExtraTypeInfoType : uint8_t {
 	INVALID_TYPE_INFO = 0,
@@ -38,15 +36,9 @@ struct ExtraTypeInfo {
 
 public:
 	bool Equals(ExtraTypeInfo *other_p) const;
-	//! Serializes a ExtraTypeInfo to a stand-alone binary blob
-	virtual void Serialize(FieldWriter &writer) const;
-	//! Serializes a ExtraTypeInfo to a stand-alone binary blob
-	static void Serialize(ExtraTypeInfo *info, FieldWriter &writer);
-	//! Deserializes a blob back into an ExtraTypeInfo
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
 
-	virtual void FormatSerialize(FormatSerializer &serializer) const;
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
+	virtual void Serialize(Serializer &serializer) const;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 
 	template <class TARGET>
 	TARGET &Cast() {
@@ -70,11 +62,8 @@ struct DecimalTypeInfo : public ExtraTypeInfo {
 	uint8_t scale;
 
 public:
-	void Serialize(FieldWriter &writer) const override;
-	void FormatSerialize(FormatSerializer &serializer) const override;
-
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -89,11 +78,8 @@ struct StringTypeInfo : public ExtraTypeInfo {
 	string collation;
 
 public:
-	void Serialize(FieldWriter &writer) const override;
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
-
-	void FormatSerialize(FormatSerializer &serializer) const override;
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -108,11 +94,8 @@ struct ListTypeInfo : public ExtraTypeInfo {
 	LogicalType child_type;
 
 public:
-	void Serialize(FieldWriter &writer) const override;
-	void FormatSerialize(FormatSerializer &serializer) const override;
-
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -127,12 +110,8 @@ struct StructTypeInfo : public ExtraTypeInfo {
 	child_list_t<LogicalType> child_types;
 
 public:
-	void Serialize(FieldWriter &writer) const override;
-	void FormatSerialize(FormatSerializer &serializer) const override;
-
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
-
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &deserializer);
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &deserializer);
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -147,11 +126,8 @@ struct AggregateStateTypeInfo : public ExtraTypeInfo {
 	aggregate_state_t state_type;
 
 public:
-	void Serialize(FieldWriter &writer) const override;
-	void FormatSerialize(FormatSerializer &serializer) const override;
-
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -166,11 +142,8 @@ struct UserTypeInfo : public ExtraTypeInfo {
 	string user_type_name;
 
 public:
-	void Serialize(FieldWriter &writer) const override;
-	void FormatSerialize(FormatSerializer &serializer) const override;
-
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -195,11 +168,8 @@ public:
 
 	static LogicalType CreateType(Vector &ordered_data, idx_t size);
 
-	void Serialize(FieldWriter &writer) const override;
-	static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
-
-	void FormatSerialize(FormatSerializer &serializer) const override;
-	static shared_ptr<ExtraTypeInfo> FormatDeserialize(FormatDeserializer &source);
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 
 protected:
 	// Equalities are only used in enums with different catalog entries

package/src/duckdb/src/include/duckdb/common/index_vector.hpp

@@ -10,8 +10,8 @@
 
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/vector.hpp"
-#include "duckdb/common/serializer/format_serializer.hpp"
-#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -61,11 +61,11 @@ public:
 		return internal_vector.end();
 	}
 
-	void FormatSerialize(FormatSerializer &serializer) const {
+	void Serialize(Serializer &serializer) const {
 		serializer.WriteProperty(100, "internal_vector", internal_vector);
 	}
 
-	static IndexVector<T, INDEX_TYPE> FormatDeserialize(FormatDeserializer &deserializer) {
+	static IndexVector<T, INDEX_TYPE> Deserialize(Deserializer &deserializer) {
 		IndexVector<T, INDEX_TYPE> result;
 		deserializer.ReadProperty(100, "internal_vector", result.internal_vector);
 		return result;

package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp

@@ -31,9 +31,7 @@ struct HivePartitioningIndex {
 	idx_t index;
 
 	DUCKDB_API void Serialize(Serializer &serializer) const;
-	DUCKDB_API static HivePartitioningIndex Deserialize(Deserializer &source);
-	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
-	DUCKDB_API static HivePartitioningIndex FormatDeserialize(FormatDeserializer &deserializer);
+	DUCKDB_API static HivePartitioningIndex Deserialize(Deserializer &deserializer);
 };
 
 //! The bind data for the multi-file reader, obtained through MultiFileReader::BindReader
@@ -44,9 +42,7 @@ struct MultiFileReaderBindData {
 	vector<HivePartitioningIndex> hive_partitioning_indexes;
 
 	DUCKDB_API void Serialize(Serializer &serializer) const;
-	DUCKDB_API static MultiFileReaderBindData Deserialize(Deserializer &source);
-	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
-	DUCKDB_API static MultiFileReaderBindData FormatDeserialize(FormatDeserializer &deserializer);
+	DUCKDB_API static MultiFileReaderBindData Deserialize(Deserializer &deserializer);
 };
 
 struct MultiFileFilterEntry {

package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp

@@ -14,8 +14,6 @@
 #include "duckdb/main/client_context.hpp"
 
 namespace duckdb {
-class Serializer;
-class Deserializer;
 struct BindInfo;
 
 struct MultiFileReaderOptions {
@@ -28,8 +26,6 @@ struct MultiFileReaderOptions {
 
 	DUCKDB_API void Serialize(Serializer &serializer) const;
 	DUCKDB_API static MultiFileReaderOptions Deserialize(Deserializer &source);
-	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
-	DUCKDB_API static MultiFileReaderOptions FormatDeserialize(FormatDeserializer &source);
 	DUCKDB_API void AddBatchInfo(BindInfo &bind_info) const;
 	DUCKDB_API void AutoDetectHivePartitioning(const vector<string> &files, ClientContext &context);
 	DUCKDB_API static bool AutoDetectHivePartitioningInternal(const vector<string> &files, ClientContext &context);

package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp

@@ -8,50 +8,64 @@
 
 #pragma once
 
-#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 #include "duckdb/common/serializer/encoding_util.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
 
 namespace duckdb {
 class ClientContext;
 
-class BinaryDeserializer : public FormatDeserializer {
+class BinaryDeserializer : public Deserializer {
 public:
+	explicit BinaryDeserializer(ReadStream &stream) : stream(stream) {
+		deserialize_enum_from_string = false;
+	}
+
 	template <class T>
 	unique_ptr<T> Deserialize() {
 		OnObjectBegin();
-		auto result = T::FormatDeserialize(*this);
+		auto result = T::Deserialize(*this);
 		OnObjectEnd();
 		D_ASSERT(nesting_level == 0); // make sure we are at the root level
 		return result;
 	}
 
 	template <class T>
-	static unique_ptr<T> Deserialize(data_ptr_t ptr, idx_t length) {
-		BinaryDeserializer deserializer(ptr, length);
+	static unique_ptr<T> Deserialize(ReadStream &stream) {
+		BinaryDeserializer deserializer(stream);
 		return deserializer.template Deserialize<T>();
 	}
 
 	template <class T>
-	static unique_ptr<T> Deserialize(ClientContext &context, bound_parameter_map_t &parameters, data_ptr_t ptr,
-	                                 idx_t length) {
-		BinaryDeserializer deserializer(ptr, length);
+	static unique_ptr<T> Deserialize(ReadStream &stream, ClientContext &context, bound_parameter_map_t &parameters) {
+		BinaryDeserializer deserializer(stream);
 		deserializer.Set<ClientContext &>(context);
 		deserializer.Set<bound_parameter_map_t &>(parameters);
 		return deserializer.template Deserialize<T>();
 	}
 
-private:
-	explicit BinaryDeserializer(data_ptr_t ptr, idx_t length) : ptr(ptr), end_ptr(ptr + length) {
-		deserialize_enum_from_string = false;
+	void Begin() {
+		OnObjectBegin();
+	}
+
+	void End() {
+		OnObjectEnd();
+		D_ASSERT(nesting_level == 0); // make sure we are at the root level
+	}
+
+	ReadStream &GetStream() {
+		return stream;
 	}
 
-	data_ptr_t ptr;
-	data_ptr_t end_ptr;
+private:
+	ReadStream &stream;
 	idx_t nesting_level = 0;
 
 	// Allow peeking 1 field ahead
 	bool has_buffered_field = false;
 	field_id_t buffered_field = 0;
+
+private:
 	field_id_t PeekField() {
 		if (!has_buffered_field) {
 			buffered_field = ReadPrimitive<field_id_t>();
@@ -74,6 +88,10 @@ private:
 		return ReadPrimitive<field_id_t>();
 	}
 
+	void ReadData(data_ptr_t buffer, idx_t read_size) {
+		stream.ReadData(buffer, read_size);
+	}
+
 	template <class T>
 	T ReadPrimitive() {
 		T value;
@@ -81,19 +99,22 @@ private:
 		return value;
 	}
 
-	void ReadData(data_ptr_t buffer, idx_t read_size) {
-		if (ptr + read_size > end_ptr) {
-			throw InternalException("Failed to deserialize: not enough data in buffer to fulfill read request");
-		}
-		memcpy(buffer, ptr, read_size);
-		ptr += read_size;
-	}
-
 	template <class T>
 	T VarIntDecode() {
+		// FIXME: maybe we should pass a source to EncodingUtil instead
+		uint8_t buffer[16];
+		idx_t varint_size;
+		for (varint_size = 0; varint_size < 16; varint_size++) {
+			ReadData(buffer + varint_size, 1);
+			if (!(buffer[varint_size] & 0x80)) {
+				varint_size++;
+				break;
+			}
+		}
 		T value;
-		auto read_size = EncodingUtil::DecodeLEB128<T>(ptr, value);
-		ptr += read_size;
+		auto read_size = EncodingUtil::DecodeLEB128<T>(buffer, value);
+		D_ASSERT(read_size == varint_size);
+		(void)read_size;
 		return value;
 	}
 

package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp

@@ -8,13 +8,20 @@
 
 #pragma once
 
-#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/serializer/encoding_util.hpp"
 
 namespace duckdb {
 
-struct BinarySerializer : public FormatSerializer {
+class BinarySerializer : public Serializer {
+public:
+	explicit BinarySerializer(WriteStream &stream, bool serialize_default_values_p = false) : stream(stream) {
+		serialize_default_values = serialize_default_values_p;
+		serialize_enum_as_string = false;
+	}
+
 private:
 	struct DebugState {
 		unordered_set<const char *> seen_field_tags;
@@ -22,19 +29,17 @@ private:
 		vector<pair<const char *, field_id_t>> seen_fields;
 	};
 
-	vector<DebugState> debug_stack;
-	vector<data_t> data;
+	void WriteData(const_data_ptr_t buffer, idx_t write_size) {
+		stream.WriteData(buffer, write_size);
+	}
 
 	template <class T>
 	void Write(T element) {
 		static_assert(std::is_trivially_destructible<T>(), "Write element must be trivially destructible");
-		WriteDataInternal(const_data_ptr_cast(&element), sizeof(T));
+		WriteData(const_data_ptr_cast(&element), sizeof(T));
 	}
-	void WriteDataInternal(const_data_ptr_t buffer, idx_t write_size) {
-		data.insert(data.end(), buffer, buffer + write_size);
-	}
-	void WriteDataInternal(const char *ptr, idx_t write_size) {
-		WriteDataInternal(const_data_ptr_cast(ptr), write_size);
+	void WriteData(const char *ptr, idx_t write_size) {
+		WriteData(const_data_ptr_cast(ptr), write_size);
 	}
 
 	template <class T>
@@ -42,27 +47,26 @@ private:
 		uint8_t buffer[16];
 		auto write_size = EncodingUtil::EncodeLEB128<T>(buffer, value);
 		D_ASSERT(write_size <= sizeof(buffer));
-		WriteDataInternal(buffer, write_size);
-	}
-
-	explicit BinarySerializer(bool serialize_default_values_p) {
-		serialize_default_values = serialize_default_values_p;
-		serialize_enum_as_string = false;
+		WriteData(buffer, write_size);
 	}
 
 public:
-	//! Serializes the given object into a binary blob, optionally serializing default values if
-	//! serialize_default_values is set to true, otherwise properties set to their provided default value
-	//! will not be serialized
 	template <class T>
-	static vector<data_t> Serialize(T &obj, bool serialize_default_values) {
-		BinarySerializer serializer(serialize_default_values);
+	static void Serialize(const T &value, WriteStream &stream, bool serialize_default_values = false) {
+		BinarySerializer serializer(stream, serialize_default_values);
 		serializer.OnObjectBegin();
-		obj.FormatSerialize(serializer);
+		value.Serialize(serializer);
 		serializer.OnObjectEnd();
-		return std::move(serializer.data);
 	}
 
+	void Begin() {
+		OnObjectBegin();
+	}
+	void End() {
+		OnObjectEnd();
+	}
+
+protected:
 	//-------------------------------------------------------------------------
 	// Nested Type Hooks
 	//-------------------------------------------------------------------------
@@ -99,6 +103,10 @@ public:
 	void WriteValue(const char *value) final;
 	void WriteValue(bool value) final;
 	void WriteDataPtr(const_data_ptr_t ptr, idx_t count) final;
+
+private:
+	vector<DebugState> debug_stack;
+	WriteStream &stream;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp

@@ -9,13 +9,14 @@
 #pragma once
 
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
 
 namespace duckdb {
 
-class BufferedFileReader : public Deserializer {
+class BufferedFileReader : public ReadStream {
 public:
-	BufferedFileReader(FileSystem &fs, const char *path, optional_ptr<ClientContext> context,
-	                   FileLockType lock_type = FileLockType::READ_LOCK, optional_ptr<FileOpener> opener = nullptr);
+	BufferedFileReader(FileSystem &fs, const char *path, FileLockType lock_type = FileLockType::READ_LOCK,
+	                   optional_ptr<FileOpener> opener = nullptr);
 
 	FileSystem &fs;
 	unsafe_unique_array<data_t> data;
@@ -35,12 +36,9 @@ public:
 	void Seek(uint64_t location);
 	uint64_t CurrentOffset();
 
-	ClientContext &GetContext() override;
-
 private:
 	idx_t file_size;
 	idx_t total_read;
-	optional_ptr<ClientContext> context;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp

@@ -8,14 +8,14 @@
 
 #pragma once
 
-#include "duckdb/common/serializer.hpp"
+#include "duckdb/common/serializer/write_stream.hpp"
 #include "duckdb/common/file_system.hpp"
 
 namespace duckdb {
 
 #define FILE_BUFFER_SIZE 4096
 
-class BufferedFileWriter : public Serializer {
+class BufferedFileWriter : public WriteStream {
 public:
 	static constexpr uint8_t DEFAULT_OPEN_FLAGS = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE;
 
@@ -31,7 +31,7 @@ public:
 	unique_ptr<FileHandle> handle;
 
 public:
-	DUCKDB_API void WriteData(const_data_ptr_t buffer, uint64_t write_size) override;
+	DUCKDB_API void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
 	//! Flush the buffer to disk and sync the file to ensure writing is completed
 	DUCKDB_API void Sync();
 	//! Flush the buffer to the file (without sync)

package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp

@@ -74,6 +74,23 @@ inline void DeserializationData::Unset<LogicalOperatorType>() {
 	enums.pop();
 }
 
+template <>
+inline void DeserializationData::Set(CatalogType type) {
+	enums.push(idx_t(type));
+}
+
+template <>
+inline CatalogType DeserializationData::Get() {
+	AssertNotEmpty(enums);
+	return CatalogType(enums.top());
+}
+
+template <>
+inline void DeserializationData::Unset<CatalogType>() {
+	AssertNotEmpty(enums);
+	enums.pop();
+}
+
 template <>
 inline void DeserializationData::Set(ClientContext &context) {
 	contexts.push(context);

package/src/duckdb/src/include/duckdb/common/serializer/{format_deserializer.hpp → deserializer.hpp}

@@ -8,8 +8,6 @@
 
 #pragma once
 
-#include "duckdb/common/field_writer.hpp"
-#include "duckdb/common/serializer.hpp"
 #include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/serializer/serialization_traits.hpp"
 #include "duckdb/common/serializer/deserialization_data.hpp"
@@ -19,18 +17,21 @@
 
 namespace duckdb {
 
-class FormatDeserializer {
+class Deserializer {
 protected:
 	bool deserialize_enum_from_string = false;
 	DeserializationData data;
 
 public:
+	virtual ~Deserializer() {
+	}
+
 	class List {
-		friend FormatDeserializer;
+		friend Deserializer;
 
 	private:
-		FormatDeserializer &deserializer;
-		explicit List(FormatDeserializer &deserializer) : deserializer(deserializer) {
+		Deserializer &deserializer;
+		explicit List(Deserializer &deserializer) : deserializer(deserializer) {
 		}
 
 	public:
@@ -58,7 +59,6 @@ public:
 		OnPropertyBegin(field_id, tag);
 		auto ret = Read<T>();
 		OnPropertyEnd();
-		;
 		return ret;
 	}
 
@@ -134,11 +134,11 @@ public:
 	}
 
 private:
-	// Deserialize anything implementing a FormatDeserialize method
+	// Deserialize anything implementing a Deserialize method
 	template <typename T = void>
 	inline typename std::enable_if<has_deserialize<T>::value, T>::type Read() {
 		OnObjectBegin();
-		auto val = T::FormatDeserialize(*this);
+		auto val = T::Deserialize(*this);
 		OnObjectEnd();
 		return val;
 	}
@@ -150,7 +150,7 @@ private:
 		auto is_present = OnNullableBegin();
 		if (is_present) {
 			OnObjectBegin();
-			ptr = ELEMENT_TYPE::FormatDeserialize(*this);
+			ptr = ELEMENT_TYPE::Deserialize(*this);
 			OnObjectEnd();
 		}
 		OnNullableEnd();
@@ -165,7 +165,7 @@ private:
 		auto is_present = OnNullableBegin();
 		if (is_present) {
 			OnObjectBegin();
-			ptr = ELEMENT_TYPE::FormatDeserialize(*this);
+			ptr = ELEMENT_TYPE::Deserialize(*this);
 			OnObjectEnd();
 		}
 		OnNullableEnd();
@@ -416,14 +416,14 @@ protected:
 };
 
 template <class FUNC>
-void FormatDeserializer::List::ReadObject(FUNC f) {
+void Deserializer::List::ReadObject(FUNC f) {
 	deserializer.OnObjectBegin();
 	f(deserializer);
 	deserializer.OnObjectEnd();
 }
 
 template <class T>
-T FormatDeserializer::List::ReadElement() {
+T Deserializer::List::ReadElement() {
 	return deserializer.Read<T>();
 }
 

package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/serializer/buffer_stream.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/serializer/write_stream.hpp"
+#include "duckdb/common/serializer/read_stream.hpp"
+#include "duckdb/common/typedefs.hpp"
+
+namespace duckdb {
+
+class MemoryStream : public WriteStream, public ReadStream {
+private:
+	idx_t position;
+	idx_t capacity;
+	bool owns_data;
+	data_ptr_t data;
+
+public:
+	// Create a new owning MemoryStream with an internal  backing buffer with the specified capacity. The stream will
+	// own the backing buffer, resize it when needed and free its memory when the stream is destroyed
+	explicit MemoryStream(idx_t capacity = 512);
+
+	// Create a new non-owning MemoryStream over the specified external buffer and capacity. The stream will not take
+	// ownership of the backing buffer, will not attempt to resize it and will not free the memory when the stream
+	// is destroyed
+	explicit MemoryStream(data_ptr_t buffer, idx_t capacity);
+
+	~MemoryStream() override;
+
+	// Write data to the stream.
+	// Throws if the write would exceed the capacity of the stream and the backing buffer is not owned by the stream
+	void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
+
+	// Read data from the stream.
+	// Throws if the read would exceed the capacity of the stream
+	void ReadData(data_ptr_t buffer, idx_t read_size) override;
+
+	// Rewind the stream to the start, keeping the capacity and the backing buffer intact
+	void Rewind();
+
+	// Release ownership of the backing buffer and turn a owning stream into a non-owning one.
+	// The stream will no longer be responsible for freeing the data.
+	// The stream will also no longer attempt to automatically resize the buffer when the capacity is reached.
+	void Release();
+
+	// Get a pointer to the underlying backing buffer
+	data_ptr_t GetData() const;
+
+	// Get the current position in the stream
+	idx_t GetPosition() const;
+
+	// Get the capacity of the stream
+	idx_t GetCapacity() const;
+};
+
+} // namespace duckdb