duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp

@@ -28,43 +28,15 @@ protected:
 public:
 	// Serialize a value
 	template <class T>
-	typename std::enable_if<!std::is_enum<T>::value, void>::type WriteProperty(const char *tag, T &value) {
-		SetTag(tag);
+	void WriteProperty(const field_id_t field_id, const char *tag, const T &value) {
+		SetTag(field_id, tag);
 		WriteValue(value);
 	}
 
-	// Serialize an enum
-	template <class T>
-	typename std::enable_if<std::is_enum<T>::value, void>::type WriteProperty(const char *tag, T value) {
-		SetTag(tag);
-		if (serialize_enum_as_string) {
-			// Use the enum serializer to lookup tostring function
-			auto str = EnumUtil::ToChars(value);
-			WriteValue(str);
-		} else {
-			// Use the underlying type
-			WriteValue(static_cast<typename std::underlying_type<T>::type>(value));
-		}
-	}
-
 	// Optional pointer
-	template <class T>
-	void WriteOptionalProperty(const char *tag, T *ptr) {
-		SetTag(tag);
-		if (ptr == nullptr) {
-			OnOptionalBegin(false);
-			OnOptionalEnd(false);
-		} else {
-			OnOptionalBegin(true);
-			WriteValue(*ptr);
-			OnOptionalEnd(true);
-		}
-	}
-
-	// Optional unique_ptr
-	template <class T>
-	void WriteOptionalProperty(const char *tag, const unique_ptr<T> &ptr) {
-		SetTag(tag);
+	template <class POINTER>
+	void WriteOptionalProperty(const field_id_t field_id, const char *tag, POINTER &&ptr) {
+		SetTag(field_id, tag);
 		if (ptr == nullptr) {
 			OnOptionalBegin(false);
 			OnOptionalEnd(false);
@@ -76,12 +48,34 @@ public:
 	}
 
 	// Special case: data_ptr_T
-	void WriteProperty(const char *tag, const_data_ptr_t ptr, idx_t count) {
-		SetTag(tag);
+	void WriteProperty(const field_id_t field_id, const char *tag, const_data_ptr_t ptr, idx_t count) {
+		SetTag(field_id, tag);
 		WriteDataPtr(ptr, count);
 	}
 
+	// Manually begin an object - should be followed by EndObject
+	void BeginObject(const field_id_t field_id, const char *tag) {
+		SetTag(field_id, tag);
+		OnObjectBegin();
+	}
+
+	void EndObject() {
+		OnObjectEnd();
+	}
+
 protected:
+	template <typename T>
+	typename std::enable_if<std::is_enum<T>::value, void>::type WriteValue(const T value) {
+		if (serialize_enum_as_string) {
+			// Use the enum serializer to lookup tostring function
+			auto str = EnumUtil::ToChars(value);
+			WriteValue(str);
+		} else {
+			// Use the underlying type
+			WriteValue(static_cast<typename std::underlying_type<T>::type>(value));
+		}
+	}
+
 	// Unique Pointer Ref
 	template <typename T>
 	void WriteValue(const unique_ptr<T> &ptr) {
@@ -122,10 +116,20 @@ protected:
 		OnListEnd(count);
 	}
 
+	template <class T>
+	void WriteValue(const unsafe_vector<T> &vec) {
+		auto count = vec.size();
+		OnListBegin(count);
+		for (auto &item : vec) {
+			WriteValue(item);
+		}
+		OnListEnd(count);
+	}
+
 	// UnorderedSet
 	// Serialized the same way as a list/vector
 	template <class T, class HASH, class CMP>
-	void WriteValue(const unordered_set<T, HASH, CMP> &set) {
+	void WriteValue(const duckdb::unordered_set<T, HASH, CMP> &set) {
 		auto count = set.size();
 		OnListBegin(count);
 		for (auto &item : set) {
@@ -137,7 +141,7 @@ protected:
 	// Set
 	// Serialized the same way as a list/vector
 	template <class T, class HASH, class CMP>
-	void WriteValue(const set<T, HASH, CMP> &set) {
+	void WriteValue(const duckdb::set<T, HASH, CMP> &set) {
 		auto count = set.size();
 		OnListBegin(count);
 		for (auto &item : set) {
@@ -148,7 +152,25 @@ protected:
 
 	// Map
 	template <class K, class V, class HASH, class CMP>
-	void WriteValue(const std::unordered_map<K, V, HASH, CMP> &map) {
+	void WriteValue(const duckdb::unordered_map<K, V, HASH, CMP> &map) {
+		auto count = map.size();
+		OnMapBegin(count);
+		for (auto &item : map) {
+			OnMapEntryBegin();
+			OnMapKeyBegin();
+			WriteValue(item.first);
+			OnMapKeyEnd();
+			OnMapValueBegin();
+			WriteValue(item.second);
+			OnMapValueEnd();
+			OnMapEntryEnd();
+		}
+		OnMapEnd(count);
+	}
+
+	// Map
+	template <class K, class V, class HASH, class CMP>
+	void WriteValue(const duckdb::map<K, V, HASH, CMP> &map) {
 		auto count = map.size();
 		OnMapBegin(count);
 		for (auto &item : map) {
@@ -166,7 +188,7 @@ protected:
 
 	// class or struct implementing `FormatSerialize(FormatSerializer& FormatSerializer)`;
 	template <typename T>
-	typename std::enable_if<has_serialize<T>::value>::type WriteValue(T &value) {
+	typename std::enable_if<has_serialize<T>::value>::type WriteValue(const T &value) {
 		// Else, we defer to the .FormatSerialize method
 		OnObjectBegin();
 		value.FormatSerialize(*this);
@@ -174,7 +196,8 @@ protected:
 	}
 
 	// Handle setting a "tag" (optional)
-	virtual void SetTag(const char *tag) {
+	virtual void SetTag(const field_id_t field_id, const char *tag) {
+		(void)field_id;
 		(void)tag;
 	}
 
@@ -243,6 +266,16 @@ protected:
 	virtual void WriteValue(const char *str) = 0;
 	virtual void WriteValue(interval_t value) = 0;
 	virtual void WriteDataPtr(const_data_ptr_t ptr, idx_t count) = 0;
+	void WriteValue(LogicalIndex value) {
+		WriteValue(value.index);
+	}
+	void WriteValue(PhysicalIndex value) {
+		WriteValue(value.index);
+	}
 };
 
+// We need to special case vector<bool> because elements of vector<bool> cannot be referenced
+template <>
+void FormatSerializer::WriteValue(const vector<bool> &vec);
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp

@@ -1,14 +1,19 @@
 #pragma once
 #include <type_traits>
+#include <cstdint>
+
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
-
+#include "duckdb/common/set.hpp"
+#include "duckdb/common/shared_ptr.hpp"
 namespace duckdb {
 
 class FormatSerializer;   // Forward declare
 class FormatDeserializer; // Forward declare
 
+typedef uint32_t field_id_t;
+
 // Backport to c++11
 template <class...>
 using void_t = void;
@@ -51,11 +56,28 @@ struct is_vector<typename duckdb::vector<T>> : std::true_type {
 	typedef T ELEMENT_TYPE;
 };
 
+template <typename T>
+struct is_unsafe_vector : std::false_type {};
+template <typename T>
+struct is_unsafe_vector<typename duckdb::unsafe_vector<T>> : std::true_type {
+	typedef T ELEMENT_TYPE;
+};
+
 // Check if T is a unordered map, and provide access to the inner type
 template <typename T>
 struct is_unordered_map : std::false_type {};
 template <typename... Args>
-struct is_unordered_map<typename std::unordered_map<Args...>> : std::true_type {
+struct is_unordered_map<typename duckdb::unordered_map<Args...>> : std::true_type {
+	typedef typename std::tuple_element<0, std::tuple<Args...>>::type KEY_TYPE;
+	typedef typename std::tuple_element<1, std::tuple<Args...>>::type VALUE_TYPE;
+	typedef typename std::tuple_element<2, std::tuple<Args...>>::type HASH_TYPE;
+	typedef typename std::tuple_element<3, std::tuple<Args...>>::type EQUAL_TYPE;
+};
+
+template <typename T>
+struct is_map : std::false_type {};
+template <typename... Args>
+struct is_map<typename duckdb::map<Args...>> : std::true_type {
 	typedef typename std::tuple_element<0, std::tuple<Args...>>::type KEY_TYPE;
 	typedef typename std::tuple_element<1, std::tuple<Args...>>::type VALUE_TYPE;
 	typedef typename std::tuple_element<2, std::tuple<Args...>>::type HASH_TYPE;
@@ -90,7 +112,7 @@ struct is_pair<std::pair<T, U>> : std::true_type {
 template <typename T>
 struct is_unordered_set : std::false_type {};
 template <typename... Args>
-struct is_unordered_set<std::unordered_set<Args...>> : std::true_type {
+struct is_unordered_set<duckdb::unordered_set<Args...>> : std::true_type {
 	typedef typename std::tuple_element<0, std::tuple<Args...>>::type ELEMENT_TYPE;
 	typedef typename std::tuple_element<1, std::tuple<Args...>>::type HASH_TYPE;
 	typedef typename std::tuple_element<2, std::tuple<Args...>>::type EQUAL_TYPE;
@@ -99,7 +121,7 @@ struct is_unordered_set<std::unordered_set<Args...>> : std::true_type {
 template <typename T>
 struct is_set : std::false_type {};
 template <typename... Args>
-struct is_set<std::set<Args...>> : std::true_type {
+struct is_set<duckdb::set<Args...>> : std::true_type {
 	typedef typename std::tuple_element<0, std::tuple<Args...>>::type ELEMENT_TYPE;
 	typedef typename std::tuple_element<1, std::tuple<Args...>>::type HASH_TYPE;
 	typedef typename std::tuple_element<2, std::tuple<Args...>>::type EQUAL_TYPE;

package/src/duckdb/src/include/duckdb/common/serializer.hpp

@@ -22,8 +22,6 @@ private:
 	uint64_t version = 0L;
 
 public:
-	bool is_query_plan = false;
-
 	virtual ~Serializer() {
 	}
 
@@ -119,11 +117,6 @@ public:
 		throw InternalException("This deserializer does not have a client-context");
 	};
 
-	//! Gets the catalog for the deserializer
-	virtual optional_ptr<Catalog> GetCatalog() {
-		return nullptr;
-	};
-
 	template <class T>
 	T Read() {
 		T value;

package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp

@@ -42,8 +42,8 @@ public:
 	using Orders = vector<BoundOrderByNode>;
 	using Types = vector<LogicalType>;
 
-	using GroupingPartition = unique_ptr<PartitionedColumnData>;
-	using GroupingAppend = unique_ptr<PartitionedColumnDataAppendState>;
+	using GroupingPartition = unique_ptr<PartitionedTupleData>;
+	using GroupingAppend = unique_ptr<PartitionedTupleDataAppendState>;
 
 	static void GenerateOrderings(Orders &partitions, Orders &orders,
 	                              const vector<unique_ptr<Expression>> &partition_bys, const Orders &order_bys,
@@ -53,10 +53,14 @@ public:
 	                         const vector<BoundOrderByNode> &order_bys, const Types &payload_types,
 	                         const vector<unique_ptr<BaseStatistics>> &partitions_stats, idx_t estimated_cardinality);
 
+	unique_ptr<RadixPartitionedTupleData> CreatePartition(idx_t new_bits) const;
+	void SyncPartitioning(const PartitionGlobalSinkState &other);
+
 	void UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
 	void CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
 
-	void BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &global_sort);
+	void BuildSortState(TupleDataCollection &group_data, GlobalSortState &global_sort) const;
+	void BuildSortState(TupleDataCollection &group_data, PartitionGlobalHashGroup &global_sort);
 
 	ClientContext &context;
 	BufferManager &buffer_manager;
@@ -64,9 +68,11 @@ public:
 	mutex lock;
 
 	// OVER(PARTITION BY...) (hash grouping)
-	unique_ptr<RadixPartitionedColumnData> grouping_data;
+	unique_ptr<RadixPartitionedTupleData> grouping_data;
 	//! Payload plus hash column
-	Types grouping_types;
+	TupleDataLayout grouping_types;
+	//! The number of radix bits if this partition is being synced with another
+	idx_t fixed_bits;
 
 	// OVER(...) (sorting)
 	Orders partitions;
@@ -83,6 +89,7 @@ public:
 
 	// Threading
 	idx_t memory_per_thread;
+	idx_t max_bits;
 	atomic<idx_t> count;
 
 private:
@@ -102,8 +109,8 @@ public:
 	ExpressionExecutor executor;
 	DataChunk group_chunk;
 	DataChunk payload_chunk;
-	unique_ptr<PartitionedColumnData> local_partition;
-	unique_ptr<PartitionedColumnDataAppendState> local_append;
+	unique_ptr<PartitionedTupleData> local_partition;
+	unique_ptr<PartitionedTupleDataAppendState> local_append;
 
 	// OVER(...) (sorting)
 	size_t sort_cols;
@@ -127,7 +134,7 @@ class PartitionLocalMergeState;
 
 class PartitionGlobalMergeState {
 public:
-	using GroupDataPtr = unique_ptr<ColumnDataCollection>;
+	using GroupDataPtr = unique_ptr<TupleDataCollection>;
 
 	PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data, hash_t hash_bin);
 
@@ -175,10 +182,18 @@ public:
 
 class PartitionGlobalMergeStates {
 public:
+	struct Callback {
+		virtual bool HasError() const {
+			return false;
+		}
+	};
+
 	using PartitionGlobalMergeStatePtr = unique_ptr<PartitionGlobalMergeState>;
 
 	explicit PartitionGlobalMergeStates(PartitionGlobalSinkState &sink);
 
+	bool ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback);
+
 	vector<PartitionGlobalMergeStatePtr> states;
 };
 

package/src/duckdb/src/include/duckdb/common/stack_checker.hpp

@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/stack_checker.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+namespace duckdb {
+
+template <class RECURSIVE_CLASS>
+class StackChecker {
+public:
+	StackChecker(RECURSIVE_CLASS &recursive_class_p, idx_t stack_usage_p)
+	    : recursive_class(recursive_class_p), stack_usage(stack_usage_p) {
+		recursive_class.stack_depth += stack_usage;
+	}
+	~StackChecker() {
+		recursive_class.stack_depth -= stack_usage;
+	}
+	StackChecker(StackChecker &&other) noexcept
+	    : recursive_class(other.recursive_class), stack_usage(other.stack_usage) {
+		other.stack_usage = 0;
+	}
+	StackChecker(const StackChecker &) = delete;
+
+private:
+	RECURSIVE_CLASS &recursive_class;
+	idx_t stack_usage;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/string_util.hpp

@@ -16,6 +16,17 @@
 
 namespace duckdb {
 
+#ifndef DUCKDB_QUOTE_DEFINE
+// Preprocessor trick to allow text to be converted to C-string / string
+// Expecte use is:
+//	#ifdef SOME_DEFINE
+//	string str = DUCKDB_QUOTE_DEFINE(SOME_DEFINE)
+//	...do something with str
+//	#endif SOME_DEFINE
+#define DUCKDB_QUOTE_DEFINE_IMPL(x) #x
+#define DUCKDB_QUOTE_DEFINE(x)      DUCKDB_QUOTE_DEFINE_IMPL(x)
+#endif
+
 /**
  * String Utility Functions
  * Note that these are not the most efficient implementations (i.e., they copy

package/src/duckdb/src/include/duckdb/common/type_util.hpp

@@ -1,3 +1,11 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/type_util.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
 #include "duckdb/common/types.hpp"

package/src/duckdb/src/include/duckdb/common/typedefs.hpp

@@ -1,3 +1,11 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/typedefs.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
 #pragma once
 
 #include <cstdint>

package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp

@@ -24,7 +24,7 @@ struct BatchedChunkScanState {
 //! Scans over a BatchedDataCollection are ordered by batch index
 class BatchedDataCollection {
 public:
-	DUCKDB_API BatchedDataCollection(vector<LogicalType> types);
+	DUCKDB_API BatchedDataCollection(ClientContext &context, vector<LogicalType> types, bool buffer_managed = false);
 
 	//! Appends a datachunk with the given batch index to the batched collection
 	DUCKDB_API void Append(DataChunk &input, idx_t batch_index);
@@ -51,7 +51,9 @@ private:
 		ColumnDataAppendState append_state;
 	};
 
+	ClientContext &context;
 	vector<LogicalType> types;
+	bool buffer_managed;
 	//! The data of the batched chunk collection - a set of batch_index -> ColumnDataCollection pointers
 	map<idx_t, unique_ptr<ColumnDataCollection>> data;
 	//! The last batch collection that was inserted into

package/src/duckdb/src/include/duckdb/common/types/bit.hpp

@@ -8,8 +8,12 @@
 
 #pragma once
 
+#include "duckdb/common/assert.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/hugeint.hpp"
+#include "duckdb/common/limits.hpp"
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/types/string_type.hpp"
 
 namespace duckdb {
 
@@ -37,7 +41,33 @@ public:
 	//! Convert a string to a bit. This function should ONLY be called after calling GetBitSize, since it does NOT
 	//! perform data validation.
 	DUCKDB_API static void ToBit(string_t str, string_t &output);
+
 	DUCKDB_API static string ToBit(string_t str);
+
+	//! output needs to have enough space allocated before calling this function (blob size + 1)
+	DUCKDB_API static void BlobToBit(string_t blob, string_t &output);
+
+	DUCKDB_API static string BlobToBit(string_t blob);
+
+	//! output_str needs to have enough space allocated before calling this function (sizeof(T) + 1)
+	template <class T>
+	static void NumericToBit(T numeric, string_t &output_str);
+
+	template <class T>
+	static string NumericToBit(T numeric);
+
+	//! bit is expected to fit inside of output num (bit size <= sizeof(T) + 1)
+	template <class T>
+	static void BitToNumeric(string_t bit, T &output_num);
+
+	template <class T>
+	static T BitToNumeric(string_t bit);
+
+	//! bit is expected to fit inside of output_blob (bit size = output_blob + 1)
+	static void BitToBlob(string_t bit, string_t &output_blob);
+
+	static string BitToBlob(string_t bit);
+
 	//! Creates a new bitstring of determined length
 	DUCKDB_API static void BitString(const string_t &input, const idx_t &len, string_t &result);
 	DUCKDB_API static void SetEmptyBitString(string_t &target, string_t &input);
@@ -58,5 +88,56 @@ private:
 	static idx_t GetBitInternal(string_t bit_string, idx_t n);
 	static void SetBitInternal(string_t &bit_string, idx_t n, idx_t new_value);
 	static idx_t GetBitIndex(idx_t n);
+	static uint8_t GetFirstByte(const string_t &str);
 };
+
+//===--------------------------------------------------------------------===//
+// Bit Template definitions
+//===--------------------------------------------------------------------===//
+template <class T>
+void Bit::NumericToBit(T numeric, string_t &output_str) {
+	D_ASSERT(output_str.GetSize() >= sizeof(T) + 1);
+
+	auto output = output_str.GetDataWriteable();
+	auto data = const_data_ptr_cast(&numeric);
+
+	*output = 0; // set padding to 0
+	++output;
+	for (idx_t idx = 0; idx < sizeof(T); ++idx) {
+		output[idx] = data[sizeof(T) - idx - 1];
+	}
+	Bit::Finalize(output_str);
+}
+
+template <class T>
+string Bit::NumericToBit(T numeric) {
+	auto bit_len = sizeof(T) + 1;
+	auto buffer = make_unsafe_uniq_array<char>(bit_len);
+	string_t output_str(buffer.get(), bit_len);
+	Bit::NumericToBit(numeric, output_str);
+	return output_str.GetString();
+}
+
+template <class T>
+T Bit::BitToNumeric(string_t bit) {
+	T output;
+	Bit::BitToNumeric(bit, output);
+	return (output);
+}
+
+template <class T>
+void Bit::BitToNumeric(string_t bit, T &output_num) {
+	D_ASSERT(bit.GetSize() <= sizeof(T) + 1);
+
+	output_num = 0;
+	auto data = const_data_ptr_cast(bit.GetData());
+	auto output = data_ptr_cast(&output_num);
+
+	idx_t padded_byte_idx = sizeof(T) - bit.GetSize() + 1;
+	output[sizeof(T) - 1 - padded_byte_idx] = GetFirstByte(bit);
+	for (idx_t idx = padded_byte_idx + 1; idx < sizeof(T); ++idx) {
+		output[sizeof(T) - 1 - idx] = data[1 + idx - padded_byte_idx];
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp

@@ -43,9 +43,19 @@ public:
 	void MakeShared() {
 		shared = true;
 	}
+	bool IsShared() const {
+		return shared;
+	}
 	idx_t BlockCount() const {
 		return blocks.size();
 	}
+	idx_t SizeInBytes() const {
+		idx_t total_size = 0;
+		for (const auto &block : blocks) {
+			total_size += block.size;
+		}
+		return total_size;
+	}
 
 public:
 	void AllocateData(idx_t size, uint32_t &block_id, uint32_t &offset, ChunkManagementState *chunk_state);
@@ -53,7 +63,7 @@ public:
 	void Initialize(ColumnDataAllocator &other);
 	void InitializeChunkState(ChunkManagementState &state, ChunkMetaData &meta_data);
 	data_ptr_t GetDataPointer(ChunkManagementState &state, uint32_t block_id, uint32_t offset);
-	void UnswizzlePointers(ChunkManagementState &state, Vector &result, uint16_t v_offset, uint16_t count,
+	void UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
 	                       uint32_t block_id, uint32_t offset);
 
 	//! Deletes the block with the given id

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp

@@ -61,6 +61,9 @@ public:
 		return types.size();
 	}
 
+	//! The size (in bytes) of this ColumnDataCollection
+	idx_t SizeInBytes() const;
+
 	//! Get the allocator
 	DUCKDB_API Allocator &GetAllocator() const;
 
@@ -143,9 +146,17 @@ public:
 	//! Initialize the column data collection
 	void Initialize(vector<LogicalType> types);
 
-	//! Get a vector of references to every chunk (segment, index in segment), and optionally sort by block id
+	//! Get references to the string heaps in this ColumnDataCollection
+	vector<shared_ptr<StringHeap>> GetHeapReferences();
+	//! Get the allocator type of this ColumnDataCollection
+	ColumnDataAllocatorType GetAllocatorType() const;
+
+	//! Get a vector of the segments in this ColumnDataCollection
 	const vector<unique_ptr<ColumnDataCollectionSegment>> &GetSegments() const;
 
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static unique_ptr<ColumnDataCollection> FormatDeserialize(FormatDeserializer &deserializer);
+
 private:
 	//! Creates a new segment within the ColumnDataCollection
 	void CreateSegment();

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp

@@ -94,7 +94,7 @@ public:
 	//! The set of child indices
 	vector<VectorDataIndex> child_indices;
 	//! The string heap for the column data collection (only used for IN_MEMORY_ALLOCATOR)
-	StringHeap heap;
+	shared_ptr<StringHeap> heap;
 
 public:
 	void AllocateNewChunk();
@@ -126,6 +126,8 @@ public:
 	}
 
 	idx_t ChunkCount() const;
+	idx_t SizeInBytes() const;
+
 	void FetchChunk(idx_t chunk_idx, DataChunk &result);
 	void FetchChunk(idx_t chunk_idx, DataChunk &result, const vector<column_t> &column_ids);
 

package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp

@@ -20,7 +20,9 @@ enum class ColumnDataAllocatorType : uint8_t {
 	BUFFER_MANAGER_ALLOCATOR,
 	//! Use an in-memory allocator, allocating data for every chunk
 	//! This causes the column data collection to allocate blocks that are not tied to a buffer manager
-	IN_MEMORY_ALLOCATOR
+	IN_MEMORY_ALLOCATOR,
+	//! Use a buffer manager to allocate vectors, but use a StringHeap for strings
+	HYBRID
 };
 
 enum class ColumnDataScanProperties : uint8_t {

package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp

@@ -14,8 +14,6 @@
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/winapi.hpp"
 
-struct ArrowArray;
-
 namespace duckdb {
 class Allocator;
 class ClientContext;
@@ -153,7 +151,7 @@ public:
 
 	//! Converts this DataChunk to a printable string representation
 	DUCKDB_API string ToString() const;
-	DUCKDB_API void Print();
+	DUCKDB_API void Print() const;
 
 	DataChunk(const DataChunk &) = delete;