duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/storage/compression/bitpacking_hugeint.cpp

@@ -0,0 +1,295 @@
+#include "duckdb/common/bitpacking.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Unpacking
+//===--------------------------------------------------------------------===//
+
+static void UnpackSingle(const uint32_t *__restrict &in, hugeint_t *__restrict out, uint16_t delta, uint16_t shr) {
+	if (delta + shr < 32) {
+		*out = ((static_cast<hugeint_t>(in[0])) >> shr) % (hugeint_t(1) << delta);
+	}
+
+	else if (delta + shr >= 32 && delta + shr < 64) {
+		*out = static_cast<hugeint_t>(in[0]) >> shr;
+		++in;
+
+		if (delta + shr > 32) {
+			const uint16_t NEXT_SHR = shr + delta - 32;
+			*out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (32 - shr);
+		}
+	}
+
+	else if (delta + shr >= 64 && delta + shr < 96) {
+		*out = static_cast<hugeint_t>(in[0]) >> shr;
+		*out |= static_cast<hugeint_t>(in[1]) << (32 - shr);
+		in += 2;
+
+		if (delta + shr > 64) {
+			const uint16_t NEXT_SHR = delta + shr - 64;
+			*out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (64 - shr);
+		}
+	}
+
+	else if (delta + shr >= 96 && delta + shr < 128) {
+		*out = static_cast<hugeint_t>(in[0]) >> shr;
+		*out |= static_cast<hugeint_t>(in[1]) << (32 - shr);
+		*out |= static_cast<hugeint_t>(in[2]) << (64 - shr);
+		in += 3;
+
+		if (delta + shr > 96) {
+			const uint16_t NEXT_SHR = delta + shr - 96;
+			*out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (96 - shr);
+		}
+	}
+
+	else if (delta + shr >= 128) {
+		*out = static_cast<hugeint_t>(in[0]) >> shr;
+		*out |= static_cast<hugeint_t>(in[1]) << (32 - shr);
+		*out |= static_cast<hugeint_t>(in[2]) << (64 - shr);
+		*out |= static_cast<hugeint_t>(in[3]) << (96 - shr);
+		in += 4;
+
+		if (delta + shr > 128) {
+			const uint16_t NEXT_SHR = delta + shr - 128;
+			*out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (128 - shr);
+		}
+	}
+}
+
+static void UnpackLast(const uint32_t *__restrict &in, hugeint_t *__restrict out, uint16_t delta) {
+	const uint8_t LAST_IDX = 31;
+	const uint16_t SHIFT = (delta * 31) % 32;
+	out[LAST_IDX] = in[0] >> SHIFT;
+	if (delta > 32) {
+		out[LAST_IDX] |= static_cast<hugeint_t>(in[1]) << (32 - SHIFT);
+	}
+	if (delta > 64) {
+		out[LAST_IDX] |= static_cast<hugeint_t>(in[2]) << (64 - SHIFT);
+	}
+	if (delta > 96) {
+		out[LAST_IDX] |= static_cast<hugeint_t>(in[3]) << (96 - SHIFT);
+	}
+}
+
+// Unpacks for specific deltas
+static void UnpackDelta0(const uint32_t *__restrict in, hugeint_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		out[i] = 0;
+	}
+}
+
+static void UnpackDelta32(const uint32_t *__restrict in, hugeint_t *__restrict out) {
+	for (uint8_t k = 0; k < 32; ++k) {
+		out[k] = static_cast<hugeint_t>(in[k]);
+	}
+}
+
+static void UnpackDelta64(const uint32_t *__restrict in, hugeint_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		const uint8_t OFFSET = i * 2;
+		out[i] = in[OFFSET];
+		out[i] |= static_cast<hugeint_t>(in[OFFSET + 1]) << 32;
+	}
+}
+
+static void UnpackDelta96(const uint32_t *__restrict in, hugeint_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		const uint8_t OFFSET = i * 3;
+		out[i] = in[OFFSET];
+		out[i] |= static_cast<hugeint_t>(in[OFFSET + 1]) << 32;
+		out[i] |= static_cast<hugeint_t>(in[OFFSET + 2]) << 64;
+	}
+}
+
+static void UnpackDelta128(const uint32_t *__restrict in, hugeint_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		const uint8_t OFFSET = i * 4;
+		out[i] = in[OFFSET];
+		out[i] |= static_cast<hugeint_t>(in[OFFSET + 1]) << 32;
+		out[i] |= static_cast<hugeint_t>(in[OFFSET + 2]) << 64;
+		out[i] |= static_cast<hugeint_t>(in[OFFSET + 3]) << 96;
+	}
+}
+
+//===--------------------------------------------------------------------===//
+// Packing
+//===--------------------------------------------------------------------===//
+
+static void PackSingle(const hugeint_t in, uint32_t *__restrict &out, uint16_t delta, uint16_t shl, hugeint_t mask) {
+	if (delta + shl < 32) {
+
+		if (shl == 0) {
+			out[0] = static_cast<uint32_t>(in & mask);
+		} else {
+			out[0] |= static_cast<uint32_t>((in & mask) << shl);
+		}
+
+	} else if (delta + shl >= 32 && delta + shl < 64) {
+
+		if (shl == 0) {
+			out[0] = static_cast<uint32_t>(in & mask);
+		} else {
+			out[0] |= static_cast<uint32_t>((in & mask) << shl);
+		}
+		++out;
+
+		if (delta + shl > 32) {
+			*out = static_cast<uint32_t>((in & mask) >> (32 - shl));
+		}
+	}
+
+	else if (delta + shl >= 64 && delta + shl < 96) {
+
+		if (shl == 0) {
+			out[0] = static_cast<uint32_t>(in & mask);
+		} else {
+			out[0] |= static_cast<uint32_t>(in << shl);
+		}
+
+		out[1] = static_cast<uint32_t>((in & mask) >> (32 - shl));
+		out += 2;
+
+		if (delta + shl > 64) {
+			*out = static_cast<uint32_t>((in & mask) >> (64 - shl));
+		}
+	}
+
+	else if (delta + shl >= 96 && delta + shl < 128) {
+		if (shl == 0) {
+			out[0] = static_cast<uint32_t>(in & mask);
+		} else {
+			out[0] |= static_cast<uint32_t>(in << shl);
+		}
+
+		out[1] = static_cast<uint32_t>((in & mask) >> (32 - shl));
+		out[2] = static_cast<uint32_t>((in & mask) >> (64 - shl));
+		out += 3;
+
+		if (delta + shl > 96) {
+			*out = static_cast<uint32_t>((in & mask) >> (96 - shl));
+		}
+	}
+
+	else if (delta + shl >= 128) {
+		// shl == 0 won't ever happen here considering a delta of 128 calls PackDelta128
+		out[0] |= static_cast<uint32_t>(in << shl);
+		out[1] = static_cast<uint32_t>((in & mask) >> (32 - shl));
+		out[2] = static_cast<uint32_t>((in & mask) >> (64 - shl));
+		out[3] = static_cast<uint32_t>((in & mask) >> (96 - shl));
+		out += 4;
+
+		if (delta + shl > 128) {
+			*out = static_cast<uint32_t>((in & mask) >> (128 - shl));
+		}
+	}
+}
+
+static void PackLast(const hugeint_t *__restrict in, uint32_t *__restrict out, uint16_t delta) {
+	const uint8_t LAST_IDX = 31;
+	const uint16_t SHIFT = (delta * 31) % 32;
+	out[0] |= static_cast<uint32_t>(in[LAST_IDX] << SHIFT);
+	if (delta > 32) {
+		out[1] = static_cast<uint32_t>(in[LAST_IDX] >> (32 - SHIFT));
+	}
+	if (delta > 64) {
+		out[2] = static_cast<uint32_t>(in[LAST_IDX] >> (64 - SHIFT));
+	}
+	if (delta > 96) {
+		out[3] = static_cast<uint32_t>(in[LAST_IDX] >> (96 - SHIFT));
+	}
+}
+
+// Packs for specific deltas
+static void PackDelta32(const hugeint_t *__restrict in, uint32_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		out[i] = static_cast<uint32_t>(in[i]);
+	}
+}
+
+static void PackDelta64(const hugeint_t *__restrict in, uint32_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		const uint8_t OFFSET = 2 * i;
+		out[OFFSET] = static_cast<uint32_t>(in[i]);
+		out[OFFSET + 1] = static_cast<uint32_t>(in[i] >> 32);
+	}
+}
+
+static void PackDelta96(const hugeint_t *__restrict in, uint32_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		const uint8_t OFFSET = 3 * i;
+		out[OFFSET] = static_cast<uint32_t>(in[i]);
+		out[OFFSET + 1] = static_cast<uint32_t>(in[i] >> 32);
+		out[OFFSET + 2] = static_cast<uint32_t>(in[i] >> 64);
+	}
+}
+
+static void PackDelta128(const hugeint_t *__restrict in, uint32_t *__restrict out) {
+	for (uint8_t i = 0; i < 32; ++i) {
+		const uint8_t OFFSET = 4 * i;
+		out[OFFSET] = static_cast<uint32_t>(in[i]);
+		out[OFFSET + 1] = static_cast<uint32_t>(in[i] >> 32);
+		out[OFFSET + 2] = static_cast<uint32_t>(in[i] >> 64);
+		out[OFFSET + 3] = static_cast<uint32_t>(in[i] >> 96);
+	}
+}
+
+//===--------------------------------------------------------------------===//
+// HugeIntPacker
+//===--------------------------------------------------------------------===//
+
+void HugeIntPacker::Pack(const hugeint_t *__restrict in, uint32_t *__restrict out, bitpacking_width_t width) {
+	D_ASSERT(width <= 128);
+	switch (width) {
+	case 0:
+		break;
+	case 32:
+		PackDelta32(in, out);
+		break;
+	case 64:
+		PackDelta64(in, out);
+		break;
+	case 96:
+		PackDelta96(in, out);
+		break;
+	case 128:
+		PackDelta128(in, out);
+		break;
+	default:
+		for (idx_t oindex = 0; oindex < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE - 1; ++oindex) {
+			PackSingle(in[oindex], out, width, (width * oindex) % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE,
+			           (hugeint_t(1) << width) - 1);
+		}
+		PackLast(in, out, width);
+	}
+}
+
+void HugeIntPacker::Unpack(const uint32_t *__restrict in, hugeint_t *__restrict out, bitpacking_width_t width) {
+	D_ASSERT(width <= 128);
+	switch (width) {
+	case 0:
+		UnpackDelta0(in, out);
+		break;
+	case 32:
+		UnpackDelta32(in, out);
+		break;
+	case 64:
+		UnpackDelta64(in, out);
+		break;
+	case 96:
+		UnpackDelta96(in, out);
+		break;
+	case 128:
+		UnpackDelta128(in, out);
+		break;
+	default:
+		for (idx_t oindex = 0; oindex < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE - 1; ++oindex) {
+			UnpackSingle(in, out + oindex, width,
+			             (width * oindex) % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE);
+		}
+		UnpackLast(in, out, width);
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -65,7 +65,7 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 	auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
 	if (type.InternalType() == PhysicalType::VARCHAR) {
 		auto &state = compressed_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
-		state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().GetBlockManager());
+		state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(checkpointer.GetRowGroup().GetBlockManager());
 	}
 	current_segment = std::move(compressed_segment);
 	current_segment->InitializeAppend(append_state);
@@ -73,6 +73,11 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 
 void UncompressedCompressState::FlushSegment(idx_t segment_size) {
 	auto &state = checkpointer.GetCheckpointState();
+	if (current_segment->type.InternalType() == PhysicalType::VARCHAR) {
+		auto &segment_state = current_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
+		segment_state.overflow_writer->Flush();
+		segment_state.overflow_writer.reset();
+	}
 	state.FlushSegment(std::move(current_segment), segment_size);
 }
 

package/src/duckdb/src/storage/compression/fsst.cpp

@@ -461,7 +461,7 @@ void FSSTStorage::Compress(CompressionState &state_p, Vector &scan_vector, idx_t
 	    &sizes_in[0],         /* IN: byte-lengths of the inputs */
 	    &strings_in[0],       /* IN: input string start pointers. */
 	    compress_buffer_size, /* IN: byte-length of output buffer. */
-	    &compress_buffer[0],  /* OUT: memorxy buffer to put the compressed strings in (one after the other). */
+	    &compress_buffer[0],  /* OUT: memory buffer to put the compressed strings in (one after the other). */
 	    &sizes_out[0],        /* OUT: byte-lengths of the compressed strings. */
 	    &strings_out[0]       /* OUT: output string start pointers. Will all point into [output,output+size). */
 	);

package/src/duckdb/src/storage/compression/rle.cpp

@@ -166,8 +166,8 @@ struct RLECompressState : public CompressionState {
 	void WriteValue(T value, rle_count_t count, bool is_null) {
 		// write the RLE entry
 		auto handle_ptr = handle.Ptr() + RLEConstants::RLE_HEADER_SIZE;
-		auto data_pointer = (T *)handle_ptr;
-		auto index_pointer = (rle_count_t *)(handle_ptr + max_rle_count * sizeof(T));
+		auto data_pointer = reinterpret_cast<T *>(handle_ptr);
+		auto index_pointer = reinterpret_cast<rle_count_t *>(handle_ptr + max_rle_count * sizeof(T));
 		data_pointer[entry_count] = value;
 		index_pointer[entry_count] = count;
 		entry_count++;
@@ -257,7 +257,7 @@ struct RLEScanState : public SegmentScanState {
 
 	void Skip(ColumnSegment &segment, idx_t skip_count) {
 		auto data = handle.Ptr() + segment.GetBlockOffset();
-		auto index_pointer = (rle_count_t *)(data + rle_count_offset);
+		auto index_pointer = reinterpret_cast<rle_count_t *>(data + rle_count_offset);
 
 		for (idx_t i = 0; i < skip_count; i++) {
 			// assign the current value
@@ -272,7 +272,6 @@ struct RLEScanState : public SegmentScanState {
 	}
 
 	BufferHandle handle;
-	uint32_t rle_offset;
 	idx_t entry_pos;
 	idx_t position_in_entry;
 	uint32_t rle_count_offset;
@@ -293,14 +292,63 @@ void RLESkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
 	scan_state.Skip(segment, skip_count);
 }
 
+template <bool ENTIRE_VECTOR>
+static bool CanEmitConstantVector(idx_t position, idx_t run_length, idx_t scan_count) {
+	if (!ENTIRE_VECTOR) {
+		return false;
+	}
+	if (scan_count != STANDARD_VECTOR_SIZE) {
+		// Only when we can fill an entire Vector can we emit a ConstantVector, because subsequent scans require the
+		// input Vector to be flat
+		return false;
+	}
+	D_ASSERT(position < run_length);
+	auto remaining_in_run = run_length - position;
+	// The amount of values left in this run are equal or greater than the amount of values we need to scan
+	return remaining_in_run >= scan_count;
+}
+
 template <class T>
-void RLEScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
-                    idx_t result_offset) {
+inline static void ForwardToNextRun(RLEScanState<T> &scan_state) {
+	// handled all entries in this RLE value
+	// move to the next entry
+	scan_state.entry_pos++;
+	scan_state.position_in_entry = 0;
+}
+
+template <class T>
+inline static bool ExhaustedRun(RLEScanState<T> &scan_state, rle_count_t *index_pointer) {
+	return scan_state.position_in_entry >= index_pointer[scan_state.entry_pos];
+}
+
+template <class T>
+static void RLEScanConstant(RLEScanState<T> &scan_state, rle_count_t *index_pointer, T *data_pointer, idx_t scan_count,
+                            Vector &result) {
+	result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	auto result_data = ConstantVector::GetData<T>(result);
+	result_data[0] = data_pointer[scan_state.entry_pos];
+	scan_state.position_in_entry += scan_count;
+	if (ExhaustedRun(scan_state, index_pointer)) {
+		ForwardToNextRun(scan_state);
+	}
+	return;
+}
+
+template <class T, bool ENTIRE_VECTOR>
+void RLEScanPartialInternal(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
+                            idx_t result_offset) {
 	auto &scan_state = state.scan_state->Cast<RLEScanState<T>>();
 
 	auto data = scan_state.handle.Ptr() + segment.GetBlockOffset();
-	auto data_pointer = (T *)(data + RLEConstants::RLE_HEADER_SIZE);
-	auto index_pointer = (rle_count_t *)(data + scan_state.rle_count_offset);
+	auto data_pointer = reinterpret_cast<T *>(data + RLEConstants::RLE_HEADER_SIZE);
+	auto index_pointer = reinterpret_cast<rle_count_t *>(data + scan_state.rle_count_offset);
+
+	// If we are scanning an entire Vector and it contains only a single run
+	if (CanEmitConstantVector<ENTIRE_VECTOR>(scan_state.position_in_entry, index_pointer[scan_state.entry_pos],
+	                                         scan_count)) {
+		RLEScanConstant<T>(scan_state, index_pointer, data_pointer, scan_count, result);
+		return;
+	}
 
 	auto result_data = FlatVector::GetData<T>(result);
 	result.SetVectorType(VectorType::FLAT_VECTOR);
@@ -308,19 +356,21 @@ void RLEScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_c
 		// assign the current value
 		result_data[result_offset + i] = data_pointer[scan_state.entry_pos];
 		scan_state.position_in_entry++;
-		if (scan_state.position_in_entry >= index_pointer[scan_state.entry_pos]) {
-			// handled all entries in this RLE value
-			// move to the next entry
-			scan_state.entry_pos++;
-			scan_state.position_in_entry = 0;
+		if (ExhaustedRun(scan_state, index_pointer)) {
+			ForwardToNextRun(scan_state);
 		}
 	}
 }
 
+template <class T>
+void RLEScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
+                    idx_t result_offset) {
+	return RLEScanPartialInternal<T, false>(segment, state, scan_count, result, result_offset);
+}
+
 template <class T>
 void RLEScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
-	// FIXME: emit constant vector if repetition of single value is >= scan_count
-	RLEScanPartial<T>(segment, state, scan_count, result, 0);
+	RLEScanPartialInternal<T, true>(segment, state, scan_count, result, 0);
 }
 
 //===--------------------------------------------------------------------===//
@@ -332,7 +382,7 @@ void RLEFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id,
 	scan_state.Skip(segment, row_id);
 
 	auto data = scan_state.handle.Ptr() + segment.GetBlockOffset();
-	auto data_pointer = (T *)(data + RLEConstants::RLE_HEADER_SIZE);
+	auto data_pointer = reinterpret_cast<T *>(data + RLEConstants::RLE_HEADER_SIZE);
 	auto result_data = FlatVector::GetData<T>(result);
 	result_data[result_idx] = data_pointer[scan_state.entry_pos];
 }

package/src/duckdb/src/storage/compression/string_uncompressed.cpp

@@ -2,7 +2,9 @@
 
 #include "duckdb/common/pair.hpp"
 #include "duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp"
-#include "miniz_wrapper.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 
 namespace duckdb {
 
@@ -141,9 +143,22 @@ void UncompressedStringStorage::StringFetchRow(ColumnSegment &segment, ColumnFet
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
+struct SerializedStringSegmentState : public ColumnSegmentState {
+	SerializedStringSegmentState() {
+	}
+	explicit SerializedStringSegmentState(vector<block_id_t> blocks_p) : blocks(std::move(blocks_p)) {
+	}
+
+	vector<block_id_t> blocks;
+
+	void Serialize(Serializer &serializer) const override {
+		serializer.WriteProperty(1, "overflow_blocks", blocks);
+	}
+};
 
-unique_ptr<CompressedSegmentState> UncompressedStringStorage::StringInitSegment(ColumnSegment &segment,
-                                                                                block_id_t block_id) {
+unique_ptr<CompressedSegmentState>
+UncompressedStringStorage::StringInitSegment(ColumnSegment &segment, block_id_t block_id,
+                                             optional_ptr<ColumnSegmentState> segment_state) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	if (block_id == INVALID_BLOCK) {
 		auto handle = buffer_manager.Pin(segment.block);
@@ -152,7 +167,12 @@ unique_ptr<CompressedSegmentState> UncompressedStringStorage::StringInitSegment(
 		dictionary.end = segment.SegmentSize();
 		SetDictionary(segment, handle, dictionary);
 	}
-	return make_uniq<UncompressedStringSegmentState>();
+	auto result = make_uniq<UncompressedStringSegmentState>();
+	if (segment_state) {
+		auto &serialized_state = segment_state->Cast<SerializedStringSegmentState>();
+		result->on_disk_blocks = std::move(serialized_state.blocks);
+	}
+	return std::move(result);
 }
 
 idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentStatistics &stats) {
@@ -179,6 +199,32 @@ idx_t UncompressedStringStorage::FinalizeAppend(ColumnSegment &segment, SegmentS
 	return total_size;
 }
 
+//===--------------------------------------------------------------------===//
+// Serialization & Cleanup
+//===--------------------------------------------------------------------===//
+unique_ptr<ColumnSegmentState> UncompressedStringStorage::SerializeState(ColumnSegment &segment) {
+	auto &state = segment.GetSegmentState()->Cast<UncompressedStringSegmentState>();
+	if (state.on_disk_blocks.empty()) {
+		// no on-disk blocks - nothing to write
+		return nullptr;
+	}
+	return make_uniq<SerializedStringSegmentState>(state.on_disk_blocks);
+}
+
+unique_ptr<ColumnSegmentState> UncompressedStringStorage::DeserializeState(Deserializer &deserializer) {
+	auto result = make_uniq<SerializedStringSegmentState>();
+	deserializer.ReadProperty(1, "overflow_blocks", result->blocks);
+	return std::move(result);
+}
+
+void UncompressedStringStorage::CleanupState(ColumnSegment &segment) {
+	auto &state = segment.GetSegmentState()->Cast<UncompressedStringSegmentState>();
+	auto &block_manager = segment.GetBlockManager();
+	for (auto &block_id : state.on_disk_blocks) {
+		block_manager.MarkBlockAsModified(block_id);
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Get Function
 //===--------------------------------------------------------------------===//
@@ -192,7 +238,9 @@ CompressionFunction StringUncompressed::GetFunction(PhysicalType data_type) {
 	                           UncompressedStringStorage::StringScanPartial, UncompressedStringStorage::StringFetchRow,
 	                           UncompressedFunctions::EmptySkip, UncompressedStringStorage::StringInitSegment,
 	                           UncompressedStringStorage::StringInitAppend, UncompressedStringStorage::StringAppend,
-	                           UncompressedStringStorage::FinalizeAppend);
+	                           UncompressedStringStorage::FinalizeAppend, nullptr,
+	                           UncompressedStringStorage::SerializeState, UncompressedStringStorage::DeserializeState,
+	                           UncompressedStringStorage::CleanupState);
 }
 
 //===--------------------------------------------------------------------===//
@@ -226,7 +274,7 @@ void UncompressedStringStorage::WriteString(ColumnSegment &segment, string_t str
 	auto &state = segment.GetSegmentState()->Cast<UncompressedStringSegmentState>();
 	if (state.overflow_writer) {
 		// overflow writer is set: write string there
-		state.overflow_writer->WriteString(string, result_block, result_offset);
+		state.overflow_writer->WriteString(state, string, result_block, result_offset);
 	} else {
 		// default overflow behavior: use in-memory buffer to store the overflow string
 		WriteStringMemory(segment, string, result_block, result_offset);
@@ -251,7 +299,7 @@ void UncompressedStringStorage::WriteStringMemory(ColumnSegment &segment, string
 		new_block->size = alloc_size;
 		// allocate an in-memory buffer for it
 		handle = buffer_manager.Allocate(alloc_size, false, &block);
-		state.overflow_blocks[block->BlockId()] = new_block.get();
+		state.overflow_blocks.insert(make_pair(block->BlockId(), reference<StringBlock>(*new_block)));
 		new_block->block = std::move(block);
 		new_block->next = std::move(state.head);
 		state.head = std::move(new_block);
@@ -282,61 +330,46 @@ string_t UncompressedStringStorage::ReadOverflowString(ColumnSegment &segment, V
 	if (block < MAXIMUM_BLOCK) {
 		// read the overflow string from disk
 		// pin the initial handle and read the length
-		auto block_handle = block_manager.RegisterBlock(block);
+		auto block_handle = state.GetHandle(block_manager, block);
 		auto handle = buffer_manager.Pin(block_handle);
 
 		// read header
-		uint32_t compressed_size = Load<uint32_t>(handle.Ptr() + offset);
-		uint32_t uncompressed_size = Load<uint32_t>(handle.Ptr() + offset + sizeof(uint32_t));
-		uint32_t remaining = compressed_size;
-		offset += 2 * sizeof(uint32_t);
-
-		data_ptr_t decompression_ptr;
-		unsafe_unique_array<data_t> decompression_buffer;
-
-		// If string is in single block we decompress straight from it, else we copy first
-		if (remaining <= Storage::BLOCK_SIZE - sizeof(block_id_t) - offset) {
-			decompression_ptr = handle.Ptr() + offset;
-		} else {
-			decompression_buffer = make_unsafe_uniq_array<data_t>(compressed_size);
-			auto target_ptr = decompression_buffer.get();
-
-			// now append the string to the single buffer
-			while (remaining > 0) {
-				idx_t to_write = MinValue<idx_t>(remaining, Storage::BLOCK_SIZE - sizeof(block_id_t) - offset);
-				memcpy(target_ptr, handle.Ptr() + offset, to_write);
-
-				remaining -= to_write;
-				offset += to_write;
-				target_ptr += to_write;
-				if (remaining > 0) {
-					// read the next block
-					block_id_t next_block = Load<block_id_t>(handle.Ptr() + offset);
-					block_handle = block_manager.RegisterBlock(next_block);
-					handle = buffer_manager.Pin(block_handle);
-					offset = 0;
-				}
+		uint32_t length = Load<uint32_t>(handle.Ptr() + offset);
+		uint32_t remaining = length;
+		offset += sizeof(uint32_t);
+
+		// allocate a buffer to store the string
+		auto alloc_size = MaxValue<idx_t>(Storage::BLOCK_SIZE, length);
+		// allocate a buffer to store the compressed string
+		// TODO: profile this to check if we need to reuse buffer
+		auto target_handle = buffer_manager.Allocate(alloc_size);
+		auto target_ptr = target_handle.Ptr();
+
+		// now append the string to the single buffer
+		while (remaining > 0) {
+			idx_t to_write = MinValue<idx_t>(remaining, Storage::BLOCK_SIZE - sizeof(block_id_t) - offset);
+			memcpy(target_ptr, handle.Ptr() + offset, to_write);
+			remaining -= to_write;
+			offset += to_write;
+			target_ptr += to_write;
+			if (remaining > 0) {
+				// read the next block
+				block_id_t next_block = Load<block_id_t>(handle.Ptr() + offset);
+				block_handle = state.GetHandle(block_manager, next_block);
+				handle = buffer_manager.Pin(block_handle);
+				offset = 0;
 			}
-			decompression_ptr = decompression_buffer.get();
 		}
 
-		// overflow strings on disk are gzipped, decompress here
-		auto decompressed_target_handle =
-		    buffer_manager.Allocate(MaxValue<idx_t>(Storage::BLOCK_SIZE, uncompressed_size));
-		auto decompressed_target_ptr = decompressed_target_handle.Ptr();
-		MiniZStream s;
-		s.Decompress(const_char_ptr_cast(decompression_ptr), compressed_size, char_ptr_cast(decompressed_target_ptr),
-		             uncompressed_size);
-
-		auto final_buffer = decompressed_target_handle.Ptr();
-		StringVector::AddHandle(result, std::move(decompressed_target_handle));
-		return ReadString(final_buffer, 0, uncompressed_size);
+		auto final_buffer = target_handle.Ptr();
+		StringVector::AddHandle(result, std::move(target_handle));
+		return ReadString(final_buffer, 0, length);
 	} else {
 		// read the overflow string from memory
 		// first pin the handle, if it is not pinned yet
 		auto entry = state.overflow_blocks.find(block);
 		D_ASSERT(entry != state.overflow_blocks.end());
-		auto handle = buffer_manager.Pin(entry->second->block);
+		auto handle = buffer_manager.Pin(entry->second.get().block);
 		auto final_buffer = handle.Ptr();
 		StringVector::AddHandle(result, std::move(handle));
 		return ReadStringWithLength(final_buffer, offset);