duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp

@@ -7,11 +7,12 @@
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/metadata/metadata_reader.hpp"
 
 namespace duckdb {
 
-TableDataWriter::TableDataWriter(TableCatalogEntry &table_p, optional_ptr<ClientContext> client_context_p)
-    : table(table_p.Cast<DuckTableEntry>()), client_context(client_context_p) {
+TableDataWriter::TableDataWriter(TableCatalogEntry &table_p, QueryContext context)
+    : table(table_p.Cast<DuckTableEntry>()), context(context.GetClientContext()) {
 	D_ASSERT(table_p.IsDuckTable());
 }
 
@@ -23,10 +24,6 @@ void TableDataWriter::WriteTableData(Serializer &metadata_serializer) {
 	table.GetStorage().Checkpoint(*this, metadata_serializer);
 }
 
-CompressionType TableDataWriter::GetColumnCompressionType(idx_t i) {
-	return table.GetColumn(LogicalIndex(i)).CompressionType();
-}
-
 void TableDataWriter::AddRowGroup(RowGroupPointer &&row_group_pointer, unique_ptr<RowGroupWriter> writer) {
 	row_group_pointers.push_back(std::move(row_group_pointer));
 }
@@ -36,8 +33,8 @@ DatabaseInstance &TableDataWriter::GetDatabase() {
 }
 
 unique_ptr<TaskExecutor> TableDataWriter::CreateTaskExecutor() {
-	if (client_context) {
-		return make_uniq<TaskExecutor>(*client_context);
+	if (context) {
+		return make_uniq<TaskExecutor>(*context);
 	}
 	return make_uniq<TaskExecutor>(TaskScheduler::GetScheduler(GetDatabase()));
 }
@@ -57,33 +54,57 @@ CheckpointType SingleFileTableDataWriter::GetCheckpointType() const {
 	return checkpoint_manager.GetCheckpointType();
 }
 
-void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo *info,
-                                              Serializer &serializer) {
-
-	// store the current position in the metadata writer
-	// this is where the row groups for this table start
-	auto pointer = table_data_writer.GetMetaBlockPointer();
-
-	// Serialize statistics as a single unit
-	BinarySerializer stats_serializer(table_data_writer, serializer.GetOptions());
-	stats_serializer.Begin();
-	global_stats.Serialize(stats_serializer);
-	stats_serializer.End();
-
-	// now start writing the row group pointers to disk
-	table_data_writer.Write<uint64_t>(row_group_pointers.size());
-	idx_t total_rows = 0;
-	for (auto &row_group_pointer : row_group_pointers) {
-		auto row_group_count = row_group_pointer.row_start + row_group_pointer.tuple_count;
-		if (row_group_count > total_rows) {
-			total_rows = row_group_count;
-		}
+MetadataManager &SingleFileTableDataWriter::GetMetadataManager() {
+	return checkpoint_manager.GetMetadataManager();
+}
 
-		// Each RowGroup is its own unit
-		BinarySerializer row_group_serializer(table_data_writer, serializer.GetOptions());
-		row_group_serializer.Begin();
-		RowGroup::Serialize(row_group_pointer, row_group_serializer);
-		row_group_serializer.End();
+void SingleFileTableDataWriter::WriteUnchangedTable(MetaBlockPointer pointer, idx_t total_rows) {
+	existing_pointer = pointer;
+	existing_rows = total_rows;
+}
+
+void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stats, DataTableInfo &info,
+                                              RowGroupCollection &collection, Serializer &serializer) {
+	MetaBlockPointer pointer;
+	idx_t total_rows;
+	if (!existing_pointer.IsValid()) {
+		// write the metadata
+		// store the current position in the metadata writer
+		// this is where the row groups for this table start
+		pointer = table_data_writer.GetMetaBlockPointer();
+
+		// Serialize statistics as a single unit
+		BinarySerializer stats_serializer(table_data_writer, serializer.GetOptions());
+		stats_serializer.Begin();
+		global_stats.Serialize(stats_serializer);
+		stats_serializer.End();
+
+		// now start writing the row group pointers to disk
+		table_data_writer.Write<uint64_t>(row_group_pointers.size());
+		total_rows = 0;
+		for (auto &row_group_pointer : row_group_pointers) {
+			auto row_group_count = row_group_pointer.row_start + row_group_pointer.tuple_count;
+			if (row_group_count > total_rows) {
+				total_rows = row_group_count;
+			}
+
+			// Each RowGroup is its own unit
+			BinarySerializer row_group_serializer(table_data_writer, serializer.GetOptions());
+			row_group_serializer.Begin();
+			RowGroup::Serialize(row_group_pointer, row_group_serializer);
+			row_group_serializer.End();
+		}
+		collection.FinalizeCheckpoint(pointer);
+	} else {
+		// we have existing metadata and the table is unchanged - write a pointer to the existing metadata
+		pointer = existing_pointer;
+		total_rows = existing_rows.GetIndex();
+
+		// label the blocks as used again to prevent them from being freed
+		auto &metadata_manager = checkpoint_manager.GetMetadataManager();
+		MetadataReader reader(metadata_manager, pointer);
+		auto blocks = reader.GetRemainingBlocks();
+		metadata_manager.ClearModifiedBlocks(blocks);
 	}
 
 	// Now begin the metadata as a unit
@@ -96,7 +117,7 @@ void SingleFileTableDataWriter::FinalizeTable(const TableStatistics &global_stat
 	if (!v1_0_0_storage) {
 		options.emplace("v1_0_0_storage", v1_0_0_storage);
 	}
-	auto index_storage_infos = info->GetIndexes().GetStorageInfos(options);
+	auto index_storage_infos = info.GetIndexes().SerializeToDisk(context, options);
 
 #ifdef DUCKDB_BLOCK_VERIFICATION
 	for (auto &entry : index_storage_infos) {

package/src/duckdb/src/storage/checkpoint/write_overflow_strings_to_disk.cpp

@@ -109,7 +109,7 @@ void WriteOverflowStringsToDisk::Flush() {
 		}
 		// write to disk
 		auto &block_manager = partial_block_manager.GetBlockManager();
-		block_manager.Write(handle.GetFileBuffer(), block_id);
+		block_manager.Write(QueryContext(), handle.GetFileBuffer(), block_id);
 	}
 	block_id = INVALID_BLOCK;
 	offset = 0;

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -31,16 +31,18 @@
 #include "duckdb/transaction/meta_transaction.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
 #include "duckdb/catalog/dependency_manager.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
+#include "duckdb/main/settings.hpp"
 
 namespace duckdb {
 
 void ReorderTableEntries(catalog_entry_vector_t &tables);
 
-SingleFileCheckpointWriter::SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context_p,
-                                                       AttachedDatabase &db, BlockManager &block_manager,
-                                                       CheckpointType checkpoint_type)
-    : CheckpointWriter(db), client_context(client_context_p),
-      partial_block_manager(block_manager, PartialBlockType::FULL_CHECKPOINT), checkpoint_type(checkpoint_type) {
+SingleFileCheckpointWriter::SingleFileCheckpointWriter(QueryContext context, AttachedDatabase &db,
+                                                       BlockManager &block_manager, CheckpointType checkpoint_type)
+    : CheckpointWriter(db), context(context.GetClientContext()),
+      partial_block_manager(context, block_manager, PartialBlockType::FULL_CHECKPOINT),
+      checkpoint_type(checkpoint_type) {
 }
 
 BlockManager &SingleFileCheckpointWriter::GetBlockManager() {
@@ -129,7 +131,6 @@ static catalog_entry_vector_t GetCatalogEntries(vector<reference<SchemaCatalogEn
 }
 
 void SingleFileCheckpointWriter::CreateCheckpoint() {
-	auto &config = DBConfig::Get(db);
 	auto &storage_manager = db.GetStorageManager().Cast<SingleFileStorageManager>();
 	if (storage_manager.InMemory()) {
 		return;
@@ -201,8 +202,8 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 		wal->WriteCheckpoint(meta_block);
 		wal->Flush();
 	}
-
-	if (config.options.checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_HEADER) {
+	auto debug_checkpoint_abort = DBConfig::GetSetting<DebugCheckpointAbortSetting>(db.GetDatabase());
+	if (debug_checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_HEADER) {
 		throw FatalException("Checkpoint aborted before header write because of PRAGMA checkpoint_abort flag");
 	}
 
@@ -211,7 +212,7 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	header.meta_block = meta_block.block_pointer;
 	header.block_alloc_size = block_manager.GetBlockAllocSize();
 	header.vector_size = STANDARD_VECTOR_SIZE;
-	block_manager.WriteHeader(header);
+	block_manager.WriteHeader(context, header);
 
 #ifdef DUCKDB_BLOCK_VERIFICATION
 	// extend verify_block_usage_count
@@ -241,7 +242,7 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	block_manager.VerifyBlocks(verify_block_usage_count);
 #endif
 
-	if (config.options.checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_TRUNCATE) {
+	if (debug_checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_TRUNCATE) {
 		throw FatalException("Checkpoint aborted before truncate because of PRAGMA checkpoint_abort flag");
 	}
 
@@ -594,7 +595,7 @@ void CheckpointReader::ReadTableData(CatalogTransaction transaction, Deserialize
 	auto &reader = dynamic_cast<MetadataReader &>(binary_deserializer.GetStream());
 
 	MetadataReader table_data_reader(reader.GetMetadataManager(), table_pointer);
-	TableDataReader data_reader(table_data_reader, bound_info);
+	TableDataReader data_reader(table_data_reader, bound_info, table_pointer);
 	data_reader.ReadTableData();
 
 	bound_info.data->total_rows = total_rows;

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -257,7 +257,10 @@ public:
 			    BitpackingPrimitives::MinimumBitWidth<T, false>(static_cast<T>(min_max_delta_diff));
 			auto regular_required_bitwidth = BitpackingPrimitives::MinimumBitWidth(min_max_diff);
 
-			if (delta_required_bitwidth < regular_required_bitwidth && mode != BitpackingMode::FOR) {
+			//! `min_max_diff` is uninitialized if `can_do_for` isn't true
+			bool prefer_for = can_do_for && delta_required_bitwidth >= regular_required_bitwidth;
+
+			if (!prefer_for && mode != BitpackingMode::FOR) {
 				SubtractFrameOfReference(delta_buffer, minimum_delta);
 
 				OP::WriteDeltaFor(reinterpret_cast<T *>(delta_buffer), compression_buffer_validity,

package/src/duckdb/src/storage/compression/dict_fsst/compression.cpp

@@ -305,12 +305,14 @@ static inline bool RequiresHigherBitWidth(bitpacking_width_t bitwidth, uint32_t
 }
 
 template <DictionaryAppendState APPEND_STATE>
-static inline bool AddLookup(DictFSSTCompressionState &state, idx_t lookup, const bool recalculate_indices_space) {
+static inline bool AddLookup(DictFSSTCompressionState &state, idx_t lookup, const bool recalculate_indices_space,
+                             bool fail_on_no_space) {
 	D_ASSERT(lookup != DConstants::INVALID_INDEX);
 
 	//! This string exists in the dictionary
 	idx_t new_dictionary_indices_space = state.dictionary_indices_space;
-	if (APPEND_STATE != DictionaryAppendState::ENCODED_ALL_UNIQUE && recalculate_indices_space) {
+	auto get_bitpacking_size = APPEND_STATE != DictionaryAppendState::ENCODED_ALL_UNIQUE && recalculate_indices_space;
+	if (get_bitpacking_size) {
 		new_dictionary_indices_space =
 		    BitpackingPrimitives::GetRequiredSize(state.tuple_count + 1, state.dictionary_indices_width);
 	}
@@ -336,6 +338,12 @@ static inline bool AddLookup(DictFSSTCompressionState &state, idx_t lookup, cons
 		available_space -= FSST_SYMBOL_TABLE_SIZE;
 	}
 	if (required_space > available_space) {
+		if (fail_on_no_space) {
+			throw FatalException("AddLookup in DictFSST failed: required: %d, available: %d, indices: %d, bitpacking: "
+			                     "%b, dict offset: %d, str length: %d",
+			                     required_space, available_space, new_dictionary_indices_space, get_bitpacking_size,
+			                     state.dictionary_offset, state.string_lengths_space);
+		}
 		return false;
 	}
 
@@ -349,7 +357,7 @@ static inline bool AddLookup(DictFSSTCompressionState &state, idx_t lookup, cons
 
 template <DictionaryAppendState APPEND_STATE>
 static inline bool AddToDictionary(DictFSSTCompressionState &state, const string_t &str,
-                                   const bool recalculate_indices_space) {
+                                   const bool recalculate_indices_space, bool fail_on_no_space) {
 	uint32_t str_len = UnsafeNumericCast<uint32_t>(str.GetSize());
 	if (APPEND_STATE == DictionaryAppendState::ENCODED) {
 		//! We delay encoding of new entries.
@@ -413,6 +421,12 @@ static inline bool AddToDictionary(DictFSSTCompressionState &state, const string
 		available_space -= FSST_SYMBOL_TABLE_SIZE;
 	}
 	if (required_space > available_space) {
+		if (fail_on_no_space) {
+			throw FatalException("AddToDictionary in DictFSST failed: required: %d, available: %d, dict offset + "
+			                     "str_len: %d, new str length: %d, new dict indices: %d",
+			                     required_space, available_space, state.dictionary_offset + str_len,
+			                     new_string_lengths_space, new_dictionary_indices_space);
+		}
 		return false;
 	}
 
@@ -422,7 +436,7 @@ static inline bool AddToDictionary(DictFSSTCompressionState &state, const string
 		if (str.IsInlined()) {
 			state.dictionary_encoding_buffer.push_back(str);
 		} else {
-			state.dictionary_encoding_buffer.push_back(state.uncompressed_dictionary_copy.AddString(str));
+			state.dictionary_encoding_buffer.push_back(state.uncompressed_dictionary_copy.AddBlob(str));
 		}
 		if (!state.to_encode_string_sum) {
 			//! As specified in fsst.h
@@ -461,7 +475,8 @@ static inline bool AddToDictionary(DictFSSTCompressionState &state, const string
 }
 
 bool DictFSSTCompressionState::CompressInternal(UnifiedVectorFormat &vector_format, const string_t &str, bool is_null,
-                                                EncodedInput &encoded_input, const idx_t i, idx_t count) {
+                                                EncodedInput &encoded_input, const idx_t i, idx_t count,
+                                                bool fail_on_no_space) {
 	auto strings = UnifiedVectorFormat::GetData<string_t>(vector_format);
 	idx_t lookup = DConstants::INVALID_INDEX;
 
@@ -484,17 +499,21 @@ bool DictFSSTCompressionState::CompressInternal(UnifiedVectorFormat &vector_form
 	case DictionaryAppendState::REGULAR: {
 		if (append_state == DictionaryAppendState::REGULAR) {
 			if (lookup != DConstants::INVALID_INDEX) {
-				return AddLookup<DictionaryAppendState::REGULAR>(*this, lookup, recalculate_indices_space);
+				return AddLookup<DictionaryAppendState::REGULAR>(*this, lookup, recalculate_indices_space,
+				                                                 fail_on_no_space);
 			} else {
 				//! This string does not exist in the dictionary, add it
-				return AddToDictionary<DictionaryAppendState::REGULAR>(*this, str, recalculate_indices_space);
+				return AddToDictionary<DictionaryAppendState::REGULAR>(*this, str, recalculate_indices_space,
+				                                                       fail_on_no_space);
 			}
 		} else {
 			if (lookup != DConstants::INVALID_INDEX) {
-				return AddLookup<DictionaryAppendState::NOT_ENCODED>(*this, lookup, recalculate_indices_space);
+				return AddLookup<DictionaryAppendState::NOT_ENCODED>(*this, lookup, recalculate_indices_space,
+				                                                     fail_on_no_space);
 			} else {
 				//! This string does not exist in the dictionary, add it
-				return AddToDictionary<DictionaryAppendState::NOT_ENCODED>(*this, str, recalculate_indices_space);
+				return AddToDictionary<DictionaryAppendState::NOT_ENCODED>(*this, str, recalculate_indices_space,
+				                                                           fail_on_no_space);
 			}
 		}
 	}
@@ -505,10 +524,12 @@ bool DictFSSTCompressionState::CompressInternal(UnifiedVectorFormat &vector_form
 
 		bool fits;
 		if (lookup != DConstants::INVALID_INDEX) {
-			fits = AddLookup<DictionaryAppendState::ENCODED>(*this, lookup, recalculate_indices_space);
+			fits =
+			    AddLookup<DictionaryAppendState::ENCODED>(*this, lookup, recalculate_indices_space, fail_on_no_space);
 		} else {
 			//! Not in the dictionary, add it
-			fits = AddToDictionary<DictionaryAppendState::ENCODED>(*this, str, recalculate_indices_space);
+			fits = AddToDictionary<DictionaryAppendState::ENCODED>(*this, str, recalculate_indices_space,
+			                                                       fail_on_no_space);
 		}
 		if (fits) {
 			return fits;
@@ -523,10 +544,12 @@ bool DictFSSTCompressionState::CompressInternal(UnifiedVectorFormat &vector_form
 		// we flush these and try again to see if the size went down enough
 		FlushEncodingBuffer();
 		if (lookup != DConstants::INVALID_INDEX) {
-			return AddLookup<DictionaryAppendState::ENCODED>(*this, lookup, recalculate_indices_space);
+			return AddLookup<DictionaryAppendState::ENCODED>(*this, lookup, recalculate_indices_space,
+			                                                 fail_on_no_space);
 		} else {
 			//! Not in the dictionary, add it
-			return AddToDictionary<DictionaryAppendState::ENCODED>(*this, str, recalculate_indices_space);
+			return AddToDictionary<DictionaryAppendState::ENCODED>(*this, str, recalculate_indices_space,
+			                                                       fail_on_no_space);
 		}
 	}
 	case DictionaryAppendState::ENCODED_ALL_UNIQUE: {
@@ -535,8 +558,7 @@ bool DictFSSTCompressionState::CompressInternal(UnifiedVectorFormat &vector_form
 
 #ifdef DEBUG
 		auto temp_decoder = alloca(sizeof(duckdb_fsst_decoder_t));
-		duckdb_fsst_import((duckdb_fsst_decoder_t *)temp_decoder, fsst_serialized_symbol_table.get());
-
+		duckdb_fsst_import(reinterpret_cast<duckdb_fsst_decoder_t *>(temp_decoder), fsst_serialized_symbol_table.get());
 		vector<unsigned char> decompress_buffer;
 #endif
 
@@ -589,12 +611,12 @@ bool DictFSSTCompressionState::CompressInternal(UnifiedVectorFormat &vector_form
 				//! Verify that we can decompress the string
 				auto &uncompressed_str = strings[encoded_input.offset + j];
 				decompress_buffer.resize(uncompressed_str.GetSize() + 1 + 100);
-				auto decoded_std_string =
-				    FSSTPrimitives::DecompressValue((void *)temp_decoder, (const char *)compressed_ptrs[j],
-				                                    (idx_t)compressed_sizes[j], decompress_buffer);
+				auto decoded_std_string = FSSTPrimitives::DecompressValue(
+				    (void *)temp_decoder, reinterpret_cast<const char *>(compressed_ptrs[j]),
+				    (idx_t)compressed_sizes[j], decompress_buffer);
 
 				D_ASSERT(decoded_std_string.size() == uncompressed_str.GetSize());
-				string_t decompressed_string((const char *)decompress_buffer.data(),
+				string_t decompressed_string(reinterpret_cast<const char *>(decompress_buffer.data()),
 				                             UnsafeNumericCast<uint32_t>(uncompressed_str.GetSize()));
 				D_ASSERT(decompressed_string == uncompressed_str);
 #endif
@@ -615,14 +637,15 @@ bool DictFSSTCompressionState::CompressInternal(UnifiedVectorFormat &vector_form
 			                                    compressed_string.GetSize(), decompress_buffer);
 
 			D_ASSERT(decoded_std_string.size() == uncompressed_string.GetSize());
-			string_t decompressed_string((const char *)decompress_buffer.data(),
+			string_t decompressed_string(reinterpret_cast<const char *>(decompress_buffer.data()),
 			                             UnsafeNumericCast<uint32_t>(uncompressed_string.GetSize()));
 			D_ASSERT(decompressed_string == uncompressed_string);
 		}
 
 #endif
 		auto &string = encoded_input.data[i - encoded_input.offset];
-		return AddToDictionary<DictionaryAppendState::ENCODED_ALL_UNIQUE>(*this, string, recalculate_indices_space);
+		return AddToDictionary<DictionaryAppendState::ENCODED_ALL_UNIQUE>(*this, string, recalculate_indices_space,
+		                                                                  fail_on_no_space);
 	}
 	};
 	throw InternalException("Unreachable");
@@ -820,22 +843,22 @@ void DictFSSTCompressionState::Compress(Vector &scan_vector, idx_t count) {
 		auto &str = strings[idx];
 		auto is_null = !vector_format.validity.RowIsValid(idx);
 		do {
-			if (CompressInternal(vector_format, str, is_null, encoded_input, i, count)) {
+			if (CompressInternal(vector_format, str, is_null, encoded_input, i, count, false)) {
 				break;
 			}
 
 			if (append_state == DictionaryAppendState::REGULAR) {
 				append_state = TryEncode();
 				D_ASSERT(append_state != DictionaryAppendState::REGULAR);
-				if (CompressInternal(vector_format, str, is_null, encoded_input, i, count)) {
+				if (CompressInternal(vector_format, str, is_null, encoded_input, i, count, false)) {
 					break;
 				}
 			}
 			Flush(false);
 			encoded_input.data.clear();
 			encoded_input.offset = 0;
-			if (!CompressInternal(vector_format, str, is_null, encoded_input, i, count)) {
-				throw FatalException("Compressing directly after Flush doesn't fit");
+			if (!CompressInternal(vector_format, str, is_null, encoded_input, i, count, true)) {
+				throw FatalException("Compressing directly after Flush doesn't fit - expected to throw earlier!");
 			}
 		} while (false);
 		if (!is_null) {

package/src/duckdb/src/storage/compression/dict_fsst/decompression.cpp

@@ -134,6 +134,13 @@ const SelectionVector &CompressedStringScanState::GetSelVec(idx_t start, idx_t s
 		sel_t *sel_vec_ptr = sel_vec->data();
 		BitpackingPrimitives::UnPackBuffer<sel_t>(data_ptr_cast(sel_vec_ptr), sel_buf_src, decompress_count,
 		                                          dictionary_indices_width);
+
+		if (start_offset != 0) {
+			for (idx_t i = 0; i < scan_count; i++) {
+				sel_vec->set_index(i, sel_vec->get_index(i + start_offset));
+			}
+		}
+
 		return *sel_vec;
 	}
 	}
@@ -146,14 +153,8 @@ void CompressedStringScanState::ScanToFlatVector(Vector &result, idx_t result_of
 	// Create a decompression buffer of sufficient size if we don't already have one.
 	auto &selvec = GetSelVec(start, scan_count);
 
-	idx_t start_offset = start;
-	if (mode != DictFSSTMode::FSST_ONLY) {
-		// Handling non-bitpacking-group-aligned start values;
-		start_offset %= BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
-	} else {
-		//! (index 0 is reserved for NULL, which we don't have in this mode)
-		start_offset++;
-	}
+	//! (index 0 is reserved for NULL, which we don't have in this mode)
+	const idx_t start_offset = mode == DictFSSTMode::FSST_ONLY ? start + 1 : 0;
 
 	if (dictionary) {
 		// We have prepared the full dictionary, we can reference these strings directly
@@ -203,13 +204,10 @@ void CompressedStringScanState::Select(Vector &result, idx_t start, const Select
 	}
 }
 
-bool CompressedStringScanState::AllowDictionaryScan(idx_t start, idx_t scan_count) {
+bool CompressedStringScanState::AllowDictionaryScan(idx_t scan_count) {
 	if (mode == DictFSSTMode::FSST_ONLY) {
 		return false;
 	}
-	if (start % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE != 0) {
-		return false;
-	}
 	if (scan_count != STANDARD_VECTOR_SIZE) {
 		return false;
 	}
@@ -221,7 +219,6 @@ bool CompressedStringScanState::AllowDictionaryScan(idx_t start, idx_t scan_coun
 
 void CompressedStringScanState::ScanToDictionaryVector(ColumnSegment &segment, Vector &result, idx_t result_offset,
                                                        idx_t start, idx_t scan_count) {
-	D_ASSERT(start % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE == 0);
 	D_ASSERT(scan_count == STANDARD_VECTOR_SIZE);
 	D_ASSERT(result_offset == 0);
 

package/src/duckdb/src/storage/compression/dict_fsst.cpp

@@ -131,7 +131,7 @@ void DictFSSTCompressionStorage::StringScanPartial(ColumnSegment &segment, Colum
 	auto &scan_state = state.scan_state->Cast<CompressedStringScanState>();
 
 	auto start = segment.GetRelativeIndex(state.row_index);
-	if (!ALLOW_DICT_VECTORS || !scan_state.AllowDictionaryScan(start, scan_count)) {
+	if (!ALLOW_DICT_VECTORS || !scan_state.AllowDictionaryScan(scan_count)) {
 		scan_state.ScanToFlatVector(result, result_offset, start, scan_count);
 	} else {
 		scan_state.ScanToDictionaryVector(segment, result, result_offset, start, scan_count);
@@ -179,7 +179,7 @@ static void DictFSSTFilter(ColumnSegment &segment, ColumnScanState &state, idx_t
                            TableFilterState &filter_state) {
 	auto &scan_state = state.scan_state->Cast<CompressedStringScanState>();
 	auto start = segment.GetRelativeIndex(state.row_index);
-	if (scan_state.AllowDictionaryScan(start, vector_count)) {
+	if (scan_state.AllowDictionaryScan(vector_count)) {
 		// only pushdown filters on dictionaries
 		if (!scan_state.filter_result) {
 			// no filter result yet - apply filter to the dictionary

package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp

@@ -72,9 +72,12 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 	                                                                info.GetBlockManager());
 	if (type.InternalType() == PhysicalType::VARCHAR) {
 		auto &state = compressed_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
-		auto &partial_block_manager = checkpoint_data.GetCheckpointState().GetPartialBlockManager();
-		state.block_manager = partial_block_manager.GetBlockManager();
-		state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(partial_block_manager);
+		auto &storage_manager = checkpoint_data.GetStorageManager();
+		if (!storage_manager.InMemory()) {
+			auto &partial_block_manager = checkpoint_data.GetCheckpointState().GetPartialBlockManager();
+			state.block_manager = partial_block_manager.GetBlockManager();
+			state.overflow_writer = make_uniq<WriteOverflowStringsToDisk>(partial_block_manager);
+		}
 	}
 	current_segment = std::move(compressed_segment);
 	current_segment->InitializeAppend(append_state);
@@ -84,8 +87,10 @@ void UncompressedCompressState::FlushSegment(idx_t segment_size) {
 	auto &state = checkpoint_data.GetCheckpointState();
 	if (current_segment->type.InternalType() == PhysicalType::VARCHAR) {
 		auto &segment_state = current_segment->GetSegmentState()->Cast<UncompressedStringSegmentState>();
-		segment_state.overflow_writer->Flush();
-		segment_state.overflow_writer.reset();
+		if (segment_state.overflow_writer) {
+			segment_state.overflow_writer->Flush();
+			segment_state.overflow_writer.reset();
+		}
 	}
 	append_state.child_appends.clear();
 	append_state.append_state.reset();

package/src/duckdb/src/storage/compression/fsst.cpp

@@ -8,8 +8,10 @@
 #include "duckdb/storage/checkpoint/write_overflow_strings_to_disk.hpp"
 #include "duckdb/storage/string_uncompressed.hpp"
 #include "duckdb/storage/table/column_data_checkpointer.hpp"
+#include "duckdb/main/settings.hpp"
 
 #include "fsst.h"
+
 #include "miniz_wrapper.hpp"
 
 namespace duckdb {
@@ -644,8 +646,7 @@ void FSSTStorage::StringScanPartial(ColumnSegment &segment, ColumnScanState &sta
 
 	bool enable_fsst_vectors;
 	if (ALLOW_FSST_VECTORS) {
-		auto &config = DBConfig::GetConfig(segment.db);
-		enable_fsst_vectors = config.options.enable_fsst_vectors;
+		enable_fsst_vectors = DBConfig::GetSetting<EnableFSSTVectorsSetting>(segment.db);
 	} else {
 		enable_fsst_vectors = false;
 	}

package/src/duckdb/src/storage/compression/rle.cpp

@@ -134,7 +134,7 @@ struct RLECompressState : public CompressionState {
 
 	idx_t MaxRLECount() {
 		auto entry_size = sizeof(T) + sizeof(rle_count_t);
-		return (info.GetBlockSize() - RLEConstants::RLE_HEADER_SIZE) / entry_size;
+		return AlignValueFloor((info.GetBlockSize() - RLEConstants::RLE_HEADER_SIZE) / entry_size);
 	}
 
 	RLECompressState(ColumnDataCheckpointData &checkpoint_data_p, const CompressionInfo &info)

package/src/duckdb/src/storage/compression/zstd.cpp

@@ -474,7 +474,7 @@ public:
 
 		// Write the current page to disk
 		auto &block_manager = partial_block_manager.GetBlockManager();
-		block_manager.Write(buffer.GetFileBuffer(), block_id);
+		block_manager.Write(QueryContext(), buffer.GetFileBuffer(), block_id);
 	}
 
 	void FlushVector() {
@@ -900,16 +900,15 @@ public:
 		for (idx_t i = 0; i < count; i++) {
 			uncompressed_length += string_lengths[i];
 		}
-		auto empty_string = StringVector::EmptyString(result, uncompressed_length);
-		auto uncompressed_data = empty_string.GetDataWriteable();
+		auto &buffer = StringVector::GetStringBuffer(result);
+		auto uncompressed_data = buffer.AllocateShrinkableBuffer(uncompressed_length);
 		auto string_data = FlatVector::GetData<string_t>(result);
 
-		DecompressString(scan_state, reinterpret_cast<data_ptr_t>(uncompressed_data), uncompressed_length);
+		DecompressString(scan_state, uncompressed_data, uncompressed_length);
 
 		idx_t offset = 0;
-		auto uncompressed_data_const = empty_string.GetData();
 		for (idx_t i = 0; i < count; i++) {
-			string_data[result_offset + i] = string_t(uncompressed_data_const + offset, string_lengths[i]);
+			string_data[result_offset + i] = string_t(char_ptr_cast(uncompressed_data + offset), string_lengths[i]);
 			offset += string_lengths[i];
 		}
 		scan_state.scanned_count += count;