duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/storage/statistics/string_stats.cpp

@@ -1,14 +1,13 @@
 #include "duckdb/storage/statistics/string_stats.hpp"
 
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/main/error_manager.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "utf8proc_wrapper.hpp"
 
-#include "duckdb/common/serializer/serializer.hpp"
-#include "duckdb/common/serializer/deserializer.hpp"
-
 namespace duckdb {
 
 BaseStatistics StringStats::CreateUnknown(LogicalType type) {
@@ -115,7 +114,6 @@ void StringStats::Deserialize(Deserializer &deserializer, BaseStatistics &base)
 }
 
 static int StringValueComparison(const_data_ptr_t data, idx_t len, const_data_ptr_t comparison) {
-	D_ASSERT(len <= StringStatsData::MAX_STRING_MINMAX_SIZE);
 	for (idx_t i = 0; i < len; i++) {
 		if (data[i] < comparison[i]) {
 			return -1;
@@ -169,6 +167,9 @@ void StringStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
 	if (other.GetType().id() == LogicalTypeId::VALIDITY) {
 		return;
 	}
+	if (other.GetType().id() == LogicalTypeId::SQLNULL) {
+		return;
+	}
 	auto &string_data = StringStats::GetDataUnsafe(stats);
 	auto &other_data = StringStats::GetDataUnsafe(other);
 	if (StringValueComparison(other_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.min) < 0) {
@@ -185,12 +186,17 @@ void StringStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
 FilterPropagateResult StringStats::CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,
                                                 const string &constant) {
 	auto &string_data = StringStats::GetDataUnsafe(stats);
+	return CheckZonemap(string_data.min, StringStatsData::MAX_STRING_MINMAX_SIZE, string_data.max,
+	                    StringStatsData::MAX_STRING_MINMAX_SIZE, comparison_type, constant);
+}
+
+FilterPropagateResult StringStats::CheckZonemap(const_data_ptr_t min_data, idx_t min_len, const_data_ptr_t max_data,
+                                                idx_t max_len, ExpressionType comparison_type, const string &constant) {
 	auto data = const_data_ptr_cast(constant.c_str());
-	auto size = constant.size();
+	idx_t size = constant.size();
 
-	idx_t value_size = size > StringStatsData::MAX_STRING_MINMAX_SIZE ? StringStatsData::MAX_STRING_MINMAX_SIZE : size;
-	int min_comp = StringValueComparison(data, value_size, string_data.min);
-	int max_comp = StringValueComparison(data, value_size, string_data.max);
+	int min_comp = StringValueComparison(data, MinValue(min_len, size), min_data);
+	int max_comp = StringValueComparison(data, MinValue(max_len, size), max_data);
 	switch (comparison_type) {
 	case ExpressionType::COMPARE_EQUAL:
 		if (min_comp >= 0 && max_comp <= 0) {

package/src/duckdb/src/storage/statistics/struct_stats.cpp

@@ -85,7 +85,8 @@ void StructStats::Merge(BaseStatistics &stats, const BaseStatistics &other) {
 	if (other.GetType().id() == LogicalTypeId::VALIDITY) {
 		return;
 	}
-	D_ASSERT(stats.GetType() == other.GetType());
+	D_ASSERT(stats.GetType().id() == other.GetType().id());
+	D_ASSERT(StructType::GetChildCount(stats.GetType()) == StructType::GetChildCount(other.GetType()));
 	auto child_count = StructType::GetChildCount(stats.GetType());
 	for (idx_t i = 0; i < child_count; i++) {
 		stats.child_stats[i].Merge(other.child_stats[i]);

package/src/duckdb/src/storage/storage_info.cpp

@@ -1,4 +1,6 @@
 #include "duckdb/storage/storage_info.hpp"
+
+#include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/common/optional_idx.hpp"
 
 namespace duckdb {
@@ -20,18 +22,19 @@ struct SerializationVersionInfo {
 
 // START OF STORAGE VERSION INFO
 static const StorageVersionInfo storage_version_info[] = {
-    {"v0.0.4", 1},  {"v0.1.0", 1},  {"v0.1.1", 1},  {"v0.1.2", 1},  {"v0.1.3", 1},   {"v0.1.4", 1},   {"v0.1.5", 1},
-    {"v0.1.6", 1},  {"v0.1.7", 1},  {"v0.1.8", 1},  {"v0.1.9", 1},  {"v0.2.0", 1},   {"v0.2.1", 1},   {"v0.2.2", 4},
-    {"v0.2.3", 6},  {"v0.2.4", 11}, {"v0.2.5", 13}, {"v0.2.6", 15}, {"v0.2.7", 17},  {"v0.2.8", 18},  {"v0.2.9", 21},
-    {"v0.3.0", 25}, {"v0.3.1", 27}, {"v0.3.2", 31}, {"v0.3.3", 33}, {"v0.3.4", 33},  {"v0.3.5", 33},  {"v0.4.0", 33},
-    {"v0.5.0", 38}, {"v0.5.1", 38}, {"v0.6.0", 39}, {"v0.6.1", 39}, {"v0.7.0", 43},  {"v0.7.1", 43},  {"v0.8.0", 51},
-    {"v0.8.1", 51}, {"v0.9.0", 64}, {"v0.9.1", 64}, {"v0.9.2", 64}, {"v0.10.0", 64}, {"v0.10.1", 64}, {"v0.10.2", 64},
-    {nullptr, 0}};
+    {"v0.0.4", 1},   {"v0.1.0", 1},  {"v0.1.1", 1},  {"v0.1.2", 1},  {"v0.1.3", 1},   {"v0.1.4", 1},   {"v0.1.5", 1},
+    {"v0.1.6", 1},   {"v0.1.7", 1},  {"v0.1.8", 1},  {"v0.1.9", 1},  {"v0.2.0", 1},   {"v0.2.1", 1},   {"v0.2.2", 4},
+    {"v0.2.3", 6},   {"v0.2.4", 11}, {"v0.2.5", 13}, {"v0.2.6", 15}, {"v0.2.7", 17},  {"v0.2.8", 18},  {"v0.2.9", 21},
+    {"v0.3.0", 25},  {"v0.3.1", 27}, {"v0.3.2", 31}, {"v0.3.3", 33}, {"v0.3.4", 33},  {"v0.3.5", 33},  {"v0.4.0", 33},
+    {"v0.5.0", 38},  {"v0.5.1", 38}, {"v0.6.0", 39}, {"v0.6.1", 39}, {"v0.7.0", 43},  {"v0.7.1", 43},  {"v0.8.0", 51},
+    {"v0.8.1", 51},  {"v0.9.0", 64}, {"v0.9.1", 64}, {"v0.9.2", 64}, {"v0.10.0", 64}, {"v0.10.1", 64}, {"v0.10.2", 64},
+    {"v0.10.3", 64}, {"v1.0.0", 64}, {"v1.1.0", 64}, {nullptr, 0}};
 // END OF STORAGE VERSION INFO
 
 // START OF SERIALIZATION VERSION INFO
-static const SerializationVersionInfo serialization_version_info[] = {
-    {"v0.10.0", 1}, {"v0.10.1", 1}, {"v0.10.2", 1}, {"latest", 2}, {nullptr, 0}};
+static const SerializationVersionInfo serialization_version_info[] = {{"v0.10.0", 1}, {"v0.10.1", 1}, {"v0.10.2", 1},
+                                                                      {"v0.10.3", 2}, {"v1.0.0", 2},  {"v1.1.0", 3},
+                                                                      {"latest", 3},  {nullptr, 0}};
 // END OF SERIALIZATION VERSION INFO
 
 optional_idx GetStorageVersion(const char *version_string) {
@@ -82,4 +85,26 @@ string GetDuckDBVersion(idx_t version_number) {
 	return result;
 }
 
+void Storage::VerifyBlockAllocSize(const idx_t block_alloc_size) {
+	if (!IsPowerOfTwo(block_alloc_size)) {
+		throw InvalidInputException("the block size must be a power of two, got %llu", block_alloc_size);
+	}
+	if (block_alloc_size < MIN_BLOCK_ALLOC_SIZE) {
+		throw InvalidInputException(
+		    "the block size must be greater or equal than the minimum block size of %llu, got %llu",
+		    MIN_BLOCK_ALLOC_SIZE, block_alloc_size);
+	}
+	if (block_alloc_size > MAX_BLOCK_ALLOC_SIZE) {
+		throw InvalidInputException(
+		    "the block size must be lesser or equal than the maximum block size of %llu, got %llu",
+		    MAX_BLOCK_ALLOC_SIZE, block_alloc_size);
+	}
+	auto max_value = NumericCast<idx_t>(NumericLimits<int32_t>().Maximum());
+	if (block_alloc_size > max_value) {
+		throw InvalidInputException(
+		    "the block size must not be greater than the maximum 32-bit signed integer value of %llu, got %llu",
+		    max_value, block_alloc_size);
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/storage_manager.cpp

@@ -13,19 +13,21 @@
 #include "duckdb/storage/object_cache.hpp"
 #include "duckdb/storage/single_file_block_manager.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
-
+#include "duckdb/common/serializer/memory_stream.hpp"
 #include "duckdb/storage/storage_extension.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 
 namespace duckdb {
 
 StorageManager::StorageManager(AttachedDatabase &db, string path_p, bool read_only)
     : db(db), path(std::move(path_p)), read_only(read_only) {
+
 	if (path.empty()) {
 		path = IN_MEMORY_PATH;
-	} else {
-		auto &fs = FileSystem::Get(db);
-		this->path = fs.ExpandPath(path);
+		return;
 	}
+	auto &fs = FileSystem::Get(db);
+	path = fs.ExpandPath(path);
 }
 
 StorageManager::~StorageManager() {
@@ -58,15 +60,12 @@ bool ObjectCache::ObjectCacheEnabled(ClientContext &context) {
 	return context.db->config.options.object_cache_enable;
 }
 
-int64_t StorageManager::GetWALSize() {
-	if (!wal && !GetWAL()) {
-		return 0;
-	}
-	if (!wal->Initialized()) {
-		D_ASSERT(!FileSystem::Get(db).FileExists(GetWALPath()));
+idx_t StorageManager::GetWALSize() {
+	auto wal_ptr = GetWAL();
+	if (!wal_ptr) {
 		return 0;
 	}
-	return wal->GetWriter().GetFileSize();
+	return wal_ptr->GetWALSize();
 }
 
 optional_ptr<WriteAheadLog> StorageManager::GetWAL() {
@@ -77,11 +76,6 @@ optional_ptr<WriteAheadLog> StorageManager::GetWAL() {
 	if (!wal) {
 		auto wal_path = GetWALPath();
 		wal = make_uniq<WriteAheadLog>(db, wal_path);
-
-		// If the WAL file exists, then we initialize it.
-		if (FileSystem::Get(db).FileExists(wal_path)) {
-			wal->Initialize();
-		}
 	}
 	return wal.get();
 }
@@ -95,8 +89,12 @@ void StorageManager::ResetWAL() {
 }
 
 string StorageManager::GetWALPath() {
-
-	std::size_t question_mark_pos = path.find('?');
+	// we append the ".wal" **before** a question mark in case of GET parameters
+	// but only if we are not in a windows long path (which starts with \\?\)
+	std::size_t question_mark_pos = std::string::npos;
+	if (!StringUtil::StartsWith(path, "\\\\?\\")) {
+		question_mark_pos = path.find('?');
+	}
 	auto wal_path = path;
 	if (question_mark_pos != std::string::npos) {
 		wal_path.insert(question_mark_pos, ".wal");
@@ -111,14 +109,14 @@ bool StorageManager::InMemory() {
 	return path == IN_MEMORY_PATH;
 }
 
-void StorageManager::Initialize() {
+void StorageManager::Initialize(const optional_idx block_alloc_size) {
 	bool in_memory = InMemory();
 	if (in_memory && read_only) {
 		throw CatalogException("Cannot launch in-memory database in read-only mode!");
 	}
 
-	// create or load the database from disk, if not in-memory mode
-	LoadDatabase();
+	// Create or load the database from disk, if not in-memory mode.
+	LoadDatabase(block_alloc_size);
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -145,9 +143,9 @@ SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string
     : StorageManager(db, std::move(path), read_only) {
 }
 
-void SingleFileStorageManager::LoadDatabase() {
+void SingleFileStorageManager::LoadDatabase(const optional_idx block_alloc_size) {
 	if (InMemory()) {
-		block_manager = make_uniq<InMemoryBlockManager>(BufferManager::GetBufferManager(db));
+		block_manager = make_uniq<InMemoryBlockManager>(BufferManager::GetBufferManager(db), DEFAULT_BLOCK_ALLOC_SIZE);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 		return;
 	}
@@ -165,7 +163,8 @@ void SingleFileStorageManager::LoadDatabase() {
 	options.use_direct_io = config.options.use_direct_io;
 	options.debug_initialize = config.options.debug_initialize;
 
-	// first check if the database exists
+	// Check if the database file already exists.
+	// Note: a file can also exist if there was a ROLLBACK on a previous transaction creating that file.
 	if (!read_only && !fs.FileExists(path)) {
 		// file does not exist and we are in read-write mode
 		// create a new file
@@ -178,21 +177,39 @@ void SingleFileStorageManager::LoadDatabase() {
 			fs.RemoveFile(wal_path);
 		}
 
-		// initialize the block manager while creating a new db file
+		// Set the block allocation size for the new database file.
+		if (block_alloc_size.IsValid()) {
+			// Use the option provided by the user.
+			options.block_alloc_size = block_alloc_size;
+		} else {
+			// No explicit option provided: use the default option.
+			options.block_alloc_size = config.options.default_block_alloc_size;
+		}
+
+		// Initialize the block manager before creating a new database.
 		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
 		sf_block_manager->CreateNewDatabase();
 		block_manager = std::move(sf_block_manager);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
+
 	} else {
-		// either the file exists, or we are in read-only mode
-		// try to read the existing file on disk
+		// Either the file exists, or we are in read-only mode, so we
+		// try to read the existing file on disk.
 
-		// initialize the block manager while loading the current db file
+		// Initialize the block manager while loading the database file.
+		// We'll construct the SingleFileBlockManager with the default block allocation size,
+		// and later adjust it when reading the file header.
 		auto sf_block_manager = make_uniq<SingleFileBlockManager>(db, path, options);
 		sf_block_manager->LoadExistingDatabase();
 		block_manager = std::move(sf_block_manager);
 		table_io_manager = make_uniq<SingleFileTableIOManager>(*block_manager);
 
+		if (block_alloc_size.IsValid() && block_alloc_size.GetIndex() != block_manager->GetBlockAllocSize()) {
+			throw InvalidInputException(
+			    "block size parameter does not match the file's block size, got %llu, expected %llu",
+			    block_alloc_size.GetIndex(), block_manager->GetBlockAllocSize());
+		}
+
 		// load the db from storage
 		auto checkpoint_reader = SingleFileCheckpointReader(*this);
 		checkpoint_reader.LoadFromStorage();
@@ -213,68 +230,120 @@ void SingleFileStorageManager::LoadDatabase() {
 
 ///////////////////////////////////////////////////////////////////////////////
 
-class SingleFileStorageCommitState : public StorageCommitState {
-	idx_t initial_wal_size = 0;
-	idx_t initial_written = 0;
-	optional_ptr<WriteAheadLog> log;
-	bool checkpoint;
+enum class WALCommitState { IN_PROGRESS, FLUSHED, TRUNCATED };
 
-public:
-	SingleFileStorageCommitState(StorageManager &storage_manager, bool checkpoint);
-	~SingleFileStorageCommitState() override {
-		// If log is non-null, then commit threw an exception before flushing.
-		if (log) {
-			auto &wal = *log.get();
-			wal.skip_writing = false;
-			if (wal.GetTotalWritten() > initial_written) {
-				// remove any entries written into the WAL by truncating it
-				wal.Truncate(NumericCast<int64_t>(initial_wal_size));
-			}
-		}
+struct OptimisticallyWrittenRowGroupData {
+	OptimisticallyWrittenRowGroupData(idx_t start, idx_t count, unique_ptr<PersistentCollectionData> row_group_data_p)
+	    : start(start), count(count), row_group_data(std::move(row_group_data_p)) {
 	}
 
+	idx_t start;
+	idx_t count;
+	unique_ptr<PersistentCollectionData> row_group_data;
+};
+
+class SingleFileStorageCommitState : public StorageCommitState {
+public:
+	SingleFileStorageCommitState(StorageManager &storage, WriteAheadLog &log);
+	~SingleFileStorageCommitState() override;
+
+	//! Revert the commit
+	void RevertCommit() override;
 	// Make the commit persistent
 	void FlushCommit() override;
+
+	void AddRowGroupData(DataTable &table, idx_t start_index, idx_t count,
+	                     unique_ptr<PersistentCollectionData> row_group_data) override;
+	optional_ptr<PersistentCollectionData> GetRowGroupData(DataTable &table, idx_t start_index, idx_t &count) override;
+	bool HasRowGroupData() override;
+
+private:
+	idx_t initial_wal_size = 0;
+	idx_t initial_written = 0;
+	WriteAheadLog &wal;
+	WALCommitState state;
+	reference_map_t<DataTable, unordered_map<idx_t, OptimisticallyWrittenRowGroupData>> optimistically_written_data;
 };
 
-SingleFileStorageCommitState::SingleFileStorageCommitState(StorageManager &storage_manager, bool checkpoint)
-    : checkpoint(checkpoint) {
+SingleFileStorageCommitState::SingleFileStorageCommitState(StorageManager &storage, WriteAheadLog &wal)
+    : wal(wal), state(WALCommitState::IN_PROGRESS) {
+	auto initial_size = storage.GetWALSize();
+	initial_written = wal.GetTotalWritten();
+	initial_wal_size = initial_size;
+}
 
-	log = storage_manager.GetWAL();
-	if (!log) {
+SingleFileStorageCommitState::~SingleFileStorageCommitState() {
+	if (state != WALCommitState::IN_PROGRESS) {
 		return;
 	}
+	try {
+		// truncate the WAL in case of a destructor
+		RevertCommit();
+	} catch (...) { // NOLINT
+	}
+}
 
-	auto initial_size = storage_manager.GetWALSize();
-	initial_written = log->GetTotalWritten();
-	initial_wal_size = initial_size < 0 ? 0 : idx_t(initial_size);
-
-	if (checkpoint) {
-		// True, if we are checkpointing after the current commit.
-		// If true, we don't need to write to the WAL, saving unnecessary writes to disk.
-		log->skip_writing = true;
+void SingleFileStorageCommitState::RevertCommit() {
+	if (state != WALCommitState::IN_PROGRESS) {
+		return;
+	}
+	if (wal.GetTotalWritten() > initial_written) {
+		// remove any entries written into the WAL by truncating it
+		wal.Truncate(initial_wal_size);
 	}
+	state = WALCommitState::TRUNCATED;
 }
 
-// Make the commit persistent
 void SingleFileStorageCommitState::FlushCommit() {
-	if (log) {
-		// flush the WAL if any changes were made
-		if (log->GetTotalWritten() > initial_written) {
-			(void)checkpoint;
-			D_ASSERT(!checkpoint);
-			D_ASSERT(!log->skip_writing);
-			log->Flush();
-		}
-		log->skip_writing = false;
+	if (state != WALCommitState::IN_PROGRESS) {
+		return;
+	}
+	wal.Flush();
+	state = WALCommitState::FLUSHED;
+}
+
+void SingleFileStorageCommitState::AddRowGroupData(DataTable &table, idx_t start_index, idx_t count,
+                                                   unique_ptr<PersistentCollectionData> row_group_data) {
+	if (row_group_data->HasUpdates()) {
+		// cannot serialize optimistic block pointers if in-memory updates exist
+		return;
+	}
+	if (table.HasIndexes()) {
+		// cannot serialize optimistic block pointers if the table has indexes
+		return;
+	}
+	auto &entries = optimistically_written_data[table];
+	auto entry = entries.find(start_index);
+	if (entry != entries.end()) {
+		throw InternalException("FIXME: AddOptimisticallyWrittenRowGroup is writing a duplicate row group");
+	}
+	entries.insert(
+	    make_pair(start_index, OptimisticallyWrittenRowGroupData(start_index, count, std::move(row_group_data))));
+}
+
+optional_ptr<PersistentCollectionData> SingleFileStorageCommitState::GetRowGroupData(DataTable &table,
+                                                                                     idx_t start_index, idx_t &count) {
+	auto entry = optimistically_written_data.find(table);
+	if (entry == optimistically_written_data.end()) {
+		// no data for this table
+		return nullptr;
+	}
+	auto &row_groups = entry->second;
+	auto start_entry = row_groups.find(start_index);
+	if (start_entry == row_groups.end()) {
+		// this row group was not optimistically written
+		return nullptr;
 	}
-	// Null so that the destructor will not truncate the log.
-	log = nullptr;
+	count = start_entry->second.count;
+	return start_entry->second.row_group_data.get();
 }
 
-unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(Transaction &transaction,
-                                                                               bool checkpoint) {
-	return make_uniq<SingleFileStorageCommitState>(*this, checkpoint);
+bool SingleFileStorageCommitState::HasRowGroupData() {
+	return !optimistically_written_data.empty();
+}
+
+unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(WriteAheadLog &wal) {
+	return make_uniq<SingleFileStorageCommitState>(*this, wal);
 }
 
 bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id) {
@@ -289,7 +358,7 @@ void SingleFileStorageManager::CreateCheckpoint(CheckpointOptions options) {
 		db.GetStorageExtension()->OnCheckpointStart(db, options);
 	}
 	auto &config = DBConfig::Get(db);
-	if (GetWALSize() > 0 || config.options.force_checkpoint || options.action == CheckpointAction::FORCE_CHECKPOINT) {
+	if (GetWALSize() > 0 || config.options.force_checkpoint || options.action == CheckpointAction::ALWAYS_CHECKPOINT) {
 		// we only need to checkpoint if there is anything in the WAL
 		try {
 			SingleFileCheckpointWriter checkpointer(db, *block_manager, options.type);
@@ -313,7 +382,7 @@ DatabaseSize SingleFileStorageManager::GetDatabaseSize() {
 	DatabaseSize ds;
 	if (!InMemory()) {
 		ds.total_blocks = block_manager->TotalBlocks();
-		ds.block_size = Storage::BLOCK_ALLOC_SIZE;
+		ds.block_size = block_manager->GetBlockAllocSize();
 		ds.free_blocks = block_manager->FreeBlocks();
 		ds.used_blocks = ds.total_blocks - ds.free_blocks;
 		ds.bytes = (ds.total_blocks * ds.block_size);
@@ -339,4 +408,8 @@ shared_ptr<TableIOManager> SingleFileStorageManager::GetTableIOManager(BoundCrea
 	return shared_ptr<TableIOManager>(shared_ptr<char>(nullptr), table_io_manager.get());
 }
 
+BlockManager &SingleFileStorageManager::GetBlockManager() {
+	return *block_manager;
+}
+
 } // namespace duckdb

package/src/duckdb/src/storage/table/array_column_data.cpp

@@ -24,10 +24,17 @@ void ArrayColumnData::SetStart(idx_t new_start) {
 	validity.SetStart(new_start);
 }
 
-bool ArrayColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
+FilterPropagateResult ArrayColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
 	// FIXME: There is nothing preventing us from supporting this, but it's not implemented yet.
 	// table filters are not supported yet for fixed size list columns
-	return false;
+	return FilterPropagateResult::NO_PRUNING_POSSIBLE;
+}
+
+void ArrayColumnData::InitializePrefetch(PrefetchState &prefetch_state, ColumnScanState &scan_state, idx_t rows) {
+	ColumnData::InitializePrefetch(prefetch_state, scan_state, rows);
+	validity.InitializePrefetch(prefetch_state, scan_state.child_states[0], rows);
+	auto array_size = ArrayType::GetSize(type);
+	child_column->InitializePrefetch(prefetch_state, scan_state.child_states[1], rows * array_size);
 }
 
 void ArrayColumnData::InitializeScan(ColumnScanState &state) {
@@ -59,11 +66,12 @@ void ArrayColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row
 	validity.InitializeScanWithOffset(state.child_states[0], row_idx);
 
 	auto array_size = ArrayType::GetSize(type);
-	auto child_offset = (row_idx - start) * array_size;
+	auto child_count = (row_idx - start) * array_size;
 
-	D_ASSERT(child_offset <= child_column->GetMaxEntry());
-	if (child_offset < child_column->GetMaxEntry()) {
-		child_column->InitializeScanWithOffset(state.child_states[1], start + child_offset);
+	D_ASSERT(child_count <= child_column->GetMaxEntry());
+	if (child_count < child_column->GetMaxEntry()) {
+		const auto child_offset = start + child_count;
+		child_column->InitializeScanWithOffset(state.child_states[1], child_offset);
 	}
 }
 
@@ -164,7 +172,10 @@ void ArrayColumnData::FetchRow(TransactionData transaction, ColumnFetchState &st
 	// We need to fetch between [row_id * array_size, (row_id + 1) * array_size)
 	auto child_state = make_uniq<ColumnScanState>();
 	child_state->Initialize(child_type, nullptr);
-	child_column->InitializeScanWithOffset(*child_state, UnsafeNumericCast<idx_t>(row_id) * array_size);
+
+	const auto child_offset = start + (UnsafeNumericCast<idx_t>(row_id) - start) * array_size;
+
+	child_column->InitializeScanWithOffset(*child_state, child_offset);
 	Vector child_scan(child_type, array_size);
 	child_column->ScanCount(*child_state, child_scan, array_size);
 	VectorOperations::Copy(child_scan, child_vec, array_size, 0, result_idx * array_size);
@@ -191,11 +202,11 @@ public:
 		return stats.ToUnique();
 	}
 
-	void WriteDataPointers(RowGroupWriter &writer, Serializer &serializer) override {
-		serializer.WriteObject(101, "validity",
-		                       [&](Serializer &serializer) { validity_state->WriteDataPointers(writer, serializer); });
-		serializer.WriteObject(102, "child_column",
-		                       [&](Serializer &serializer) { child_state->WriteDataPointers(writer, serializer); });
+	PersistentColumnData ToPersistentData() override {
+		PersistentColumnData data(PhysicalType::ARRAY);
+		data.child_columns.push_back(validity_state->ToPersistentData());
+		data.child_columns.push_back(child_state->ToPersistentData());
+		return data;
 	}
 };
 
@@ -213,13 +224,22 @@ unique_ptr<ColumnCheckpointState> ArrayColumnData::Checkpoint(RowGroup &row_grou
 	return std::move(checkpoint_state);
 }
 
-void ArrayColumnData::DeserializeColumn(Deserializer &deserializer, BaseStatistics &target_stats) {
-	deserializer.ReadObject(101, "validity",
-	                        [&](Deserializer &source) { validity.DeserializeColumn(source, target_stats); });
+bool ArrayColumnData::IsPersistent() {
+	return validity.IsPersistent() && child_column->IsPersistent();
+}
+
+PersistentColumnData ArrayColumnData::Serialize() {
+	PersistentColumnData persistent_data(PhysicalType::ARRAY);
+	persistent_data.child_columns.push_back(validity.Serialize());
+	persistent_data.child_columns.push_back(child_column->Serialize());
+	return persistent_data;
+}
 
+void ArrayColumnData::InitializeColumn(PersistentColumnData &column_data, BaseStatistics &target_stats) {
+	D_ASSERT(column_data.pointers.empty());
+	validity.InitializeColumn(column_data.child_columns[0], target_stats);
 	auto &child_stats = ArrayStats::GetChildStats(target_stats);
-	deserializer.ReadObject(102, "child_column",
-	                        [&](Deserializer &source) { child_column->DeserializeColumn(source, child_stats); });
+	child_column->InitializeColumn(column_data.child_columns[1], child_stats);
 	this->count = validity.count.load();
 }
 

package/src/duckdb/src/storage/table/chunk_info.cpp

@@ -32,6 +32,10 @@ static bool UseVersion(TransactionData transaction, transaction_t id) {
 	return TransactionVersionOperator::UseInsertedVersion(transaction.start_time, transaction.transaction_id, id);
 }
 
+bool ChunkInfo::Cleanup(transaction_t lowest_transaction, unique_ptr<ChunkInfo> &result) const {
+	return false;
+}
+
 void ChunkInfo::Write(WriteStream &writer) const {
 	writer.Write<ChunkInfoType>(type);
 }
@@ -95,6 +99,18 @@ idx_t ChunkConstantInfo::GetCommittedDeletedCount(idx_t max_count) {
 	return delete_id < TRANSACTION_ID_START ? max_count : 0;
 }
 
+bool ChunkConstantInfo::Cleanup(transaction_t lowest_transaction, unique_ptr<ChunkInfo> &result) const {
+	if (delete_id != NOT_DELETED_ID) {
+		// the chunk info is labeled as deleted - we need to keep it around
+		return false;
+	}
+	if (insert_id > lowest_transaction) {
+		// there are still transactions active that need this ChunkInfo
+		return false;
+	}
+	return true;
+}
+
 void ChunkConstantInfo::Write(WriteStream &writer) const {
 	D_ASSERT(HasDeletes());
 	ChunkInfo::Write(writer);
@@ -233,6 +249,28 @@ void ChunkVectorInfo::CommitAppend(transaction_t commit_id, idx_t start, idx_t e
 	}
 }
 
+bool ChunkVectorInfo::Cleanup(transaction_t lowest_transaction, unique_ptr<ChunkInfo> &result) const {
+	if (any_deleted) {
+		// if any rows are deleted we can't clean-up
+		return false;
+	}
+	// check if the insertion markers have to be used by all transactions going forward
+	if (!same_inserted_id) {
+		for (idx_t idx = 1; idx < STANDARD_VECTOR_SIZE; idx++) {
+			if (inserted[idx] > lowest_transaction) {
+				// transaction was inserted after the lowest transaction start
+				// we still need to use an older version - cannot compress
+				return false;
+			}
+		}
+	} else if (insert_id > lowest_transaction) {
+		// transaction was inserted after the lowest transaction start
+		// we still need to use an older version - cannot compress
+		return false;
+	}
+	return true;
+}
+
 bool ChunkVectorInfo::HasDeletes() const {
 	return any_deleted;
 }