duckdb 1.4.2-dev4.0 → 1.4.2

package/src/duckdb/src/storage/compression/zstd.cpp

@@ -142,6 +142,11 @@ public:
 unique_ptr<AnalyzeState> ZSTDStorage::StringInitAnalyze(ColumnData &col_data, PhysicalType type) {
 	// check if the storage version we are writing to supports sztd
 	auto &storage = col_data.GetStorageManager();
+	auto &block_manager = col_data.GetBlockManager();
+	if (block_manager.InMemory()) {
+		//! Can't use ZSTD in in-memory environment
+		return nullptr;
+	}
 	if (storage.GetStorageVersion() < 4) {
 		// compatibility mode with old versions - disable zstd
 		return nullptr;
@@ -249,6 +254,7 @@ public:
 
 public:
 	void ResetOutBuffer() {
+		D_ASSERT(GetCurrentOffset() <= GetWritableSpace(info));
 		out_buffer.dst = current_buffer_ptr;
 		out_buffer.pos = 0;
 
@@ -347,6 +353,7 @@ public:
 	void InitializeVector() {
 		D_ASSERT(!in_vector);
 		if (vector_count + 1 >= total_vector_count) {
+			//! Last vector
 			vector_size = analyze_state->count - (ZSTD_VECTOR_SIZE * vector_count);
 		} else {
 			vector_size = ZSTD_VECTOR_SIZE;
@@ -355,6 +362,7 @@ public:
 		current_offset = UnsafeNumericCast<page_offset_t>(
 		    AlignValue<idx_t, sizeof(string_length_t)>(UnsafeNumericCast<idx_t>(current_offset)));
 		current_buffer_ptr = current_buffer->Ptr() + current_offset;
+		D_ASSERT(GetCurrentOffset() <= GetWritableSpace(info));
 		compressed_size = 0;
 		uncompressed_size = 0;
 
@@ -413,15 +421,11 @@ public:
 				throw InvalidInputException("ZSTD Compression failed: %s",
 				                            duckdb_zstd::ZSTD_getErrorName(compress_result));
 			}
+			D_ASSERT(GetCurrentOffset() <= GetWritableSpace(info));
 			if (compress_result == 0) {
 				// Finished
 				break;
 			}
-			if (out_buffer.pos != out_buffer.size) {
-				throw InternalException("Expected ZSTD_compressStream2 to fully utilize the current buffer, but pos is "
-				                        "%d, while size is %d",
-				                        out_buffer.pos, out_buffer.size);
-			}
 			NewPage();
 		}
 	}
@@ -691,7 +695,7 @@ public:
 	explicit ZSTDScanState(ColumnSegment &segment)
 	    : state(segment.GetSegmentState()->Cast<UncompressedStringSegmentState>()),
 	      block_manager(segment.GetBlockManager()), buffer_manager(BufferManager::GetBufferManager(segment.db)),
-	      segment_block_offset(segment.GetBlockOffset()) {
+	      segment_block_offset(segment.GetBlockOffset()), segment(segment) {
 		decompression_context = duckdb_zstd::ZSTD_createDCtx();
 		segment_handle = buffer_manager.Pin(segment.block);
 
@@ -791,14 +795,23 @@ public:
 
 		auto vector_size = metadata.count;
 
+		auto string_lengths_size = (sizeof(string_length_t) * vector_size);
 		scan_state.string_lengths = reinterpret_cast<string_length_t *>(scan_state.current_buffer_ptr);
-		scan_state.current_buffer_ptr += (sizeof(string_length_t) * vector_size);
+		scan_state.current_buffer_ptr += string_lengths_size;
 
 		// Update the in_buffer to point to the start of the compressed data frame
 		idx_t current_offset = UnsafeNumericCast<idx_t>(scan_state.current_buffer_ptr - handle_start);
 		scan_state.in_buffer.src = scan_state.current_buffer_ptr;
 		scan_state.in_buffer.pos = 0;
-		scan_state.in_buffer.size = block_manager.GetBlockSize() - sizeof(block_id_t) - current_offset;
+		if (scan_state.metadata.block_offset + string_lengths_size + scan_state.metadata.compressed_size >
+		    (segment.SegmentSize() - sizeof(block_id_t))) {
+			//! We know that the compressed size is too big to fit on the current page
+			scan_state.in_buffer.size =
+			    MinValue(metadata.compressed_size, block_manager.GetBlockSize() - sizeof(block_id_t) - current_offset);
+		} else {
+			scan_state.in_buffer.size =
+			    MinValue(metadata.compressed_size, block_manager.GetBlockSize() - current_offset);
+		}
 
 		// Initialize the context for streaming decompression
 		duckdb_zstd::ZSTD_DCtx_reset(decompression_context, duckdb_zstd::ZSTD_reset_session_only);
@@ -832,7 +845,7 @@ public:
 		scan_state.in_buffer.src = ptr;
 		scan_state.in_buffer.pos = 0;
 
-		idx_t page_size = block_manager.GetBlockSize() - sizeof(block_id_t);
+		idx_t page_size = segment.SegmentSize() - sizeof(block_id_t);
 		idx_t remaining_compressed_data = scan_state.metadata.compressed_size - scan_state.compressed_scan_count;
 		scan_state.in_buffer.size = MinValue<idx_t>(page_size, remaining_compressed_data);
 	}
@@ -842,6 +855,7 @@ public:
 			return;
 		}
 
+		auto &in_buffer = scan_state.in_buffer;
 		duckdb_zstd::ZSTD_outBuffer out_buffer;
 
 		out_buffer.dst = destination;
@@ -849,18 +863,25 @@ public:
 		out_buffer.size = uncompressed_length;
 
 		while (true) {
-			idx_t old_pos = scan_state.in_buffer.pos;
+			idx_t old_pos = in_buffer.pos;
 			size_t res = duckdb_zstd::ZSTD_decompressStream(
 			    /* zds = */ decompression_context,
 			    /* output =*/&out_buffer,
-			    /* input =*/&scan_state.in_buffer);
-			scan_state.compressed_scan_count += scan_state.in_buffer.pos - old_pos;
+			    /* input =*/&in_buffer);
+			scan_state.compressed_scan_count += in_buffer.pos - old_pos;
 			if (duckdb_zstd::ZSTD_isError(res)) {
 				throw InvalidInputException("ZSTD Decompression failed: %s", duckdb_zstd::ZSTD_getErrorName(res));
 			}
 			if (out_buffer.pos == out_buffer.size) {
+				//! Done decompressing the relevant portion
+				break;
+			}
+			if (!res) {
+				D_ASSERT(out_buffer.pos == out_buffer.size);
+				D_ASSERT(in_buffer.pos == in_buffer.size);
 				break;
 			}
+			D_ASSERT(in_buffer.pos == in_buffer.size);
 			// Did not fully decompress, it needs a new page to read from
 			LoadNextPageForVector(scan_state);
 		}
@@ -956,6 +977,7 @@ public:
 	idx_t segment_count;
 	//! The amount of tuples consumed
 	idx_t scanned_count = 0;
+	ColumnSegment &segment;
 
 	//! Buffer for skipping data
 	AllocatedData skip_buffer;

package/src/duckdb/src/storage/data_table.cpp

@@ -1195,7 +1195,7 @@ ErrorData DataTable::AppendToIndexes(TableIndexList &indexes, optional_ptr<Table
 		if (!index.IsBound()) {
 			// Buffer only the key columns, and store their mapping.
 			auto &unbound_index = index.Cast<UnboundIndex>();
-			unbound_index.BufferChunk(index_chunk, row_ids, mapped_column_ids);
+			unbound_index.BufferChunk(index_chunk, row_ids, mapped_column_ids, BufferedIndexReplay::INSERT_ENTRY);
 			return false;
 		}
 

package/src/duckdb/src/storage/local_storage.cpp

@@ -154,12 +154,25 @@ void LocalTableStorage::FlushBlocks() {
 ErrorData LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source,
                                              TableIndexList &index_list, const vector<LogicalType> &table_types,
                                              row_t &start_row) {
-	// In this function, we only care about scanning the indexed columns of a table.
+	// mapped_column_ids contains the physical column indices of each Indexed column in the table.
+	// This mapping is used to retrieve the physical column index for the corresponding vector of an index chunk scan.
+	// For example, if we are processing data for index_chunk.data[i], we can retrieve the physical column index
+	// by getting the value at mapped_column_ids[i].
+	// An important note is that the index_chunk orderings are created in accordance with this mapping, not the other
+	// way around. (Check the scan code below, where the mapped_column_ids is passed as a parameter to the scan.
+	// The index_chunk inside of that lambda is ordered according to the mapping that is a parameter to the scan).
+
+	// mapped_column_ids is used in two places:
+	// 1) To create the physical table chunk in this function.
+	// 2) If we are in an unbound state (i.e., WAL replay is happening right now), this mapping and the index_chunk
+	//	  are buffered in unbound_index. However, there can also be buffered deletes happening, so it is important
+	//    to maintain a canonical representation of the mapping, which is just sorting.
 	auto indexed_columns = index_list.GetRequiredColumns();
 	vector<StorageIndex> mapped_column_ids;
 	for (auto &col : indexed_columns) {
 		mapped_column_ids.emplace_back(col);
 	}
+	std::sort(mapped_column_ids.begin(), mapped_column_ids.end());
 
 	// However, because the bound expressions of the indexes (and their bound
 	// column references) are in relation to ALL table columns, we create an
@@ -168,6 +181,7 @@ ErrorData LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGr
 	DataChunk table_chunk;
 	table_chunk.InitializeEmpty(table_types);
 
+	// index_chunk scans are created here in the mapped_column_ids ordering (see note above).
 	ErrorData error;
 	source.Scan(transaction, mapped_column_ids, [&](DataChunk &index_chunk) -> bool {
 		D_ASSERT(index_chunk.ColumnCount() == mapped_column_ids.size());
@@ -195,7 +209,6 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
                                         bool append_to_table) {
 	// In this function, we might scan all table columns,
 	// as we might also append to the table itself (append_to_table).
-
 	auto &table = table_ref.get();
 	if (append_to_table) {
 		table.InitializeAppend(transaction, append_state);

package/src/duckdb/src/storage/metadata/metadata_manager.cpp

@@ -104,7 +104,11 @@ MetadataHandle MetadataManager::Pin(QueryContext context, const MetadataPointer
 	shared_ptr<BlockHandle> block_handle;
 	{
 		lock_guard<mutex> guard(block_lock);
-		auto &block = blocks[UnsafeNumericCast<int64_t>(pointer.block_index)];
+		auto entry = blocks.find(UnsafeNumericCast<int64_t>(pointer.block_index));
+		if (entry == blocks.end()) {
+			throw InternalException("Trying to pin block %llu - but the block did not exist", pointer.block_index);
+		}
+		auto &block = entry->second;
 #ifdef DEBUG
 		for (auto &free_block : block.free_blocks) {
 			if (free_block == pointer.index) {
@@ -272,15 +276,18 @@ void MetadataManager::Flush() {
 			}
 			continue;
 		}
-		auto handle = buffer_manager.Pin(block.block);
+		auto block_handle = block.block;
+		auto handle = buffer_manager.Pin(block_handle);
 		// zero-initialize the few leftover bytes
 		memset(handle.Ptr() + total_metadata_size, 0, block_manager.GetBlockSize() - total_metadata_size);
 		D_ASSERT(kv.first == block.block_id);
-		if (block.block->BlockId() >= MAXIMUM_BLOCK) {
-			auto new_block =
-			    block_manager.ConvertToPersistent(QueryContext(), kv.first, block.block, std::move(handle));
-
+		if (block_handle->BlockId() >= MAXIMUM_BLOCK) {
 			// Convert the temporary block to a persistent block.
+			// we cannot use ConvertToPersistent as another thread might still be reading the block
+			// so we use the safe version of ConvertToPersistent
+			auto new_block = block_manager.ConvertToPersistent(QueryContext(), kv.first, std::move(block_handle),
+			                                                   std::move(handle), ConvertToPersistentMode::THREAD_SAFE);
+
 			guard.lock();
 			block.block = std::move(new_block);
 			guard.unlock();
@@ -366,6 +373,7 @@ void MetadataBlock::FreeBlocksFromInteger(idx_t free_list) {
 }
 
 void MetadataManager::MarkBlocksAsModified() {
+	unique_lock<mutex> guard(block_lock);
 	// for any blocks that were modified in the last checkpoint - set them to free blocks currently
 	for (auto &kv : modified_blocks) {
 		auto block_id = kv.first;
@@ -379,7 +387,10 @@ void MetadataManager::MarkBlocksAsModified() {
 		if (new_free_blocks == NumericLimits<idx_t>::Maximum()) {
 			// if new free_blocks is all blocks - mark entire block as modified
 			blocks.erase(entry);
+
+			guard.unlock();
 			block_manager.MarkBlockAsModified(block_id);
+			guard.lock();
 		} else {
 			// set the new set of free blocks
 			block.FreeBlocksFromInteger(new_free_blocks);
@@ -414,6 +425,18 @@ void MetadataManager::ClearModifiedBlocks(const vector<MetaBlockPointer> &pointe
 	}
 }
 
+bool MetadataManager::BlockHasBeenCleared(const MetaBlockPointer &pointer) {
+	unique_lock<mutex> guard(block_lock);
+	auto block_id = pointer.GetBlockId();
+	auto block_index = pointer.GetBlockIndex();
+	auto entry = modified_blocks.find(block_id);
+	if (entry == modified_blocks.end()) {
+		throw InternalException("BlockHasBeenCleared - Block id %llu not found in modified_blocks", block_id);
+	}
+	auto &modified_list = entry->second;
+	return (modified_list & (1ULL << block_index)) == 0ULL;
+}
+
 vector<MetadataBlockInfo> MetadataManager::GetMetadataInfo() const {
 	vector<MetadataBlockInfo> result;
 	unique_lock<mutex> guard(block_lock);

package/src/duckdb/src/storage/metadata/metadata_reader.cpp

@@ -4,11 +4,8 @@ namespace duckdb {
 
 MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer,
                                optional_ptr<vector<MetaBlockPointer>> read_pointers_p, BlockReaderType type)
-    : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true),
-      read_pointers(read_pointers_p), index(0), offset(0), next_offset(pointer.offset), capacity(0) {
-	if (read_pointers) {
-		read_pointers->push_back(pointer);
-	}
+    : manager(manager), type(type), next_pointer(pointer), has_next_block(true), read_pointers(read_pointers_p),
+      index(0), offset(0), next_offset(pointer.offset), capacity(0) {
 }
 
 MetadataReader::MetadataReader(MetadataManager &manager, BlockPointer pointer)
@@ -59,11 +56,10 @@ MetaBlockPointer MetadataReader::GetMetaBlockPointer() {
 vector<MetaBlockPointer> MetadataReader::GetRemainingBlocks(MetaBlockPointer last_block) {
 	vector<MetaBlockPointer> result;
 	while (has_next_block) {
-		auto next_block_pointer = manager.GetDiskPointer(next_pointer, UnsafeNumericCast<uint32_t>(next_offset));
-		if (last_block.IsValid() && next_block_pointer.block_pointer == last_block.block_pointer) {
+		if (last_block.IsValid() && next_pointer.block_pointer == last_block.block_pointer) {
 			break;
 		}
-		result.push_back(next_block_pointer);
+		result.push_back(next_pointer);
 		ReadNextBlock();
 	}
 	return result;
@@ -77,18 +73,18 @@ void MetadataReader::ReadNextBlock(QueryContext context) {
 	if (!has_next_block) {
 		throw IOException("No more data remaining in MetadataReader");
 	}
-	block = manager.Pin(context, next_pointer);
-	index = next_pointer.index;
+	if (read_pointers) {
+		read_pointers->push_back(next_pointer);
+	}
+	auto next_disk_pointer = FromDiskPointer(next_pointer);
+	block = manager.Pin(context, next_disk_pointer);
+	index = next_disk_pointer.index;
 
 	idx_t next_block = Load<idx_t>(BasePtr());
 	if (next_block == idx_t(-1)) {
 		has_next_block = false;
 	} else {
-		next_pointer = FromDiskPointer(MetaBlockPointer(next_block, 0));
-		MetaBlockPointer next_block_pointer(next_block, 0);
-		if (read_pointers) {
-			read_pointers->push_back(next_block_pointer);
-		}
+		next_pointer = MetaBlockPointer(next_block, 0);
 	}
 	if (next_offset < sizeof(block_id_t)) {
 		next_offset = sizeof(block_id_t);

package/src/duckdb/src/storage/metadata/metadata_writer.cpp

@@ -32,7 +32,7 @@ MetaBlockPointer MetadataWriter::GetMetaBlockPointer() {
 
 void MetadataWriter::SetWrittenPointers(optional_ptr<vector<MetaBlockPointer>> written_pointers_p) {
 	written_pointers = written_pointers_p;
-	if (written_pointers && capacity > 0) {
+	if (written_pointers && capacity > 0 && offset < capacity) {
 		written_pointers->push_back(manager.GetDiskPointer(current_pointer));
 	}
 }

package/src/duckdb/src/storage/serialization/serialize_query_node.cpp

@@ -38,28 +38,10 @@ unique_ptr<QueryNode> QueryNode::Deserialize(Deserializer &deserializer) {
 	}
 	result->modifiers = std::move(modifiers);
 	result->cte_map = std::move(cte_map);
+	ExtractCTENodes(result);
 	return result;
 }
 
-void CTENode::Serialize(Serializer &serializer) const {
-	QueryNode::Serialize(serializer);
-	serializer.WritePropertyWithDefault<string>(200, "cte_name", ctename);
-	serializer.WritePropertyWithDefault<unique_ptr<QueryNode>>(201, "query", query);
-	serializer.WritePropertyWithDefault<unique_ptr<QueryNode>>(202, "child", child);
-	serializer.WritePropertyWithDefault<vector<string>>(203, "aliases", aliases);
-	serializer.WritePropertyWithDefault<CTEMaterialize>(204, "materialized", materialized, CTEMaterialize::CTE_MATERIALIZE_DEFAULT);
-}
-
-unique_ptr<QueryNode> CTENode::Deserialize(Deserializer &deserializer) {
-	auto result = duckdb::unique_ptr<CTENode>(new CTENode());
-	deserializer.ReadPropertyWithDefault<string>(200, "cte_name", result->ctename);
-	deserializer.ReadPropertyWithDefault<unique_ptr<QueryNode>>(201, "query", result->query);
-	deserializer.ReadPropertyWithDefault<unique_ptr<QueryNode>>(202, "child", result->child);
-	deserializer.ReadPropertyWithDefault<vector<string>>(203, "aliases", result->aliases);
-	deserializer.ReadPropertyWithExplicitDefault<CTEMaterialize>(204, "materialized", result->materialized, CTEMaterialize::CTE_MATERIALIZE_DEFAULT);
-	return std::move(result);
-}
-
 void RecursiveCTENode::Serialize(Serializer &serializer) const {
 	QueryNode::Serialize(serializer);
 	serializer.WritePropertyWithDefault<string>(200, "cte_name", ctename);

package/src/duckdb/src/storage/single_file_block_manager.cpp

@@ -66,8 +66,8 @@ void DeserializeEncryptionData(ReadStream &stream, data_t *dest, idx_t size) {
 
 void GenerateDBIdentifier(uint8_t *db_identifier) {
 	memset(db_identifier, 0, MainHeader::DB_IDENTIFIER_LEN);
-	duckdb_mbedtls::MbedTlsWrapper::AESStateMBEDTLS::GenerateRandomDataStatic(db_identifier,
-	                                                                          MainHeader::DB_IDENTIFIER_LEN);
+	RandomEngine engine;
+	engine.RandomData(db_identifier, MainHeader::DB_IDENTIFIER_LEN);
 }
 
 void EncryptCanary(MainHeader &main_header, const shared_ptr<EncryptionState> &encryption_state,
@@ -362,6 +362,15 @@ void SingleFileBlockManager::CheckAndAddEncryptionKey(MainHeader &main_header) {
 void SingleFileBlockManager::CreateNewDatabase(QueryContext context) {
 	auto flags = GetFileFlags(true);
 
+	auto encryption_enabled = options.encryption_options.encryption_enabled;
+	if (encryption_enabled) {
+		if (!db.GetDatabase().GetEncryptionUtil()->SupportsEncryption() && !options.read_only) {
+			throw InvalidConfigurationException(
+			    "The database was opened with encryption enabled, but DuckDB currently has a read-only crypto module "
+			    "loaded. Please re-open using READONLY, or ensure httpfs is loaded using `LOAD httpfs`.");
+		}
+	}
+
 	// open the RDBMS handle
 	auto &fs = FileSystem::Get(db);
 	handle = fs.OpenFile(path, flags);
@@ -376,7 +385,6 @@ void SingleFileBlockManager::CreateNewDatabase(QueryContext context) {
 	// Derive the encryption key and add it to the cache.
 	// Not used for plain databases.
 	data_t derived_key[MainHeader::DEFAULT_ENCRYPTION_KEY_LENGTH];
-	auto encryption_enabled = options.encryption_options.encryption_enabled;
 
 	// We need the unique database identifier, if the storage version is new enough.
 	// If encryption is enabled, we also use it as the salt.
@@ -487,6 +495,15 @@ void SingleFileBlockManager::LoadExistingDatabase(QueryContext context) {
 	if (main_header.IsEncrypted()) {
 		if (options.encryption_options.encryption_enabled) {
 			//! Encryption is set
+
+			//! Check if our encryption module can write, if not, we should throw here
+			if (!db.GetDatabase().GetEncryptionUtil()->SupportsEncryption() && !options.read_only) {
+				throw InvalidConfigurationException(
+				    "The database is encrypted, but DuckDB currently has a read-only crypto module loaded. Either "
+				    "re-open the database using `ATTACH '..' (READONLY)`, or ensure httpfs is loaded using `LOAD "
+				    "httpfs`.");
+			}
+
 			//! Check if the given key upon attach is correct
 			// Derive the encryption key and add it to cache
 			CheckAndAddEncryptionKey(main_header);
@@ -506,6 +523,19 @@ void SingleFileBlockManager::LoadExistingDatabase(QueryContext context) {
 			                       path, EncryptionTypes::CipherToString(config_cipher),
 			                       EncryptionTypes::CipherToString(stored_cipher));
 		}
+
+		// This avoids the cipher from being downgrades by an attacker FIXME: we likely want to have a propervalidation
+		// of the cipher used instead of this trick to avoid downgrades
+		if (stored_cipher != EncryptionTypes::GCM) {
+			if (config_cipher == EncryptionTypes::INVALID) {
+				throw CatalogException(
+				    "Cannot open encrypted database \"%s\" without explicitly specifying the "
+				    "encryption cipher for security reasons. Please make sure you understand the security implications "
+				    "and re-attach the database specifying the desired cipher.",
+				    path);
+			}
+		}
+
 		// this is ugly, but the storage manager does not know the cipher type before
 		db.GetStorageManager().SetCipher(stored_cipher);
 	}

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -542,8 +542,10 @@ unique_ptr<FileBuffer> StandardBufferManager::ReadTemporaryBuffer(QueryContext c
                                                                   BlockHandle &block,
                                                                   unique_ptr<FileBuffer> reusable_buffer) {
 	D_ASSERT(!temporary_directory.path.empty());
-	D_ASSERT(temporary_directory.handle.get());
 	auto id = block.BlockId();
+	if (!temporary_directory.handle) {
+		throw InternalException("ReadTemporaryBuffer called but temporary directory has not been instantiated yet");
+	}
 	if (temporary_directory.handle->GetTempFile().HasTemporaryBuffer(id)) {
 		// This is a block that was offloaded to a regular .tmp file, the file contains blocks of a fixed size
 		return temporary_directory.handle->GetTempFile().ReadTemporaryBuffer(context, id, std::move(reusable_buffer));

package/src/duckdb/src/storage/storage_info.cpp

@@ -83,6 +83,8 @@ static const StorageVersionInfo storage_version_info[] = {
 	{"v1.3.1", 66},
 	{"v1.3.2", 66},
 	{"v1.4.0", 67},
+	{"v1.4.1", 67},
+	{"v1.4.2", 67},
 	{nullptr, 0}
 };
 // END OF STORAGE VERSION INFO
@@ -108,6 +110,8 @@ static const SerializationVersionInfo serialization_version_info[] = {
 	{"v1.3.1", 5},
 	{"v1.3.2", 5},
 	{"v1.4.0", 6},
+	{"v1.4.1", 6},
+	{"v1.4.2", 6},
 	{"latest", 6},
 	{nullptr, 0}
 };

package/src/duckdb/src/storage/storage_manager.cpp

@@ -142,6 +142,14 @@ bool StorageManager::InMemory() const {
 	return path == IN_MEMORY_PATH;
 }
 
+inline void ClearUserKey(shared_ptr<string> const &encryption_key) {
+	if (encryption_key && !encryption_key->empty()) {
+		duckdb_mbedtls::MbedTlsWrapper::AESStateMBEDTLS::SecureClearData(data_ptr_cast(&(*encryption_key)[0]),
+		                                                                 encryption_key->size());
+		encryption_key->clear();
+	}
+}
+
 void StorageManager::Initialize(QueryContext context) {
 	bool in_memory = InMemory();
 	if (in_memory && read_only) {

package/src/duckdb/src/storage/table/array_column_data.cpp

@@ -120,7 +120,7 @@ void ArrayColumnData::Select(TransactionData transaction, idx_t vector_index, Co
 				// not consecutive - break
 				break;
 			}
-			end_idx = next_idx;
+			end_idx = next_idx + 1;
 		}
 		consecutive_ranges++;
 	}

package/src/duckdb/src/storage/table/column_data.cpp

@@ -495,10 +495,10 @@ void ColumnData::InitializeAppend(ColumnAppendState &state) {
 void ColumnData::AppendData(BaseStatistics &append_stats, ColumnAppendState &state, UnifiedVectorFormat &vdata,
                             idx_t append_count) {
 	idx_t offset = 0;
-	this->count += append_count;
 	while (true) {
 		// append the data from the vector
 		idx_t copied_elements = state.current->Append(state, vdata, offset, append_count);
+		this->count += copied_elements;
 		append_stats.Merge(state.current->stats.statistics);
 		if (copied_elements == append_count) {
 			// finished copying everything
@@ -868,7 +868,8 @@ bool PersistentCollectionData::HasUpdates() const {
 }
 
 PersistentColumnData ColumnData::Serialize() {
-	PersistentColumnData result(type.InternalType(), GetDataPointers());
+	auto result = count ? PersistentColumnData(type.InternalType(), GetDataPointers())
+	                    : PersistentColumnData(type.InternalType());
 	result.has_updates = HasUpdates();
 	return result;
 }

package/src/duckdb/src/storage/table/column_data_checkpointer.cpp

@@ -109,9 +109,10 @@ CompressionType ForceCompression(StorageManager &storage_manager,
                                  CompressionType compression_type) {
 	// One of the force_compression flags has been set
 	// check if this compression method is available
-	// if (CompressionTypeIsDeprecated(compression_type, storage_manager)) {
+	// auto compression_availability_result = CompressionTypeIsAvailable(compression_type, storage_manager);
+	// if (!compression_availability_result.IsAvailable()) {
 	//	throw InvalidInputException("The forced compression method (%s) is not available in the current storage
-	// version", 	                            CompressionTypeToString(compression_type));
+	// version", CompressionTypeToString(compression_type));
 	//}
 
 	bool found = false;