duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -28,8 +28,8 @@
 #include "duckdb/storage/checkpoint/table_data_writer.hpp"
 #include "duckdb/storage/metadata/metadata_reader.hpp"
 #include "duckdb/storage/table/column_checkpoint_state.hpp"
-#include "duckdb/transaction/transaction_manager.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
+#include "duckdb/transaction/transaction_manager.hpp"
 
 namespace duckdb {
 
@@ -204,10 +204,38 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 	// finally write the updated header
 	DatabaseHeader header;
 	header.meta_block = meta_block.block_pointer;
-	header.block_size = Storage::BLOCK_ALLOC_SIZE;
+	header.block_alloc_size = block_manager.GetBlockAllocSize();
 	header.vector_size = STANDARD_VECTOR_SIZE;
 	block_manager.WriteHeader(header);
 
+#ifdef DUCKDB_BLOCK_VERIFICATION
+	// extend verify_block_usage_count
+	auto metadata_info = storage_manager.GetMetadataInfo();
+	for (auto &info : metadata_info) {
+		verify_block_usage_count[info.block_id]++;
+	}
+	for (auto &entry_ref : catalog_entries) {
+		auto &entry = entry_ref.get();
+		if (entry.type == CatalogType::TABLE_ENTRY) {
+			auto &table = entry.Cast<DuckTableEntry>();
+			auto &storage = table.GetStorage();
+			auto segment_info = storage.GetColumnSegmentInfo();
+			for (auto &segment : segment_info) {
+				verify_block_usage_count[segment.block_id]++;
+				if (StringUtil::Contains(segment.segment_info, "Overflow String Block Ids: ")) {
+					auto overflow_blocks = StringUtil::Replace(segment.segment_info, "Overflow String Block Ids: ", "");
+					auto splits = StringUtil::Split(overflow_blocks, ", ");
+					for (auto &split : splits) {
+						auto overflow_block_id = std::stoll(split);
+						verify_block_usage_count[overflow_block_id]++;
+					}
+				}
+			}
+		}
+	}
+	block_manager.VerifyBlocks(verify_block_usage_count);
+#endif
+
 	if (config.options.checkpoint_abort == CheckpointAbort::DEBUG_ABORT_BEFORE_TRUNCATE) {
 		throw FatalException("Checkpoint aborted before truncate because of PRAGMA checkpoint_abort flag");
 	}
@@ -243,6 +271,12 @@ void SingleFileCheckpointReader::LoadFromStorage() {
 		return;
 	}
 
+	if (block_manager.IsRemote()) {
+		auto metadata_blocks = metadata_manager.GetBlocks();
+		auto &buffer_manager = BufferManager::GetBufferManager(storage.GetDatabase());
+		buffer_manager.Prefetch(metadata_blocks);
+	}
+
 	// create the MetadataReader to read from the storage
 	MetadataReader reader(metadata_manager, meta_block);
 	auto transaction = CatalogTransaction::GetSystemTransaction(catalog.GetDatabase());
@@ -395,7 +429,6 @@ void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog_entry, Serial
 }
 
 void CheckpointReader::ReadIndex(CatalogTransaction transaction, Deserializer &deserializer) {
-
 	// we need to keep the tag "index", even though it is slightly misleading.
 	auto create_info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(100, "index");
 	auto &info = create_info->Cast<CreateIndexInfo>();
@@ -403,7 +436,7 @@ void CheckpointReader::ReadIndex(CatalogTransaction transaction, Deserializer &d
 	// also, we have to read the root_block_pointer, which will not be valid for newer storage versions.
 	// This leads to different code paths in this function.
 	auto root_block_pointer =
-	    deserializer.ReadPropertyWithDefault<BlockPointer>(101, "root_block_pointer", BlockPointer());
+	    deserializer.ReadPropertyWithExplicitDefault<BlockPointer>(101, "root_block_pointer", BlockPointer());
 
 	// create the index in the catalog
 
@@ -419,27 +452,19 @@ void CheckpointReader::ReadIndex(CatalogTransaction transaction, Deserializer &d
 	}
 
 	// now we can look for the index in the catalog and assign the table info
-	auto &index = catalog.CreateIndex(transaction, info)->Cast<DuckIndexEntry>();
-	auto data_table_info = table.GetStorage().GetDataTableInfo();
-	index.info = make_shared_ptr<IndexDataTableInfo>(data_table_info, info.index_name);
-
-	// insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints
-	for (auto &parsed_expr : info.parsed_expressions) {
-		index.parsed_expressions.push_back(parsed_expr->Copy());
-	}
-	D_ASSERT(!info.parsed_expressions.empty());
+	auto &index = schema.CreateIndex(transaction, info, table)->Cast<DuckIndexEntry>();
 	auto &data_table = table.GetStorage();
 
 	IndexStorageInfo index_storage_info;
 	if (root_block_pointer.IsValid()) {
-		// this code path is necessary to read older duckdb files
-		index_storage_info.name = info.index_name;
+		// Read older duckdb files.
+		index_storage_info.name = index.name;
 		index_storage_info.root_block_ptr = root_block_pointer;
 
 	} else {
-		// get the matching index storage info
+		// Read the matching index storage info.
 		for (auto const &elem : data_table.GetDataTableInfo()->GetIndexStorageInfo()) {
-			if (elem.name == info.index_name) {
+			if (elem.name == index.name) {
 				index_storage_info = elem;
 				break;
 			}
@@ -448,10 +473,9 @@ void CheckpointReader::ReadIndex(CatalogTransaction transaction, Deserializer &d
 
 	D_ASSERT(index_storage_info.IsValid() && !index_storage_info.name.empty());
 
-	// Create an unbound index and add it to the table
+	// Create an unbound index and add it to the table.
 	auto unbound_index = make_uniq<UnboundIndex>(std::move(create_info), index_storage_info,
 	                                             TableIOManager::Get(data_table), data_table.db);
-
 	data_table.GetDataTableInfo()->GetIndexes().AddIndex(std::move(unbound_index));
 }
 
@@ -528,18 +552,19 @@ void CheckpointReader::ReadTableData(CatalogTransaction transaction, Deserialize
 	auto table_pointer = deserializer.ReadProperty<MetaBlockPointer>(101, "table_pointer");
 	auto total_rows = deserializer.ReadProperty<idx_t>(102, "total_rows");
 
-	// old file read
-	auto index_pointers = deserializer.ReadPropertyWithDefault<vector<BlockPointer>>(103, "index_pointers", {});
-	// new file read
+	// Cover reading old storage files.
+	auto index_pointers = deserializer.ReadPropertyWithExplicitDefault<vector<BlockPointer>>(103, "index_pointers", {});
+	// Cover reading new storage files.
 	auto index_storage_infos =
-	    deserializer.ReadPropertyWithDefault<vector<IndexStorageInfo>>(104, "index_storage_infos", {});
+	    deserializer.ReadPropertyWithExplicitDefault<vector<IndexStorageInfo>>(104, "index_storage_infos", {});
 
 	if (!index_storage_infos.empty()) {
 		bound_info.indexes = index_storage_infos;
 
 	} else {
-		// old duckdb file containing index pointers
+		// This is an old duckdb file containing index pointers and deprecated storage.
 		for (idx_t i = 0; i < index_pointers.size(); i++) {
+			// Deprecated storage is always true for old duckdb files.
 			IndexStorageInfo index_storage_info;
 			index_storage_info.root_block_ptr = index_pointers[i];
 			bound_info.indexes.push_back(index_storage_info);

package/src/duckdb/src/storage/compression/alp/alp.cpp

@@ -1,14 +1,9 @@
-#include "duckdb/storage/compression/alp/alp_compress.hpp"
-#include "duckdb/storage/compression/alp/alp_scan.hpp"
-#include "duckdb/storage/compression/alp/alp_fetch.hpp"
-#include "duckdb/storage/compression/alp/alp_analyze.hpp"
-
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/function/compression_function.hpp"
-#include "duckdb/main/config.hpp"
-
-#include <cmath>
-#include <functional>
+#include "duckdb/storage/compression/alp/alp_analyze.hpp"
+#include "duckdb/storage/compression/alp/alp_compress.hpp"
+#include "duckdb/storage/compression/alp/alp_fetch.hpp"
+#include "duckdb/storage/compression/alp/alp_scan.hpp"
 
 namespace duckdb {
 
@@ -44,8 +39,8 @@ CompressionFunction AlpCompressionFun::GetFunction(PhysicalType type) {
 	}
 }
 
-bool AlpCompressionFun::TypeIsSupported(PhysicalType type) {
-	switch (type) {
+bool AlpCompressionFun::TypeIsSupported(const PhysicalType physical_type) {
+	switch (physical_type) {
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
 		return true;

package/src/duckdb/src/storage/compression/alprd.cpp

@@ -1,15 +1,11 @@
+#include "duckdb/common/limits.hpp"
+#include "duckdb/function/compression/compression.hpp"
+#include "duckdb/function/compression_function.hpp"
 #include "duckdb/storage/compression/alprd/alprd_analyze.hpp"
 #include "duckdb/storage/compression/alprd/alprd_compress.hpp"
 #include "duckdb/storage/compression/alprd/alprd_fetch.hpp"
 #include "duckdb/storage/compression/alprd/alprd_scan.hpp"
 
-#include "duckdb/common/limits.hpp"
-#include "duckdb/function/compression/compression.hpp"
-#include "duckdb/function/compression_function.hpp"
-
-#include <cmath>
-#include <functional>
-
 namespace duckdb {
 
 template <class T>
@@ -44,8 +40,8 @@ CompressionFunction AlpRDCompressionFun::GetFunction(PhysicalType type) {
 	}
 }
 
-bool AlpRDCompressionFun::TypeIsSupported(PhysicalType type) {
-	switch (type) {
+bool AlpRDCompressionFun::TypeIsSupported(const PhysicalType physical_type) {
+	switch (physical_type) {
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
 		return true;

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -3,9 +3,9 @@
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/common/operator/add.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
 #include "duckdb/common/operator/multiply.hpp"
 #include "duckdb/common/operator/subtract.hpp"
-#include "duckdb/common/operator/cast_operators.hpp"
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/function/compression_function.hpp"
 #include "duckdb/main/config.hpp"
@@ -217,7 +217,7 @@ public:
 
 	template <class T_INNER>
 	void SubtractFrameOfReference(T_INNER *buffer, T_INNER frame_of_reference) {
-		static_assert(IsIntegral<T_INNER>::value, "Integral type required.");
+		static_assert(NumericLimits<T_INNER>::IsIntegral(), "Integral type required.");
 
 		using T_U = typename MakeUnsigned<T_INNER>::type;
 
@@ -317,6 +317,7 @@ public:
 //===--------------------------------------------------------------------===//
 template <class T>
 struct BitpackingAnalyzeState : public AnalyzeState {
+	explicit BitpackingAnalyzeState(const CompressionInfo &info) : AnalyzeState(info) {};
 	BitpackingState<T> state;
 };
 
@@ -324,7 +325,8 @@ template <class T>
 unique_ptr<AnalyzeState> BitpackingInitAnalyze(ColumnData &col_data, PhysicalType type) {
 	auto &config = DBConfig::GetConfig(col_data.GetDatabase());
 
-	auto state = make_uniq<BitpackingAnalyzeState<T>>();
+	CompressionInfo info(col_data.GetBlockManager().GetBlockSize());
+	auto state = make_uniq<BitpackingAnalyzeState<T>>(info);
 	state->state.mode = config.options.force_bitpacking_mode;
 
 	return std::move(state);
@@ -333,6 +335,15 @@ unique_ptr<AnalyzeState> BitpackingInitAnalyze(ColumnData &col_data, PhysicalTyp
 template <class T>
 bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
 	auto &analyze_state = state.Cast<BitpackingAnalyzeState<T>>();
+
+	// We use BITPACKING_METADATA_GROUP_SIZE tuples, which can exceed the block size.
+	// In that case, we disable bitpacking.
+	// we are conservative here by multiplying by 2
+	auto type_size = GetTypeIdSize(input.GetType().InternalType());
+	if (type_size * BITPACKING_METADATA_GROUP_SIZE * 2 > state.info.GetBlockSize()) {
+		return false;
+	}
+
 	UnifiedVectorFormat vdata;
 	input.ToUnifiedFormat(count, vdata);
 
@@ -362,8 +373,8 @@ idx_t BitpackingFinalAnalyze(AnalyzeState &state) {
 template <class T, bool WRITE_STATISTICS, class T_S = typename MakeSigned<T>::type>
 struct BitpackingCompressState : public CompressionState {
 public:
-	explicit BitpackingCompressState(ColumnDataCheckpointer &checkpointer)
-	    : checkpointer(checkpointer),
+	explicit BitpackingCompressState(ColumnDataCheckpointer &checkpointer, const CompressionInfo &info)
+	    : CompressionState(info), checkpointer(checkpointer),
 	      function(checkpointer.GetCompressionFunction(CompressionType::COMPRESSION_BITPACKING)) {
 		CreateEmptySegment(checkpointer.GetRowGroup().start);
 
@@ -465,31 +476,34 @@ public:
 			state->current_segment->count += count;
 
 			if (WRITE_STATISTICS && !state->state.all_invalid) {
-				NumericStats::Update<T>(state->current_segment->stats.statistics, state->state.minimum);
-				NumericStats::Update<T>(state->current_segment->stats.statistics, state->state.maximum);
+				state->current_segment->stats.statistics.template UpdateNumericStats<T>(state->state.maximum);
+				state->current_segment->stats.statistics.template UpdateNumericStats<T>(state->state.minimum);
 			}
 		}
 	};
 
 	bool CanStore(idx_t data_bytes, idx_t meta_bytes) {
 		auto required_data_bytes = AlignValue<idx_t>(UnsafeNumericCast<idx_t>((data_ptr + data_bytes) - data_ptr));
-		auto required_meta_bytes = Storage::BLOCK_SIZE - UnsafeNumericCast<idx_t>(metadata_ptr - data_ptr) + meta_bytes;
+		auto required_meta_bytes = info.GetBlockSize() - UnsafeNumericCast<idx_t>(metadata_ptr - data_ptr) + meta_bytes;
 
 		return required_data_bytes + required_meta_bytes <=
-		       Storage::BLOCK_SIZE - BitpackingPrimitives::BITPACKING_HEADER_SIZE;
+		       info.GetBlockSize() - BitpackingPrimitives::BITPACKING_HEADER_SIZE;
 	}
 
 	void CreateEmptySegment(idx_t row_start) {
 		auto &db = checkpointer.GetDatabase();
 		auto &type = checkpointer.GetType();
-		auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
+
+		auto compressed_segment =
+		    ColumnSegment::CreateTransientSegment(db, type, row_start, info.GetBlockSize(), info.GetBlockSize());
 		compressed_segment->function = function;
 		current_segment = std::move(compressed_segment);
+
 		auto &buffer_manager = BufferManager::GetBufferManager(db);
 		handle = buffer_manager.Pin(current_segment->block);
 
 		data_ptr = handle.Ptr() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
-		metadata_ptr = handle.Ptr() + Storage::BLOCK_SIZE;
+		metadata_ptr = handle.Ptr() + info.GetBlockSize();
 	}
 
 	void Append(UnifiedVectorFormat &vdata, idx_t count) {
@@ -518,7 +532,7 @@ public:
 
 		idx_t unaligned_offset = NumericCast<idx_t>(data_ptr - base_ptr);
 		idx_t metadata_offset = AlignValue(unaligned_offset);
-		idx_t metadata_size = NumericCast<idx_t>(base_ptr + Storage::BLOCK_SIZE - metadata_ptr);
+		idx_t metadata_size = NumericCast<idx_t>(base_ptr + info.GetBlockSize() - metadata_ptr);
 		idx_t total_segment_size = metadata_offset + metadata_size;
 
 		// Asserting things are still sane here
@@ -549,7 +563,7 @@ public:
 template <class T, bool WRITE_STATISTICS>
 unique_ptr<CompressionState> BitpackingInitCompression(ColumnDataCheckpointer &checkpointer,
                                                        unique_ptr<AnalyzeState> state) {
-	return make_uniq<BitpackingCompressState<T, WRITE_STATISTICS>>(checkpointer);
+	return make_uniq<BitpackingCompressState<T, WRITE_STATISTICS>>(checkpointer, state->info);
 }
 
 template <class T, bool WRITE_STATISTICS>
@@ -613,12 +627,12 @@ public:
 	explicit BitpackingScanState(ColumnSegment &segment) : current_segment(segment) {
 		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 		handle = buffer_manager.Pin(segment.block);
-		auto dataptr = handle.Ptr();
+		auto data_ptr = handle.Ptr();
 
 		// load offset to bitpacking widths pointer
-		auto bitpacking_metadata_offset = Load<idx_t>(dataptr + segment.GetBlockOffset());
+		auto bitpacking_metadata_offset = Load<idx_t>(data_ptr + segment.GetBlockOffset());
 		bitpacking_metadata_ptr =
-		    dataptr + segment.GetBlockOffset() + bitpacking_metadata_offset - sizeof(bitpacking_metadata_encoded_t);
+		    data_ptr + segment.GetBlockOffset() + bitpacking_metadata_offset - sizeof(bitpacking_metadata_encoded_t);
 
 		// load the first group
 		LoadNextGroup();
@@ -630,7 +644,6 @@ public:
 	T decompression_buffer[BITPACKING_METADATA_GROUP_SIZE];
 
 	bitpacking_metadata_t current_group;
-
 	bitpacking_width_t current_width;
 	T current_frame_of_reference;
 	T current_constant;
@@ -642,11 +655,11 @@ public:
 
 public:
 	//! Loads the metadata for the current metadata group. This will set bitpacking_metadata_ptr to the next group.
-	//! this will also load any metadata that is at the start of a compressed buffer (e.g. the width, for, or constant
-	//! value) depending on the bitpacking mode for that group
+	//! It also loads any metadata at the start of a compressed buffer (e.g. the width, for, or constant value)
+	//! depending on the bitpacking mode of that group.
 	void LoadNextGroup() {
 		D_ASSERT(bitpacking_metadata_ptr > handle.Ptr() &&
-		         bitpacking_metadata_ptr < handle.Ptr() + Storage::BLOCK_SIZE);
+		         bitpacking_metadata_ptr < handle.Ptr() + current_segment.GetBlockManager().GetBlockSize());
 		current_group_offset = 0;
 		current_group = DecodeMeta(reinterpret_cast<bitpacking_metadata_encoded_t *>(bitpacking_metadata_ptr));
 
@@ -976,17 +989,8 @@ CompressionFunction BitpackingFun::GetFunction(PhysicalType type) {
 	}
 }
 
-bool BitpackingFun::TypeIsSupported(PhysicalType type) {
-
-	// we calculate on BITPACKING_METADATA_GROUP_SIZE tuples, but they can exceed the block size,
-	// in which case we have to disable bitpacking for that data type
-	// we are conservative here by multiplying by 2
-	auto type_size = GetTypeIdSize(type);
-	if (type_size * BITPACKING_METADATA_GROUP_SIZE * 2 > Storage::BLOCK_SIZE) {
-		return false;
-	}
-
-	switch (type) {
+bool BitpackingFun::TypeIsSupported(const PhysicalType physical_type) {
+	switch (physical_type) {
 	case PhysicalType::BOOL:
 	case PhysicalType::INT8:
 	case PhysicalType::INT16:

package/src/duckdb/src/storage/compression/chimp/chimp.cpp

@@ -1,11 +1,9 @@
-#include "duckdb/storage/compression/chimp/chimp.hpp"
-#include "duckdb/storage/compression/chimp/chimp_compress.hpp"
-#include "duckdb/storage/compression/chimp/chimp_scan.hpp"
-#include "duckdb/storage/compression/chimp/chimp_fetch.hpp"
-#include "duckdb/storage/compression/chimp/chimp_analyze.hpp"
-
 #include "duckdb/function/compression/compression.hpp"
 #include "duckdb/function/compression_function.hpp"
+#include "duckdb/storage/compression/chimp/chimp_analyze.hpp"
+#include "duckdb/storage/compression/chimp/chimp_compress.hpp"
+#include "duckdb/storage/compression/chimp/chimp_fetch.hpp"
+#include "duckdb/storage/compression/chimp/chimp_scan.hpp"
 
 namespace duckdb {
 
@@ -28,8 +26,8 @@ CompressionFunction ChimpCompressionFun::GetFunction(PhysicalType type) {
 	}
 }
 
-bool ChimpCompressionFun::TypeIsSupported(PhysicalType type) {
-	switch (type) {
+bool ChimpCompressionFun::TypeIsSupported(const PhysicalType physical_type) {
+	switch (physical_type) {
 	case PhysicalType::FLOAT:
 	case PhysicalType::DOUBLE:
 		return true;