duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/storage/index_storage_info.hpp

@@ -8,10 +8,11 @@
 
 #pragma once
 
+#include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/types/value.hpp"
+#include "duckdb/common/unordered_set.hpp"
 #include "duckdb/storage/block.hpp"
 #include "duckdb/storage/storage_info.hpp"
-#include "duckdb/common/unordered_set.hpp"
 
 namespace duckdb {
 
@@ -38,26 +39,29 @@ struct IndexBufferInfo {
 	idx_t allocation_size;
 };
 
-//! Information to serialize an index
+//! Index (de)serialization information.
 struct IndexStorageInfo {
 	IndexStorageInfo() {};
-	explicit IndexStorageInfo(string name) : name(std::move(name)) {};
+	explicit IndexStorageInfo(const string &name) : name(name) {};
 
-	//! The name of the index
+	//! The name.
 	string name;
-	//! The root of the index
+	//! The storage root.
 	idx_t root;
-	//! Information to serialize the index memory held by the fixed-size allocators
+	//! Any index specialization can provide additional key-Value settings via this map.
+	case_insensitive_map_t<Value> options;
+	//! Serialization information for fixed-size allocator memory.
 	vector<FixedSizeAllocatorInfo> allocator_infos;
 
-	//! Contains all buffer pointers and their allocation size for serializing to the WAL
-	//! First dimension: all fixed-size allocators, second dimension: the buffers of each allocator
+	//! Contains all buffer pointers and their allocation size for serializing to the WAL.
+	//! First dimension: All fixed-size allocators.
+	//! Second dimension: The buffers of each fixed-size allocator.
 	vector<vector<IndexBufferInfo>> buffers;
 
-	//! The root block pointer of the index, which is necessary to support older storage files
+	//! The root block pointer of the index. Necessary to support older storage files.
 	BlockPointer root_block_ptr;
 
-	//! Returns true, if the struct contains index information
+	//! Returns true, if IndexStorageInfo holds information to deserialize an index.
 	bool IsValid() const {
 		return root_block_ptr.IsValid() || !allocator_infos.empty();
 	}

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -44,8 +44,6 @@ class MetadataManager {
 public:
 	//! The amount of metadata blocks per storage block
 	static constexpr const idx_t METADATA_BLOCK_COUNT = 64;
-	//! The size of metadata blocks
-	static constexpr const idx_t METADATA_BLOCK_SIZE = AlignValueFloor(Storage::BLOCK_SIZE / METADATA_BLOCK_COUNT);
 
 public:
 	MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager);
@@ -58,8 +56,8 @@ public:
 	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
 	MetadataPointer RegisterDiskPointer(MetaBlockPointer pointer);
 
-	static BlockPointer ToBlockPointer(MetaBlockPointer meta_pointer);
-	static MetaBlockPointer FromBlockPointer(BlockPointer block_pointer);
+	static BlockPointer ToBlockPointer(MetaBlockPointer meta_pointer, const idx_t metadata_block_size);
+	static MetaBlockPointer FromBlockPointer(BlockPointer block_pointer, const idx_t metadata_block_size);
 
 	//! Flush all blocks to disk
 	void Flush();
@@ -68,11 +66,14 @@ public:
 	void ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers);
 
 	vector<MetadataBlockInfo> GetMetadataInfo() const;
+	vector<shared_ptr<BlockHandle>> GetBlocks() const;
 	idx_t BlockCount();
 
 	void Write(WriteStream &sink);
 	void Read(ReadStream &source);
 
+	idx_t GetMetadataBlockSize() const;
+
 protected:
 	BlockManager &block_manager;
 	BufferManager &buffer_manager;
@@ -81,6 +82,7 @@ protected:
 
 protected:
 	block_id_t AllocateNewBlock();
+	block_id_t PeekNextBlockId();
 	block_id_t GetNextBlockId();
 
 	void AddBlock(MetadataBlock new_block, bool if_exists = false);
@@ -88,8 +90,4 @@ protected:
 	void ConvertToTransient(MetadataBlock &block);
 };
 
-//! Detect mismatching constant values
-static_assert(MetadataManager::METADATA_BLOCK_SIZE * MetadataManager::METADATA_BLOCK_COUNT <= Storage::BLOCK_SIZE,
-              "metadata block count exceeds total block alloc size");
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -92,8 +92,6 @@ enum class PartialBlockType { FULL_CHECKPOINT, APPEND_TO_TABLE };
 //! In any case, they must share a block manager.
 class PartialBlockManager {
 public:
-	//! 20% free / 80% utilization
-	static constexpr const idx_t DEFAULT_MAX_PARTIAL_BLOCK_SIZE = Storage::BLOCK_SIZE / 5 * 4;
 	//! Max number of shared references to a block. No effective limit by default.
 	static constexpr const idx_t DEFAULT_MAX_USE_COUNT = 1u << 20;
 	//! No point letting map size grow unbounded. We'll drop blocks with the
@@ -102,7 +100,7 @@ public:
 
 public:
 	PartialBlockManager(BlockManager &block_manager, PartialBlockType partial_block_type,
-	                    uint32_t max_partial_block_size = DEFAULT_MAX_PARTIAL_BLOCK_SIZE,
+	                    optional_idx max_partial_block_size = optional_idx(),
 	                    uint32_t max_use_count = DEFAULT_MAX_USE_COUNT);
 	virtual ~PartialBlockManager();
 
@@ -128,6 +126,12 @@ public:
 		return unique_lock<mutex>(partial_block_lock);
 	}
 
+	//! Returns a reference to the underlying block manager.
+	BlockManager &GetBlockManager() const;
+
+	//! Registers a block as "written" by this partial block manager
+	void AddWrittenBlock(block_id_t block);
+
 protected:
 	BlockManager &block_manager;
 	PartialBlockType partial_block_type;
@@ -151,7 +155,6 @@ protected:
 	bool GetPartialBlock(idx_t segment_size, unique_ptr<PartialBlock> &state);
 
 	bool HasBlockAllocation(uint32_t segment_size);
-	void AddWrittenBlock(block_id_t block);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/segment/uncompressed.hpp

@@ -38,16 +38,13 @@ public:
 struct StringUncompressed {
 public:
 	static CompressionFunction GetFunction(PhysicalType data_type);
+	static idx_t GetStringBlockLimit(const idx_t block_size) {
+		return MinValue(AlignValueFloor(block_size / 4), DEFAULT_STRING_BLOCK_LIMIT);
+	}
 
 public:
 	//! The default maximum string size for sufficiently big block sizes
-	static constexpr uint16_t DEFAULT_STRING_BLOCK_LIMIT = 4096;
-	//! The maximum string size within a block. We offload bigger strings to overflow blocks
-	static constexpr uint16_t STRING_BLOCK_LIMIT =
-	    MinValue(AlignValueFloor(Storage::BLOCK_SIZE / 4), idx_t(DEFAULT_STRING_BLOCK_LIMIT));
+	static constexpr idx_t DEFAULT_STRING_BLOCK_LIMIT = 4096;
 };
 
-//! Detect mismatching constant values
-static_assert(StringUncompressed::STRING_BLOCK_LIMIT != 0, "the string block limit cannot be 0");
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -26,6 +26,7 @@ struct StorageManagerOptions {
 	bool read_only = false;
 	bool use_direct_io = false;
 	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
+	optional_idx block_alloc_size = optional_idx();
 };
 
 //! SingleFileBlockManager is an implementation for a BlockManager which manages blocks in a single file
@@ -34,10 +35,13 @@ class SingleFileBlockManager : public BlockManager {
 	static constexpr uint64_t BLOCK_START = Storage::FILE_HEADER_SIZE * 3;
 
 public:
-	SingleFileBlockManager(AttachedDatabase &db, string path, StorageManagerOptions options);
+	SingleFileBlockManager(AttachedDatabase &db, const string &path, const StorageManagerOptions &options);
 
 	FileOpenFlags GetFileFlags(bool create_new) const;
+	//! Creates a new database.
 	void CreateNewDatabase();
+	//! Loads an existing database. We pass the provided block allocation size as a parameter
+	//! to detect inconsistencies with the file header.
 	void LoadExistingDatabase();
 
 	//! Creates a new Block using the specified block_id and returns a pointer
@@ -45,10 +49,14 @@ public:
 	unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) override;
 	//! Return the next free block id
 	block_id_t GetFreeBlockId() override;
+	//! Check the next free block id - but do not assign or allocate it
+	block_id_t PeekFreeBlockId() override;
 	//! Returns whether or not a specified block is the root block
 	bool IsRootBlock(MetaBlockPointer root) override;
 	//! Mark a block as free (immediately re-writeable)
 	void MarkBlockAsFree(block_id_t block_id) override;
+	//! Mark a block as used (no longer re-writeable)
+	void MarkBlockAsUsed(block_id_t block_id) override;
 	//! Mark a block as modified (re-writeable after a checkpoint)
 	void MarkBlockAsModified(block_id_t block_id) override;
 	//! Increase the reference count of a block. The block should hold at least one reference
@@ -57,31 +65,48 @@ public:
 	idx_t GetMetaBlock() override;
 	//! Read the content of the block from disk
 	void Read(Block &block) override;
+	//! Read the content of a range of blocks into a buffer
+	void ReadBlocks(FileBuffer &buffer, block_id_t start_block, idx_t block_count) override;
 	//! Write the given block to disk
 	void Write(FileBuffer &block, block_id_t block_id) override;
 	//! Write the header to disk, this is the final step of the checkpointing process
 	void WriteHeader(DatabaseHeader header) override;
+	//! Sync changes to the underlying file
+	void FileSync() override;
 	//! Truncate the underlying database file after a checkpoint
 	void Truncate() override;
 
+	bool InMemory() override {
+		return false;
+	}
 	//! Returns the number of total blocks
 	idx_t TotalBlocks() override;
 	//! Returns the number of free blocks
 	idx_t FreeBlocks() override;
+	//! Whether or not the attached database is a remote file
+	bool IsRemote() override;
 
 private:
-	//! Load the free list from the file
+	//! Loads the free list of the file.
 	void LoadFreeList();
-
-	void Initialize(DatabaseHeader &header);
+	//! Initializes the database header. We pass the provided block allocation size as a parameter
+	//!	to detect inconsistencies with the file header.
+	void Initialize(const DatabaseHeader &header, const optional_idx block_alloc_size);
 
 	void ReadAndChecksum(FileBuffer &handle, uint64_t location) const;
 	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
 
+	idx_t GetBlockLocation(block_id_t block_id);
+
 	//! Return the blocks to which we will write the free list and modified blocks
 	vector<MetadataHandle> GetFreeListBlocks();
 	void TrimFreeBlocks();
 
+	void IncreaseBlockReferenceCountInternal(block_id_t block_id);
+
+	//! Verify the block usage count
+	void VerifyBlocks(const unordered_map<block_id_t, idx_t> &block_usage_count) override;
+
 private:
 	AttachedDatabase &db;
 	//! The active DatabaseHeader, either 0 (h1) or 1 (h2)

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -43,15 +43,22 @@ public:
 	static unique_ptr<FileBuffer> ReadTemporaryBufferInternal(BufferManager &buffer_manager, FileHandle &handle,
 	                                                          idx_t position, idx_t size,
 	                                                          unique_ptr<FileBuffer> reusable_buffer);
-	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
-	//! This buffer can be small (smaller than BLOCK_SIZE)
-	//! Unpin and pin are nops on this block of memory
-	shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size) final;
+
+	//! Registers a transient memory buffer.
+	shared_ptr<BlockHandle> RegisterTransientMemory(const idx_t size, const idx_t block_size) final;
+	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed.
+	//! This buffer can be small (smaller than the block size of the temporary block manager).
+	//! Unpin and Pin are NOPs on this block of memory.
+	shared_ptr<BlockHandle> RegisterSmallMemory(const idx_t size) final;
 
 	idx_t GetUsedMemory() const final;
 	idx_t GetMaxMemory() const final;
 	idx_t GetUsedSwap() final;
 	optional_idx GetMaxSwap() const final;
+	//! Returns the block allocation size for buffer-managed blocks.
+	idx_t GetBlockAllocSize() const final;
+	//! Returns the block size for buffer-managed blocks.
+	idx_t GetBlockSize() const final;
 
 	//! Allocate an in-memory buffer with a single pin.
 	//! The allocated memory is released when the buffer handle is destroyed.
@@ -62,6 +69,7 @@ public:
 	void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) final;
 
 	BufferHandle Pin(shared_ptr<BlockHandle> &handle) final;
+	void Prefetch(vector<shared_ptr<BlockHandle>> &handles) final;
 	void Unpin(shared_ptr<BlockHandle> &handle) final;
 
 	//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
@@ -102,14 +110,18 @@ protected:
 	                                             ARGS...);
 
 	//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
-	//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
-	//! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
+	//! not the buffer can be destroyed instead of evicted,
+	//! if true, it will be destroyed,
+	//! if false, it will be written to a temporary file so it can be reloaded
+	//! If we want to change this, e.g., to immediately destroy the buffer upon unpinning,
+	//! we can call BlockHandle::SetDestroyBufferUpon
+	//! The resulting buffer will already be allocated, but needs to be pinned in order to be used.
 	//! This needs to be private to prevent creating blocks without ever pinning them:
 	//! blocks that are never pinned are never added to the eviction queue
 	shared_ptr<BlockHandle> RegisterMemory(MemoryTag tag, idx_t block_size, bool can_destroy);
 
 	//! Garbage collect eviction queue
-	void PurgeQueue() final;
+	void PurgeQueue(FileBufferType type) final;
 
 	BufferPool &GetBufferPool() const final;
 	TemporaryMemoryManager &GetTemporaryMemoryManager() final;
@@ -139,6 +151,9 @@ protected:
 	//! overwrites the data within with garbage. Any readers that do not hold the pin will notice
 	void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
 
+	void BatchRead(vector<shared_ptr<BlockHandle>> &handles, const map<block_id_t, idx_t> &load_map,
+	               block_id_t first_block, block_id_t last_block);
+
 protected:
 	// These are stored here because temp_directory creation is lazy
 	// so we need to store information related to the temporary directory before it's created

package/src/duckdb/src/include/duckdb/storage/statistics/base_statistics.hpp

@@ -9,9 +9,9 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
-#include "duckdb/common/types.hpp"
-#include "duckdb/common/operator/comparison_operators.hpp"
 #include "duckdb/common/enums/expression_type.hpp"
+#include "duckdb/common/operator/comparison_operators.hpp"
+#include "duckdb/common/types.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/storage/statistics/numeric_stats.hpp"
 #include "duckdb/storage/statistics/string_stats.hpp"
@@ -112,6 +112,12 @@ public:
 	idx_t GetDistinctCount();
 	static BaseStatistics FromConstant(const Value &input);
 
+	template <class T>
+	void UpdateNumericStats(T new_value) {
+		D_ASSERT(GetStatsType() == StatisticsType::NUMERIC_STATS);
+		NumericStats::Update(stats_union.numeric_data, new_value);
+	}
+
 private:
 	BaseStatistics();
 	explicit BaseStatistics(LogicalType type);
@@ -145,4 +151,11 @@ private:
 	unsafe_unique_array<BaseStatistics> child_stats;
 };
 
+template <>
+inline void BaseStatistics::UpdateNumericStats<interval_t>(interval_t new_value) {
+}
+template <>
+inline void BaseStatistics::UpdateNumericStats<list_entry_t>(list_entry_t new_value) {
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp

@@ -9,8 +9,8 @@
 #pragma once
 
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/mutex.hpp"
 #include "duckdb/common/types/hyperloglog.hpp"
-#include "duckdb/storage/statistics/base_statistics.hpp"
 
 namespace duckdb {
 class Vector;
@@ -47,7 +47,13 @@ public:
 
 private:
 	//! For distinct statistics we sample the input to speed up insertions
-	static constexpr const double SAMPLE_RATE = 0.1;
+	static constexpr double BASE_SAMPLE_RATE = 0.1;
+	//! For integers, we sample more: likely to be join keys (and hashing is cheaper than, e.g., strings)
+	static constexpr double INTEGRAL_SAMPLE_RATE = 0.3;
+	//! For concurrent access
+	mutable mutex lock;
+	//! Preallocated vector for hashes
+	Vector hash_vec;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp

@@ -8,11 +8,11 @@
 
 #pragma once
 
-#include "duckdb/storage/statistics/numeric_stats_union.hpp"
-#include "duckdb/common/enums/filter_propagate_result.hpp"
 #include "duckdb/common/enums/expression_type.hpp"
+#include "duckdb/common/enums/filter_propagate_result.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
 #include "duckdb/common/types/value.hpp"
+#include "duckdb/storage/statistics/numeric_stats_union.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -66,17 +66,11 @@ struct NumericStats {
 
 	template <class T>
 	static inline void UpdateValue(T new_value, T &min, T &max) {
-		if (LessThan::Operation(new_value, min)) {
-			min = new_value;
-		}
-		if (GreaterThan::Operation(new_value, max)) {
-			max = new_value;
-		}
+		min = LessThan::Operation(new_value, min) ? new_value : min;
+		max = GreaterThan::Operation(new_value, max) ? new_value : max;
 	}
-
 	template <class T>
-	static inline void Update(BaseStatistics &stats, T new_value) {
-		auto &nstats = NumericStats::GetDataUnsafe(stats);
+	static inline void Update(NumericStatsData &nstats, T new_value) {
 		UpdateValue<T>(new_value, nstats.min.GetReferenceUnsafe<T>(), nstats.max.GetReferenceUnsafe<T>());
 	}
 
@@ -104,9 +98,4 @@ private:
 	static void TemplatedVerify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
 };
 
-template <>
-void NumericStats::Update<interval_t>(BaseStatistics &stats, interval_t new_value);
-template <>
-void NumericStats::Update<list_entry_t>(BaseStatistics &stats, list_entry_t new_value);
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp

@@ -36,30 +36,68 @@ struct NumericValueUnion {
 };
 
 template <>
-DUCKDB_API bool &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline bool &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.boolean;
+}
+
 template <>
-DUCKDB_API int8_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline int8_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.tinyint;
+}
+
 template <>
-DUCKDB_API int16_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline int16_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.smallint;
+}
+
 template <>
-DUCKDB_API int32_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline int32_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.integer;
+}
+
 template <>
-DUCKDB_API int64_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline int64_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.bigint;
+}
+
 template <>
-DUCKDB_API hugeint_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline hugeint_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.hugeint;
+}
+
 template <>
-DUCKDB_API uhugeint_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline uhugeint_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.uhugeint;
+}
+
 template <>
-DUCKDB_API uint8_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline uint8_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.utinyint;
+}
+
 template <>
-DUCKDB_API uint16_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline uint16_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.usmallint;
+}
+
 template <>
-DUCKDB_API uint32_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline uint32_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.uinteger;
+}
+
 template <>
-DUCKDB_API uint64_t &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline uint64_t &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.ubigint;
+}
+
 template <>
-DUCKDB_API float &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline float &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.float_;
+}
+
 template <>
-DUCKDB_API double &NumericValueUnion::GetReferenceUnsafe();
+DUCKDB_API inline double &NumericValueUnion::GetReferenceUnsafe() {
+	return value_.double_;
+}
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -9,11 +9,11 @@
 #pragma once
 
 #include "duckdb/common/common.hpp"
-#include "duckdb/common/exception.hpp"
-#include "duckdb/common/types/hugeint.hpp"
-#include "duckdb/common/enums/filter_propagate_result.hpp"
 #include "duckdb/common/enums/expression_type.hpp"
+#include "duckdb/common/enums/filter_propagate_result.hpp"
+#include "duckdb/common/exception.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
+#include "duckdb/common/types/hugeint.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -63,6 +63,9 @@ struct StringStats {
 
 	DUCKDB_API static FilterPropagateResult CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,
 	                                                     const string &value);
+	DUCKDB_API static FilterPropagateResult CheckZonemap(const_data_ptr_t min_data, idx_t min_len,
+	                                                     const_data_ptr_t max_data, idx_t max_len,
+	                                                     ExpressionType comparison_type, const string &value);
 
 	DUCKDB_API static void Update(BaseStatistics &stats, const string_t &value);
 	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -9,9 +9,9 @@
 #pragma once
 
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/limits.hpp"
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/vector_size.hpp"
-#include "duckdb/common/limits.hpp"
 
 namespace duckdb {
 struct FileHandle;
@@ -22,21 +22,19 @@ struct FileHandle;
 #define INVALID_BLOCK (-1)
 //! The maximum block id is 2^62
 #define MAXIMUM_BLOCK 4611686018427388000LL
-//! The default block size
-#define DEFAULT_BLOCK_ALLOC_SIZE 262144
+
+//! The default block allocation size.
+#define DEFAULT_BLOCK_ALLOC_SIZE 262144ULL
+//! The configurable block allocation size.
+#ifndef DUCKDB_BLOCK_ALLOC_SIZE
+#define DUCKDB_BLOCK_ALLOC_SIZE DEFAULT_BLOCK_ALLOC_SIZE
+#endif
 
 using block_id_t = int64_t;
 
 struct Storage {
 	//! The size of a hard disk sector, only really needed for Direct IO
 	constexpr static idx_t SECTOR_SIZE = 4096U;
-	//! Block header size for blocks written to the storage
-	constexpr static idx_t BLOCK_HEADER_SIZE = sizeof(uint64_t);
-	//! Size of a memory slot managed by the StorageManager. This is the quantum of allocation for Blocks on DuckDB. We
-	//! default to 256KB. (1 << 18)
-	constexpr static idx_t BLOCK_ALLOC_SIZE = DEFAULT_BLOCK_ALLOC_SIZE;
-	//! The actual memory space that is available within the blocks
-	constexpr static idx_t BLOCK_SIZE = BLOCK_ALLOC_SIZE - BLOCK_HEADER_SIZE;
 	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
 	//! to the page size, which is 4KB. (1 << 12)
 	constexpr static idx_t FILE_HEADER_SIZE = 4096U;
@@ -44,6 +42,18 @@ struct Storage {
 	constexpr static const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
 	//! The number of vectors per row group
 	constexpr static const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
+
+	//! The minimum block allocation size. This is the minimum size we test in our nightly tests.
+	constexpr static idx_t MIN_BLOCK_ALLOC_SIZE = 16384ULL;
+	//! The maximum block allocation size. This is the maximum size currently supported by duckdb.
+	constexpr static idx_t MAX_BLOCK_ALLOC_SIZE = 262144ULL;
+	//! The default block header size for blocks written to storage.
+	constexpr static idx_t DEFAULT_BLOCK_HEADER_SIZE = sizeof(idx_t);
+	//! The default block size.
+	constexpr static idx_t DEFAULT_BLOCK_SIZE = DEFAULT_BLOCK_ALLOC_SIZE - DEFAULT_BLOCK_HEADER_SIZE;
+
+	//! Ensures that a user-provided block allocation size matches all requirements.
+	static void VerifyBlockAllocSize(const idx_t block_alloc_size);
 };
 
 //! The version number of the database storage format
@@ -58,7 +68,7 @@ vector<string> GetSerializationCandidates();
 struct MainHeader {
 	static constexpr idx_t MAX_VERSION_SIZE = 32;
 	static constexpr idx_t MAGIC_BYTE_SIZE = 4;
-	static constexpr idx_t MAGIC_BYTE_OFFSET = Storage::BLOCK_HEADER_SIZE;
+	static constexpr idx_t MAGIC_BYTE_OFFSET = Storage::DEFAULT_BLOCK_HEADER_SIZE;
 	static constexpr idx_t FLAG_COUNT = 4;
 	//! The magic bytes in front of the file should be "DUCK"
 	static const char MAGIC_BYTES[];
@@ -97,8 +107,8 @@ struct DatabaseHeader {
 	//! The number of blocks that is in the file as of this database header. If the file is larger than BLOCK_SIZE *
 	//! block_count any blocks appearing AFTER block_count are implicitly part of the free_list.
 	uint64_t block_count;
-	//! The block size of the database file
-	idx_t block_size;
+	//! The allocation size of blocks in this database file. Defaults to default_block_alloc_size (DBConfig).
+	idx_t block_alloc_size;
 	//! The vector size of the database file
 	idx_t vector_size;
 
@@ -114,11 +124,11 @@ struct DatabaseHeader {
 #if (STANDARD_ROW_GROUPS_SIZE < STANDARD_VECTOR_SIZE)
 #error Row groups must be able to hold at least one vector
 #endif
-
-static_assert(Storage::BLOCK_ALLOC_SIZE % Storage::SECTOR_SIZE == 0,
-              "the block allocation size has to be a multiple of the sector size");
-static_assert(Storage::BLOCK_SIZE < idx_t(NumericLimits<int32_t>::Maximum()),
-              "the block size cannot exceed the maximum signed integer value,"
-              "as some comparisons require casts");
+#if (DEFAULT_BLOCK_ALLOC_SIZE & (DEFAULT_BLOCK_ALLOC_SIZE - 1) != 0)
+#error The default block allocation size must be a power of two
+#endif
+#if (DUCKDB_BLOCK_ALLOC_SIZE & (DUCKDB_BLOCK_ALLOC_SIZE - 1) != 0)
+#error The duckdb block allocation size must be a power of two
+#endif
 
 } // namespace duckdb