duckdb 0.8.2-dev2673.0 → 0.8.2-dev2809.0

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -14,7 +14,7 @@
 #include "duckdb/common/sort/sort.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/common/types/constraint_conflict_info.hpp"
 
@@ -126,7 +126,7 @@ public:
 	}
 
 	//! Serializes the index and returns the pair of block_id offset positions
-	virtual BlockPointer Serialize(MetaBlockWriter &writer);
+	virtual BlockPointer Serialize(MetadataWriter &writer);
 	//! Returns the serialized data pointer to the block and offset of the serialized index
 	BlockPointer GetSerializedDataPointer() const {
 		return serialized_data_pointer;

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/serializer.hpp"
+#include "duckdb/storage/block.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/common/set.hpp"
+#include "duckdb/storage/buffer/buffer_handle.hpp"
+
+namespace duckdb {
+class DatabaseInstance;
+
+struct MetadataBlock {
+	shared_ptr<BlockHandle> block;
+	block_id_t block_id;
+	vector<uint8_t> free_blocks;
+
+	void Serialize(Serializer &serializer);
+	static MetadataBlock Deserialize(Deserializer &source);
+
+	idx_t FreeBlocksToInteger();
+	void FreeBlocksFromInteger(idx_t blocks);
+};
+
+struct MetadataPointer {
+	idx_t block_index : 56;
+	uint8_t index : 8;
+};
+
+struct MetadataHandle {
+	MetadataPointer pointer;
+	BufferHandle handle;
+};
+
+class MetadataManager {
+public:
+	//! The size of metadata blocks
+	static constexpr const idx_t METADATA_BLOCK_SIZE = 4088;
+	//! The amount of metadata blocks per storage block
+	static constexpr const idx_t METADATA_BLOCK_COUNT = 64;
+
+public:
+	MetadataManager(BlockManager &block_manager, BufferManager &buffer_manager);
+	~MetadataManager();
+
+	MetadataHandle AllocateHandle();
+	MetadataHandle Pin(MetadataPointer pointer);
+
+	MetaBlockPointer GetDiskPointer(MetadataPointer pointer, uint32_t offset = 0);
+	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
+	MetadataPointer RegisterDiskPointer(MetaBlockPointer pointer);
+
+	static BlockPointer ToBlockPointer(MetaBlockPointer meta_pointer);
+	static MetaBlockPointer FromBlockPointer(BlockPointer block_pointer);
+
+	//! Flush all blocks to disk
+	void Flush();
+
+	void MarkBlocksAsModified();
+
+	idx_t BlockCount();
+
+	void Serialize(Serializer &serializer);
+	void Deserialize(Deserializer &source);
+
+protected:
+	BlockManager &block_manager;
+	BufferManager &buffer_manager;
+	unordered_map<block_id_t, MetadataBlock> blocks;
+	unordered_map<block_id_t, idx_t> modified_blocks;
+
+protected:
+	block_id_t AllocateNewBlock();
+	block_id_t GetNextBlockId();
+
+	void AddBlock(MetadataBlock new_block, bool if_exists = false);
+	void AddAndRegisterBlock(MetadataBlock block);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp

@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_reader.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+
+namespace duckdb {
+
+enum class BlockReaderType { EXISTING_BLOCKS, REGISTER_BLOCKS };
+
+class MetadataReader : public Deserializer {
+public:
+	MetadataReader(MetadataManager &manager, MetaBlockPointer pointer,
+	               BlockReaderType type = BlockReaderType::EXISTING_BLOCKS);
+	MetadataReader(MetadataManager &manager, BlockPointer pointer);
+	~MetadataReader() override;
+
+public:
+	//! Read content of size read_size into the buffer
+	void ReadData(data_ptr_t buffer, idx_t read_size) override;
+
+	MetaBlockPointer GetMetaBlockPointer();
+
+	MetadataManager &GetMetadataManager() {
+		return manager;
+	}
+
+private:
+	data_ptr_t BasePtr();
+	data_ptr_t Ptr();
+
+	void ReadNextBlock();
+
+	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
+
+private:
+	MetadataManager &manager;
+	BlockReaderType type;
+	MetadataHandle block;
+	MetadataPointer next_pointer;
+	bool has_next_block;
+	idx_t index;
+	idx_t offset;
+	idx_t next_offset;
+	idx_t capacity;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/metadata/metadata_writer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/metadata/metadata_manager.hpp"
+
+namespace duckdb {
+
+class MetadataWriter : public Serializer {
+public:
+	explicit MetadataWriter(MetadataManager &manager);
+	~MetadataWriter() override;
+
+public:
+	void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
+
+	void Flush();
+
+	BlockPointer GetBlockPointer();
+	MetaBlockPointer GetMetaBlockPointer();
+
+protected:
+	virtual MetadataHandle NextHandle();
+
+private:
+	data_ptr_t BasePtr();
+	data_ptr_t Ptr();
+
+	void NextBlock();
+
+private:
+	MetadataManager &manager;
+	MetadataHandle block;
+	MetadataPointer current_pointer;
+	idx_t capacity;
+	idx_t offset;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp

@@ -11,14 +11,14 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
 class ClientContext;
 class ColumnSegment;
-class MetaBlockReader;
+class MetadataReader;
 class SchemaCatalogEntry;
 class SequenceCatalogEntry;
 class TableCatalogEntry;

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -20,6 +20,7 @@
 namespace duckdb {
 
 class DatabaseInstance;
+struct MetadataHandle;
 
 struct StorageManagerOptions {
 	bool read_only = false;
@@ -45,7 +46,7 @@ public:
 	//! Return the next free block id
 	block_id_t GetFreeBlockId() override;
 	//! Returns whether or not a specified block is the root block
-	bool IsRootBlock(block_id_t root) override;
+	bool IsRootBlock(MetaBlockPointer root) override;
 	//! Mark a block as free (immediately re-writeable)
 	void MarkBlockAsFree(block_id_t block_id) override;
 	//! Mark a block as modified (re-writeable after a checkpoint)
@@ -53,7 +54,7 @@ public:
 	//! Increase the reference count of a block. The block should hold at least one reference
 	void IncreaseBlockReferenceCount(block_id_t block_id) override;
 	//! Return the meta block id
-	block_id_t GetMetaBlock() override;
+	idx_t GetMetaBlock() override;
 	//! Read the content of the block from disk
 	void Read(Block &block) override;
 	//! Write the given block to disk
@@ -78,7 +79,7 @@ private:
 	void ChecksumAndWrite(FileBuffer &handle, uint64_t location) const;
 
 	//! Return the blocks to which we will write the free list and modified blocks
-	vector<block_id_t> GetFreeListBlocks();
+	vector<MetadataHandle> GetFreeListBlocks();
 
 private:
 	AttachedDatabase &db;
@@ -99,11 +100,11 @@ private:
 	//! The list of blocks that will be added to the free list
 	unordered_set<block_id_t> modified_blocks;
 	//! The current meta block id
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! The current maximum block id, this id will be given away first after the free_list runs out
 	block_id_t max_block;
 	//! The block id where the free list can be found
-	block_id_t free_list_id;
+	idx_t free_list_id;
 	//! The current header iteration count
 	uint64_t iteration_count;
 	//! The storage manager options

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -65,9 +65,9 @@ struct DatabaseHeader {
 	//! The iteration count, increases by 1 every time the storage is checkpointed.
 	uint64_t iteration;
 	//! A pointer to the initial meta block
-	block_id_t meta_block;
+	idx_t meta_block;
 	//! A pointer to the block containing the free list
-	block_id_t free_list;
+	idx_t free_list;
 	//! The number of blocks that is in the file as of this database header. If the file is larger than BLOCK_SIZE *
 	//! block_count any blocks appearing AFTER block_count are implicitly part of the free_list.
 	uint64_t block_count;

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -65,7 +65,7 @@ public:
 
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) = 0;
-	virtual bool IsCheckpointClean(block_id_t checkpoint_id) = 0;
+	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
 	virtual void CreateCheckpoint(bool delete_wal = false, bool force_checkpoint = false) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -109,7 +109,7 @@ public:
 public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(Transaction &transaction, bool checkpoint) override;
-	bool IsCheckpointClean(block_id_t checkpoint_id) override;
+	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
 	void CreateCheckpoint(bool delete_wal, bool force_checkpoint) override;
 	DatabaseSize GetDatabaseSize() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;

package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/storage/data_pointer.hpp"
 #include "duckdb/storage/table/table_statistics.hpp"
+#include "duckdb/storage/metadata/metadata_manager.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -24,8 +25,7 @@ public:
 	TableStatistics table_stats;
 	idx_t total_rows;
 	idx_t row_group_count;
-	block_id_t block_id;
-	idx_t offset;
+	MetaBlockPointer block_pointer;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -162,7 +162,7 @@ private:
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
-	vector<BlockPointer> column_pointers;
+	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
 };
 

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -27,6 +27,7 @@ class DuckTransaction;
 class BoundConstraint;
 class RowGroupSegmentTree;
 struct ColumnSegmentInfo;
+class MetadataManager;
 
 class RowGroupCollection {
 public:
@@ -106,6 +107,7 @@ public:
 	BlockManager &GetBlockManager() {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager();
 	DataTableInfo &GetTableInfo() {
 		return *info;
 	}

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -14,7 +14,7 @@
 namespace duckdb {
 struct DataTableInfo;
 class PersistentTableData;
-class MetaBlockReader;
+class MetadataReader;
 
 class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
 public:
@@ -29,7 +29,7 @@ protected:
 	RowGroupCollection &collection;
 	idx_t current_row_group;
 	idx_t max_row_group;
-	unique_ptr<MetaBlockReader> reader;
+	unique_ptr<MetadataReader> reader;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -47,7 +47,7 @@ public:
 	void VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, DataChunk &chunk, ConflictManager &conflict_manager);
 
 	//! Serialize all indexes owned by this table, returns a vector of block info of all indexes
-	vector<BlockPointer> SerializeIndexes(duckdb::MetaBlockWriter &writer);
+	vector<BlockPointer> SerializeIndexes(duckdb::MetadataWriter &writer);
 
 	vector<column_t> GetRequiredColumns();
 

package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp

@@ -13,6 +13,7 @@
 namespace duckdb {
 class BlockManager;
 class DataTable;
+class MetadataManager;
 
 class TableIOManager {
 public:
@@ -27,6 +28,8 @@ public:
 
 	//! The block manager used for storing row group data
 	virtual BlockManager &GetBlockManagerForRowData() = 0;
+
+	virtual MetadataManager &GetMetadataManager() = 0;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -39,8 +39,7 @@ class TransactionManager;
 class ReplayState {
 public:
 	ReplayState(AttachedDatabase &db, ClientContext &context, Deserializer &source)
-	    : db(db), context(context), catalog(db.GetCatalog()), source(source), deserialize_only(false),
-	      checkpoint_id(INVALID_BLOCK) {
+	    : db(db), context(context), catalog(db.GetCatalog()), source(source), deserialize_only(false) {
 	}
 
 	AttachedDatabase &db;
@@ -49,7 +48,7 @@ public:
 	Deserializer &source;
 	optional_ptr<TableCatalogEntry> current_table;
 	bool deserialize_only;
-	block_id_t checkpoint_id;
+	MetaBlockPointer checkpoint_id;
 
 public:
 	void ReplayEntry(WALType entry_type);
@@ -157,7 +156,7 @@ public:
 	void Delete();
 	void Flush();
 
-	void WriteCheckpoint(block_id_t meta_block);
+	void WriteCheckpoint(MetaBlockPointer meta_block);
 
 protected:
 	AttachedDatabase &database;

package/src/duckdb/src/include/duckdb.h

@@ -308,7 +308,8 @@ typedef enum { DuckDBSuccess = 0, DuckDBError = 1 } duckdb_state;
 typedef enum {
 	DUCKDB_PENDING_RESULT_READY = 0,
 	DUCKDB_PENDING_RESULT_NOT_READY = 1,
-	DUCKDB_PENDING_ERROR = 2
+	DUCKDB_PENDING_ERROR = 2,
+	DUCKDB_PENDING_NO_TASKS_AVAILABLE = 3
 } duckdb_pending_state;
 
 //===--------------------------------------------------------------------===//
@@ -1260,6 +1261,15 @@ Otherwise, all remaining tasks must be executed first.
 */
 DUCKDB_API duckdb_state duckdb_execute_pending(duckdb_pending_result pending_result, duckdb_result *out_result);
 
+/*!
+Returns whether a duckdb_pending_state is finished executing. For example if `pending_state` is
+DUCKDB_PENDING_RESULT_READY, this function will return true.
+
+* pending_state: The pending state on which to decide whether to finish execution.
+* returns: Boolean indicating pending execution should be considered finished.
+*/
+DUCKDB_API bool duckdb_pending_execution_is_finished(duckdb_pending_state pending_state);
+
 //===--------------------------------------------------------------------===//
 // Value Interface
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/capi/pending-c.cpp

@@ -92,6 +92,8 @@ duckdb_pending_state duckdb_pending_execute_task(duckdb_pending_result pending_r
 	switch (return_value) {
 	case PendingExecutionResult::RESULT_READY:
 		return DUCKDB_PENDING_RESULT_READY;
+	case PendingExecutionResult::NO_TASKS_AVAILABLE:
+		return DUCKDB_PENDING_NO_TASKS_AVAILABLE;
 	case PendingExecutionResult::RESULT_NOT_READY:
 		return DUCKDB_PENDING_RESULT_NOT_READY;
 	default:
@@ -99,6 +101,21 @@ duckdb_pending_state duckdb_pending_execute_task(duckdb_pending_result pending_r
 	}
 }
 
+bool duckdb_pending_execution_is_finished(duckdb_pending_state pending_state) {
+	switch (pending_state) {
+	case DUCKDB_PENDING_RESULT_READY:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::RESULT_READY);
+	case DUCKDB_PENDING_NO_TASKS_AVAILABLE:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::NO_TASKS_AVAILABLE);
+	case DUCKDB_PENDING_RESULT_NOT_READY:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::RESULT_NOT_READY);
+	case DUCKDB_PENDING_ERROR:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::EXECUTION_ERROR);
+	default:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::EXECUTION_ERROR);
+	}
+}
+
 duckdb_state duckdb_execute_pending(duckdb_pending_result pending_result, duckdb_result *out_result) {
 	if (!pending_result || !out_result) {
 		return DuckDBError;

package/src/duckdb/src/main/pending_query_result.cpp

@@ -55,7 +55,8 @@ PendingExecutionResult PendingQueryResult::ExecuteTaskInternal(ClientContextLock
 
 unique_ptr<QueryResult> PendingQueryResult::ExecuteInternal(ClientContextLock &lock) {
 	CheckExecutableInternal(lock);
-	while (ExecuteTaskInternal(lock) == PendingExecutionResult::RESULT_NOT_READY) {
+	// Busy wait while execution is not finished
+	while (!IsFinished(ExecuteTaskInternal(lock))) {
 	}
 	if (HasError()) {
 		return make_uniq<MaterializedQueryResult>(error);
@@ -74,4 +75,11 @@ void PendingQueryResult::Close() {
 	context.reset();
 }
 
+bool PendingQueryResult::IsFinished(PendingExecutionResult result) {
+	if (result == PendingExecutionResult::RESULT_READY || result == PendingExecutionResult::EXECUTION_ERROR) {
+		return true;
+	}
+	return false;
+}
+
 } // namespace duckdb