duckdb 0.5.2-dev547.0 → 0.5.2-dev561.0

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "83aff54f1"
-#define DUCKDB_VERSION "v0.5.2-dev547"
+#define DUCKDB_SOURCE_ID "c2e70c43d"
+#define DUCKDB_VERSION "v0.5.2-dev561"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -17831,12 +17831,13 @@ public:
 
 
 namespace duckdb {
+class BlockManager;
 class ColumnData;
 class DatabaseInstance;
 class DataTable;
 struct DataTableInfo;
 class ExpressionExecutor;
-class TableDataWriter;
+class RowGroupWriter;
 class UpdateSegment;
 class Vector;
 struct RowGroupPointer;
@@ -17853,15 +17854,17 @@ public:
 	static constexpr const idx_t ROW_GROUP_SIZE = STANDARD_VECTOR_SIZE * ROW_GROUP_VECTOR_COUNT;
 
 public:
-	RowGroup(DatabaseInstance &db, DataTableInfo &table_info, idx_t start, idx_t count);
-	RowGroup(DatabaseInstance &db, DataTableInfo &table_info, const vector<LogicalType> &types,
-	         RowGroupPointer &pointer);
+	RowGroup(DatabaseInstance &db, BlockManager &block_manager, DataTableInfo &table_info, idx_t start, idx_t count);
+	RowGroup(DatabaseInstance &db, BlockManager &block_manager, DataTableInfo &table_info,
+	         const vector<LogicalType> &types, RowGroupPointer &&pointer);
 	RowGroup(RowGroup &row_group, idx_t start);
 	~RowGroup();
 
 private:
 	//! The database instance
 	DatabaseInstance &db;
+	//! The block manager
+	BlockManager &block_manager;
 	//! The table info of this row_group
 	DataTableInfo &table_info;
 	//! The version info of the row_group (inserted and deleted tuple info)
@@ -17875,6 +17878,9 @@ public:
 	DatabaseInstance &GetDatabase() {
 		return db;
 	}
+	BlockManager &GetBlockManager() {
+		return block_manager;
+	}
 	DataTableInfo &GetTableInfo() {
 		return table_info;
 	}
@@ -17930,7 +17936,7 @@ public:
 	//! Delete the given set of rows in the version manager
 	idx_t Delete(TransactionData transaction, DataTable *table, row_t *row_ids, idx_t count);
 
-	RowGroupPointer Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats);
+	RowGroupPointer Checkpoint(RowGroupWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &source, const vector<ColumnDefinition> &columns);
 
@@ -18060,13 +18066,14 @@ namespace duckdb {
 struct ParallelTableScanState;
 
 class PersistentTableData;
+class TableDataWriter;
 class TableIndexList;
 class TableStatistics;
 
 class RowGroupCollection {
 public:
-	RowGroupCollection(shared_ptr<DataTableInfo> info, vector<LogicalType> types, idx_t row_start,
-	                   idx_t total_rows = 0);
+	RowGroupCollection(shared_ptr<DataTableInfo> info, BlockManager &block_manager, vector<LogicalType> types,
+	                   idx_t row_start, idx_t total_rows = 0);
 
 public:
 	idx_t GetTotalRows() const;
@@ -18106,8 +18113,7 @@ public:
 	void UpdateColumn(TransactionData transaction, Vector &row_ids, const vector<column_t> &column_path,
 	                  DataChunk &updates, TableStatistics &stats);
 
-	void Checkpoint(TableDataWriter &writer, vector<RowGroupPointer> &row_group_pointers,
-	                vector<unique_ptr<BaseStatistics>> &global_stats);
+	void Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats);
 
 	void CommitDropColumn(idx_t index);
 	void CommitDropTable();
@@ -18124,6 +18130,8 @@ public:
 	void VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint);
 
 private:
+	//! BlockManager
+	BlockManager &block_manager;
 	//! The number of rows in the table
 	atomic<idx_t> total_rows;
 	shared_ptr<DataTableInfo> info;
@@ -19611,9 +19619,8 @@ public:
 	BufferManager &buffer_manager;
 
 public:
-	virtual void StartCheckpoint() = 0;
 	//! Creates a new block inside the block manager
-	virtual unique_ptr<Block> CreateBlock(block_id_t block_id) = 0;
+	virtual unique_ptr<Block> CreateBlock(block_id_t block_id, FileBuffer *source_buffer) = 0;
 	//! Return the next free block id
 	virtual block_id_t GetFreeBlockId() = 0;
 	//! Returns whether or not a specified block is the root block
@@ -20189,25 +20196,34 @@ private:
 namespace duckdb {
 class DatabaseInstance;
 
-//! This struct is responsible for writing metadata to disk
+//! This struct is responsible for writing data to disk in a stream of blocks.
 class MetaBlockWriter : public Serializer {
 public:
-	MetaBlockWriter(DatabaseInstance &db, block_id_t initial_block_id = INVALID_BLOCK);
+	MetaBlockWriter(BlockManager &block_manager, block_id_t initial_block_id = INVALID_BLOCK);
 	~MetaBlockWriter() override;
 
-	DatabaseInstance &db;
+	BlockManager &block_manager;
+
+protected:
 	unique_ptr<Block> block;
 	set<block_id_t> written_blocks;
 	idx_t offset;
 
 public:
 	BlockPointer GetBlockPointer();
-	void Flush();
+	virtual void Flush();
 
 	void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
 
+	void MarkWrittenBlocks() {
+		for (auto &block_id : written_blocks) {
+			block_manager.MarkBlockAsModified(block_id);
+		}
+	}
+
 protected:
 	virtual block_id_t GetNextBlockId();
+	void AdvanceBlock();
 };
 
 } // namespace duckdb
@@ -20217,6 +20233,7 @@ protected:
 namespace duckdb {
 
 class ClientContext;
+class TableIOManager;
 class Transaction;
 
 struct IndexLock;
@@ -20224,12 +20241,14 @@ struct IndexLock;
 //! The index is an abstract base class that serves as the basis for indexes
 class Index {
 public:
-	Index(IndexType type, const vector<column_t> &column_ids, const vector<unique_ptr<Expression>> &unbound_expressions,
-	      IndexConstraintType constraint_type);
+	Index(IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids,
+	      const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type);
 	virtual ~Index() = default;
 
 	//! The type of the index
 	IndexType type;
+	//! Associated table io manager
+	TableIOManager &table_io_manager;
 	//! Column identifiers to extract from the base table
 	vector<column_t> column_ids;
 	//! unordered_set of column_ids used by the index
@@ -25740,7 +25759,6 @@ public:
 
 
 
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -25989,9 +26007,9 @@ public:
 	//! The block that this segment relates to
 	shared_ptr<BlockHandle> block;
 
-	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, block_id_t id, idx_t offset,
-	                                                         const LogicalType &type_p, idx_t start, idx_t count,
-	                                                         CompressionType compression_type,
+	static unique_ptr<ColumnSegment> CreatePersistentSegment(DatabaseInstance &db, BlockManager &block_manager,
+	                                                         block_id_t id, idx_t offset, const LogicalType &type_p,
+	                                                         idx_t start, idx_t count, CompressionType compression_type,
 	                                                         unique_ptr<BaseStatistics> statistics);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start);
 	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
@@ -26022,15 +26040,20 @@ public:
 
 	//! Convert a transient in-memory segment into a persistent segment blocked by an on-disk block.
 	//! Only used during checkpointing.
-	void ConvertToPersistent(block_id_t block_id);
-	//! Convert a transient in-memory segment into a persistent segment blocked by an on-disk block.
-	void ConvertToPersistent(shared_ptr<BlockHandle> block, block_id_t block_id, uint32_t offset_in_block);
+	void ConvertToPersistent(BlockManager *block_manager, block_id_t block_id);
+	//! Updates pointers to refer to the given block and offset. This is only used
+	//! when sharing a block among segments. This is invoked only AFTER the block is written.
+	void MarkAsPersistent(shared_ptr<BlockHandle> block, uint32_t offset_in_block);
 
 	block_id_t GetBlockId() {
 		D_ASSERT(segment_type == ColumnSegmentType::PERSISTENT);
 		return block_id;
 	}
 
+	BlockManager &GetBlockManager() const {
+		return block->block_manager;
+	}
+
 	idx_t GetBlockOffset() {
 		D_ASSERT(segment_type == ColumnSegmentType::PERSISTENT || offset == 0);
 		return offset;
@@ -26047,9 +26070,9 @@ public:
 	}
 
 public:
-	ColumnSegment(DatabaseInstance &db, LogicalType type, ColumnSegmentType segment_type, idx_t start, idx_t count,
-	              CompressionFunction *function, unique_ptr<BaseStatistics> statistics, block_id_t block_id,
-	              idx_t offset);
+	ColumnSegment(DatabaseInstance &db, shared_ptr<BlockHandle> block, LogicalType type, ColumnSegmentType segment_type,
+	              idx_t start, idx_t count, CompressionFunction *function, unique_ptr<BaseStatistics> statistics,
+	              block_id_t block_id, idx_t offset);
 	ColumnSegment(ColumnSegment &other, idx_t start);
 
 private:
@@ -26071,26 +26094,33 @@ private:
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/data_table_info.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
 
 
 namespace duckdb {
-class ClientContext;
-class ColumnDefinition;
-class DataTable;
-class RowGroup;
-class StorageManager;
-class TableCatalogEntry;
-class Transaction;
-class WriteAheadLog;
-class TableDataWriter;
+class DatabaseInstance;
+class TableIOManager;
 
 struct DataTableInfo {
-	DataTableInfo(DatabaseInstance &db, string schema, string table)
-	    : db(db), cardinality(0), schema(move(schema)), table(move(table)) {
+	DataTableInfo(DatabaseInstance &db, shared_ptr<TableIOManager> table_io_manager_p, string schema, string table)
+	    : db(db), table_io_manager(move(table_io_manager_p)), cardinality(0), schema(move(schema)), table(move(table)) {
 	}
 
 	//! The database instance of the table
 	DatabaseInstance &db;
+	//! The table IO manager
+	shared_ptr<TableIOManager> table_io_manager;
 	//! The amount of elements in the table. Note that this number signifies the amount of COMMITTED entries in the
 	//! table. It can be inaccurate inside of transactions. More work is needed to properly support that.
 	atomic<idx_t> cardinality;
@@ -26106,12 +26136,28 @@ struct DataTableInfo {
 	}
 };
 
+} // namespace duckdb
+
+
+namespace duckdb {
+class ClientContext;
+class ColumnDefinition;
+class DataTable;
+class RowGroup;
+class StorageManager;
+class TableCatalogEntry;
+class TableIOManager;
+class Transaction;
+class WriteAheadLog;
+class TableDataWriter;
+
 //! DataTable represents a physical table on disk
 class DataTable {
 public:
 	//! Constructs a new data table from an (optional) set of persistent segments
-	DataTable(DatabaseInstance &db, const string &schema, const string &table,
-	          vector<ColumnDefinition> column_definitions_p, unique_ptr<PersistentTableData> data = nullptr);
+	DataTable(DatabaseInstance &db, shared_ptr<TableIOManager> table_io_manager, const string &schema,
+	          const string &table, vector<ColumnDefinition> column_definitions_p,
+	          unique_ptr<PersistentTableData> data = nullptr);
 	//! Constructs a DataTable as a delta on an existing data table with a newly added column
 	DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value);
 	//! Constructs a DataTable as a delta on an existing data table but with one column removed
@@ -26122,10 +26168,10 @@ public:
 	//! Constructs a DataTable as a delta on an existing data table but with one column added new constraint
 	DataTable(ClientContext &context, DataTable &parent, unique_ptr<BoundConstraint> constraint);
 
+	//! The table info
 	shared_ptr<DataTableInfo> info;
-
+	//! The set of physical columns stored by this DataTable
 	vector<ColumnDefinition> column_definitions;
-
 	//! A reference to the database instance
 	DatabaseInstance &db;