duckdb 0.8.2-dev87.0 → 0.9.0

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -36,11 +36,12 @@ class Vector;
 struct ColumnCheckpointState;
 struct RowGroupPointer;
 struct TransactionData;
-struct VersionNode;
 class CollectionScanState;
 class TableFilterSet;
 struct ColumnFetchState;
 struct RowGroupAppendState;
+class MetadataManager;
+class RowVersionManager;
 
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
@@ -50,11 +51,6 @@ struct RowGroupWriteData {
 class RowGroup : public SegmentBase<RowGroup> {
 public:
 	friend class ColumnData;
-	friend class VersionDeleteState;
-
-public:
-	static constexpr const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
-	static constexpr const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
 
 public:
 	RowGroup(RowGroupCollection &collection, idx_t start, idx_t count);
@@ -65,7 +61,7 @@ private:
 	//! The RowGroupCollection this row-group is a part of
 	reference<RowGroupCollection> collection;
 	//! The version info of the row_group (inserted and deleted tuple info)
-	shared_ptr<VersionNode> version_info;
+	shared_ptr<RowVersionManager> version_info;
 	//! The column data of the row_group
 	vector<shared_ptr<ColumnData>> columns;
 
@@ -74,7 +70,6 @@ public:
 	RowGroupCollection &GetCollection() {
 		return collection.get();
 	}
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();
 	DataTableInfo &GetTableInfo();
 
@@ -82,7 +77,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -123,9 +118,8 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	bool AllDeleted();
 	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
-	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
-	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
 
 	void InitializeAppend(RowGroupAppendState &append_state);
 	void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
@@ -147,8 +141,17 @@ public:
 
 	void NextVector(CollectionScanState &state);
 
+	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
+	RowVersionManager &GetOrCreateVersionInfo();
+
+	// Serialization
+	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
+	static RowGroupPointer Deserialize(Deserializer &deserializer);
+
 private:
-	ChunkInfo *GetChunkInfo(idx_t vector_idx);
+	shared_ptr<RowVersionManager> &GetVersionInfo();
+	shared_ptr<RowVersionManager> &GetOrCreateVersionInfoPtr();
+
 	ColumnData &GetColumn(storage_t c);
 	idx_t GetColumnCount() const;
 	vector<shared_ptr<ColumnData>> &GetColumns();
@@ -156,20 +159,17 @@ private:
 	template <TableScanType TYPE>
 	void TemplatedScan(TransactionData transaction, CollectionScanState &state, DataChunk &result);
 
-	static void CheckpointDeletes(VersionNode *versions, Serializer &serializer);
-	static shared_ptr<VersionNode> DeserializeDeletes(Deserializer &source);
+	vector<MetaBlockPointer> CheckpointDeletes(MetadataManager &manager);
+
+	bool HasUnloadedDeletes() const;
 
 private:
 	mutex row_group_lock;
 	mutex stats_lock;
-	vector<BlockPointer> column_pointers;
+	vector<MetaBlockPointer> column_pointers;
 	unique_ptr<atomic<bool>[]> is_loaded;
-};
-
-struct VersionNode {
-	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
-
-	void SetStart(idx_t start);
+	vector<MetaBlockPointer> deletes_pointers;
+	atomic<bool> deletes_is_loaded;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -27,6 +27,7 @@ class DuckTransaction;
 class BoundConstraint;
 class RowGroupSegmentTree;
 struct ColumnSegmentInfo;
+class MetadataManager;
 
 class RowGroupCollection {
 public:
@@ -45,7 +46,6 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
-	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
@@ -73,7 +73,7 @@ public:
 	//! FinalizeAppend flushes an append with a variable number of rows.
 	void FinalizeAppend(TransactionData transaction, TableAppendState &state);
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
-	void RevertAppendInternal(idx_t start_row, idx_t count);
+	void RevertAppendInternal(idx_t start_row);
 
 	void MergeStorage(RowGroupCollection &data);
 
@@ -93,7 +93,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression *default_value);
+	                                         Expression &default_value);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
@@ -104,10 +104,10 @@ public:
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	AttachedDatabase &GetAttached();
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager() {
 		return block_manager;
 	}
+	MetadataManager &GetMetadataManager();
 	DataTableInfo &GetTableInfo() {
 		return *info;
 	}

package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp

@@ -14,7 +14,7 @@
 namespace duckdb {
 struct DataTableInfo;
 class PersistentTableData;
-class MetaBlockReader;
+class MetadataReader;
 
 class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
 public:
@@ -29,7 +29,7 @@ protected:
 	RowGroupCollection &collection;
 	idx_t current_row_group;
 	idx_t max_row_group;
-	unique_ptr<MetaBlockReader> reader;
+	unique_ptr<MetadataReader> reader;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp

@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/table/row_version_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/vector_size.hpp"
+#include "duckdb/storage/table/chunk_info.hpp"
+#include "duckdb/storage/storage_info.hpp"
+#include "duckdb/common/mutex.hpp"
+
+namespace duckdb {
+
+class MetadataManager;
+struct MetaBlockPointer;
+
+class RowVersionManager {
+public:
+	explicit RowVersionManager(idx_t start);
+
+	idx_t GetStart() {
+		return start;
+	}
+	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
+
+	idx_t GetSelVector(TransactionData transaction, idx_t vector_idx, SelectionVector &sel_vector, idx_t max_count);
+	idx_t GetCommittedSelVector(transaction_t start_time, transaction_t transaction_id, idx_t vector_idx,
+	                            SelectionVector &sel_vector, idx_t max_count);
+	bool Fetch(TransactionData transaction, idx_t row);
+
+	void AppendVersionInfo(TransactionData transaction, idx_t count, idx_t row_group_start, idx_t row_group_end);
+	void CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_t count);
+	void RevertAppend(idx_t start_row);
+
+	idx_t DeleteRows(idx_t vector_idx, transaction_t transaction_id, row_t rows[], idx_t count);
+	void CommitDelete(idx_t vector_idx, transaction_t commit_id, row_t rows[], idx_t count);
+
+	vector<MetaBlockPointer> Checkpoint(MetadataManager &manager);
+	static shared_ptr<RowVersionManager> Deserialize(MetaBlockPointer delete_pointer, MetadataManager &manager,
+	                                                 idx_t start);
+
+private:
+	mutex version_lock;
+	idx_t start;
+	unique_ptr<ChunkInfo> vector_info[Storage::ROW_GROUP_VECTOR_COUNT];
+	bool has_changes;
+	vector<MetaBlockPointer> storage_pointers;
+
+private:
+	optional_ptr<ChunkInfo> GetChunkInfo(idx_t vector_idx);
+	ChunkVectorInfo &GetVectorInfo(idx_t vector_idx);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp

@@ -54,11 +54,11 @@ public:
 	void CheckpointScan(ColumnSegment &segment, ColumnScanState &state, idx_t row_group_start, idx_t count,
 	                    Vector &scan_vector) override;
 
-	void DeserializeColumn(Deserializer &source) override;
-
 	void GetColumnSegmentInfo(duckdb::idx_t row_group_index, vector<duckdb::idx_t> col_path,
 	                          vector<duckdb::ColumnSegmentInfo> &result) override;
 
+	void DeserializeColumn(Deserializer &deserializer) override;
+
 	void Verify(RowGroup &parent) override;
 };
 

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -47,7 +47,7 @@ public:
 	void VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, DataChunk &chunk, ConflictManager &conflict_manager);
 
 	//! Serialize all indexes owned by this table, returns a vector of block info of all indexes
-	vector<BlockPointer> SerializeIndexes(duckdb::MetaBlockWriter &writer);
+	vector<BlockPointer> SerializeIndexes(duckdb::MetadataWriter &writer);
 
 	vector<column_t> GetRequiredColumns();
 

package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp

@@ -16,6 +16,8 @@
 namespace duckdb {
 class ColumnList;
 class PersistentTableData;
+class Serializer;
+class Deserializer;
 
 class TableStatisticsLock {
 public:
@@ -47,8 +49,8 @@ public:
 
 	unique_ptr<TableStatisticsLock> GetLock();
 
-	void Serialize(Serializer &serializer);
-	void Deserialize(Deserializer &source, ColumnList &columns);
+	void Serialize(Serializer &serializer) const;
+	void Deserialize(Deserializer &deserializer, ColumnList &columns);
 
 private:
 	//! The statistics lock

package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp

@@ -101,7 +101,7 @@ struct UpdateNodeData {
 };
 
 struct UpdateNode {
-	unique_ptr<UpdateNodeData> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
+	unique_ptr<UpdateNodeData> info[Storage::ROW_GROUP_VECTOR_COUNT];
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp

@@ -13,6 +13,7 @@
 namespace duckdb {
 class BlockManager;
 class DataTable;
+class MetadataManager;
 
 class TableIOManager {
 public:
@@ -27,6 +28,8 @@ public:
 
 	//! The block manager used for storing row group data
 	virtual BlockManager &GetBlockManagerForRowData() = 0;
+
+	virtual MetadataManager &GetMetadataManager() = 0;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp

@@ -28,6 +28,7 @@ struct ColumnSegmentInfo {
 	bool persistent;
 	block_id_t block_id;
 	idx_t block_offset;
+	string segment_info;
 };
 
 struct IndexInfo {

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -24,7 +24,6 @@ namespace duckdb {
 struct AlterInfo;
 
 class AttachedDatabase;
-class BufferedSerializer;
 class Catalog;
 class DatabaseInstance;
 class SchemaCatalogEntry;
@@ -38,54 +37,52 @@ class TransactionManager;
 
 class ReplayState {
 public:
-	ReplayState(AttachedDatabase &db, ClientContext &context, Deserializer &source)
-	    : db(db), context(context), catalog(db.GetCatalog()), source(source), deserialize_only(false),
-	      checkpoint_id(INVALID_BLOCK) {
+	ReplayState(AttachedDatabase &db, ClientContext &context)
+	    : db(db), context(context), catalog(db.GetCatalog()), deserialize_only(false) {
 	}
 
 	AttachedDatabase &db;
 	ClientContext &context;
 	Catalog &catalog;
-	Deserializer &source;
 	optional_ptr<TableCatalogEntry> current_table;
 	bool deserialize_only;
-	block_id_t checkpoint_id;
+	MetaBlockPointer checkpoint_id;
 
 public:
-	void ReplayEntry(WALType entry_type);
+	void ReplayEntry(WALType entry_type, BinaryDeserializer &deserializer);
 
 protected:
-	virtual void ReplayCreateTable();
-	void ReplayDropTable();
-	void ReplayAlter();
+	virtual void ReplayCreateTable(BinaryDeserializer &deserializer);
+	void ReplayDropTable(BinaryDeserializer &deserializer);
+	void ReplayAlter(BinaryDeserializer &deserializer);
 
-	void ReplayCreateView();
-	void ReplayDropView();
+	void ReplayCreateView(BinaryDeserializer &deserializer);
+	void ReplayDropView(BinaryDeserializer &deserializer);
 
-	void ReplayCreateSchema();
-	void ReplayDropSchema();
+	void ReplayCreateSchema(BinaryDeserializer &deserializer);
+	void ReplayDropSchema(BinaryDeserializer &deserializer);
 
-	void ReplayCreateType();
-	void ReplayDropType();
+	void ReplayCreateType(BinaryDeserializer &deserializer);
+	void ReplayDropType(BinaryDeserializer &deserializer);
 
-	void ReplayCreateSequence();
-	void ReplayDropSequence();
-	void ReplaySequenceValue();
+	void ReplayCreateSequence(BinaryDeserializer &deserializer);
+	void ReplayDropSequence(BinaryDeserializer &deserializer);
+	void ReplaySequenceValue(BinaryDeserializer &deserializer);
 
-	void ReplayCreateMacro();
-	void ReplayDropMacro();
+	void ReplayCreateMacro(BinaryDeserializer &deserializer);
+	void ReplayDropMacro(BinaryDeserializer &deserializer);
 
-	void ReplayCreateTableMacro();
-	void ReplayDropTableMacro();
+	void ReplayCreateTableMacro(BinaryDeserializer &deserializer);
+	void ReplayDropTableMacro(BinaryDeserializer &deserializer);
 
-	void ReplayCreateIndex();
-	void ReplayDropIndex();
+	void ReplayCreateIndex(BinaryDeserializer &deserializer);
+	void ReplayDropIndex(BinaryDeserializer &deserializer);
 
-	void ReplayUseTable();
-	void ReplayInsert();
-	void ReplayDelete();
-	void ReplayUpdate();
-	void ReplayCheckpoint();
+	void ReplayUseTable(BinaryDeserializer &deserializer);
+	void ReplayInsert(BinaryDeserializer &deserializer);
+	void ReplayDelete(BinaryDeserializer &deserializer);
+	void ReplayUpdate(BinaryDeserializer &deserializer);
+	void ReplayCheckpoint(BinaryDeserializer &deserializer);
 };
 
 //! The WriteAheadLog (WAL) is a log that is used to provide durability. Prior
@@ -137,7 +134,7 @@ public:
 	//! Sets the table used for subsequent insert/delete/update commands
 	void WriteSetTable(string &schema, string &table);
 
-	void WriteAlter(data_ptr_t ptr, idx_t data_size);
+	void WriteAlter(const AlterInfo &info);
 
 	void WriteInsert(DataChunk &chunk);
 	void WriteDelete(DataChunk &chunk);
@@ -157,7 +154,7 @@ public:
 	void Delete();
 	void Flush();
 
-	void WriteCheckpoint(block_id_t meta_block);
+	void WriteCheckpoint(MetaBlockPointer meta_block);
 
 protected:
 	AttachedDatabase &database;

package/src/duckdb/src/include/duckdb/transaction/commit_state.hpp

@@ -23,7 +23,7 @@ struct UpdateInfo;
 
 class CommitState {
 public:
-	explicit CommitState(ClientContext &context, transaction_t commit_id, optional_ptr<WriteAheadLog> log = nullptr);
+	explicit CommitState(transaction_t commit_id, optional_ptr<WriteAheadLog> log = nullptr);
 
 	optional_ptr<WriteAheadLog> log;
 	transaction_t commit_id;
@@ -35,9 +35,6 @@ public:
 	unique_ptr<DataChunk> delete_chunk;
 	unique_ptr<DataChunk> update_chunk;
 
-private:
-	ClientContext &context;
-
 public:
 	template <bool HAS_LOG>
 	void CommitEntry(UndoFlags type, data_ptr_t data);
@@ -49,8 +46,6 @@ private:
 	void WriteCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data);
 	void WriteDelete(DeleteInfo &info);
 	void WriteUpdate(UpdateInfo &info);
-
-	void AppendRowId(row_t rowid);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/transaction/delete_info.hpp

@@ -11,12 +11,13 @@
 #include "duckdb/common/constants.hpp"
 
 namespace duckdb {
-class ChunkVectorInfo;
 class DataTable;
+class RowVersionManager;
 
 struct DeleteInfo {
 	DataTable *table;
-	ChunkVectorInfo *vinfo;
+	RowVersionManager *version_info;
+	idx_t vector_idx;
 	idx_t count;
 	idx_t base_row;
 	row_t rows[1];

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -11,12 +11,13 @@
 #include "duckdb/transaction/transaction.hpp"
 
 namespace duckdb {
+class RowVersionManager;
 
 class DuckTransaction : public Transaction {
 public:
 	DuckTransaction(TransactionManager &manager, ClientContext &context, transaction_t start_time,
 	                transaction_t transaction_id);
-	~DuckTransaction();
+	~DuckTransaction() override;
 
 	//! The start timestamp of this transaction
 	transaction_t start_time;
@@ -49,7 +50,8 @@ public:
 
 	bool ChangesMade();
 
-	void PushDelete(DataTable &table, ChunkVectorInfo *vinfo, row_t rows[], idx_t count, idx_t base_row);
+	void PushDelete(DataTable &table, RowVersionManager &info, idx_t vector_idx, row_t rows[], idx_t count,
+	                idx_t base_row);
 	void PushAppend(DataTable &table, idx_t row_start, idx_t row_count);
 	UpdateInfo *CreateUpdateInfo(idx_t type_size, idx_t entries);
 

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -32,7 +32,7 @@ public:
 	LocalTableStorage(DataTable &table, LocalTableStorage &parent, idx_t drop_idx);
 	// Create a LocalTableStorage from an ADD COLUMN
 	LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, ColumnDefinition &new_column,
-	                  optional_ptr<Expression> default_value);
+	                  Expression &default_value);
 	~LocalTableStorage();
 
 	reference<DataTable> table_ref;
@@ -88,7 +88,7 @@ private:
 class LocalStorage {
 public:
 	// Threshold to merge row groups instead of appending
-	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
+	static constexpr const idx_t MERGE_THRESHOLD = Storage::ROW_GROUP_SIZE;
 
 public:
 	struct CommitState {
@@ -143,8 +143,7 @@ public:
 
 	idx_t AddedRows(DataTable &table);
 
-	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
-	               optional_ptr<Expression> default_value);
+	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column, Expression &default_value);
 	void DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t removed_column);
 	void ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t changed_idx, const LogicalType &target_type,
 	                const vector<column_t> &bound_columns, Expression &cast_expr);

package/src/duckdb/src/include/duckdb/transaction/undo_buffer.hpp

@@ -48,7 +48,6 @@ public:
 	void Rollback() noexcept;
 
 private:
-	ClientContext &context;
 	ArenaAllocator allocator;
 
 private:

package/src/duckdb/src/include/duckdb/verification/prepared_statement_verifier.hpp

@@ -21,7 +21,7 @@ public:
 	         const std::function<unique_ptr<QueryResult>(const string &, unique_ptr<SQLStatement>)> &run) override;
 
 private:
-	vector<unique_ptr<ParsedExpression>> values;
+	case_insensitive_map_t<unique_ptr<ParsedExpression>> values;
 	unique_ptr<SQLStatement> prepare_statement;
 	unique_ptr<SQLStatement> execute_statement;
 	unique_ptr<SQLStatement> dealloc_statement;

package/src/duckdb/src/include/duckdb/verification/statement_verifier.hpp

@@ -18,7 +18,6 @@ enum class VerificationType : uint8_t {
 	ORIGINAL,
 	COPIED,
 	DESERIALIZED,
-	DESERIALIZED_V2,
 	PARSED,
 	UNOPTIMIZED,
 	NO_OPERATOR_CACHING,