duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/storage/wal_replay.cpp

@@ -4,12 +4,17 @@
 #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
+#include "duckdb/common/checksum.hpp"
 #include "duckdb/common/printer.hpp"
 #include "duckdb/common/serializer/binary_deserializer.hpp"
 #include "duckdb/common/serializer/buffered_file_reader.hpp"
+#include "duckdb/common/serializer/memory_stream.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/index/index_type_set.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/client_context.hpp"
+#include "duckdb/main/config.hpp"
 #include "duckdb/main/connection.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/parser/parsed_data/alter_table_info.hpp"
@@ -20,14 +25,10 @@
 #include "duckdb/planner/expression_binder/index_binder.hpp"
 #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/storage/write_ahead_log.hpp"
-#include "duckdb/common/serializer/memory_stream.hpp"
-#include "duckdb/common/checksum.hpp"
-#include "duckdb/execution/index/index_type_set.hpp"
-#include "duckdb/execution/index/art/art.hpp"
-#include "duckdb/main/config.hpp"
 #include "duckdb/storage/table/delete_state.hpp"
+#include "duckdb/storage/write_ahead_log.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 
 namespace duckdb {
 
@@ -86,10 +87,9 @@ public:
 		// compute and verify the checksum
 		auto computed_checksum = Checksum(buffer.get(), size);
 		if (stored_checksum != computed_checksum) {
-			throw SerializationException(
-			    "Corrupt WAL file: entry at byte position %llu computed checksum %llu does not match "
-			    "stored checksum %llu",
-			    offset, computed_checksum, stored_checksum);
+			throw IOException("Corrupt WAL file: entry at byte position %llu computed checksum %llu does not match "
+			                  "stored checksum %llu",
+			                  offset, computed_checksum, stored_checksum);
 		}
 		return WriteAheadLogDeserializer(state_p, std::move(buffer), size, deserialize_only);
 	}
@@ -143,6 +143,7 @@ protected:
 
 	void ReplayUseTable();
 	void ReplayInsert();
+	void ReplayRowGroupData();
 	void ReplayDelete();
 	void ReplayUpdate();
 	void ReplayCheckpoint();
@@ -310,6 +311,9 @@ void WriteAheadLogDeserializer::ReplayEntry(WALType entry_type) {
 	case WALType::INSERT_TUPLE:
 		ReplayInsert();
 		break;
+	case WALType::ROW_GROUP_DATA:
+		ReplayRowGroupData();
+		break;
 	case WALType::DELETE_TUPLE:
 		ReplayDelete();
 		break;
@@ -551,7 +555,7 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() {
 
 			// read the data into a buffer handle
 			shared_ptr<BlockHandle> block_handle;
-			buffer_manager.Allocate(MemoryTag::ART_INDEX, Storage::BLOCK_SIZE, false, &block_handle);
+			buffer_manager.Allocate(MemoryTag::ART_INDEX, block_manager->GetBlockSize(), false, &block_handle);
 			auto buffer_handle = buffer_manager.Pin(block_handle);
 			auto data_ptr = buffer_handle.Ptr();
 
@@ -581,20 +585,7 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() {
 
 	// create the index in the catalog
 	auto &table = catalog.GetEntry<TableCatalogEntry>(context, create_info->schema, info.table).Cast<DuckTableEntry>();
-	auto &index = catalog.CreateIndex(context, info)->Cast<DuckIndexEntry>();
-	index.info = make_shared_ptr<IndexDataTableInfo>(table.GetStorage().GetDataTableInfo(), index.name);
-
-	// insert the parsed expressions into the index so that we can (de)serialize them during consecutive checkpoints
-	for (auto &parsed_expr : info.parsed_expressions) {
-		index.parsed_expressions.push_back(parsed_expr->Copy());
-	}
-
-	// obtain the parsed expressions of the ART from the index metadata
-	vector<unique_ptr<ParsedExpression>> parsed_expressions;
-	for (auto &parsed_expr : info.parsed_expressions) {
-		parsed_expressions.push_back(parsed_expr->Copy());
-	}
-	D_ASSERT(!parsed_expressions.empty());
+	auto &index = table.schema.CreateIndex(context, info, table)->Cast<DuckIndexEntry>();
 
 	// add the table to the bind context to bind the parsed expressions
 	auto binder = Binder::CreateBinder(context);
@@ -612,9 +603,10 @@ void WriteAheadLogDeserializer::ReplayCreateIndex() {
 
 	// bind the parsed expressions to create unbound expressions
 	vector<unique_ptr<Expression>> unbound_expressions;
-	unbound_expressions.reserve(parsed_expressions.size());
-	for (auto &expr : parsed_expressions) {
-		unbound_expressions.push_back(idx_binder.Bind(expr));
+	unbound_expressions.reserve(index.parsed_expressions.size());
+	for (auto &expr : index.parsed_expressions) {
+		auto copy = expr->Copy();
+		unbound_expressions.push_back(idx_binder.Bind(copy));
 	}
 
 	auto &data_table = table.GetStorage();
@@ -666,6 +658,54 @@ void WriteAheadLogDeserializer::ReplayInsert() {
 	state.current_table->GetStorage().LocalAppend(*state.current_table, context, chunk, bound_constraints);
 }
 
+static void MarkBlocksAsUsed(BlockManager &manager, const PersistentColumnData &col_data) {
+	for (auto &pointer : col_data.pointers) {
+		auto block_id = pointer.block_pointer.block_id;
+		if (block_id != INVALID_BLOCK) {
+			manager.MarkBlockAsUsed(block_id);
+		}
+		if (pointer.segment_state) {
+			for (auto &block : pointer.segment_state->blocks) {
+				manager.MarkBlockAsUsed(block);
+			}
+		}
+	}
+	for (auto &child_column : col_data.child_columns) {
+		MarkBlocksAsUsed(manager, child_column);
+	}
+}
+
+void WriteAheadLogDeserializer::ReplayRowGroupData() {
+	auto &block_manager = db.GetStorageManager().GetBlockManager();
+	PersistentCollectionData data;
+	deserializer.Set<DatabaseInstance &>(db.GetDatabase());
+	CompressionInfo compression_info(block_manager.GetBlockSize());
+	deserializer.Set<const CompressionInfo &>(compression_info);
+	deserializer.ReadProperty(101, "row_group_data", data);
+	deserializer.Unset<const CompressionInfo>();
+	deserializer.Unset<DatabaseInstance>();
+	if (DeserializeOnly()) {
+		// label blocks in data as used - they will be used after the WAL replay is finished
+		// we need to do this during the deserialization phase to ensure the blocks will not be overwritten
+		// by previous deserialization steps
+		for (auto &group : data.row_group_data) {
+			for (auto &col_data : group.column_data) {
+				MarkBlocksAsUsed(block_manager, col_data);
+			}
+		}
+		return;
+	}
+	if (!state.current_table) {
+		throw InternalException("Corrupt WAL: insert without table");
+	}
+	auto &storage = state.current_table->GetStorage();
+	auto &table_info = storage.GetDataTableInfo();
+	RowGroupCollection new_row_groups(table_info, block_manager, storage.GetTypes(), 0);
+	new_row_groups.Initialize(data);
+	TableIndexList index_list;
+	storage.MergeStorage(new_row_groups, index_list, nullptr);
+}
+
 void WriteAheadLogDeserializer::ReplayDelete() {
 	DataChunk chunk;
 	deserializer.ReadObject(101, "chunk", [&](Deserializer &object) { chunk.Deserialize(object); });

package/src/duckdb/src/storage/write_ahead_log.cpp

@@ -15,48 +15,69 @@
 #include "duckdb/storage/table_io_manager.hpp"
 #include "duckdb/common/checksum.hpp"
 #include "duckdb/common/serializer/memory_stream.hpp"
+#include "duckdb/storage/table/column_data.hpp"
 
 namespace duckdb {
 
 const uint64_t WAL_VERSION_NUMBER = 2;
 
 WriteAheadLog::WriteAheadLog(AttachedDatabase &database, const string &wal_path)
-    : skip_writing(false), database(database), wal_path(wal_path) {
+    : database(database), wal_path(wal_path), wal_size(0), initialized(false) {
 }
 
 WriteAheadLog::~WriteAheadLog() {
 }
 
 BufferedFileWriter &WriteAheadLog::Initialize() {
+	if (initialized) {
+		return *writer;
+	}
+	lock_guard<mutex> lock(wal_lock);
 	if (!writer) {
 		writer = make_uniq<BufferedFileWriter>(FileSystem::Get(database), wal_path,
 		                                       FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE |
 		                                           FileFlags::FILE_FLAGS_APPEND);
+		wal_size = writer->GetFileSize();
+		initialized = true;
+	}
+	return *writer;
+}
+
+//! Gets the total bytes written to the WAL since startup
+idx_t WriteAheadLog::GetWALSize() {
+	if (!Initialized()) {
+		auto &fs = FileSystem::Get(database);
+		if (!fs.FileExists(wal_path)) {
+			return 0;
+		}
+		Initialize();
 	}
-	return GetWriter();
+	return wal_size;
 }
 
 idx_t WriteAheadLog::GetTotalWritten() {
-	if (!writer) {
+	if (!Initialized()) {
 		return 0;
 	}
 	return writer->GetTotalWritten();
 }
 
-void WriteAheadLog::Truncate(int64_t size) {
-	if (!writer) {
+void WriteAheadLog::Truncate(idx_t size) {
+	if (!Initialized()) {
 		return;
 	}
 	writer->Truncate(size);
+	wal_size = writer->GetFileSize();
 }
 
 void WriteAheadLog::Delete() {
-	if (!writer) {
+	if (!Initialized()) {
 		return;
 	}
 	writer.reset();
 	auto &fs = FileSystem::Get(database);
 	fs.RemoveFile(wal_path);
+	wal_size = 0;
 }
 
 //===--------------------------------------------------------------------===//
@@ -68,17 +89,11 @@ public:
 	}
 
 	void WriteData(const_data_ptr_t buffer, idx_t write_size) override {
-		if (wal.skip_writing) {
-			return;
-		}
 		// buffer data into the memory stream
 		memory_stream.WriteData(buffer, write_size);
 	}
 
 	void Flush() {
-		if (wal.skip_writing) {
-			return;
-		}
 		if (!stream) {
 			stream = wal.Initialize();
 		}
@@ -103,11 +118,7 @@ private:
 
 class WriteAheadLogSerializer {
 public:
-	WriteAheadLogSerializer(WriteAheadLog &wal, WALType wal_type)
-	    : wal(wal), checksum_writer(wal), serializer(checksum_writer) {
-		if (wal.skip_writing) {
-			return;
-		}
+	WriteAheadLogSerializer(WriteAheadLog &wal, WALType wal_type) : checksum_writer(wal), serializer(checksum_writer) {
 		if (!wal.Initialized()) {
 			wal.Initialize();
 		}
@@ -118,34 +129,21 @@ public:
 	}
 
 	void End() {
-		if (wal.skip_writing) {
-			return;
-		}
-		D_ASSERT(wal.Initialized());
 		serializer.End();
 		checksum_writer.Flush();
 	}
 
 	template <class T>
 	void WriteProperty(const field_id_t field_id, const char *tag, const T &value) {
-		if (wal.skip_writing) {
-			return;
-		}
-		D_ASSERT(wal.Initialized());
 		serializer.WriteProperty(field_id, tag, value);
 	}
 
 	template <class FUNC>
 	void WriteList(const field_id_t field_id, const char *tag, idx_t count, FUNC func) {
-		if (wal.skip_writing) {
-			return;
-		}
-		D_ASSERT(wal.Initialized());
 		serializer.WriteList(field_id, tag, count, func);
 	}
 
 private:
-	WriteAheadLog &wal;
 	ChecksumWriter checksum_writer;
 	BinarySerializer serializer;
 };
@@ -261,11 +259,12 @@ void WriteAheadLog::WriteDropTableMacro(const TableMacroCatalogEntry &entry) {
 // Indexes
 //===--------------------------------------------------------------------===//
 
-void SerializeIndexToWAL(WriteAheadLogSerializer &serializer, const unique_ptr<Index> &index) {
+void SerializeIndexToWAL(WriteAheadLogSerializer &serializer, Index &index,
+                         const case_insensitive_map_t<Value> &options) {
 
 	// We will never write an index to the WAL that is not bound
-	D_ASSERT(index->IsBound());
-	auto index_storage_info = index->Cast<BoundIndex>().GetStorageInfo(true);
+	D_ASSERT(index.IsBound());
+	const auto index_storage_info = index.Cast<BoundIndex>().GetStorageInfo(options, true);
 	serializer.WriteProperty(102, "index_storage_info", index_storage_info);
 
 	serializer.WriteList(103, "index_storage", index_storage_info.buffers.size(), [&](Serializer::List &list, idx_t i) {
@@ -277,25 +276,27 @@ void SerializeIndexToWAL(WriteAheadLogSerializer &serializer, const unique_ptr<I
 }
 
 void WriteAheadLog::WriteCreateIndex(const IndexCatalogEntry &entry) {
-	if (skip_writing) {
-		return;
-	}
-
 	WriteAheadLogSerializer serializer(*this, WALType::CREATE_INDEX);
 	serializer.WriteProperty(101, "index_catalog_entry", &entry);
 
+	auto db_options = database.GetDatabase().config.options;
+	auto v1_0_0_storage = db_options.serialization_compatibility.serialization_version < 3;
+	case_insensitive_map_t<Value> options;
+	if (!v1_0_0_storage) {
+		options.emplace("v1_0_0_storage", v1_0_0_storage);
+	}
+
 	// now serialize the index data to the persistent storage and write the index metadata
 	auto &duck_index_entry = entry.Cast<DuckIndexEntry>();
-	auto &indexes = duck_index_entry.GetDataTableInfo().GetIndexes().Indexes();
+	auto &table_idx_list = duck_index_entry.GetDataTableInfo().GetIndexes();
 
-	// get the matching index and serialize its storage info
-	for (auto const &index : indexes) {
-		if (duck_index_entry.name == index->GetIndexName()) {
-			SerializeIndexToWAL(serializer, index);
-			break;
+	table_idx_list.Scan([&](Index &index) {
+		if (duck_index_entry.name == index.GetIndexName()) {
+			SerializeIndexToWAL(serializer, index, options);
+			return true;
 		}
-	}
-
+		return false;
+	});
 	serializer.End();
 }
 
@@ -366,6 +367,14 @@ void WriteAheadLog::WriteInsert(DataChunk &chunk) {
 	serializer.End();
 }
 
+void WriteAheadLog::WriteRowGroupData(const PersistentCollectionData &data) {
+	D_ASSERT(!data.row_group_data.empty());
+
+	WriteAheadLogSerializer serializer(*this, WALType::ROW_GROUP_DATA);
+	serializer.WriteProperty(101, "row_group_data", data);
+	serializer.End();
+}
+
 void WriteAheadLog::WriteDelete(DataChunk &chunk) {
 	D_ASSERT(chunk.size() > 0);
 	D_ASSERT(chunk.ColumnCount() == 1 && chunk.data[0].GetType() == LogicalType::ROW_TYPE);
@@ -401,10 +410,9 @@ void WriteAheadLog::WriteAlter(const AlterInfo &info) {
 // FLUSH
 //===--------------------------------------------------------------------===//
 void WriteAheadLog::Flush() {
-	if (skip_writing) {
+	if (!writer) {
 		return;
 	}
-	D_ASSERT(writer);
 
 	// write an empty entry
 	WriteAheadLogSerializer serializer(*this, WALType::WAL_FLUSH);
@@ -412,6 +420,7 @@ void WriteAheadLog::Flush() {
 
 	// flushes all changes made to the WAL to disk
 	writer->Sync();
+	wal_size = writer->GetFileSize();
 }
 
 } // namespace duckdb

package/src/duckdb/src/transaction/cleanup_state.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/transaction/cleanup_state.hpp"
 #include "duckdb/transaction/delete_info.hpp"
 #include "duckdb/transaction/update_info.hpp"
+#include "duckdb/transaction/append_info.hpp"
 
 #include "duckdb/storage/data_table.hpp"
 
@@ -12,7 +13,8 @@
 
 namespace duckdb {
 
-CleanupState::CleanupState() : current_table(nullptr), count(0) {
+CleanupState::CleanupState(transaction_t lowest_active_transaction)
+    : lowest_active_transaction(lowest_active_transaction), current_table(nullptr), count(0) {
 }
 
 CleanupState::~CleanupState() {
@@ -29,6 +31,12 @@ void CleanupState::CleanupEntry(UndoFlags type, data_ptr_t data) {
 		entry.set->CleanupEntry(entry);
 		break;
 	}
+	case UndoFlags::INSERT_TUPLE: {
+		auto info = reinterpret_cast<AppendInfo *>(data);
+		// mark the tuples as committed
+		info->table->CleanupAppend(lowest_active_transaction, info->start_row, info->count);
+		break;
+	}
 	case UndoFlags::DELETE_TUPLE: {
 		auto info = reinterpret_cast<DeleteInfo *>(data);
 		CleanupDelete(*info);