duckdb 0.5.2-dev737.0 → 0.5.2-dev746.0

package/src/duckdb.cpp

@@ -17587,9 +17587,15 @@ public:
 	                           idx_t result_idx);
 	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id);
 
-	static idx_t StringAppend(ColumnSegment &segment, SegmentStatistics &stats, UnifiedVectorFormat &data, idx_t offset,
-	                          idx_t count) {
-		return StringAppendBase(segment, stats, data, offset, count);
+	static unique_ptr<CompressionAppendState> StringInitAppend(ColumnSegment &segment) {
+		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
+		auto handle = buffer_manager.Pin(segment.block);
+		return make_unique<CompressionAppendState>(move(handle));
+	}
+
+	static idx_t StringAppend(CompressionAppendState &append_state, ColumnSegment &segment, SegmentStatistics &stats,
+	                          UnifiedVectorFormat &data, idx_t offset, idx_t count) {
+		return StringAppendBase(append_state.handle, segment, stats, data, offset, count);
 	}
 
 	template <bool DUPLICATE_ELIMINATE = false>
@@ -17598,7 +17604,12 @@ public:
 	                              std::unordered_map<string, int32_t> *seen_strings = nullptr) {
 		auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 		auto handle = buffer_manager.Pin(segment.block);
-
+		return StringAppendBase(handle, segment, stats, data, offset, count);
+	}
+	template <bool DUPLICATE_ELIMINATE = false>
+	static idx_t StringAppendBase(BufferHandle &handle, ColumnSegment &segment, SegmentStatistics &stats,
+	                              UnifiedVectorFormat &data, idx_t offset, idx_t count,
+	                              std::unordered_map<string, int32_t> *seen_strings = nullptr) {
 		D_ASSERT(segment.GetBlockOffset() == 0);
 		auto source_data = (string_t *)data.data;
 		auto result_data = (int32_t *)(handle.Ptr() + DICTIONARY_HEADER_SIZE);
@@ -78452,10 +78463,12 @@ public:
 public:
 	// Sink interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
-	void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
 	                    DataChunk &input) const override;
+	void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          GlobalSinkState &gstate) const override;
 
 	bool IsSink() const override {
 		return true;
@@ -78478,17 +78491,27 @@ public:
 
 namespace duckdb {
 
+PhysicalInsert::PhysicalInsert(vector<LogicalType> types, TableCatalogEntry *table, vector<idx_t> column_index_map,
+                               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality,
+                               bool return_chunk)
+    : PhysicalOperator(PhysicalOperatorType::INSERT, move(types), estimated_cardinality),
+      column_index_map(std::move(column_index_map)), table(table), bound_defaults(move(bound_defaults)),
+      return_chunk(return_chunk) {
+}
+
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
 class InsertGlobalState : public GlobalSinkState {
 public:
 	explicit InsertGlobalState(ClientContext &context, const vector<LogicalType> &return_types)
-	    : insert_count(0), return_collection(context, return_types) {
+	    : insert_count(0), initialized(false), return_collection(context, return_types) {
 	}
 
 	mutex lock;
 	idx_t insert_count;
+	LocalAppendState append_state;
+	bool initialized;
 	ColumnDataCollection return_collection;
 };
 
@@ -78504,12 +78527,12 @@ public:
 	ExpressionExecutor default_executor;
 };
 
-PhysicalInsert::PhysicalInsert(vector<LogicalType> types, TableCatalogEntry *table, vector<idx_t> column_index_map,
-                               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality,
-                               bool return_chunk)
-    : PhysicalOperator(PhysicalOperatorType::INSERT, move(types), estimated_cardinality),
-      column_index_map(std::move(column_index_map)), table(table), bound_defaults(move(bound_defaults)),
-      return_chunk(return_chunk) {
+unique_ptr<GlobalSinkState> PhysicalInsert::GetGlobalSinkState(ClientContext &context) const {
+	return make_unique<InsertGlobalState>(context, GetTypes());
+}
+
+unique_ptr<LocalSinkState> PhysicalInsert::GetLocalSinkState(ExecutionContext &context) const {
+	return make_unique<InsertLocalState>(Allocator::Get(context.client), table->GetTypes(), bound_defaults);
 }
 
 SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
@@ -78550,7 +78573,11 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &
 	}
 
 	lock_guard<mutex> glock(gstate.lock);
-	table->storage->Append(*table, context.client, istate.insert_chunk);
+	if (!gstate.initialized) {
+		table->storage->InitializeLocalAppend(gstate.append_state, context.client);
+		gstate.initialized = true;
+	}
+	table->storage->LocalAppend(gstate.append_state, *table, context.client, istate.insert_chunk);
 
 	if (return_chunk) {
 		gstate.return_collection.Append(istate.insert_chunk);
@@ -78560,14 +78587,6 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-unique_ptr<GlobalSinkState> PhysicalInsert::GetGlobalSinkState(ClientContext &context) const {
-	return make_unique<InsertGlobalState>(context, GetTypes());
-}
-
-unique_ptr<LocalSinkState> PhysicalInsert::GetLocalSinkState(ExecutionContext &context) const {
-	return make_unique<InsertLocalState>(Allocator::Get(context.client), table->GetTypes(), bound_defaults);
-}
-
 void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
 	auto &state = (InsertLocalState &)lstate;
 	auto &client_profiler = QueryProfiler::Get(context.client);
@@ -78575,6 +78594,15 @@ void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate,
 	client_profiler.Flush(context.thread.profiler);
 }
 
+SinkFinalizeType PhysicalInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+                                          GlobalSinkState &state) const {
+	auto &gstate = (InsertGlobalState &)state;
+	if (gstate.initialized) {
+		table->storage->FinalizeLocalAppend(gstate.append_state);
+	}
+	return SinkFinalizeType::READY;
+}
+
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
@@ -78775,7 +78803,7 @@ SinkResultType PhysicalUpdate::Sink(ExecutionContext &context, GlobalSinkState &
 		for (idx_t i = 0; i < columns.size(); i++) {
 			mock_chunk.data[columns[i]].Reference(update_chunk.data[i]);
 		}
-		table.Append(tableref, context.client, mock_chunk);
+		table.LocalAppend(tableref, context.client, mock_chunk);
 	} else {
 		if (return_chunk) {
 			mock_chunk.SetCardinality(update_chunk);
@@ -80430,9 +80458,10 @@ public:
 
 public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
-
 	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
 	                    DataChunk &input) const override;
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          GlobalSinkState &gstate) const override;
 
 	bool IsSink() const override {
 		return true;
@@ -80461,13 +80490,14 @@ PhysicalCreateTableAs::PhysicalCreateTableAs(LogicalOperator &op, SchemaCatalogE
 //===--------------------------------------------------------------------===//
 class CreateTableAsGlobalState : public GlobalSinkState {
 public:
-	CreateTableAsGlobalState() {
-		inserted_count = 0;
+	CreateTableAsGlobalState() : inserted_count(0), initialized(false) {
 	}
 
 	mutex append_lock;
 	TableCatalogEntry *table;
 	int64_t inserted_count;
+	LocalAppendState append_state;
+	bool initialized;
 };
 
 unique_ptr<GlobalSinkState> PhysicalCreateTableAs::GetGlobalSinkState(ClientContext &context) const {
@@ -80482,11 +80512,24 @@ SinkResultType PhysicalCreateTableAs::Sink(ExecutionContext &context, GlobalSink
 	auto &sink = (CreateTableAsGlobalState &)state;
 	D_ASSERT(sink.table);
 	lock_guard<mutex> client_guard(sink.append_lock);
-	sink.table->storage->Append(*sink.table, context.client, input);
+	if (!sink.initialized) {
+		sink.table->storage->InitializeLocalAppend(sink.append_state, context.client);
+		sink.initialized = true;
+	}
+	sink.table->storage->LocalAppend(sink.append_state, *sink.table, context.client, input);
 	sink.inserted_count += input.size();
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
+SinkFinalizeType PhysicalCreateTableAs::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+                                                 GlobalSinkState &state) const {
+	auto &gstate = (CreateTableAsGlobalState &)state;
+	if (gstate.initialized) {
+		gstate.table->storage->FinalizeLocalAppend(gstate.append_state);
+	}
+	return SinkFinalizeType::READY;
+}
+
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
@@ -125062,9 +125105,7 @@ void Appender::FlushInternal(ColumnDataCollection &collection) {
 }
 
 void InternalAppender::FlushInternal(ColumnDataCollection &collection) {
-	for (auto &chunk : collection.Chunks()) {
-		table.storage->Append(table, context, chunk);
-	}
+	table.storage->LocalAppend(table, context, collection);
 }
 
 void BaseAppender::Close() {
@@ -129455,9 +129496,7 @@ void ClientContext::Append(TableDescription &description, ColumnDataCollection &
 				throw Exception("Failed to append: table entry has different number of columns!");
 			}
 		}
-		for (auto &chunk : collection.Chunks()) {
-			table_entry->storage->Append(*table_entry, *this, chunk);
-		}
+		table_entry->storage->LocalAppend(*table_entry, *this, collection);
 	});
 }
 
@@ -130935,7 +130974,7 @@ DBConfig::DBConfig(std::unordered_map<string, string> &config_dict, bool read_on
 DBConfig::~DBConfig() {
 }
 
-DatabaseInstance::DatabaseInstance() {
+DatabaseInstance::DatabaseInstance() : is_invalidated(false) {
 }
 
 DatabaseInstance::~DatabaseInstance() {
@@ -198390,6 +198429,18 @@ idx_t FixedSizeFinalAnalyze(AnalyzeState &state_p) {
 //===--------------------------------------------------------------------===//
 // Compress
 //===--------------------------------------------------------------------===//
+struct UncompressedCompressState : public CompressionState {
+	explicit UncompressedCompressState(ColumnDataCheckpointer &checkpointer);
+
+	ColumnDataCheckpointer &checkpointer;
+	unique_ptr<ColumnSegment> current_segment;
+	ColumnAppendState append_state;
+
+	virtual void CreateEmptySegment(idx_t row_start);
+	void FlushSegment(idx_t segment_size);
+	void Finalize(idx_t segment_size);
+};
+
 UncompressedCompressState::UncompressedCompressState(ColumnDataCheckpointer &checkpointer)
     : checkpointer(checkpointer) {
 	CreateEmptySegment(checkpointer.GetRowGroup().start);
@@ -198404,6 +198455,7 @@ void UncompressedCompressState::CreateEmptySegment(idx_t row_start) {
 		state.overflow_writer = make_unique<WriteOverflowStringsToDisk>(checkpointer.GetColumnData().block_manager);
 	}
 	current_segment = move(compressed_segment);
+	current_segment->InitializeAppend(append_state);
 }
 
 void UncompressedCompressState::FlushSegment(idx_t segment_size) {
@@ -198426,17 +198478,16 @@ void UncompressedFunctions::Compress(CompressionState &state_p, Vector &data, id
 	UnifiedVectorFormat vdata;
 	data.ToUnifiedFormat(count, vdata);
 
-	ColumnAppendState append_state;
 	idx_t offset = 0;
 	while (count > 0) {
-		idx_t appended = state.current_segment->Append(append_state, vdata, offset, count);
+		idx_t appended = state.current_segment->Append(state.append_state, vdata, offset, count);
 		if (appended == count) {
 			// appended everything: finished
 			return;
 		}
 		auto next_start = state.current_segment->start + state.current_segment->count;
 		// the segment is full: flush it to disk
-		state.FlushSegment(state.current_segment->FinalizeAppend());
+		state.FlushSegment(state.current_segment->FinalizeAppend(state.append_state));
 
 		// now create a new segment and continue appending
 		state.CreateEmptySegment(next_start);
@@ -198447,7 +198498,7 @@ void UncompressedFunctions::Compress(CompressionState &state_p, Vector &data, id
 
 void UncompressedFunctions::FinalizeCompress(CompressionState &state_p) {
 	auto &state = (UncompressedCompressState &)state_p;
-	state.Finalize(state.current_segment->FinalizeAppend());
+	state.Finalize(state.current_segment->FinalizeAppend(state.append_state));
 }
 
 //===--------------------------------------------------------------------===//
@@ -198517,6 +198568,12 @@ void FixedSizeFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t ro
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
+static unique_ptr<CompressionAppendState> FixedSizeInitAppend(ColumnSegment &segment) {
+	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
+	auto handle = buffer_manager.Pin(segment.block);
+	return make_unique<CompressionAppendState>(move(handle));
+}
+
 template <class T>
 static void AppendLoop(SegmentStatistics &stats, data_ptr_t target, idx_t target_offset, UnifiedVectorFormat &adata,
                        idx_t offset, idx_t count) {
@@ -198559,13 +198616,11 @@ void AppendLoop<list_entry_t>(SegmentStatistics &stats, data_ptr_t target, idx_t
 }
 
 template <class T>
-idx_t FixedSizeAppend(ColumnSegment &segment, SegmentStatistics &stats, UnifiedVectorFormat &data, idx_t offset,
-                      idx_t count) {
-	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
-	auto handle = buffer_manager.Pin(segment.block);
+idx_t FixedSizeAppend(CompressionAppendState &append_state, ColumnSegment &segment, SegmentStatistics &stats,
+                      UnifiedVectorFormat &data, idx_t offset, idx_t count) {
 	D_ASSERT(segment.GetBlockOffset() == 0);
 
-	auto target_ptr = handle.Ptr();
+	auto target_ptr = append_state.handle.Ptr();
 	idx_t max_tuple_count = Storage::BLOCK_SIZE / sizeof(T);
 	idx_t copy_count = MinValue<idx_t>(count, max_tuple_count - segment.count);
 
@@ -198588,7 +198643,7 @@ CompressionFunction FixedSizeGetFunction(PhysicalType data_type) {
 	                           FixedSizeAnalyze, FixedSizeFinalAnalyze<T>, UncompressedFunctions::InitCompression,
 	                           UncompressedFunctions::Compress, UncompressedFunctions::FinalizeCompress,
 	                           FixedSizeInitScan, FixedSizeScan<T>, FixedSizeScanPartial<T>, FixedSizeFetchRow<T>,
-	                           UncompressedFunctions::EmptySkip, nullptr, FixedSizeAppend<T>,
+	                           UncompressedFunctions::EmptySkip, nullptr, FixedSizeInitAppend, FixedSizeAppend<T>,
 	                           FixedSizeFinalizeAppend<T>, nullptr);
 }
 
@@ -200114,7 +200169,8 @@ CompressionFunction StringUncompressed::GetFunction(PhysicalType data_type) {
 	                           UncompressedStringStorage::StringInitScan, UncompressedStringStorage::StringScan,
 	                           UncompressedStringStorage::StringScanPartial, UncompressedStringStorage::StringFetchRow,
 	                           UncompressedFunctions::EmptySkip, UncompressedStringStorage::StringInitSegment,
-	                           UncompressedStringStorage::StringAppend, UncompressedStringStorage::FinalizeAppend);
+	                           UncompressedStringStorage::StringInitAppend, UncompressedStringStorage::StringAppend,
+	                           UncompressedStringStorage::FinalizeAppend);
 }
 
 //===--------------------------------------------------------------------===//
@@ -200758,6 +200814,12 @@ void ValidityFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
+static unique_ptr<CompressionAppendState> ValidityInitAppend(ColumnSegment &segment) {
+	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
+	auto handle = buffer_manager.Pin(segment.block);
+	return make_unique<CompressionAppendState>(move(handle));
+}
+
 unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, block_id_t block_id) {
 	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
 	if (block_id == INVALID_BLOCK) {
@@ -200767,8 +200829,8 @@ unique_ptr<CompressedSegmentState> ValidityInitSegment(ColumnSegment &segment, b
 	return nullptr;
 }
 
-idx_t ValidityAppend(ColumnSegment &segment, SegmentStatistics &stats, UnifiedVectorFormat &data, idx_t offset,
-                     idx_t vcount) {
+idx_t ValidityAppend(CompressionAppendState &append_state, ColumnSegment &segment, SegmentStatistics &stats,
+                     UnifiedVectorFormat &data, idx_t offset, idx_t vcount) {
 	D_ASSERT(segment.GetBlockOffset() == 0);
 	auto &validity_stats = (ValidityStatistics &)*stats.statistics;
 
@@ -200780,10 +200842,8 @@ idx_t ValidityAppend(ColumnSegment &segment, SegmentStatistics &stats, UnifiedVe
 		validity_stats.has_no_null = true;
 		return append_count;
 	}
-	auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
-	auto handle = buffer_manager.Pin(segment.block);
 
-	ValidityMask mask((validity_t *)handle.Ptr());
+	ValidityMask mask((validity_t *)append_state.handle.Ptr());
 	for (idx_t i = 0; i < append_count; i++) {
 		auto idx = data.sel->get_index(offset + i);
 		if (!data.validity.RowIsValidUnsafe(idx)) {
@@ -200833,8 +200893,8 @@ CompressionFunction ValidityUncompressed::GetFunction(PhysicalType data_type) {
 	                           ValidityAnalyze, ValidityFinalAnalyze, UncompressedFunctions::InitCompression,
 	                           UncompressedFunctions::Compress, UncompressedFunctions::FinalizeCompress,
 	                           ValidityInitScan, ValidityScan, ValidityScanPartial, ValidityFetchRow,
-	                           UncompressedFunctions::EmptySkip, ValidityInitSegment, ValidityAppend,
-	                           ValidityFinalizeAppend, ValidityRevertAppend);
+	                           UncompressedFunctions::EmptySkip, ValidityInitSegment, ValidityInitAppend,
+	                           ValidityAppend, ValidityFinalizeAppend, ValidityRevertAppend);
 }
 
 } // namespace duckdb
@@ -201408,14 +201468,20 @@ void DataTable::VerifyAppendConstraints(TableCatalogEntry &table, ClientContext
 	}
 }
 
-void DataTable::Append(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) {
+void DataTable::InitializeLocalAppend(LocalAppendState &state, ClientContext &context) {
+	if (!is_root) {
+		throw TransactionException("Transaction conflict: adding entries to a table that has been altered!");
+	}
+	auto &transaction = Transaction::GetTransaction(context);
+	transaction.storage.InitializeAppend(state, this);
+}
+
+void DataTable::LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context,
+                            DataChunk &chunk) {
 	if (chunk.size() == 0) {
 		return;
 	}
-	// FIXME: could be an assertion instead?
-	if (chunk.ColumnCount() != table.StandardColumnCount()) {
-		throw InternalException("Mismatch in column count for append");
-	}
+	D_ASSERT(chunk.ColumnCount() == table.StandardColumnCount());
 	if (!is_root) {
 		throw TransactionException("Transaction conflict: adding entries to a table that has been altered!");
 	}
@@ -201426,8 +201492,27 @@ void DataTable::Append(TableCatalogEntry &table, ClientContext &context, DataChu
 	VerifyAppendConstraints(table, context, chunk);
 
 	// append to the transaction local data
-	auto &transaction = Transaction::GetTransaction(context);
-	transaction.storage.Append(this, chunk);
+	LocalStorage::Append(state, chunk);
+}
+
+void DataTable::FinalizeLocalAppend(LocalAppendState &state) {
+	LocalStorage::FinalizeAppend(state);
+}
+
+void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) {
+	LocalAppendState append_state;
+	table.storage->InitializeLocalAppend(append_state, context);
+	table.storage->LocalAppend(append_state, table, context, chunk);
+	table.storage->FinalizeLocalAppend(append_state);
+}
+
+void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection) {
+	LocalAppendState append_state;
+	table.storage->InitializeLocalAppend(append_state, context);
+	for (auto &chunk : collection.Chunks()) {
+		table.storage->LocalAppend(append_state, table, context, chunk);
+	}
+	table.storage->FinalizeLocalAppend(append_state);
 }
 
 void DataTable::AppendLock(TableAppendState &state) {
@@ -201446,9 +201531,9 @@ void DataTable::InitializeAppend(Transaction &transaction, TableAppendState &sta
 	row_groups->InitializeAppend(transaction, state, append_count);
 }
 
-void DataTable::Append(Transaction &transaction, DataChunk &chunk, TableAppendState &state) {
+void DataTable::Append(DataChunk &chunk, TableAppendState &state) {
 	D_ASSERT(is_root);
-	row_groups->Append(transaction, chunk, state, stats);
+	row_groups->Append(chunk, state, stats);
 }
 
 void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::function<void(DataChunk &chunk)> &function) {
@@ -202097,27 +202182,33 @@ bool LocalStorage::NextParallelScan(ClientContext &context, DataTable *table, Pa
 	return storage->row_groups->NextParallelScan(context, state, scan_state);
 }
 
-void LocalStorage::Append(DataTable *table, DataChunk &chunk) {
+void LocalStorage::InitializeAppend(LocalAppendState &state, DataTable *table) {
 	auto entry = table_storage.find(table);
-	LocalTableStorage *storage;
 	if (entry == table_storage.end()) {
 		auto new_storage = make_shared<LocalTableStorage>(*table);
-		storage = new_storage.get();
+		state.storage = new_storage.get();
 		table_storage.insert(make_pair(table, move(new_storage)));
 	} else {
-		storage = entry->second.get();
+		state.storage = entry->second.get();
 	}
+	state.storage->row_groups->InitializeAppend(state.append_state);
+}
+
+void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 	// append to unique indices (if any)
+	auto storage = state.storage;
 	idx_t base_id = MAX_ROW_ID + storage->row_groups->GetTotalRows();
 	if (!DataTable::AppendToIndexes(storage->indexes, chunk, base_id)) {
 		throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
 	}
 
 	//! Append to the chunk
-	TableAppendState state;
+	storage->row_groups->Append(chunk, state.append_state, storage->stats);
+}
+
+void LocalStorage::FinalizeAppend(LocalAppendState &state) {
 	TransactionData transaction_data(0, 0);
-	storage->row_groups->InitializeAppend(transaction_data, state, chunk.size());
-	storage->row_groups->Append(transaction_data, chunk, state, storage->stats);
+	state.storage->row_groups->FinalizeAppend(transaction_data, state.append_state);
 }
 
 LocalTableStorage *LocalStorage::GetStorage(DataTable *table) {
@@ -202213,7 +202304,7 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 				return false;
 			}
 			// append to base table
-			table.Append(transaction, chunk, append_state);
+			table.Append(chunk, append_state);
 			return true;
 		});
 		if (constraint_violated) {
@@ -202238,8 +202329,7 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	transaction.PushAppend(&table, append_state.row_start, append_count);
 }
 
-void LocalStorage::Commit(LocalStorage::CommitState &commit_state, Transaction &transaction, WriteAheadLog *log,
-                          transaction_t commit_id) {
+void LocalStorage::Commit(LocalStorage::CommitState &commit_state, Transaction &transaction) {
 	// commit local storage, iterate over all entries in the table storage map
 	for (auto &entry : table_storage) {
 		auto table = entry.first;
@@ -206001,27 +206091,33 @@ void ColumnSegment::FetchRow(ColumnFetchState &state, row_t row_id, Vector &resu
 	function->fetch_row(*this, state, row_id - this->start, result, result_idx);
 }
 
+//===--------------------------------------------------------------------===//
+// Append
+//===--------------------------------------------------------------------===//
 void ColumnSegment::InitializeAppend(ColumnAppendState &state) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
+	if (!function->init_append) {
+		throw InternalException("Attempting to init append to a segment without init_append method");
+	}
+	state.append_state = function->init_append(*this);
 }
 
-//===--------------------------------------------------------------------===//
-// Append
-//===--------------------------------------------------------------------===//
 idx_t ColumnSegment::Append(ColumnAppendState &state, UnifiedVectorFormat &append_data, idx_t offset, idx_t count) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
 	if (!function->append) {
 		throw InternalException("Attempting to append to a segment without append method");
 	}
-	return function->append(*this, stats, append_data, offset, count);
+	return function->append(*state.append_state, *this, stats, append_data, offset, count);
 }
 
-idx_t ColumnSegment::FinalizeAppend() {
+idx_t ColumnSegment::FinalizeAppend(ColumnAppendState &state) {
 	D_ASSERT(segment_type == ColumnSegmentType::TRANSIENT);
 	if (!function->finalize_append) {
 		throw InternalException("Attempting to call FinalizeAppend on a segment without a finalize_append method");
 	}
-	return function->finalize_append(*this, stats);
+	auto result_count = function->finalize_append(*this, stats);
+	state.append_state.reset();
+	return result_count;
 }
 
 void ColumnSegment::RevertAppend(idx_t start_row) {
@@ -207272,14 +207368,14 @@ void RowGroup::FetchRow(TransactionData transaction, ColumnFetchState &state, co
 	}
 }
 
-void RowGroup::AppendVersionInfo(TransactionData transaction, idx_t row_group_start, idx_t count,
-                                 transaction_t commit_id) {
+void RowGroup::AppendVersionInfo(TransactionData transaction, idx_t count) {
+	idx_t row_group_start = this->count.load();
 	idx_t row_group_end = row_group_start + count;
+	if (row_group_end > RowGroup::ROW_GROUP_SIZE) {
+		row_group_end = RowGroup::ROW_GROUP_SIZE;
+	}
 	lock_guard<mutex> lock(row_group_lock);
 
-	this->count += count;
-	D_ASSERT(this->count <= RowGroup::ROW_GROUP_SIZE);
-
 	// create the version_info if it doesn't exist yet
 	if (!version_info) {
 		version_info = make_unique<VersionNode>();
@@ -207293,7 +207389,7 @@ void RowGroup::AppendVersionInfo(TransactionData transaction, idx_t row_group_st
 		if (start == 0 && end == STANDARD_VECTOR_SIZE) {
 			// entire vector is encapsulated by append: append a single constant
 			auto constant_info = make_unique<ChunkConstantInfo>(this->start + vector_idx * STANDARD_VECTOR_SIZE);
-			constant_info->insert_id = commit_id;
+			constant_info->insert_id = transaction.transaction_id;
 			constant_info->delete_id = NOT_DELETED_ID;
 			version_info->info[vector_idx] = move(constant_info);
 		} else {
@@ -207309,9 +207405,10 @@ void RowGroup::AppendVersionInfo(TransactionData transaction, idx_t row_group_st
 				// use existing vector
 				info = (ChunkVectorInfo *)version_info->info[vector_idx].get();
 			}
-			info->Append(start, end, commit_id);
+			info->Append(start, end, transaction.transaction_id);
 		}
 	}
+	this->count = row_group_end;
 }
 
 void RowGroup::CommitAppend(transaction_t commit_id, idx_t row_group_start, idx_t count) {
@@ -207347,8 +207444,7 @@ void RowGroup::RevertAppend(idx_t row_group_start) {
 	Verify();
 }
 
-void RowGroup::InitializeAppend(TransactionData transaction, RowGroupAppendState &append_state,
-                                idx_t remaining_append_count) {
+void RowGroup::InitializeAppend(RowGroupAppendState &append_state) {
 	append_state.row_group = this;
 	append_state.offset_in_row_group = this->count;
 	// for each column, initialize the append state
@@ -207356,9 +207452,6 @@ void RowGroup::InitializeAppend(TransactionData transaction, RowGroupAppendState
 	for (idx_t i = 0; i < columns.size(); i++) {
 		columns[i]->InitializeAppend(append_state.states[i]);
 	}
-	// append the version info for this row_group
-	idx_t append_count = MinValue<idx_t>(remaining_append_count, RowGroup::ROW_GROUP_SIZE - this->count);
-	AppendVersionInfo(transaction, this->count, append_count, transaction.transaction_id);
 }
 
 void RowGroup::Append(RowGroupAppendState &state, DataChunk &chunk, idx_t append_count) {
@@ -207730,10 +207823,14 @@ void RowGroupCollection::AppendRowGroup(idx_t start_row) {
 
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
+	idx_t current_total_rows = 0;
 	for (auto segment = row_groups->GetRootSegment(); segment; segment = segment->next.get()) {
 		auto &row_group = (RowGroup &)*segment;
 		row_group.Verify();
+		D_ASSERT(row_group.start == this->row_start + current_total_rows);
+		current_total_rows += row_group.count;
 	}
+	D_ASSERT(current_total_rows == total_rows.load());
 #endif
 }
 
@@ -207835,14 +207932,22 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
+TableAppendState::TableAppendState()
+    : row_group_append_state(*this), total_append_count(0), start_row_group(nullptr), transaction(0, 0), remaining(0) {
+}
+
+TableAppendState::~TableAppendState() {
+	D_ASSERT(Exception::UncaughtException() || remaining == 0);
+}
+
 bool RowGroupCollection::IsEmpty() const {
 	return row_groups->GetRootSegment() == nullptr;
 }
 
 void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppendState &state, idx_t append_count) {
-	state.remaining_append_count = append_count;
 	state.row_start = total_rows;
 	state.current_row = state.row_start;
+	state.total_append_count = 0;
 
 	// start writing to the row_groups
 	lock_guard<mutex> row_group_lock(row_groups->node_lock);
@@ -207850,19 +207955,29 @@ void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppe
 		// empty row group collection: empty first row group
 		AppendRowGroup(row_start);
 	}
-	auto last_row_group = (RowGroup *)row_groups->GetLastSegment();
-	D_ASSERT(this->row_start + total_rows == last_row_group->start + last_row_group->count);
-	last_row_group->InitializeAppend(transaction, state.row_group_append_state, state.remaining_append_count);
-	total_rows += append_count;
+	state.start_row_group = (RowGroup *)row_groups->GetLastSegment();
+	D_ASSERT(this->row_start + total_rows == state.start_row_group->start + state.start_row_group->count);
+	state.start_row_group->InitializeAppend(state.row_group_append_state);
+	state.remaining = append_count;
+	if (state.remaining > 0) {
+		state.transaction = transaction;
+		state.start_row_group->AppendVersionInfo(transaction, state.remaining);
+		total_rows += state.remaining;
+	}
+}
+
+void RowGroupCollection::InitializeAppend(TableAppendState &state) {
+	TransactionData tdata(0, 0);
+	InitializeAppend(tdata, state, 0);
 }
 
-void RowGroupCollection::Append(TransactionData transaction, DataChunk &chunk, TableAppendState &state,
-                                TableStatistics &stats) {
+void RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, TableStatistics &stats) {
 	D_ASSERT(chunk.ColumnCount() == types.size());
 	chunk.Verify();
 
 	idx_t append_count = chunk.size();
 	idx_t remaining = chunk.size();
+	state.total_append_count += append_count;
 	while (true) {
 		auto current_row_group = state.row_group_append_state.row_group;
 		// check how much we can fit into the current row_group
@@ -207876,8 +207991,10 @@ void RowGroupCollection::Append(TransactionData transaction, DataChunk &chunk, T
 				current_row_group->MergeIntoStatistics(i, *stats.GetStats(i).stats);
 			}
 		}
-		state.remaining_append_count -= append_count;
 		remaining -= append_count;
+		if (state.remaining > 0) {
+			state.remaining -= append_count;
+		}
 		if (remaining > 0) {
 			// we expect max 1 iteration of this loop (i.e. a single chunk should never overflow more than one
 			// row_group)
@@ -207891,11 +208008,15 @@ void RowGroupCollection::Append(TransactionData transaction, DataChunk &chunk, T
 				chunk.Slice(sel, remaining);
 			}
 			// append a new row_group
-			AppendRowGroup(current_row_group->start + current_row_group->count);
+			auto next_start = current_row_group->start + state.row_group_append_state.offset_in_row_group;
+			AppendRowGroup(next_start);
 			// set up the append state for this row_group
 			lock_guard<mutex> row_group_lock(row_groups->node_lock);
 			auto last_row_group = (RowGroup *)row_groups->GetLastSegment();
-			last_row_group->InitializeAppend(transaction, state.row_group_append_state, state.remaining_append_count);
+			last_row_group->InitializeAppend(state.row_group_append_state);
+			if (state.remaining > 0) {
+				last_row_group->AppendVersionInfo(state.transaction, state.remaining);
+			}
 			continue;
 		} else {
 			break;
@@ -207912,6 +208033,22 @@ void RowGroupCollection::Append(TransactionData transaction, DataChunk &chunk, T
 	}
 }
 
+void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppendState &state) {
+	D_ASSERT(state.transaction.transaction_id == 0);
+	auto remaining = state.total_append_count;
+	auto row_group = state.start_row_group;
+	while (remaining > 0) {
+		auto append_count = MinValue<idx_t>(remaining, RowGroup::ROW_GROUP_SIZE - row_group->count);
+		row_group->AppendVersionInfo(transaction, append_count);
+		remaining -= append_count;
+		row_group = (RowGroup *)row_group->next.get();
+	}
+	total_rows += state.total_append_count;
+
+	state.total_append_count = 0;
+	state.start_row_group = nullptr;
+}
+
 void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count) {
 	auto row_group = (RowGroup *)row_groups->GetSegment(row_start);
 	D_ASSERT(row_group);
@@ -210929,7 +211066,7 @@ void ReplayState::ReplayInsert() {
 	}
 
 	// append to the current table
-	current_table->storage->Append(*current_table, context, chunk);
+	current_table->storage->LocalAppend(*current_table, context, chunk);
 }
 
 void ReplayState::ReplayDelete() {
@@ -211901,6 +212038,13 @@ void RollbackState::RollbackEntry(UndoFlags type, data_ptr_t data) {
 
 namespace duckdb {
 
+TransactionData::TransactionData(Transaction &transaction_p) // NOLINT
+    : transaction(&transaction_p), transaction_id(transaction_p.transaction_id), start_time(transaction_p.start_time) {
+}
+TransactionData::TransactionData(transaction_t transaction_id_p, transaction_t start_time_p)
+    : transaction(nullptr), transaction_id(transaction_id_p), start_time(start_time_p) {
+}
+
 Transaction::Transaction(weak_ptr<ClientContext> context_p, transaction_t start_time, transaction_t transaction_id,
                          timestamp_t start_timestamp, idx_t catalog_version)
     : context(move(context_p)), start_time(start_time), transaction_id(transaction_id), commit_id(0),
@@ -211980,7 +212124,7 @@ string Transaction::Commit(DatabaseInstance &db, transaction_t commit_id, bool c
 	LocalStorage::CommitState commit_state;
 	auto storage_commit_state = storage_manager.GenStorageCommitState(*this, checkpoint);
 	try {
-		storage.Commit(commit_state, *this, log, commit_id);
+		storage.Commit(commit_state, *this);
 		undo_buffer.Commit(iterator_state, log, commit_id);
 		if (log) {
 			// commit any sequences that were used to the WAL