duckdb 0.5.2-dev1040.0 → 0.5.2-dev1080.0

package/src/duckdb.cpp

@@ -6442,7 +6442,6 @@ void DependencyManager::AddOwnership(ClientContext &context, CatalogEntry *owner
 
 
 
-#include <execinfo.h>
 #endif
 
 namespace duckdb {
@@ -6487,8 +6486,6 @@ struct AllocatorDebugInfo {
 	AllocatorDebugInfo();
 	~AllocatorDebugInfo();
 
-	static string GetStackTrace(int max_depth = 128);
-
 	void AllocateData(data_ptr_t pointer, idx_t size);
 	void FreeData(data_ptr_t pointer, idx_t size);
 	void ReallocateData(data_ptr_t pointer, data_ptr_t new_pointer, idx_t old_size, idx_t new_size);
@@ -6585,7 +6582,7 @@ AllocatorDebugInfo::~AllocatorDebugInfo() {
 	if (allocation_count != 0) {
 		printf("Outstanding allocations found for Allocator\n");
 		for (auto &entry : pointers) {
-			printf("Allocation of size %ld at address %p\n", entry.second.first, (void *)entry.first);
+			printf("Allocation of size %llu at address %p\n", entry.second.first, (void *)entry.first);
 			printf("Stack trace:\n%s\n", entry.second.second.c_str());
 			printf("\n");
 		}
@@ -6597,28 +6594,11 @@ AllocatorDebugInfo::~AllocatorDebugInfo() {
 	D_ASSERT(allocation_count == 0);
 }
 
-string AllocatorDebugInfo::GetStackTrace(int max_depth) {
-#ifdef DUCKDB_DEBUG_ALLOCATION
-	string result;
-	auto callstack = unique_ptr<void *[]>(new void *[max_depth]);
-	int frames = backtrace(callstack.get(), max_depth);
-	char **strs = backtrace_symbols(callstack.get(), frames);
-	for (int i = 0; i < frames; i++) {
-		result += strs[i];
-		result += "\n";
-	}
-	free(strs);
-	return result;
-#else
-	throw InternalException("GetStackTrace not supported without DUCKDB_DEBUG_ALLOCATION");
-#endif
-}
-
 void AllocatorDebugInfo::AllocateData(data_ptr_t pointer, idx_t size) {
 	allocation_count += size;
 #ifdef DUCKDB_DEBUG_ALLOCATION
 	lock_guard<mutex> l(pointer_lock);
-	pointers[pointer] = make_pair(size, GetStackTrace());
+	pointers[pointer] = make_pair(size, Exception::GetStackTrace());
 #endif
 }
 
@@ -8122,7 +8102,8 @@ void DuckDBAssertInternal(bool condition, const char *condition_name, const char
 	if (condition) {
 		return;
 	}
-	throw InternalException("Assertion triggered in file \"%s\" on line %d: %s", file, linenr, condition_name);
+	throw InternalException("Assertion triggered in file \"%s\" on line %d: %s%s", file, linenr, condition_name,
+	                        Exception::GetStackTrace());
 }
 
 } // namespace duckdb
@@ -9671,6 +9652,9 @@ string StatementReturnTypeToString(StatementReturnType type) {
 #include <stdio.h>
 #include <stdlib.h>
 #endif
+#ifdef DUCKDB_DEBUG_STACKTRACE
+#include <execinfo.h>
+#endif
 
 namespace duckdb {
 
@@ -9699,6 +9683,24 @@ bool Exception::UncaughtException() {
 #endif
 }
 
+string Exception::GetStackTrace(int max_depth) {
+#ifdef DUCKDB_DEBUG_STACKTRACE
+	string result;
+	auto callstack = unique_ptr<void *[]>(new void *[max_depth]);
+	int frames = backtrace(callstack.get(), max_depth);
+	char **strs = backtrace_symbols(callstack.get(), frames);
+	for (int i = 0; i < frames; i++) {
+		result += strs[i];
+		result += "\n";
+	}
+	free(strs);
+	return "\n" + result;
+#else
+	// Stack trace not available. Toggle DUCKDB_DEBUG_STACKTRACE in exception.cpp to enable stack traces.
+	return "";
+#endif
+}
+
 string Exception::ConstructMessageRecursive(const string &msg, vector<ExceptionFormatValue> &values) {
 	return ExceptionFormatValue::Format(msg, values);
 }
@@ -39187,7 +39189,9 @@ public:
 	bool ParallelSink() const override {
 		return true;
 	}
-
+	bool IsOrderPreserving() const override {
+		return false;
+	}
 	string ParamsToString() const override;
 
 public:
@@ -39352,6 +39356,11 @@ struct aggr_ht_entry_32 {
 
 enum HtEntryType { HT_WIDTH_32, HT_WIDTH_64 };
 
+struct AggregateHTScanState {
+	mutex lock;
+	idx_t scan_position = 0;
+};
+
 class GroupedAggregateHashTable : public BaseAggregateHashTable {
 public:
 	//! The hash table load factor, when a resize is triggered
@@ -39381,7 +39390,7 @@ public:
 	//! Scan the HT starting from the scan_position until the result and group
 	//! chunks are filled. scan_position will be updated by this function.
 	//! Returns the amount of elements found.
-	idx_t Scan(idx_t &scan_position, DataChunk &result);
+	idx_t Scan(AggregateHTScanState &scan_state, DataChunk &result);
 
 	//! Fetch the aggregates for specific groups from the HT and place them in the result
 	void FetchAggregates(DataChunk &groups, DataChunk &result);
@@ -56453,28 +56462,32 @@ void GroupedAggregateHashTable::Partition(vector<GroupedAggregateHashTable *> &p
 	D_ASSERT(total_count == entries);
 }
 
-idx_t GroupedAggregateHashTable::Scan(idx_t &scan_position, DataChunk &result) {
-	if (scan_position >= entries) {
-		return 0;
-	}
-	auto remaining = entries - scan_position;
-	auto this_n = MinValue((idx_t)STANDARD_VECTOR_SIZE, remaining);
-
+idx_t GroupedAggregateHashTable::Scan(AggregateHTScanState &scan_state, DataChunk &result) {
+	idx_t this_n;
 	Vector addresses(LogicalType::POINTER);
 	auto data_pointers = FlatVector::GetData<data_ptr_t>(addresses);
+	{
+		lock_guard<mutex> l(scan_state.lock);
+		if (scan_state.scan_position >= entries) {
+			return 0;
+		}
+		auto remaining = entries - scan_state.scan_position;
+		this_n = MinValue((idx_t)STANDARD_VECTOR_SIZE, remaining);
 
-	auto chunk_idx = scan_position / tuples_per_block;
-	auto chunk_offset = (scan_position % tuples_per_block) * tuple_size;
-	D_ASSERT(chunk_offset + tuple_size <= Storage::BLOCK_SIZE);
+		auto chunk_idx = scan_state.scan_position / tuples_per_block;
+		auto chunk_offset = (scan_state.scan_position % tuples_per_block) * tuple_size;
+		D_ASSERT(chunk_offset + tuple_size <= Storage::BLOCK_SIZE);
 
-	auto read_ptr = payload_hds_ptrs[chunk_idx++];
-	for (idx_t i = 0; i < this_n; i++) {
-		data_pointers[i] = read_ptr + chunk_offset;
-		chunk_offset += tuple_size;
-		if (chunk_offset >= tuples_per_block * tuple_size) {
-			read_ptr = payload_hds_ptrs[chunk_idx++];
-			chunk_offset = 0;
+		auto read_ptr = payload_hds_ptrs[chunk_idx++];
+		for (idx_t i = 0; i < this_n; i++) {
+			data_pointers[i] = read_ptr + chunk_offset;
+			chunk_offset += tuple_size;
+			if (chunk_offset >= tuples_per_block * tuple_size) {
+				read_ptr = payload_hds_ptrs[chunk_idx++];
+				chunk_offset = 0;
+			}
 		}
+		scan_state.scan_position += this_n;
 	}
 
 	result.SetCardinality(this_n);
@@ -56487,8 +56500,6 @@ idx_t GroupedAggregateHashTable::Scan(idx_t &scan_position, DataChunk &result) {
 	}
 
 	RowOperations::FinalizeStates(layout, addresses, result, group_cols);
-
-	scan_position += this_n;
 	return this_n;
 }
 
@@ -63356,12 +63367,12 @@ public:
 	}
 
 	const PhysicalHashAggregate &op;
-	std::atomic<size_t> state_index;
+	mutex lock;
+	atomic<idx_t> state_index;
 
 	vector<unique_ptr<GlobalSourceState>> radix_states;
 
 public:
-#if 0
 	idx_t MaxThreads() override {
 		// If there are no tables, we only need one thread.
 		if (op.radix_tables.empty()) {
@@ -63373,10 +63384,8 @@ public:
 		for (size_t sidx = 0; sidx < op.radix_tables.size(); ++sidx) {
 			count += op.radix_tables[sidx].Size(*ht_state.radix_states[sidx]);
 		}
-
-		return (count + STANDARD_VECTOR_SIZE - 1 ) / STANDARD_VECTOR_SIZE;
+		return MaxValue<idx_t>(1, count / RowGroup::ROW_GROUP_SIZE);
 	}
-#endif
 };
 
 unique_ptr<GlobalSourceState> PhysicalHashAggregate::GetGlobalSourceState(ClientContext &context) const {
@@ -63404,12 +63413,24 @@ void PhysicalHashAggregate::GetData(ExecutionContext &context, DataChunk &chunk,
 	auto &ht_state = (HashAggregateGlobalState &)*sink_state;
 	auto &gstate = (PhysicalHashAggregateGlobalSourceState &)gstate_p;
 	auto &lstate = (PhysicalHashAggregateLocalSourceState &)lstate_p;
-	for (size_t sidx = gstate.state_index; sidx < radix_tables.size(); sidx = ++gstate.state_index) {
-		radix_tables[sidx].GetData(context, chunk, *ht_state.radix_states[sidx], *gstate.radix_states[sidx],
-		                           *lstate.radix_states[sidx]);
+	while (true) {
+		idx_t radix_idx = gstate.state_index;
+		if (radix_idx >= radix_tables.size()) {
+			break;
+		}
+		radix_tables[radix_idx].GetData(context, chunk, *ht_state.radix_states[radix_idx],
+		                                *gstate.radix_states[radix_idx], *lstate.radix_states[radix_idx]);
 		if (chunk.size() != 0) {
 			return;
 		}
+		// move to the next table
+		lock_guard<mutex> l(gstate.lock);
+		radix_idx++;
+		if (radix_idx > gstate.state_index) {
+			// we have not yet worked on the table
+			// move the global index forwards
+			gstate.state_index = radix_idx;
+		}
 	}
 }
 
@@ -63495,6 +63516,10 @@ public:
 		return true;
 	}
 
+	bool IsOrderPreserving() const override {
+		return false;
+	}
+
 public:
 	//! The group types
 	vector<LogicalType> group_types;
@@ -68921,6 +68946,10 @@ public:
 	                               PhysicalOperator &op);
 	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
 	vector<const PhysicalOperator *> GetSources() const override;
+
+	bool IsOrderPreserving() const override {
+		return false;
+	}
 };
 
 } // namespace duckdb
@@ -68946,10 +68975,6 @@ public:
 	//! Construct the remainder of a Full Outer Join based on which tuples in the RHS found no match
 	static void ConstructFullOuterJoinResult(bool *found_match, ColumnDataCollection &input, DataChunk &result,
 	                                         ColumnDataScanState &scan_state);
-
-	bool IsOrderPreserving() const override {
-		return false;
-	}
 };
 
 } // namespace duckdb
@@ -69680,6 +69705,10 @@ public:
 		return true;
 	}
 
+	bool IsOrderPreserving() const override {
+		return false;
+	}
+
 public:
 	// Sink Interface
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -72797,6 +72826,10 @@ public:
 		return true;
 	}
 
+	bool IsOrderPreserving() const override {
+		return false;
+	}
+
 public:
 	void BuildPipelines(Executor &executor, Pipeline &current, PipelineBuildState &state) override;
 	vector<const PhysicalOperator *> GetSources() const override;
@@ -78505,22 +78538,37 @@ vector<const PhysicalOperator *> PhysicalExport::GetSources() const {
 
 
 
+
 namespace duckdb {
 
 //! Physically insert a set of data into a table
 class PhysicalInsert : public PhysicalOperator {
 public:
+	//! INSERT INTO
 	PhysicalInsert(vector<LogicalType> types, TableCatalogEntry *table, vector<idx_t> column_index_map,
-	               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality, bool return_chunk);
+	               vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality, bool return_chunk,
+	               bool parallel);
+	//! CREATE TABLE AS
+	PhysicalInsert(LogicalOperator &op, SchemaCatalogEntry *schema, unique_ptr<BoundCreateTableInfo> info,
+	               idx_t estimated_cardinality, bool parallel);
 
 	//! The map from insert column index to table column index
 	vector<idx_t> column_index_map;
 	//! The table to insert into
-	TableCatalogEntry *table;
+	TableCatalogEntry *insert_table;
+	//! The insert types
+	vector<LogicalType> insert_types;
 	//! The default expressions of the columns for which no value is provided
 	vector<unique_ptr<Expression>> bound_defaults;
 	//! If the returning statement is present, return the whole chunk
 	bool return_chunk;
+	//! Table schema, in case of CREATE TABLE AS
+	SchemaCatalogEntry *schema;
+	//! Create table info, in case of CREATE TABLE AS
+	unique_ptr<BoundCreateTableInfo> info;
+	//! Whether or not the INSERT can be executed in parallel
+	//! This insert is not order preserving if executed in parallel
+	bool parallel;
 
 public:
 	// Source interface
@@ -78543,7 +78591,7 @@ public:
 	}
 
 	bool ParallelSink() const override {
-		return false;
+		return parallel;
 	}
 };
 
@@ -78557,14 +78605,32 @@ public:
 
 
 
+
+
+
+
 namespace duckdb {
 
 PhysicalInsert::PhysicalInsert(vector<LogicalType> types, TableCatalogEntry *table, vector<idx_t> column_index_map,
                                vector<unique_ptr<Expression>> bound_defaults, idx_t estimated_cardinality,
-                               bool return_chunk)
+                               bool return_chunk, bool parallel)
     : PhysicalOperator(PhysicalOperatorType::INSERT, move(types), estimated_cardinality),
-      column_index_map(std::move(column_index_map)), table(table), bound_defaults(move(bound_defaults)),
-      return_chunk(return_chunk) {
+      column_index_map(std::move(column_index_map)), insert_table(table), insert_types(table->GetTypes()),
+      bound_defaults(move(bound_defaults)), return_chunk(return_chunk), parallel(parallel) {
+}
+
+PhysicalInsert::PhysicalInsert(LogicalOperator &op, SchemaCatalogEntry *schema, unique_ptr<BoundCreateTableInfo> info_p,
+                               idx_t estimated_cardinality, bool parallel)
+    : PhysicalOperator(PhysicalOperatorType::INSERT, op.types, estimated_cardinality), insert_table(nullptr),
+      return_chunk(false), schema(schema), info(move(info_p)), parallel(parallel) {
+	auto &create_info = (CreateTableInfo &)*info->base;
+	for (auto &col : create_info.columns) {
+		if (col.Generated()) {
+			continue;
+		}
+		insert_types.push_back(col.GetType());
+		bound_defaults.push_back(make_unique<BoundConstantExpression>(Value(col.GetType())));
+	}
 }
 
 //===--------------------------------------------------------------------===//
@@ -78577,43 +78643,58 @@ public:
 	}
 
 	mutex lock;
+	TableCatalogEntry *table;
 	idx_t insert_count;
-	LocalAppendState append_state;
 	bool initialized;
+	LocalAppendState append_state;
 	ColumnDataCollection return_collection;
 };
 
 class InsertLocalState : public LocalSinkState {
 public:
-	InsertLocalState(Allocator &allocator, const vector<LogicalType> &types,
+	InsertLocalState(ClientContext &context, const vector<LogicalType> &types,
 	                 const vector<unique_ptr<Expression>> &bound_defaults)
-	    : default_executor(allocator, bound_defaults) {
-		insert_chunk.Initialize(allocator, types);
+	    : default_executor(Allocator::Get(context), bound_defaults) {
+		insert_chunk.Initialize(Allocator::Get(context), types);
 	}
 
 	DataChunk insert_chunk;
 	ExpressionExecutor default_executor;
+	TableAppendState local_append_state;
+	unique_ptr<RowGroupCollection> local_collection;
+	unique_ptr<OptimisticDataWriter> writer;
 };
 
 unique_ptr<GlobalSinkState> PhysicalInsert::GetGlobalSinkState(ClientContext &context) const {
-	return make_unique<InsertGlobalState>(context, GetTypes());
+	auto result = make_unique<InsertGlobalState>(context, GetTypes());
+	if (info) {
+		// CREATE TABLE AS
+		D_ASSERT(!insert_table);
+		auto &catalog = Catalog::GetCatalog(context);
+		result->table = (TableCatalogEntry *)catalog.CreateTable(context, schema, info.get());
+	} else {
+		D_ASSERT(insert_table);
+		result->table = insert_table;
+	}
+	return move(result);
 }
 
 unique_ptr<LocalSinkState> PhysicalInsert::GetLocalSinkState(ExecutionContext &context) const {
-	return make_unique<InsertLocalState>(Allocator::Get(context.client), table->GetTypes(), bound_defaults);
+	return make_unique<InsertLocalState>(context.client, insert_types, bound_defaults);
 }
 
-SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p,
                                     DataChunk &chunk) const {
 	auto &gstate = (InsertGlobalState &)state;
-	auto &istate = (InsertLocalState &)lstate;
+	auto &lstate = (InsertLocalState &)lstate_p;
 
 	chunk.Flatten();
-	istate.default_executor.SetChunk(chunk);
+	lstate.default_executor.SetChunk(chunk);
 
-	istate.insert_chunk.Reset();
-	istate.insert_chunk.SetCardinality(chunk);
+	lstate.insert_chunk.Reset();
+	lstate.insert_chunk.SetCardinality(chunk);
 
+	auto table = gstate.table;
 	if (!column_index_map.empty()) {
 		// columns specified by the user, use column_index_map
 		for (idx_t i = 0; i < table->columns.size(); i++) {
@@ -78624,48 +78705,101 @@ SinkResultType PhysicalInsert::Sink(ExecutionContext &context, GlobalSinkState &
 			auto storage_idx = col.StorageOid();
 			if (column_index_map[i] == DConstants::INVALID_INDEX) {
 				// insert default value
-				istate.default_executor.ExecuteExpression(i, istate.insert_chunk.data[storage_idx]);
+				lstate.default_executor.ExecuteExpression(i, lstate.insert_chunk.data[storage_idx]);
 			} else {
 				// get value from child chunk
 				D_ASSERT((idx_t)column_index_map[i] < chunk.ColumnCount());
-				D_ASSERT(istate.insert_chunk.data[storage_idx].GetType() == chunk.data[column_index_map[i]].GetType());
-				istate.insert_chunk.data[storage_idx].Reference(chunk.data[column_index_map[i]]);
+				D_ASSERT(lstate.insert_chunk.data[storage_idx].GetType() == chunk.data[column_index_map[i]].GetType());
+				lstate.insert_chunk.data[storage_idx].Reference(chunk.data[column_index_map[i]]);
 			}
 		}
 	} else {
 		// no columns specified, just append directly
-		for (idx_t i = 0; i < istate.insert_chunk.ColumnCount(); i++) {
-			D_ASSERT(istate.insert_chunk.data[i].GetType() == chunk.data[i].GetType());
-			istate.insert_chunk.data[i].Reference(chunk.data[i]);
+		for (idx_t i = 0; i < lstate.insert_chunk.ColumnCount(); i++) {
+			D_ASSERT(lstate.insert_chunk.data[i].GetType() == chunk.data[i].GetType());
+			lstate.insert_chunk.data[i].Reference(chunk.data[i]);
 		}
 	}
 
-	lock_guard<mutex> glock(gstate.lock);
-	if (!gstate.initialized) {
-		table->storage->InitializeLocalAppend(gstate.append_state, context.client);
-		gstate.initialized = true;
-	}
-	table->storage->LocalAppend(gstate.append_state, *table, context.client, istate.insert_chunk);
+	if (!parallel) {
+		if (!gstate.initialized) {
+			table->storage->InitializeLocalAppend(gstate.append_state, context.client);
+			gstate.initialized = true;
+		}
+		table->storage->LocalAppend(gstate.append_state, *table, context.client, lstate.insert_chunk);
 
-	if (return_chunk) {
-		gstate.return_collection.Append(istate.insert_chunk);
+		if (return_chunk) {
+			gstate.return_collection.Append(lstate.insert_chunk);
+		}
+		gstate.insert_count += chunk.size();
+	} else {
+		D_ASSERT(!return_chunk);
+		// parallel append
+		if (!lstate.local_collection) {
+			auto &table_info = table->storage->info;
+			auto &block_manager = TableIOManager::Get(*table->storage).GetBlockManagerForRowData();
+			lstate.local_collection = make_unique<RowGroupCollection>(table_info, block_manager, insert_types, 0);
+			lstate.local_collection->InitializeEmpty();
+			lstate.local_collection->InitializeAppend(lstate.local_append_state);
+			lstate.writer = make_unique<OptimisticDataWriter>(gstate.table->storage.get());
+		}
+		auto new_row_group = lstate.local_collection->Append(lstate.insert_chunk, lstate.local_append_state);
+		if (new_row_group) {
+			lstate.writer->CheckFlushToDisk(*lstate.local_collection);
+		}
 	}
 
-	gstate.insert_count += chunk.size();
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
-void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const {
-	auto &state = (InsertLocalState &)lstate;
+void PhysicalInsert::Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const {
+	auto &gstate = (InsertGlobalState &)gstate_p;
+	auto &lstate = (InsertLocalState &)lstate_p;
 	auto &client_profiler = QueryProfiler::Get(context.client);
-	context.thread.profiler.Flush(this, &state.default_executor, "default_executor", 1);
+	context.thread.profiler.Flush(this, &lstate.default_executor, "default_executor", 1);
 	client_profiler.Flush(context.thread.profiler);
+
+	if (!parallel) {
+		return;
+	}
+	if (!lstate.local_collection) {
+		return;
+	}
+	// parallel append: finalize the append
+	TransactionData tdata(0, 0);
+	lstate.local_collection->FinalizeAppend(tdata, lstate.local_append_state);
+
+	auto append_count = lstate.local_collection->GetTotalRows();
+
+	if (append_count < LocalStorage::MERGE_THRESHOLD) {
+		// we have few rows - append to the local storage directly
+		lock_guard<mutex> lock(gstate.lock);
+		gstate.insert_count += append_count;
+		auto table = gstate.table;
+		table->storage->InitializeLocalAppend(gstate.append_state, context.client);
+		auto &transaction = Transaction::GetTransaction(context.client);
+		lstate.local_collection->Scan(transaction, [&](DataChunk &insert_chunk) {
+			table->storage->LocalAppend(gstate.append_state, *table, context.client, insert_chunk);
+			return true;
+		});
+		table->storage->FinalizeLocalAppend(gstate.append_state);
+	} else {
+		// we have many rows - flush the row group collection to disk (if required) and merge into the transaction-local
+		// state
+		lstate.writer->FlushToDisk(*lstate.local_collection);
+		lstate.writer->FinalFlush();
+
+		lock_guard<mutex> lock(gstate.lock);
+		gstate.insert_count += append_count;
+		gstate.table->storage->LocalMerge(context.client, *lstate.local_collection);
+	}
 }
 
 SinkFinalizeType PhysicalInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
                                           GlobalSinkState &state) const {
 	auto &gstate = (InsertGlobalState &)state;
-	if (gstate.initialized) {
+	if (!parallel && gstate.initialized) {
+		auto table = gstate.table;
 		table->storage->FinalizeLocalAppend(gstate.append_state);
 	}
 	return SinkFinalizeType::READY;
@@ -80485,142 +80619,6 @@ void PhysicalCreateTable::GetData(ExecutionContext &context, DataChunk &chunk, G
 	state.finished = true;
 }
 
-} // namespace duckdb
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/execution/operator/schema/physical_create_table_as.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-namespace duckdb {
-
-//! Physically CREATE TABLE AS statement
-class PhysicalCreateTableAs : public PhysicalOperator {
-public:
-	PhysicalCreateTableAs(LogicalOperator &op, SchemaCatalogEntry *schema, unique_ptr<BoundCreateTableInfo> info,
-	                      idx_t estimated_cardinality);
-
-	//! Schema to insert to
-	SchemaCatalogEntry *schema;
-	//! Table name to create
-	unique_ptr<BoundCreateTableInfo> info;
-
-public:
-	// Source interface
-	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
-	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	             LocalSourceState &lstate) const override;
-
-public:
-	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
-	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
-	                    DataChunk &input) const override;
-	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-	                          GlobalSinkState &gstate) const override;
-
-	bool IsSink() const override {
-		return true;
-	}
-	bool ParallelSink() const override {
-		return true;
-	}
-};
-} // namespace duckdb
-
-
-
-
-
-
-namespace duckdb {
-
-PhysicalCreateTableAs::PhysicalCreateTableAs(LogicalOperator &op, SchemaCatalogEntry *schema,
-                                             unique_ptr<BoundCreateTableInfo> info, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::CREATE_TABLE_AS, op.types, estimated_cardinality), schema(schema),
-      info(move(info)) {
-}
-
-//===--------------------------------------------------------------------===//
-// Sink
-//===--------------------------------------------------------------------===//
-class CreateTableAsGlobalState : public GlobalSinkState {
-public:
-	CreateTableAsGlobalState() : inserted_count(0), initialized(false) {
-	}
-
-	mutex append_lock;
-	TableCatalogEntry *table;
-	int64_t inserted_count;
-	LocalAppendState append_state;
-	bool initialized;
-};
-
-unique_ptr<GlobalSinkState> PhysicalCreateTableAs::GetGlobalSinkState(ClientContext &context) const {
-	auto sink = make_unique<CreateTableAsGlobalState>();
-	auto &catalog = Catalog::GetCatalog(context);
-	sink->table = (TableCatalogEntry *)catalog.CreateTable(context, schema, info.get());
-	return move(sink);
-}
-
-SinkResultType PhysicalCreateTableAs::Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p,
-                                           DataChunk &input) const {
-	auto &sink = (CreateTableAsGlobalState &)state;
-	D_ASSERT(sink.table);
-	lock_guard<mutex> client_guard(sink.append_lock);
-	if (!sink.initialized) {
-		sink.table->storage->InitializeLocalAppend(sink.append_state, context.client);
-		sink.initialized = true;
-	}
-	sink.table->storage->LocalAppend(sink.append_state, *sink.table, context.client, input);
-	sink.inserted_count += input.size();
-	return SinkResultType::NEED_MORE_INPUT;
-}
-
-SinkFinalizeType PhysicalCreateTableAs::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-                                                 GlobalSinkState &state) const {
-	auto &gstate = (CreateTableAsGlobalState &)state;
-	if (gstate.initialized) {
-		gstate.table->storage->FinalizeLocalAppend(gstate.append_state);
-	}
-	return SinkFinalizeType::READY;
-}
-
-//===--------------------------------------------------------------------===//
-// Source
-//===--------------------------------------------------------------------===//
-class CreateTableAsSourceState : public GlobalSourceState {
-public:
-	CreateTableAsSourceState() : finished(false) {
-	}
-
-	bool finished;
-};
-
-unique_ptr<GlobalSourceState> PhysicalCreateTableAs::GetGlobalSourceState(ClientContext &context) const {
-	return make_unique<CreateTableAsSourceState>();
-}
-
-void PhysicalCreateTableAs::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-                                    LocalSourceState &lstate) const {
-	auto &state = (CreateTableAsSourceState &)gstate;
-	auto &sink = (CreateTableAsGlobalState &)*sink_state;
-	if (state.finished) {
-		return;
-	}
-	if (sink.table) {
-		chunk.SetCardinality(1);
-		chunk.SetValue(0, 0, Value::BIGINT(sink.inserted_count));
-	}
-	state.finished = true;
-}
-
 } // namespace duckdb
 //===----------------------------------------------------------------------===//
 //                         DuckDB
@@ -82779,6 +82777,7 @@ protected:
 } // namespace duckdb
 
 
+
 namespace duckdb {
 
 static void ExtractDependencies(Expression &expr, unordered_set<CatalogEntry *> &dependencies) {
@@ -82804,9 +82803,15 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateTabl
 	    catalog.GetEntry(context, CatalogType::TABLE_ENTRY, create_info.schema, create_info.table, true);
 	bool replace = op.info->Base().on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT;
 	if ((!existing_entry || replace) && !op.children.empty()) {
-		D_ASSERT(op.children.size() == 1);
-		auto create = make_unique<PhysicalCreateTableAs>(op, op.schema, move(op.info), op.estimated_cardinality);
 		auto plan = CreatePlan(*op.children[0]);
+
+		auto &config = DBConfig::GetConfig(context);
+		bool plan_preserves_order = plan->AllOperatorsPreserveOrder();
+		bool parallel_streaming_insert = !config.options.preserve_insertion_order || !plan_preserves_order;
+
+		D_ASSERT(op.children.size() == 1);
+		auto create = make_unique<PhysicalInsert>(op, op.schema, move(op.info), op.estimated_cardinality,
+		                                          parallel_streaming_insert);
 		create->children.push_back(move(plan));
 		return move(create);
 	} else {
@@ -84296,6 +84301,8 @@ protected:
 } // namespace duckdb
 
 
+
+
 namespace duckdb {
 
 unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalInsert &op) {
@@ -84305,9 +84312,21 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalInsert &op
 		plan = CreatePlan(*op.children[0]);
 	}
 
+	auto &config = DBConfig::GetConfig(context);
+	bool plan_preserves_order = plan->AllOperatorsPreserveOrder();
+	bool parallel_streaming_insert = !config.options.preserve_insertion_order || !plan_preserves_order;
+	if (!op.table->storage->info->indexes.Empty()) {
+		// not for tables with indexes currently
+		parallel_streaming_insert = false;
+	}
+	if (op.return_chunk) {
+		// not supported for RETURNING
+		parallel_streaming_insert = false;
+	}
+
 	dependencies.insert(op.table);
 	auto insert = make_unique<PhysicalInsert>(op.types, op.table, op.column_index_map, move(op.bound_defaults),
-	                                          op.estimated_cardinality, op.return_chunk);
+	                                          op.estimated_cardinality, op.return_chunk, parallel_streaming_insert);
 	if (plan) {
 		insert->children.push_back(move(plan));
 	}
@@ -86075,14 +86094,16 @@ bool RadixPartitionedHashTable::ForceSingleHT(GlobalSinkState &state) const {
 class RadixHTGlobalSourceState : public GlobalSourceState {
 public:
 	explicit RadixHTGlobalSourceState(Allocator &allocator, const RadixPartitionedHashTable &ht)
-	    : ht_index(0), ht_scan_position(0), finished(false) {
+	    : ht_index(0), initialized(false), finished(false) {
 	}
 
 	//! Heavy handed for now.
 	mutex lock;
 	//! The current position to scan the HT for output tuples
-	idx_t ht_index;
-	idx_t ht_scan_position;
+	atomic<idx_t> ht_index;
+	//! The set of aggregate scan states
+	unique_ptr<AggregateHTScanState[]> ht_scan_states;
+	atomic<bool> initialized;
 	atomic<bool> finished;
 };
 
@@ -86168,23 +86189,39 @@ void RadixPartitionedHashTable::GetData(ExecutionContext &context, DataChunk &ch
 	idx_t elements_found = 0;
 
 	lstate.scan_chunk.Reset();
+	if (!state.initialized) {
+		lock_guard<mutex> l(state.lock);
+		if (!state.ht_scan_states) {
+			state.ht_scan_states =
+			    unique_ptr<AggregateHTScanState[]>(new AggregateHTScanState[gstate.finalized_hts.size()]);
+		} else {
+			D_ASSERT(state.initialized);
+		}
+		state.initialized = true;
+	}
 	while (true) {
-		lock_guard<mutex> glock(state.lock);
-		if (state.ht_index == gstate.finalized_hts.size()) {
+		idx_t ht_index = state.ht_index;
+		if (ht_index >= gstate.finalized_hts.size()) {
 			state.finished = true;
 			return;
 		}
-		D_ASSERT(gstate.finalized_hts[state.ht_index]);
-		elements_found = gstate.finalized_hts[state.ht_index]->Scan(state.ht_scan_position, lstate.scan_chunk);
-
+		D_ASSERT(state.ht_index < gstate.finalized_hts.size());
+		D_ASSERT(state.ht_scan_states);
+		auto &ht = gstate.finalized_hts[ht_index];
+		auto &scan_state = state.ht_scan_states[ht_index];
+		D_ASSERT(ht);
+		elements_found = ht->Scan(scan_state, lstate.scan_chunk);
 		if (elements_found > 0) {
 			break;
 		}
-		if (!gstate.multi_scan) {
-			gstate.finalized_hts[state.ht_index].reset();
+		// move to the next hash table
+		lock_guard<mutex> l(state.lock);
+		ht_index++;
+		if (ht_index > state.ht_index) {
+			// we have not yet worked on the table
+			// move the global index forwards
+			state.ht_index = ht_index;
 		}
-		state.ht_index++;
-		state.ht_scan_position = 0;
 	}
 
 	// compute the final projection list
@@ -141762,10 +141799,9 @@ static void PrintRow(std::ostream &ss, const string &annotation, int id, const s
 
 static void ExtractFunctions(std::ostream &ss, ExpressionInfo &info, int &fun_id, int depth) {
 	if (info.hasfunction) {
-		D_ASSERT(info.sample_tuples_count != 0);
-		PrintRow(ss, "Function", fun_id++, info.function_name,
-		         int(info.function_time) / double(info.sample_tuples_count), info.sample_tuples_count,
-		         info.tuples_count, "", depth);
+		double time = info.sample_tuples_count == 0 ? 0 : int(info.function_time) / double(info.sample_tuples_count);
+		PrintRow(ss, "Function", fun_id++, info.function_name, time, info.sample_tuples_count, info.tuples_count, "",
+		         depth);
 	}
 	if (info.children.empty()) {
 		return;
@@ -141792,10 +141828,11 @@ static void ToJSONRecursive(QueryProfiler::TreeNode &node, std::ostream &ss, int
 			continue;
 		}
 		for (auto &expr_timer : expr_executor->roots) {
-			D_ASSERT(expr_timer->sample_tuples_count != 0);
-			PrintRow(ss, "ExpressionRoot", expression_counter++, expr_timer->name,
-			         int(expr_timer->time) / double(expr_timer->sample_tuples_count), expr_timer->sample_tuples_count,
-			         expr_timer->tuples_count, expr_timer->extra_info, depth + 1);
+			double time = expr_timer->sample_tuples_count == 0
+			                  ? 0
+			                  : double(expr_timer->time) / double(expr_timer->sample_tuples_count);
+			PrintRow(ss, "ExpressionRoot", expression_counter++, expr_timer->name, time,
+			         expr_timer->sample_tuples_count, expr_timer->tuples_count, expr_timer->extra_info, depth + 1);
 			// Extract all functions inside the tree
 			ExtractFunctions(ss, *expr_timer->root, function_counter, depth + 1);
 		}
@@ -197829,7 +197866,7 @@ public:
 
 protected:
 	//! Append a transient segment
-	void AppendTransientSegment(idx_t start_row);
+	void AppendTransientSegment(SegmentLock &l, idx_t start_row);
 
 	//! Scans a base vector from the column
 	idx_t ScanVector(ColumnScanState &state, Vector &result, idx_t remaining);
@@ -197867,7 +197904,7 @@ public:
 	RowGroup &GetRowGroup();
 	ColumnCheckpointState &GetCheckpointState();
 
-	void Checkpoint(unique_ptr<SegmentBase> segment);
+	void Checkpoint(vector<SegmentNode> nodes);
 
 private:
 	void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
@@ -197882,7 +197919,7 @@ private:
 	ColumnCheckpointState &state;
 	bool is_validity;
 	Vector intermediate;
-	unique_ptr<SegmentBase> owned_segment;
+	vector<SegmentNode> nodes;
 	vector<CompressionFunction *> compression_functions;
 	ColumnCheckpointInfo &checkpoint_info;
 };
@@ -202468,16 +202505,16 @@ public:
 	template <class OP = EmptyRLEWriter>
 	void Update(T *data, ValidityMask &validity, idx_t idx) {
 		if (validity.RowIsValid(idx)) {
-			all_null = false;
-			if (seen_count == 0) {
+			if (all_null) {
 				// no value seen yet
-				// assign the current value, and set the seen_count to 1
+				// assign the current value, and increment the seen_count
 				// note that we increment last_seen_count rather than setting it to 1
 				// this is intentional: this is the first VALID value we see
 				// but it might not be the first value in case of nulls!
 				last_value = data[idx];
-				seen_count = 1;
+				seen_count++;
 				last_seen_count++;
+				all_null = false;
 			} else if (last_value == data[idx]) {
 				// the last value is identical to this value: increment the last_seen_count
 				last_seen_count++;
@@ -203881,33 +203918,24 @@ DataTable::DataTable(DatabaseInstance &db, shared_ptr<TableIOManager> table_io_m
 	    make_shared<RowGroupCollection>(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0);
 	if (data && !data->row_groups.empty()) {
 		this->row_groups->Initialize(*data);
-		stats.Initialize(types, *data);
-	}
-	if (stats.Empty()) {
+	} else {
+		this->row_groups->InitializeEmpty();
 		D_ASSERT(row_groups->GetTotalRows() == 0);
-
-		stats.InitializeEmpty(types);
 	}
 	row_groups->Verify();
 }
 
 DataTable::DataTable(ClientContext &context, DataTable &parent, ColumnDefinition &new_column, Expression *default_value)
     : info(parent.info), db(parent.db), is_root(true) {
+	// add the column definitions from this DataTable
 	for (auto &column_def : parent.column_definitions) {
 		column_definitions.emplace_back(column_def.Copy());
 	}
+	column_definitions.emplace_back(new_column.Copy());
 	// prevent any new tuples from being added to the parent
 	lock_guard<mutex> parent_lock(parent.append_lock);
-	// add the new column to this DataTable
-	auto new_column_type = new_column.Type();
-	auto new_column_idx = parent.column_definitions.size();
-
-	stats.InitializeAddColumn(parent.stats, new_column_type);
-
-	// add the column definitions from this DataTable
-	column_definitions.emplace_back(new_column.Copy());
 
-	this->row_groups = parent.row_groups->AddColumn(new_column, default_value, stats.GetStats(new_column_idx));
+	this->row_groups = parent.row_groups->AddColumn(new_column, default_value);
 
 	// also add this column to client local storage
 	auto &local_storage = LocalStorage::Get(context);
@@ -203937,9 +203965,6 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t removed_co
 		return false;
 	});
 
-	// erase the stats from this DataTable
-	stats.InitializeRemoveColumn(parent.stats, removed_column);
-
 	// erase the column definitions from this DataTable
 	D_ASSERT(removed_column < column_definitions.size());
 	column_definitions.erase(column_definitions.begin() + removed_column);
@@ -203973,7 +203998,6 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, unique_ptr<Bound
 	for (auto &column_def : parent.column_definitions) {
 		column_definitions.emplace_back(column_def.Copy());
 	}
-	stats.InitializeAddConstraint(parent.stats);
 
 	// Verify the new constraint against current persistent/local data
 	VerifyNewConstraint(context, parent, constraint.get());
@@ -204008,9 +204032,7 @@ DataTable::DataTable(ClientContext &context, DataTable &parent, idx_t changed_id
 
 	// set up the statistics for the table
 	// the column that had its type changed will have the new statistics computed during conversion
-	stats.InitializeAlterType(parent.stats, changed_idx, target_type);
-	this->row_groups =
-	    parent.row_groups->AlterType(changed_idx, target_type, bound_columns, cast_expr, stats.GetStats(changed_idx));
+	this->row_groups = parent.row_groups->AlterType(changed_idx, target_type, bound_columns, cast_expr);
 
 	// scan the original table, and fill the new column with the transformed value
 	auto &local_storage = LocalStorage::Get(context);
@@ -204357,6 +204379,11 @@ void DataTable::FinalizeLocalAppend(LocalAppendState &state) {
 	LocalStorage::FinalizeAppend(state);
 }
 
+void DataTable::LocalMerge(ClientContext &context, RowGroupCollection &collection) {
+	auto &local_storage = LocalStorage::Get(context);
+	local_storage.LocalMerge(this, collection);
+}
+
 void DataTable::LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk) {
 	LocalAppendState append_state;
 	table.storage->InitializeLocalAppend(append_state, context);
@@ -204392,7 +204419,7 @@ void DataTable::InitializeAppend(Transaction &transaction, TableAppendState &sta
 
 void DataTable::Append(DataChunk &chunk, TableAppendState &state) {
 	D_ASSERT(is_root);
-	row_groups->Append(chunk, state, stats);
+	row_groups->Append(chunk, state);
 }
 
 void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::function<void(DataChunk &chunk)> &function) {
@@ -204439,9 +204466,8 @@ void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::functi
 	}
 }
 
-void DataTable::MergeStorage(RowGroupCollection &data, TableIndexList &indexes, TableStatistics &other_stats) {
+void DataTable::MergeStorage(RowGroupCollection &data, TableIndexList &indexes) {
 	row_groups->MergeStorage(data);
-	stats.MergeStats(other_stats);
 	row_groups->Verify();
 }
 
@@ -204740,7 +204766,7 @@ void DataTable::Update(TableCatalogEntry &table, ClientContext &context, Vector
 	// we need to figure out for each id to which row group it belongs
 	// usually all (or many) ids belong to the same row group
 	// we iterate over the ids and check for every id if it belongs to the same row group as their predecessor
-	row_groups->Update(transaction, ids, column_ids, updates, stats);
+	row_groups->Update(transaction, ids, column_ids, updates);
 }
 
 void DataTable::UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
@@ -204761,7 +204787,7 @@ void DataTable::UpdateColumn(TableCatalogEntry &table, ClientContext &context, V
 
 	updates.Flatten();
 	row_ids.Flatten(updates.size());
-	row_groups->UpdateColumn(transaction, row_ids, column_path, updates, stats);
+	row_groups->UpdateColumn(transaction, row_ids, column_path, updates);
 }
 
 //===--------------------------------------------------------------------===//
@@ -204778,13 +204804,12 @@ unique_ptr<BaseStatistics> DataTable::GetStatistics(ClientContext &context, colu
 	if (column_id == COLUMN_IDENTIFIER_ROW_ID) {
 		return nullptr;
 	}
-	return stats.CopyStats(column_id);
+	return row_groups->CopyStats(column_id);
 }
 
 void DataTable::SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun) {
 	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
-	auto stats_guard = stats.GetLock();
-	set_fun(*stats.GetStats(column_id).stats);
+	row_groups->SetStatistics(column_id, set_fun);
 }
 
 //===--------------------------------------------------------------------===//
@@ -204795,7 +204820,7 @@ void DataTable::Checkpoint(TableDataWriter &writer) {
 	// FIXME: we might want to combine adjacent row groups in case they have had deletions...
 	vector<unique_ptr<BaseStatistics>> global_stats;
 	for (idx_t i = 0; i < column_definitions.size(); i++) {
-		global_stats.push_back(stats.CopyStats(i));
+		global_stats.push_back(row_groups->CopyStats(i));
 	}
 
 	row_groups->Checkpoint(writer, global_stats);
@@ -204933,16 +204958,96 @@ BlockPointer Index::Serialize(duckdb::MetaBlockWriter &writer) {
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// OptimisticDataWriter
+//===--------------------------------------------------------------------===//
+OptimisticDataWriter::OptimisticDataWriter(DataTable *table) : table(table) {
+}
+
+OptimisticDataWriter::OptimisticDataWriter(DataTable *table, OptimisticDataWriter &parent)
+    : table(table), partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
+	if (partial_manager) {
+		partial_manager->FlushPartialBlocks();
+	}
+}
+
+OptimisticDataWriter::~OptimisticDataWriter() {
+}
+
+void OptimisticDataWriter::CheckFlushToDisk(RowGroupCollection &row_groups) {
+	// we finished writing a complete row group
+	// check if we should pre-emptively write it to disk
+	if (table->info->IsTemporary() || StorageManager::GetStorageManager(table->db).InMemory()) {
+		return;
+	}
+	// we should! write the second-to-last row group to disk
+	// allocate the partial block-manager if none is allocated yet
+	if (!partial_manager) {
+		auto &block_manager = table->info->table_io_manager->GetBlockManagerForRowData();
+		partial_manager = make_unique<PartialBlockManager>(block_manager);
+	}
+	// flush second-to-last row group
+	auto row_group = row_groups.GetRowGroup(-2);
+	FlushToDisk(row_group);
+}
+
+void OptimisticDataWriter::FlushToDisk(RowGroup *row_group) {
+	// flush the specified row group
+	D_ASSERT(row_group);
+	//! The set of column compression types (if any)
+	vector<CompressionType> compression_types;
+	D_ASSERT(compression_types.empty());
+	for (auto &column : table->column_definitions) {
+		compression_types.push_back(column.CompressionType());
+	}
+	auto row_group_pointer = row_group->WriteToDisk(*partial_manager, compression_types);
+
+	// update the set of written blocks
+	for (idx_t col_idx = 0; col_idx < row_group_pointer.statistics.size(); col_idx++) {
+		row_group_pointer.states[col_idx]->GetBlockIds(written_blocks);
+	}
+}
+
+void OptimisticDataWriter::FlushToDisk(RowGroupCollection &row_groups) {
+	if (!partial_manager) {
+		// no partial manager - nothing to flush
+		return;
+	}
+	// flush the last row group
+	FlushToDisk(row_groups.GetRowGroup(-1));
+}
+
+void OptimisticDataWriter::FinalFlush() {
+	if (!partial_manager) {
+		return;
+	}
+	// then flush the partial manager
+	partial_manager->FlushPartialBlocks();
+	partial_manager.reset();
+}
+
+void OptimisticDataWriter::Rollback() {
+	if (partial_manager) {
+		partial_manager->Clear();
+		partial_manager.reset();
+	}
+	if (!written_blocks.empty()) {
+		auto &block_manager = table->info->table_io_manager->GetBlockManagerForRowData();
+		for (auto block_id : written_blocks) {
+			block_manager.MarkBlockAsModified(block_id);
+		}
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Local Table Storage
 //===--------------------------------------------------------------------===//
 LocalTableStorage::LocalTableStorage(DataTable &table)
-    : table(&table), allocator(Allocator::Get(table.db)), deleted_rows(0) {
+    : table(&table), allocator(Allocator::Get(table.db)), deleted_rows(0), optimistic_writer(&table) {
 	auto types = table.GetTypes();
 	row_groups = make_shared<RowGroupCollection>(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(),
 	                                             types, MAX_ROW_ID, 0);
-
-	stats.InitializeEmpty(types);
+	row_groups->InitializeEmpty();
 	table.info->indexes.Scan([&](Index &index) {
 		D_ASSERT(index.type == IndexType::ART);
 		auto &art = (ART &)index;
@@ -204963,24 +205068,15 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
                                      const LogicalType &target_type, const vector<column_t> &bound_columns,
                                      Expression &cast_expr)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
-	if (partial_manager) {
-		partial_manager->FlushPartialBlocks();
-	}
-	stats.InitializeAlterType(parent.stats, changed_idx, target_type);
-	row_groups =
-	    parent.row_groups->AlterType(changed_idx, target_type, bound_columns, cast_expr, stats.GetStats(changed_idx));
+      optimistic_writer(table, parent.optimistic_writer) {
+	row_groups = parent.row_groups->AlterType(changed_idx, target_type, bound_columns, cast_expr);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
 }
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t drop_idx)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
-	if (partial_manager) {
-		partial_manager->FlushPartialBlocks();
-	}
-	stats.InitializeRemoveColumn(parent.stats, drop_idx);
+      optimistic_writer(table, parent.optimistic_writer) {
 	row_groups = parent.row_groups->RemoveColumn(drop_idx);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
@@ -204989,10 +205085,8 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, ColumnDefinition &new_column,
                                      Expression *default_value)
     : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
-      partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
-	idx_t new_column_idx = parent.table->column_definitions.size();
-	stats.InitializeAddColumn(parent.stats, new_column.GetType());
-	row_groups = parent.row_groups->AddColumn(new_column, default_value, stats.GetStats(new_column_idx));
+      optimistic_writer(table, parent.optimistic_writer) {
+	row_groups = parent.row_groups->AddColumn(new_column, default_value);
 	parent.row_groups.reset();
 	indexes.Move(parent.indexes);
 }
@@ -205021,6 +205115,143 @@ idx_t LocalTableStorage::EstimatedSize() {
 	return appended_rows * row_size;
 }
 
+void LocalTableStorage::CheckFlushToDisk() {
+	if (deleted_rows != 0) {
+		// we have deletes - we cannot merge row groups
+		return;
+	}
+	optimistic_writer.CheckFlushToDisk(*row_groups);
+}
+
+void LocalTableStorage::FlushToDisk() {
+	optimistic_writer.FlushToDisk(*row_groups);
+	optimistic_writer.FinalFlush();
+}
+
+void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendState &append_state, idx_t append_count,
+                                        bool append_to_table) {
+	bool constraint_violated = false;
+	if (append_to_table) {
+		table->InitializeAppend(transaction, append_state, append_count);
+	}
+	if (append_to_table) {
+		// appending: need to scan entire
+		row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool {
+			// append this chunk to the indexes of the table
+			if (!table->AppendToIndexes(chunk, append_state.current_row)) {
+				constraint_violated = true;
+				return false;
+			}
+			// append to base table
+			table->Append(chunk, append_state);
+			return true;
+		});
+	} else {
+		// only need to scan for index append
+		// figure out which columns we need to scan for the set of indexes
+		auto columns = table->info->indexes.GetRequiredColumns();
+		// create an empty mock chunk that contains all the correct types for the table
+		DataChunk mock_chunk;
+		mock_chunk.InitializeEmpty(table->GetTypes());
+		row_groups->Scan(transaction, columns, [&](DataChunk &chunk) -> bool {
+			// construct the mock chunk by referencing the required columns
+			for (idx_t i = 0; i < columns.size(); i++) {
+				mock_chunk.data[columns[i]].Reference(chunk.data[i]);
+			}
+			mock_chunk.SetCardinality(chunk);
+			// append this chunk to the indexes of the table
+			if (!table->AppendToIndexes(mock_chunk, append_state.current_row)) {
+				constraint_violated = true;
+				return false;
+			}
+			append_state.current_row += chunk.size();
+			return true;
+		});
+	}
+	if (constraint_violated) {
+		// need to revert the append
+		row_t current_row = append_state.row_start;
+		// remove the data from the indexes, if there are any indexes
+		row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool {
+			// append this chunk to the indexes of the table
+			table->RemoveFromIndexes(append_state, chunk, current_row);
+
+			current_row += chunk.size();
+			if (current_row >= append_state.current_row) {
+				// finished deleting all rows from the index: abort now
+				return false;
+			}
+			return true;
+		});
+		if (append_to_table) {
+			table->RevertAppendInternal(append_state.row_start, append_count);
+		}
+		throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
+	}
+}
+
+void LocalTableStorage::Rollback() {
+	optimistic_writer.Rollback();
+}
+
+//===--------------------------------------------------------------------===//
+// LocalTableManager
+//===--------------------------------------------------------------------===//
+LocalTableStorage *LocalTableManager::GetStorage(DataTable *table) {
+	lock_guard<mutex> l(table_storage_lock);
+	auto entry = table_storage.find(table);
+	return entry == table_storage.end() ? nullptr : entry->second.get();
+}
+
+LocalTableStorage *LocalTableManager::GetOrCreateStorage(DataTable *table) {
+	lock_guard<mutex> l(table_storage_lock);
+	auto entry = table_storage.find(table);
+	if (entry == table_storage.end()) {
+		auto new_storage = make_shared<LocalTableStorage>(*table);
+		auto storage = new_storage.get();
+		table_storage.insert(make_pair(table, move(new_storage)));
+		return storage;
+	} else {
+		return entry->second.get();
+	}
+}
+
+bool LocalTableManager::IsEmpty() {
+	lock_guard<mutex> l(table_storage_lock);
+	return table_storage.empty();
+}
+
+shared_ptr<LocalTableStorage> LocalTableManager::MoveEntry(DataTable *table) {
+	lock_guard<mutex> l(table_storage_lock);
+	auto entry = table_storage.find(table);
+	if (entry == table_storage.end()) {
+		return nullptr;
+	}
+	auto storage_entry = move(entry->second);
+	table_storage.erase(table);
+	return storage_entry;
+}
+
+unordered_map<DataTable *, shared_ptr<LocalTableStorage>> LocalTableManager::MoveEntries() {
+	lock_guard<mutex> l(table_storage_lock);
+	return move(table_storage);
+}
+
+idx_t LocalTableManager::EstimatedSize() {
+	lock_guard<mutex> l(table_storage_lock);
+	idx_t estimated_size = 0;
+	for (auto &storage : table_storage) {
+		estimated_size += storage.second->EstimatedSize();
+	}
+	return estimated_size;
+}
+
+void LocalTableManager::InsertEntry(DataTable *table, shared_ptr<LocalTableStorage> entry) {
+	lock_guard<mutex> l(table_storage_lock);
+	D_ASSERT(table_storage.find(table) == table_storage.end());
+	table_storage[table] = move(entry);
+}
+
 //===--------------------------------------------------------------------===//
 // LocalStorage
 //===--------------------------------------------------------------------===//
@@ -205036,11 +205267,10 @@ LocalStorage &LocalStorage::Get(ClientContext &context) {
 }
 
 void LocalStorage::InitializeScan(DataTable *table, CollectionScanState &state, TableFilterSet *table_filters) {
-	auto entry = table_storage.find(table);
-	if (entry == table_storage.end()) {
+	auto storage = table_manager.GetStorage(table);
+	if (storage == nullptr) {
 		return;
 	}
-	auto storage = entry->second.get();
 	storage->InitializeScan(state, table_filters);
 }
 
@@ -205049,7 +205279,7 @@ void LocalStorage::Scan(CollectionScanState &state, const vector<column_t> &colu
 }
 
 void LocalStorage::InitializeParallelScan(DataTable *table, ParallelCollectionScanState &state) {
-	auto storage = GetStorage(table);
+	auto storage = table_manager.GetStorage(table);
 	if (!storage) {
 		state.max_row = 0;
 		state.vector_index = 0;
@@ -205061,7 +205291,7 @@ void LocalStorage::InitializeParallelScan(DataTable *table, ParallelCollectionSc
 
 bool LocalStorage::NextParallelScan(ClientContext &context, DataTable *table, ParallelCollectionScanState &state,
                                     CollectionScanState &scan_state) {
-	auto storage = GetStorage(table);
+	auto storage = table_manager.GetStorage(table);
 	if (!storage) {
 		return false;
 	}
@@ -205069,15 +205299,8 @@ bool LocalStorage::NextParallelScan(ClientContext &context, DataTable *table, Pa
 }
 
 void LocalStorage::InitializeAppend(LocalAppendState &state, DataTable *table) {
-	auto entry = table_storage.find(table);
-	if (entry == table_storage.end()) {
-		auto new_storage = make_shared<LocalTableStorage>(*table);
-		state.storage = new_storage.get();
-		table_storage.insert(make_pair(table, move(new_storage)));
-	} else {
-		state.storage = entry->second.get();
-	}
-	state.storage->row_groups->InitializeAppend(state.append_state);
+	state.storage = table_manager.GetOrCreateStorage(table);
+	state.storage->row_groups->InitializeAppend(TransactionData(transaction), state.append_state, 0);
 }
 
 void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
@@ -205089,7 +205312,7 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 	}
 
 	//! Append the chunk to the local storage
-	auto new_row_group = storage->row_groups->Append(chunk, state.append_state, storage->stats);
+	auto new_row_group = storage->row_groups->Append(chunk, state.append_state);
 
 	//! Check if we should pre-emptively flush blocks to disk
 	if (new_row_group) {
@@ -205097,80 +205320,29 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 	}
 }
 
-void LocalTableStorage::CheckFlushToDisk() {
-	// we finished writing a complete row group
-	// check if we should pre-emptively write it to disk
-	if (table->info->IsTemporary() || StorageManager::GetStorageManager(table->db).InMemory()) {
-		return;
-	}
-	if (deleted_rows != 0) {
-		// we have deletes - we cannot merge
-		return;
-	}
-	// we should! write the second-to-last row group to disk
-	// allocate the partial block-manager if none is allocated yet
-	if (!partial_manager) {
-		auto &block_manager = table->info->table_io_manager->GetBlockManagerForRowData();
-		partial_manager = make_unique<PartialBlockManager>(block_manager);
-	}
-	// flush second-to-last row group
-	auto row_group = row_groups->GetRowGroup(-2);
-	FlushToDisk(row_group);
+void LocalStorage::FinalizeAppend(LocalAppendState &state) {
+	state.storage->row_groups->FinalizeAppend(state.append_state.transaction, state.append_state);
 }
 
-void LocalTableStorage::FlushToDisk(RowGroup *row_group) {
-	// flush the specified row group
-	D_ASSERT(row_group);
-	D_ASSERT(deleted_rows == 0);
-	D_ASSERT(partial_manager);
-	//! The set of column compression types (if any)
-	vector<CompressionType> compression_types;
-	D_ASSERT(compression_types.empty());
-	for (auto &column : table->column_definitions) {
-		compression_types.push_back(column.CompressionType());
-	}
-	auto row_group_pointer = row_group->WriteToDisk(*partial_manager, compression_types);
-	for (idx_t col_idx = 0; col_idx < row_group_pointer.statistics.size(); col_idx++) {
-		row_group_pointer.states[col_idx]->GetBlockIds(written_blocks);
-		stats.MergeStats(col_idx, *row_group_pointer.statistics[col_idx]);
-	}
-}
-void LocalTableStorage::FlushToDisk() {
-	// no partial manager - nothing to flush
-	if (!partial_manager) {
-		return;
-	}
-	// flush the last row group
-	FlushToDisk(row_groups->GetRowGroup(-1));
-	// then flush the partial manager
-	partial_manager->FlushPartialBlocks();
-	partial_manager.reset();
+void LocalStorage::LocalMerge(DataTable *table, RowGroupCollection &collection) {
+	auto storage = table_manager.GetOrCreateStorage(table);
+	storage->row_groups->MergeStorage(collection);
 }
 
-void LocalStorage::FinalizeAppend(LocalAppendState &state) {
-	TransactionData transaction_data(0, 0);
-	state.storage->row_groups->FinalizeAppend(transaction_data, state.append_state);
+bool LocalStorage::ChangesMade() noexcept {
+	return !table_manager.IsEmpty();
 }
 
-LocalTableStorage *LocalStorage::GetStorage(DataTable *table) {
-	auto entry = table_storage.find(table);
-	return entry == table_storage.end() ? nullptr : entry->second.get();
+bool LocalStorage::Find(DataTable *table) {
+	return table_manager.GetStorage(table) != nullptr;
 }
 
 idx_t LocalStorage::EstimatedSize() {
-	idx_t estimated_size = 0;
-	for (auto &storage : table_storage) {
-		estimated_size += storage.second->EstimatedSize();
-	}
-	return estimated_size;
-}
-
-bool LocalTableStorage::HasWrittenBlocks() {
-	return partial_manager || !written_blocks.empty();
+	return table_manager.EstimatedSize();
 }
 
 idx_t LocalStorage::Delete(DataTable *table, Vector &row_ids, idx_t count) {
-	auto storage = GetStorage(table);
+	auto storage = table_manager.GetStorage(table);
 	D_ASSERT(storage);
 
 	// delete from unique indices (if any)
@@ -205185,119 +205357,14 @@ idx_t LocalStorage::Delete(DataTable *table, Vector &row_ids, idx_t count) {
 }
 
 void LocalStorage::Update(DataTable *table, Vector &row_ids, const vector<column_t> &column_ids, DataChunk &updates) {
-	auto storage = GetStorage(table);
+	auto storage = table_manager.GetStorage(table);
 	D_ASSERT(storage);
 
 	auto ids = FlatVector::GetData<row_t>(row_ids);
-	storage->row_groups->Update(TransactionData(0, 0), ids, column_ids, updates, storage->stats);
-}
-
-template <class T>
-bool LocalTableStorage::ScanTableStorage(Transaction &transaction, const vector<column_t> &column_ids, T &&fun) {
-	auto all_types = table->GetTypes();
-	vector<LogicalType> scan_types;
-	for (idx_t i = 0; i < column_ids.size(); i++) {
-		scan_types.push_back(all_types[column_ids[i]]);
-	}
-	DataChunk chunk;
-	chunk.Initialize(allocator, scan_types);
-
-	// initialize the scan
-	TableScanState state;
-	state.Initialize(column_ids, nullptr);
-	InitializeScan(state.local_state, nullptr);
-
-	while (true) {
-		chunk.Reset();
-		state.local_state.Scan(transaction, chunk);
-		if (chunk.size() == 0) {
-			return true;
-		}
-		if (!fun(chunk)) {
-			return false;
-		}
-	}
-}
-
-template <class T>
-bool LocalTableStorage::ScanTableStorage(Transaction &transaction, T &&fun) {
-	vector<column_t> column_ids;
-	column_ids.reserve(table->column_definitions.size());
-	for (idx_t i = 0; i < table->column_definitions.size(); i++) {
-		column_ids.push_back(i);
-	}
-	return ScanTableStorage(transaction, column_ids, fun);
-}
-
-void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendState &append_state, idx_t append_count,
-                                        bool append_to_table) {
-	bool constraint_violated = false;
-	if (append_to_table) {
-		table->InitializeAppend(transaction, append_state, append_count);
-	}
-	if (append_to_table) {
-		// appending: need to scan entire
-		ScanTableStorage(transaction, [&](DataChunk &chunk) -> bool {
-			// append this chunk to the indexes of the table
-			if (!table->AppendToIndexes(chunk, append_state.current_row)) {
-				constraint_violated = true;
-				return false;
-			}
-			// append to base table
-			table->Append(chunk, append_state);
-			return true;
-		});
-	} else {
-		// only need to scan for index append
-		// figure out which columns we need to scan for the set of indexes
-		auto columns = table->info->indexes.GetRequiredColumns();
-		// create an empty mock chunk that contains all the correct types for the table
-		DataChunk mock_chunk;
-		mock_chunk.InitializeEmpty(table->GetTypes());
-		ScanTableStorage(transaction, columns, [&](DataChunk &chunk) -> bool {
-			// construct the mock chunk by referencing the required columns
-			for (idx_t i = 0; i < columns.size(); i++) {
-				mock_chunk.data[columns[i]].Reference(chunk.data[i]);
-			}
-			mock_chunk.SetCardinality(chunk);
-			// append this chunk to the indexes of the table
-			if (!table->AppendToIndexes(mock_chunk, append_state.current_row)) {
-				constraint_violated = true;
-				return false;
-			}
-			append_state.current_row += chunk.size();
-			return true;
-		});
-	}
-	if (constraint_violated) {
-		// need to revert the append
-		row_t current_row = append_state.row_start;
-		// remove the data from the indexes, if there are any indexes
-		ScanTableStorage(transaction, [&](DataChunk &chunk) -> bool {
-			// append this chunk to the indexes of the table
-			table->RemoveFromIndexes(append_state, chunk, current_row);
-
-			current_row += chunk.size();
-			if (current_row >= append_state.current_row) {
-				// finished deleting all rows from the index: abort now
-				return false;
-			}
-			return true;
-		});
-		if (append_to_table) {
-			table->RevertAppendInternal(append_state.row_start, append_count);
-		}
-		throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
-	}
+	storage->row_groups->Update(TransactionData(0, 0), ids, column_ids, updates);
 }
 
 void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
-	// bulk append threshold: a full row group
-	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
-
-	auto storage_entry = move(table_storage[&table]);
-	table_storage[&table].reset();
-
 	if (storage.row_groups->GetTotalRows() <= storage.deleted_rows) {
 		return;
 	}
@@ -205317,13 +205384,12 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 			storage.AppendToIndexes(transaction, append_state, append_count, false);
 		}
 		// finally move over the row groups
-		table.MergeStorage(*storage.row_groups, storage.indexes, storage.stats);
+		table.MergeStorage(*storage.row_groups, storage.indexes);
 	} else {
-		if (storage.partial_manager || !storage.written_blocks.empty()) {
-			// we have written data but cannot merge to disk after all
-			// revert the data we have already written
-			storage.Rollback();
-		}
+		// check if we have written data
+		// if we have, we cannot merge to disk after all
+		// so we need to revert the data we have already written
+		storage.Rollback();
 		// append to the indexes and append to the base table
 		storage.AppendToIndexes(transaction, append_state, append_count, true);
 	}
@@ -205331,102 +205397,91 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 }
 
 void LocalStorage::Commit(LocalStorage::CommitState &commit_state, Transaction &transaction) {
-	// commit local storage, iterate over all entries in the table storage map
+	// commit local storage
+	// iterate over all entries in the table storage map and commit them
+	// after this, the local storage is no longer required and can be cleared
+	auto table_storage = table_manager.MoveEntries();
 	for (auto &entry : table_storage) {
 		auto table = entry.first;
 		auto storage = entry.second.get();
 		Flush(*table, *storage);
+
+		entry.second.reset();
 	}
-	// finished commit: clear local storage
-	table_storage.clear();
 }
 
 void LocalStorage::Rollback() {
+	// rollback local storage
+	// after this, the local storage is no longer required and can be cleared
+	auto table_storage = table_manager.MoveEntries();
 	for (auto &entry : table_storage) {
 		auto storage = entry.second.get();
 		if (!storage) {
 			continue;
 		}
 		storage->Rollback();
-	}
-}
 
-void LocalTableStorage::Rollback() {
-	if (partial_manager) {
-		partial_manager->Clear();
-		partial_manager.reset();
-	}
-	auto &block_manager = table->info->table_io_manager->GetBlockManagerForRowData();
-	for (auto block_id : written_blocks) {
-		block_manager.MarkBlockAsModified(block_id);
+		entry.second.reset();
 	}
 }
 
 idx_t LocalStorage::AddedRows(DataTable *table) {
-	auto entry = table_storage.find(table);
-	if (entry == table_storage.end()) {
+	auto storage = table_manager.GetStorage(table);
+	if (!storage) {
 		return 0;
 	}
-	return entry->second->row_groups->GetTotalRows() - entry->second->deleted_rows;
+	return storage->row_groups->GetTotalRows() - storage->deleted_rows;
 }
 
 void LocalStorage::MoveStorage(DataTable *old_dt, DataTable *new_dt) {
 	// check if there are any pending appends for the old version of the table
-	auto entry = table_storage.find(old_dt);
-	if (entry == table_storage.end()) {
+	auto new_storage = table_manager.MoveEntry(old_dt);
+	if (!new_storage) {
 		return;
 	}
 	// take over the storage from the old entry
-	auto new_storage = move(entry->second);
 	new_storage->table = new_dt;
-	table_storage.erase(entry);
-	table_storage[new_dt] = move(new_storage);
+	table_manager.InsertEntry(new_dt, move(new_storage));
 }
 
 void LocalStorage::AddColumn(DataTable *old_dt, DataTable *new_dt, ColumnDefinition &new_column,
                              Expression *default_value) {
 	// check if there are any pending appends for the old version of the table
-	auto entry = table_storage.find(old_dt);
-	if (entry == table_storage.end()) {
+	auto storage = table_manager.MoveEntry(old_dt);
+	if (!storage) {
 		return;
 	}
-	auto storage = move(entry->second);
 	auto new_storage = make_unique<LocalTableStorage>(*new_dt, *storage, new_column, default_value);
-
-	table_storage[new_dt] = move(new_storage);
-	table_storage.erase(old_dt);
+	table_manager.InsertEntry(new_dt, move(new_storage));
 }
 
 void LocalStorage::DropColumn(DataTable *old_dt, DataTable *new_dt, idx_t removed_column) {
 	// check if there are any pending appends for the old version of the table
-	auto entry = table_storage.find(old_dt);
-	if (entry == table_storage.end()) {
+	auto storage = table_manager.MoveEntry(old_dt);
+	if (!storage) {
 		return;
 	}
-	auto storage = move(entry->second);
 	auto new_storage = make_unique<LocalTableStorage>(*new_dt, *storage, removed_column);
-
-	table_storage[new_dt] = move(new_storage);
-	table_storage.erase(old_dt);
+	table_manager.InsertEntry(new_dt, move(new_storage));
 }
 
 void LocalStorage::ChangeType(DataTable *old_dt, DataTable *new_dt, idx_t changed_idx, const LogicalType &target_type,
                               const vector<column_t> &bound_columns, Expression &cast_expr) {
 	// check if there are any pending appends for the old version of the table
-	auto entry = table_storage.find(old_dt);
-	if (entry == table_storage.end()) {
+	auto storage = table_manager.MoveEntry(old_dt);
+	if (!storage) {
 		return;
 	}
-	auto storage = move(entry->second);
 	auto new_storage =
 	    make_unique<LocalTableStorage>(*new_dt, *storage, changed_idx, target_type, bound_columns, cast_expr);
-
-	table_storage[new_dt] = move(new_storage);
-	table_storage.erase(old_dt);
+	table_manager.InsertEntry(new_dt, move(new_storage));
 }
 
 void LocalStorage::FetchChunk(DataTable *table, Vector &row_ids, idx_t count, DataChunk &verify_chunk) {
-	auto storage = GetStorage(table);
+	auto storage = table_manager.GetStorage(table);
+	if (!storage) {
+		throw InternalException("LocalStorage::FetchChunk - local storage not found");
+	}
 
 	ColumnFetchState fetch_state;
 	vector<column_t> col_ids;
@@ -205439,13 +205494,15 @@ void LocalStorage::FetchChunk(DataTable *table, Vector &row_ids, idx_t count, Da
 }
 
 TableIndexList &LocalStorage::GetIndexes(DataTable *table) {
-	auto storage = GetStorage(table);
-
+	auto storage = table_manager.GetStorage(table);
+	if (!storage) {
+		throw InternalException("LocalStorage::GetIndexes - local storage not found");
+	}
 	return storage->indexes;
 }
 
 void LocalStorage::VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint) {
-	auto storage = GetStorage(&parent);
+	auto storage = table_manager.GetStorage(&parent);
 	if (!storage) {
 		return;
 	}
@@ -207771,9 +207828,6 @@ idx_t ChunkVectorInfo::Delete(transaction_t transaction_id, row_t rows[], idx_t
 			// tuple was already deleted by another transaction
 			throw TransactionException("Conflict on tuple deletion!");
 		}
-		if (inserted[rows[i]] >= TRANSACTION_ID_START) {
-			throw TransactionException("Deleting non-committed tuples is not supported (for now...)");
-		}
 		// after verifying that there are no conflicts we mark the tuple as deleted
 		deleted[rows[i]] = transaction_id;
 		rows[deleted_tuples] = rows[i];
@@ -208297,7 +208351,7 @@ ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
     : block_manager(other.block_manager), info(other.info), column_index(other.column_index), start(start),
       type(move(other.type)), parent(parent), updates(move(other.updates)), version(parent ? parent->version + 1 : 0) {
 	idx_t offset = 0;
-	for (auto segment = other.data.GetRootSegment(); segment; segment = segment->next.get()) {
+	for (auto segment = other.data.GetRootSegment(); segment; segment = segment->Next()) {
 		auto &other = (ColumnSegment &)*segment;
 		this->data.AppendSegment(ColumnSegment::CreateSegment(other, start + offset));
 		offset += segment->count;
@@ -208327,8 +208381,9 @@ void ColumnData::IncrementVersion() {
 }
 
 idx_t ColumnData::GetMaxEntry() {
-	auto first_segment = data.GetRootSegment();
-	auto last_segment = data.GetLastSegment();
+	auto l = data.Lock();
+	auto first_segment = data.GetRootSegment(l);
+	auto last_segment = data.GetLastSegment(l);
 	if (!first_segment) {
 		D_ASSERT(!last_segment);
 		return 0;
@@ -208380,15 +208435,18 @@ idx_t ColumnData::ScanVector(ColumnScanState &state, Vector &result, idx_t remai
 		         state.row_index <= state.current->start + state.current->count);
 		idx_t scan_count = MinValue<idx_t>(remaining, state.current->start + state.current->count - state.row_index);
 		idx_t result_offset = initial_remaining - remaining;
-		state.current->Scan(state, scan_count, result, result_offset, scan_count == initial_remaining);
+		if (scan_count > 0) {
+			state.current->Scan(state, scan_count, result, result_offset, scan_count == initial_remaining);
+
+			state.row_index += scan_count;
+			remaining -= scan_count;
+		}
 
-		state.row_index += scan_count;
-		remaining -= scan_count;
 		if (remaining > 0) {
 			if (!state.current->next) {
 				break;
 			}
-			state.current = (ColumnSegment *)state.current->next.get();
+			state.current = (ColumnSegment *)state.current->Next();
 			state.current->InitializeScan(state);
 			state.segment_checked = false;
 			D_ASSERT(state.row_index >= state.current->start &&
@@ -208488,17 +208546,17 @@ void ColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, Vector
 }
 
 void ColumnData::InitializeAppend(ColumnAppendState &state) {
-	lock_guard<mutex> tree_lock(data.node_lock);
-	if (data.nodes.empty()) {
+	auto l = data.Lock();
+	if (data.IsEmpty(l)) {
 		// no segments yet, append an empty segment
-		AppendTransientSegment(start);
+		AppendTransientSegment(l, start);
 	}
-	auto segment = (ColumnSegment *)data.GetLastSegment();
+	auto segment = (ColumnSegment *)data.GetLastSegment(l);
 	if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
 		// no transient segments yet
 		auto total_rows = segment->start + segment->count;
-		AppendTransientSegment(total_rows);
-		state.current = (ColumnSegment *)data.GetLastSegment();
+		AppendTransientSegment(l, total_rows);
+		state.current = (ColumnSegment *)data.GetLastSegment(l);
 	} else {
 		state.current = (ColumnSegment *)segment;
 	}
@@ -208521,9 +208579,9 @@ void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, Uni
 
 		// we couldn't fit everything we wanted in the current column segment, create a new one
 		{
-			lock_guard<mutex> tree_lock(data.node_lock);
-			AppendTransientSegment(state.current->start + state.current->count);
-			state.current = (ColumnSegment *)data.GetLastSegment();
+			auto l = data.Lock();
+			AppendTransientSegment(l, state.current->start + state.current->count);
+			state.current = (ColumnSegment *)data.GetLastSegment(l);
 			state.current->InitializeAppend(state);
 		}
 		offset += copied_elements;
@@ -208532,23 +208590,23 @@ void ColumnData::AppendData(BaseStatistics &stats, ColumnAppendState &state, Uni
 }
 
 void ColumnData::RevertAppend(row_t start_row) {
-	lock_guard<mutex> tree_lock(data.node_lock);
+	auto l = data.Lock();
 	// check if this row is in the segment tree at all
-	if (idx_t(start_row) >= data.nodes.back().row_start + data.nodes.back().node->count) {
+	auto last_segment = data.GetLastSegment(l);
+	if (idx_t(start_row) >= last_segment->start + last_segment->count) {
 		// the start row is equal to the final portion of the column data: nothing was ever appended here
-		D_ASSERT(idx_t(start_row) == data.nodes.back().row_start + data.nodes.back().node->count);
+		D_ASSERT(idx_t(start_row) == last_segment->start + last_segment->count);
 		return;
 	}
 	// find the segment index that the current row belongs to
-	idx_t segment_index = data.GetSegmentIndex(start_row);
-	auto segment = data.nodes[segment_index].node;
+	idx_t segment_index = data.GetSegmentIndex(l, start_row);
+	auto segment = data.GetSegmentByIndex(l, segment_index);
 	auto &transient = (ColumnSegment &)*segment;
 	D_ASSERT(transient.segment_type == ColumnSegmentType::TRANSIENT);
 
 	// remove any segments AFTER this segment: they should be deleted entirely
-	if (segment_index < data.nodes.size() - 1) {
-		data.nodes.erase(data.nodes.begin() + segment_index + 1, data.nodes.end());
-	}
+	data.EraseSegments(l, segment_index);
+
 	segment->next = nullptr;
 	transient.RevertAppend(start_row);
 }
@@ -208602,9 +208660,9 @@ unique_ptr<BaseStatistics> ColumnData::GetUpdateStatistics() {
 	return updates ? updates->GetStatistics() : nullptr;
 }
 
-void ColumnData::AppendTransientSegment(idx_t start_row) {
+void ColumnData::AppendTransientSegment(SegmentLock &l, idx_t start_row) {
 	auto new_segment = ColumnSegment::CreateTransientSegment(GetDatabase(), type, start_row);
-	data.AppendSegment(move(new_segment));
+	data.AppendSegment(l, move(new_segment));
 }
 
 void ColumnData::CommitDropColumn() {
@@ -208616,7 +208674,7 @@ void ColumnData::CommitDropColumn() {
 				block_manager.MarkBlockAsModified(block_id);
 			}
 		}
-		segment = (ColumnSegment *)segment->next.get();
+		segment = (ColumnSegment *)segment->Next();
 	}
 }
 
@@ -208642,17 +208700,19 @@ unique_ptr<ColumnCheckpointState> ColumnData::Checkpoint(RowGroup &row_group,
 	auto checkpoint_state = CreateCheckpointState(row_group, partial_block_manager);
 	checkpoint_state->global_stats = BaseStatistics::CreateEmpty(type, StatisticsType::LOCAL_STATS);
 
-	if (!data.root_node) {
+	auto l = data.Lock();
+	auto nodes = data.MoveSegments(l);
+	if (nodes.empty()) {
 		// empty table: flush the empty list
 		return checkpoint_state;
 	}
 	lock_guard<mutex> update_guard(update_lock);
 
 	ColumnDataCheckpointer checkpointer(*this, row_group, *checkpoint_state, checkpoint_info);
-	checkpointer.Checkpoint(move(data.root_node));
+	checkpointer.Checkpoint(move(nodes));
 
 	// replace the old tree with the new one
-	data.Replace(checkpoint_state->new_tree);
+	data.Replace(l, checkpoint_state->new_tree);
 	version++;
 
 	return checkpoint_state;
@@ -208743,13 +208803,14 @@ void ColumnData::GetStorageInfo(idx_t row_group_index, vector<idx_t> col_path, v
 		result.push_back(move(column_info));
 
 		segment_idx++;
-		segment = (ColumnSegment *)segment->next.get();
+		segment = (ColumnSegment *)segment->Next();
 	}
 }
 
 void ColumnData::Verify(RowGroup &parent) {
 #ifdef DEBUG
 	D_ASSERT(this->start == parent.start);
+	data.Verify();
 	auto root = data.GetRootSegment();
 	if (root) {
 		D_ASSERT(root != nullptr);
@@ -208761,7 +208822,7 @@ void ColumnData::Verify(RowGroup &parent) {
 			if (!root->next) {
 				D_ASSERT(prev_end == parent.start + parent.count);
 			}
-			root = root->next.get();
+			root = root->Next();
 		}
 	}
 #endif
@@ -208853,7 +208914,8 @@ ColumnCheckpointState &ColumnDataCheckpointer::GetCheckpointState() {
 
 void ColumnDataCheckpointer::ScanSegments(const std::function<void(Vector &, idx_t)> &callback) {
 	Vector scan_vector(intermediate.GetType(), nullptr);
-	for (auto segment = (ColumnSegment *)owned_segment.get(); segment; segment = (ColumnSegment *)segment->next.get()) {
+	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
+		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
 		ColumnScanState scan_state;
 		scan_state.current = segment;
 		segment->InitializeScan(scan_state);
@@ -208977,7 +209039,8 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	// if there are any persistent segments, we will mark their old block ids as modified
 	// since the segments will be rewritten their old on disk data is no longer required
 	auto &block_manager = col_data.block_manager;
-	for (auto segment = (ColumnSegment *)owned_segment.get(); segment; segment = (ColumnSegment *)segment->next.get()) {
+	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
+		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
 		if (segment->segment_type == ColumnSegmentType::PERSISTENT) {
 			// persistent segment has updates: mark it as modified and rewrite the block with the merged updates
 			auto block_id = segment->GetBlockId();
@@ -209003,11 +209066,12 @@ void ColumnDataCheckpointer::WriteToDisk() {
 	    [&](Vector &scan_vector, idx_t count) { best_function->compress(*compress_state, scan_vector, count); });
 	best_function->compress_finalize(*compress_state);
 
-	owned_segment.reset();
+	nodes.clear();
 }
 
 bool ColumnDataCheckpointer::HasChanges() {
-	for (auto segment = (ColumnSegment *)owned_segment.get(); segment; segment = (ColumnSegment *)segment->next.get()) {
+	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
+		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
 		if (segment->segment_type == ColumnSegmentType::TRANSIENT) {
 			// transient segment: always need to write to disk
 			return true;
@@ -209026,10 +209090,8 @@ bool ColumnDataCheckpointer::HasChanges() {
 void ColumnDataCheckpointer::WritePersistentSegments() {
 	// all segments are persistent and there are no updates
 	// we only need to write the metadata
-	auto segment = (ColumnSegment *)owned_segment.get();
-	while (segment) {
-		auto next_segment = move(segment->next);
-
+	for (idx_t segment_idx = 0; segment_idx < nodes.size(); segment_idx++) {
+		auto segment = (ColumnSegment *)nodes[segment_idx].node.get();
 		D_ASSERT(segment->segment_type == ColumnSegmentType::PERSISTENT);
 
 		// set up the data pointer directly using the data from the persistent segment
@@ -209045,19 +209107,15 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
 		state.global_stats->Merge(*segment->stats.statistics);
 
 		// directly append the current segment to the new tree
-		state.new_tree.AppendSegment(move(owned_segment));
+		state.new_tree.AppendSegment(move(nodes[segment_idx].node));
 
 		state.data_pointers.push_back(move(pointer));
-
-		// move to the next segment in the list
-		owned_segment = move(next_segment);
-		segment = (ColumnSegment *)owned_segment.get();
 	}
 }
 
-void ColumnDataCheckpointer::Checkpoint(unique_ptr<SegmentBase> segment) {
-	D_ASSERT(!owned_segment);
-	this->owned_segment = move(segment);
+void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode> nodes) {
+	D_ASSERT(!nodes.empty());
+	this->nodes = move(nodes);
 	// first check if any of the segments have changes
 	if (!HasChanges()) {
 		// no changes: only need to write the metadata for this column
@@ -210917,21 +210975,27 @@ Allocator &RowGroupCollection::GetAllocator() const {
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Initialize(PersistentTableData &data) {
 	D_ASSERT(this->row_start == 0);
+	auto l = row_groups->Lock();
 	for (auto &row_group_pointer : data.row_groups) {
 		auto new_row_group = make_unique<RowGroup>(info->db, block_manager, *info, types, move(row_group_pointer));
 		auto row_group_count = new_row_group->start + new_row_group->count;
 		if (row_group_count > this->total_rows) {
 			this->total_rows = row_group_count;
 		}
-		row_groups->AppendSegment(move(new_row_group));
+		row_groups->AppendSegment(l, move(new_row_group));
 	}
+	stats.Initialize(types, data);
 }
 
-void RowGroupCollection::AppendRowGroup(idx_t start_row) {
+void RowGroupCollection::InitializeEmpty() {
+	stats.InitializeEmpty(types);
+}
+
+void RowGroupCollection::AppendRowGroup(SegmentLock &l, idx_t start_row) {
 	D_ASSERT(start_row >= row_start);
 	auto new_row_group = make_unique<RowGroup>(info->db, block_manager, *info, start_row, 0);
 	new_row_group->InitializeEmpty(types);
-	row_groups->AppendSegment(move(new_row_group));
+	row_groups->AppendSegment(l, move(new_row_group));
 }
 
 RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
@@ -210941,7 +211005,8 @@ RowGroup *RowGroupCollection::GetRowGroup(int64_t index) {
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
 	idx_t current_total_rows = 0;
-	for (auto segment = row_groups->GetRootSegment(); segment; segment = segment->next.get()) {
+	row_groups->Verify();
+	for (auto segment = row_groups->GetRootSegment(); segment; segment = segment->Next()) {
 		auto &row_group = (RowGroup &)*segment;
 		row_group.Verify();
 		D_ASSERT(row_group.start == this->row_start + current_total_rows);
@@ -210960,12 +211025,12 @@ void RowGroupCollection::InitializeScan(CollectionScanState &state, const vector
 	D_ASSERT(row_group);
 	state.max_row = row_start + total_rows;
 	while (row_group && !row_group->InitializeScan(state.row_group_state)) {
-		row_group = (RowGroup *)row_group->next.get();
+		row_group = (RowGroup *)row_group->Next();
 	}
 }
 
 void RowGroupCollection::InitializeCreateIndexScan(CreateIndexScanState &state) {
-	state.delete_lock = std::unique_lock<mutex>(row_groups->node_lock);
+	state.segment_lock = row_groups->Lock();
 }
 
 void RowGroupCollection::InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids,
@@ -211011,11 +211076,11 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec
 		if (ClientConfig::GetConfig(context).verify_parallelism) {
 			state.vector_index++;
 			if (state.vector_index * STANDARD_VECTOR_SIZE >= state.current_row_group->count) {
-				state.current_row_group = (RowGroup *)state.current_row_group->next.get();
+				state.current_row_group = (RowGroup *)state.current_row_group->Next();
 				state.vector_index = 0;
 			}
 		} else {
-			state.current_row_group = (RowGroup *)state.current_row_group->next.get();
+			state.current_row_group = (RowGroup *)state.current_row_group->Next();
 		}
 		if (!need_to_scan) {
 			// filters allow us to skip this row group: move to the next row group
@@ -211026,6 +211091,41 @@ bool RowGroupCollection::NextParallelScan(ClientContext &context, ParallelCollec
 	return false;
 }
 
+bool RowGroupCollection::Scan(Transaction &transaction, const vector<column_t> &column_ids,
+                              const std::function<bool(DataChunk &chunk)> &fun) {
+	vector<LogicalType> scan_types;
+	for (idx_t i = 0; i < column_ids.size(); i++) {
+		scan_types.push_back(types[column_ids[i]]);
+	}
+	DataChunk chunk;
+	chunk.Initialize(GetAllocator(), scan_types);
+
+	// initialize the scan
+	TableScanState state;
+	state.Initialize(column_ids, nullptr);
+	InitializeScan(state.local_state, column_ids, nullptr);
+
+	while (true) {
+		chunk.Reset();
+		state.local_state.Scan(transaction, chunk);
+		if (chunk.size() == 0) {
+			return true;
+		}
+		if (!fun(chunk)) {
+			return false;
+		}
+	}
+}
+
+bool RowGroupCollection::Scan(Transaction &transaction, const std::function<bool(DataChunk &chunk)> &fun) {
+	vector<column_t> column_ids;
+	column_ids.reserve(types.size());
+	for (idx_t i = 0; i < types.size(); i++) {
+		column_ids.push_back(i);
+	}
+	return Scan(transaction, column_ids, fun);
+}
+
 //===--------------------------------------------------------------------===//
 // Fetch
 //===--------------------------------------------------------------------===//
@@ -211036,7 +211136,16 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 	idx_t count = 0;
 	for (idx_t i = 0; i < fetch_count; i++) {
 		auto row_id = row_ids[i];
-		auto row_group = (RowGroup *)row_groups->GetSegment(row_id);
+		RowGroup *row_group;
+		{
+			idx_t segment_index;
+			auto l = row_groups->Lock();
+			if (!row_groups->TryGetSegmentIndex(l, row_id, segment_index)) {
+				// in parallel append scenarios it is possible for the row_id
+				continue;
+			}
+			row_group = (RowGroup *)row_groups->GetSegmentByIndex(l, segment_index);
+		}
 		if (!row_group->Fetch(transaction, row_id - row_group->start)) {
 			continue;
 		}
@@ -211058,7 +211167,12 @@ TableAppendState::~TableAppendState() {
 }
 
 bool RowGroupCollection::IsEmpty() const {
-	return row_groups->GetRootSegment() == nullptr;
+	auto l = row_groups->Lock();
+	return IsEmpty(l);
+}
+
+bool RowGroupCollection::IsEmpty(SegmentLock &l) const {
+	return row_groups->IsEmpty(l);
 }
 
 void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppendState &state, idx_t append_count) {
@@ -211067,17 +211181,17 @@ void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppe
 	state.total_append_count = 0;
 
 	// start writing to the row_groups
-	lock_guard<mutex> row_group_lock(row_groups->node_lock);
-	if (IsEmpty()) {
+	auto l = row_groups->Lock();
+	if (IsEmpty(l)) {
 		// empty row group collection: empty first row group
-		AppendRowGroup(row_start);
+		AppendRowGroup(l, row_start);
 	}
-	state.start_row_group = (RowGroup *)row_groups->GetLastSegment();
+	state.start_row_group = (RowGroup *)row_groups->GetLastSegment(l);
 	D_ASSERT(this->row_start + total_rows == state.start_row_group->start + state.start_row_group->count);
 	state.start_row_group->InitializeAppend(state.row_group_append_state);
 	state.remaining = append_count;
+	state.transaction = transaction;
 	if (state.remaining > 0) {
-		state.transaction = transaction;
 		state.start_row_group->AppendVersionInfo(transaction, state.remaining);
 		total_rows += state.remaining;
 	}
@@ -211088,7 +211202,7 @@ void RowGroupCollection::InitializeAppend(TableAppendState &state) {
 	InitializeAppend(tdata, state, 0);
 }
 
-bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, TableStatistics &stats) {
+bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state) {
 	D_ASSERT(chunk.ColumnCount() == types.size());
 	chunk.Verify();
 
@@ -211128,10 +211242,11 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, Table
 			// append a new row_group
 			new_row_group = true;
 			auto next_start = current_row_group->start + state.row_group_append_state.offset_in_row_group;
-			AppendRowGroup(next_start);
+
+			auto l = row_groups->Lock();
+			AppendRowGroup(l, next_start);
 			// set up the append state for this row_group
-			lock_guard<mutex> row_group_lock(row_groups->node_lock);
-			auto last_row_group = (RowGroup *)row_groups->GetLastSegment();
+			auto last_row_group = (RowGroup *)row_groups->GetLastSegment(l);
 			last_row_group->InitializeAppend(state.row_group_append_state);
 			if (state.remaining > 0) {
 				last_row_group->AppendVersionInfo(state.transaction, state.remaining);
@@ -211154,19 +211269,20 @@ bool RowGroupCollection::Append(DataChunk &chunk, TableAppendState &state, Table
 }
 
 void RowGroupCollection::FinalizeAppend(TransactionData transaction, TableAppendState &state) {
-	D_ASSERT(state.transaction.transaction_id == 0);
 	auto remaining = state.total_append_count;
 	auto row_group = state.start_row_group;
 	while (remaining > 0) {
 		auto append_count = MinValue<idx_t>(remaining, RowGroup::ROW_GROUP_SIZE - row_group->count);
 		row_group->AppendVersionInfo(transaction, append_count);
 		remaining -= append_count;
-		row_group = (RowGroup *)row_group->next.get();
+		row_group = (RowGroup *)row_group->Next();
 	}
 	total_rows += state.total_append_count;
 
 	state.total_append_count = 0;
 	state.start_row_group = nullptr;
+
+	Verify();
 }
 
 void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count) {
@@ -211185,7 +211301,7 @@ void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start,
 		if (remaining == 0) {
 			break;
 		}
-		row_group = (RowGroup *)row_group->next.get();
+		row_group = (RowGroup *)row_group->Next();
 	}
 }
 
@@ -211195,16 +211311,15 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 	}
 	total_rows = start_row;
 
-	lock_guard<mutex> tree_lock(row_groups->node_lock);
+	auto l = row_groups->Lock();
 	// find the segment index that the current row belongs to
-	idx_t segment_index = row_groups->GetSegmentIndex(start_row);
-	auto segment = row_groups->nodes[segment_index].node;
+	idx_t segment_index = row_groups->GetSegmentIndex(l, start_row);
+	auto segment = row_groups->GetSegmentByIndex(l, segment_index);
 	auto &info = (RowGroup &)*segment;
 
 	// remove any segments AFTER this segment: they should be deleted entirely
-	if (segment_index < row_groups->nodes.size() - 1) {
-		row_groups->nodes.erase(row_groups->nodes.begin() + segment_index + 1, row_groups->nodes.end());
-	}
+	row_groups->EraseSegments(l, segment_index);
+
 	info.next = nullptr;
 	info.RevertAppend(start_row);
 }
@@ -211212,12 +211327,13 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 void RowGroupCollection::MergeStorage(RowGroupCollection &data) {
 	D_ASSERT(data.types == types);
 	auto index = row_start + total_rows.load();
-	for (auto segment = data.row_groups->GetRootSegment(); segment; segment = segment->next.get()) {
+	for (auto segment = data.row_groups->GetRootSegment(); segment; segment = segment->Next()) {
 		auto &row_group = (RowGroup &)*segment;
 		auto new_group = make_unique<RowGroup>(row_group, index);
 		index += new_group->count;
 		row_groups->AppendSegment(move(new_group));
 	}
+	stats.MergeStats(data.stats);
 	total_rows += data.total_rows.load();
 }
 
@@ -211255,7 +211371,7 @@ idx_t RowGroupCollection::Delete(TransactionData transaction, DataTable *table,
 // Update
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Update(TransactionData transaction, row_t *ids, const vector<column_t> &column_ids,
-                                DataChunk &updates, TableStatistics &stats) {
+                                DataChunk &updates) {
 	idx_t pos = 0;
 	do {
 		idx_t start = pos;
@@ -211327,7 +211443,7 @@ void RowGroupCollection::RemoveFromIndexes(TableIndexList &indexes, Vector &row_
 }
 
 void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_ids, const vector<column_t> &column_path,
-                                      DataChunk &updates, TableStatistics &stats) {
+                                      DataChunk &updates) {
 	auto first_id = FlatVector::GetValue<row_t>(row_ids, 0);
 	if (first_id >= MAX_ROW_ID) {
 		throw NotImplementedException("Cannot update a column-path on transaction local data");
@@ -211345,7 +211461,7 @@ void RowGroupCollection::UpdateColumn(TransactionData transaction, Vector &row_i
 //===--------------------------------------------------------------------===//
 void RowGroupCollection::Checkpoint(TableDataWriter &writer, vector<unique_ptr<BaseStatistics>> &global_stats) {
 	for (auto row_group = (RowGroup *)row_groups->GetRootSegment(); row_group;
-	     row_group = (RowGroup *)row_group->next.get()) {
+	     row_group = (RowGroup *)row_group->Next()) {
 		auto rowg_writer = writer.GetRowGroupWriter(*row_group);
 		auto pointer = row_group->Checkpoint(*rowg_writer, global_stats);
 		writer.AddRowGroup(move(pointer), move(rowg_writer));
@@ -211359,7 +211475,7 @@ void RowGroupCollection::CommitDropColumn(idx_t index) {
 	auto segment = (RowGroup *)row_groups->GetRootSegment();
 	while (segment) {
 		segment->CommitDropColumn(index);
-		segment = (RowGroup *)segment->next.get();
+		segment = (RowGroup *)segment->Next();
 	}
 }
 
@@ -211367,7 +211483,7 @@ void RowGroupCollection::CommitDropTable() {
 	auto segment = (RowGroup *)row_groups->GetRootSegment();
 	while (segment) {
 		segment->CommitDrop();
-		segment = (RowGroup *)segment->next.get();
+		segment = (RowGroup *)segment->Next();
 	}
 }
 
@@ -211383,7 +211499,7 @@ vector<vector<Value>> RowGroupCollection::GetStorageInfo() {
 		row_group->GetStorageInfo(row_group_index, result);
 		row_group_index++;
 
-		row_group = (RowGroup *)row_group->next.get();
+		row_group = (RowGroup *)row_group->Next();
 	}
 
 	return result;
@@ -211392,8 +211508,7 @@ vector<vector<Value>> RowGroupCollection::GetStorageInfo() {
 //===--------------------------------------------------------------------===//
 // Alter
 //===--------------------------------------------------------------------===//
-shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ColumnDefinition &new_column, Expression *default_value,
-                                                             ColumnStatistics &stats) {
+shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ColumnDefinition &new_column, Expression *default_value) {
 	idx_t new_column_idx = types.size();
 	auto new_types = types;
 	new_types.push_back(new_column.GetType());
@@ -211408,16 +211523,19 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AddColumn(ColumnDefinition &n
 		executor.AddExpression(*default_value);
 	}
 
+	result->stats.InitializeAddColumn(stats, new_column.GetType());
+	auto &new_column_stats = result->stats.GetStats(new_column_idx);
+
 	// fill the column with its DEFAULT value, or NULL if none is specified
 	auto new_stats = make_unique<SegmentStatistics>(new_column.GetType());
 	auto current_row_group = (RowGroup *)row_groups->GetRootSegment();
 	while (current_row_group) {
 		auto new_row_group = current_row_group->AddColumn(new_column, executor, default_value, default_vector);
 		// merge in the statistics
-		new_row_group->MergeIntoStatistics(new_column_idx, *stats.stats);
+		new_row_group->MergeIntoStatistics(new_column_idx, *new_column_stats.stats);
 
 		result->row_groups->AppendSegment(move(new_row_group));
-		current_row_group = (RowGroup *)current_row_group->next.get();
+		current_row_group = (RowGroup *)current_row_group->Next();
 	}
 	return result;
 }
@@ -211428,24 +211546,25 @@ shared_ptr<RowGroupCollection> RowGroupCollection::RemoveColumn(idx_t col_idx) {
 	new_types.erase(new_types.begin() + col_idx);
 
 	auto result = make_shared<RowGroupCollection>(info, block_manager, move(new_types), row_start, total_rows.load());
+	result->stats.InitializeRemoveColumn(stats, col_idx);
 
 	auto current_row_group = (RowGroup *)row_groups->GetRootSegment();
 	while (current_row_group) {
 		auto new_row_group = current_row_group->RemoveColumn(col_idx);
 		result->row_groups->AppendSegment(move(new_row_group));
-		current_row_group = (RowGroup *)current_row_group->next.get();
+		current_row_group = (RowGroup *)current_row_group->Next();
 	}
 	return result;
 }
 
 shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(idx_t changed_idx, const LogicalType &target_type,
-                                                             vector<column_t> bound_columns, Expression &cast_expr,
-                                                             ColumnStatistics &stats) {
+                                                             vector<column_t> bound_columns, Expression &cast_expr) {
 	D_ASSERT(changed_idx < types.size());
 	auto new_types = types;
 	new_types[changed_idx] = target_type;
 
 	auto result = make_shared<RowGroupCollection>(info, block_manager, move(new_types), row_start, total_rows.load());
+	result->stats.InitializeAlterType(stats, changed_idx, target_type);
 
 	vector<LogicalType> scan_types;
 	for (idx_t i = 0; i < bound_columns.size(); i++) {
@@ -211467,12 +211586,13 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(idx_t changed_idx,
 
 	// now alter the type of the column within all of the row_groups individually
 	auto current_row_group = (RowGroup *)row_groups->GetRootSegment();
+	auto &changed_stats = result->stats.GetStats(changed_idx);
 	while (current_row_group) {
 		auto new_row_group = current_row_group->AlterType(target_type, changed_idx, executor,
 		                                                  scan_state.table_state.row_group_state, scan_chunk);
-		new_row_group->MergeIntoStatistics(changed_idx, *stats.stats);
+		new_row_group->MergeIntoStatistics(changed_idx, *changed_stats.stats);
 		result->row_groups->AppendSegment(move(new_row_group));
-		current_row_group = (RowGroup *)current_row_group->next.get();
+		current_row_group = (RowGroup *)current_row_group->Next();
 	}
 
 	return result;
@@ -211494,9 +211614,9 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst
 	vector<column_t> cids;
 	cids.push_back(not_null_constraint.index);
 	// Use ScanCommitted to scan the latest committed data
-	InitializeCreateIndexScan(state);
 	state.Initialize(cids, nullptr);
 	InitializeScan(state.table_state, cids, nullptr);
+	InitializeCreateIndexScan(state);
 	while (true) {
 		scan_chunk.Reset();
 		state.table_state.ScanCommitted(scan_chunk, TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
@@ -211511,6 +211631,19 @@ void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConst
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Statistics
+//===--------------------------------------------------------------------===//
+unique_ptr<BaseStatistics> RowGroupCollection::CopyStats(column_t column_id) {
+	return stats.CopyStats(column_id);
+}
+
+void RowGroupCollection::SetStatistics(column_t column_id, const std::function<void(BaseStatistics &)> &set_fun) {
+	D_ASSERT(column_id != COLUMN_IDENTIFIER_ROW_ID);
+	auto stats_guard = stats.GetLock();
+	set_fun(*stats.GetStats(column_id).stats);
+}
+
 } // namespace duckdb
 
 
@@ -211549,7 +211682,7 @@ void ColumnScanState::NextInternal(idx_t count) {
 	}
 	row_index += count;
 	while (row_index >= current->start + current->count) {
-		current = (ColumnSegment *)current->next.get();
+		current = (ColumnSegment *)current->Next();
 		initialized = false;
 		segment_checked = false;
 		if (!current) {
@@ -211606,7 +211739,7 @@ bool CollectionScanState::Scan(Transaction &transaction, DataChunk &result) {
 			return true;
 		} else {
 			do {
-				current_row_group = row_group_state.row_group = (RowGroup *)current_row_group->next.get();
+				current_row_group = row_group_state.row_group = (RowGroup *)current_row_group->Next();
 				if (current_row_group) {
 					bool scan_row_group = current_row_group->InitializeScan(row_group_state);
 					if (scan_row_group) {
@@ -211627,7 +211760,7 @@ bool CollectionScanState::ScanCommitted(DataChunk &result, TableScanType type) {
 		if (result.size() > 0) {
 			return true;
 		} else {
-			current_row_group = row_group_state.row_group = (RowGroup *)current_row_group->next.get();
+			current_row_group = row_group_state.row_group = (RowGroup *)current_row_group->Next();
 			if (current_row_group) {
 				current_row_group->InitializeScan(row_group_state);
 			}
@@ -211640,37 +211773,74 @@ bool CollectionScanState::ScanCommitted(DataChunk &result, TableScanType type) {
 
 
 
+
 namespace duckdb {
 
+SegmentLock SegmentTree::Lock() {
+	return SegmentLock(node_lock);
+}
+
+bool SegmentTree::IsEmpty(SegmentLock &) {
+	return nodes.empty();
+}
+
+SegmentBase *SegmentTree::GetRootSegment(SegmentLock &l) {
+	return nodes.empty() ? nullptr : nodes[0].node.get();
+}
+
+vector<SegmentNode> SegmentTree::MoveSegments(SegmentLock &) {
+	return move(nodes);
+}
+
 SegmentBase *SegmentTree::GetRootSegment() {
-	return root_node.get();
+	auto l = Lock();
+	return GetRootSegment(l);
 }
 
-SegmentBase *SegmentTree::GetSegmentByIndex(int64_t index) {
+SegmentBase *SegmentTree::GetSegmentByIndex(SegmentLock &, int64_t index) {
 	if (index < 0) {
 		index = nodes.size() + index;
 		if (index < 0) {
 			return nullptr;
 		}
-		return nodes[index].node;
+		return nodes[index].node.get();
 	} else {
 		if (idx_t(index) >= nodes.size()) {
 			return nullptr;
 		}
-		return nodes[index].node;
+		return nodes[index].node.get();
+	}
+}
+SegmentBase *SegmentTree::GetSegmentByIndex(int64_t index) {
+	auto l = Lock();
+	return GetSegmentByIndex(l, index);
+}
+
+SegmentBase *SegmentTree::GetLastSegment(SegmentLock &l) {
+	if (nodes.empty()) {
+		return nullptr;
 	}
+	return nodes.back().node.get();
 }
 
 SegmentBase *SegmentTree::GetLastSegment() {
-	return nodes.empty() ? nullptr : nodes.back().node;
+	auto l = Lock();
+	return GetLastSegment(l);
+}
+
+SegmentBase *SegmentTree::GetSegment(SegmentLock &l, idx_t row_number) {
+	return nodes[GetSegmentIndex(l, row_number)].node.get();
 }
 
 SegmentBase *SegmentTree::GetSegment(idx_t row_number) {
-	lock_guard<mutex> tree_lock(node_lock);
-	return nodes[GetSegmentIndex(row_number)].node;
+	auto l = Lock();
+	return GetSegment(l, row_number);
 }
 
-idx_t SegmentTree::GetSegmentIndex(idx_t row_number) {
+bool SegmentTree::TryGetSegmentIndex(SegmentLock &, idx_t row_number, idx_t &result) {
+	if (nodes.empty()) {
+		return false;
+	}
 	D_ASSERT(!nodes.empty());
 	D_ASSERT(row_number >= nodes[0].row_start);
 	D_ASSERT(row_number < nodes.back().row_start + nodes.back().node->count);
@@ -211687,44 +211857,97 @@ idx_t SegmentTree::GetSegmentIndex(idx_t row_number) {
 		} else if (row_number >= entry.row_start + entry.node->count) {
 			lower = index + 1;
 		} else {
-			return index;
+			result = index;
+			return true;
 		}
 	}
-	throw InternalException("Could not find node in column segment tree!");
+	return false;
 }
 
-bool SegmentTree::HasSegment(SegmentBase *segment) {
-	lock_guard<mutex> tree_lock(node_lock);
+idx_t SegmentTree::GetSegmentIndex(SegmentLock &l, idx_t row_number) {
+	idx_t segment_index;
+	if (TryGetSegmentIndex(l, row_number, segment_index)) {
+		return segment_index;
+	}
+	string error;
+	error = StringUtil::Format("Attempting to find row number \"%lld\" in %lld nodes\n", row_number, nodes.size());
+	for (idx_t i = 0; i < nodes.size(); i++) {
+		error +=
+		    StringUtil::Format("Node %lld: Start %lld, Count %lld", i, nodes[i].row_start, nodes[i].node->count.load());
+	}
+	throw InternalException("Could not find node in column segment tree!\n%s%s", error, Exception::GetStackTrace());
+}
+
+idx_t SegmentTree::GetSegmentIndex(idx_t row_number) {
+	auto l = Lock();
+	return GetSegmentIndex(l, row_number);
+}
+
+bool SegmentTree::HasSegment(SegmentLock &, SegmentBase *segment) {
 	for (auto &node : nodes) {
-		if (node.node == segment) {
+		if (node.node.get() == segment) {
 			return true;
 		}
 	}
 	return false;
 }
 
-void SegmentTree::AppendSegment(unique_ptr<SegmentBase> segment) {
+bool SegmentTree::HasSegment(SegmentBase *segment) {
+	auto l = Lock();
+	return HasSegment(l, segment);
+}
+
+void SegmentTree::AppendSegment(SegmentLock &, unique_ptr<SegmentBase> segment) {
 	D_ASSERT(segment);
 	// add the node to the list of nodes
+	if (!nodes.empty()) {
+		nodes.back().node->next = segment.get();
+	}
 	SegmentNode node;
 	node.row_start = segment->start;
-	node.node = segment.get();
-	nodes.push_back(node);
+	node.node = move(segment);
+	nodes.push_back(move(node));
+}
 
-	if (nodes.size() > 1) {
-		// add the node as the next pointer of the last node
-		D_ASSERT(!nodes[nodes.size() - 2].node->next);
-		nodes[nodes.size() - 2].node->next = move(segment);
-	} else {
-		root_node = move(segment);
+void SegmentTree::AppendSegment(unique_ptr<SegmentBase> segment) {
+	auto l = Lock();
+	AppendSegment(l, move(segment));
+}
+
+void SegmentTree::EraseSegments(SegmentLock &, idx_t segment_start) {
+	if (segment_start >= nodes.size() - 1) {
+		return;
 	}
+	nodes.erase(nodes.begin() + segment_start + 1, nodes.end());
 }
 
-void SegmentTree::Replace(SegmentTree &other) {
-	root_node = move(other.root_node);
+void SegmentTree::Replace(SegmentLock &, SegmentTree &other) {
 	nodes = move(other.nodes);
 }
 
+void SegmentTree::Replace(SegmentTree &other) {
+	auto l = Lock();
+	Replace(l, other);
+}
+
+void SegmentTree::Verify(SegmentLock &) {
+#ifdef DEBUG
+	idx_t base_start = nodes.empty() ? 0 : nodes[0].node->start;
+	for (idx_t i = 0; i < nodes.size(); i++) {
+		D_ASSERT(nodes[i].row_start == nodes[i].node->start);
+		D_ASSERT(nodes[i].node->start == base_start);
+		base_start += nodes[i].node->count;
+	}
+#endif
+}
+
+void SegmentTree::Verify() {
+#ifdef DEBUG
+	auto l = Lock();
+	Verify(l);
+#endif
+}
+
 } // namespace duckdb