duckdb 0.5.2-dev325.0 → 0.5.2-dev339.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev325.0",
+  "version": "0.5.2-dev339.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -128298,7 +128298,6 @@ PreservedError ClientContext::VerifyQuery(ClientContextLock &lock, const string
 	bool any_failed = original->Run(*this, query, [&](const string &q, unique_ptr<SQLStatement> s) {
 		return RunStatementInternal(lock, q, move(s), false, false);
 	});
-
 	// Execute the verifiers
 	for (auto &verifier : statement_verifiers) {
 		bool failed = verifier->Run(*this, query, [&](const string &q, unique_ptr<SQLStatement> s) {
@@ -128318,11 +128317,7 @@ PreservedError ClientContext::VerifyQuery(ClientContextLock &lock, const string
 		}
 	} else {
 		if (db->IsInvalidated()) {
-			for (auto &verifier : statement_verifiers) {
-				if (verifier->materialized_result->HasError()) {
-					return verifier->materialized_result->GetErrorObject();
-				}
-			}
+			return original->materialized_result->GetErrorObject();
 		}
 	}
 
@@ -195078,6 +195073,8 @@ class ColumnData {
 
 public:
 	ColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type, ColumnData *parent);
+	ColumnData(ColumnData &other, idx_t start, ColumnData *parent);
+
 	virtual ~ColumnData();
 
 	//! Table info for the column
@@ -195159,8 +195156,10 @@ public:
 
 	static shared_ptr<ColumnData> CreateColumn(DataTableInfo &info, idx_t column_index, idx_t start_row,
 	                                           const LogicalType &type, ColumnData *parent = nullptr);
+	static shared_ptr<ColumnData> CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
 	static unique_ptr<ColumnData> CreateColumnUnique(DataTableInfo &info, idx_t column_index, idx_t start_row,
 	                                                 const LogicalType &type, ColumnData *parent = nullptr);
+	static unique_ptr<ColumnData> CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent = nullptr);
 
 protected:
 	//! Append a transient segment
@@ -198462,6 +198461,7 @@ namespace duckdb {
 class ValidityColumnData : public ColumnData {
 public:
 	ValidityColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, ColumnData *parent);
+	ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 public:
 	bool CheckZonemap(ColumnScanState &state, TableFilter &filter) override;
@@ -198477,6 +198477,7 @@ class StandardColumnData : public ColumnData {
 public:
 	StandardColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
 	                   ColumnData *parent = nullptr);
+	StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 	//! The validity column data
 	ValidityColumnData validity;
@@ -198543,7 +198544,6 @@ DataTable::DataTable(DatabaseInstance &db, const string &schema, const string &t
 	if (stats.Empty()) {
 		D_ASSERT(row_groups->GetTotalRows() == 0);
 
-		row_groups->InitializeEmpty();
 		stats.InitializeEmpty(types);
 	}
 	row_groups->Verify();
@@ -198999,12 +198999,19 @@ void DataTable::Append(TableCatalogEntry &table, ClientContext &context, DataChu
 	transaction.storage.Append(this, chunk);
 }
 
-void DataTable::InitializeAppend(Transaction &transaction, TableAppendState &state, idx_t append_count) {
-	// obtain the append lock for this table
+void DataTable::AppendLock(TableAppendState &state) {
 	state.append_lock = unique_lock<mutex>(append_lock);
 	if (!is_root) {
 		throw TransactionException("Transaction conflict: adding entries to a table that has been altered!");
 	}
+	state.row_start = row_groups->GetTotalRows();
+}
+
+void DataTable::InitializeAppend(Transaction &transaction, TableAppendState &state, idx_t append_count) {
+	// obtain the append lock for this table
+	if (!state.append_lock) {
+		throw InternalException("DataTable::AppendLock should be called before DataTable::InitializeAppend");
+	}
 	row_groups->InitializeAppend(transaction, state, append_count);
 }
 
@@ -199056,6 +199063,15 @@ void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::functi
 	}
 }
 
+void DataTable::MergeStorage(RowGroupCollection &data, TableIndexList &indexes, TableStatistics &other_stats) {
+	row_groups->MergeStorage(data);
+	stats.MergeStats(other_stats);
+	row_groups->Verify();
+	if (!indexes.Empty()) {
+		throw InternalException("FIXME: merge indexes");
+	}
+}
+
 void DataTable::WriteToLog(WriteAheadLog &log, idx_t row_start, idx_t count) {
 	if (log.skip_writing) {
 		return;
@@ -199567,7 +199583,6 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
     : table(table), allocator(Allocator::Get(table.db)), deleted_rows(0) {
 	auto types = table.GetTypes();
 	row_groups = make_shared<RowGroupCollection>(table.info, types, MAX_ROW_ID, 0);
-	row_groups->InitializeEmpty();
 	stats.InitializeEmpty(types);
 	table.info->indexes.Scan([&](Index &index) {
 		D_ASSERT(index.type == IndexType::ART);
@@ -199758,44 +199773,51 @@ bool LocalStorage::ScanTableStorage(DataTable &table, LocalTableStorage &storage
 }
 
 void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
+	auto storage_entry = move(table_storage[&table]);
+	table_storage[&table].reset();
+
 	if (storage.row_groups->GetTotalRows() <= storage.deleted_rows) {
 		return;
 	}
 	idx_t append_count = storage.row_groups->GetTotalRows() - storage.deleted_rows;
-	TableAppendState append_state;
-	table.InitializeAppend(transaction, append_state, append_count);
 
-	bool constraint_violated = false;
-	ScanTableStorage(table, storage, [&](DataChunk &chunk) -> bool {
-		// append this chunk to the indexes of the table
-		if (!table.AppendToIndexes(chunk, append_state.current_row)) {
-			constraint_violated = true;
-			return false;
-		}
-		// append to base table
-		table.Append(transaction, chunk, append_state);
-		return true;
-	});
-	if (constraint_violated) {
-		// need to revert the append
-		row_t current_row = append_state.row_start;
-		// remove the data from the indexes, if there are any indexes
+	TableAppendState append_state;
+	table.AppendLock(append_state);
+	if (append_state.row_start == 0 && storage.table.info->indexes.Empty() && storage.deleted_rows == 0) {
+		// table is currently empty: move over the storage directly
+		table.MergeStorage(*storage.row_groups, storage.indexes, storage.stats);
+	} else {
+		bool constraint_violated = false;
+		table.InitializeAppend(transaction, append_state, append_count);
 		ScanTableStorage(table, storage, [&](DataChunk &chunk) -> bool {
 			// append this chunk to the indexes of the table
-			table.RemoveFromIndexes(append_state, chunk, current_row);
-
-			current_row += chunk.size();
-			if (current_row >= append_state.current_row) {
-				// finished deleting all rows from the index: abort now
+			if (!table.AppendToIndexes(chunk, append_state.current_row)) {
+				constraint_violated = true;
 				return false;
 			}
+			// append to base table
+			table.Append(transaction, chunk, append_state);
 			return true;
 		});
-		table.RevertAppendInternal(append_state.row_start, append_count);
-		table_storage[&table].reset();
-		throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
+		if (constraint_violated) {
+			// need to revert the append
+			row_t current_row = append_state.row_start;
+			// remove the data from the indexes, if there are any indexes
+			ScanTableStorage(table, storage, [&](DataChunk &chunk) -> bool {
+				// append this chunk to the indexes of the table
+				table.RemoveFromIndexes(append_state, chunk, current_row);
+
+				current_row += chunk.size();
+				if (current_row >= append_state.current_row) {
+					// finished deleting all rows from the index: abort now
+					return false;
+				}
+				return true;
+			});
+			table.RevertAppendInternal(append_state.row_start, append_count);
+			throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
+		}
 	}
-	table_storage[&table].reset();
 	transaction.PushAppend(&table, append_state.row_start, append_count);
 }
 
@@ -202071,6 +202093,7 @@ class ListColumnData : public ColumnData {
 public:
 	ListColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
 	               ColumnData *parent = nullptr);
+	ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 	//! The child-column of the list
 	unique_ptr<ColumnData> child_column;
@@ -202279,6 +202302,7 @@ class StructColumnData : public ColumnData {
 public:
 	StructColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row, LogicalType type,
 	                 ColumnData *parent = nullptr);
+	StructColumnData(ColumnData &original, idx_t start_row, ColumnData *parent = nullptr);
 
 	//! The sub-columns of the struct
 	vector<unique_ptr<ColumnData>> sub_columns;
@@ -202441,6 +202465,17 @@ ColumnData::ColumnData(DataTableInfo &info, idx_t column_index, idx_t start_row,
     : info(info), column_index(column_index), start(start_row), type(move(type)), parent(parent) {
 }
 
+ColumnData::ColumnData(ColumnData &other, idx_t start, ColumnData *parent)
+    : info(other.info), column_index(other.column_index), start(start), type(move(other.type)), parent(parent),
+      updates(move(other.updates)) {
+	idx_t offset = 0;
+	for (auto segment = other.data.GetRootSegment(); segment; segment = segment->next.get()) {
+		auto &other = (ColumnSegment &)*segment;
+		this->data.AppendSegment(ColumnSegment::CreateSegment(other, start + offset));
+		offset += segment->count;
+	}
+}
+
 ColumnData::~ColumnData() {
 }
 
@@ -202460,8 +202495,15 @@ const LogicalType &ColumnData::RootType() const {
 }
 
 idx_t ColumnData::GetMaxEntry() {
+	auto first_segment = data.GetRootSegment();
 	auto last_segment = data.GetLastSegment();
-	return last_segment ? last_segment->start + last_segment->count : start;
+	if (!first_segment) {
+		D_ASSERT(!last_segment);
+		return 0;
+	} else {
+		D_ASSERT(last_segment->start >= first_segment->start);
+		return last_segment->start + last_segment->count - first_segment->start;
+	}
 }
 
 void ColumnData::InitializeScan(ColumnScanState &state) {
@@ -202875,10 +202917,6 @@ void ColumnData::Verify(RowGroup &parent) {
 			}
 			root = root->next.get();
 		}
-	} else {
-		if (type.InternalType() != PhysicalType::STRUCT) {
-			D_ASSERT(parent.count == 0);
-		}
 	}
 #endif
 }
@@ -202896,16 +202934,36 @@ static RET CreateColumnInternal(DataTableInfo &info, idx_t column_index, idx_t s
 	return OP::template Create<StandardColumnData>(info, column_index, start_row, type, parent);
 }
 
+template <class RET, class OP>
+static RET CreateColumnInternal(ColumnData &other, idx_t start_row, ColumnData *parent) {
+	if (other.type.InternalType() == PhysicalType::STRUCT) {
+		return OP::template Create<StructColumnData>(other, start_row, parent);
+	} else if (other.type.InternalType() == PhysicalType::LIST) {
+		return OP::template Create<ListColumnData>(other, start_row, parent);
+	} else if (other.type.id() == LogicalTypeId::VALIDITY) {
+		return OP::template Create<ValidityColumnData>(other, start_row, parent);
+	}
+	return OP::template Create<StandardColumnData>(other, start_row, parent);
+}
+
 shared_ptr<ColumnData> ColumnData::CreateColumn(DataTableInfo &info, idx_t column_index, idx_t start_row,
                                                 const LogicalType &type, ColumnData *parent) {
 	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(info, column_index, start_row, type, parent);
 }
 
+shared_ptr<ColumnData> ColumnData::CreateColumn(ColumnData &other, idx_t start_row, ColumnData *parent) {
+	return CreateColumnInternal<shared_ptr<ColumnData>, SharedConstructor>(other, start_row, parent);
+}
+
 unique_ptr<ColumnData> ColumnData::CreateColumnUnique(DataTableInfo &info, idx_t column_index, idx_t start_row,
                                                       const LogicalType &type, ColumnData *parent) {
 	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(info, column_index, start_row, type, parent);
 }
 
+unique_ptr<ColumnData> ColumnData::CreateColumnUnique(ColumnData &other, idx_t start_row, ColumnData *parent) {
+	return CreateColumnInternal<unique_ptr<ColumnData>, UniqueConstructor>(other, start_row, parent);
+}
+
 } // namespace duckdb
 
 
@@ -203196,6 +203254,10 @@ unique_ptr<ColumnSegment> ColumnSegment::CreateTransientSegment(DatabaseInstance
 	                                  INVALID_BLOCK, 0);
 }
 
+unique_ptr<ColumnSegment> ColumnSegment::CreateSegment(ColumnSegment &other, idx_t start) {
+	return make_unique<ColumnSegment>(other, start);
+}
+
 ColumnSegment::ColumnSegment(DatabaseInstance &db, LogicalType type_p, ColumnSegmentType segment_type, idx_t start,
                              idx_t count, CompressionFunction *function_p, unique_ptr<BaseStatistics> statistics,
                              block_id_t block_id_p, idx_t offset_p)
@@ -203222,6 +203284,12 @@ ColumnSegment::ColumnSegment(DatabaseInstance &db, LogicalType type_p, ColumnSeg
 	}
 }
 
+ColumnSegment::ColumnSegment(ColumnSegment &other, idx_t start)
+    : SegmentBase(start, other.count), db(other.db), type(move(other.type)), type_size(other.type_size),
+      segment_type(other.segment_type), function(other.function), stats(move(other.stats)), block(move(other.block)),
+      block_id(other.block_id), offset(other.offset), segment_state(move(other.segment_state)) {
+}
+
 ColumnSegment::~ColumnSegment() {
 }
 
@@ -203625,6 +203693,12 @@ ListColumnData::ListColumnData(DataTableInfo &info, idx_t column_index, idx_t st
 	child_column = ColumnData::CreateColumnUnique(info, 1, start_row, child_type, this);
 }
 
+ListColumnData::ListColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
+    : ColumnData(original, start_row, parent), validity(((ListColumnData &)original).validity, start_row, this) {
+	auto &list_data = (ListColumnData &)original;
+	child_column = ColumnData::CreateColumnUnique(*list_data.child_column, start_row, this);
+}
+
 bool ListColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
 	// table filters are not supported yet for list columns
 	return false;
@@ -203674,7 +203748,7 @@ void ListColumnData::InitializeScanWithOffset(ColumnScanState &state, idx_t row_
 	D_ASSERT(child_offset <= child_column->GetMaxEntry());
 	ColumnScanState child_state;
 	if (child_offset < child_column->GetMaxEntry()) {
-		child_column->InitializeScanWithOffset(child_state, child_offset);
+		child_column->InitializeScanWithOffset(child_state, start + child_offset);
 	}
 	state.child_states.push_back(move(child_state));
 }
@@ -203719,7 +203793,8 @@ idx_t ListColumnData::ScanCount(ColumnScanState &state, Vector &result, idx_t co
 	if (child_scan_count > 0) {
 		auto &child_entry = ListVector::GetEntry(result);
 		D_ASSERT(child_entry.GetType().InternalType() == PhysicalType::STRUCT ||
-		         state.child_states[1].row_index + child_scan_count <= child_column->GetMaxEntry());
+		         state.child_states[1].row_index + child_scan_count <=
+		             child_column->start + child_column->GetMaxEntry());
 		child_column->ScanCount(state.child_states[1], child_entry, child_scan_count);
 	}
 
@@ -203908,9 +203983,9 @@ void ListColumnData::FetchRow(TransactionData transaction, ColumnFetchState &sta
 		auto &child_type = ListType::GetChildType(result.GetType());
 		Vector child_scan(child_type, child_scan_count);
 		// seek the scan towards the specified position and read [length] entries
-		child_column->InitializeScanWithOffset(*child_state, original_offset);
+		child_column->InitializeScanWithOffset(*child_state, start + original_offset);
 		D_ASSERT(child_type.InternalType() == PhysicalType::STRUCT ||
-		         child_state->row_index + child_scan_count <= child_column->GetMaxEntry());
+		         child_state->row_index + child_scan_count - this->start <= child_column->GetMaxEntry());
 		child_column->ScanCount(*child_state, child_scan, child_scan_count);
 
 		ListVector::Append(result, child_scan, child_scan_count);
@@ -204040,6 +204115,28 @@ RowGroup::RowGroup(DatabaseInstance &db, DataTableInfo &table_info, const vector
 	Verify();
 }
 
+RowGroup::RowGroup(RowGroup &row_group, idx_t start)
+    : SegmentBase(start, row_group.count), db(row_group.db), table_info(row_group.table_info),
+      version_info(move(row_group.version_info)), stats(move(row_group.stats)) {
+	for (auto &column : row_group.columns) {
+		this->columns.push_back(ColumnData::CreateColumn(*column, start));
+	}
+	if (version_info) {
+		version_info->SetStart(start);
+	}
+	Verify();
+}
+
+void VersionNode::SetStart(idx_t start) {
+	idx_t current_start = start;
+	for (idx_t i = 0; i < RowGroup::ROW_GROUP_VECTOR_COUNT; i++) {
+		if (info[i]) {
+			info[i]->start = current_start;
+		}
+		current_start += STANDARD_VECTOR_SIZE;
+	}
+}
+
 RowGroup::~RowGroup() {
 }
 
@@ -204933,10 +205030,6 @@ void RowGroupCollection::Initialize(PersistentTableData &data) {
 	}
 }
 
-void RowGroupCollection::InitializeEmpty() {
-	AppendRowGroup(row_start);
-}
-
 void RowGroupCollection::AppendRowGroup(idx_t start_row) {
 	D_ASSERT(start_row >= row_start);
 	auto new_row_group = make_unique<RowGroup>(info->db, *info, start_row, 0);
@@ -204946,7 +205039,10 @@ void RowGroupCollection::AppendRowGroup(idx_t start_row) {
 
 void RowGroupCollection::Verify() {
 #ifdef DEBUG
-	D_ASSERT(row_groups->GetRootSegment() != nullptr);
+	for (auto segment = row_groups->GetRootSegment(); segment; segment = segment->next.get()) {
+		auto &row_group = (RowGroup &)*segment;
+		row_group.Verify();
+	}
 #endif
 }
 
@@ -204956,6 +205052,7 @@ void RowGroupCollection::Verify() {
 void RowGroupCollection::InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids,
                                         TableFilterSet *table_filters) {
 	auto row_group = (RowGroup *)row_groups->GetRootSegment();
+	D_ASSERT(row_group);
 	state.max_row = row_start + total_rows;
 	while (row_group && !row_group->InitializeScan(state.row_group_state)) {
 		row_group = (RowGroup *)row_group->next.get();
@@ -204969,6 +205066,7 @@ void RowGroupCollection::InitializeCreateIndexScan(CreateIndexScanState &state)
 void RowGroupCollection::InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids,
                                                   idx_t start_row, idx_t end_row) {
 	auto row_group = (RowGroup *)row_groups->GetSegment(start_row);
+	D_ASSERT(row_group);
 	state.max_row = end_row;
 	idx_t start_vector = (start_row - row_group->start) / STANDARD_VECTOR_SIZE;
 	if (!row_group->InitializeScanWithOffset(state.row_group_state, start_vector)) {
@@ -205046,6 +205144,10 @@ void RowGroupCollection::Fetch(TransactionData transaction, DataChunk &result, c
 //===--------------------------------------------------------------------===//
 // Append
 //===--------------------------------------------------------------------===//
+bool RowGroupCollection::IsEmpty() const {
+	return row_groups->GetRootSegment() == nullptr;
+}
+
 void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppendState &state, idx_t append_count) {
 	state.remaining_append_count = append_count;
 	state.row_start = total_rows;
@@ -205053,6 +205155,10 @@ void RowGroupCollection::InitializeAppend(TransactionData transaction, TableAppe
 
 	// start writing to the row_groups
 	lock_guard<mutex> row_group_lock(row_groups->node_lock);
+	if (IsEmpty()) {
+		// empty row group collection: empty first row group
+		AppendRowGroup(row_start);
+	}
 	auto last_row_group = (RowGroup *)row_groups->GetLastSegment();
 	D_ASSERT(this->row_start + total_rows == last_row_group->start + last_row_group->count);
 	last_row_group->InitializeAppend(transaction, state.row_group_append_state, state.remaining_append_count);
@@ -205117,6 +205223,7 @@ void RowGroupCollection::Append(TransactionData transaction, DataChunk &chunk, T
 
 void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count) {
 	auto row_group = (RowGroup *)row_groups->GetSegment(row_start);
+	D_ASSERT(row_group);
 	idx_t current_row = row_start;
 	idx_t remaining = count;
 	while (true) {
@@ -205159,6 +205266,18 @@ void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
 	info.RevertAppend(start_row);
 }
 
+void RowGroupCollection::MergeStorage(RowGroupCollection &data) {
+	D_ASSERT(data.types == types);
+	auto index = row_start;
+	for (auto segment = data.row_groups->GetRootSegment(); segment; segment = segment->next.get()) {
+		auto &row_group = (RowGroup &)*segment;
+		auto new_group = make_unique<RowGroup>(row_group, index);
+		index += new_group->count;
+		row_groups->AppendSegment(move(new_group));
+	}
+	total_rows += data.total_rows.load();
+}
+
 //===--------------------------------------------------------------------===//
 // Delete
 //===--------------------------------------------------------------------===//
@@ -205417,6 +205536,9 @@ shared_ptr<RowGroupCollection> RowGroupCollection::AlterType(idx_t changed_idx,
 }
 
 void RowGroupCollection::VerifyNewConstraint(DataTable &parent, const BoundConstraint &constraint) {
+	if (total_rows == 0) {
+		return;
+	}
 	// scan the original table, check if there's any null value
 	auto &not_null_constraint = (BoundNotNullConstraint &)constraint;
 	vector<LogicalType> scan_types;
@@ -205651,6 +205773,10 @@ StandardColumnData::StandardColumnData(DataTableInfo &info, idx_t column_index,
     : ColumnData(info, column_index, start_row, move(type), parent), validity(info, 0, start_row, this) {
 }
 
+StandardColumnData::StandardColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
+    : ColumnData(original, start_row, parent), validity(((StandardColumnData &)original).validity, start_row, this) {
+}
+
 bool StandardColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
 	if (!state.segment_checked) {
 		if (!state.current) {
@@ -205873,6 +205999,14 @@ StructColumnData::StructColumnData(DataTableInfo &info, idx_t column_index, idx_
 	}
 }
 
+StructColumnData::StructColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
+    : ColumnData(original, start_row, parent), validity(((StructColumnData &)original).validity, start_row, this) {
+	auto &struct_data = (StructColumnData &)original;
+	for (auto &child_col : struct_data.sub_columns) {
+		sub_columns.push_back(ColumnData::CreateColumnUnique(*child_col, start_row, this));
+	}
+}
+
 bool StructColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
 	// table filters are not supported yet for struct columns
 	return false;
@@ -206220,6 +206354,14 @@ void TableStatistics::InitializeAddConstraint(TableStatistics &parent) {
 	}
 }
 
+void TableStatistics::MergeStats(TableStatistics &other) {
+	auto l = GetLock();
+	D_ASSERT(column_stats.size() == other.column_stats.size());
+	for (idx_t i = 0; i < column_stats.size(); i++) {
+		column_stats[i]->stats->Merge(*other.column_stats[i]->stats);
+	}
+}
+
 void TableStatistics::MergeStats(idx_t i, BaseStatistics &stats) {
 	auto l = GetLock();
 	MergeStats(*l, i, stats);
@@ -207543,6 +207685,10 @@ ValidityColumnData::ValidityColumnData(DataTableInfo &info, idx_t column_index,
     : ColumnData(info, column_index, start_row, LogicalType(LogicalTypeId::VALIDITY), parent) {
 }
 
+ValidityColumnData::ValidityColumnData(ColumnData &original, idx_t start_row, ColumnData *parent)
+    : ColumnData(original, start_row, parent) {
+}
+
 bool ValidityColumnData::CheckZonemap(ColumnScanState &state, TableFilter &filter) {
 	return true;
 }

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "689b242d2"
-#define DUCKDB_VERSION "v0.5.2-dev325"
+#define DUCKDB_SOURCE_ID "23ca324e8"
+#define DUCKDB_VERSION "v0.5.2-dev339"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -17759,6 +17759,7 @@ public:
 	RowGroup(DatabaseInstance &db, DataTableInfo &table_info, idx_t start, idx_t count);
 	RowGroup(DatabaseInstance &db, DataTableInfo &table_info, const vector<LogicalType> &types,
 	         RowGroupPointer &pointer);
+	RowGroup(RowGroup &row_group, idx_t start);
 	~RowGroup();
 
 private:
@@ -17872,6 +17873,8 @@ private:
 
 struct VersionNode {
 	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
+
+	void SetStart(idx_t start);
 };
 
 } // namespace duckdb
@@ -17973,7 +17976,8 @@ public:
 	Allocator &GetAllocator() const;
 
 	void Initialize(PersistentTableData &data);
-	void InitializeEmpty();
+
+	bool IsEmpty() const;
 
 	void AppendRowGroup(idx_t start_row);
 	void Verify();
@@ -17995,6 +17999,8 @@ public:
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
 	void RevertAppendInternal(idx_t start_row, idx_t count);
 
+	void MergeStorage(RowGroupCollection &data);
+
 	void RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count);
 
 	idx_t Delete(TransactionData transaction, DataTable *table, row_t *ids, idx_t count);
@@ -20288,6 +20294,7 @@ public:
 	void InitializeAlterType(TableStatistics &parent, idx_t changed_idx, const LogicalType &new_type);
 	void InitializeAddConstraint(TableStatistics &parent);
 
+	void MergeStats(TableStatistics &other);
 	void MergeStats(idx_t i, BaseStatistics &stats);
 	void MergeStats(TableStatisticsLock &lock, idx_t i, BaseStatistics &stats);
 
@@ -25872,6 +25879,7 @@ public:
 	                                                         CompressionType compression_type,
 	                                                         unique_ptr<BaseStatistics> statistics);
 	static unique_ptr<ColumnSegment> CreateTransientSegment(DatabaseInstance &db, const LogicalType &type, idx_t start);
+	static unique_ptr<ColumnSegment> CreateSegment(ColumnSegment &other, idx_t start);
 
 public:
 	void InitializeScan(ColumnScanState &state);
@@ -25927,6 +25935,7 @@ public:
 	ColumnSegment(DatabaseInstance &db, LogicalType type, ColumnSegmentType segment_type, idx_t start, idx_t count,
 	              CompressionFunction *function, unique_ptr<BaseStatistics> statistics, block_id_t block_id,
 	              idx_t offset);
+	ColumnSegment(ColumnSegment &other, idx_t start);
 
 private:
 	void Scan(ColumnScanState &state, idx_t scan_count, Vector &result);
@@ -26048,6 +26057,8 @@ public:
 	void UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
 	                  const vector<column_t> &column_path, DataChunk &updates);
 
+	//! Fetches an append lock
+	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
 	void InitializeAppend(Transaction &transaction, TableAppendState &state, idx_t append_count);
 	//! Append a chunk to the table using the AppendState obtained from BeginAppend
@@ -26063,6 +26074,9 @@ public:
 
 	void ScanTableSegment(idx_t start_row, idx_t count, const std::function<void(DataChunk &chunk)> &function);
 
+	//! Merge a row group collection directly into this table - appending it to the end of the table without copying
+	void MergeStorage(RowGroupCollection &data, TableIndexList &indexes, TableStatistics &stats);
+
 	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table, returns
 	//! whether or not the append succeeded
 	bool AppendToIndexes(DataChunk &chunk, row_t row_start);