duckdb 0.5.2-dev809.0 → 0.5.2-dev833.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev809.0",
+  "version": "0.5.2-dev833.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -29106,7 +29106,7 @@ struct RowOperations {
 namespace duckdb {
 
 template <class OP, class RETURN_TYPE, typename... ARGS>
-RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
+RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&...args) {
 	D_ASSERT(radix_bits <= sizeof(hash_t) * 8);
 	switch (radix_bits) {
 	case 1:
@@ -29135,7 +29135,7 @@ RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
 }
 
 template <class OP, class RETURN_TYPE, idx_t radix_bits_1, typename... ARGS>
-RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&... args) {
+RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&...args) {
 	D_ASSERT(radix_bits_2 <= sizeof(hash_t) * 8);
 	switch (radix_bits_2) {
 	case 1:
@@ -29164,7 +29164,7 @@ RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&... args) {
 }
 
 template <class OP, class RETURN_TYPE, typename... ARGS>
-RETURN_TYPE DoubleRadixBitsSwitch1(idx_t radix_bits_1, idx_t radix_bits_2, ARGS &&... args) {
+RETURN_TYPE DoubleRadixBitsSwitch1(idx_t radix_bits_1, idx_t radix_bits_2, ARGS &&...args) {
 	D_ASSERT(radix_bits_1 <= sizeof(hash_t) * 8);
 	switch (radix_bits_1) {
 	case 1:
@@ -49455,6 +49455,9 @@ void Vector::Initialize(bool zero_data, idx_t capacity) {
 			memset(data, 0, capacity * type_size);
 		}
 	}
+	if (capacity > STANDARD_VECTOR_SIZE) {
+		validity.Resize(STANDARD_VECTOR_SIZE, capacity);
+	}
 }
 
 struct DataArrays {
@@ -93661,6 +93664,13 @@ unique_ptr<FunctionData> HistogramBindFunction(ClientContext &context, Aggregate
                                                vector<unique_ptr<Expression>> &arguments) {
 
 	D_ASSERT(arguments.size() == 1);
+
+	if (arguments[0]->return_type.id() == LogicalTypeId::LIST ||
+	    arguments[0]->return_type.id() == LogicalTypeId::STRUCT ||
+	    arguments[0]->return_type.id() == LogicalTypeId::MAP) {
+		throw NotImplementedException("Unimplemented type for histogram %s", arguments[0]->return_type.ToString());
+	}
+
 	child_list_t<LogicalType> struct_children;
 	struct_children.push_back({"key", LogicalType::LIST(arguments[0]->return_type)});
 	struct_children.push_back({"value", LogicalType::LIST(LogicalType::UBIGINT)});
@@ -95714,6 +95724,11 @@ struct VectorCastHelpers {
 	}
 };
 
+struct VectorStringifiedListParser {
+	static idx_t CountParts(const string_t &input);
+	static bool SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start, Vector &child);
+};
+
 } // namespace duckdb
 
 
@@ -96484,20 +96499,8 @@ BoundCastInfo DefaultCasts::EnumCastSwitch(BindCastInput &input, const LogicalTy
 
 namespace duckdb {
 
-struct ListBoundCastData : public BoundCastData {
-	explicit ListBoundCastData(BoundCastInfo child_cast) : child_cast_info(move(child_cast)) {
-	}
-
-	BoundCastInfo child_cast_info;
-
-public:
-	unique_ptr<BoundCastData> Copy() const override {
-		return make_unique<ListBoundCastData>(child_cast_info.Copy());
-	}
-};
-
-unique_ptr<BoundCastData> BindListToListCast(BindCastInput &input, const LogicalType &source,
-                                             const LogicalType &target) {
+unique_ptr<BoundCastData> ListBoundCastData::BindListToListCast(BindCastInput &input, const LogicalType &source,
+                                                                const LogicalType &target) {
 	vector<BoundCastInfo> child_cast_info;
 	auto &source_child_type = ListType::GetChildType(source);
 	auto &result_child_type = ListType::GetChildType(target);
@@ -96608,11 +96611,11 @@ static bool ListToVarcharCast(Vector &source, Vector &result, idx_t count, CastP
 BoundCastInfo DefaultCasts::ListCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
 	switch (target.id()) {
 	case LogicalTypeId::LIST:
-		return BoundCastInfo(ListToListCast, BindListToListCast(input, source, target));
+		return BoundCastInfo(ListToListCast, ListBoundCastData::BindListToListCast(input, source, target));
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::JSON:
-		return BoundCastInfo(ListToVarcharCast,
-		                     BindListToListCast(input, source, LogicalType::LIST(LogicalType::VARCHAR)));
+		return BoundCastInfo(ListToVarcharCast, ListBoundCastData::BindListToListCast(
+		                                            input, source, LogicalType::LIST(LogicalType::VARCHAR)));
 	default:
 		return DefaultCasts::TryVectorNullCast;
 	}
@@ -96950,9 +96953,97 @@ static BoundCastInfo VectorStringCastNumericSwitch(BindCastInput &input, const L
 	}
 }
 
+bool StringListCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result, ValidityMask &result_mask,
+                        idx_t count, CastParameters &parameters, const SelectionVector *sel) {
+
+	idx_t total_list_size = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			continue;
+		}
+		total_list_size += VectorStringifiedListParser::CountParts(source_data[idx]);
+	}
+
+	Vector varchar_vector(LogicalType::VARCHAR, total_list_size);
+
+	ListVector::Reserve(result, total_list_size);
+	ListVector::SetListSize(result, total_list_size);
+
+	auto list_data = ListVector::GetData(result);
+	auto child_data = FlatVector::GetData<string_t>(varchar_vector);
+
+	bool all_converted = true;
+	idx_t total = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			result_mask.SetInvalid(i);
+			continue;
+		}
+
+		list_data[i].offset = total;
+		auto valid =
+		    VectorStringifiedListParser::SplitStringifiedList(source_data[idx], child_data, total, varchar_vector);
+		if (!valid) {
+			string text = "Type VARCHAR with value '" + source_data[idx].GetString() +
+			              "' can't be cast to the destination type LIST";
+			HandleVectorCastError::Operation<string_t>(text, result_mask, idx, parameters.error_message, all_converted);
+		}
+		list_data[i].length = total - list_data[i].offset; // length is the amount of parts coming from this string
+	}
+	D_ASSERT(total_list_size == total);
+
+	auto &result_child = ListVector::GetEntry(result);
+	auto &cast_data = (ListBoundCastData &)*parameters.cast_data;
+	CastParameters child_parameters(parameters, cast_data.child_cast_info.cast_data.get());
+	return cast_data.child_cast_info.function(varchar_vector, result_child, total_list_size, child_parameters) &&
+	       all_converted;
+}
+
+bool StringListCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
+	D_ASSERT(source.GetType().id() == LogicalTypeId::VARCHAR);
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+
+	switch (source.GetVectorType()) {
+	case VectorType::CONSTANT_VECTOR: {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+
+		auto source_data = ConstantVector::GetData<string_t>(source);
+		auto &source_mask = ConstantVector::Validity(source);
+		auto &result_mask = ConstantVector::Validity(result);
+
+		return StringListCastLoop(source_data, source_mask, result, result_mask, 1, parameters, nullptr);
+	}
+	default: {
+		UnifiedVectorFormat unified_source;
+		result.SetVectorType(VectorType::FLAT_VECTOR);
+
+		source.ToUnifiedFormat(count, unified_source);
+		auto source_sel = unified_source.sel;
+		auto source_data = (string_t *)unified_source.data;
+		auto &source_mask = unified_source.validity;
+		auto &result_mask = FlatVector::Validity(result);
+
+		return StringListCastLoop(source_data, source_mask, result, result_mask, count, parameters, source_sel);
+	}
+	}
+}
+
+BoundCastInfo StringToListCast(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
+	// second argument allows for a secondary casting function to be passed in the CastParameters
+	return BoundCastInfo(&StringListCast,
+	                     ListBoundCastData::BindListToListCast(input, LogicalType::LIST(LogicalType::VARCHAR), target));
+}
+
 BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const LogicalType &source,
                                              const LogicalType &target) {
-	// now switch on the target type
 	switch (target.id()) {
 	case LogicalTypeId::DATE:
 		return BoundCastInfo(&VectorCastHelpers::TryCastErrorLoop<string_t, date_t, duckdb::TryCastErrorMessage>);
@@ -96980,6 +97071,8 @@ BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const Logical
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::JSON:
 		return &DefaultCasts::ReinterpretCast;
+	case LogicalTypeId::LIST:
+		return StringToListCast(input, source, target);
 	default:
 		return VectorStringCastNumericSwitch(input, source, target);
 	}
@@ -97341,6 +97434,144 @@ BoundCastInfo DefaultCasts::UUIDCastSwitch(BindCastInput &input, const LogicalTy
 } // namespace duckdb
 
 
+namespace duckdb {
+
+struct CountPartOperation {
+	idx_t count = 0;
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+		count++;
+	}
+};
+
+struct SplitStringOperation {
+	SplitStringOperation(string_t *child_data, idx_t &child_start, Vector &child)
+	    : child_data(child_data), child_start(child_start), child(child) {
+	}
+
+	string_t *child_data;
+	idx_t &child_start;
+	Vector &child;
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+
+		if ((pos - start_pos) >= 4 && buf[start_pos] == 'N' && buf[start_pos + 1] == 'U' && buf[start_pos + 2] == 'L' &&
+		    buf[start_pos + 3] == 'L') {
+			FlatVector::SetNull(child, child_start, true);
+			child_start++;
+			return;
+		}
+		child_data[child_start] = StringVector::AddString(child, buf + start_pos, pos - start_pos);
+		child_start++;
+	}
+};
+
+static bool SkipToCloseQuotes(idx_t &pos, const char *buf, idx_t &len) {
+	char quote = buf[pos];
+	pos++;
+
+	while (pos < len) {
+		if (buf[pos] == quote) {
+			return true;
+		}
+		pos++;
+	}
+	return false;
+}
+
+static bool SkipToClose(idx_t &idx, const char *buf, idx_t &len, idx_t &lvl) {
+	while (idx < len) {
+		if (buf[idx] == '[') {
+			if (!SkipToClose(++idx, buf, len, lvl)) {
+				return false;
+			}
+			lvl++;
+			idx++;
+		}
+		if (buf[idx] == '"' || buf[idx] == '\'') {
+			SkipToCloseQuotes(idx, buf, len);
+		}
+		if (buf[idx] == ']') {
+			lvl--;
+			return true;
+		}
+		idx++;
+	}
+	return false;
+}
+
+template <class OP>
+static bool SplitStringifiedListInternal(const string_t &input, OP &state) {
+	const char *buf = input.GetDataUnsafe();
+	idx_t len = input.GetSize();
+	idx_t lvl = 1;
+	idx_t pos = 0;
+
+	while (pos < len && StringUtil::CharacterIsSpace(buf[pos])) {
+		pos++;
+	}
+	if (pos == len || buf[pos] != '[') {
+		return false;
+	}
+	pos++;
+	while (pos < len && StringUtil::CharacterIsSpace(buf[pos])) {
+		pos++;
+	}
+
+	idx_t start_pos = pos;
+	while (pos < len) {
+		if (buf[pos] == '[') {
+			if (!SkipToClose(++pos, buf, len, ++lvl)) {
+				return false;
+			}
+		} else if (buf[pos] == '"' || buf[pos] == '\'') {
+			SkipToCloseQuotes(pos, buf, len);
+		} else if (buf[pos] == ',' || buf[pos] == ']') {
+			idx_t trailing_whitespace = 0;
+			while (StringUtil::CharacterIsSpace(buf[pos - trailing_whitespace - 1])) {
+				trailing_whitespace++;
+			}
+			if (!(buf[pos] == ']' && start_pos == (pos))) {
+				state.HandleValue(buf, start_pos, pos - trailing_whitespace);
+			} // else the list is empty
+			if (buf[pos] == ']') {
+				lvl--;
+				break;
+			}
+			while (pos + 1 < len && StringUtil::CharacterIsSpace(buf[pos + 1])) {
+				pos++;
+			}
+			start_pos = pos + 1;
+		}
+		pos++;
+	}
+	pos++;
+	while (pos < len) {
+		if (!StringUtil::CharacterIsSpace(buf[pos])) {
+			return false;
+		}
+		pos++;
+	}
+	if (lvl != 0) {
+		return false;
+	}
+	return true;
+}
+
+bool VectorStringifiedListParser::SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start,
+                                                       Vector &child) {
+	SplitStringOperation state(child_data, child_start, child);
+	return SplitStringifiedListInternal<SplitStringOperation>(input, state);
+}
+
+idx_t VectorStringifiedListParser::CountParts(const string_t &input) {
+	CountPartOperation state;
+	SplitStringifiedListInternal<CountPartOperation>(input, state);
+	return state.count;
+}
+} // namespace duckdb
+
+
 namespace duckdb {
 
 //! The target type determines the preferred implicit casts
@@ -201589,6 +201820,7 @@ void DataTable::AppendLock(TableAppendState &state) {
 		throw TransactionException("Transaction conflict: adding entries to a table that has been altered!");
 	}
 	state.row_start = row_groups->GetTotalRows();
+	state.current_row = state.row_start;
 }
 
 void DataTable::InitializeAppend(Transaction &transaction, TableAppendState &state, idx_t append_count) {
@@ -201652,9 +201884,6 @@ void DataTable::MergeStorage(RowGroupCollection &data, TableIndexList &indexes,
 	row_groups->MergeStorage(data);
 	stats.MergeStats(other_stats);
 	row_groups->Verify();
-	if (!indexes.Empty()) {
-		throw InternalException("FIXME: merge indexes");
-	}
 }
 
 void DataTable::WriteToLog(WriteAheadLog &log, idx_t row_start, idx_t count) {
@@ -202149,7 +202378,7 @@ namespace duckdb {
 // Local Table Storage
 //===--------------------------------------------------------------------===//
 LocalTableStorage::LocalTableStorage(DataTable &table)
-    : table(table), allocator(Allocator::Get(table.db)), deleted_rows(0) {
+    : table(&table), allocator(Allocator::Get(table.db)), deleted_rows(0) {
 	auto types = table.GetTypes();
 	row_groups = make_shared<RowGroupCollection>(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(),
 	                                             types, MAX_ROW_ID, 0);
@@ -202174,7 +202403,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t changed_idx,
                                      const LogicalType &target_type, const vector<column_t> &bound_columns,
                                      Expression &cast_expr)
-    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows),
+    : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
       partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
 	if (partial_manager) {
 		partial_manager->FlushPartialBlocks();
@@ -202187,7 +202416,7 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 }
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, idx_t drop_idx)
-    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows),
+    : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
       partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
 	if (partial_manager) {
 		partial_manager->FlushPartialBlocks();
@@ -202200,9 +202429,9 @@ LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &paren
 
 LocalTableStorage::LocalTableStorage(DataTable &new_dt, LocalTableStorage &parent, ColumnDefinition &new_column,
                                      Expression *default_value)
-    : table(new_dt), allocator(Allocator::Get(table.db)), deleted_rows(parent.deleted_rows),
+    : table(&new_dt), allocator(Allocator::Get(table->db)), deleted_rows(parent.deleted_rows),
       partial_manager(move(parent.partial_manager)), written_blocks(move(parent.written_blocks)) {
-	idx_t new_column_idx = parent.table.column_definitions.size();
+	idx_t new_column_idx = parent.table->column_definitions.size();
 	stats.InitializeAddColumn(parent.stats, new_column.GetType());
 	row_groups = parent.row_groups->AddColumn(new_column, default_value, stats.GetStats(new_column_idx));
 	parent.row_groups.reset();
@@ -202312,11 +202541,7 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 void LocalTableStorage::CheckFlushToDisk() {
 	// we finished writing a complete row group
 	// check if we should pre-emptively write it to disk
-	if (table.info->IsTemporary() || StorageManager::GetStorageManager(table.db).InMemory()) {
-		return;
-	}
-	if (!table.info->indexes.Empty()) {
-		// we have indexes - we cannot merge
+	if (table->info->IsTemporary() || StorageManager::GetStorageManager(table->db).InMemory()) {
 		return;
 	}
 	if (deleted_rows != 0) {
@@ -202326,7 +202551,7 @@ void LocalTableStorage::CheckFlushToDisk() {
 	// we should! write the second-to-last row group to disk
 	// allocate the partial block-manager if none is allocated yet
 	if (!partial_manager) {
-		auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
+		auto &block_manager = table->info->table_io_manager->GetBlockManagerForRowData();
 		partial_manager = make_unique<PartialBlockManager>(block_manager);
 	}
 	// flush second-to-last row group
@@ -202338,12 +202563,11 @@ void LocalTableStorage::FlushToDisk(RowGroup *row_group) {
 	// flush the specified row group
 	D_ASSERT(row_group);
 	D_ASSERT(deleted_rows == 0);
-	D_ASSERT(table.info->indexes.Empty());
 	D_ASSERT(partial_manager);
 	//! The set of column compression types (if any)
 	vector<CompressionType> compression_types;
 	D_ASSERT(compression_types.empty());
-	for (auto &column : table.column_definitions) {
+	for (auto &column : table->column_definitions) {
 		compression_types.push_back(column.CompressionType());
 	}
 	auto row_group_pointer = row_group->WriteToDisk(*partial_manager, compression_types);
@@ -202410,24 +202634,23 @@ void LocalStorage::Update(DataTable *table, Vector &row_ids, const vector<column
 }
 
 template <class T>
-bool LocalStorage::ScanTableStorage(DataTable &table, LocalTableStorage &storage, T &&fun) {
-	vector<column_t> column_ids;
-	column_ids.reserve(table.column_definitions.size());
-	for (idx_t i = 0; i < table.column_definitions.size(); i++) {
-		column_ids.push_back(i);
+bool LocalTableStorage::ScanTableStorage(Transaction &transaction, const vector<column_t> &column_ids, T &&fun) {
+	auto all_types = table->GetTypes();
+	vector<LogicalType> scan_types;
+	for (idx_t i = 0; i < column_ids.size(); i++) {
+		scan_types.push_back(all_types[column_ids[i]]);
 	}
-
 	DataChunk chunk;
-	chunk.Initialize(storage.allocator, table.GetTypes());
+	chunk.Initialize(allocator, scan_types);
 
 	// initialize the scan
 	TableScanState state;
 	state.Initialize(column_ids, nullptr);
-	storage.InitializeScan(state.local_state, nullptr);
+	InitializeScan(state.local_state, nullptr);
 
 	while (true) {
 		chunk.Reset();
-		Scan(state.local_state, column_ids, chunk);
+		state.local_state.Scan(transaction, chunk);
 		if (chunk.size() == 0) {
 			return true;
 		}
@@ -202437,6 +202660,78 @@ bool LocalStorage::ScanTableStorage(DataTable &table, LocalTableStorage &storage
 	}
 }
 
+template <class T>
+bool LocalTableStorage::ScanTableStorage(Transaction &transaction, T &&fun) {
+	vector<column_t> column_ids;
+	column_ids.reserve(table->column_definitions.size());
+	for (idx_t i = 0; i < table->column_definitions.size(); i++) {
+		column_ids.push_back(i);
+	}
+	return ScanTableStorage(transaction, column_ids, fun);
+}
+
+void LocalTableStorage::AppendToIndexes(Transaction &transaction, TableAppendState &append_state, idx_t append_count,
+                                        bool append_to_table) {
+	bool constraint_violated = false;
+	if (append_to_table) {
+		table->InitializeAppend(transaction, append_state, append_count);
+	}
+	if (append_to_table) {
+		// appending: need to scan entire
+		ScanTableStorage(transaction, [&](DataChunk &chunk) -> bool {
+			// append this chunk to the indexes of the table
+			if (!table->AppendToIndexes(chunk, append_state.current_row)) {
+				constraint_violated = true;
+				return false;
+			}
+			// append to base table
+			table->Append(chunk, append_state);
+			return true;
+		});
+	} else {
+		// only need to scan for index append
+		// figure out which columns we need to scan for the set of indexes
+		auto columns = table->info->indexes.GetRequiredColumns();
+		// create an empty mock chunk that contains all the correct types for the table
+		DataChunk mock_chunk;
+		mock_chunk.InitializeEmpty(table->GetTypes());
+		ScanTableStorage(transaction, columns, [&](DataChunk &chunk) -> bool {
+			// construct the mock chunk by referencing the required columns
+			for (idx_t i = 0; i < columns.size(); i++) {
+				mock_chunk.data[columns[i]].Reference(chunk.data[i]);
+			}
+			mock_chunk.SetCardinality(chunk);
+			// append this chunk to the indexes of the table
+			if (!table->AppendToIndexes(mock_chunk, append_state.current_row)) {
+				constraint_violated = true;
+				return false;
+			}
+			append_state.current_row += chunk.size();
+			return true;
+		});
+	}
+	if (constraint_violated) {
+		// need to revert the append
+		row_t current_row = append_state.row_start;
+		// remove the data from the indexes, if there are any indexes
+		ScanTableStorage(transaction, [&](DataChunk &chunk) -> bool {
+			// append this chunk to the indexes of the table
+			table->RemoveFromIndexes(append_state, chunk, current_row);
+
+			current_row += chunk.size();
+			if (current_row >= append_state.current_row) {
+				// finished deleting all rows from the index: abort now
+				return false;
+			}
+			return true;
+		});
+		if (append_to_table) {
+			table->RevertAppendInternal(append_state.row_start, append_count);
+		}
+		throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
+	}
+}
+
 void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	// bulk append threshold: a full row group
 	static constexpr const idx_t MERGE_THRESHOLD = RowGroup::ROW_GROUP_SIZE;
@@ -202452,10 +202747,17 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 	TableAppendState append_state;
 	table.AppendLock(append_state);
 	if ((append_state.row_start == 0 || storage.row_groups->GetTotalRows() >= MERGE_THRESHOLD) &&
-	    storage.table.info->indexes.Empty() && storage.deleted_rows == 0) {
+	    storage.deleted_rows == 0) {
 		// table is currently empty OR we are bulk appending: move over the storage directly
 		// first flush any out-standing storage nodes
 		storage.FlushToDisk();
+		// now append to the indexes (if there are any)
+		// FIXME: we should be able to merge the transaction-local index directly into the main table index
+		// as long we just rewrite some row-ids
+		if (!table.info->indexes.Empty()) {
+			storage.AppendToIndexes(transaction, append_state, append_count, false);
+		}
+		// finally move over the row groups
 		table.MergeStorage(*storage.row_groups, storage.indexes, storage.stats);
 	} else {
 		if (storage.partial_manager || !storage.written_blocks.empty()) {
@@ -202463,36 +202765,8 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 			// revert the data we have already written
 			storage.Rollback();
 		}
-		bool constraint_violated = false;
-		table.InitializeAppend(transaction, append_state, append_count);
-		ScanTableStorage(table, storage, [&](DataChunk &chunk) -> bool {
-			// append this chunk to the indexes of the table
-			if (!table.AppendToIndexes(chunk, append_state.current_row)) {
-				constraint_violated = true;
-				return false;
-			}
-			// append to base table
-			table.Append(chunk, append_state);
-			return true;
-		});
-		if (constraint_violated) {
-			// need to revert the append
-			row_t current_row = append_state.row_start;
-			// remove the data from the indexes, if there are any indexes
-			ScanTableStorage(table, storage, [&](DataChunk &chunk) -> bool {
-				// append this chunk to the indexes of the table
-				table.RemoveFromIndexes(append_state, chunk, current_row);
-
-				current_row += chunk.size();
-				if (current_row >= append_state.current_row) {
-					// finished deleting all rows from the index: abort now
-					return false;
-				}
-				return true;
-			});
-			table.RevertAppendInternal(append_state.row_start, append_count);
-			throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
-		}
+		// append to the indexes and append to the base table
+		storage.AppendToIndexes(transaction, append_state, append_count, true);
 	}
 	transaction.PushAppend(&table, append_state.row_start, append_count);
 }
@@ -202523,7 +202797,7 @@ void LocalTableStorage::Rollback() {
 		partial_manager->Clear();
 		partial_manager.reset();
 	}
-	auto &block_manager = table.info->table_io_manager->GetBlockManagerForRowData();
+	auto &block_manager = table->info->table_io_manager->GetBlockManagerForRowData();
 	for (auto block_id : written_blocks) {
 		block_manager.MarkBlockAsModified(block_id);
 	}
@@ -202545,6 +202819,7 @@ void LocalStorage::MoveStorage(DataTable *old_dt, DataTable *new_dt) {
 	}
 	// take over the storage from the old entry
 	auto new_storage = move(entry->second);
+	new_storage->table = new_dt;
 	table_storage.erase(entry);
 	table_storage[new_dt] = move(new_storage);
 }
@@ -202596,7 +202871,7 @@ void LocalStorage::FetchChunk(DataTable *table, Vector &row_ids, idx_t count, Da
 
 	ColumnFetchState fetch_state;
 	vector<column_t> col_ids;
-	vector<LogicalType> types = storage->table.GetTypes();
+	vector<LogicalType> types = storage->table->GetTypes();
 	for (idx_t i = 0; i < types.size(); i++) {
 		col_ids.push_back(i);
 	}
@@ -210911,6 +211186,22 @@ void TableIndexList::VerifyForeignKey(const vector<idx_t> &fk_keys, bool is_appe
 	}
 }
 
+vector<column_t> TableIndexList::GetRequiredColumns() {
+	lock_guard<mutex> lock(indexes_lock);
+	set<column_t> unique_indexes;
+	for (auto &index : indexes) {
+		for (auto col_index : index->column_ids) {
+			unique_indexes.insert(col_index);
+		}
+	}
+	vector<column_t> result;
+	result.reserve(unique_indexes.size());
+	for (auto column_index : unique_indexes) {
+		result.emplace_back(column_index);
+	}
+	return result;
+}
+
 vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetaBlockWriter &writer) {
 	vector<BlockPointer> blocks_info;
 	for (auto &index : indexes) {