duckdb 0.6.2-dev1376.0 → 0.6.2-dev1461.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1376.0",
+  "version": "0.6.2-dev1461.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/types/conflict_info.cpp

@@ -0,0 +1,18 @@
+#include "duckdb/common/types/constraint_conflict_info.hpp"
+#include "duckdb/storage/index.hpp"
+
+namespace duckdb {
+
+bool ConflictInfo::ConflictTargetMatches(Index &index) const {
+	if (only_check_unique && !index.IsUnique()) {
+		// We only support checking ON CONFLICT for Unique/Primary key constraints
+		return false;
+	}
+	if (column_ids.empty()) {
+		return true;
+	}
+	// Check whether the column ids match
+	return column_ids == index.column_id_set;
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/types/conflict_manager.cpp

@@ -0,0 +1,257 @@
+#include "duckdb/common/types/conflict_manager.hpp"
+#include "duckdb/storage/index.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/common/types/constraint_conflict_info.hpp"
+
+namespace duckdb {
+
+ConflictManager::ConflictManager(VerifyExistenceType lookup_type, idx_t input_size, ConflictInfo *conflict_info)
+    : lookup_type(lookup_type), input_size(input_size), conflict_info(conflict_info), conflicts(input_size, false),
+      mode(ConflictManagerMode::THROW) {
+}
+
+ManagedSelection &ConflictManager::InternalSelection() {
+	if (!conflicts.Initialized()) {
+		conflicts.Initialize(input_size);
+	}
+	return conflicts;
+}
+
+const unordered_set<idx_t> &ConflictManager::InternalConflictSet() const {
+	D_ASSERT(conflict_set);
+	return *conflict_set;
+}
+
+Vector &ConflictManager::InternalRowIds() {
+	if (!row_ids) {
+		row_ids = make_unique<Vector>(LogicalType::ROW_TYPE, input_size);
+	}
+	return *row_ids;
+}
+
+Vector &ConflictManager::InternalIntermediate() {
+	if (!intermediate_vector) {
+		intermediate_vector = make_unique<Vector>(LogicalType::BOOLEAN, true, true, input_size);
+	}
+	return *intermediate_vector;
+}
+
+const ConflictInfo &ConflictManager::GetConflictInfo() const {
+	D_ASSERT(conflict_info);
+	return *conflict_info;
+}
+
+void ConflictManager::FinishLookup() {
+	if (mode == ConflictManagerMode::THROW) {
+		return;
+	}
+	if (!SingleIndexTarget()) {
+		return;
+	}
+	if (conflicts.Count() != 0) {
+		// We have recorded conflicts from the one index we're interested in
+		// We set this so we don't duplicate the conflicts when there are duplicate indexes
+		// that also match our conflict target
+		single_index_finished = true;
+	}
+}
+
+void ConflictManager::SetMode(ConflictManagerMode mode) {
+	// Only allow SCAN when we have conflict info
+	D_ASSERT(mode != ConflictManagerMode::SCAN || conflict_info != nullptr);
+	this->mode = mode;
+}
+
+void ConflictManager::AddToConflictSet(idx_t chunk_index) {
+	if (!conflict_set) {
+		conflict_set = make_unique<unordered_set<idx_t>>();
+	}
+	auto &set = *conflict_set;
+	set.insert(chunk_index);
+}
+
+void ConflictManager::AddConflictInternal(idx_t chunk_index, row_t row_id) {
+	D_ASSERT(mode == ConflictManagerMode::SCAN);
+
+	// Only when we should not throw on conflict should we get here
+	D_ASSERT(!ShouldThrow(chunk_index));
+	AddToConflictSet(chunk_index);
+	if (SingleIndexTarget()) {
+		// If we have identical indexes, only the conflicts of the first index should be recorded
+		// as the other index(es) would produce the exact same conflicts anyways
+		if (single_index_finished) {
+			return;
+		}
+
+		// We can be more efficient because we don't need to merge conflicts of multiple indexes
+		auto &selection = InternalSelection();
+		auto &row_ids = InternalRowIds();
+		auto data = FlatVector::GetData<row_t>(row_ids);
+		data[selection.Count()] = row_id;
+		selection.Append(chunk_index);
+	} else {
+		auto &intermediate = InternalIntermediate();
+		auto data = FlatVector::GetData<bool>(intermediate);
+		// Mark this index in the chunk as producing a conflict
+		data[chunk_index] = true;
+		if (row_id_map.empty()) {
+			row_id_map.resize(input_size);
+		}
+		row_id_map[chunk_index] = row_id;
+	}
+}
+
+bool ConflictManager::IsConflict(LookupResultType type) {
+	switch (type) {
+	case LookupResultType::LOOKUP_NULL: {
+		if (ShouldIgnoreNulls()) {
+			return false;
+		}
+		// If nulls are not ignored, treat this as a hit instead
+		return IsConflict(LookupResultType::LOOKUP_HIT);
+	}
+	case LookupResultType::LOOKUP_HIT: {
+		return true;
+	}
+	case LookupResultType::LOOKUP_MISS: {
+		// FIXME: If we record a miss as a conflict when the verify type is APPEND_FK, then we can simplify the checks
+		// in VerifyForeignKeyConstraint This also means we should not record a hit as a conflict when the verify type
+		// is APPEND_FK
+		return false;
+	}
+	default: {
+		throw NotImplementedException("Type not implemented for LookupResultType");
+	}
+	}
+}
+
+bool ConflictManager::AddHit(idx_t chunk_index, row_t row_id) {
+	D_ASSERT(chunk_index < input_size);
+	// First check if this causes a conflict
+	if (!IsConflict(LookupResultType::LOOKUP_HIT)) {
+		return false;
+	}
+
+	// Then check if we should throw on a conflict
+	if (ShouldThrow(chunk_index)) {
+		return true;
+	}
+	if (mode == ConflictManagerMode::THROW) {
+		// When our mode is THROW, and the chunk index is part of the previously scanned conflicts
+		// then we ignore the conflict instead
+		D_ASSERT(!ShouldThrow(chunk_index));
+		return false;
+	}
+	D_ASSERT(conflict_info);
+	// Because we don't throw, we need to register the conflict
+	AddConflictInternal(chunk_index, row_id);
+	return false;
+}
+
+bool ConflictManager::AddMiss(idx_t chunk_index) {
+	D_ASSERT(chunk_index < input_size);
+	return IsConflict(LookupResultType::LOOKUP_MISS);
+}
+
+bool ConflictManager::AddNull(idx_t chunk_index) {
+	D_ASSERT(chunk_index < input_size);
+	if (!IsConflict(LookupResultType::LOOKUP_NULL)) {
+		return false;
+	}
+	return AddHit(chunk_index, DConstants::INVALID_INDEX);
+}
+
+bool ConflictManager::SingleIndexTarget() const {
+	D_ASSERT(conflict_info);
+	// We are only interested in a specific index
+	return !conflict_info->column_ids.empty();
+}
+
+bool ConflictManager::ShouldThrow(idx_t chunk_index) const {
+	if (mode == ConflictManagerMode::SCAN) {
+		return false;
+	}
+	D_ASSERT(mode == ConflictManagerMode::THROW);
+	if (conflict_set == nullptr) {
+		// No conflicts were scanned, so this conflict is not in the set
+		return true;
+	}
+	auto &set = InternalConflictSet();
+	if (set.count(chunk_index)) {
+		return false;
+	}
+	// None of the scanned conflicts arose from this insert tuple
+	return true;
+}
+
+bool ConflictManager::ShouldIgnoreNulls() const {
+	switch (lookup_type) {
+	case VerifyExistenceType::APPEND:
+		return true;
+	case VerifyExistenceType::APPEND_FK:
+		return false;
+	case VerifyExistenceType::DELETE_FK:
+		return true;
+	default:
+		throw InternalException("Type not implemented for VerifyExistenceType");
+	}
+}
+
+Vector &ConflictManager::RowIds() {
+	D_ASSERT(finalized);
+	return *row_ids;
+}
+
+const ManagedSelection &ConflictManager::Conflicts() const {
+	D_ASSERT(finalized);
+	return conflicts;
+}
+
+idx_t ConflictManager::ConflictCount() const {
+	return conflicts.Count();
+}
+
+void ConflictManager::Finalize() {
+	D_ASSERT(!finalized);
+	if (SingleIndexTarget()) {
+		// Selection vector has been directly populated already, no need to finalize
+		finalized = true;
+		return;
+	}
+	finalized = true;
+	if (!intermediate_vector) {
+		// No conflicts were found, we're done
+		return;
+	}
+	auto &intermediate = InternalIntermediate();
+	auto data = FlatVector::GetData<bool>(intermediate);
+	auto &selection = InternalSelection();
+	// Create the selection vector from the encountered conflicts
+	for (idx_t i = 0; i < input_size; i++) {
+		if (data[i]) {
+			selection.Append(i);
+		}
+	}
+	// Now create the row_ids Vector, aligned with the selection vector
+	auto &row_ids = InternalRowIds();
+	auto row_id_data = FlatVector::GetData<row_t>(row_ids);
+
+	for (idx_t i = 0; i < selection.Count(); i++) {
+		D_ASSERT(!row_id_map.empty());
+		auto index = selection[i];
+		D_ASSERT(index < row_id_map.size());
+		auto row_id = row_id_map[index];
+		row_id_data[i] = row_id;
+	}
+	intermediate_vector.reset();
+}
+
+VerifyExistenceType ConflictManager::LookupType() const {
+	return this->lookup_type;
+}
+
+void ConflictManager::SetIndexCount(idx_t count) {
+	index_count = count;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/column_binding_resolver.cpp

@@ -3,6 +3,7 @@
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_create_index.hpp"
 #include "duckdb/planner/operator/logical_delim_join.hpp"
+#include "duckdb/planner/operator/logical_insert.hpp"
 
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
@@ -59,6 +60,25 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 		bindings = op.GetColumnBindings();
 		VisitOperatorExpressions(op);
 		return;
+	} else if (op.type == LogicalOperatorType::LOGICAL_INSERT) {
+		//! We want to execute the normal path, but also add a dummy 'excluded' binding if there is a
+		// ON CONFLICT DO UPDATE clause
+		auto &insert_op = (LogicalInsert &)op;
+		if (insert_op.action_type != OnConflictAction::THROW) {
+			VisitOperatorChildren(op);
+			auto dummy_bindings = LogicalOperator::GenerateColumnBindings(
+			    insert_op.excluded_table_index, insert_op.table->columns.PhysicalColumnCount());
+			bindings.insert(bindings.begin(), dummy_bindings.begin(), dummy_bindings.end());
+			if (insert_op.on_conflict_condition) {
+				VisitExpression(&insert_op.on_conflict_condition);
+			}
+			if (insert_op.do_update_condition) {
+				VisitExpression(&insert_op.do_update_condition);
+			}
+			VisitOperatorExpressions(op);
+			bindings = op.GetColumnBindings();
+			return;
+		}
 	}
 	// general case
 	// first visit the children of this operator

package/src/duckdb/src/execution/expression_executor/execute_operator.cpp

@@ -20,7 +20,7 @@ void ExpressionExecutor::Execute(const BoundOperatorExpression &expr, Expression
 	// IN has n children
 	if (expr.type == ExpressionType::COMPARE_IN || expr.type == ExpressionType::COMPARE_NOT_IN) {
 		if (expr.children.size() < 2) {
-			throw Exception("IN needs at least two children");
+			throw InvalidInputException("IN needs at least two children");
 		}
 
 		Vector left(expr.children[0]->return_type);

package/src/duckdb/src/execution/index/art/art.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/execution/index/art/art_key.hpp"
+#include "duckdb/common/types/conflict_manager.hpp"
 
 #include <algorithm>
 #include <cstring>
@@ -375,15 +376,26 @@ bool ART::Append(IndexLock &lock, DataChunk &appended_data, Vector &row_identifi
 }
 
 void ART::VerifyAppend(DataChunk &chunk) {
-	VerifyExistence(chunk, VerifyExistenceType::APPEND);
+	ConflictManager conflict_manager(VerifyExistenceType::APPEND, chunk.size());
+	LookupValues(chunk, conflict_manager);
 }
 
-void ART::VerifyAppendForeignKey(DataChunk &chunk, string *err_msg_ptr) {
-	VerifyExistence(chunk, VerifyExistenceType::APPEND_FK, err_msg_ptr);
+void ART::VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) {
+	D_ASSERT(conflict_manager.LookupType() == VerifyExistenceType::APPEND);
+	LookupValues(chunk, conflict_manager);
 }
 
-void ART::VerifyDeleteForeignKey(DataChunk &chunk, string *err_msg_ptr) {
-	VerifyExistence(chunk, VerifyExistenceType::DELETE_FK, err_msg_ptr);
+void ART::VerifyAppendForeignKey(DataChunk &chunk) {
+	ConflictManager conflict_manager(VerifyExistenceType::APPEND_FK, chunk.size());
+	LookupValues(chunk, conflict_manager);
+}
+
+void ART::VerifyDeleteForeignKey(DataChunk &chunk) {
+	if (!IsUnique()) {
+		return;
+	}
+	ConflictManager conflict_manager(VerifyExistenceType::DELETE_FK, chunk.size());
+	LookupValues(chunk, conflict_manager);
 }
 
 bool ART::InsertToLeaf(Leaf &leaf, row_t row_id) {
@@ -789,69 +801,90 @@ bool ART::Scan(Transaction &transaction, DataTable &table, IndexScanState &table
 	return true;
 }
 
-void ART::VerifyExistence(DataChunk &chunk, VerifyExistenceType verify_type, string *err_msg_ptr) {
-	if (verify_type != VerifyExistenceType::DELETE_FK && !IsUnique()) {
-		return;
+string ART::GenerateErrorKeyName(DataChunk &input, idx_t row) {
+	// re-executing the expressions is not very fast, but we're going to throw anyways, so we don't care
+	DataChunk expression_chunk;
+	expression_chunk.Initialize(Allocator::DefaultAllocator(), logical_types);
+	ExecuteExpressions(input, expression_chunk);
+
+	string key_name;
+	for (idx_t k = 0; k < expression_chunk.ColumnCount(); k++) {
+		if (k > 0) {
+			key_name += ", ";
+		}
+		key_name += unbound_expressions[k]->GetName() + ": " + expression_chunk.data[k].GetValue(row).ToString();
+	}
+	return key_name;
+}
+
+string ART::GenerateConstraintErrorMessage(VerifyExistenceType verify_type, const string &key_name) {
+	switch (verify_type) {
+	case VerifyExistenceType::APPEND: {
+		// This node already exists in the tree
+		string type = IsPrimary() ? "primary key" : "unique";
+		return StringUtil::Format("Duplicate key \"%s\" violates %s constraint", key_name, type);
+	}
+	case VerifyExistenceType::APPEND_FK: {
+		// The node we tried to insert does not exist in the foreign table
+		return StringUtil::Format(
+		    "Violates foreign key constraint because key \"%s\" does not exist in referenced table", key_name);
+	}
+	case VerifyExistenceType::DELETE_FK: {
+		// The node we tried to delete still exists in the foreign table
+		return StringUtil::Format("Violates foreign key constraint because key \"%s\" exists in table has foreign key",
+		                          key_name);
+	}
+	default:
+		throw NotImplementedException("Type not implemented for VerifyExistenceType");
 	}
+}
 
+void ART::LookupValues(DataChunk &input, ConflictManager &conflict_manager) {
 	DataChunk expression_chunk;
 	expression_chunk.Initialize(Allocator::DefaultAllocator(), logical_types);
 
 	// unique index, check
 	lock_guard<mutex> l(lock);
+
 	// first resolve the expressions for the index
-	ExecuteExpressions(chunk, expression_chunk);
+	ExecuteExpressions(input, expression_chunk);
 
 	// generate the keys for the given input
 	ArenaAllocator arena_allocator(BufferAllocator::Get(db));
 	vector<Key> keys(expression_chunk.size());
 	GenerateKeys(arena_allocator, expression_chunk, keys);
 
-	for (idx_t i = 0; i < chunk.size(); i++) {
+	idx_t found_conflict = DConstants::INVALID_INDEX;
+	for (idx_t i = 0; found_conflict == DConstants::INVALID_INDEX && i < input.size(); i++) {
 		if (keys[i].Empty()) {
+			if (conflict_manager.AddNull(i)) {
+				found_conflict = i;
+			}
 			continue;
 		}
-		Node *node_ptr = Lookup(tree, keys[i], 0);
-		bool throw_exception =
-		    verify_type == VerifyExistenceType::APPEND_FK ? node_ptr == nullptr : node_ptr != nullptr;
-		if (!throw_exception) {
-			continue;
-		}
-		string key_name;
-		for (idx_t k = 0; k < expression_chunk.ColumnCount(); k++) {
-			if (k > 0) {
-				key_name += ", ";
+		Leaf *leaf_ptr = Lookup(tree, keys[i], 0);
+		if (leaf_ptr == nullptr) {
+			if (conflict_manager.AddMiss(i)) {
+				found_conflict = i;
 			}
-			key_name += unbound_expressions[k]->GetName() + ": " + expression_chunk.data[k].GetValue(i).ToString();
-		}
-		string exception_msg;
-		switch (verify_type) {
-		case VerifyExistenceType::APPEND: {
-			// node already exists in tree
-			string type = IsPrimary() ? "primary key" : "unique";
-			exception_msg = "duplicate key \"" + key_name + "\" violates ";
-			exception_msg += type + " constraint";
-			break;
-		}
-		case VerifyExistenceType::APPEND_FK: {
-			// found node no exists in tree
-			exception_msg =
-			    "violates foreign key constraint because key \"" + key_name + "\" does not exist in referenced table";
-			break;
-		}
-		case VerifyExistenceType::DELETE_FK: {
-			// found node exists in tree
-			exception_msg =
-			    "violates foreign key constraint because key \"" + key_name + "\" exist in table has foreign key";
-			break;
-		}
+			continue;
 		}
-		if (err_msg_ptr) {
-			err_msg_ptr[i] = exception_msg;
-		} else {
-			throw ConstraintException(exception_msg);
+		// When we find a node, we need to update the 'matches' and 'row_ids'
+		// NOTE: Leafs can have more than one row_id, but for UNIQUE/PRIMARY KEY they will only have one
+		D_ASSERT(leaf_ptr->count == 1);
+		auto row_id = leaf_ptr->GetRowId(0);
+		if (conflict_manager.AddHit(i, row_id)) {
+			found_conflict = i;
 		}
 	}
+	conflict_manager.FinishLookup();
+	if (found_conflict == DConstants::INVALID_INDEX) {
+		// No conflicts detected
+		return;
+	}
+	auto key_name = GenerateErrorKeyName(input, found_conflict);
+	auto exception_msg = GenerateConstraintErrorMessage(conflict_manager.LookupType(), key_name);
+	throw ConstraintException(exception_msg);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -128,7 +128,7 @@ TextSearchShiftArray::TextSearchShiftArray() {
 
 TextSearchShiftArray::TextSearchShiftArray(string search_term) : length(search_term.size()) {
 	if (length > 255) {
-		throw Exception("Size of delimiter/quote/escape in CSV reader is limited to 255 bytes");
+		throw InvalidInputException("Size of delimiter/quote/escape in CSV reader is limited to 255 bytes");
 	}
 	// initialize the shifts array
 	shifts = unique_ptr<uint8_t[]>(new uint8_t[length * 255]);
@@ -248,7 +248,7 @@ void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
 	if (options.auto_detect) {
 		return_types = SniffCSV(requested_types);
 		if (return_types.empty()) {
-			throw Exception("Failed to detect column types from CSV: is the file a valid CSV file?");
+			throw InvalidInputException("Failed to detect column types from CSV: is the file a valid CSV file?");
 		}
 		if (cached_chunks.empty()) {
 			JumpToBeginning(options.skip_rows, options.header);

package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp

@@ -302,6 +302,8 @@ SinkResultType PhysicalBatchInsert::Sink(ExecutionContext &context, GlobalSinkSt
 	}
 	lstate.current_index = lstate.batch_index;
 	table->storage->VerifyAppendConstraints(*table, context.client, lstate.insert_chunk);
+	// TODO: call method that returns for which values the constraint check failed
+	// so we can support ON CONFLICT in here
 	auto new_row_group = lstate.current_collection->Append(lstate.insert_chunk, lstate.current_append_state);
 	if (new_row_group) {
 		lstate.writer->CheckFlushToDisk(*lstate.current_collection);