duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/common/types/conflict_manager.cpp

@@ -0,0 +1,257 @@
+#include "duckdb/common/types/conflict_manager.hpp"
+#include "duckdb/storage/index.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/common/types/constraint_conflict_info.hpp"
+
+namespace duckdb {
+
+ConflictManager::ConflictManager(VerifyExistenceType lookup_type, idx_t input_size, ConflictInfo *conflict_info)
+    : lookup_type(lookup_type), input_size(input_size), conflict_info(conflict_info), conflicts(input_size, false),
+      mode(ConflictManagerMode::THROW) {
+}
+
+ManagedSelection &ConflictManager::InternalSelection() {
+	if (!conflicts.Initialized()) {
+		conflicts.Initialize(input_size);
+	}
+	return conflicts;
+}
+
+const unordered_set<idx_t> &ConflictManager::InternalConflictSet() const {
+	D_ASSERT(conflict_set);
+	return *conflict_set;
+}
+
+Vector &ConflictManager::InternalRowIds() {
+	if (!row_ids) {
+		row_ids = make_unique<Vector>(LogicalType::ROW_TYPE, input_size);
+	}
+	return *row_ids;
+}
+
+Vector &ConflictManager::InternalIntermediate() {
+	if (!intermediate_vector) {
+		intermediate_vector = make_unique<Vector>(LogicalType::BOOLEAN, true, true, input_size);
+	}
+	return *intermediate_vector;
+}
+
+const ConflictInfo &ConflictManager::GetConflictInfo() const {
+	D_ASSERT(conflict_info);
+	return *conflict_info;
+}
+
+void ConflictManager::FinishLookup() {
+	if (mode == ConflictManagerMode::THROW) {
+		return;
+	}
+	if (!SingleIndexTarget()) {
+		return;
+	}
+	if (conflicts.Count() != 0) {
+		// We have recorded conflicts from the one index we're interested in
+		// We set this so we don't duplicate the conflicts when there are duplicate indexes
+		// that also match our conflict target
+		single_index_finished = true;
+	}
+}
+
+void ConflictManager::SetMode(ConflictManagerMode mode) {
+	// Only allow SCAN when we have conflict info
+	D_ASSERT(mode != ConflictManagerMode::SCAN || conflict_info != nullptr);
+	this->mode = mode;
+}
+
+void ConflictManager::AddToConflictSet(idx_t chunk_index) {
+	if (!conflict_set) {
+		conflict_set = make_unique<unordered_set<idx_t>>();
+	}
+	auto &set = *conflict_set;
+	set.insert(chunk_index);
+}
+
+void ConflictManager::AddConflictInternal(idx_t chunk_index, row_t row_id) {
+	D_ASSERT(mode == ConflictManagerMode::SCAN);
+
+	// Only when we should not throw on conflict should we get here
+	D_ASSERT(!ShouldThrow(chunk_index));
+	AddToConflictSet(chunk_index);
+	if (SingleIndexTarget()) {
+		// If we have identical indexes, only the conflicts of the first index should be recorded
+		// as the other index(es) would produce the exact same conflicts anyways
+		if (single_index_finished) {
+			return;
+		}
+
+		// We can be more efficient because we don't need to merge conflicts of multiple indexes
+		auto &selection = InternalSelection();
+		auto &row_ids = InternalRowIds();
+		auto data = FlatVector::GetData<row_t>(row_ids);
+		data[selection.Count()] = row_id;
+		selection.Append(chunk_index);
+	} else {
+		auto &intermediate = InternalIntermediate();
+		auto data = FlatVector::GetData<bool>(intermediate);
+		// Mark this index in the chunk as producing a conflict
+		data[chunk_index] = true;
+		if (row_id_map.empty()) {
+			row_id_map.resize(input_size);
+		}
+		row_id_map[chunk_index] = row_id;
+	}
+}
+
+bool ConflictManager::IsConflict(LookupResultType type) {
+	switch (type) {
+	case LookupResultType::LOOKUP_NULL: {
+		if (ShouldIgnoreNulls()) {
+			return false;
+		}
+		// If nulls are not ignored, treat this as a hit instead
+		return IsConflict(LookupResultType::LOOKUP_HIT);
+	}
+	case LookupResultType::LOOKUP_HIT: {
+		return true;
+	}
+	case LookupResultType::LOOKUP_MISS: {
+		// FIXME: If we record a miss as a conflict when the verify type is APPEND_FK, then we can simplify the checks
+		// in VerifyForeignKeyConstraint This also means we should not record a hit as a conflict when the verify type
+		// is APPEND_FK
+		return false;
+	}
+	default: {
+		throw NotImplementedException("Type not implemented for LookupResultType");
+	}
+	}
+}
+
+bool ConflictManager::AddHit(idx_t chunk_index, row_t row_id) {
+	D_ASSERT(chunk_index < input_size);
+	// First check if this causes a conflict
+	if (!IsConflict(LookupResultType::LOOKUP_HIT)) {
+		return false;
+	}
+
+	// Then check if we should throw on a conflict
+	if (ShouldThrow(chunk_index)) {
+		return true;
+	}
+	if (mode == ConflictManagerMode::THROW) {
+		// When our mode is THROW, and the chunk index is part of the previously scanned conflicts
+		// then we ignore the conflict instead
+		D_ASSERT(!ShouldThrow(chunk_index));
+		return false;
+	}
+	D_ASSERT(conflict_info);
+	// Because we don't throw, we need to register the conflict
+	AddConflictInternal(chunk_index, row_id);
+	return false;
+}
+
+bool ConflictManager::AddMiss(idx_t chunk_index) {
+	D_ASSERT(chunk_index < input_size);
+	return IsConflict(LookupResultType::LOOKUP_MISS);
+}
+
+bool ConflictManager::AddNull(idx_t chunk_index) {
+	D_ASSERT(chunk_index < input_size);
+	if (!IsConflict(LookupResultType::LOOKUP_NULL)) {
+		return false;
+	}
+	return AddHit(chunk_index, DConstants::INVALID_INDEX);
+}
+
+bool ConflictManager::SingleIndexTarget() const {
+	D_ASSERT(conflict_info);
+	// We are only interested in a specific index
+	return !conflict_info->column_ids.empty();
+}
+
+bool ConflictManager::ShouldThrow(idx_t chunk_index) const {
+	if (mode == ConflictManagerMode::SCAN) {
+		return false;
+	}
+	D_ASSERT(mode == ConflictManagerMode::THROW);
+	if (conflict_set == nullptr) {
+		// No conflicts were scanned, so this conflict is not in the set
+		return true;
+	}
+	auto &set = InternalConflictSet();
+	if (set.count(chunk_index)) {
+		return false;
+	}
+	// None of the scanned conflicts arose from this insert tuple
+	return true;
+}
+
+bool ConflictManager::ShouldIgnoreNulls() const {
+	switch (lookup_type) {
+	case VerifyExistenceType::APPEND:
+		return true;
+	case VerifyExistenceType::APPEND_FK:
+		return false;
+	case VerifyExistenceType::DELETE_FK:
+		return true;
+	default:
+		throw InternalException("Type not implemented for VerifyExistenceType");
+	}
+}
+
+Vector &ConflictManager::RowIds() {
+	D_ASSERT(finalized);
+	return *row_ids;
+}
+
+const ManagedSelection &ConflictManager::Conflicts() const {
+	D_ASSERT(finalized);
+	return conflicts;
+}
+
+idx_t ConflictManager::ConflictCount() const {
+	return conflicts.Count();
+}
+
+void ConflictManager::Finalize() {
+	D_ASSERT(!finalized);
+	if (SingleIndexTarget()) {
+		// Selection vector has been directly populated already, no need to finalize
+		finalized = true;
+		return;
+	}
+	finalized = true;
+	if (!intermediate_vector) {
+		// No conflicts were found, we're done
+		return;
+	}
+	auto &intermediate = InternalIntermediate();
+	auto data = FlatVector::GetData<bool>(intermediate);
+	auto &selection = InternalSelection();
+	// Create the selection vector from the encountered conflicts
+	for (idx_t i = 0; i < input_size; i++) {
+		if (data[i]) {
+			selection.Append(i);
+		}
+	}
+	// Now create the row_ids Vector, aligned with the selection vector
+	auto &row_ids = InternalRowIds();
+	auto row_id_data = FlatVector::GetData<row_t>(row_ids);
+
+	for (idx_t i = 0; i < selection.Count(); i++) {
+		D_ASSERT(!row_id_map.empty());
+		auto index = selection[i];
+		D_ASSERT(index < row_id_map.size());
+		auto row_id = row_id_map[index];
+		row_id_data[i] = row_id;
+	}
+	intermediate_vector.reset();
+}
+
+VerifyExistenceType ConflictManager::LookupType() const {
+	return this->lookup_type;
+}
+
+void ConflictManager::SetIndexCount(idx_t count) {
+	index_count = count;
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/types/data_chunk.cpp

@@ -48,7 +48,7 @@ void DataChunk::Initialize(Allocator &allocator, vector<LogicalType>::const_iter
 	for (; begin != end; begin++) {
 		VectorCache cache(allocator, *begin, capacity);
 		data.emplace_back(cache);
-		vector_caches.push_back(move(cache));
+		vector_caches.push_back(std::move(cache));
 	}
 }
 
@@ -117,8 +117,8 @@ void DataChunk::Reference(DataChunk &chunk) {
 void DataChunk::Move(DataChunk &chunk) {
 	SetCardinality(chunk);
 	SetCapacity(chunk);
-	data = move(chunk.data);
-	vector_caches = move(chunk.vector_caches);
+	data = std::move(chunk.data);
+	vector_caches = std::move(chunk.vector_caches);
 
 	chunk.Destroy();
 }
@@ -152,8 +152,8 @@ void DataChunk::Split(DataChunk &other, idx_t split_idx) {
 	D_ASSERT(split_idx < data.size());
 	const idx_t num_cols = data.size();
 	for (idx_t col_idx = split_idx; col_idx < num_cols; col_idx++) {
-		other.data.push_back(move(data[col_idx]));
-		other.vector_caches.push_back(move(vector_caches[col_idx]));
+		other.data.push_back(std::move(data[col_idx]));
+		other.vector_caches.push_back(std::move(vector_caches[col_idx]));
 	}
 	for (idx_t col_idx = split_idx; col_idx < num_cols; col_idx++) {
 		data.pop_back();
@@ -167,8 +167,8 @@ void DataChunk::Fuse(DataChunk &other) {
 	D_ASSERT(other.size() == size());
 	const idx_t num_cols = other.data.size();
 	for (idx_t col_idx = 0; col_idx < num_cols; ++col_idx) {
-		data.emplace_back(move(other.data[col_idx]));
-		vector_caches.emplace_back(move(other.vector_caches[col_idx]));
+		data.emplace_back(std::move(other.data[col_idx]));
+		vector_caches.emplace_back(std::move(other.vector_caches[col_idx]));
 	}
 	other.Destroy();
 }
@@ -307,6 +307,16 @@ void DataChunk::Hash(Vector &result) {
 	}
 }
 
+void DataChunk::Hash(vector<idx_t> &column_ids, Vector &result) {
+	D_ASSERT(result.GetType().id() == LogicalType::HASH);
+	D_ASSERT(column_ids.size() > 0);
+
+	VectorOperations::Hash(data[column_ids[0]], result, size());
+	for (idx_t i = 1; i < column_ids.size(); i++) {
+		VectorOperations::CombineHash(result, data[column_ids[i]], size());
+	}
+}
+
 void DataChunk::Verify() {
 #ifdef DEBUG
 	D_ASSERT(size() <= capacity);

package/src/duckdb/src/common/types/date.cpp

@@ -425,13 +425,18 @@ int64_t Date::Epoch(date_t date) {
 }
 
 int64_t Date::EpochNanoseconds(date_t date) {
-	return ((int64_t)date.days) * (Interval::MICROS_PER_DAY * 1000);
+	int64_t result;
+	if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, Interval::MICROS_PER_DAY * 1000,
+	                                                               result)) {
+		throw ConversionException("Could not convert DATE (%s) to nanoseconds", Date::ToString(date));
+	}
+	return result;
 }
 
 int64_t Date::EpochMicroseconds(date_t date) {
 	int64_t result;
 	if (!TryMultiplyOperator::Operation<int64_t, int64_t, int64_t>(date.days, Interval::MICROS_PER_DAY, result)) {
-		throw ConversionException("Could not convert DATE to microseconds");
+		throw ConversionException("Could not convert DATE (%s) to microseconds", Date::ToString(date));
 	}
 	return result;
 }

package/src/duckdb/src/common/types/hash.cpp

@@ -40,7 +40,7 @@ template <>
 hash_t Hash(float val) {
 	static_assert(sizeof(float) == sizeof(uint32_t), "");
 	FloatingPointEqualityTransform<float>::OP(val);
-	uint32_t uval = *((uint32_t *)&val);
+	uint32_t uval = Load<uint32_t>((const_data_ptr_t)&val);
 	return murmurhash64(uval);
 }
 
@@ -48,7 +48,7 @@ template <>
 hash_t Hash(double val) {
 	static_assert(sizeof(double) == sizeof(uint64_t), "");
 	FloatingPointEqualityTransform<double>::OP(val);
-	uint64_t uval = *((uint64_t *)&val);
+	uint64_t uval = Load<uint64_t>((const_data_ptr_t)&val);
 	return murmurhash64(uval);
 }
 

package/src/duckdb/src/common/types/partitioned_column_data.cpp

@@ -1,13 +1,14 @@
 #include "duckdb/common/types/partitioned_column_data.hpp"
 
 #include "duckdb/common/radix_partitioning.hpp"
+#include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 
 namespace duckdb {
 
 PartitionedColumnData::PartitionedColumnData(PartitionedColumnDataType type_p, ClientContext &context_p,
                                              vector<LogicalType> types_p)
-    : type(type_p), context(context_p), types(move(types_p)), allocators(make_shared<PartitionAllocators>()) {
+    : type(type_p), context(context_p), types(std::move(types_p)), allocators(make_shared<PartitionAllocators>()) {
 }
 
 PartitionedColumnData::PartitionedColumnData(const PartitionedColumnData &other)
@@ -18,6 +19,8 @@ unique_ptr<PartitionedColumnData> PartitionedColumnData::CreateShared() {
 	switch (type) {
 	case PartitionedColumnDataType::RADIX:
 		return make_unique<RadixPartitionedColumnData>((RadixPartitionedColumnData &)*this);
+	case PartitionedColumnDataType::HIVE:
+		return make_unique<HivePartitionedColumnData>((HivePartitionedColumnData &)*this);
 	default:
 		throw NotImplementedException("CreateShared for this type of PartitionedColumnData");
 	}
@@ -141,10 +144,12 @@ void PartitionedColumnData::FlushAppendState(PartitionedColumnDataAppendState &s
 void PartitionedColumnData::Combine(PartitionedColumnData &other) {
 	// Now combine the state's partitions into this
 	lock_guard<mutex> guard(lock);
+
 	if (partitions.empty()) {
 		// This is the first merge, we just copy them over
-		partitions = move(other.partitions);
+		partitions = std::move(other.partitions);
 	} else {
+		D_ASSERT(partitions.size() == other.partitions.size());
 		// Combine the append state's partitions into this PartitionedColumnData
 		for (idx_t i = 0; i < other.partitions.size(); i++) {
 			partitions[i]->Combine(*other.partitions[i]);

package/src/duckdb/src/common/types/row_data_collection.cpp

@@ -67,7 +67,7 @@ vector<BufferHandle> RowDataCollection::Build(idx_t added_count, data_ptr_t key_
 				// now append to the block
 				idx_t append_count = AppendToBlock(last_block, handle, append_entries, remaining, entry_sizes);
 				remaining -= append_count;
-				handles.push_back(move(handle));
+				handles.push_back(std::move(handle));
 			}
 		}
 		while (remaining > 0) {
@@ -83,9 +83,9 @@ vector<BufferHandle> RowDataCollection::Build(idx_t added_count, data_ptr_t key_
 			remaining -= append_count;
 
 			if (keep_pinned) {
-				pinned_blocks.push_back(move(handle));
+				pinned_blocks.push_back(std::move(handle));
 			} else {
-				handles.push_back(move(handle));
+				handles.push_back(std::move(handle));
 			}
 		}
 	}
@@ -121,8 +121,8 @@ void RowDataCollection::Merge(RowDataCollection &other) {
 		temp.count = other.count;
 		temp.block_capacity = other.block_capacity;
 		temp.entry_size = other.entry_size;
-		temp.blocks = move(other.blocks);
-		temp.pinned_blocks = move(other.pinned_blocks);
+		temp.blocks = std::move(other.blocks);
+		temp.pinned_blocks = std::move(other.pinned_blocks);
 	}
 	other.Clear();
 
@@ -131,10 +131,10 @@ void RowDataCollection::Merge(RowDataCollection &other) {
 	block_capacity = MaxValue(block_capacity, temp.block_capacity);
 	entry_size = MaxValue(entry_size, temp.entry_size);
 	for (auto &block : temp.blocks) {
-		blocks.emplace_back(move(block));
+		blocks.emplace_back(std::move(block));
 	}
 	for (auto &handle : temp.pinned_blocks) {
-		pinned_blocks.emplace_back(move(handle));
+		pinned_blocks.emplace_back(std::move(handle));
 	}
 }
 

package/src/duckdb/src/common/types/row_data_collection_scanner.cpp

@@ -56,7 +56,7 @@ void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block
 				auto heap_offset = heap_ptr - heap_handle.Ptr();
 				RowOperations::SwizzleHeapPointer(layout, data_ptr, heap_ptr, data_block->count, heap_offset);
 			} else {
-				swizzled_string_heap.pinned_blocks.emplace_back(move(heap_handle));
+				swizzled_string_heap.pinned_blocks.emplace_back(std::move(heap_handle));
 			}
 
 			// Update counter
@@ -103,7 +103,7 @@ void RowDataCollectionScanner::AlignHeapBlocks(RowDataCollection &swizzled_block
 			auto new_heap_ptr = new_heap_handle.Ptr();
 			if (swizzled_string_heap.keep_pinned) {
 				// Since the heap blocks are pinned, we can unswizzle the data again.
-				swizzled_string_heap.pinned_blocks.emplace_back(move(new_heap_handle));
+				swizzled_string_heap.pinned_blocks.emplace_back(std::move(new_heap_handle));
 				RowOperations::UnswizzlePointers(layout, base_row_ptr, new_heap_ptr, data_block->count);
 				RowOperations::UnswizzleHeapPointer(layout, base_row_ptr, new_heap_ptr, data_block->count);
 			}
@@ -271,4 +271,12 @@ void RowDataCollectionScanner::Scan(DataChunk &chunk) {
 	}
 }
 
+void RowDataCollectionScanner::Reset(bool flush_p) {
+	flush = flush_p;
+	total_scanned = 0;
+
+	read_state.block_idx = 0;
+	read_state.entry_idx = 0;
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/types/row_layout.cpp

@@ -18,7 +18,7 @@ RowLayout::RowLayout()
 
 void RowLayout::Initialize(vector<LogicalType> types_p, Aggregates aggregates_p, bool align) {
 	offsets.clear();
-	types = move(types_p);
+	types = std::move(types_p);
 
 	// Null mask at the front - 1 bit per value.
 	flag_width = ValidityBytes::ValidityMaskSize(types.size());
@@ -61,7 +61,7 @@ void RowLayout::Initialize(vector<LogicalType> types_p, Aggregates aggregates_p,
 	data_width = row_width - flag_width;
 
 	// Aggregate fields.
-	aggregates = move(aggregates_p);
+	aggregates = std::move(aggregates_p);
 	for (auto &aggregate : aggregates) {
 		offsets.push_back(row_width);
 		row_width += aggregate.payload_size;
@@ -80,11 +80,11 @@ void RowLayout::Initialize(vector<LogicalType> types_p, Aggregates aggregates_p,
 }
 
 void RowLayout::Initialize(vector<LogicalType> types_p, bool align) {
-	Initialize(move(types_p), Aggregates(), align);
+	Initialize(std::move(types_p), Aggregates(), align);
 }
 
 void RowLayout::Initialize(Aggregates aggregates_p, bool align) {
-	Initialize(vector<LogicalType>(), move(aggregates_p), align);
+	Initialize(vector<LogicalType>(), std::move(aggregates_p), align);
 }
 
 } // namespace duckdb

package/src/duckdb/src/common/types/string_type.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/common/types/string_type.hpp"
+#include "duckdb/common/types/bit.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/algorithm.hpp"
 #include "utf8proc_wrapper.hpp"

package/src/duckdb/src/common/types/time.cpp

@@ -1,14 +1,15 @@
-#include "duckdb/common/types/date.hpp"
 #include "duckdb/common/types/time.hpp"
-#include "duckdb/common/types/timestamp.hpp"
-#include "duckdb/common/types/interval.hpp"
-#include "duckdb/common/types/cast_helpers.hpp"
-#include "duckdb/common/string_util.hpp"
+
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types/cast_helpers.hpp"
+#include "duckdb/common/types/date.hpp"
+#include "duckdb/common/types/interval.hpp"
+#include "duckdb/common/types/timestamp.hpp"
 
+#include <cctype>
 #include <cstring>
 #include <sstream>
-#include <cctype>
 
 namespace duckdb {
 
@@ -115,6 +116,9 @@ bool Time::TryConvertTime(const char *buf, idx_t len, idx_t &pos, dtime_t &resul
 			// last chance, check if we can parse as timestamp
 			timestamp_t timestamp;
 			if (Timestamp::TryConvertTimestamp(buf, len, timestamp)) {
+				if (!Timestamp::IsFinite(timestamp)) {
+					return false;
+				}
 				result = Timestamp::GetTime(timestamp);
 				return true;
 			}

package/src/duckdb/src/common/types/validity_mask.cpp

@@ -24,7 +24,7 @@ void ValidityMask::Combine(const ValidityMask &other, idx_t count) {
 	}
 	// have to merge
 	// create a new validity mask that contains the combined mask
-	auto owned_data = move(validity_data);
+	auto owned_data = std::move(validity_data);
 	auto data = GetData();
 	auto other_data = other.GetData();
 
@@ -61,7 +61,7 @@ void ValidityMask::Resize(idx_t old_size, idx_t new_size) {
 		for (idx_t entry_idx = old_size_count; entry_idx < new_size_count; entry_idx++) {
 			new_owned_data[entry_idx] = ValidityData::MAX_ENTRY;
 		}
-		validity_data = move(new_validity_data);
+		validity_data = std::move(new_validity_data);
 		validity_mask = validity_data->owned_data.get();
 	} else {
 		Initialize(new_size);