duckdb 1.1.2-dev6.0 → 1.1.4-dev11.0

package/src/duckdb/src/include/duckdb/common/vector_operations/unary_executor.hpp

@@ -11,6 +11,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/common/enums/function_errors.hpp"
 
 #include <functional>
 
@@ -90,6 +91,7 @@ private:
 		}
 	}
 
+#ifndef DUCKDB_SMALLER_BINARY
 	template <class INPUT_TYPE, class RESULT_TYPE, class OPWRAPPER, class OP>
 	static inline void ExecuteFlat(const INPUT_TYPE *__restrict ldata, RESULT_TYPE *__restrict result_data, idx_t count,
 	                               ValidityMask &mask, ValidityMask &result_mask, void *dataptr, bool adds_nulls) {
@@ -135,9 +137,11 @@ private:
 			}
 		}
 	}
+#endif
 
 	template <class INPUT_TYPE, class RESULT_TYPE, class OPWRAPPER, class OP>
-	static inline void ExecuteStandard(Vector &input, Vector &result, idx_t count, void *dataptr, bool adds_nulls) {
+	static inline void ExecuteStandard(Vector &input, Vector &result, idx_t count, void *dataptr, bool adds_nulls,
+	                                   FunctionErrors errors = FunctionErrors::CAN_THROW_RUNTIME_ERROR) {
 		switch (input.GetVectorType()) {
 		case VectorType::CONSTANT_VECTOR: {
 			result.SetVectorType(VectorType::CONSTANT_VECTOR);
@@ -153,6 +157,7 @@ private:
 			}
 			break;
 		}
+#ifndef DUCKDB_SMALLER_BINARY
 		case VectorType::FLAT_VECTOR: {
 			result.SetVectorType(VectorType::FLAT_VECTOR);
 			auto result_data = FlatVector::GetData<RESULT_TYPE>(result);
@@ -162,6 +167,35 @@ private:
 			                                                    FlatVector::Validity(result), dataptr, adds_nulls);
 			break;
 		}
+		case VectorType::DICTIONARY_VECTOR: {
+			// dictionary vector - we can run the function ONLY on the dictionary in some cases
+			// we can only do this if the function does not throw errors
+			// we can execute the function on a value that is in the dictionary but that is not referenced
+			// if the function can throw errors - this will result in us (incorrectly) throwing an error
+			if (errors == FunctionErrors::CANNOT_ERROR) {
+				static constexpr idx_t DICTIONARY_THRESHOLD = 2;
+				auto dict_size = DictionaryVector::DictionarySize(input);
+				if (dict_size.IsValid() && dict_size.GetIndex() * DICTIONARY_THRESHOLD <= count) {
+					// we can operate directly on the dictionary if we have a dictionary size
+					// but this only makes sense if the dictionary size is smaller than the count by some factor
+					auto &dictionary_values = DictionaryVector::Child(input);
+					if (dictionary_values.GetVectorType() == VectorType::FLAT_VECTOR) {
+						// execute the function over the dictionary
+						auto result_data = FlatVector::GetData<RESULT_TYPE>(result);
+						auto ldata = FlatVector::GetData<INPUT_TYPE>(dictionary_values);
+						ExecuteFlat<INPUT_TYPE, RESULT_TYPE, OPWRAPPER, OP>(
+						    ldata, result_data, dict_size.GetIndex(), FlatVector::Validity(dictionary_values),
+						    FlatVector::Validity(result), dataptr, adds_nulls);
+						// slice the result with the original offsets
+						auto &offsets = DictionaryVector::SelVector(input);
+						result.Dictionary(result, dict_size.GetIndex(), offsets, count);
+						break;
+					}
+				}
+			}
+			DUCKDB_EXPLICIT_FALLTHROUGH;
+		}
+#endif
 		default: {
 			UnifiedVectorFormat vdata;
 			input.ToUnifiedFormat(count, vdata);
@@ -184,9 +218,10 @@ public:
 	}
 
 	template <class INPUT_TYPE, class RESULT_TYPE, class FUNC = std::function<RESULT_TYPE(INPUT_TYPE)>>
-	static void Execute(Vector &input, Vector &result, idx_t count, FUNC fun) {
-		ExecuteStandard<INPUT_TYPE, RESULT_TYPE, UnaryLambdaWrapper, FUNC>(input, result, count,
-		                                                                   reinterpret_cast<void *>(&fun), false);
+	static void Execute(Vector &input, Vector &result, idx_t count, FUNC fun,
+	                    FunctionErrors errors = FunctionErrors::CAN_THROW_RUNTIME_ERROR) {
+		ExecuteStandard<INPUT_TYPE, RESULT_TYPE, UnaryLambdaWrapper, FUNC>(
+		    input, result, count, reinterpret_cast<void *>(&fun), false, errors);
 	}
 
 	template <class INPUT_TYPE, class RESULT_TYPE, class OP>
@@ -206,6 +241,78 @@ public:
 		UnaryExecutor::GenericExecute<INPUT_TYPE, RESULT_TYPE, UnaryStringOperator<OP>>(input, result, count,
 		                                                                                (void *)&result);
 	}
+
+private:
+	// Select logic copied from TernaryExecutor, but with a lambda instead of a static functor
+	template <class INPUT_TYPE, class FUNC = std::function<bool(INPUT_TYPE)>, bool NO_NULL, bool HAS_TRUE_SEL,
+	          bool HAS_FALSE_SEL>
+	static inline idx_t SelectLoop(const INPUT_TYPE *__restrict input_data, const SelectionVector *result_sel,
+	                               const idx_t count, FUNC fun, const SelectionVector &input_sel,
+	                               ValidityMask &input_validity, SelectionVector *true_sel,
+	                               SelectionVector *false_sel) {
+		idx_t true_count = 0, false_count = 0;
+		for (idx_t i = 0; i < count; i++) {
+			const auto result_idx = result_sel->get_index(i);
+			const auto idx = input_sel.get_index(i);
+			const bool comparison_result = (NO_NULL || input_validity.RowIsValid(idx)) && fun(input_data[idx]);
+			if (HAS_TRUE_SEL) {
+				true_sel->set_index(true_count, result_idx);
+				true_count += comparison_result;
+			}
+			if (HAS_FALSE_SEL) {
+				false_sel->set_index(false_count, result_idx);
+				false_count += !comparison_result;
+			}
+		}
+		if (HAS_TRUE_SEL) {
+			return true_count;
+		} else {
+			return count - false_count;
+		}
+	}
+
+	template <class INPUT_TYPE, class FUNC = std::function<bool(INPUT_TYPE)>, bool NO_NULL>
+	static inline idx_t SelectLoopSelSwitch(UnifiedVectorFormat &input_data, const SelectionVector *sel,
+	                                        const idx_t count, FUNC fun, SelectionVector *true_sel,
+	                                        SelectionVector *false_sel) {
+		if (true_sel && false_sel) {
+			return SelectLoop<INPUT_TYPE, FUNC, NO_NULL, true, true>(
+			    UnifiedVectorFormat::GetData<INPUT_TYPE>(input_data), sel, count, fun, *input_data.sel,
+			    input_data.validity, true_sel, false_sel);
+		} else if (true_sel) {
+			return SelectLoop<INPUT_TYPE, FUNC, NO_NULL, true, false>(
+			    UnifiedVectorFormat::GetData<INPUT_TYPE>(input_data), sel, count, fun, *input_data.sel,
+			    input_data.validity, true_sel, false_sel);
+		} else {
+			D_ASSERT(false_sel);
+			return SelectLoop<INPUT_TYPE, FUNC, NO_NULL, false, true>(
+			    UnifiedVectorFormat::GetData<INPUT_TYPE>(input_data), sel, count, fun, *input_data.sel,
+			    input_data.validity, true_sel, false_sel);
+		}
+	}
+
+	template <class INPUT_TYPE, class FUNC = std::function<bool(INPUT_TYPE)>>
+	static inline idx_t SelectLoopSwitch(UnifiedVectorFormat &input_data, const SelectionVector *sel, const idx_t count,
+	                                     FUNC fun, SelectionVector *true_sel, SelectionVector *false_sel) {
+		if (!input_data.validity.AllValid()) {
+			return SelectLoopSelSwitch<INPUT_TYPE, FUNC, false>(input_data, sel, count, fun, true_sel, false_sel);
+		} else {
+			return SelectLoopSelSwitch<INPUT_TYPE, FUNC, true>(input_data, sel, count, fun, true_sel, false_sel);
+		}
+	}
+
+public:
+	template <class INPUT_TYPE, class FUNC = std::function<bool(INPUT_TYPE)>>
+	static idx_t Select(Vector &input, const SelectionVector *sel, const idx_t count, FUNC fun,
+	                    SelectionVector *true_sel, SelectionVector *false_sel) {
+		if (!sel) {
+			sel = FlatVector::IncrementalSelectionVector();
+		}
+		UnifiedVectorFormat input_data;
+		input.ToUnifiedFormat(count, input_data);
+
+		return SelectLoopSwitch<INPUT_TYPE, FUNC>(input_data, sel, count, fun, true_sel, false_sel);
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/vector_operations/vector_operations.hpp

@@ -129,7 +129,6 @@ struct VectorOperations {
 	                                    idx_t count, optional_ptr<SelectionVector> true_sel,
 	                                    optional_ptr<SelectionVector> false_sel,
 	                                    optional_ptr<ValidityMask> null_mask = nullptr);
-
 	// true := A > B with nulls being minimal
 	static idx_t DistinctGreaterThanNullsFirst(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel,
 	                                           idx_t count, optional_ptr<SelectionVector> true_sel,

package/src/duckdb/src/include/duckdb/execution/adaptive_filter.hpp

@@ -33,6 +33,8 @@ public:
 	void EndFilter(AdaptiveFilterState state);
 
 private:
+	bool disable_permutations = false;
+
 	//! used for adaptive expression reordering
 	idx_t iteration_count = 0;
 	idx_t swap_idx = 0;

package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp

@@ -43,7 +43,7 @@ public:
 
 public:
 	//! The hash table load factor, when a resize is triggered
-	constexpr static double LOAD_FACTOR = 1.5;
+	constexpr static double LOAD_FACTOR = 1.25;
 
 	//! Get the layout of this HT
 	const TupleDataLayout &GetLayout() const;
@@ -57,6 +57,7 @@ public:
 	idx_t Capacity() const;
 	//! Threshold at which to resize the HT
 	idx_t ResizeThreshold() const;
+	static idx_t ResizeThreshold(idx_t capacity);
 
 	//! Add the given data to the HT, computing the aggregates grouped by the
 	//! data in the group chunk. When resize = true, aggregates will not be
@@ -64,6 +65,9 @@ public:
 	idx_t AddChunk(DataChunk &groups, DataChunk &payload, const unsafe_vector<idx_t> &filter);
 	idx_t AddChunk(DataChunk &groups, Vector &group_hashes, DataChunk &payload, const unsafe_vector<idx_t> &filter);
 	idx_t AddChunk(DataChunk &groups, DataChunk &payload, AggregateType filter);
+	optional_idx TryAddCompressedGroups(DataChunk &groups, DataChunk &payload, const unsafe_vector<idx_t> &filter);
+	optional_idx TryAddDictionaryGroups(DataChunk &groups, DataChunk &payload, const unsafe_vector<idx_t> &filter);
+	optional_idx TryAddConstantGroups(DataChunk &groups, DataChunk &payload, const unsafe_vector<idx_t> &filter);
 
 	//! Fetch the aggregates for specific groups from the HT and place them in the result
 	void FetchAggregates(DataChunk &groups, DataChunk &result);
@@ -76,36 +80,53 @@ public:
 	idx_t FindOrCreateGroups(DataChunk &groups, Vector &addresses_out, SelectionVector &new_groups_out);
 	void FindOrCreateGroups(DataChunk &groups, Vector &addresses_out);
 
-	unique_ptr<PartitionedTupleData> &GetPartitionedData();
+	const PartitionedTupleData &GetPartitionedData() const;
+	unique_ptr<PartitionedTupleData> AcquirePartitionedData();
+	void Abandon();
+	void Repartition();
 	shared_ptr<ArenaAllocator> GetAggregateAllocator();
 
 	//! Resize the HT to the specified size. Must be larger than the current size.
 	void Resize(idx_t size);
 	//! Resets the pointer table of the HT to all 0's
 	void ClearPointerTable();
-	//! Resets the group count to 0
-	void ResetCount();
 	//! Set the radix bits for this HT
 	void SetRadixBits(idx_t radix_bits);
-	//! Initializes the PartitionedTupleData
-	void InitializePartitionedData();
+	//! Get the radix bits for this HT
+	idx_t GetRadixBits() const;
+	//! Get the total amount of data sunk into this HT
+	idx_t GetSinkCount() const;
+	//! Skips lookups from here on out
+	void SkipLookups();
 
 	//! Executes the filter(if any) and update the aggregates
 	void Combine(GroupedAggregateHashTable &other);
 	void Combine(TupleDataCollection &other_data, optional_ptr<atomic<double>> progress = nullptr);
 
-	//! Unpins the data blocks
-	void UnpinData();
-
 private:
 	//! Efficiently matches groups
 	RowMatcher row_matcher;
 
+	struct AggregateDictionaryState {
+		AggregateDictionaryState();
+
+		//! The current dictionary vector id (if any)
+		string dictionary_id;
+		DataChunk unique_values;
+		Vector hashes;
+		Vector new_dictionary_pointers;
+		SelectionVector unique_entries;
+		unique_ptr<Vector> dictionary_addresses;
+		unsafe_unique_array<bool> found_entry;
+		idx_t capacity = 0;
+	};
+
 	//! Append state
 	struct AggregateHTAppendState {
 		AggregateHTAppendState();
 
-		PartitionedTupleDataAppendState append_state;
+		PartitionedTupleDataAppendState partitioned_append_state;
+		PartitionedTupleDataAppendState unpartitioned_append_state;
 
 		Vector ht_offsets;
 		Vector hash_salts;
@@ -116,12 +137,16 @@ private:
 		Vector addresses;
 		unsafe_unique_array<UnifiedVectorFormat> group_data;
 		DataChunk group_chunk;
+		AggregateDictionaryState dict_state;
 	} state;
 
+	//! If we have this many or more radix bits, we use the unpartitioned data collection too
+	static constexpr idx_t UNPARTITIONED_RADIX_BITS_THRESHOLD = 3;
 	//! The number of radix bits to partition by
 	idx_t radix_bits;
 	//! The data of the HT
 	unique_ptr<PartitionedTupleData> partitioned_data;
+	unique_ptr<PartitionedTupleData> unpartitioned_data;
 
 	//! Predicates for matching groups (always ExpressionType::COMPARE_EQUAL)
 	vector<ExpressionType> predicates;
@@ -138,6 +163,11 @@ private:
 	//! Bitmask for getting relevant bits from the hashes to determine the position
 	hash_t bitmask;
 
+	//! How many tuples went into this HT (before de-duplication)
+	idx_t sink_count;
+	//! If true, we just append, skipping HT lookups
+	bool skip_lookups;
+
 	//! The active arena allocator used by the aggregates for their internal state
 	shared_ptr<ArenaAllocator> aggregate_allocator;
 	//! Owning arena allocators that this HT has data from
@@ -149,8 +179,16 @@ private:
 	//! Destroy the HT
 	void Destroy();
 
+	//! Initializes the PartitionedTupleData
+	void InitializePartitionedData();
+	//! Initializes the PartitionedTupleData that only has 1 partition
+	void InitializeUnpartitionedData();
 	//! Apply bitmask to get the entry in the HT
 	inline idx_t ApplyBitMask(hash_t hash) const;
+	//! Reinserts tuples (triggered by Resize)
+	void ReinsertTuples(PartitionedTupleData &data);
+
+	void UpdateAggregates(DataChunk &payload, const unsafe_vector<idx_t> &filter);
 
 	//! Does the actual group matching / creation
 	idx_t FindOrCreateGroupsInternal(DataChunk &groups, Vector &group_hashes, Vector &addresses,

package/src/duckdb/src/include/duckdb/execution/executor.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/common/reference_map.hpp"
 #include "duckdb/main/query_result.hpp"
 #include "duckdb/execution/task_error_manager.hpp"
+#include "duckdb/execution/progress_data.hpp"
 #include "duckdb/parallel/pipeline.hpp"
 
 #include <condition_variable>
@@ -86,7 +87,7 @@ public:
 	void AddToBeRescheduled(shared_ptr<Task> &task);
 
 	//! Returns the progress of the pipelines
-	bool GetPipelinesProgress(double &current_progress, uint64_t &current_cardinality, uint64_t &total_cardinality);
+	idx_t GetPipelinesProgress(ProgressData &progress);
 
 	void CompletePipeline() {
 		completed_pipelines++;

package/src/duckdb/src/include/duckdb/execution/expression_executor.hpp

@@ -21,7 +21,6 @@ class ExecutionContext;
 //! ExpressionExecutor is responsible for executing a set of expressions and storing the result in a data chunk
 class ExpressionExecutor {
 	friend class BoundIndex;
-	friend class CreateIndexLocalSinkState;
 
 public:
 	DUCKDB_API explicit ExpressionExecutor(ClientContext &context);

package/src/duckdb/src/include/duckdb/execution/ht_entry.hpp

@@ -13,6 +13,11 @@
 
 namespace duckdb {
 
+#if !defined(DISABLE_POINTER_SALT) && defined(__ANDROID__)
+// Google, why does Android need 18446744 TB of address space?
+#define DISABLE_POINTER_SALT
+#endif
+
 //! The ht_entry_t struct represents an individual entry within a hash table.
 /*!
     This struct is used by the JoinHashTable and AggregateHashTable to store entries within the hash table. It stores
@@ -21,31 +26,38 @@ namespace duckdb {
 */
 struct ht_entry_t { // NOLINT
 public:
-	//! Upper 16 bits are salt
+#ifdef DISABLE_POINTER_SALT
+	//! No salt, all pointer
+	static constexpr const hash_t SALT_MASK = 0x0000000000000000;
+	static constexpr const hash_t POINTER_MASK = 0xFFFFFFFFFFFFFFFF;
+#else
+	//! Upper 16 bits are salt, lower 48 bits are the pointer
 	static constexpr const hash_t SALT_MASK = 0xFFFF000000000000;
-	//! Lower 48 bits are the pointer
 	static constexpr const hash_t POINTER_MASK = 0x0000FFFFFFFFFFFF;
+#endif
 
-	explicit inline ht_entry_t(hash_t value_p) noexcept : value(value_p) {
+	ht_entry_t() noexcept : value(0) {
 	}
 
-	// Add a default constructor for 32-bit linux test case
-	ht_entry_t() noexcept : value(0) {
+	explicit ht_entry_t(hash_t value_p) noexcept : value(value_p) {
 	}
 
-	inline bool IsOccupied() const {
-		return value != 0;
+	ht_entry_t(const hash_t &salt, const data_ptr_t &pointer)
+	    : value(cast_pointer_to_uint64(pointer) | (salt & SALT_MASK)) {
 	}
 
-	// Returns a pointer based on the stored value without checking cell occupancy.
-	// This can return a nullptr if the cell is not occupied.
-	inline data_ptr_t GetPointerOrNull() const {
-		return cast_uint64_to_pointer(value & POINTER_MASK);
+	inline bool IsOccupied() const {
+		return value != 0;
 	}
 
-	// Returns a pointer based on the stored value if the cell is occupied
+	//! Returns a pointer based on the stored value (asserts if the cell is occupied)
 	inline data_ptr_t GetPointer() const {
 		D_ASSERT(IsOccupied());
+		return GetPointerOrNull();
+	}
+
+	//! Returns a pointer based on the stored value
+	inline data_ptr_t GetPointerOrNull() const {
 		return cast_uint64_to_pointer(value & POINTER_MASK);
 	}
 
@@ -59,15 +71,10 @@ public:
 	}
 
 	// Returns the salt, leaves upper salt bits intact, sets lower bits to all 1's
-	static inline hash_t ExtractSalt(hash_t hash) {
+	static inline hash_t ExtractSalt(const hash_t &hash) {
 		return hash | POINTER_MASK;
 	}
 
-	// Returns the salt, leaves upper salt bits intact, sets lower bits to all 0's
-	static inline hash_t ExtractSaltWithNulls(hash_t hash) {
-		return hash & SALT_MASK;
-	}
-
 	inline hash_t GetSalt() const {
 		return ExtractSalt(value);
 	}
@@ -81,15 +88,6 @@ public:
 		value = salt;
 	}
 
-	static inline ht_entry_t GetDesiredEntry(const data_ptr_t &pointer, const hash_t &salt) {
-		auto desired = cast_pointer_to_uint64(pointer) | (salt & SALT_MASK);
-		return ht_entry_t(desired);
-	}
-
-	static inline ht_entry_t GetEmptyEntry() {
-		return ht_entry_t(0);
-	}
-
 private:
 	hash_t value;
 };

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -14,14 +14,10 @@
 
 namespace duckdb {
 
-enum class VerifyExistenceType : uint8_t {
-	// Appends to a table.
-	APPEND = 0,
-	// Appends to a table that has a foreign key.
-	APPEND_FK = 1,
-	// Delete from a table that has a foreign key.
-	DELETE_FK = 2
-};
+enum class VerifyExistenceType : uint8_t { APPEND = 0, APPEND_FK = 1, DELETE_FK = 2 };
+enum class ARTConflictType : uint8_t { NO_CONFLICT = 0, CONSTRAINT = 1, TRANSACTION = 2 };
+enum class ARTAppendMode : uint8_t { DEFAULT = 0, IGNORE_DUPLICATES = 1, INSERT_DUPLICATES = 2 };
+
 class ConflictManager;
 class ARTKey;
 class ARTKeySection;
@@ -66,6 +62,8 @@ public:
 	bool owns_data;
 	//! The number of bytes fitting in the prefix.
 	uint8_t prefix_count;
+	//! The append mode.
+	ARTAppendMode append_mode;
 
 public:
 	//! Try to initialize a scan on the ART with the given expression and filter.
@@ -74,15 +72,23 @@ public:
 	//! If all row IDs were fetched, it return true, else false.
 	bool Scan(IndexScanState &state, idx_t max_count, unsafe_vector<row_t> &row_ids);
 
-	//! Append a chunk by first executing the ART's expressions.
-	ErrorData Append(IndexLock &lock, DataChunk &input, Vector &row_ids) override;
+	//! Appends data to the locked index.
+	ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids) override;
+	//! Appends data to the locked index and verifies constraint violations against a delete index.
+	ErrorData AppendWithDeleteIndex(IndexLock &l, DataChunk &chunk, Vector &row_ids,
+	                                optional_ptr<BoundIndex> delete_index) override;
+
+	//! Internally inserts a chunk.
+	ARTConflictType Insert(Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id, const GateStatus status,
+	                       optional_ptr<ART> delete_art);
 	//! Insert a chunk.
-	bool Insert(Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id, const GateStatus status);
-	ErrorData Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;
+	ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids) override;
+	//! Insert a chunk and verifies constraint violations against a delete index.
+	ErrorData Insert(IndexLock &l, DataChunk &data, Vector &row_ids, optional_ptr<BoundIndex> delete_index) override;
 
-	//! Constraint verification for a chunk.
-	void VerifyAppend(DataChunk &chunk) override;
-	void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) override;
+	//! Verify that data can be appended to the index without a constraint violation.
+	void VerifyAppend(DataChunk &chunk, optional_ptr<BoundIndex> delete_index,
+	                  optional_ptr<ConflictManager> manager) override;
 
 	//! Delete a chunk from the ART.
 	void Delete(IndexLock &lock, DataChunk &entries, Vector &row_ids) override;
@@ -124,12 +130,16 @@ private:
 
 	void InsertIntoEmpty(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
 	                     const GateStatus status);
-	bool InsertIntoNode(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
-	                    const GateStatus status);
+	ARTConflictType InsertIntoInlined(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
+	                                  const GateStatus status, optional_ptr<ART> delete_art);
+	ARTConflictType InsertIntoNode(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
+	                               const GateStatus status, optional_ptr<ART> delete_art);
 
 	string GenerateErrorKeyName(DataChunk &input, idx_t row);
 	string GenerateConstraintErrorMessage(VerifyExistenceType verify_type, const string &key_name);
-	void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) override;
+	void VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> delete_art, ConflictManager &manager,
+	                optional_idx &conflict_idx, idx_t i);
+	void VerifyConstraint(DataChunk &chunk, optional_ptr<BoundIndex> delete_index, ConflictManager &manager) override;
 	string GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index,
 	                                     DataChunk &input) override;
 

package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp

@@ -143,6 +143,7 @@ public:
 	inline void SetGateStatus(const GateStatus status) {
 		switch (status) {
 		case GateStatus::GATE_SET:
+			D_ASSERT(GetType() != NType::LEAF_INLINED);
 			SetMetadata(GetMetadata() | AND_GATE);
 			break;
 		case GateStatus::GATE_NOT_SET:

package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp

@@ -84,8 +84,8 @@ public:
 	static GateStatus Split(ART &art, reference<Node> &node, Node &child, const uint8_t pos);
 
 	//! Insert a key into a prefix.
-	static bool Insert(ART &art, Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id,
-	                   const GateStatus status);
+	static ARTConflictType Insert(ART &art, Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id,
+	                              const GateStatus status, optional_ptr<ART> delete_art);
 
 	//! Returns the string representation of the node, or only traverses and verifies the node and its subtree
 	static string VerifyAndToString(ART &art, const Node &node, const bool only_verify);

package/src/duckdb/src/include/duckdb/execution/index/bound_index.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/storage/index.hpp
+// duckdb/execution/index/bound_index.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -15,8 +15,8 @@
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/storage/table_storage_info.hpp"
 #include "duckdb/storage/index.hpp"
+#include "duckdb/storage/table_storage_info.hpp"
 
 namespace duckdb {
 
@@ -64,19 +64,24 @@ public:
 		return index_constraint_type;
 	}
 
-public: // Index interface
-	//! Obtain a lock on the index
+public:
+	//! Obtains a lock on the index.
 	void InitializeLock(IndexLock &state);
-	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
-	virtual ErrorData Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
-	//! Obtains a lock and calls Append while holding that lock
-	ErrorData Append(DataChunk &entries, Vector &row_identifiers);
-	//! Verify that data can be appended to the index without a constraint violation
-	virtual void VerifyAppend(DataChunk &chunk) = 0;
-	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
-	virtual void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) = 0;
-	//! Performs constraint checking for a chunk of input data
-	virtual void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) = 0;
+	//! Appends data to the locked index.
+	virtual ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids) = 0;
+	//! Obtains a lock and calls Append while holding that lock.
+	ErrorData Append(DataChunk &chunk, Vector &row_ids);
+	//! Appends data to the locked index and verifies constraint violations against a delete index.
+	virtual ErrorData AppendWithDeleteIndex(IndexLock &l, DataChunk &chunk, Vector &row_ids,
+	                                        optional_ptr<BoundIndex> delete_index);
+	//! Obtains a lock and calls Append with an delete_index while holding that lock.
+	ErrorData AppendWithDeleteIndex(DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index);
+
+	//! Verify that data can be appended to the index without a constraint violation.
+	virtual void VerifyAppend(DataChunk &chunk, optional_ptr<BoundIndex> delete_index,
+	                          optional_ptr<ConflictManager> manager);
+	//! Verifies the constraint for a chunk of data.
+	virtual void VerifyConstraint(DataChunk &chunk, optional_ptr<BoundIndex> delete_index, ConflictManager &manager);
 
 	//! Deletes all data from the index. The lock obtained from InitializeLock must be held
 	virtual void CommitDrop(IndexLock &index_lock) = 0;
@@ -87,8 +92,10 @@ public: // Index interface
 	//! Obtains a lock and calls Delete while holding that lock
 	void Delete(DataChunk &entries, Vector &row_identifiers);
 
-	//! Insert a chunk of entries into the index
-	virtual ErrorData Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
+	//! Insert a chunk.
+	virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids) = 0;
+	//! Insert a chunk and verifies constraint violations against a delete index.
+	virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index);
 
 	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
 	//! index must also be locked during the merge

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp

@@ -94,6 +94,10 @@ public:
 		return total_segment_count;
 	}
 
+	inline idx_t GetMaxSegmentsPerBuffer() const {
+		return available_segments_per_buffer;
+	}
+
 	//! Returns the upper bound of the available buffer IDs, i.e., upper_bound > max_buffer_id
 	idx_t GetUpperBoundBufferId() const;
 	//! Merge another FixedSizeAllocator into this allocator. Both must have the same segment size

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp

@@ -89,7 +89,7 @@ public:
 	//! Pin a buffer (if not in-memory)
 	void Pin();
 	//! Returns the first free offset in a bitmask
-	uint32_t GetOffset(const idx_t bitmask_count);
+	uint32_t GetOffset(const idx_t bitmask_count, const idx_t available_segments);
 	//! Sets the allocation size, if dirty
 	void SetAllocationSize(const idx_t available_segments, const idx_t segment_size, const idx_t bitmask_offset);
 
@@ -102,7 +102,7 @@ private:
 private:
 	//! Sets all uninitialized regions of a buffer in the respective partial block allocation
 	void SetUninitializedRegions(PartialBlockForIndex &p_block_for_index, const idx_t segment_size, const idx_t offset,
-	                             const idx_t bitmask_offset);
+	                             const idx_t bitmask_offset, const idx_t available_segments);
 };
 
 } // namespace duckdb