duckdb 1.1.4-dev9.0 → 1.2.0

package/src/duckdb/src/function/window/window_segment_tree.cpp

@@ -79,17 +79,26 @@ public:
 
 	void Combine(WindowSegmentTreePart &other, idx_t count);
 
-	void Evaluate(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends, Vector &result,
-	              idx_t count, idx_t row_idx, FramePart frame_part);
+	void Evaluate(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends, const idx_t *bounds,
+	              Vector &result, idx_t count, idx_t row_idx, FramePart frame_part);
 
 protected:
 	//! Initialises the accumulation state vector (statef)
 	void Initialize(idx_t count);
 	//! Accumulate upper tree levels
 	void EvaluateUpperLevels(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
-	                         idx_t count, idx_t row_idx, FramePart frame_part);
-	void EvaluateLeaves(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends, idx_t count,
-	                    idx_t row_idx, FramePart frame_part, FramePart leaf_part);
+	                         const idx_t *bounds, idx_t count, idx_t row_idx, FramePart frame_part);
+	void EvaluateLeaves(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
+	                    const idx_t *bounds, idx_t count, idx_t row_idx, FramePart frame_part, FramePart leaf_part);
+
+	static inline const idx_t *FrameBegins(const idx_t *begins, const idx_t *ends, const idx_t *bounds,
+	                                       FramePart frame_part) {
+		return frame_part == FramePart::RIGHT ? bounds : begins;
+	}
+	static inline const idx_t *FrameEnds(const idx_t *begins, const idx_t *ends, const idx_t *bounds,
+	                                     FramePart frame_part) {
+		return frame_part == FramePart::LEFT ? bounds : ends;
+	}
 
 public:
 	//! Allocator for aggregates
@@ -400,8 +409,12 @@ void WindowSegmentTreeState::Evaluate(const WindowSegmentTreeGlobalState &gtstat
 	}
 
 	if (gtstate.aggregator.exclude_mode != WindowExcludeMode::NO_OTHER) {
+		// If we exclude the current row, then both left and right need to contain it.
+		const bool exclude_current = gtstate.aggregator.exclude_mode == WindowExcludeMode::CURRENT_ROW;
+
 		// 1. evaluate the tree left of the excluded part
-		part->Evaluate(gtstate, window_begin, peer_begin, result, count, row_idx, WindowSegmentTreePart::LEFT);
+		auto middle = exclude_current ? peer_end : peer_begin;
+		part->Evaluate(gtstate, window_begin, middle, window_end, result, count, row_idx, WindowSegmentTreePart::LEFT);
 
 		// 2. set up a second state for the right of the excluded part
 		if (!right_part) {
@@ -409,32 +422,35 @@ void WindowSegmentTreeState::Evaluate(const WindowSegmentTreeGlobalState &gtstat
 		}
 
 		// 3. evaluate the tree right of the excluded part
-		right_part->Evaluate(gtstate, peer_end, window_end, result, count, row_idx, WindowSegmentTreePart::RIGHT);
+		middle = exclude_current ? peer_begin : peer_end;
+		right_part->Evaluate(gtstate, middle, window_end, window_begin, result, count, row_idx,
+		                     WindowSegmentTreePart::RIGHT);
 
 		// 4. combine the buffer state into the Segment Tree State
 		part->Combine(*right_part, count);
 	} else {
-		part->Evaluate(gtstate, window_begin, window_end, result, count, row_idx, WindowSegmentTreePart::FULL);
+		part->Evaluate(gtstate, window_begin, window_end, nullptr, result, count, row_idx, WindowSegmentTreePart::FULL);
 	}
 
 	part->Finalize(result, count);
 }
 
 void WindowSegmentTreePart::Evaluate(const WindowSegmentTreeGlobalState &tree, const idx_t *begins, const idx_t *ends,
-                                     Vector &result, idx_t count, idx_t row_idx, FramePart frame_part) {
+                                     const idx_t *bounds, Vector &result, idx_t count, idx_t row_idx,
+                                     FramePart frame_part) {
 	Initialize(count);
 
 	if (order_insensitive) {
 		//	First pass: aggregate the segment tree nodes with sharing
-		EvaluateUpperLevels(tree, begins, ends, count, row_idx, frame_part);
+		EvaluateUpperLevels(tree, begins, ends, bounds, count, row_idx, frame_part);
 
 		//	Second pass: aggregate the ragged leaves
-		EvaluateLeaves(tree, begins, ends, count, row_idx, frame_part, FramePart::FULL);
+		EvaluateLeaves(tree, begins, ends, bounds, count, row_idx, frame_part, FramePart::FULL);
 	} else {
 		//	Evaluate leaves in order
-		EvaluateLeaves(tree, begins, ends, count, row_idx, frame_part, FramePart::LEFT);
-		EvaluateUpperLevels(tree, begins, ends, count, row_idx, frame_part);
-		EvaluateLeaves(tree, begins, ends, count, row_idx, frame_part, FramePart::RIGHT);
+		EvaluateLeaves(tree, begins, ends, bounds, count, row_idx, frame_part, FramePart::LEFT);
+		EvaluateUpperLevels(tree, begins, ends, bounds, count, row_idx, frame_part);
+		EvaluateLeaves(tree, begins, ends, bounds, count, row_idx, frame_part, FramePart::RIGHT);
 	}
 }
 
@@ -447,13 +463,18 @@ void WindowSegmentTreePart::Initialize(idx_t count) {
 }
 
 void WindowSegmentTreePart::EvaluateUpperLevels(const WindowSegmentTreeGlobalState &tree, const idx_t *begins,
-                                                const idx_t *ends, idx_t count, idx_t row_idx, FramePart frame_part) {
+                                                const idx_t *ends, const idx_t *bounds, idx_t count, idx_t row_idx,
+                                                FramePart frame_part) {
 	auto fdata = FlatVector::GetData<data_ptr_t>(statef);
 
 	const auto exclude_mode = tree.tree.exclude_mode;
 	const bool begin_on_curr_row = frame_part == FramePart::RIGHT && exclude_mode == WindowExcludeMode::CURRENT_ROW;
 	const bool end_on_curr_row = frame_part == FramePart::LEFT && exclude_mode == WindowExcludeMode::CURRENT_ROW;
 
+	// We need the full range of the frame to clamp
+	auto frame_begins = FrameBegins(begins, ends, bounds, frame_part);
+	auto frame_ends = FrameEnds(begins, ends, bounds, frame_part);
+
 	const auto max_level = tree.levels_flat_start.size() + 1;
 	right_stack.resize(max_level, {0, 0});
 
@@ -467,8 +488,8 @@ void WindowSegmentTreePart::EvaluateUpperLevels(const WindowSegmentTreeGlobalSta
 	for (idx_t rid = 0, cur_row = row_idx; rid < count; ++rid, ++cur_row) {
 		auto state_ptr = fdata[rid];
 
-		auto begin = begin_on_curr_row ? cur_row + 1 : begins[rid];
-		auto end = end_on_curr_row ? cur_row : ends[rid];
+		auto begin = MaxValue(begin_on_curr_row ? cur_row + 1 : begins[rid], frame_begins[rid]);
+		auto end = MinValue(end_on_curr_row ? cur_row : ends[rid], frame_ends[rid]);
 		if (begin >= end) {
 			continue;
 		}
@@ -541,8 +562,8 @@ void WindowSegmentTreePart::EvaluateUpperLevels(const WindowSegmentTreeGlobalSta
 }
 
 void WindowSegmentTreePart::EvaluateLeaves(const WindowSegmentTreeGlobalState &tree, const idx_t *begins,
-                                           const idx_t *ends, idx_t count, idx_t row_idx, FramePart frame_part,
-                                           FramePart leaf_part) {
+                                           const idx_t *ends, const idx_t *bounds, idx_t count, idx_t row_idx,
+                                           FramePart frame_part, FramePart leaf_part) {
 
 	auto fdata = FlatVector::GetData<data_ptr_t>(statef);
 
@@ -558,12 +579,19 @@ void WindowSegmentTreePart::EvaluateLeaves(const WindowSegmentTreeGlobalState &t
 	// current row
 	const bool add_curr_row = compute_left && frame_part == FramePart::RIGHT && exclude_mode == WindowExcludeMode::TIES;
 
+	// We need the full range of the frame to clamp
+	auto frame_begins = FrameBegins(begins, ends, bounds, frame_part);
+	auto frame_ends = FrameEnds(begins, ends, bounds, frame_part);
+
 	for (idx_t rid = 0, cur_row = row_idx; rid < count; ++rid, ++cur_row) {
 		auto state_ptr = fdata[rid];
 
-		const auto begin = begin_on_curr_row ? cur_row + 1 : begins[rid];
-		const auto end = end_on_curr_row ? cur_row : ends[rid];
-		if (add_curr_row) {
+		const auto frame_begin = frame_begins[rid];
+		auto begin = MaxValue(begin_on_curr_row ? cur_row + 1 : begins[rid], frame_begin);
+
+		const auto frame_end = frame_ends[rid];
+		auto end = MinValue(end_on_curr_row ? cur_row : ends[rid], frame_end);
+		if (add_curr_row && frame_begin <= cur_row && cur_row < frame_end) {
 			WindowSegmentValue(tree, 0, cur_row, cur_row + 1, state_ptr);
 		}
 		if (begin >= end) {

package/src/duckdb/src/function/window/window_token_tree.cpp

@@ -122,13 +122,14 @@ idx_t WindowTokenTree::Rank(const idx_t lower, const idx_t upper, const idx_t ro
 
 template <typename TREE>
 static idx_t NextPeer(const TREE &tree, const idx_t lower, const idx_t upper, const idx_t row_idx) {
-	idx_t rank = 0;
+	// We return an index, not a relative position
+	idx_t idx = lower;
 	// Because tokens are dense, we can find the next peer by adding 1 to the probed token value
 	const auto needle = tree.LowestLevel()[row_idx] + 1;
 	tree.AggregateLowerBound(lower, upper, needle, [&](idx_t level, const idx_t run_begin, const idx_t run_pos) {
-		rank += run_pos - run_begin;
+		idx += run_pos - run_begin;
 	});
-	return rank;
+	return idx;
 }
 
 idx_t WindowTokenTree::PeerEnd(const idx_t lower, const idx_t upper, const idx_t row_idx) const {

package/src/duckdb/src/include/duckdb/catalog/catalog.hpp

@@ -315,7 +315,11 @@ public:
 		return CatalogLookupBehavior::STANDARD;
 	}
 
+	//! Returns the default schema of the catalog
+	virtual string GetDefaultSchema() const;
+
 	//! The default table is used for `SELECT * FROM <catalog_name>;`
+	//! FIXME: these should be virtual methods
 	DUCKDB_API bool HasDefaultTable() const;
 	DUCKDB_API void SetDefaultTable(const string &schema, const string &name);
 	DUCKDB_API string GetDefaultTable() const;

package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp

@@ -53,7 +53,9 @@ public:
 		return set_paths;
 	}
 	DUCKDB_API const CatalogSearchEntry &GetDefault();
+	//! FIXME: this method is deprecated
 	DUCKDB_API string GetDefaultSchema(const string &catalog);
+	DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog);
 	DUCKDB_API string GetDefaultCatalog(const string &schema);
 
 	DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog);

package/src/duckdb/src/include/duckdb/catalog/dependency_list.hpp

@@ -31,6 +31,7 @@ public:
 public:
 	explicit LogicalDependency(CatalogEntry &entry);
 	LogicalDependency();
+	LogicalDependency(optional_ptr<Catalog> catalog, CatalogEntryInfo entry, string catalog_str);
 	bool operator==(const LogicalDependency &other) const;
 
 public:

package/src/duckdb/src/include/duckdb/common/arrow/arrow_type_extension.hpp

@@ -65,11 +65,13 @@ typedef void (*populate_arrow_schema_t)(DuckDBArrowSchemaHolder &root_holder, Ar
                                         const LogicalType &type, ClientContext &context,
                                         const ArrowTypeExtension &extension);
 
-typedef shared_ptr<ArrowType> (*get_type_t)(const ArrowSchema &schema, const ArrowSchemaMetadata &schema_metadata);
+typedef unique_ptr<ArrowType> (*get_type_t)(const ArrowSchema &schema, const ArrowSchemaMetadata &schema_metadata);
 
 class ArrowTypeExtension {
 public:
 	ArrowTypeExtension() {};
+	//! This type is not registered, so we just use whatever is the format and hope for the best
+	explicit ArrowTypeExtension(ArrowExtensionMetadata &extension_metadata, unique_ptr<ArrowType> type);
 	//! We either have simple extensions where we only return one type
 	ArrowTypeExtension(string extension_name, string arrow_format, shared_ptr<ArrowTypeExtensionData> type);
 	ArrowTypeExtension(string vendor_name, string type_name, string arrow_format,
@@ -84,7 +86,7 @@ public:
 
 	ArrowExtensionMetadata GetInfo() const;
 
-	shared_ptr<ArrowType> GetType(const ArrowSchema &schema, const ArrowSchemaMetadata &schema_metadata) const;
+	unique_ptr<ArrowType> GetType(const ArrowSchema &schema, const ArrowSchemaMetadata &schema_metadata) const;
 
 	shared_ptr<ArrowTypeExtensionData> GetTypeExtension() const;
 

package/src/duckdb/src/include/duckdb/common/enum_util.hpp

@@ -32,8 +32,6 @@ struct EnumUtil {
     static string ToString(T value) { return string(ToChars<T>(value)); }
 };
 
-enum class ARTAppendMode : uint8_t;
-
 enum class ARTConflictType : uint8_t;
 
 enum class AccessMode : uint8_t;
@@ -188,6 +186,8 @@ enum class GateStatus : uint8_t;
 
 enum class HLLStorageType : uint8_t;
 
+enum class IndexAppendMode : uint8_t;
+
 enum class IndexConstraintType : uint8_t;
 
 enum class InsertColumnOrder : uint8_t;
@@ -391,9 +391,6 @@ enum class WindowBoundary : uint8_t;
 enum class WindowExcludeMode : uint8_t;
 
 
-template<>
-const char* EnumUtil::ToChars<ARTAppendMode>(ARTAppendMode value);
-
 template<>
 const char* EnumUtil::ToChars<ARTConflictType>(ARTConflictType value);
 
@@ -625,6 +622,9 @@ const char* EnumUtil::ToChars<GateStatus>(GateStatus value);
 template<>
 const char* EnumUtil::ToChars<HLLStorageType>(HLLStorageType value);
 
+template<>
+const char* EnumUtil::ToChars<IndexAppendMode>(IndexAppendMode value);
+
 template<>
 const char* EnumUtil::ToChars<IndexConstraintType>(IndexConstraintType value);
 
@@ -929,9 +929,6 @@ template<>
 const char* EnumUtil::ToChars<WindowExcludeMode>(WindowExcludeMode value);
 
 
-template<>
-ARTAppendMode EnumUtil::FromString<ARTAppendMode>(const char *value);
-
 template<>
 ARTConflictType EnumUtil::FromString<ARTConflictType>(const char *value);
 
@@ -1163,6 +1160,9 @@ GateStatus EnumUtil::FromString<GateStatus>(const char *value);
 template<>
 HLLStorageType EnumUtil::FromString<HLLStorageType>(const char *value);
 
+template<>
+IndexAppendMode EnumUtil::FromString<IndexAppendMode>(const char *value);
+
 template<>
 IndexConstraintType EnumUtil::FromString<IndexConstraintType>(const char *value);
 

package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp

@@ -47,7 +47,6 @@ public:
 	      identifier(other.identifier) {
 	}
 
-	MultiFileReaderColumnDefinition(MultiFileReaderColumnDefinition &&other) noexcept = default;
 	MultiFileReaderColumnDefinition &operator=(const MultiFileReaderColumnDefinition &other) {
 		if (this != &other) {
 			name = other.name;
@@ -58,7 +57,6 @@ public:
 		}
 		return *this;
 	}
-	MultiFileReaderColumnDefinition &operator=(MultiFileReaderColumnDefinition &&other) noexcept = default;
 
 public:
 	static vector<MultiFileReaderColumnDefinition> ColumnsFromNamesAndTypes(const vector<string> &names,

package/src/duckdb/src/include/duckdb/common/serializer/deserializer.hpp

@@ -81,7 +81,7 @@ public:
 	}
 
 	template <typename T>
-	inline T ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T &&default_value) {
+	inline T ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T default_value) {
 		if (!OnOptionalPropertyBegin(field_id, tag)) {
 			OnOptionalPropertyEnd(false);
 			return std::forward<T>(default_value);
@@ -104,7 +104,7 @@ public:
 	}
 
 	template <typename T>
-	inline void ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T &ret, T &&default_value) {
+	inline void ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, T &ret, T default_value) {
 		if (!OnOptionalPropertyBegin(field_id, tag)) {
 			ret = std::forward<T>(default_value);
 			OnOptionalPropertyEnd(false);
@@ -116,7 +116,7 @@ public:
 
 	template <typename T>
 	inline void ReadPropertyWithExplicitDefault(const field_id_t field_id, const char *tag, CSVOption<T> &ret,
-	                                            T &&default_value) {
+	                                            T default_value) {
 		if (!OnOptionalPropertyBegin(field_id, tag)) {
 			ret = std::forward<T>(default_value);
 			OnOptionalPropertyEnd(false);
@@ -159,6 +159,11 @@ public:
 		return data.Get<T>();
 	}
 
+	template <class T>
+	optional_ptr<T> TryGet() {
+		return data.TryGet<T>();
+	}
+
 	//! Unset a serialization property
 	template <class T>
 	void Unset() {

package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp

@@ -13,17 +13,22 @@
 #include "duckdb/common/typedefs.hpp"
 
 namespace duckdb {
+class Allocator;
 
 class MemoryStream : public WriteStream, public ReadStream {
 private:
+	optional_ptr<Allocator> allocator;
 	idx_t position;
 	idx_t capacity;
-	bool owns_data;
 	data_ptr_t data;
 
 public:
 	static constexpr idx_t DEFAULT_INITIAL_CAPACITY = 512;
 
+	// Create a new owning MemoryStream with an internal  backing buffer with the specified capacity. The stream will
+	// own the backing buffer, resize it when needed and free its memory when the stream is destroyed
+	explicit MemoryStream(Allocator &allocator, idx_t capacity = DEFAULT_INITIAL_CAPACITY);
+
 	// Create a new owning MemoryStream with an internal  backing buffer with the specified capacity. The stream will
 	// own the backing buffer, resize it when needed and free its memory when the stream is destroyed
 	explicit MemoryStream(idx_t capacity = DEFAULT_INITIAL_CAPACITY);

package/src/duckdb/src/include/duckdb/common/serializer/serialization_data.hpp

@@ -30,6 +30,7 @@ struct SerializationData {
 
 	stack<reference<ClientContext>> contexts;
 	stack<reference<DatabaseInstance>> databases;
+	stack<reference<Catalog>> catalogs;
 	stack<idx_t> enums;
 	stack<reference<bound_parameter_map_t>> parameter_data;
 	stack<const_reference<LogicalType>> types;
@@ -42,6 +43,9 @@ struct SerializationData {
 	template <class T>
 	T Get() = delete;
 
+	template <class T>
+	optional_ptr<T> TryGet() = delete;
+
 	template <class T>
 	void Unset() = delete;
 
@@ -163,6 +167,27 @@ inline void SerializationData::Unset<ClientContext>() {
 	contexts.pop();
 }
 
+template <>
+inline void SerializationData::Set(Catalog &catalog) {
+	catalogs.emplace(catalog);
+}
+
+template <>
+inline Catalog &SerializationData::Get() {
+	AssertNotEmpty(catalogs);
+	return catalogs.top();
+}
+
+template <>
+inline optional_ptr<Catalog> SerializationData::TryGet() {
+	return catalogs.empty() ? nullptr : &catalogs.top().get();
+}
+
+template <>
+inline void SerializationData::Unset<Catalog>() {
+	AssertNotEmpty(catalogs);
+	catalogs.pop();
+}
 template <>
 inline void SerializationData::Set(DatabaseInstance &db) {
 	databases.emplace(db);

package/src/duckdb/src/include/duckdb/common/serializer/serializer.hpp

@@ -28,6 +28,9 @@ namespace duckdb {
 
 class SerializationOptions {
 public:
+	SerializationOptions() = default;
+	explicit SerializationOptions(AttachedDatabase &db);
+
 	bool serialize_enum_as_string = false;
 	bool serialize_default_values = false;
 	SerializationCompatibility serialization_compatibility = SerializationCompatibility::Default();
@@ -68,6 +71,9 @@ public:
 	};
 
 public:
+	SerializationOptions GetOptions() {
+		return options;
+	}
 	SerializationData &GetSerializationData() {
 		return data;
 	}
@@ -99,7 +105,7 @@ public:
 	}
 
 	template <class T>
-	void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const T &value, const T &&default_value) {
+	void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const T &value, const T &default_value) {
 		// If current value is default, don't write it
 		if (!options.serialize_default_values && (value == default_value)) {
 			OnOptionalPropertyBegin(field_id, tag, false);
@@ -114,7 +120,7 @@ public:
 	// Specialization for Value (default Value comparison throws when comparing nulls)
 	template <class T>
 	void WritePropertyWithDefault(const field_id_t field_id, const char *tag, const CSVOption<T> &value,
-	                              const T &&default_value) {
+	                              const T &default_value) {
 		// If current value is default, don't write it
 		if (!options.serialize_default_values && (value == default_value)) {
 			OnOptionalPropertyBegin(field_id, tag, false);
@@ -377,7 +383,7 @@ void Serializer::WriteValue(const vector<bool> &vec);
 // Specialization for Value (default Value comparison throws when comparing nulls)
 template <>
 void Serializer::WritePropertyWithDefault<Value>(const field_id_t field_id, const char *tag, const Value &value,
-                                                 const Value &&default_value);
+                                                 const Value &default_value);
 
 // List Impl
 template <class FUNC>

package/src/duckdb/src/include/duckdb/common/types/selection_vector.hpp

@@ -32,7 +32,7 @@ struct SelectionVector {
 		Initialize(count);
 	}
 	SelectionVector(idx_t start, idx_t count) {
-		Initialize(STANDARD_VECTOR_SIZE);
+		Initialize(MaxValue<idx_t>(count, STANDARD_VECTOR_SIZE));
 		for (idx_t i = 0; i < count; i++) {
 			set_index(i, start + i);
 		}

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -16,7 +16,6 @@ namespace duckdb {
 
 enum class VerifyExistenceType : uint8_t { APPEND = 0, APPEND_FK = 1, DELETE_FK = 2 };
 enum class ARTConflictType : uint8_t { NO_CONFLICT = 0, CONSTRAINT = 1, TRANSACTION = 2 };
-enum class ARTAppendMode : uint8_t { DEFAULT = 0, IGNORE_DUPLICATES = 1, INSERT_DUPLICATES = 2 };
 
 class ConflictManager;
 class ARTKey;
@@ -62,8 +61,6 @@ public:
 	bool owns_data;
 	//! The number of bytes fitting in the prefix.
 	uint8_t prefix_count;
-	//! The append mode.
-	ARTAppendMode append_mode;
 
 public:
 	//! Try to initialize a scan on the ART with the given expression and filter.
@@ -74,21 +71,19 @@ public:
 
 	//! Appends data to the locked index.
 	ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids) override;
-	//! Appends data to the locked index and verifies constraint violations against a delete index.
-	ErrorData AppendWithDeleteIndex(IndexLock &l, DataChunk &chunk, Vector &row_ids,
-	                                optional_ptr<BoundIndex> delete_index) override;
+	//! Appends data to the locked index and verifies constraint violations.
+	ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info) override;
 
 	//! Internally inserts a chunk.
 	ARTConflictType Insert(Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id, const GateStatus status,
-	                       optional_ptr<ART> delete_art);
+	                       optional_ptr<ART> delete_art, const IndexAppendMode append_mode);
 	//! Insert a chunk.
 	ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids) override;
-	//! Insert a chunk and verifies constraint violations against a delete index.
-	ErrorData Insert(IndexLock &l, DataChunk &data, Vector &row_ids, optional_ptr<BoundIndex> delete_index) override;
+	//! Insert a chunk and verifies constraint violations.
+	ErrorData Insert(IndexLock &l, DataChunk &data, Vector &row_ids, IndexAppendInfo &info) override;
 
 	//! Verify that data can be appended to the index without a constraint violation.
-	void VerifyAppend(DataChunk &chunk, optional_ptr<BoundIndex> delete_index,
-	                  optional_ptr<ConflictManager> manager) override;
+	void VerifyAppend(DataChunk &chunk, IndexAppendInfo &info, optional_ptr<ConflictManager> manager) override;
 
 	//! Delete a chunk from the ART.
 	void Delete(IndexLock &lock, DataChunk &entries, Vector &row_ids) override;
@@ -131,15 +126,17 @@ private:
 	void InsertIntoEmpty(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
 	                     const GateStatus status);
 	ARTConflictType InsertIntoInlined(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
-	                                  const GateStatus status, optional_ptr<ART> delete_art);
+	                                  const GateStatus status, optional_ptr<ART> delete_art,
+	                                  const IndexAppendMode append_mode);
 	ARTConflictType InsertIntoNode(Node &node, const ARTKey &key, const idx_t depth, const ARTKey &row_id,
-	                               const GateStatus status, optional_ptr<ART> delete_art);
+	                               const GateStatus status, optional_ptr<ART> delete_art,
+	                               const IndexAppendMode append_mode);
 
 	string GenerateErrorKeyName(DataChunk &input, idx_t row);
 	string GenerateConstraintErrorMessage(VerifyExistenceType verify_type, const string &key_name);
 	void VerifyLeaf(const Node &leaf, const ARTKey &key, optional_ptr<ART> delete_art, ConflictManager &manager,
 	                optional_idx &conflict_idx, idx_t i);
-	void VerifyConstraint(DataChunk &chunk, optional_ptr<BoundIndex> delete_index, ConflictManager &manager) override;
+	void VerifyConstraint(DataChunk &chunk, IndexAppendInfo &info, ConflictManager &manager) override;
 	string GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index,
 	                                     DataChunk &input) override;
 

package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp

@@ -41,7 +41,7 @@ public:
 		return art.prefix_count;
 	}
 	static idx_t GetMismatchWithOther(const Prefix &l_prefix, const Prefix &r_prefix, const idx_t max_count);
-	static idx_t GetMismatchWithKey(ART &art, const Node &node, const ARTKey &key, idx_t &depth);
+	static optional_idx GetMismatchWithKey(ART &art, const Node &node, const ARTKey &key, idx_t &depth);
 	static uint8_t GetByte(const ART &art, const Node &node, const uint8_t pos);
 
 public:
@@ -64,8 +64,8 @@ public:
 	//! 1. a non-prefix node.
 	//! 2. a mismatching byte.
 	//! Early-out, if the next prefix is a gate node.
-	static idx_t Traverse(ART &art, reference<const Node> &node, const ARTKey &key, idx_t &depth);
-	static idx_t TraverseMutable(ART &art, reference<Node> &node, const ARTKey &key, idx_t &depth);
+	static optional_idx Traverse(ART &art, reference<const Node> &node, const ARTKey &key, idx_t &depth);
+	static optional_idx TraverseMutable(ART &art, reference<Node> &node, const ARTKey &key, idx_t &depth);
 
 	//! Traverse two prefixes to find
 	//! 1. that they match.
@@ -85,7 +85,8 @@ public:
 
 	//! Insert a key into a prefix.
 	static ARTConflictType Insert(ART &art, Node &node, const ARTKey &key, idx_t depth, const ARTKey &row_id,
-	                              const GateStatus status, optional_ptr<ART> delete_art);
+	                              const GateStatus status, optional_ptr<ART> delete_art,
+	                              const IndexAppendMode append_mode);
 
 	//! Returns the string representation of the node, or only traverses and verifies the node and its subtree
 	static string VerifyAndToString(ART &art, const Node &node, const bool only_verify);

package/src/duckdb/src/include/duckdb/execution/index/bound_index.hpp

@@ -28,6 +28,19 @@ class ConflictManager;
 struct IndexLock;
 struct IndexScanState;
 
+enum class IndexAppendMode : uint8_t { DEFAULT = 0, IGNORE_DUPLICATES = 1, INSERT_DUPLICATES = 2 };
+
+class IndexAppendInfo {
+public:
+	IndexAppendInfo() : append_mode(IndexAppendMode::DEFAULT), delete_index(nullptr) {};
+	IndexAppendInfo(const IndexAppendMode append_mode, const optional_ptr<BoundIndex> delete_index)
+	    : append_mode(append_mode), delete_index(delete_index) {};
+
+public:
+	IndexAppendMode append_mode;
+	optional_ptr<BoundIndex> delete_index;
+};
+
 //! The index is an abstract base class that serves as the basis for indexes
 class BoundIndex : public Index {
 public:
@@ -71,17 +84,15 @@ public:
 	virtual ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids) = 0;
 	//! Obtains a lock and calls Append while holding that lock.
 	ErrorData Append(DataChunk &chunk, Vector &row_ids);
-	//! Appends data to the locked index and verifies constraint violations against a delete index.
-	virtual ErrorData AppendWithDeleteIndex(IndexLock &l, DataChunk &chunk, Vector &row_ids,
-	                                        optional_ptr<BoundIndex> delete_index);
-	//! Obtains a lock and calls Append with an delete_index while holding that lock.
-	ErrorData AppendWithDeleteIndex(DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index);
+	//! Appends data to the locked index and verifies constraint violations.
+	virtual ErrorData Append(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info);
+	//! Obtains a lock and calls Append while holding that lock.
+	ErrorData Append(DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info);
 
 	//! Verify that data can be appended to the index without a constraint violation.
-	virtual void VerifyAppend(DataChunk &chunk, optional_ptr<BoundIndex> delete_index,
-	                          optional_ptr<ConflictManager> manager);
+	virtual void VerifyAppend(DataChunk &chunk, IndexAppendInfo &info, optional_ptr<ConflictManager> manager);
 	//! Verifies the constraint for a chunk of data.
-	virtual void VerifyConstraint(DataChunk &chunk, optional_ptr<BoundIndex> delete_index, ConflictManager &manager);
+	virtual void VerifyConstraint(DataChunk &chunk, IndexAppendInfo &info, ConflictManager &manager);
 
 	//! Deletes all data from the index. The lock obtained from InitializeLock must be held
 	virtual void CommitDrop(IndexLock &index_lock) = 0;
@@ -94,8 +105,8 @@ public:
 
 	//! Insert a chunk.
 	virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids) = 0;
-	//! Insert a chunk and verifies constraint violations against a delete index.
-	virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, optional_ptr<BoundIndex> delete_index);
+	//! Insert a chunk and verifies constraint violations.
+	virtual ErrorData Insert(IndexLock &l, DataChunk &chunk, Vector &row_ids, IndexAppendInfo &info);
 
 	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
 	//! index must also be locked during the merge

package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp

@@ -54,8 +54,9 @@ public:
 		D_ASSERT(ptr.GetOffset() < available_segments_per_buffer);
 		D_ASSERT(buffers.find(ptr.GetBufferId()) != buffers.end());
 
-		auto &buffer = buffers.find(ptr.GetBufferId())->second;
-		auto buffer_ptr = buffer.Get(dirty);
+		auto buffer_it = buffers.find(ptr.GetBufferId());
+		D_ASSERT(buffer_it != buffers.end());
+		auto buffer_ptr = buffer_it->second->Get(dirty);
 		return buffer_ptr + ptr.GetOffset() * segment_size + bitmask_offset;
 	}
 
@@ -71,11 +72,11 @@ public:
 		D_ASSERT(buffers.find(ptr.GetBufferId()) != buffers.end());
 
 		auto &buffer = buffers.find(ptr.GetBufferId())->second;
-		if (!buffer.InMemory()) {
+		if (!buffer->InMemory()) {
 			return nullptr;
 		}
 
-		auto buffer_ptr = buffer.Get();
+		auto buffer_ptr = buffer->Get();
 		auto raw_ptr = buffer_ptr + ptr.GetOffset() * segment_size + bitmask_offset;
 		return raw_ptr;
 	}
@@ -152,7 +153,7 @@ private:
 	idx_t total_segment_count;
 
 	//! Buffers containing the segments
-	unordered_map<idx_t, FixedSizeBuffer> buffers;
+	unordered_map<idx_t, unique_ptr<FixedSizeBuffer>> buffers;
 	//! Buffers with free space
 	unordered_set<idx_t> buffers_with_free_space;
 	//! Buffers qualifying for a vacuum (helper field to allow for fast NeedsVacuum checks)