duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/execution/aggregate_hashtable.cpp

@@ -482,7 +482,7 @@ struct FlushMoveState {
 	bool Scan() {
 		if (collection.Scan(scan_state, groups)) {
 			collection.Gather(scan_state.chunk_state.row_locations, *FlatVector::IncrementalSelectionVector(),
-			                  groups.size(), hash_col_idx, hashes, *FlatVector::IncrementalSelectionVector());
+			                  groups.size(), hash_col_idx, hashes, *FlatVector::IncrementalSelectionVector(), nullptr);
 			return true;
 		}
 
@@ -512,7 +512,7 @@ void GroupedAggregateHashTable::Combine(GroupedAggregateHashTable &other) {
 	}
 }
 
-void GroupedAggregateHashTable::Combine(TupleDataCollection &other_data) {
+void GroupedAggregateHashTable::Combine(TupleDataCollection &other_data, optional_ptr<atomic<double>> progress) {
 	D_ASSERT(other_data.GetLayout().GetAggrWidth() == layout.GetAggrWidth());
 	D_ASSERT(other_data.GetLayout().GetDataWidth() == layout.GetDataWidth());
 	D_ASSERT(other_data.GetLayout().GetRowWidth() == layout.GetRowWidth());
@@ -523,6 +523,9 @@ void GroupedAggregateHashTable::Combine(TupleDataCollection &other_data) {
 
 	FlushMoveState fm_state(other_data);
 	RowOperationsState row_state(*aggregate_allocator);
+
+	idx_t chunk_idx = 0;
+	const auto chunk_count = other_data.ChunkCount();
 	while (fm_state.Scan()) {
 		FindOrCreateGroups(fm_state.groups, fm_state.hashes, fm_state.group_addresses, fm_state.new_groups_sel);
 		RowOperations::CombineStates(row_state, layout, fm_state.scan_state.chunk_state.row_locations,
@@ -531,6 +534,10 @@ void GroupedAggregateHashTable::Combine(TupleDataCollection &other_data) {
 			RowOperations::DestroyStates(row_state, layout, fm_state.scan_state.chunk_state.row_locations,
 			                             fm_state.groups.size());
 		}
+
+		if (progress) {
+			*progress = double(++chunk_idx) / double(chunk_count);
+		}
 	}
 
 	Verify();

package/src/duckdb/src/execution/column_binding_resolver.cpp

@@ -1,16 +1,14 @@
 #include "duckdb/execution/column_binding_resolver.hpp"
 
-#include "duckdb/planner/operator/logical_comparison_join.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/common/to_string.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/operator/logical_any_join.hpp"
+#include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_create_index.hpp"
-#include "duckdb/planner/operator/logical_insert.hpp"
 #include "duckdb/planner/operator/logical_extension_operator.hpp"
-
-#include "duckdb/planner/expression/bound_columnref_expression.hpp"
-#include "duckdb/planner/expression/bound_reference_expression.hpp"
-
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
-#include "duckdb/common/to_string.hpp"
+#include "duckdb/planner/operator/logical_insert.hpp"
 
 namespace duckdb {
 
@@ -20,8 +18,7 @@ ColumnBindingResolver::ColumnBindingResolver() {
 void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 	switch (op.type) {
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
-	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
-	case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
+	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN: {
 		// special case: comparison join
 		auto &comp_join = op.Cast<LogicalComparisonJoin>();
 		// first get the bindings of the LHS and resolve the LHS expressions
@@ -42,6 +39,40 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 		bindings = op.GetColumnBindings();
 		return;
 	}
+	case LogicalOperatorType::LOGICAL_DELIM_JOIN: {
+		auto &comp_join = op.Cast<LogicalComparisonJoin>();
+		// depending on whether the delim join has been flipped, get the appropriate bindings
+		if (comp_join.delim_flipped) {
+			VisitOperator(*comp_join.children[1]);
+			for (auto &cond : comp_join.conditions) {
+				VisitExpression(&cond.right);
+			}
+		} else {
+			VisitOperator(*comp_join.children[0]);
+			for (auto &cond : comp_join.conditions) {
+				VisitExpression(&cond.left);
+			}
+		}
+		// visit the duplicate eliminated columns
+		for (auto &expr : comp_join.duplicate_eliminated_columns) {
+			VisitExpression(&expr);
+		}
+		// now get the other side
+		if (comp_join.delim_flipped) {
+			VisitOperator(*comp_join.children[0]);
+			for (auto &cond : comp_join.conditions) {
+				VisitExpression(&cond.left);
+			}
+		} else {
+			VisitOperator(*comp_join.children[1]);
+			for (auto &cond : comp_join.conditions) {
+				VisitExpression(&cond.right);
+			}
+		}
+		// finally update the bindings with the result bindings of the join
+		bindings = op.GetColumnBindings();
+		return;
+	}
 	case LogicalOperatorType::LOGICAL_ANY_JOIN: {
 		// ANY join, this join is different because we evaluate the expression on the bindings of BOTH join sides at
 		// once i.e. we set the bindings first to the bindings of the entire join, and then resolve the expressions of
@@ -53,6 +84,9 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 			auto right_bindings = op.children[1]->GetColumnBindings();
 			bindings.insert(bindings.end(), right_bindings.begin(), right_bindings.end());
 		}
+		if (any_join.join_type == JoinType::RIGHT_SEMI || any_join.join_type == JoinType::RIGHT_ANTI) {
+			throw InternalException("RIGHT SEMI/ANTI any join not supported yet");
+		}
 		VisitOperatorExpressions(op);
 		return;
 	}

package/src/duckdb/src/execution/expression_executor/execute_between.cpp

@@ -1,3 +1,4 @@
+#include "duckdb/common/uhugeint.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression/bound_between_expression.hpp"
@@ -66,6 +67,9 @@ static idx_t BetweenLoopTypeSwitch(Vector &input, Vector &lower, Vector &upper,
 	case PhysicalType::UINT64:
 		return TernaryExecutor::Select<uint64_t, uint64_t, uint64_t, OP>(input, lower, upper, sel, count, true_sel,
 		                                                                 false_sel);
+	case PhysicalType::UINT128:
+		return TernaryExecutor::Select<uhugeint_t, uhugeint_t, uhugeint_t, OP>(input, lower, upper, sel, count,
+		                                                                       true_sel, false_sel);
 	case PhysicalType::FLOAT:
 		return TernaryExecutor::Select<float, float, float, OP>(input, lower, upper, sel, count, true_sel, false_sel);
 	case PhysicalType::DOUBLE:

package/src/duckdb/src/execution/expression_executor/execute_case.cpp

@@ -1,3 +1,4 @@
+#include "duckdb/common/uhugeint.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression/bound_case_expression.hpp"
@@ -170,6 +171,9 @@ void ExpressionExecutor::FillSwitch(Vector &vector, Vector &result, const Select
 	case PhysicalType::INT128:
 		TemplatedFillLoop<hugeint_t>(vector, result, sel, count);
 		break;
+	case PhysicalType::UINT128:
+		TemplatedFillLoop<uhugeint_t>(vector, result, sel, count);
+		break;
 	case PhysicalType::FLOAT:
 		TemplatedFillLoop<float>(vector, result, sel, count);
 		break;

package/src/duckdb/src/execution/expression_executor/execute_comparison.cpp

@@ -1,3 +1,4 @@
+#include "duckdb/common/uhugeint.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression/bound_comparison_expression.hpp"
@@ -85,6 +86,8 @@ static idx_t TemplatedSelectOperation(Vector &left, Vector &right, const Selecti
 		return BinaryExecutor::Select<uint64_t, uint64_t, OP>(left, right, sel, count, true_sel, false_sel);
 	case PhysicalType::INT128:
 		return BinaryExecutor::Select<hugeint_t, hugeint_t, OP>(left, right, sel, count, true_sel, false_sel);
+	case PhysicalType::UINT128:
+		return BinaryExecutor::Select<uhugeint_t, uhugeint_t, OP>(left, right, sel, count, true_sel, false_sel);
 	case PhysicalType::FLOAT:
 		return BinaryExecutor::Select<float, float, OP>(left, right, sel, count, true_sel, false_sel);
 	case PhysicalType::DOUBLE:
@@ -95,6 +98,7 @@ static idx_t TemplatedSelectOperation(Vector &left, Vector &right, const Selecti
 		return BinaryExecutor::Select<string_t, string_t, OP>(left, right, sel, count, true_sel, false_sel);
 	case PhysicalType::LIST:
 	case PhysicalType::STRUCT:
+	case PhysicalType::ARRAY:
 		return NestedSelectOperation<OP>(left, right, sel, count, true_sel, false_sel);
 	default:
 		throw InternalException("Invalid type for comparison");

package/src/duckdb/src/execution/expression_executor.cpp

@@ -171,7 +171,8 @@ unique_ptr<ExpressionState> ExpressionExecutor::InitializeState(const Expression
 void ExpressionExecutor::Execute(const Expression &expr, ExpressionState *state, const SelectionVector *sel,
                                  idx_t count, Vector &result) {
 #ifdef DEBUG
-	//! The result Vector must be "clean"
+	// the result vector has to be used for the first time or has to be reset
+	// otherwise, the validity mask might contain previous (now incorrect) data
 	if (result.GetVectorType() == VectorType::FLAT_VECTOR) {
 		D_ASSERT(FlatVector::Validity(result).CheckAllValid(count));
 	}

package/src/duckdb/src/execution/index/art/art.cpp

@@ -1,23 +1,21 @@
 #include "duckdb/execution/index/art/art.hpp"
 
-#include "duckdb/common/radix.hpp"
+#include "duckdb/common/types/conflict_manager.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"
-#include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/execution/index/art/art_key.hpp"
-#include "duckdb/execution/index/art/prefix.hpp"
+#include "duckdb/execution/index/art/iterator.hpp"
 #include "duckdb/execution/index/art/leaf.hpp"
-#include "duckdb/execution/index/art/node4.hpp"
 #include "duckdb/execution/index/art/node16.hpp"
-#include "duckdb/execution/index/art/node48.hpp"
 #include "duckdb/execution/index/art/node256.hpp"
-#include "duckdb/execution/index/art/iterator.hpp"
-#include "duckdb/common/types/conflict_manager.hpp"
-#include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/execution/index/art/node4.hpp"
+#include "duckdb/execution/index/art/node48.hpp"
+#include "duckdb/execution/index/art/prefix.hpp"
+#include "duckdb/storage/arena_allocator.hpp"
 #include "duckdb/storage/metadata/metadata_reader.hpp"
+#include "duckdb/storage/table/scan_state.hpp"
 #include "duckdb/storage/table_io_manager.hpp"
-
-#include <algorithm>
+#include "duckdb/optimizer/matcher/expression_matcher.hpp"
 
 namespace duckdb {
 
@@ -33,15 +31,16 @@ struct ARTIndexScanState : public IndexScanState {
 	Iterator iterator;
 };
 
-ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
-         const vector<unique_ptr<Expression>> &unbound_expressions, const IndexConstraintType constraint_type,
+//===--------------------------------------------------------------------===//
+// ART
+//===--------------------------------------------------------------------===//
+
+ART::ART(const string &name, const IndexConstraintType index_constraint_type, const vector<column_t> &column_ids,
+         TableIOManager &table_io_manager, const vector<unique_ptr<Expression>> &unbound_expressions,
          AttachedDatabase &db, const shared_ptr<array<unique_ptr<FixedSizeAllocator>, ALLOCATOR_COUNT>> &allocators_ptr,
-         const BlockPointer &pointer)
-    : Index(db, IndexType::ART, table_io_manager, column_ids, unbound_expressions, constraint_type),
+         const IndexStorageInfo &info)
+    : Index(name, ART::TYPE_NAME, index_constraint_type, column_ids, table_io_manager, unbound_expressions, db),
       allocators(allocators_ptr), owns_data(false) {
-	if (!Radix::IsLittleEndian()) {
-		throw NotImplementedException("ART indexes are not supported on big endian architectures");
-	}
 
 	// initialize all allocators
 	if (!allocators) {
@@ -58,8 +57,16 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 		allocators = make_shared<array<unique_ptr<FixedSizeAllocator>, ALLOCATOR_COUNT>>(std::move(allocator_array));
 	}
 
-	if (pointer.IsValid()) {
-		Deserialize(pointer);
+	// deserialize lazily
+	if (info.IsValid()) {
+
+		if (!info.root_block_ptr.IsValid()) {
+			InitAllocators(info);
+
+		} else {
+			// old storage file
+			Deserialize(info.root_block_ptr);
+		}
 	}
 
 	// validate the types of the key columns
@@ -75,6 +82,7 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 		case PhysicalType::UINT16:
 		case PhysicalType::UINT32:
 		case PhysicalType::UINT64:
+		case PhysicalType::UINT128:
 		case PhysicalType::FLOAT:
 		case PhysicalType::DOUBLE:
 		case PhysicalType::VARCHAR:
@@ -89,8 +97,9 @@ ART::ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 // Initialize Predicate Scans
 //===--------------------------------------------------------------------===//
 
-unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
-                                                              const ExpressionType expression_type) {
+//! Initialize a single predicate scan on the index with the given expression and column IDs
+static unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
+                                                                const ExpressionType expression_type) {
 	// initialize point lookup
 	auto result = make_uniq<ARTIndexScanState>();
 	result->values[0] = value;
@@ -98,10 +107,11 @@ unique_ptr<IndexScanState> ART::InitializeScanSinglePredicate(const Transaction
 	return std::move(result);
 }
 
-unique_ptr<IndexScanState> ART::InitializeScanTwoPredicates(const Transaction &transaction, const Value &low_value,
-                                                            const ExpressionType low_expression_type,
-                                                            const Value &high_value,
-                                                            const ExpressionType high_expression_type) {
+//! Initialize a two predicate scan on the index with the given expression and column IDs
+static unique_ptr<IndexScanState> InitializeScanTwoPredicates(const Transaction &transaction, const Value &low_value,
+                                                              const ExpressionType low_expression_type,
+                                                              const Value &high_value,
+                                                              const ExpressionType high_expression_type) {
 	// initialize range lookup
 	auto result = make_uniq<ARTIndexScanState>();
 	result->values[0] = low_value;
@@ -111,6 +121,93 @@ unique_ptr<IndexScanState> ART::InitializeScanTwoPredicates(const Transaction &t
 	return std::move(result);
 }
 
+unique_ptr<IndexScanState> ART::TryInitializeScan(const Transaction &transaction, const Expression &index_expr,
+                                                  const Expression &filter_expr) {
+
+	Value low_value, high_value, equal_value;
+	ExpressionType low_comparison_type = ExpressionType::INVALID, high_comparison_type = ExpressionType::INVALID;
+	// try to find a matching index for any of the filter expressions
+
+	// create a matcher for a comparison with a constant
+	ComparisonExpressionMatcher matcher;
+	// match on a comparison type
+	matcher.expr_type = make_uniq<ComparisonExpressionTypeMatcher>();
+	// match on a constant comparison with the indexed expression
+	matcher.matchers.push_back(make_uniq<ExpressionEqualityMatcher>(const_cast<Expression &>(index_expr)));
+	matcher.matchers.push_back(make_uniq<ConstantExpressionMatcher>());
+
+	matcher.policy = SetMatcher::Policy::UNORDERED;
+
+	vector<reference<Expression>> bindings;
+	if (matcher.Match(const_cast<Expression &>(filter_expr), bindings)) {
+		// range or equality comparison with constant value
+		// we can use our index here
+		// bindings[0] = the expression
+		// bindings[1] = the index expression
+		// bindings[2] = the constant
+		auto &comparison = bindings[0].get().Cast<BoundComparisonExpression>();
+		auto constant_value = bindings[2].get().Cast<BoundConstantExpression>().value;
+		auto comparison_type = comparison.type;
+		if (comparison.left->type == ExpressionType::VALUE_CONSTANT) {
+			// the expression is on the right side, we flip them around
+			comparison_type = FlipComparisonExpression(comparison_type);
+		}
+		if (comparison_type == ExpressionType::COMPARE_EQUAL) {
+			// equality value
+			// equality overrides any other bounds so we just break here
+			equal_value = constant_value;
+		} else if (comparison_type == ExpressionType::COMPARE_GREATERTHANOREQUALTO ||
+		           comparison_type == ExpressionType::COMPARE_GREATERTHAN) {
+			// greater than means this is a lower bound
+			low_value = constant_value;
+			low_comparison_type = comparison_type;
+		} else {
+			// smaller than means this is an upper bound
+			high_value = constant_value;
+			high_comparison_type = comparison_type;
+		}
+	} else if (filter_expr.type == ExpressionType::COMPARE_BETWEEN) {
+		// BETWEEN expression
+		auto &between = filter_expr.Cast<BoundBetweenExpression>();
+		if (!between.input->Equals(index_expr)) {
+			// expression doesn't match the index expression
+			return nullptr;
+		}
+		if (between.lower->type != ExpressionType::VALUE_CONSTANT ||
+		    between.upper->type != ExpressionType::VALUE_CONSTANT) {
+			// not a constant comparison
+			return nullptr;
+		}
+		low_value = (between.lower->Cast<BoundConstantExpression>()).value;
+		low_comparison_type = between.lower_inclusive ? ExpressionType::COMPARE_GREATERTHANOREQUALTO
+		                                              : ExpressionType::COMPARE_GREATERTHAN;
+		high_value = (between.upper->Cast<BoundConstantExpression>()).value;
+		high_comparison_type =
+		    between.upper_inclusive ? ExpressionType::COMPARE_LESSTHANOREQUALTO : ExpressionType::COMPARE_LESSTHAN;
+	}
+
+	if (!equal_value.IsNull() || !low_value.IsNull() || !high_value.IsNull()) {
+		// we can scan this index using this predicate: try a scan
+		unique_ptr<IndexScanState> index_state;
+		if (!equal_value.IsNull()) {
+			// equality predicate
+			index_state = InitializeScanSinglePredicate(transaction, equal_value, ExpressionType::COMPARE_EQUAL);
+		} else if (!low_value.IsNull() && !high_value.IsNull()) {
+			// two-sided predicate
+			index_state = InitializeScanTwoPredicates(transaction, low_value, low_comparison_type, high_value,
+			                                          high_comparison_type);
+		} else if (!low_value.IsNull()) {
+			// less than predicate
+			index_state = InitializeScanSinglePredicate(transaction, low_value, low_comparison_type);
+		} else {
+			D_ASSERT(!high_value.IsNull());
+			index_state = InitializeScanSinglePredicate(transaction, high_value, high_comparison_type);
+		}
+		return index_state;
+	}
+	return nullptr;
+}
+
 //===--------------------------------------------------------------------===//
 // Keys
 //===--------------------------------------------------------------------===//
@@ -188,6 +285,9 @@ void ART::GenerateKeys(ArenaAllocator &allocator, DataChunk &input, vector<ARTKe
 	case PhysicalType::UINT64:
 		TemplatedGenerateKeys<uint64_t>(allocator, input.data[0], input.size(), keys);
 		break;
+	case PhysicalType::UINT128:
+		TemplatedGenerateKeys<uhugeint_t>(allocator, input.data[0], input.size(), keys);
+		break;
 	case PhysicalType::FLOAT:
 		TemplatedGenerateKeys<float>(allocator, input.data[0], input.size(), keys);
 		break;
@@ -234,6 +334,9 @@ void ART::GenerateKeys(ArenaAllocator &allocator, DataChunk &input, vector<ARTKe
 		case PhysicalType::UINT64:
 			ConcatenateKeys<uint64_t>(allocator, input.data[i], input.size(), keys);
 			break;
+		case PhysicalType::UINT128:
+			ConcatenateKeys<uhugeint_t>(allocator, input.data[i], input.size(), keys);
+			break;
 		case PhysicalType::FLOAT:
 			ConcatenateKeys<float>(allocator, input.data[i], input.size(), keys);
 			break;
@@ -367,7 +470,7 @@ bool ART::ConstructFromSorted(idx_t count, vector<ARTKey> &keys, Vector &row_ide
 //===--------------------------------------------------------------------===//
 // Insert / Verification / Constraint Checking
 //===--------------------------------------------------------------------===//
-PreservedError ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
+ErrorData ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 
 	D_ASSERT(row_ids.GetType().InternalType() == ROW_TYPE);
 	D_ASSERT(logical_types[0] == input.data[0].GetType());
@@ -408,8 +511,8 @@ PreservedError ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 	}
 
 	if (failed_index != DConstants::INVALID_INDEX) {
-		return PreservedError(ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicate key \"%s\"",
-		                                          AppendRowError(input, failed_index)));
+		return ErrorData(ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicate key \"%s\"",
+		                                     AppendRowError(input, failed_index)));
 	}
 
 #ifdef DEBUG
@@ -423,10 +526,10 @@ PreservedError ART::Insert(IndexLock &lock, DataChunk &input, Vector &row_ids) {
 	}
 #endif
 
-	return PreservedError();
+	return ErrorData();
 }
 
-PreservedError ART::Append(IndexLock &lock, DataChunk &appended_data, Vector &row_identifiers) {
+ErrorData ART::Append(IndexLock &lock, DataChunk &appended_data, Vector &row_identifiers) {
 	DataChunk expression_result;
 	expression_result.Initialize(Allocator::DefaultAllocator(), logical_types);
 
@@ -655,6 +758,8 @@ static ARTKey CreateKey(ArenaAllocator &allocator, PhysicalType type, Value &val
 		return ARTKey::CreateARTKey<uint64_t>(allocator, value.type(), value);
 	case PhysicalType::INT128:
 		return ARTKey::CreateARTKey<hugeint_t>(allocator, value.type(), value);
+	case PhysicalType::UINT128:
+		return ARTKey::CreateARTKey<uhugeint_t>(allocator, value.type(), value);
 	case PhysicalType::FLOAT:
 		return ARTKey::CreateARTKey<float>(allocator, value.type(), value);
 	case PhysicalType::DOUBLE:
@@ -896,11 +1001,11 @@ string ART::GenerateConstraintErrorMessage(VerifyExistenceType verify_type, cons
 	case VerifyExistenceType::APPEND: {
 		// APPEND to PK/UNIQUE table, but node/key already exists in PK/UNIQUE table
 		string type = IsPrimary() ? "primary key" : "unique";
-		return StringUtil::Format(
-		    "Duplicate key \"%s\" violates %s constraint. "
-		    "If this is an unexpected constraint violation please double "
-		    "check with the known index limitations section in our documentation (docs - sql - indexes).",
-		    key_name, type);
+		return StringUtil::Format("Duplicate key \"%s\" violates %s constraint. "
+		                          "If this is an unexpected constraint violation please double "
+		                          "check with the known index limitations section in our documentation "
+		                          "(https://duckdb.org/docs/sql/indexes).",
+		                          key_name, type);
 	}
 	case VerifyExistenceType::APPEND_FK: {
 		// APPEND_FK to FK table, node/key does not exist in PK/UNIQUE table
@@ -971,46 +1076,67 @@ void ART::CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_m
 }
 
 //===--------------------------------------------------------------------===//
-// Serialization
+// Helper functions for (de)serialization
 //===--------------------------------------------------------------------===//
 
-BlockPointer ART::Serialize(MetadataWriter &writer) {
+IndexStorageInfo ART::GetStorageInfo(const bool get_buffers) {
 
-	D_ASSERT(owns_data);
+	// set the name and root node
+	IndexStorageInfo info;
+	info.name = name;
+	info.root = tree.Get();
 
-	// early-out, if all allocators are empty
-	if (!tree.HasMetadata()) {
-		root_block_pointer = BlockPointer();
-		return root_block_pointer;
+	if (!get_buffers) {
+		// store the data on disk as partial blocks and set the block ids
+		WritePartialBlocks();
+
+	} else {
+		// set the correct allocation sizes and get the map containing all buffers
+		for (const auto &allocator : *allocators) {
+			info.buffers.push_back(allocator->InitSerializationToWAL());
+		}
 	}
 
-	lock_guard<mutex> l(lock);
+	for (const auto &allocator : *allocators) {
+		info.allocator_infos.push_back(allocator->GetInfo());
+	}
+
+	return info;
+}
+
+void ART::WritePartialBlocks() {
+
+	// use the partial block manager to serialize all allocator data
 	auto &block_manager = table_io_manager.GetIndexBlockManager();
 	PartialBlockManager partial_block_manager(block_manager, CheckpointType::FULL_CHECKPOINT);
 
-	vector<BlockPointer> allocator_pointers;
 	for (auto &allocator : *allocators) {
-		allocator_pointers.push_back(allocator->Serialize(partial_block_manager, writer));
+		allocator->SerializeBuffers(partial_block_manager);
 	}
 	partial_block_manager.FlushPartialBlocks();
+}
 
-	root_block_pointer = writer.GetBlockPointer();
-	writer.Write(tree);
-	for (auto &allocator_pointer : allocator_pointers) {
-		writer.Write(allocator_pointer);
-	}
+void ART::InitAllocators(const IndexStorageInfo &info) {
+
+	// set the root node
+	tree.Set(info.root);
 
-	return root_block_pointer;
+	// initialize the allocators
+	D_ASSERT(info.allocator_infos.size() == ALLOCATOR_COUNT);
+	for (idx_t i = 0; i < info.allocator_infos.size(); i++) {
+		(*allocators)[i]->Init(info.allocator_infos[i]);
+	}
 }
 
 void ART::Deserialize(const BlockPointer &pointer) {
 
 	D_ASSERT(pointer.IsValid());
-	MetadataReader reader(table_io_manager.GetMetadataManager(), pointer);
+	auto &metadata_manager = table_io_manager.GetMetadataManager();
+	MetadataReader reader(metadata_manager, pointer);
 	tree = reader.Read<Node>();
 
 	for (idx_t i = 0; i < ALLOCATOR_COUNT; i++) {
-		(*allocators)[i]->Deserialize(reader.Read<BlockPointer>());
+		(*allocators)[i]->Deserialize(metadata_manager, reader.Read<BlockPointer>());
 	}
 }
 
@@ -1069,6 +1195,21 @@ void ART::Vacuum(IndexLock &state) {
 	FinalizeVacuum(flags);
 }
 
+//===--------------------------------------------------------------------===//
+// Size
+//===--------------------------------------------------------------------===//
+
+idx_t ART::GetInMemorySize(IndexLock &index_lock) {
+
+	D_ASSERT(owns_data);
+
+	idx_t in_memory_size = 0;
+	for (auto &allocator : *allocators) {
+		in_memory_size += allocator->GetInMemorySize();
+	}
+	return in_memory_size;
+}
+
 //===--------------------------------------------------------------------===//
 // Merging
 //===--------------------------------------------------------------------===//
@@ -1129,4 +1270,12 @@ string ART::VerifyAndToStringInternal(const bool only_verify) {
 	return "[empty]";
 }
 
+string ART::GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index, DataChunk &input) {
+	auto key_name = GenerateErrorKeyName(input, failed_index);
+	auto exception_msg = GenerateConstraintErrorMessage(verify_type, key_name);
+	return exception_msg;
+}
+
+constexpr const char *ART::TYPE_NAME;
+
 } // namespace duckdb

package/src/duckdb/src/execution/index/art/art_key.cpp

@@ -14,21 +14,28 @@ ARTKey::ARTKey(ArenaAllocator &allocator, const uint32_t &len) : len(len) {
 
 template <>
 ARTKey ARTKey::CreateARTKey(ArenaAllocator &allocator, const LogicalType &type, string_t value) {
-	uint32_t len = value.GetSize() + 1;
+	auto string_data = const_data_ptr_cast(value.GetData());
+	auto string_len = value.GetSize();
+	// we need to escape \00 and \01
+	idx_t escape_count = 0;
+	for (idx_t r = 0; r < string_len; r++) {
+		if (string_data[r] <= 1) {
+			escape_count++;
+		}
+	}
+	idx_t len = string_len + escape_count + 1;
 	auto data = allocator.Allocate(len);
-	memcpy(data, value.GetData(), len - 1);
-
-	// FIXME: rethink this
-	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
-		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain zero bytes
-		for (uint32_t i = 0; i < len - 1; i++) {
-			if (data[i] == '\0') {
-				throw NotImplementedException("ART indexes cannot contain BLOBs with zero bytes.");
-			}
+	// copy over the data and add in escapes
+	idx_t pos = 0;
+	for (idx_t r = 0; r < string_len; r++) {
+		if (string_data[r] <= 1) {
+			// escape
+			data[pos++] = '\01';
 		}
+		data[pos++] = string_data[r];
 	}
-
-	data[len - 1] = '\0';
+	// end with a null-terminator
+	data[pos] = '\0';
 	return ARTKey(data, len);
 }
 
@@ -39,21 +46,7 @@ ARTKey ARTKey::CreateARTKey(ArenaAllocator &allocator, const LogicalType &type,
 
 template <>
 void ARTKey::CreateARTKey(ArenaAllocator &allocator, const LogicalType &type, ARTKey &key, string_t value) {
-	key.len = value.GetSize() + 1;
-	key.data = allocator.Allocate(key.len);
-	memcpy(key.data, value.GetData(), key.len - 1);
-
-	// FIXME: rethink this
-	if (type == LogicalType::BLOB || type == LogicalType::VARCHAR) {
-		// indexes cannot contain BLOBs (or BLOBs cast to VARCHARs) that contain zero bytes
-		for (uint32_t i = 0; i < key.len - 1; i++) {
-			if (key.data[i] == '\0') {
-				throw NotImplementedException("ART indexes cannot contain BLOBs with zero bytes.");
-			}
-		}
-	}
-
-	key.data[key.len - 1] = '\0';
+	key = ARTKey::CreateARTKey<string_t>(allocator, type, value);
 }
 
 template <>