duckdb 0.7.1-dev240.0 → 0.7.1-dev284.0

package/src/duckdb/src/include/duckdb/execution/index/art/art_key.hpp

@@ -28,24 +28,24 @@ public:
 
 public:
 	template <class T>
-	static inline Key CreateKey(ArenaAllocator &allocator, T element) {
+	static inline Key CreateKey(ArenaAllocator &allocator, const LogicalType &type, T element) {
 		auto data = Key::CreateData<T>(allocator, element);
 		return Key(data, sizeof(element));
 	}
 
 	template <class T>
-	static inline Key CreateKey(ArenaAllocator &allocator, const Value &element) {
-		return CreateKey(allocator, element.GetValueUnsafe<T>());
+	static inline Key CreateKey(ArenaAllocator &allocator, const LogicalType &type, const Value &element) {
+		return CreateKey(allocator, type, element.GetValueUnsafe<T>());
 	}
 
 	template <class T>
-	static inline void CreateKey(ArenaAllocator &allocator, Key &key, T element) {
+	static inline void CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, T element) {
 		key.data = Key::CreateData<T>(allocator, element);
 		key.len = sizeof(element);
 	}
 
 	template <class T>
-	static inline void CreateKey(ArenaAllocator &allocator, Key &key, const Value element) {
+	static inline void CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, const Value element) {
 		key.data = Key::CreateData<T>(allocator, element.GetValueUnsafe<T>());
 		key.len = sizeof(element);
 	}
@@ -76,12 +76,9 @@ private:
 };
 
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, string_t value);
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, string_t value);
 template <>
-Key Key::CreateKey(ArenaAllocator &allocator, const char *value);
+Key Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, const char *value);
 template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, string_t value);
-template <>
-void Key::CreateKey(ArenaAllocator &allocator, Key &key, const char *value);
-
+void Key::CreateKey(ArenaAllocator &allocator, const LogicalType &type, Key &key, string_t value);
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -108,6 +108,9 @@ public:
 
 	unique_ptr<BoundCreateTableInfo> BindCreateTableInfo(unique_ptr<CreateInfo> info);
 	unique_ptr<BoundCreateTableInfo> BindCreateTableInfo(unique_ptr<CreateInfo> info, SchemaCatalogEntry *schema);
+
+	vector<unique_ptr<Expression>> BindCreateIndexExpressions(TableCatalogEntry *table, CreateIndexInfo *info);
+
 	void BindCreateViewInfo(CreateViewInfo &base);
 	SchemaCatalogEntry *BindSchema(CreateInfo &info);
 	SchemaCatalogEntry *BindCreateFunctionInfo(CreateInfo &info);

package/src/duckdb/src/include/duckdb/planner/expression_binder/index_binder.hpp

@@ -10,20 +10,27 @@
 
 #include "duckdb/planner/expression_binder.hpp"
 #include "duckdb/common/unordered_map.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 
 namespace duckdb {
 class BoundColumnRefExpression;
 
-//! The INDEX binder is responsible for binding an expression within an Index statement
+//! The IndexBinder is responsible for binding an expression within an index statement
 class IndexBinder : public ExpressionBinder {
 public:
-	IndexBinder(Binder &binder, ClientContext &context);
+	IndexBinder(Binder &binder, ClientContext &context, TableCatalogEntry *table = nullptr,
+	            CreateIndexInfo *info = nullptr);
 
 protected:
 	BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
 	                          bool root_expression = false) override;
-
 	string UnsupportedAggregateMessage() override;
+
+private:
+	// only for WAL replay
+	TableCatalogEntry *table;
+	CreateIndexInfo *info;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -122,6 +122,12 @@ public:
 	void UpdateColumn(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
 	                  const vector<column_t> &column_path, DataChunk &updates);
 
+	//! Add an index to the DataTable. NOTE: for CREATE (UNIQUE) INDEX statements, we use the PhysicalCreateIndex
+	//! operator. This function is only used during the WAL replay, and is a much less performant index creation
+	//! approach.
+	void WALAddIndex(ClientContext &context, unique_ptr<Index> index,
+	                 const vector<unique_ptr<Expression>> &expressions);
+
 	//! Fetches an append lock
 	void AppendLock(TableAppendState &state);
 	//! Begin appending structs to this table, obtaining necessary locks, etc
@@ -176,7 +182,7 @@ public:
 	static bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type);
 
 	//! Initializes a special scan that is used to create an index on the table, it keeps locks on the table
-	void InitializeCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids);
+	void InitializeWALCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids);
 	//! Scans the next chunk for the CREATE INDEX operator
 	bool CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type);
 

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -40,11 +40,11 @@ public:
 	IndexType type;
 	//! Associated table io manager
 	TableIOManager &table_io_manager;
-	//! Column identifiers to extract from the base table
+	//! Column identifiers to extract key columns from the base table
 	vector<column_t> column_ids;
-	//! Unordered_set of column_ids used by the index
+	//! Unordered set of column_ids used by the index
 	unordered_set<column_t> column_id_set;
-	//! Unbound expressions used by the index
+	//! Unbound expressions used by the index during optimizations
 	vector<unique_ptr<Expression>> unbound_expressions;
 	//! The physical types stored in the index
 	vector<PhysicalType> types;
@@ -64,94 +64,103 @@ public:
 	bool track_memory;
 
 public:
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table when we only have one query predicate
+	//! Initialize a single predicate scan on the index with the given expression and column IDs
 	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
 	                                                                 ExpressionType expressionType) = 0;
-	//! Initialize a scan on the index with the given expression and column ids
-	//! to fetch from the base table for two query predicates
+	//! Initialize a two predicate scan on the index with the given expression and column IDs
 	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
 	                                                               ExpressionType low_expression_type,
 	                                                               const Value &high_value,
 	                                                               ExpressionType high_expression_type) = 0;
-	//! Perform a lookup on the index, fetching up to max_count result ids. Returns true if all row ids were fetched,
-	//! and false otherwise.
+	//! Performs a lookup on the index, fetching up to max_count result IDs. Returns true if all row IDs were fetched,
+	//! and false otherwise
 	virtual bool Scan(Transaction &transaction, DataTable &table, IndexScanState &state, idx_t max_count,
 	                  vector<row_t> &result_ids) = 0;
 
 	//! Obtain a lock on the index
 	virtual void InitializeLock(IndexLock &state);
-	//! Called when data is appended to the index. The lock obtained from InitializeAppend must be held
+	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
 	virtual bool Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	//! Obtains a lock and calls Append while holding that lock
 	bool Append(DataChunk &entries, Vector &row_identifiers);
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation
 	virtual void VerifyAppend(DataChunk &chunk) = 0;
-	//! Verify that data can be appended to the index
+	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
 	virtual void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) = 0;
-	//! Verify that data can be appended to the index for foreign key constraint
-	virtual void VerifyAppendForeignKey(DataChunk &chunk) = 0;
-	//! Verify that data can be delete from the index for foreign key constraint
-	virtual void VerifyDeleteForeignKey(DataChunk &chunk) = 0;
+	//! Performs constraint checking for a chunk of input data
+	virtual void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) = 0;
 
-	//! Called when data inside the index is Deleted
+	//! Delete a chunk of entries from the index. The lock obtained from InitializeLock must be held
 	virtual void Delete(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
+	//! Obtains a lock and calls Delete while holding that lock
 	void Delete(DataChunk &entries, Vector &row_identifiers);
 
-	//! Insert data into the index. Does not lock the index.
+	//! Insert a chunk of entries into the index
 	virtual bool Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
 
-	//! Merge other_index into this index.
+	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
+	//! index must also be locked during the merge
 	virtual bool MergeIndexes(IndexLock &state, Index *other_index) = 0;
+	//! Obtains a lock and calls MergeIndexes while holding that lock
 	bool MergeIndexes(Index *other_index);
 
 	//! Returns the string representation of an index
 	virtual string ToString() = 0;
-	//! Verifies that the memory_size value of the index matches its actual size
+	//! Verifies that the in-memory size value of the index matches its actual size
 	virtual void Verify() = 0;
-
-	//! Returns true if the index is affected by updates on the specified column ids, and false otherwise
+	//! Increases the memory size by the difference between the old size and the current size
+	//! and performs verifications
+	virtual void IncreaseAndVerifyMemorySize(idx_t old_memory_size) = 0;
+
+	//! Increases the in-memory size value
+	inline void IncreaseMemorySize(idx_t size) {
+		memory_size += size;
+	};
+	//! Decreases the in-memory size value
+	inline void DecreaseMemorySize(idx_t size) {
+		D_ASSERT(memory_size >= size);
+		memory_size -= size;
+	};
+
+	//! Returns true if the index is affected by updates on the specified column IDs, and false otherwise
 	bool IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const;
 
-	//! Returns how many of the input values were found in the 'input' chunk, with the option to also record what those
-	//! matches were. For this purpose, nulls count as a match, and are returned in 'null_count'
-	virtual void LookupValues(DataChunk &input, ConflictManager &conflict_manager) = 0;
-
 	//! Returns unique flag
 	bool IsUnique() {
 		return (constraint_type == IndexConstraintType::UNIQUE || constraint_type == IndexConstraintType::PRIMARY);
 	}
-	//! Returns primary flag
+	//! Returns primary key flag
 	bool IsPrimary() {
 		return (constraint_type == IndexConstraintType::PRIMARY);
 	}
-	//! Returns foreign flag
+	//! Returns foreign key flag
 	bool IsForeign() {
 		return (constraint_type == IndexConstraintType::FOREIGN);
 	}
-	//! Serializes the index and returns the pair of block_id offset positions
-	virtual BlockPointer Serialize(duckdb::MetaBlockWriter &writer);
-	BlockPointer GetBlockPointer();
 
-	//! Returns block/offset of where index was most recently serialized.
+	//! Serializes the index and returns the pair of block_id offset positions
+	virtual BlockPointer Serialize(MetaBlockWriter &writer);
+	//! Returns the serialized data pointer to the block and offset of the serialized index
 	BlockPointer GetSerializedDataPointer() const {
 		return serialized_data_pointer;
 	}
 
-protected:
+	//! Execute the index expressions on an input chunk
 	void ExecuteExpressions(DataChunk &input, DataChunk &result);
 
-	//! Lock used for updating the index
+protected:
+	//! Lock used for any changes to the index
 	mutex lock;
-
-	//! Pointer to most recently checkpointed index data.
+	//! Pointer to serialized index data
 	BlockPointer serialized_data_pointer;
 
 private:
-	//! Bound expressions used by the index
+	//! Bound expressions used during expression execution
 	vector<unique_ptr<Expression>> bound_expressions;
-	//! Expression executor for the index expressions
+	//! Expression executor to execute the index expressions
 	ExpressionExecutor executor;
 
+	//! Bind the unbound expressions of the index
 	unique_ptr<Expression> BindExpression(unique_ptr<Expression> expr);
 };
 

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/index_catalog_entry.hpp"
 #include "duckdb/main/attached_database.hpp"
 #include "duckdb/storage/storage_info.hpp"
 
@@ -77,6 +78,9 @@ protected:
 	void ReplayCreateTableMacro();
 	void ReplayDropTableMacro();
 
+	void ReplayCreateIndex();
+	void ReplayDropIndex();
+
 	void ReplayUseTable();
 	void ReplayInsert();
 	void ReplayDelete();
@@ -125,6 +129,9 @@ public:
 	void WriteCreateTableMacro(TableMacroCatalogEntry *entry);
 	void WriteDropTableMacro(TableMacroCatalogEntry *entry);
 
+	void WriteCreateIndex(IndexCatalogEntry *entry);
+	void WriteDropIndex(IndexCatalogEntry *entry);
+
 	void WriteCreateType(TypeCatalogEntry *entry);
 	void WriteDropType(TypeCatalogEntry *entry);
 	//! Sets the table used for subsequent insert/delete/update commands

package/src/duckdb/src/parser/parsed_data/create_index_info.cpp

@@ -16,6 +16,9 @@ unique_ptr<CreateInfo> CreateIndexInfo::Copy() const {
 	for (auto &expr : expressions) {
 		result->expressions.push_back(expr->Copy());
 	}
+	for (auto &expr : parsed_expressions) {
+		result->parsed_expressions.push_back(expr->Copy());
+	}
 
 	result->scan_types = scan_types;
 	result->names = names;

package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp

@@ -16,6 +16,8 @@
 #include "duckdb/parser/expression/list.hpp"
 #include "duckdb/common/index_map.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/expression_binder/index_binder.hpp"
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
 
 #include <algorithm>
 
@@ -300,4 +302,15 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
 	return BindCreateTableInfo(std::move(info), schema);
 }
 
+vector<unique_ptr<Expression>> Binder::BindCreateIndexExpressions(TableCatalogEntry *table, CreateIndexInfo *info) {
+	vector<unique_ptr<Expression>> expressions;
+
+	auto index_binder = IndexBinder(*this, this->context, table, info);
+	for (auto &expr : info->expressions) {
+		expressions.push_back(index_binder.Bind(expr));
+	}
+
+	return expressions;
+}
+
 } // namespace duckdb

package/src/duckdb/src/planner/expression_binder/index_binder.cpp

@@ -1,8 +1,14 @@
 #include "duckdb/planner/expression_binder/index_binder.hpp"
 
+#include "duckdb/parser/parsed_data/create_index_info.hpp"
+#include "duckdb/parser/expression/columnref_expression.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/column_binding.hpp"
+
 namespace duckdb {
 
-IndexBinder::IndexBinder(Binder &binder, ClientContext &context) : ExpressionBinder(binder, context) {
+IndexBinder::IndexBinder(Binder &binder, ClientContext &context, TableCatalogEntry *table, CreateIndexInfo *info)
+    : ExpressionBinder(binder, context), table(table), info(info) {
 }
 
 BindResult IndexBinder::BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth, bool root_expression) {
@@ -12,6 +18,31 @@ BindResult IndexBinder::BindExpression(unique_ptr<ParsedExpression> *expr_ptr, i
 		return BindResult("window functions are not allowed in index expressions");
 	case ExpressionClass::SUBQUERY:
 		return BindResult("cannot use subquery in index expressions");
+	case ExpressionClass::COLUMN_REF: {
+		if (table) {
+			// WAL replay
+			// we assume that the parsed expressions have qualified column names
+			// and that the columns exist in the table
+			auto &col_ref = (ColumnRefExpression &)expr;
+			auto col_idx = table->GetColumnIndex(col_ref.column_names.back());
+			auto col_type = table->GetColumn(col_idx).GetType();
+
+			// find the col_idx in the index.column_ids
+			auto col_id_idx = DConstants::INVALID_INDEX;
+			for (idx_t i = 0; i < info->column_ids.size(); i++) {
+				if (col_idx.index == info->column_ids[i]) {
+					col_id_idx = i;
+				}
+			}
+
+			if (col_id_idx == DConstants::INVALID_INDEX) {
+				throw InternalException("failed to replay CREATE INDEX statement - column id not found");
+			}
+			return BindResult(
+			    make_unique<BoundColumnRefExpression>(col_ref.alias, col_type, ColumnBinding(0, col_id_idx)));
+		}
+		return ExpressionBinder::BindExpression(expr_ptr, depth);
+	}
 	default:
 		return ExpressionBinder::BindExpression(expr_ptr, depth);
 	}

package/src/duckdb/src/storage/compression/bitpacking.cpp

@@ -447,8 +447,8 @@ public:
 
 		static void ReserveSpace(BitpackingCompressState<T> *state, idx_t data_bytes) {
 			idx_t meta_bytes = sizeof(bitpacking_metadata_encoded_t);
-			state->FlushAndCreateSegmentIfFull(data_bytes + meta_bytes);
-			D_ASSERT(data_bytes + meta_bytes <= state->RemainingSize());
+			state->FlushAndCreateSegmentIfFull(data_bytes, meta_bytes);
+			D_ASSERT(state->CanStore(data_bytes, meta_bytes));
 		}
 
 		static void UpdateStats(BitpackingCompressState<T> *state, idx_t count) {
@@ -461,9 +461,12 @@ public:
 		}
 	};
 
-	// Space remaining between the metadata_ptr growing down and data ptr growing up
-	idx_t RemainingSize() {
-		return metadata_ptr - data_ptr;
+	bool CanStore(idx_t data_bytes, idx_t meta_bytes) {
+		auto required_data_bytes = AlignValue<idx_t>((data_ptr + data_bytes) - data_ptr);
+		auto required_meta_bytes = Storage::BLOCK_SIZE - (metadata_ptr - data_ptr) + meta_bytes;
+
+		return required_data_bytes + required_meta_bytes <=
+		       Storage::BLOCK_SIZE - BitpackingPrimitives::BITPACKING_HEADER_SIZE;
 	}
 
 	void CreateEmptySegment(idx_t row_start) {
@@ -489,8 +492,8 @@ public:
 		}
 	}
 
-	void FlushAndCreateSegmentIfFull(idx_t required_space) {
-		if (RemainingSize() < required_space) {
+	void FlushAndCreateSegmentIfFull(idx_t required_data_bytes, idx_t required_meta_bytes) {
+		if (!CanStore(required_data_bytes, required_meta_bytes)) {
 			auto row_start = current_segment->start + current_segment->count;
 			FlushSegment();
 			CreateEmptySegment(row_start);
@@ -505,6 +508,12 @@ public:
 		idx_t metadata_offset = AlignValue(data_ptr - base_ptr);
 		idx_t metadata_size = base_ptr + Storage::BLOCK_SIZE - metadata_ptr;
 		idx_t total_segment_size = metadata_offset + metadata_size;
+
+		// Asserting things are still sane here
+		if (!CanStore(0, 0)) {
+			throw InternalException("Error in bitpacking size calculation");
+		}
+
 		memmove(base_ptr + metadata_offset, metadata_ptr, metadata_size);
 
 		// Store the offset of the metadata of the first group (which is at the highest address).

package/src/duckdb/src/storage/data_table.cpp

@@ -1144,15 +1144,78 @@ void DataTable::UpdateColumn(TableCatalogEntry &table, ClientContext &context, V
 }
 
 //===--------------------------------------------------------------------===//
-// Create Index Scan
+// Index Scan
 //===--------------------------------------------------------------------===//
-void DataTable::InitializeCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids) {
+void DataTable::InitializeWALCreateIndexScan(CreateIndexScanState &state, const vector<column_t> &column_ids) {
 	// we grab the append lock to make sure nothing is appended until AFTER we finish the index scan
 	state.append_lock = std::unique_lock<mutex>(append_lock);
-	row_groups->InitializeCreateIndexScan(state);
 	InitializeScan(state, column_ids);
 }
 
+void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
+                            const vector<unique_ptr<Expression>> &expressions) {
+
+	// if the data table is empty
+	if (row_groups->IsEmpty()) {
+		info->indexes.AddIndex(std::move(index));
+		return;
+	}
+
+	auto &allocator = Allocator::Get(db);
+
+	DataChunk result;
+	result.Initialize(allocator, index->logical_types);
+
+	DataChunk intermediate;
+	vector<LogicalType> intermediate_types;
+	auto column_ids = index->column_ids;
+	column_ids.push_back(COLUMN_IDENTIFIER_ROW_ID);
+	for (auto &id : index->column_ids) {
+		auto &col = column_definitions[id];
+		intermediate_types.push_back(col.Type());
+	}
+	intermediate_types.emplace_back(LogicalType::ROW_TYPE);
+	intermediate.Initialize(allocator, intermediate_types);
+
+	// initialize an index scan
+	CreateIndexScanState state;
+	InitializeWALCreateIndexScan(state, column_ids);
+
+	if (!is_root) {
+		throw InternalException("Error during WAL replay. Cannot add an index to a table that has been altered.");
+	}
+
+	// now start incrementally building the index
+	{
+		IndexLock lock;
+		index->InitializeLock(lock);
+
+		while (true) {
+			intermediate.Reset();
+			result.Reset();
+			// scan a new chunk from the table to index
+			CreateIndexScan(state, intermediate, TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
+			if (intermediate.size() == 0) {
+				// finished scanning for index creation
+				// release all locks
+				break;
+			}
+			// resolve the expressions for this chunk
+			index->ExecuteExpressions(intermediate, result);
+
+			// insert into the index
+			if (!index->Insert(lock, result, intermediate.data[intermediate.ColumnCount() - 1])) {
+				throw InternalException("Error during WAL replay. Can't create unique index, table contains "
+				                        "duplicate data on indexed column(s).");
+			}
+		}
+	}
+	info->indexes.AddIndex(std::move(index));
+}
+
+//===--------------------------------------------------------------------===//
+// Statistics
+//===--------------------------------------------------------------------===//
 unique_ptr<BaseStatistics> DataTable::GetStatistics(ClientContext &context, column_t column_id) {
 	if (column_id == COLUMN_IDENTIFIER_ROW_ID) {
 		return nullptr;

package/src/duckdb/src/storage/index.cpp

@@ -86,7 +86,7 @@ bool Index::IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const {
 	return false;
 }
 
-BlockPointer Index::Serialize(duckdb::MetaBlockWriter &writer) {
+BlockPointer Index::Serialize(MetaBlockWriter &writer) {
 	throw NotImplementedException("The implementation of this index serialization does not exist.");
 }
 

package/src/duckdb/src/storage/local_storage.cpp

@@ -127,7 +127,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 				unbound_expressions.push_back(expr->Copy());
 			}
 			indexes.AddIndex(make_unique<ART>(art.column_ids, art.table_io_manager, std::move(unbound_expressions),
-			                                  art.constraint_type, art.db, false));
+			                                  art.constraint_type, art.db, true));
 		}
 		return false;
 	});

package/src/duckdb/src/storage/table_index_list.cpp

@@ -60,8 +60,7 @@ void TableIndexList::VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, Data
 		throw InternalException("Internal Foreign Key error: could not find index to verify...");
 	}
 	conflict_manager.SetIndexCount(1);
-
-	index->LookupValues(chunk, conflict_manager);
+	index->CheckConstraintsForChunk(chunk, conflict_manager);
 }
 
 vector<column_t> TableIndexList::GetRequiredColumns() {

package/src/duckdb/src/storage/wal_replay.cpp

@@ -18,6 +18,8 @@
 #include "duckdb/storage/write_ahead_log.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/main/attached_database.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
 
 namespace duckdb {
 
@@ -154,6 +156,12 @@ void ReplayState::ReplayEntry(WALType entry_type) {
 	case WALType::DROP_TABLE_MACRO:
 		ReplayDropTableMacro();
 		break;
+	case WALType::CREATE_INDEX:
+		ReplayCreateIndex();
+		break;
+	case WALType::DROP_INDEX:
+		ReplayDropIndex();
+		break;
 	case WALType::USE_TABLE:
 		ReplayUseTable();
 		break;
@@ -379,6 +387,66 @@ void ReplayState::ReplayDropTableMacro() {
 	catalog.DropEntry(context, &info);
 }
 
+//===--------------------------------------------------------------------===//
+// Replay Index
+//===--------------------------------------------------------------------===//
+void ReplayState::ReplayCreateIndex() {
+
+	auto info = IndexCatalogEntry::Deserialize(source, context);
+	if (deserialize_only) {
+		return;
+	}
+
+	// get the physical table to which we'll add the index
+	auto table = catalog.GetEntry<TableCatalogEntry>(context, info->schema, info->table->table_name);
+	auto &data_table = table->GetStorage();
+
+	// bind the parsed expressions
+	if (info->expressions.empty()) {
+		for (auto &parsed_expr : info->parsed_expressions) {
+			info->expressions.push_back(parsed_expr->Copy());
+		}
+	}
+	auto binder = Binder::CreateBinder(context);
+	auto expressions = binder->BindCreateIndexExpressions(table, info.get());
+
+	// create the empty index
+	unique_ptr<Index> index;
+	switch (info->index_type) {
+	case IndexType::ART: {
+		index = make_unique<ART>(info->column_ids, TableIOManager::Get(data_table), expressions, info->constraint_type,
+		                         data_table.db, true);
+		break;
+	}
+	default:
+		throw InternalException("Unimplemented index type");
+	}
+
+	// add the index to the catalog
+	auto index_entry = (DuckIndexEntry *)catalog.CreateIndex(context, info.get());
+	index_entry->index = index.get();
+	index_entry->info = data_table.info;
+	for (auto &parsed_expr : info->parsed_expressions) {
+		index_entry->parsed_expressions.push_back(parsed_expr->Copy());
+	}
+
+	// physically add the index to the data table storage
+	data_table.WALAddIndex(context, std::move(index), expressions);
+}
+
+void ReplayState::ReplayDropIndex() {
+
+	DropInfo info;
+	info.type = CatalogType::INDEX_ENTRY;
+	info.schema = source.Read<string>();
+	info.name = source.Read<string>();
+	if (deserialize_only) {
+		return;
+	}
+
+	catalog.DropEntry(context, &info);
+}
+
 //===--------------------------------------------------------------------===//
 // Replay Data
 //===--------------------------------------------------------------------===//