duckdb 0.7.2-dev2552.0 → 0.7.2-dev2675.0

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -32,8 +32,7 @@ struct IndexScanState;
 class Index {
 public:
 	Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_manager, const vector<column_t> &column_ids,
-	      const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type,
-	      bool track_memory);
+	      const vector<unique_ptr<Expression>> &unbound_expressions, IndexConstraintType constraint_type);
 	virtual ~Index() = default;
 
 	//! The type of the index
@@ -57,25 +56,21 @@ public:
 	AttachedDatabase &db;
 	//! Buffer manager of the database instance
 	BufferManager &buffer_manager;
-	//! The size of the index in memory
-	//! This does not track the size of the index meta information, but only allocated nodes and leaves
-	idx_t memory_size;
-	//! Flag determining if this index's size is tracked by the buffer manager
-	bool track_memory;
 
 public:
 	//! Initialize a single predicate scan on the index with the given expression and column IDs
 	virtual unique_ptr<IndexScanState> InitializeScanSinglePredicate(const Transaction &transaction, const Value &value,
-	                                                                 ExpressionType expressionType) = 0;
+	                                                                 const ExpressionType expression_type) = 0;
 	//! Initialize a two predicate scan on the index with the given expression and column IDs
-	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(Transaction &transaction, const Value &low_value,
-	                                                               ExpressionType low_expression_type,
+	virtual unique_ptr<IndexScanState> InitializeScanTwoPredicates(const Transaction &transaction,
+	                                                               const Value &low_value,
+	                                                               const ExpressionType low_expression_type,
 	                                                               const Value &high_value,
-	                                                               ExpressionType high_expression_type) = 0;
+	                                                               const ExpressionType high_expression_type) = 0;
 	//! Performs a lookup on the index, fetching up to max_count result IDs. Returns true if all row IDs were fetched,
 	//! and false otherwise
-	virtual bool Scan(Transaction &transaction, DataTable &table, IndexScanState &state, idx_t max_count,
-	                  vector<row_t> &result_ids) = 0;
+	virtual bool Scan(const Transaction &transaction, const DataTable &table, IndexScanState &state,
+	                  const idx_t max_count, vector<row_t> &result_ids) = 0;
 
 	//! Obtain a lock on the index
 	virtual void InitializeLock(IndexLock &state);
@@ -104,23 +99,13 @@ public:
 	//! Obtains a lock and calls MergeIndexes while holding that lock
 	bool MergeIndexes(Index &other_index);
 
+	//! Traverses an ART and vacuums the qualifying nodes. The lock obtained from InitializeLock must be held
+	virtual void Vacuum(IndexLock &state) = 0;
+	//! Obtains a lock and calls Vacuum while holding that lock
+	void Vacuum();
+
 	//! Returns the string representation of an index
 	virtual string ToString() = 0;
-	//! Verifies that the in-memory size value of the index matches its actual size
-	virtual void Verify() = 0;
-	//! Increases the memory size by the difference between the old size and the current size
-	//! and performs verifications
-	virtual void IncreaseAndVerifyMemorySize(idx_t old_memory_size) = 0;
-
-	//! Increases the in-memory size value
-	inline void IncreaseMemorySize(idx_t size) {
-		memory_size += size;
-	};
-	//! Decreases the in-memory size value
-	inline void DecreaseMemorySize(idx_t size) {
-		D_ASSERT(memory_size >= size);
-		memory_size -= size;
-	};
 
 	//! Returns true if the index is affected by updates on the specified column IDs, and false otherwise
 	bool IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const;

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -45,8 +45,6 @@ public:
 	shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size) final override;
 
 	idx_t GetUsedMemory() const final override;
-	void IncreaseUsedMemory(idx_t amount, bool unsafe = false) final override;
-	void DecreaseUsedMemory(idx_t amount) final override;
 	idx_t GetMaxMemory() const final override;
 
 	//! Allocate an in-memory buffer with a single pin.

package/src/duckdb/src/include/duckdb/transaction/cleanup_state.hpp

@@ -10,8 +10,10 @@
 
 #include "duckdb/transaction/undo_buffer.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
+#include "duckdb/common/unordered_map.hpp"
 
 namespace duckdb {
+
 class DataTable;
 
 struct DeleteInfo;
@@ -22,6 +24,9 @@ public:
 	CleanupState();
 	~CleanupState();
 
+	// all tables with indexes that possibly need a vacuum (after e.g. a delete)
+	unordered_map<string, optional_ptr<DataTable>> indexed_tables;
+
 public:
 	void CleanupEntry(UndoFlags type, data_ptr_t data);
 

package/src/duckdb/src/include/duckdb.h

@@ -190,6 +190,23 @@ typedef struct {
 	idx_t size;
 } duckdb_string;
 
+/*
+    The internal data representation of a VARCHAR/BLOB column
+*/
+typedef struct {
+	union {
+		struct {
+			uint32_t length;
+			char prefix[4];
+			char *ptr;
+		} pointer;
+		struct {
+			uint32_t length;
+			char inlined[12];
+		} inlined;
+	} value;
+} duckdb_string_t;
+
 typedef struct {
 	void *data;
 	idx_t size;
@@ -298,6 +315,7 @@ typedef enum {
 /*!
 Creates a new database or opens an existing database file stored at the the given path.
 If no path is given a new in-memory database is created instead.
+The instantiated database should be closed with 'duckdb_close'
 
 * path: Path to the database file on disk, or `nullptr` or `:memory:` to open an in-memory database.
 * out_database: The result database object.
@@ -331,6 +349,7 @@ DUCKDB_API void duckdb_close(duckdb_database *database);
 /*!
 Opens a connection to a database. Connections are required to query the database, and store transactional state
 associated with the connection.
+The instantiated connection should be closed using 'duckdb_disconnect'
 
 * database: The database file to connect to.
 * out_connection: The result connection object.
@@ -751,6 +770,13 @@ This is the amount of tuples that will fit into a data chunk created by `duckdb_
 */
 DUCKDB_API idx_t duckdb_vector_size();
 
+/*!
+Whether or not the duckdb_string_t value is inlined.
+This means that the data of the string does not have a separate allocation.
+
+*/
+DUCKDB_API bool duckdb_string_is_inlined(duckdb_string_t string);
+
 //===--------------------------------------------------------------------===//
 // Date/Time/Timestamp Helpers
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/capi/helper-c.cpp

@@ -186,3 +186,10 @@ void duckdb_free(void *ptr) {
 idx_t duckdb_vector_size() {
 	return STANDARD_VECTOR_SIZE;
 }
+
+bool duckdb_string_is_inlined(duckdb_string_t string_p) {
+	static_assert(sizeof(duckdb_string_t) == sizeof(duckdb::string_t),
+	              "duckdb_string_t should have the same memory layout as duckdb::string_t");
+	auto &string = *(duckdb::string_t *)(&string_p);
+	return string.IsInlined();
+}

package/src/duckdb/src/parser/statement/insert_statement.cpp

@@ -27,9 +27,10 @@ InsertStatement::InsertStatement()
 }
 
 InsertStatement::InsertStatement(const InsertStatement &other)
-    : SQLStatement(other),
-      select_statement(unique_ptr_cast<SQLStatement, SelectStatement>(other.select_statement->Copy())),
-      columns(other.columns), table(other.table), schema(other.schema), catalog(other.catalog) {
+    : SQLStatement(other), select_statement(unique_ptr_cast<SQLStatement, SelectStatement>(
+                               other.select_statement ? other.select_statement->Copy() : nullptr)),
+      columns(other.columns), table(other.table), schema(other.schema), catalog(other.catalog),
+      default_values(other.default_values) {
 	cte_map = other.cte_map.Copy();
 	for (auto &expr : other.returning_list) {
 		returning_list.emplace_back(expr->Copy());
@@ -93,10 +94,15 @@ string InsertStatement::ToString() const {
 	result += " ";
 	auto values_list = GetValuesList();
 	if (values_list) {
+		D_ASSERT(!default_values);
 		values_list->alias = string();
 		result += values_list->ToString();
-	} else {
+	} else if (select_statement) {
+		D_ASSERT(!default_values);
 		result += select_statement->ToString();
+	} else {
+		D_ASSERT(default_values);
+		result += "DEFAULT VALUES";
 	}
 	if (!or_replace_shorthand_set && on_conflict_info) {
 		auto &conflict_info = *on_conflict_info;
@@ -155,7 +161,10 @@ unique_ptr<SQLStatement> InsertStatement::Copy() const {
 	return unique_ptr<InsertStatement>(new InsertStatement(*this));
 }
 
-ExpressionListRef *InsertStatement::GetValuesList() const {
+optional_ptr<ExpressionListRef> InsertStatement::GetValuesList() const {
+	if (!select_statement) {
+		return nullptr;
+	}
 	if (select_statement->node->type != QueryNodeType::SELECT_NODE) {
 		return nullptr;
 	}
@@ -178,7 +187,7 @@ ExpressionListRef *InsertStatement::GetValuesList() const {
 	if (!node.from_table || node.from_table->type != TableReferenceType::EXPRESSION_LIST) {
 		return nullptr;
 	}
-	return (ExpressionListRef *)node.from_table.get();
+	return &node.from_table->Cast<ExpressionListRef>();
 }
 
 } // namespace duckdb

package/src/duckdb/src/parser/transform/constraint/transform_constraint.cpp

@@ -108,7 +108,7 @@ unique_ptr<Constraint> Transformer::TransformConstraint(duckdb_libpgquery::PGLis
 				pk_columns.emplace_back(reinterpret_cast<duckdb_libpgquery::PGValue *>(kc->data.ptr_value)->val.str);
 			}
 		}
-		if (pk_columns.size() != fk_columns.size()) {
+		if (!pk_columns.empty() && pk_columns.size() != fk_columns.size()) {
 			throw ParserException("The number of referencing and referenced columns for foreign keys must be the same");
 		}
 		return make_uniq<ForeignKeyConstraint>(pk_columns, fk_columns, std::move(fk_info));

package/src/duckdb/src/parser/transform/expression/transform_function.cpp

@@ -13,15 +13,24 @@
 
 namespace duckdb {
 
-void Transformer::TransformWindowDef(duckdb_libpgquery::PGWindowDef *window_spec, WindowExpression *expr) {
+void Transformer::TransformWindowDef(duckdb_libpgquery::PGWindowDef *window_spec, WindowExpression *expr,
+                                     const char *window_name) {
 	D_ASSERT(window_spec);
 	D_ASSERT(expr);
 
 	// next: partitioning/ordering expressions
 	if (window_spec->partitionClause) {
+		if (window_name && !expr->partitions.empty()) {
+			throw ParserException("Cannot override PARTITION BY clause of window \"%s\"", window_name);
+		}
 		TransformExpressionList(*window_spec->partitionClause, expr->partitions);
 	}
-	TransformOrderBy(window_spec->orderClause, expr->orders);
+	if (window_spec->orderClause) {
+		if (window_name && !expr->orders.empty()) {
+			throw ParserException("Cannot override ORDER BY clause of window \"%s\"", window_name);
+		}
+		TransformOrderBy(window_spec->orderClause, expr->orders);
+	}
 }
 
 void Transformer::TransformWindowFrame(duckdb_libpgquery::PGWindowDef *window_spec, WindowExpression *expr) {
@@ -198,6 +207,7 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			D_ASSERT(window_spec);
 		}
 		auto window_ref = window_spec;
+		auto window_name = window_ref->refname;
 		if (window_ref->refname) {
 			auto it = window_clauses.find(StringUtil::Lower(string(window_spec->refname)));
 			if (it == window_clauses.end()) {
@@ -208,6 +218,9 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 		}
 		in_window_definition = true;
 		TransformWindowDef(window_ref, expr.get());
+		if (window_ref != window_spec) {
+			TransformWindowDef(window_spec, expr.get(), window_name);
+		}
 		TransformWindowFrame(window_spec, expr.get());
 		in_window_definition = false;
 		expr->query_location = root->location;
@@ -299,9 +312,9 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
 			                                               std::move(filter_expr), std::move(order_bys),
 			                                               root->agg_distinct, false, root->export_state);
 			lowercase_name = "list_sort";
-			order_bys.reset();
-			filter_expr.reset();
-			children.clear();
+			order_bys.reset();   // NOLINT
+			filter_expr.reset(); // NOLINT
+			children.clear();    // NOLINT
 			children.emplace_back(std::move(unordered));
 			children.emplace_back(std::move(sense));
 			children.emplace_back(std::move(nulls));

package/src/duckdb/src/parser/transform/statement/transform_insert.cpp

@@ -26,12 +26,6 @@ unique_ptr<InsertStatement> Transformer::TransformInsert(duckdb_libpgquery::PGNo
 	auto stmt = reinterpret_cast<duckdb_libpgquery::PGInsertStmt *>(node);
 	D_ASSERT(stmt);
 
-	if (!stmt->selectStmt) {
-		// TODO: This should be easy to add, we already support DEFAULT in the values list,
-		// this could probably just be transformed into VALUES (DEFAULT, DEFAULT, DEFAULT, ..) in the Binder
-		throw ParserException("DEFAULT VALUES clause is not supported!");
-	}
-
 	auto result = make_uniq<InsertStatement>();
 	if (stmt->withClause) {
 		TransformCTE(reinterpret_cast<duckdb_libpgquery::PGWithClause *>(stmt->withClause), result->cte_map);
@@ -49,7 +43,11 @@ unique_ptr<InsertStatement> Transformer::TransformInsert(duckdb_libpgquery::PGNo
 	if (stmt->returningList) {
 		Transformer::TransformExpressionList(*(stmt->returningList), result->returning_list);
 	}
-	result->select_statement = TransformSelect(stmt->selectStmt, false);
+	if (stmt->selectStmt) {
+		result->select_statement = TransformSelect(stmt->selectStmt, false);
+	} else {
+		result->default_values = true;
+	}
 
 	auto qname = TransformQualifiedName(stmt->relation);
 	result->table = qname.name;

package/src/duckdb/src/planner/binder/statement/bind_create.cpp

@@ -292,17 +292,30 @@ static void FindMatchingPrimaryKeyColumns(const ColumnList &columns, const vecto
 		} else {
 			pk_names = unique.columns;
 		}
-		if (pk_names.size() != fk.fk_columns.size()) {
-			// the number of referencing and referenced columns for foreign keys must be the same
-			continue;
-		}
 		if (find_primary_key) {
 			// found matching primary key
+			if (pk_names.size() != fk.fk_columns.size()) {
+				auto pk_name_str = StringUtil::Join(pk_names, ",");
+				auto fk_name_str = StringUtil::Join(fk.fk_columns, ",");
+				throw BinderException(
+				    "Failed to create foreign key: number of referencing (%s) and referenced columns (%s) differ",
+				    fk_name_str, pk_name_str);
+			}
 			fk.pk_columns = pk_names;
 			return;
 		}
-		if (fk.pk_columns != pk_names) {
-			// Name mismatch
+		if (pk_names.size() != fk.fk_columns.size()) {
+			// the number of referencing and referenced columns for foreign keys must be the same
+			continue;
+		}
+		bool equals = true;
+		for (idx_t i = 0; i < fk.pk_columns.size(); i++) {
+			if (!StringUtil::CIEquals(fk.pk_columns[i], pk_names[i])) {
+				equals = false;
+				break;
+			}
+		}
+		if (!equals) {
 			continue;
 		}
 		// found match
@@ -540,7 +553,7 @@ BoundStatement Binder::Bind(CreateStatement &stmt) {
 			D_ASSERT(fk.info.pk_keys.empty());
 			D_ASSERT(fk.info.fk_keys.empty());
 			FindForeignKeyIndexes(create_info.columns, fk.fk_columns, fk.info.fk_keys);
-			if (create_info.table == fk.info.table) {
+			if (StringUtil::CIEquals(create_info.table, fk.info.table)) {
 				// self-referential foreign key constraint
 				fk.info.type = ForeignKeyType::FK_TYPE_SELF_REFERENCE_TABLE;
 				FindMatchingPrimaryKeyColumns(create_info.columns, create_info.constraints, fk);

package/src/duckdb/src/planner/binder/statement/bind_insert.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/function/table/table_scan.hpp"
+#include "duckdb/planner/operator/logical_dummy_scan.hpp"
 #include "duckdb/planner/operator/logical_projection.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/expression_binder/returning_binder.hpp"
@@ -409,7 +410,7 @@ BoundStatement Binder::Bind(InsertStatement &stmt) {
 	AddCTEMap(stmt.cte_map);
 
 	vector<LogicalIndex> named_column_map;
-	if (!stmt.columns.empty()) {
+	if (!stmt.columns.empty() || stmt.default_values) {
 		// insertion statement specifies column list
 
 		// create a mapping of (list index) -> (column index)
@@ -448,11 +449,10 @@ BoundStatement Binder::Bind(InsertStatement &stmt) {
 
 	// bind the default values
 	BindDefaultValues(table.GetColumns(), insert->bound_defaults);
-	if (!stmt.select_statement) {
+	if (!stmt.select_statement && !stmt.default_values) {
 		result.plan = std::move(insert);
 		return result;
 	}
-
 	// Exclude the generated columns from this amount
 	idx_t expected_columns = stmt.columns.empty() ? table.GetColumns().PhysicalColumnCount() : stmt.columns.size();
 
@@ -488,14 +488,19 @@ BoundStatement Binder::Bind(InsertStatement &stmt) {
 	}
 
 	// parse select statement and add to logical plan
-	auto select_binder = Binder::CreateBinder(context, this);
-	auto root_select = select_binder->Bind(*stmt.select_statement);
-	MoveCorrelatedExpressions(*select_binder);
+	unique_ptr<LogicalOperator> root;
+	if (stmt.select_statement) {
+		auto select_binder = Binder::CreateBinder(context, this);
+		auto root_select = select_binder->Bind(*stmt.select_statement);
+		MoveCorrelatedExpressions(*select_binder);
 
-	CheckInsertColumnCountMismatch(expected_columns, root_select.types.size(), !stmt.columns.empty(),
-	                               table.name.c_str());
+		CheckInsertColumnCountMismatch(expected_columns, root_select.types.size(), !stmt.columns.empty(),
+		                               table.name.c_str());
 
-	auto root = CastLogicalOperatorToTypes(root_select.types, insert->expected_types, std::move(root_select.plan));
+		root = CastLogicalOperatorToTypes(root_select.types, insert->expected_types, std::move(root_select.plan));
+	} else {
+		root = make_uniq<LogicalDummyScan>(GenerateTableIndex());
+	}
 	insert->AddChild(std::move(root));
 
 	BindOnConflictClause(*insert, table, stmt);

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -338,27 +338,29 @@ void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
 
 void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader) {
 
-	// Deserialize the index meta data
+	// deserialize the index metadata
 	auto info = IndexCatalogEntry::Deserialize(reader, context);
 
-	// Create index in the catalog
+	// create the index in the catalog
 	auto &schema_catalog = catalog.GetSchema(context, info->schema);
 	auto &table_catalog = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, info->schema, info->table->table_name)
 	                          .Cast<DuckTableEntry>();
 	auto &index_catalog = schema_catalog.CreateIndex(context, *info, table_catalog)->Cast<DuckIndexEntry>();
 	index_catalog.info = table_catalog.GetStorage().info;
-	// Here we just gotta read the root node
+
+	// we deserialize the index lazily, i.e., we do not need to load any node information
+	// except the root block id and offset
 	auto root_block_id = reader.Read<block_id_t>();
 	auto root_offset = reader.Read<uint32_t>();
 
-	// create an adaptive radix tree around the expressions
+	// obtain the expressions of the ART from the index metadata
 	vector<unique_ptr<Expression>> unbound_expressions;
 	vector<unique_ptr<ParsedExpression>> parsed_expressions;
-
 	for (auto &p_exp : info->parsed_expressions) {
 		parsed_expressions.push_back(p_exp->Copy());
 	}
 
+	// bind the parsed expressions
 	auto binder = Binder::CreateBinder(context);
 	auto &table_ref = info->table->Cast<TableRef>();
 	auto bound_table = binder->Bind(table_ref);
@@ -370,8 +372,7 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 	}
 
 	if (parsed_expressions.empty()) {
-		// If no parsed_expressions are present, this means this is a PK/FK index, so we create the necessary bound
-		// column refs
+		// this is a PK/FK index: we create the necessary bound column ref expressions
 		unbound_expressions.reserve(info->column_ids.size());
 		for (idx_t key_nr = 0; key_nr < info->column_ids.size(); key_nr++) {
 			auto &col = table_catalog.GetColumn(LogicalIndex(info->column_ids[key_nr]));
@@ -380,17 +381,18 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 		}
 	}
 
+	// create the index and add it to the storage
 	switch (info->index_type) {
 	case IndexType::ART: {
 		auto &storage = table_catalog.GetStorage();
 		auto art = make_uniq<ART>(info->column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
-		                          info->constraint_type, storage.db, true, root_block_id, root_offset);
+		                          info->constraint_type, storage.db, root_block_id, root_offset);
 		index_catalog.index = art.get();
 		storage.info->indexes.AddIndex(std::move(art));
 		break;
 	}
 	default:
-		throw InternalException("Can't read this index type");
+		throw InternalException("Unknown index type for ReadIndex");
 	}
 }
 

package/src/duckdb/src/storage/data_table.cpp

@@ -1162,9 +1162,7 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 
 	auto &allocator = Allocator::Get(db);
 
-	DataChunk result;
-	result.Initialize(allocator, index->logical_types);
-
+	// intermediate holds scanned chunks of the underlying data to create the index
 	DataChunk intermediate;
 	vector<LogicalType> intermediate_types;
 	auto column_ids = index->column_ids;
@@ -1176,6 +1174,10 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 	intermediate_types.emplace_back(LogicalType::ROW_TYPE);
 	intermediate.Initialize(allocator, intermediate_types);
 
+	// holds the result of executing the index expression on the intermediate chunks
+	DataChunk result;
+	result.Initialize(allocator, index->logical_types);
+
 	// initialize an index scan
 	CreateIndexScanState state;
 	InitializeWALCreateIndexScan(state, column_ids);
@@ -1209,6 +1211,7 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 			}
 		}
 	}
+
 	info->indexes.AddIndex(std::move(index));
 }
 

package/src/duckdb/src/storage/index.cpp

@@ -10,10 +10,10 @@ namespace duckdb {
 
 Index::Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_manager,
              const vector<column_t> &column_ids_p, const vector<unique_ptr<Expression>> &unbound_expressions,
-             IndexConstraintType constraint_type_p, bool track_memory)
+             IndexConstraintType constraint_type_p)
 
     : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p),
-      db(db), buffer_manager(BufferManager::GetBufferManager(db)), memory_size(0), track_memory(track_memory) {
+      db(db), buffer_manager(BufferManager::GetBufferManager(db)) {
 
 	for (auto &expr : unbound_expressions) {
 		types.push_back(expr->return_type.InternalType());
@@ -49,19 +49,31 @@ void Index::Delete(DataChunk &entries, Vector &row_identifiers) {
 }
 
 bool Index::MergeIndexes(Index &other_index) {
+
 	IndexLock state;
 	InitializeLock(state);
 
 	switch (this->type) {
-	case IndexType::ART: {
-		auto &art = Cast<ART>();
-		return art.MergeIndexes(state, other_index);
-	}
+	case IndexType::ART:
+		return Cast<ART>().MergeIndexes(state, other_index);
 	default:
 		throw InternalException("Unimplemented index type for merge");
 	}
 }
 
+void Index::Vacuum() {
+
+	IndexLock state;
+	InitializeLock(state);
+
+	switch (this->type) {
+	case IndexType::ART:
+		return Cast<ART>().Vacuum(state);
+	default:
+		throw InternalException("Unimplemented index type for vacuum");
+	}
+}
+
 void Index::ExecuteExpressions(DataChunk &input, DataChunk &result) {
 	executor.Execute(input, result);
 }

package/src/duckdb/src/storage/local_storage.cpp

@@ -118,6 +118,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 	row_groups = make_shared<RowGroupCollection>(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(),
 	                                             types, MAX_ROW_ID, 0);
 	row_groups->InitializeEmpty();
+
 	table.info->indexes.Scan([&](Index &index) {
 		D_ASSERT(index.type == IndexType::ART);
 		auto &art = index.Cast<ART>();
@@ -129,7 +130,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 				unbound_expressions.push_back(expr->Copy());
 			}
 			indexes.AddIndex(make_uniq<ART>(art.column_ids, art.table_io_manager, std::move(unbound_expressions),
-			                                art.constraint_type, art.db, true));
+			                                art.constraint_type, art.db));
 		}
 		return false;
 	});
@@ -520,6 +521,12 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 		storage.AppendToIndexes(transaction, append_state, append_count, true);
 	}
 	transaction.PushAppend(table, append_state.row_start, append_count);
+
+	// possibly vacuum any excess index data
+	table.info->indexes.Scan([&](Index &index) {
+		index.Vacuum();
+		return false;
+	});
 }
 
 void LocalStorage::Commit(LocalStorage::CommitState &commit_state, DuckTransaction &transaction) {
@@ -531,7 +538,6 @@ void LocalStorage::Commit(LocalStorage::CommitState &commit_state, DuckTransacti
 		auto table = entry.first;
 		auto storage = entry.second.get();
 		Flush(table, *storage);
-
 		entry.second.reset();
 	}
 }

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -229,15 +229,6 @@ void StandardBufferManager::Unpin(shared_ptr<BlockHandle> &handle) {
 	}
 }
 
-// POTENTIALLY PROBLEMATIC
-void StandardBufferManager::IncreaseUsedMemory(idx_t size, bool unsafe) {
-	ReserveMemory(size);
-}
-
-void StandardBufferManager::DecreaseUsedMemory(idx_t size) {
-	FreeReservedMemory(size);
-}
-
 void StandardBufferManager::SetLimit(idx_t limit) {
 	buffer_pool.SetLimit(limit, InMemoryWarning());
 }

package/src/duckdb/src/storage/wal_replay.cpp

@@ -417,7 +417,7 @@ void ReplayState::ReplayCreateIndex() {
 	switch (info->index_type) {
 	case IndexType::ART: {
 		index = make_uniq<ART>(info->column_ids, TableIOManager::Get(data_table), expressions, info->constraint_type,
-		                       data_table.db, true);
+		                       data_table.db);
 		break;
 	}
 	default:

package/src/duckdb/src/transaction/cleanup_state.cpp

@@ -52,15 +52,21 @@ void CleanupState::CleanupDelete(DeleteInfo &info) {
 	auto version_table = info.table;
 	D_ASSERT(version_table->info->cardinality >= info.count);
 	version_table->info->cardinality -= info.count;
+
 	if (version_table->info->indexes.Empty()) {
 		// this table has no indexes: no cleanup to be done
 		return;
 	}
+
 	if (current_table != version_table) {
 		// table for this entry differs from previous table: flush and switch to the new table
 		Flush();
 		current_table = version_table;
 	}
+
+	// possibly vacuum any indexes in this table later
+	indexed_tables[current_table->info->table] = current_table;
+
 	count = 0;
 	for (idx_t i = 0; i < info.count; i++) {
 		row_numbers[count++] = info.vinfo->start + info.rows[i];

package/src/duckdb/src/transaction/undo_buffer.cpp

@@ -126,6 +126,14 @@ void UndoBuffer::Cleanup() {
 	CleanupState state;
 	UndoBuffer::IteratorState iterator_state;
 	IterateEntries(iterator_state, [&](UndoFlags type, data_ptr_t data) { state.CleanupEntry(type, data); });
+
+	// possibly vacuum indexes
+	for (const auto &table : state.indexed_tables) {
+		table.second->info->indexes.Scan([&](Index &index) {
+			index.Vacuum();
+			return false;
+		});
+	}
 }
 
 void UndoBuffer::Commit(UndoBuffer::IteratorState &iterator_state, optional_ptr<WriteAheadLog> log,