duckdb 0.7.2-dev2552.0 → 0.7.2-dev2699.0

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -338,27 +338,29 @@ void CheckpointWriter::WriteIndex(IndexCatalogEntry &index_catalog) {
 
 void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader) {
 
-	// Deserialize the index meta data
+	// deserialize the index metadata
 	auto info = IndexCatalogEntry::Deserialize(reader, context);
 
-	// Create index in the catalog
+	// create the index in the catalog
 	auto &schema_catalog = catalog.GetSchema(context, info->schema);
 	auto &table_catalog = catalog.GetEntry(context, CatalogType::TABLE_ENTRY, info->schema, info->table->table_name)
 	                          .Cast<DuckTableEntry>();
 	auto &index_catalog = schema_catalog.CreateIndex(context, *info, table_catalog)->Cast<DuckIndexEntry>();
 	index_catalog.info = table_catalog.GetStorage().info;
-	// Here we just gotta read the root node
+
+	// we deserialize the index lazily, i.e., we do not need to load any node information
+	// except the root block id and offset
 	auto root_block_id = reader.Read<block_id_t>();
 	auto root_offset = reader.Read<uint32_t>();
 
-	// create an adaptive radix tree around the expressions
+	// obtain the expressions of the ART from the index metadata
 	vector<unique_ptr<Expression>> unbound_expressions;
 	vector<unique_ptr<ParsedExpression>> parsed_expressions;
-
 	for (auto &p_exp : info->parsed_expressions) {
 		parsed_expressions.push_back(p_exp->Copy());
 	}
 
+	// bind the parsed expressions
 	auto binder = Binder::CreateBinder(context);
 	auto &table_ref = info->table->Cast<TableRef>();
 	auto bound_table = binder->Bind(table_ref);
@@ -370,8 +372,7 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 	}
 
 	if (parsed_expressions.empty()) {
-		// If no parsed_expressions are present, this means this is a PK/FK index, so we create the necessary bound
-		// column refs
+		// this is a PK/FK index: we create the necessary bound column ref expressions
 		unbound_expressions.reserve(info->column_ids.size());
 		for (idx_t key_nr = 0; key_nr < info->column_ids.size(); key_nr++) {
 			auto &col = table_catalog.GetColumn(LogicalIndex(info->column_ids[key_nr]));
@@ -380,17 +381,18 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
 		}
 	}
 
+	// create the index and add it to the storage
 	switch (info->index_type) {
 	case IndexType::ART: {
 		auto &storage = table_catalog.GetStorage();
 		auto art = make_uniq<ART>(info->column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
-		                          info->constraint_type, storage.db, true, root_block_id, root_offset);
+		                          info->constraint_type, storage.db, root_block_id, root_offset);
 		index_catalog.index = art.get();
 		storage.info->indexes.AddIndex(std::move(art));
 		break;
 	}
 	default:
-		throw InternalException("Can't read this index type");
+		throw InternalException("Unknown index type for ReadIndex");
 	}
 }
 

package/src/duckdb/src/storage/data_table.cpp

@@ -1162,9 +1162,7 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 
 	auto &allocator = Allocator::Get(db);
 
-	DataChunk result;
-	result.Initialize(allocator, index->logical_types);
-
+	// intermediate holds scanned chunks of the underlying data to create the index
 	DataChunk intermediate;
 	vector<LogicalType> intermediate_types;
 	auto column_ids = index->column_ids;
@@ -1176,6 +1174,10 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 	intermediate_types.emplace_back(LogicalType::ROW_TYPE);
 	intermediate.Initialize(allocator, intermediate_types);
 
+	// holds the result of executing the index expression on the intermediate chunks
+	DataChunk result;
+	result.Initialize(allocator, index->logical_types);
+
 	// initialize an index scan
 	CreateIndexScanState state;
 	InitializeWALCreateIndexScan(state, column_ids);
@@ -1209,6 +1211,7 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
 			}
 		}
 	}
+
 	info->indexes.AddIndex(std::move(index));
 }
 

package/src/duckdb/src/storage/index.cpp

@@ -10,10 +10,10 @@ namespace duckdb {
 
 Index::Index(AttachedDatabase &db, IndexType type, TableIOManager &table_io_manager,
              const vector<column_t> &column_ids_p, const vector<unique_ptr<Expression>> &unbound_expressions,
-             IndexConstraintType constraint_type_p, bool track_memory)
+             IndexConstraintType constraint_type_p)
 
     : type(type), table_io_manager(table_io_manager), column_ids(column_ids_p), constraint_type(constraint_type_p),
-      db(db), buffer_manager(BufferManager::GetBufferManager(db)), memory_size(0), track_memory(track_memory) {
+      db(db), buffer_manager(BufferManager::GetBufferManager(db)) {
 
 	for (auto &expr : unbound_expressions) {
 		types.push_back(expr->return_type.InternalType());
@@ -49,19 +49,31 @@ void Index::Delete(DataChunk &entries, Vector &row_identifiers) {
 }
 
 bool Index::MergeIndexes(Index &other_index) {
+
 	IndexLock state;
 	InitializeLock(state);
 
 	switch (this->type) {
-	case IndexType::ART: {
-		auto &art = Cast<ART>();
-		return art.MergeIndexes(state, other_index);
-	}
+	case IndexType::ART:
+		return Cast<ART>().MergeIndexes(state, other_index);
 	default:
 		throw InternalException("Unimplemented index type for merge");
 	}
 }
 
+void Index::Vacuum() {
+
+	IndexLock state;
+	InitializeLock(state);
+
+	switch (this->type) {
+	case IndexType::ART:
+		return Cast<ART>().Vacuum(state);
+	default:
+		throw InternalException("Unimplemented index type for vacuum");
+	}
+}
+
 void Index::ExecuteExpressions(DataChunk &input, DataChunk &result) {
 	executor.Execute(input, result);
 }

package/src/duckdb/src/storage/local_storage.cpp

@@ -118,6 +118,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 	row_groups = make_shared<RowGroupCollection>(table.info, TableIOManager::Get(table).GetBlockManagerForRowData(),
 	                                             types, MAX_ROW_ID, 0);
 	row_groups->InitializeEmpty();
+
 	table.info->indexes.Scan([&](Index &index) {
 		D_ASSERT(index.type == IndexType::ART);
 		auto &art = index.Cast<ART>();
@@ -129,7 +130,7 @@ LocalTableStorage::LocalTableStorage(DataTable &table)
 				unbound_expressions.push_back(expr->Copy());
 			}
 			indexes.AddIndex(make_uniq<ART>(art.column_ids, art.table_io_manager, std::move(unbound_expressions),
-			                                art.constraint_type, art.db, true));
+			                                art.constraint_type, art.db));
 		}
 		return false;
 	});
@@ -520,6 +521,12 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 		storage.AppendToIndexes(transaction, append_state, append_count, true);
 	}
 	transaction.PushAppend(table, append_state.row_start, append_count);
+
+	// possibly vacuum any excess index data
+	table.info->indexes.Scan([&](Index &index) {
+		index.Vacuum();
+		return false;
+	});
 }
 
 void LocalStorage::Commit(LocalStorage::CommitState &commit_state, DuckTransaction &transaction) {
@@ -531,7 +538,6 @@ void LocalStorage::Commit(LocalStorage::CommitState &commit_state, DuckTransacti
 		auto table = entry.first;
 		auto storage = entry.second.get();
 		Flush(table, *storage);
-
 		entry.second.reset();
 	}
 }

package/src/duckdb/src/storage/standard_buffer_manager.cpp

@@ -229,15 +229,6 @@ void StandardBufferManager::Unpin(shared_ptr<BlockHandle> &handle) {
 	}
 }
 
-// POTENTIALLY PROBLEMATIC
-void StandardBufferManager::IncreaseUsedMemory(idx_t size, bool unsafe) {
-	ReserveMemory(size);
-}
-
-void StandardBufferManager::DecreaseUsedMemory(idx_t size) {
-	FreeReservedMemory(size);
-}
-
 void StandardBufferManager::SetLimit(idx_t limit) {
 	buffer_pool.SetLimit(limit, InMemoryWarning());
 }

package/src/duckdb/src/storage/wal_replay.cpp

@@ -417,7 +417,7 @@ void ReplayState::ReplayCreateIndex() {
 	switch (info->index_type) {
 	case IndexType::ART: {
 		index = make_uniq<ART>(info->column_ids, TableIOManager::Get(data_table), expressions, info->constraint_type,
-		                       data_table.db, true);
+		                       data_table.db);
 		break;
 	}
 	default:

package/src/duckdb/src/transaction/cleanup_state.cpp

@@ -52,15 +52,21 @@ void CleanupState::CleanupDelete(DeleteInfo &info) {
 	auto version_table = info.table;
 	D_ASSERT(version_table->info->cardinality >= info.count);
 	version_table->info->cardinality -= info.count;
+
 	if (version_table->info->indexes.Empty()) {
 		// this table has no indexes: no cleanup to be done
 		return;
 	}
+
 	if (current_table != version_table) {
 		// table for this entry differs from previous table: flush and switch to the new table
 		Flush();
 		current_table = version_table;
 	}
+
+	// possibly vacuum any indexes in this table later
+	indexed_tables[current_table->info->table] = current_table;
+
 	count = 0;
 	for (idx_t i = 0; i < info.count; i++) {
 		row_numbers[count++] = info.vinfo->start + info.rows[i];

package/src/duckdb/src/transaction/undo_buffer.cpp

@@ -126,6 +126,14 @@ void UndoBuffer::Cleanup() {
 	CleanupState state;
 	UndoBuffer::IteratorState iterator_state;
 	IterateEntries(iterator_state, [&](UndoFlags type, data_ptr_t data) { state.CleanupEntry(type, data); });
+
+	// possibly vacuum indexes
+	for (const auto &table : state.indexed_tables) {
+		table.second->info->indexes.Scan([&](Index &index) {
+			index.Vacuum();
+			return false;
+		});
+	}
 }
 
 void UndoBuffer::Commit(UndoBuffer::IteratorState &iterator_state, optional_ptr<WriteAheadLog> log,

package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp

@@ -352,13 +352,13 @@
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
 
+#include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
+
 #include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
-
-#include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
 

package/src/duckdb/ub_src_execution_index_art.cpp

@@ -1,10 +1,14 @@
 #include "src/execution/index/art/art_key.cpp"
 
+#include "src/execution/index/art/node.cpp"
+
+#include "src/execution/index/art/fixed_size_allocator.cpp"
+
 #include "src/execution/index/art/iterator.cpp"
 
 #include "src/execution/index/art/leaf.cpp"
 
-#include "src/execution/index/art/node.cpp"
+#include "src/execution/index/art/leaf_segment.cpp"
 
 #include "src/execution/index/art/node4.cpp"
 
@@ -18,5 +22,7 @@
 
 #include "src/execution/index/art/prefix.cpp"
 
+#include "src/execution/index/art/prefix_segment.cpp"
+
 #include "src/execution/index/art/art.cpp"