duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -40,7 +40,7 @@ public:
 protected:
 	virtual void WriteEntry(CatalogEntry &entry, Serializer &serializer);
 	virtual void WriteSchema(SchemaCatalogEntry &schema, Serializer &serializer);
-	virtual void WriteTable(TableCatalogEntry &table, Serializer &serializer);
+	virtual void WriteTable(TableCatalogEntry &table, Serializer &serializer) = 0;
 	virtual void WriteView(ViewCatalogEntry &table, Serializer &serializer);
 	virtual void WriteSequence(SequenceCatalogEntry &table, Serializer &serializer);
 	virtual void WriteMacro(ScalarMacroCatalogEntry &table, Serializer &serializer);
@@ -60,18 +60,19 @@ protected:
 	Catalog &catalog;
 
 protected:
-	virtual void LoadCheckpoint(ClientContext &context, MetadataReader &reader);
-	virtual void ReadEntry(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadSchema(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadTable(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadView(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadSequence(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadMacro(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadTableMacro(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadIndex(ClientContext &context, Deserializer &deserializer);
-	virtual void ReadType(ClientContext &context, Deserializer &deserializer);
-
-	virtual void ReadTableData(ClientContext &context, Deserializer &deserializer, BoundCreateTableInfo &bound_info);
+	virtual void LoadCheckpoint(CatalogTransaction transaction, MetadataReader &reader);
+	virtual void ReadEntry(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadSchema(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadTable(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadView(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadSequence(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadMacro(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadTableMacro(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadIndex(CatalogTransaction transaction, Deserializer &deserializer);
+	virtual void ReadType(CatalogTransaction transaction, Deserializer &deserializer);
+
+	virtual void ReadTableData(CatalogTransaction transaction, Deserializer &deserializer,
+	                           BoundCreateTableInfo &bound_info);
 };
 
 class SingleFileCheckpointReader final : public CheckpointReader {
@@ -80,7 +81,7 @@ public:
 	    : CheckpointReader(Catalog::GetCatalog(storage.GetAttached())), storage(storage) {
 	}
 
-	void LoadFromStorage(optional_ptr<ClientContext> context = nullptr);
+	void LoadFromStorage();
 	MetadataManager &GetMetadataManager();
 
 	//! The database
@@ -96,7 +97,7 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
 	friend class SingleFileTableDataWriter;
 
 public:
-	SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager);
+	SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager, CheckpointType checkpoint_type);
 
 	//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
 	//! connection is available because right now the checkpointing cannot be done online. (TODO)
@@ -107,6 +108,12 @@ public:
 	unique_ptr<TableDataWriter> GetTableDataWriter(TableCatalogEntry &table) override;
 
 	BlockManager &GetBlockManager();
+	CheckpointType GetCheckpointType() const {
+		return checkpoint_type;
+	}
+
+public:
+	void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
 
 private:
 	//! The metadata writer is responsible for writing schema information
@@ -116,6 +123,8 @@ private:
 	//! Because this is single-file storage, we can share partial blocks across
 	//! an entire checkpoint.
 	PartialBlockManager partial_block_manager;
+	//! Checkpoint type
+	CheckpointType checkpoint_type;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/compression/alp/algorithm/alp.hpp

@@ -107,10 +107,10 @@ struct AlpCompression {
 	 */
 	static int64_t NumberToInt64(T n) {
 		if (IsImpossibleToEncode(n)) {
-			return AlpConstants::ENCODING_UPPER_LIMIT;
+			return NumericCast<int64_t>(AlpConstants::ENCODING_UPPER_LIMIT);
 		}
 		n = n + AlpTypedConstants<T>::MAGIC_NUMBER - AlpTypedConstants<T>::MAGIC_NUMBER;
-		return static_cast<int64_t>(n);
+		return NumericCast<int64_t>(n);
 	}
 
 	/*
@@ -185,7 +185,7 @@ struct AlpCompression {
 
 		// Evaluate factor/exponent compression size (we optimize for FOR)
 		uint64_t delta = (static_cast<uint64_t>(max_encoded_value) - static_cast<uint64_t>(min_encoded_value));
-		estimated_bits_per_value = std::ceil(std::log2(delta + 1));
+		estimated_bits_per_value = NumericCast<uint32_t>(std::ceil(std::log2(delta + 1)));
 		estimated_compression_size += n_values * estimated_bits_per_value;
 		estimated_compression_size +=
 		    exceptions_count * (EXACT_TYPE_BITSIZE + (AlpConstants::EXCEPTION_POSITION_SIZE * 8));
@@ -254,7 +254,8 @@ struct AlpCompression {
 	static void FindBestFactorAndExponent(const T *input_vector, idx_t n_values, State &state) {
 		//! We sample equidistant values within a vector; to do this we skip a fixed number of values
 		vector<T> vector_sample;
-		uint32_t idx_increments = MaxValue(1, (int32_t)std::ceil((double)n_values / AlpConstants::SAMPLES_PER_VECTOR));
+		auto idx_increments = MaxValue<uint32_t>(
+		    1, UnsafeNumericCast<uint32_t>(std::ceil((double)n_values / AlpConstants::SAMPLES_PER_VECTOR)));
 		for (idx_t i = 0; i < n_values; i += idx_increments) {
 			vector_sample.push_back(input_vector[i]);
 		}
@@ -358,9 +359,9 @@ struct AlpCompression {
 			BitpackingPrimitives::PackBuffer<uint64_t, false>(state.values_encoded, u_encoded_integers, n_values,
 			                                                  bit_width);
 		}
-		state.bit_width = bit_width; // in bits
-		state.bp_size = bp_size;     // in bytes
-		state.frame_of_reference = min_value;
+		state.bit_width = bit_width;                                 // in bits
+		state.bp_size = bp_size;                                     // in bytes
+		state.frame_of_reference = static_cast<uint64_t>(min_value); // understood this can be negative
 	}
 
 	/*

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_compress.hpp

@@ -183,10 +183,10 @@ public:
 		// Verify that the metadata_ptr is not smaller than the space used by the data
 		D_ASSERT(dataptr + metadata_offset <= metadata_ptr);
 
-		idx_t bytes_used_by_metadata = dataptr + Storage::BLOCK_SIZE - metadata_ptr;
+		auto bytes_used_by_metadata = UnsafeNumericCast<idx_t>(dataptr + Storage::BLOCK_SIZE - metadata_ptr);
 
 		// Initially the total segment size is the size of the block
-		idx_t total_segment_size = Storage::BLOCK_SIZE;
+		auto total_segment_size = Storage::BLOCK_SIZE;
 
 		//! We compact the block if the space used is less than a threshold
 		const auto used_space_percentage =

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_constants.hpp

@@ -70,11 +70,12 @@ struct AlpTypedConstants<float> {
 	static constexpr float MAGIC_NUMBER = 12582912.0; //! 2^22 + 2^23
 	static constexpr uint8_t MAX_EXPONENT = 10;
 
-	static constexpr const float EXP_ARR[] = {1.0,       10.0,       100.0,       1000.0,       10000.0,      100000.0,
-	                                          1000000.0, 10000000.0, 100000000.0, 1000000000.0, 10000000000.0};
+	static constexpr const float EXP_ARR[] = {1.0F,         10.0F,         100.0F,        1000.0F,
+	                                          10000.0F,     100000.0F,     1000000.0F,    10000000.0F,
+	                                          100000000.0F, 1000000000.0F, 10000000000.0F};
 
-	static constexpr float FRAC_ARR[] = {1.0,      0.1,       0.01,       0.001,       0.0001,      0.00001,
-	                                     0.000001, 0.0000001, 0.00000001, 0.000000001, 0.0000000001};
+	static constexpr float FRAC_ARR[] = {1.0F,      0.1F,       0.01F,       0.001F,       0.0001F,      0.00001F,
+	                                     0.000001F, 0.0000001F, 0.00000001F, 0.000000001F, 0.0000000001F};
 };
 
 template <>

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_fetch.hpp

@@ -28,7 +28,7 @@ void AlpFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id,
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
 
 	AlpScanState<T> scan_state(segment);
-	scan_state.Skip(segment, row_id);
+	scan_state.Skip(segment, UnsafeNumericCast<idx_t>(row_id));
 	auto result_data = FlatVector::GetData<EXACT_TYPE>(result);
 	result_data[result_idx] = (EXACT_TYPE)0;
 

package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_utils.hpp

@@ -40,9 +40,9 @@ public:
 		auto n_lookup_values =
 		    NumericCast<uint32_t>(MinValue(current_vector_n_values, (idx_t)AlpConstants::ALP_VECTOR_SIZE));
 		//! We sample equidistant values within a vector; to do this we jump a fixed number of values
-		uint32_t n_sampled_increments =
-		    MaxValue(1, (int32_t)std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR));
-		uint32_t n_sampled_values = std::ceil((double)n_lookup_values / n_sampled_increments);
+		uint32_t n_sampled_increments = MaxValue<uint32_t>(
+		    1, UnsafeNumericCast<uint32_t>(std::ceil((double)n_lookup_values / AlpConstants::SAMPLES_PER_VECTOR)));
+		uint32_t n_sampled_values = NumericCast<uint32_t>(std::ceil((double)n_lookup_values / n_sampled_increments));
 		D_ASSERT(n_sampled_values < AlpConstants::ALP_VECTOR_SIZE);
 
 		AlpSamplingParameters sampling_params = {n_lookup_values, n_sampled_increments, n_sampled_values};

package/src/duckdb/src/include/duckdb/storage/compression/alprd/algorithm/alprd.hpp

@@ -105,7 +105,8 @@ struct AlpRDCompression {
 		// The left parts bit width after compression is determined by how many elements are in the dictionary
 		uint64_t actual_dictionary_size =
 		    MinValue<uint64_t>(AlpRDConstants::MAX_DICTIONARY_SIZE, left_parts_sorted_repetitions.size());
-		uint8_t left_bit_width = MaxValue<uint8_t>(1, std::ceil(std::log2(actual_dictionary_size)));
+		uint8_t left_bit_width =
+		    MaxValue<uint8_t>(1, NumericCast<uint8_t>(std::ceil(std::log2(actual_dictionary_size))));
 
 		if (PERSIST_DICT) {
 			for (idx_t dict_idx = 0; dict_idx < actual_dictionary_size; dict_idx++) {

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp

@@ -126,13 +126,15 @@ idx_t AlpRDFinalAnalyze(AnalyzeState &state) {
 	//! Overhead per vector: Pointer to data + Exceptions count
 	double per_vector_overhead = AlpRDConstants::METADATA_POINTER_SIZE + AlpRDConstants::EXCEPTIONS_COUNT_SIZE;
 
-	uint32_t n_vectors = std::ceil((double)analyze_state.total_values_count / AlpRDConstants::ALP_VECTOR_SIZE);
+	uint32_t n_vectors =
+	    NumericCast<uint32_t>(std::ceil((double)analyze_state.total_values_count / AlpRDConstants::ALP_VECTOR_SIZE));
 
 	auto estimated_size = (estimed_compressed_bytes * factor_of_sampling) + (n_vectors * per_vector_overhead);
-	uint32_t estimated_n_blocks = std::ceil(estimated_size / (Storage::BLOCK_SIZE - per_segment_overhead));
+	uint32_t estimated_n_blocks =
+	    NumericCast<uint32_t>(std::ceil(estimated_size / (Storage::BLOCK_SIZE - per_segment_overhead)));
 
 	auto final_analyze_size = estimated_size + (estimated_n_blocks * per_segment_overhead);
-	return final_analyze_size;
+	return NumericCast<idx_t>(final_analyze_size);
 }
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp

@@ -185,10 +185,10 @@ public:
 		// Verify that the metadata_ptr is not smaller than the space used by the data
 		D_ASSERT(dataptr + metadata_offset <= metadata_ptr);
 
-		idx_t bytes_used_by_metadata = dataptr + Storage::BLOCK_SIZE - metadata_ptr;
+		auto bytes_used_by_metadata = UnsafeNumericCast<idx_t>(dataptr + Storage::BLOCK_SIZE - metadata_ptr);
 
 		// Initially the total segment size is the size of the block
-		idx_t total_segment_size = Storage::BLOCK_SIZE;
+		auto total_segment_size = Storage::BLOCK_SIZE;
 
 		//! We compact the block if the space used is less than a threshold
 		const auto used_space_percentage =

package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_fetch.hpp

@@ -27,7 +27,7 @@ template <class T>
 void AlpRDFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result, idx_t result_idx) {
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
 	AlpRDScanState<T> scan_state(segment);
-	scan_state.Skip(segment, row_id);
+	scan_state.Skip(segment, UnsafeNumericCast<idx_t>(row_id));
 	auto result_data = FlatVector::GetData<EXACT_TYPE>(result);
 	result_data[result_idx] = (EXACT_TYPE)0;
 

package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/bit_reader.hpp

@@ -150,7 +150,7 @@ public:
 			result = result << 8 | InnerReadByte(i);
 		}
 		result = result << remainder | InnerRead(remainder, bytes);
-		index += (bytes << 3) + remainder;
+		index += static_cast<uint32_t>(bytes << 3) + remainder;
 		return result;
 	}
 

package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_fetch.hpp

@@ -29,7 +29,7 @@ void ChimpFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id
 	using INTERNAL_TYPE = typename ChimpType<T>::TYPE;
 
 	ChimpScanState<T> scan_state(segment);
-	scan_state.Skip(segment, row_id);
+	scan_state.Skip(segment, UnsafeNumericCast<idx_t>(row_id));
 	auto result_data = FlatVector::GetData<INTERNAL_TYPE>(result);
 
 	if (scan_state.GroupFinished() && scan_state.total_value_count < scan_state.segment_count) {

package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_fetch.hpp

@@ -29,7 +29,7 @@ void PatasFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id
 	using EXACT_TYPE = typename FloatingToExact<T>::TYPE;
 
 	PatasScanState<T> scan_state(segment);
-	scan_state.Skip(segment, row_id);
+	scan_state.Skip(segment, UnsafeNumericCast<idx_t>(row_id));
 	auto result_data = FlatVector::GetData<EXACT_TYPE>(result);
 	result_data[result_idx] = (EXACT_TYPE)0;
 

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -41,6 +41,9 @@ class WriteAheadLog;
 class TableDataWriter;
 class ConflictManager;
 class TableScanState;
+struct TableDeleteState;
+struct ConstraintState;
+struct TableUpdateState;
 enum class VerifyExistenceType : uint8_t;
 
 //! DataTable represents a physical table on disk
@@ -60,16 +63,18 @@ public:
 	//! Constructs a DataTable as a delta on an existing data table but with one column added new constraint
 	explicit DataTable(ClientContext &context, DataTable &parent, unique_ptr<BoundConstraint> constraint);
 
-	//! The table info
-	shared_ptr<DataTableInfo> info;
-	//! The set of physical columns stored by this DataTable
-	vector<ColumnDefinition> column_definitions;
 	//! A reference to the database instance
 	AttachedDatabase &db;
 
 public:
+	AttachedDatabase &GetAttached();
+	TableIOManager &GetTableIOManager();
+
+	bool IsTemporary() const;
+
 	//! Returns a list of types of the table
 	vector<LogicalType> GetTypes();
+	const vector<ColumnDefinition> &Columns() const;
 
 	void InitializeScan(TableScanState &state, const vector<column_t> &column_ids,
 	                    TableFilterSet *table_filter = nullptr);
@@ -92,26 +97,34 @@ public:
 	           const Vector &row_ids, idx_t fetch_count, ColumnFetchState &state);
 
 	//! Initializes an append to transaction-local storage
-	void InitializeLocalAppend(LocalAppendState &state, ClientContext &context);
+	void InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context,
+	                           const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Append a DataChunk to the transaction-local storage of the table.
 	void LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
 	                 bool unsafe = false);
 	//! Finalizes a transaction-local append
 	void FinalizeLocalAppend(LocalAppendState &state);
 	//! Append a chunk to the transaction-local storage of this table
-	void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk);
+	void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Append a column data collection to the transaction-local storage of this table
-	void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection);
+	void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection,
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Merge a row group collection into the transaction-local storage
 	void LocalMerge(ClientContext &context, RowGroupCollection &collection);
 	//! Creates an optimistic writer for this table - used for optimistically writing parallel appends
 	OptimisticDataWriter &CreateOptimisticWriter(ClientContext &context);
 	void FinalizeOptimisticWriter(ClientContext &context, OptimisticDataWriter &writer);
 
+	unique_ptr<TableDeleteState> InitializeDelete(TableCatalogEntry &table, ClientContext &context,
+	                                              const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Delete the entries with the specified row identifier from the table
-	idx_t Delete(TableCatalogEntry &table, ClientContext &context, Vector &row_ids, idx_t count);
+	idx_t Delete(TableDeleteState &state, ClientContext &context, Vector &row_ids, idx_t count);
+
+	unique_ptr<TableUpdateState> InitializeUpdate(TableCatalogEntry &table, ClientContext &context,
+	                                              const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Update the entries with the specified row identifier from the table
-	void Update(TableCatalogEntry &table, ClientContext &context, Vector &row_ids,
+	void Update(TableUpdateState &state, ClientContext &context, Vector &row_ids,
 	            const vector<PhysicalIndex> &column_ids, DataChunk &data);
 	//! Update a single (sub-)column along a column path
 	//! The column_path vector is a *path* towards a column within the table
@@ -170,12 +183,17 @@ public:
 	//! Sets statistics of a physical column within the table
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
+	//! Obtains a shared lock to prevent checkpointing while operations are running
+	unique_ptr<StorageLockKey> GetSharedCheckpointLock();
+	//! Obtains a lock during a checkpoint operation that prevents other threads from reading this table
+	unique_ptr<StorageLockKey> GetCheckpointLock();
 	//! Checkpoint the table to the specified table data writer
 	void Checkpoint(TableDataWriter &writer, Serializer &serializer);
 	void CommitDropTable();
 	void CommitDropColumn(idx_t index);
 
-	idx_t GetTotalRows();
+	idx_t ColumnCount() const;
+	idx_t GetTotalRows() const;
 
 	vector<ColumnSegmentInfo> GetColumnSegmentInfo();
 	static bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type);
@@ -186,22 +204,39 @@ public:
 	//! FIXME: This is only necessary until we treat all indexes as catalog entries, allowing to alter constraints
 	bool IndexNameIsUnique(const string &name);
 
+	//! Initialize constraint verification state
+	unique_ptr<ConstraintState> InitializeConstraintState(TableCatalogEntry &table,
+	                                                      const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Verify constraints with a chunk from the Append containing all columns of the table
-	void VerifyAppendConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
-	                             ConflictManager *conflict_manager = nullptr);
+	void VerifyAppendConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk,
+	                             optional_ptr<ConflictManager> conflict_manager = nullptr);
+
+	shared_ptr<DataTableInfo> &GetDataTableInfo();
+
+	void InitializeIndexes(ClientContext &context);
+	bool HasIndexes() const;
+	void AddIndex(unique_ptr<Index> index);
+	bool HasForeignKeyIndex(const vector<PhysicalIndex> &keys, ForeignKeyType type);
+	void SetIndexStorageInfo(vector<IndexStorageInfo> index_storage_info);
+	void VacuumIndexes();
+
+	string GetTableName() const;
+	void SetTableName(string new_name);
+
+	TableStorageInfo GetStorageInfo();
 
 public:
 	static void VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &context, DataChunk &chunk,
-	                                ConflictManager *conflict_manager);
+	                                optional_ptr<ConflictManager> conflict_manager);
 
 private:
 	//! Verify the new added constraints against current persistent&local data
-	void VerifyNewConstraint(ClientContext &context, DataTable &parent, const BoundConstraint *constraint);
+	void VerifyNewConstraint(LocalStorage &local_storage, DataTable &parent, const BoundConstraint &constraint);
 	//! Verify constraints with a chunk from the Update containing only the specified column_ids
-	void VerifyUpdateConstraints(ClientContext &context, TableCatalogEntry &table, DataChunk &chunk,
+	void VerifyUpdateConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk,
 	                             const vector<PhysicalIndex> &column_ids);
 	//! Verify constraints with a chunk from the Delete containing all columns of the table
-	void VerifyDeleteConstraints(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk);
+	void VerifyDeleteConstraints(TableDeleteState &state, ClientContext &context, DataChunk &chunk);
 
 	void InitializeScanWithOffset(TableScanState &state, const vector<column_t> &column_ids, idx_t start_row,
 	                              idx_t end_row);
@@ -214,6 +249,10 @@ private:
 	                                      DataChunk &chunk);
 
 private:
+	//! The table info
+	shared_ptr<DataTableInfo> info;
+	//! The set of physical columns stored by this DataTable
+	vector<ColumnDefinition> column_definitions;
 	//! Lock for appending entries to the table
 	mutex append_lock;
 	//! The row groups of the table

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -12,7 +12,6 @@
 #include "duckdb/common/types/constraint_conflict_info.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/unordered_set.hpp"
-#include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/parser/parsed_expression.hpp"
 #include "duckdb/planner/expression.hpp"
 #include "duckdb/storage/table_storage_info.hpp"
@@ -29,127 +28,64 @@ struct IndexScanState;
 
 //! The index is an abstract base class that serves as the basis for indexes
 class Index {
-public:
-	Index(const string &name, const string &index_type, IndexConstraintType index_constraint_type,
-	      const vector<column_t> &column_ids, TableIOManager &table_io_manager,
-	      const vector<unique_ptr<Expression>> &unbound_expressions, AttachedDatabase &db);
-	virtual ~Index() = default;
+protected:
+	Index(const vector<column_t> &column_ids, TableIOManager &table_io_manager, AttachedDatabase &db);
 
-	//! The name of the index
-	string name;
-	//! The index type (ART, B+-tree, Skip-List, ...)
-	string index_type;
-	//! The index constraint type
-	IndexConstraintType index_constraint_type;
 	//! The logical column ids of the indexed table
 	vector<column_t> column_ids;
-
-	//! Associated table io manager
-	TableIOManager &table_io_manager;
 	//! Unordered set of column_ids used by the index
 	unordered_set<column_t> column_id_set;
-	//! Unbound expressions used by the index during optimizations
-	vector<unique_ptr<Expression>> unbound_expressions;
-	//! The physical types stored in the index
-	vector<PhysicalType> types;
-	//! The logical types of the expressions
-	vector<LogicalType> logical_types;
 
+public:
+	//! Associated table io manager
+	TableIOManager &table_io_manager;
 	//! Attached database instance
 	AttachedDatabase &db;
 
 public:
-	//! Returns true if the index is a unknown index, and false otherwise
-	virtual bool IsUnknown() {
-		return false;
-	}
+	virtual ~Index() = default;
 
-	//! Obtain a lock on the index
-	void InitializeLock(IndexLock &state);
-	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
-	virtual ErrorData Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
-	//! Obtains a lock and calls Append while holding that lock
-	ErrorData Append(DataChunk &entries, Vector &row_identifiers);
-	//! Verify that data can be appended to the index without a constraint violation
-	virtual void VerifyAppend(DataChunk &chunk) = 0;
-	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
-	virtual void VerifyAppend(DataChunk &chunk, ConflictManager &conflict_manager) = 0;
-	//! Performs constraint checking for a chunk of input data
-	virtual void CheckConstraintsForChunk(DataChunk &input, ConflictManager &conflict_manager) = 0;
-
-	//! Deletes all data from the index. The lock obtained from InitializeLock must be held
-	virtual void CommitDrop(IndexLock &index_lock) = 0;
-	//! Deletes all data from the index
-	void CommitDrop();
-	//! Delete a chunk of entries from the index. The lock obtained from InitializeLock must be held
-	virtual void Delete(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
-	//! Obtains a lock and calls Delete while holding that lock
-	void Delete(DataChunk &entries, Vector &row_identifiers);
-
-	//! Insert a chunk of entries into the index
-	virtual ErrorData Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
-
-	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
-	//! index must also be locked during the merge
-	virtual bool MergeIndexes(IndexLock &state, Index &other_index) = 0;
-	//! Obtains a lock and calls MergeIndexes while holding that lock
-	bool MergeIndexes(Index &other_index);
-
-	//! Traverses an ART and vacuums the qualifying nodes. The lock obtained from InitializeLock must be held
-	virtual void Vacuum(IndexLock &state) = 0;
-	//! Obtains a lock and calls Vacuum while holding that lock
-	void Vacuum();
-
-	//! Returns the in-memory usage of the index. The lock obtained from InitializeLock must be held
-	virtual idx_t GetInMemorySize(IndexLock &state) = 0;
-	//! Returns the in-memory usage of the index
-	idx_t GetInMemorySize();
-
-	//! Returns the string representation of an index, or only traverses and verifies the index
-	virtual string VerifyAndToString(IndexLock &state, const bool only_verify) = 0;
-	//! Obtains a lock and calls VerifyAndToString while holding that lock
-	string VerifyAndToString(const bool only_verify);
-
-	//! Returns true if the index is affected by updates on the specified column IDs, and false otherwise
-	bool IndexIsUpdated(const vector<PhysicalIndex> &column_ids) const;
+	//! Returns true if the index is a bound index, and false otherwise
+	virtual bool IsBound() const = 0;
+
+	//! The index type (ART, B+-tree, Skip-List, ...)
+	virtual const string &GetIndexType() const = 0;
+
+	//! The name of the index
+	virtual const string &GetIndexName() const = 0;
+
+	//! The index constraint type
+	virtual IndexConstraintType GetConstraintType() const = 0;
 
 	//! Returns unique flag
-	bool IsUnique() {
+	bool IsUnique() const {
+		auto index_constraint_type = GetConstraintType();
 		return (index_constraint_type == IndexConstraintType::UNIQUE ||
 		        index_constraint_type == IndexConstraintType::PRIMARY);
 	}
+
 	//! Returns primary key flag
-	bool IsPrimary() {
+	bool IsPrimary() const {
+		auto index_constraint_type = GetConstraintType();
 		return (index_constraint_type == IndexConstraintType::PRIMARY);
 	}
+
 	//! Returns foreign key flag
-	bool IsForeign() {
+	bool IsForeign() const {
+		auto index_constraint_type = GetConstraintType();
 		return (index_constraint_type == IndexConstraintType::FOREIGN);
 	}
 
-	//! Returns all index storage information for serialization
-	virtual IndexStorageInfo GetStorageInfo(const bool get_buffers);
-
-	//! Execute the index expressions on an input chunk
-	void ExecuteExpressions(DataChunk &input, DataChunk &result);
-	static string AppendRowError(DataChunk &input, idx_t index);
-
-	//! Throw a constraint violation exception
-	virtual string GetConstraintViolationMessage(VerifyExistenceType verify_type, idx_t failed_index,
-	                                             DataChunk &input) = 0;
-
-protected:
-	//! Lock used for any changes to the index
-	mutex lock;
+	const vector<column_t> &GetColumnIds() const {
+		return column_ids;
+	}
 
-private:
-	//! Bound expressions used during expression execution
-	vector<unique_ptr<Expression>> bound_expressions;
-	//! Expression executor to execute the index expressions
-	ExpressionExecutor executor;
+	const unordered_set<column_t> &GetColumnIdSet() const {
+		return column_id_set;
+	}
 
-	//! Bind the unbound expressions of the index
-	unique_ptr<Expression> BindExpression(unique_ptr<Expression> expr);
+	// All indexes can be dropped, even if they are unbound
+	virtual void CommitDrop() = 0;
 
 public:
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -43,7 +43,7 @@ public:
 		if (!object || object->GetObjectType() != T::ObjectType()) {
 			return nullptr;
 		}
-		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
+		return shared_ptr_cast<ObjectCacheEntry, T>(object);
 	}
 
 	template <class T, class... ARGS>
@@ -52,7 +52,7 @@ public:
 
 		auto entry = cache.find(key);
 		if (entry == cache.end()) {
-			auto value = make_shared<T>(args...);
+			auto value = make_shared_ptr<T>(args...);
 			cache[key] = value;
 			return value;
 		}
@@ -60,7 +60,7 @@ public:
 		if (!object || object->GetObjectType() != T::ObjectType()) {
 			return nullptr;
 		}
-		return std::static_pointer_cast<T, ObjectCacheEntry>(object);
+		return shared_ptr_cast<ObjectCacheEntry, T>(object);
 	}
 
 	void Put(string key, shared_ptr<ObjectCacheEntry> value) {

package/src/duckdb/src/include/duckdb/storage/optimistic_data_writer.hpp

@@ -26,7 +26,7 @@ public:
 	//! Final flush of the optimistic writer - fully flushes the partial block manager
 	void FinalFlush();
 	//! Flushes a specific row group to disk
-	void FlushToDisk(RowGroup *row_group);
+	void FlushToDisk(RowGroup &row_group);
 	//! Merge the partially written blocks from one optimistic writer into another
 	void Merge(OptimisticDataWriter &other);
 	//! Rollback