duckdb 1.1.4-dev2.0 → 1.1.4-dev9.0

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -9,11 +9,8 @@
 #pragma once
 
 #include "duckdb/common/enums/index_constraint_type.hpp"
-#include "duckdb/common/enums/scan_options.hpp"
-#include "duckdb/common/mutex.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/unique_ptr.hpp"
-#include "duckdb/storage/block.hpp"
 #include "duckdb/storage/index.hpp"
 #include "duckdb/storage/statistics/column_statistics.hpp"
 #include "duckdb/storage/table/column_segment.hpp"
@@ -25,6 +22,7 @@
 #include "duckdb/transaction/local_storage.hpp"
 
 namespace duckdb {
+
 class BoundForeignKeyConstraint;
 class ClientContext;
 class ColumnDataCollection;
@@ -59,9 +57,9 @@ public:
 	DataTable(ClientContext &context, DataTable &parent, idx_t removed_column);
 	//! Constructs a DataTable as a delta on an existing data table but with one column changed type
 	DataTable(ClientContext &context, DataTable &parent, idx_t changed_idx, const LogicalType &target_type,
-	          const vector<column_t> &bound_columns, Expression &cast_expr);
+	          const vector<StorageIndex> &bound_columns, Expression &cast_expr);
 	//! Constructs a DataTable as a delta on an existing data table but with one column added new constraint
-	explicit DataTable(ClientContext &context, DataTable &parent, unique_ptr<BoundConstraint> constraint);
+	DataTable(ClientContext &context, DataTable &parent, BoundConstraint &constraint);
 
 	//! A reference to the database instance
 	AttachedDatabase &db;
@@ -76,13 +74,11 @@ public:
 	vector<LogicalType> GetTypes();
 	const vector<ColumnDefinition> &Columns() const;
 
-	void InitializeScan(TableScanState &state, const vector<column_t> &column_ids,
-	                    TableFilterSet *table_filter = nullptr);
-	void InitializeScan(DuckTransaction &transaction, TableScanState &state, const vector<column_t> &column_ids,
+	void InitializeScan(DuckTransaction &transaction, TableScanState &state, const vector<StorageIndex> &column_ids,
 	                    TableFilterSet *table_filters = nullptr);
 
 	//! Returns the maximum amount of threads that should be assigned to scan this data table
-	idx_t MaxThreads(ClientContext &context);
+	idx_t MaxThreads(ClientContext &context) const;
 	void InitializeParallelScan(ClientContext &context, ParallelTableScanState &state);
 	bool NextParallelScan(ClientContext &context, ParallelTableScanState &state, TableScanState &scan_state);
 
@@ -93,23 +89,30 @@ public:
 	void Scan(DuckTransaction &transaction, DataChunk &result, TableScanState &state);
 
 	//! Fetch data from the specific row identifiers from the base table
-	void Fetch(DuckTransaction &transaction, DataChunk &result, const vector<column_t> &column_ids,
+	void Fetch(DuckTransaction &transaction, DataChunk &result, const vector<StorageIndex> &column_ids,
 	           const Vector &row_ids, idx_t fetch_count, ColumnFetchState &state);
 
-	//! Initializes an append to transaction-local storage
+	//! Initializes appending to transaction-local storage
 	void InitializeLocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context,
 	                           const vector<unique_ptr<BoundConstraint>> &bound_constraints);
+	//! Initializes only the delete-indexes of the transaction-local storage
+	void InitializeLocalStorage(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context,
+	                            const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Append a DataChunk to the transaction-local storage of the table.
-	void LocalAppend(LocalAppendState &state, TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
-	                 bool unsafe = false);
+	void LocalAppend(LocalAppendState &state, ClientContext &context, DataChunk &chunk, bool unsafe);
 	//! Finalizes a transaction-local append
 	void FinalizeLocalAppend(LocalAppendState &state);
-	//! Append a chunk to the transaction-local storage of this table
+	//! Append a chunk to the transaction-local storage of this table and update the delete indexes.
+	void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints, Vector &row_ids,
+	                 DataChunk &delete_chunk);
+	//! Append a chunk to the transaction-local storage of this table.
 	void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
 	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints);
-	//! Append a column data collection to the transaction-local storage of this table
+	//! Append a column data collection with default values to the transaction-local storage of this table.
 	void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection,
-	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints);
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints,
+	                 optional_ptr<const vector<LogicalIndex>> column_ids);
 	//! Merge a row group collection into the transaction-local storage
 	void LocalMerge(ClientContext &context, RowGroupCollection &collection);
 	//! Creates an optimistic writer for this table - used for optimistically writing parallel appends
@@ -149,21 +152,24 @@ public:
 	//! Commit the append
 	void CommitAppend(transaction_t commit_id, idx_t row_start, idx_t count);
 	//! Write a segment of the table to the WAL
-	void WriteToLog(WriteAheadLog &log, idx_t row_start, idx_t count, optional_ptr<StorageCommitState> commit_state);
+	void WriteToLog(DuckTransaction &transaction, WriteAheadLog &log, idx_t row_start, idx_t count,
+	                optional_ptr<StorageCommitState> commit_state);
 	//! Revert a set of appends made by the given AppendState, used to revert appends in the event of an error during
 	//! commit (e.g. because of an I/O exception)
-	void RevertAppend(idx_t start_row, idx_t count);
+	void RevertAppend(DuckTransaction &transaction, idx_t start_row, idx_t count);
 	void RevertAppendInternal(idx_t start_row);
 
-	void ScanTableSegment(idx_t start_row, idx_t count, const std::function<void(DataChunk &chunk)> &function);
+	void ScanTableSegment(DuckTransaction &transaction, idx_t start_row, idx_t count,
+	                      const std::function<void(DataChunk &chunk)> &function);
 
 	//! Merge a row group collection directly into this table - appending it to the end of the table without copying
 	void MergeStorage(RowGroupCollection &data, TableIndexList &indexes, optional_ptr<StorageCommitState> commit_state);
 
-	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table, returns
-	//! whether or not the append succeeded
-	ErrorData AppendToIndexes(DataChunk &chunk, row_t row_start);
-	static ErrorData AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start);
+	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table.
+	//! Returns empty ErrorData, if the append was successful.
+	ErrorData AppendToIndexes(optional_ptr<TableIndexList> delete_indexes, DataChunk &chunk, row_t row_start);
+	static ErrorData AppendToIndexes(TableIndexList &indexes, optional_ptr<TableIndexList> delete_indexes,
+	                                 DataChunk &chunk, row_t row_start);
 	//! Remove a chunk with the row ids [row_start, ..., row_start + chunk.size()] from all indexes of the table
 	void RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, row_t row_start);
 	//! Remove the chunk with the specified set of row identifiers from all indexes of the table
@@ -174,12 +180,16 @@ public:
 	void SetAsRoot() {
 		this->is_root = true;
 	}
+
 	bool IsRoot() {
 		return this->is_root;
 	}
 
 	//! Get statistics of a physical column within the table
 	unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, column_t column_id);
+
+	//! Get table sample
+	unique_ptr<BlockingSample> GetSample();
 	//! Sets statistics of a physical column within the table
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
@@ -208,14 +218,14 @@ public:
 	unique_ptr<ConstraintState> InitializeConstraintState(TableCatalogEntry &table,
 	                                                      const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Verify constraints with a chunk from the Append containing all columns of the table
-	void VerifyAppendConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk,
-	                             optional_ptr<ConflictManager> conflict_manager = nullptr);
+	void VerifyAppendConstraints(ConstraintState &constraint_state, ClientContext &context, DataChunk &chunk,
+	                             optional_ptr<LocalTableStorage> local_storage, optional_ptr<ConflictManager> manager);
 
 	shared_ptr<DataTableInfo> &GetDataTableInfo();
 
 	void InitializeIndexes(ClientContext &context);
 	bool HasIndexes() const;
-	void AddIndex(unique_ptr<Index> index);
+	bool HasUniqueIndexes() const;
 	bool HasForeignKeyIndex(const vector<PhysicalIndex> &keys, ForeignKeyType type);
 	void SetIndexStorageInfo(vector<IndexStorageInfo> index_storage_info);
 	void VacuumIndexes();
@@ -226,21 +236,33 @@ public:
 
 	TableStorageInfo GetStorageInfo();
 
-public:
-	static void VerifyUniqueIndexes(TableIndexList &indexes, ClientContext &context, DataChunk &chunk,
-	                                optional_ptr<ConflictManager> conflict_manager);
+	idx_t GetRowGroupSize() const;
+
+	static void VerifyUniqueIndexes(TableIndexList &indexes, optional_ptr<LocalTableStorage> storage, DataChunk &chunk,
+	                                optional_ptr<ConflictManager> manager);
+
+	//! AddIndex initializes an index and adds it to the table's index list.
+	//! It is either empty, or initialized via its index storage information.
+	void AddIndex(const ColumnList &columns, const vector<LogicalIndex> &column_indexes, const IndexConstraintType type,
+	              const IndexStorageInfo &info);
+	//! AddIndex moves an index to this table's index list.
+	void AddIndex(unique_ptr<Index> index);
+
+	//! Returns a list of the partition stats
+	vector<PartitionStatistics> GetPartitionStats(ClientContext &context);
 
 private:
 	//! Verify the new added constraints against current persistent&local data
 	void VerifyNewConstraint(LocalStorage &local_storage, DataTable &parent, const BoundConstraint &constraint);
+
 	//! Verify constraints with a chunk from the Update containing only the specified column_ids
 	void VerifyUpdateConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk,
 	                             const vector<PhysicalIndex> &column_ids);
 	//! Verify constraints with a chunk from the Delete containing all columns of the table
 	void VerifyDeleteConstraints(TableDeleteState &state, ClientContext &context, DataChunk &chunk);
 
-	void InitializeScanWithOffset(TableScanState &state, const vector<column_t> &column_ids, idx_t start_row,
-	                              idx_t end_row);
+	void InitializeScanWithOffset(DuckTransaction &transaction, TableScanState &state,
+	                              const vector<StorageIndex> &column_ids, idx_t start_row, idx_t end_row);
 
 	void VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, ClientContext &context, DataChunk &chunk,
 	                                VerifyExistenceType verify_type);

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -59,9 +59,8 @@ public:
 
 	//! Returns unique flag
 	bool IsUnique() const {
-		auto index_constraint_type = GetConstraintType();
-		return (index_constraint_type == IndexConstraintType::UNIQUE ||
-		        index_constraint_type == IndexConstraintType::PRIMARY);
+		auto type = GetConstraintType();
+		return type == IndexConstraintType::UNIQUE || type == IndexConstraintType::PRIMARY;
 	}
 
 	//! Returns primary key flag

package/src/duckdb/src/include/duckdb/storage/object_cache.hpp

@@ -74,7 +74,6 @@ public:
 	}
 
 	DUCKDB_API static ObjectCache &GetObjectCache(ClientContext &context);
-	DUCKDB_API static bool ObjectCacheEnabled(ClientContext &context);
 
 private:
 	//! Object Cache

package/src/duckdb/src/include/duckdb/storage/segment/uncompressed.hpp

@@ -14,7 +14,7 @@ namespace duckdb {
 class DatabaseInstance;
 
 struct UncompressedFunctions {
-	static unique_ptr<CompressionState> InitCompression(ColumnDataCheckpointer &checkpointer,
+	static unique_ptr<CompressionState> InitCompression(ColumnDataCheckpointData &checkpoint_data,
 	                                                    unique_ptr<AnalyzeState> state);
 	static void Compress(CompressionState &state_p, Vector &data, idx_t count);
 	static void FinalizeCompress(CompressionState &state_p);
@@ -29,6 +29,9 @@ struct FixedSizeUncompressed {
 struct ValidityUncompressed {
 public:
 	static CompressionFunction GetFunction(PhysicalType data_type);
+	static void AlignedScan(data_ptr_t input, idx_t input_start, Vector &result, idx_t scan_count);
+	static void UnalignedScan(data_ptr_t input, idx_t input_size, idx_t input_start, Vector &result,
+	                          idx_t result_offset, idx_t scan_count);
 
 public:
 	static const validity_t LOWER_MASKS[65];

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -49,7 +49,7 @@ public:
 	//! Registers an in-memory buffer that cannot be unloaded until it is destroyed.
 	//! This buffer can be small (smaller than the block size of the temporary block manager).
 	//! Unpin and Pin are NOPs on this block of memory.
-	shared_ptr<BlockHandle> RegisterSmallMemory(const idx_t size) final;
+	shared_ptr<BlockHandle> RegisterSmallMemory(MemoryTag tag, const idx_t size) final;
 
 	idx_t GetUsedMemory() const final;
 	idx_t GetMaxMemory() const final;
@@ -120,7 +120,7 @@ protected:
 	shared_ptr<BlockHandle> RegisterMemory(MemoryTag tag, idx_t block_size, bool can_destroy);
 
 	//! Garbage collect eviction queue
-	void PurgeQueue(FileBufferType type) final;
+	void PurgeQueue(const BlockHandle &handle) final;
 
 	BufferPool &GetBufferPool() const final;
 	TemporaryMemoryManager &GetTemporaryMemoryManager() final;
@@ -148,7 +148,7 @@ protected:
 
 	//! When the BlockHandle reaches 0 readers, this creates a new FileBuffer for this BlockHandle and
 	//! overwrites the data within with garbage. Any readers that do not hold the pin will notice
-	void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
+	void VerifyZeroReaders(BlockLock &l, shared_ptr<BlockHandle> &handle);
 
 	void BatchRead(vector<shared_ptr<BlockHandle>> &handles, const map<block_id_t, idx_t> &load_map,
 	               block_id_t first_block, block_id_t last_block);

package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp

@@ -24,7 +24,7 @@ public:
 
 	void Merge(ColumnStatistics &other);
 
-	void UpdateDistinctStatistics(Vector &v, idx_t count);
+	void UpdateDistinctStatistics(Vector &v, idx_t count, Vector &hashes);
 
 	BaseStatistics &Statistics();
 

package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp

@@ -34,8 +34,8 @@ public:
 
 	unique_ptr<DistinctStatistics> Copy() const;
 
-	void Update(Vector &update, idx_t count, bool sample = true);
-	void Update(UnifiedVectorFormat &update_data, const LogicalType &ptype, idx_t count, bool sample = true);
+	void UpdateSample(Vector &new_data, idx_t count, Vector &hashes);
+	void Update(Vector &new_data, idx_t count, Vector &hashes);
 
 	string ToString() const;
 	idx_t GetCount() const;
@@ -45,13 +45,14 @@ public:
 	void Serialize(Serializer &serializer) const;
 	static unique_ptr<DistinctStatistics> Deserialize(Deserializer &deserializer);
 
+private:
+	void UpdateInternal(Vector &update, idx_t count, Vector &hashes);
+
 private:
 	//! For distinct statistics we sample the input to speed up insertions
 	static constexpr double BASE_SAMPLE_RATE = 0.1;
 	//! For integers, we sample more: likely to be join keys (and hashing is cheaper than, e.g., strings)
 	static constexpr double INTEGRAL_SAMPLE_RATE = 0.3;
-	//! For concurrent access
-	mutable mutex lock;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/common/operator/comparison_operators.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/storage/statistics/numeric_stats_union.hpp"
+#include "duckdb/common/array_ptr.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -53,9 +54,23 @@ struct NumericStats {
 	//! Sets the max value of the statistics
 	DUCKDB_API static void SetMax(BaseStatistics &stats, const Value &val);
 
+	template <class T>
+	static void SetMax(BaseStatistics &stats, T val) {
+		auto &nstats = GetDataUnsafe(stats);
+		nstats.has_max = true;
+		nstats.max.GetReferenceUnsafe<T>() = val;
+	}
+
+	template <class T>
+	static void SetMin(BaseStatistics &stats, T val) {
+		auto &nstats = GetDataUnsafe(stats);
+		nstats.has_min = true;
+		nstats.min.GetReferenceUnsafe<T>() = val;
+	}
+
 	//! Check whether or not a given comparison with a constant could possibly be satisfied by rows given the statistics
 	DUCKDB_API static FilterPropagateResult CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,
-	                                                     const Value &constant);
+	                                                     array_ptr<Value> constants);
 
 	DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other_p);
 

package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/operator/comparison_operators.hpp"
 #include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/common/array_ptr.hpp"
 
 namespace duckdb {
 class BaseStatistics;
@@ -62,7 +63,7 @@ struct StringStats {
 	DUCKDB_API static string ToString(const BaseStatistics &stats);
 
 	DUCKDB_API static FilterPropagateResult CheckZonemap(const BaseStatistics &stats, ExpressionType comparison_type,
-	                                                     const string &value);
+	                                                     array_ptr<Value> constants);
 	DUCKDB_API static FilterPropagateResult CheckZonemap(const_data_ptr_t min_data, idx_t min_len,
 	                                                     const_data_ptr_t max_data, idx_t max_len,
 	                                                     ExpressionType comparison_type, const string &value);

package/src/duckdb/src/include/duckdb/storage/storage_index.hpp

@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/storage_index.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+#include "duckdb/common/vector.hpp"
+
+namespace duckdb {
+
+struct StorageIndex {
+	StorageIndex() : index(DConstants::INVALID_INDEX) {
+	}
+	explicit StorageIndex(idx_t index) : index(index) {
+	}
+	StorageIndex(idx_t index, vector<StorageIndex> child_indexes_p)
+	    : index(index), child_indexes(std::move(child_indexes_p)) {
+	}
+
+	inline bool operator==(const StorageIndex &rhs) const {
+		return index == rhs.index;
+	}
+	inline bool operator!=(const StorageIndex &rhs) const {
+		return index != rhs.index;
+	}
+	inline bool operator<(const StorageIndex &rhs) const {
+		return index < rhs.index;
+	}
+	idx_t GetPrimaryIndex() const {
+		return index;
+	}
+	PhysicalIndex ToPhysical() const {
+		return PhysicalIndex(index);
+	}
+	bool HasChildren() const {
+		return !child_indexes.empty();
+	}
+	idx_t ChildIndexCount() const {
+		return child_indexes.size();
+	}
+	const StorageIndex &GetChildIndex(idx_t idx) const {
+		return child_indexes[idx];
+	}
+	StorageIndex &GetChildIndex(idx_t idx) {
+		return child_indexes[idx];
+	}
+	const vector<StorageIndex> &GetChildIndexes() const {
+		return child_indexes;
+	}
+	void AddChildIndex(StorageIndex new_index) {
+		this->child_indexes.push_back(std::move(new_index));
+	}
+	void SetIndex(idx_t new_index) {
+		index = new_index;
+	}
+	bool IsRowIdColumn() const {
+		return index == DConstants::INVALID_INDEX;
+	}
+
+private:
+	idx_t index;
+	vector<StorageIndex> child_indexes;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -17,7 +17,7 @@ namespace duckdb {
 struct FileHandle;
 
 //! The standard row group size
-#define STANDARD_ROW_GROUPS_SIZE 122880
+#define DEFAULT_ROW_GROUP_SIZE 122880ULL
 //! The definition of an invalid block
 #define INVALID_BLOCK (-1)
 //! The maximum block id is 2^62
@@ -38,10 +38,8 @@ struct Storage {
 	//! The size of the headers. This should be small and written more or less atomically by the hard disk. We default
 	//! to the page size, which is 4KB. (1 << 12)
 	constexpr static idx_t FILE_HEADER_SIZE = 4096U;
-	//! The number of rows per row group (must be a multiple of the vector size)
-	constexpr static const idx_t ROW_GROUP_SIZE = STANDARD_ROW_GROUPS_SIZE;
-	//! The number of vectors per row group
-	constexpr static const idx_t ROW_GROUP_VECTOR_COUNT = ROW_GROUP_SIZE / STANDARD_VECTOR_SIZE;
+	//! The maximum row group size
+	constexpr static const idx_t MAX_ROW_GROUP_SIZE = 1ULL << 30ULL;
 
 	//! The minimum block allocation size. This is the minimum size we test in our nightly tests.
 	constexpr static idx_t MIN_BLOCK_ALLOC_SIZE = 16384ULL;
@@ -118,10 +116,10 @@ struct DatabaseHeader {
 
 //! Detect mismatching constant values when compiling
 
-#if (STANDARD_ROW_GROUPS_SIZE % STANDARD_VECTOR_SIZE != 0)
+#if (DEFAULT_ROW_GROUP_SIZE % STANDARD_VECTOR_SIZE != 0)
 #error The row group size must be a multiple of the vector size
 #endif
-#if (STANDARD_ROW_GROUPS_SIZE < STANDARD_VECTOR_SIZE)
+#if (DEFAULT_ROW_GROUP_SIZE < STANDARD_VECTOR_SIZE)
 #error Row groups must be able to hold at least one vector
 #endif
 #if (DEFAULT_BLOCK_ALLOC_SIZE & (DEFAULT_BLOCK_ALLOC_SIZE - 1) != 0)

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/storage/write_ahead_log.hpp"
 #include "duckdb/storage/database_size.hpp"
 #include "duckdb/common/enums/checkpoint_type.hpp"
+#include "duckdb/storage/storage_options.hpp"
 
 namespace duckdb {
 class BlockManager;
@@ -71,7 +72,7 @@ public:
 	//! Initialize a database or load an existing database from the database file path. The block_alloc_size is
 	//! either set, or invalid. If invalid, then DuckDB defaults to the default_block_alloc_size (DBConfig),
 	//! or the file's block allocation size, if it is an existing database.
-	void Initialize(const optional_idx block_alloc_size);
+	void Initialize(StorageOptions options);
 
 	DatabaseInstance &GetDatabase();
 	AttachedDatabase &GetAttached() {
@@ -106,7 +107,7 @@ public:
 	virtual BlockManager &GetBlockManager() = 0;
 
 protected:
-	virtual void LoadDatabase(const optional_idx block_alloc_size) = 0;
+	virtual void LoadDatabase(StorageOptions options) = 0;
 
 protected:
 	//! The database this storage manager belongs to
@@ -156,6 +157,6 @@ public:
 	BlockManager &GetBlockManager() override;
 
 protected:
-	void LoadDatabase(const optional_idx block_alloc_size) override;
+	void LoadDatabase(StorageOptions options) override;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/storage_options.hpp

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/storage/storage_options.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/optional_idx.hpp"
+
+namespace duckdb {
+
+struct StorageOptions {
+	//! The allocation size of blocks for this attached database file (if any)
+	optional_idx block_alloc_size;
+	//! The row group size for this attached database (if any)
+	optional_idx row_group_size;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -41,6 +41,18 @@ struct StringScanState : public SegmentScanState {
 	BufferHandle handle;
 };
 
+//===--------------------------------------------------------------------===//
+// Append
+//===--------------------------------------------------------------------===//
+struct SerializedStringSegmentState : public ColumnSegmentState {
+public:
+	SerializedStringSegmentState();
+	explicit SerializedStringSegmentState(vector<block_id_t> blocks_p);
+
+public:
+	void Serialize(Serializer &serializer) const override;
+};
+
 struct UncompressedStringStorage {
 public:
 	//! Dictionary header size at the beginning of the string segment (offset + length)
@@ -60,6 +72,9 @@ public:
 	static void StringScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
 	                              idx_t result_offset);
 	static void StringScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result);
+	static void Select(ColumnSegment &segment, ColumnScanState &state, idx_t vector_count, Vector &result,
+	                   const SelectionVector &sel, idx_t sel_count);
+
 	static void StringFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
 	                           idx_t result_idx);
 	static unique_ptr<CompressedSegmentState> StringInitSegment(ColumnSegment &segment, block_id_t block_id,
@@ -202,12 +217,25 @@ public:
 	static void WriteStringMarker(data_ptr_t target, block_id_t block_id, int32_t offset);
 	static void ReadStringMarker(data_ptr_t target, block_id_t &block_id, int32_t &offset);
 
-	static string_location_t FetchStringLocation(StringDictionaryContainer dict, data_ptr_t base_ptr,
-	                                             int32_t dict_offset, const idx_t block_size);
-	static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-	                                    data_ptr_t base_ptr, int32_t dict_offset, uint32_t string_length);
-	static string_t FetchString(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
-	                            data_ptr_t baseptr, string_location_t location, uint32_t string_length);
+	inline static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
+	                                           data_ptr_t base_ptr, int32_t dict_offset, uint32_t string_length) {
+		D_ASSERT(dict_offset <= NumericCast<int32_t>(segment.GetBlockManager().GetBlockSize()));
+		if (DUCKDB_LIKELY(dict_offset >= 0)) {
+			// regular string - fetch from dictionary
+			auto dict_end = base_ptr + dict.end;
+			auto dict_pos = dict_end - dict_offset;
+
+			auto str_ptr = char_ptr_cast(dict_pos);
+			return string_t(str_ptr, string_length);
+		} else {
+			// read overflow string
+			block_id_t block_id;
+			int32_t offset;
+			ReadStringMarker(base_ptr + dict.end - AbsValue<int32_t>(dict_offset), block_id, offset);
+
+			return ReadOverflowString(segment, result, block_id, offset);
+		}
+	}
 
 	static unique_ptr<ColumnSegmentState> SerializeState(ColumnSegment &segment);
 	static unique_ptr<ColumnSegmentState> DeserializeState(Deserializer &deserializer);

package/src/duckdb/src/include/duckdb/storage/table/append_state.hpp

@@ -72,6 +72,8 @@ struct TableAppendState {
 	TransactionData transaction;
 	//! Table statistics
 	TableStatistics stats;
+	//! Cached hash vector
+	Vector hashes;
 };
 
 struct ConstraintState {

package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/storage/table/list_column_data.hpp
+// duckdb/storage/table/array_column_data.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -21,7 +21,7 @@ public:
 
 	//! The child-column of the list
 	unique_ptr<ColumnData> child_column;
-	//! The validity column data of the struct
+	//! The validity column data of the array
 	ValidityColumnData validity;
 
 public:

package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp

@@ -40,7 +40,8 @@ protected:
 public:
 	virtual unique_ptr<BaseStatistics> GetStatistics();
 
-	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, idx_t segment_size);
+	virtual void FlushSegmentInternal(unique_ptr<ColumnSegment> segment, idx_t segment_size);
+	virtual void FlushSegment(unique_ptr<ColumnSegment> segment, BufferHandle handle, idx_t segment_size);
 	virtual PersistentColumnData ToPersistentData();
 
 	PartialBlockManager &GetPartialBlockManager() {