duckdb 1.1.4-dev2.0 → 1.1.4-dev9.0

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp

@@ -22,15 +22,18 @@ class PhysicalHashJoin : public PhysicalComparisonJoin {
 public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::HASH_JOIN;
 
+	struct JoinProjectionColumns {
+		vector<idx_t> col_idxs;
+		vector<LogicalType> col_types;
+	};
+
 public:
 	PhysicalHashJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left, unique_ptr<PhysicalOperator> right,
 	                 vector<JoinCondition> cond, JoinType join_type, const vector<idx_t> &left_projection_map,
 	                 const vector<idx_t> &right_projection_map, vector<LogicalType> delim_types,
-	                 idx_t estimated_cardinality, PerfectHashJoinStats perfect_join_stats,
-	                 unique_ptr<JoinFilterPushdownInfo> pushdown_info);
+	                 idx_t estimated_cardinality, unique_ptr<JoinFilterPushdownInfo> pushdown_info);
 	PhysicalHashJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left, unique_ptr<PhysicalOperator> right,
-	                 vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality,
-	                 PerfectHashJoinStats join_state);
+	                 vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality);
 
 	//! Initialize HT for this operator
 	unique_ptr<JoinHashTable> InitializeHashTable(ClientContext &context) const;
@@ -38,20 +41,18 @@ public:
 	//! The types of the join keys
 	vector<LogicalType> condition_types;
 
-	//! The indices for getting the payload columns
-	vector<idx_t> payload_column_idxs;
-	//! The types of the payload columns
-	vector<LogicalType> payload_types;
-
-	//! Positions of the RHS columns that need to output
-	vector<idx_t> rhs_output_columns;
-	//! The types of the output
-	vector<LogicalType> rhs_output_types;
+	//! The indices/types of the payload columns
+	JoinProjectionColumns payload_columns;
+	//! The indices/types of the lhs columns that need to be output
+	JoinProjectionColumns lhs_output_columns;
+	//! The indices/types of the rhs columns that need to be output
+	JoinProjectionColumns rhs_output_columns;
 
 	//! Duplicate eliminated types; only used for delim_joins (i.e. correlated subqueries)
 	vector<LogicalType> delim_types;
-	//! Used in perfect hash join
-	PerfectHashJoinStats perfect_join_statistics;
+
+	//! Join Keys statistics (optional)
+	vector<unique_ptr<BaseStatistics>> join_stats;
 
 public:
 	InsertionOrderPreservingMap<string> ParamsToString() const override;
@@ -75,7 +76,7 @@ protected:
 	                                                 GlobalSourceState &gstate) const override;
 	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
-	double GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
+	ProgressData GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
 
 	//! Becomes a source when it is an external join
 	bool IsSource() const override {

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp

@@ -46,7 +46,7 @@ public:
 		return true;
 	}
 
-	double GetProgress(ClientContext &context, GlobalSourceState &gstate_p) const override;
+	ProgressData GetProgress(ClientContext &context, GlobalSourceState &gstate_p) const override;
 
 public:
 	// Sink Interface

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp

@@ -35,8 +35,9 @@ public:
 	                                                 GlobalSourceState &gstate) const override;
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
-	idx_t GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	                    LocalSourceState &lstate) const override;
+	OperatorPartitionData GetPartitionData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	                                       LocalSourceState &lstate,
+	                                       const OperatorPartitionInfo &partition_info) const override;
 
 	bool IsSource() const override {
 		return true;
@@ -46,7 +47,10 @@ public:
 		return true;
 	}
 
-	bool SupportsBatchIndex() const override {
+	bool SupportsPartitioning(const OperatorPartitionInfo &partition_info) const override {
+		if (partition_info.RequiresPartitionColumns()) {
+			return false;
+		}
 		return true;
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/planner/bound_query_node.hpp"
 
 namespace duckdb {
+struct DynamicFilterData;
 
 //! Represents a physical ordering of the data. Note that this will not change
 //! the data but only add a selection vector.
@@ -21,11 +22,14 @@ public:
 
 public:
 	PhysicalTopN(vector<LogicalType> types, vector<BoundOrderByNode> orders, idx_t limit, idx_t offset,
-	             idx_t estimated_cardinality);
+	             shared_ptr<DynamicFilterData> dynamic_filter, idx_t estimated_cardinality);
+	~PhysicalTopN() override;
 
 	vector<BoundOrderByNode> orders;
 	idx_t limit;
 	idx_t offset;
+	//! Dynamic table filter (if any)
+	shared_ptr<DynamicFilterData> dynamic_filter;
 
 public:
 	// Source interface

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp

@@ -49,8 +49,8 @@ public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 	SinkNextBatchType NextBatch(ExecutionContext &context, OperatorSinkNextBatchInput &input) const override;
 
-	bool RequiresBatchIndex() const override {
-		return true;
+	OperatorPartitionInfo RequiredPartitionInfo() const override {
+		return OperatorPartitionInfo::BatchIndex();
 	}
 
 	bool IsSink() const override {

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp

@@ -60,8 +60,8 @@ public:
 	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
 	                          OperatorSinkFinalizeInput &input) const override;
 
-	bool RequiresBatchIndex() const override {
-		return true;
+	OperatorPartitionInfo RequiredPartitionInfo() const override {
+		return OperatorPartitionInfo::BatchIndex();
 	}
 
 	bool IsSink() const override {

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp

@@ -13,10 +13,58 @@
 #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
 #include "duckdb/common/index_vector.hpp"
 #include "duckdb/parser/statement/insert_statement.hpp"
+#include "duckdb/storage/table/append_state.hpp"
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
+#include "duckdb/storage/table/delete_state.hpp"
 
 namespace duckdb {
 
-class InsertLocalState;
+//===--------------------------------------------------------------------===//
+// Sink
+//===--------------------------------------------------------------------===//
+class InsertGlobalState : public GlobalSinkState {
+public:
+	explicit InsertGlobalState(ClientContext &context, const vector<LogicalType> &return_types, DuckTableEntry &table);
+
+public:
+	mutex lock;
+	DuckTableEntry &table;
+	idx_t insert_count;
+	bool initialized;
+	LocalAppendState append_state;
+	ColumnDataCollection return_collection;
+};
+
+class InsertLocalState : public LocalSinkState {
+public:
+public:
+	InsertLocalState(ClientContext &context, const vector<LogicalType> &types,
+	                 const vector<unique_ptr<Expression>> &bound_defaults,
+	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints);
+
+public:
+	ConstraintState &GetConstraintState(DataTable &table, TableCatalogEntry &table_ref);
+	TableDeleteState &GetDeleteState(DataTable &table, TableCatalogEntry &table_ref, ClientContext &context);
+
+public:
+	//! The chunk that ends up getting inserted
+	DataChunk insert_chunk;
+	//! The chunk containing the tuples that become an update (if DO UPDATE)
+	DataChunk update_chunk;
+	ExpressionExecutor default_executor;
+	TableAppendState local_append_state;
+	unique_ptr<RowGroupCollection> local_collection;
+	optional_ptr<OptimisticDataWriter> writer;
+	// Rows that have been updated by a DO UPDATE conflict
+	unordered_set<row_t> updated_rows;
+	idx_t update_count = 0;
+	unique_ptr<ConstraintState> constraint_state;
+	const vector<unique_ptr<BoundConstraint>> &bound_constraints;
+	//! The delete state for ON CONFLICT handling that is rewritten into DELETE + INSERT.
+	unique_ptr<TableDeleteState> delete_state;
+	//! The append chunk for ON CONFLICT handling that is rewritting into DELETE + INSERT.
+	DataChunk append_chunk;
+};
 
 //! Physically insert a set of data into a table
 class PhysicalInsert : public PhysicalOperator {
@@ -31,7 +79,7 @@ public:
 	               vector<LogicalType> set_types, idx_t estimated_cardinality, bool return_chunk, bool parallel,
 	               OnConflictAction action_type, unique_ptr<Expression> on_conflict_condition,
 	               unique_ptr<Expression> do_update_condition, unordered_set<column_t> on_conflict_filter,
-	               vector<column_t> columns_to_fetch);
+	               vector<column_t> columns_to_fetch, bool update_is_del_and_insert);
 	//! CREATE TABLE AS
 	PhysicalInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr<BoundCreateTableInfo> info,
 	               idx_t estimated_cardinality, bool parallel);
@@ -71,9 +119,11 @@ public:
 	unique_ptr<Expression> do_update_condition;
 	// The column ids to apply the ON CONFLICT on
 	unordered_set<column_t> conflict_target;
+	//! True, if the INSERT OR REPLACE requires delete + insert.
+	bool update_is_del_and_insert;
 
 	// Column ids from the original table to fetch
-	vector<column_t> columns_to_fetch;
+	vector<StorageIndex> columns_to_fetch;
 	// Matching types to the column ids to fetch
 	vector<LogicalType> types_to_fetch;
 
@@ -120,7 +170,8 @@ protected:
 	//! Returns the amount of updated tuples
 	void CreateUpdateChunk(ExecutionContext &context, DataChunk &chunk, TableCatalogEntry &table, Vector &row_ids,
 	                       DataChunk &result) const;
-	idx_t OnConflictHandling(TableCatalogEntry &table, ExecutionContext &context, InsertLocalState &lstate) const;
+	idx_t OnConflictHandling(TableCatalogEntry &table, ExecutionContext &context, InsertGlobalState &gstate,
+	                         InsertLocalState &lstate) const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp

@@ -35,6 +35,8 @@ public:
 	bool update_is_del_and_insert;
 	//! If the returning statement is present, return the whole chunk
 	bool return_chunk;
+	//! Set to true, if we are updating an index column.
+	bool index_update;
 
 public:
 	// Source interface

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_tableinout_function.hpp

@@ -20,7 +20,7 @@ public:
 
 public:
 	PhysicalTableInOutFunction(vector<LogicalType> types, TableFunction function_p,
-	                           unique_ptr<FunctionData> bind_data_p, vector<column_t> column_ids_p,
+	                           unique_ptr<FunctionData> bind_data_p, vector<ColumnIndex> column_ids_p,
 	                           idx_t estimated_cardinality, vector<column_t> projected_input);
 
 public:
@@ -47,7 +47,7 @@ private:
 	//! Bind data of the function
 	unique_ptr<FunctionData> bind_data;
 	//! The set of column ids to fetch
-	vector<column_t> column_ids;
+	vector<ColumnIndex> column_ids;
 	//! The set of input columns to project out
 	vector<column_t> projected_input;
 };

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_positional_scan.hpp

@@ -30,6 +30,7 @@ public:
 
 public:
 	bool Equals(const PhysicalOperator &other) const override;
+	vector<const_reference<PhysicalOperator>> GetChildren() const override;
 
 public:
 	unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
@@ -37,7 +38,7 @@ public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
 
-	double GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
+	ProgressData GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
 
 	bool IsSource() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp

@@ -13,6 +13,7 @@
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/data_table.hpp"
 #include "duckdb/common/extra_operator_info.hpp"
+#include "duckdb/common/column_index.hpp"
 
 namespace duckdb {
 
@@ -24,7 +25,7 @@ public:
 public:
 	//! Table scan that immediately projects out filter columns that are unused in the remainder of the query plan
 	PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
-	                  vector<LogicalType> returned_types, vector<column_t> column_ids, vector<idx_t> projection_ids,
+	                  vector<LogicalType> returned_types, vector<ColumnIndex> column_ids, vector<idx_t> projection_ids,
 	                  vector<string> names, unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality,
 	                  ExtraOperatorInfo extra_info, vector<Value> parameters);
 
@@ -35,14 +36,15 @@ public:
 	//! The types of ALL columns that can be returned by the table function
 	vector<LogicalType> returned_types;
 	//! The column ids used within the table function
-	vector<column_t> column_ids;
+	vector<ColumnIndex> column_ids;
 	//! The projected-out column ids
 	vector<idx_t> projection_ids;
 	//! The names of the columns
 	vector<string> names;
 	//! The table filters
 	unique_ptr<TableFilterSet> table_filters;
-	//! Currently stores info related to filters pushed down into MultiFileLists
+	//! Currently stores info related to filters pushed down into MultiFileLists and sample rate pushed down into the
+	//! table scan
 	ExtraOperatorInfo extra_info;
 	//! Parameters
 	vector<Value> parameters;
@@ -60,19 +62,18 @@ public:
 	                                                 GlobalSourceState &gstate) const override;
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
-	idx_t GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	                    LocalSourceState &lstate) const override;
+	OperatorPartitionData GetPartitionData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	                                       LocalSourceState &lstate,
+	                                       const OperatorPartitionInfo &partition_info) const override;
 
 	bool IsSource() const override {
 		return true;
 	}
 	bool ParallelSource() const override;
 
-	bool SupportsBatchIndex() const override {
-		return function.get_batch_index != nullptr;
-	}
+	bool SupportsPartitioning(const OperatorPartitionInfo &partition_info) const override;
 
-	double GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
+	ProgressData GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_art_index.hpp

@@ -8,18 +8,18 @@
 
 #pragma once
 
-#include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/execution/physical_operator.hpp"
 #include "duckdb/parser/parsed_data/create_index_info.hpp"
-
 #include "duckdb/storage/data_table.hpp"
 
 #include <fstream>
 
 namespace duckdb {
+
 class DuckTableEntry;
 
-//! Physical CREATE (UNIQUE) INDEX statement
+//! Physical index creation operator.
 class PhysicalCreateARTIndex : public PhysicalOperator {
 public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::CREATE_INDEX;
@@ -27,18 +27,21 @@ public:
 public:
 	PhysicalCreateARTIndex(LogicalOperator &op, TableCatalogEntry &table, const vector<column_t> &column_ids,
 	                       unique_ptr<CreateIndexInfo> info, vector<unique_ptr<Expression>> unbound_expressions,
-	                       idx_t estimated_cardinality, const bool sorted);
+	                       idx_t estimated_cardinality, const bool sorted,
+	                       unique_ptr<AlterTableInfo> alter_table_info = nullptr);
 
-	//! The table to create the index for
+	//! The table to create the index for.
 	DuckTableEntry &table;
-	//! The list of column IDs required for the index
+	//! The list of column IDs of the index.
 	vector<column_t> storage_ids;
-	//! Info for index creation
+	//! Index creation information.
 	unique_ptr<CreateIndexInfo> info;
-	//! Unbound expressions to be used in the optimizer
+	//! Unbound expressions of the indexed columns.
 	vector<unique_ptr<Expression>> unbound_expressions;
-	//! Whether the pipeline sorts the data prior to index creation
+	//! True, if the pipeline sorts the index data prior to index creation.
 	const bool sorted;
+	//! Alter table information for adding indexes.
+	unique_ptr<AlterTableInfo> alter_table_info;
 
 public:
 	//! Source interface, NOP for this operator
@@ -49,14 +52,14 @@ public:
 	}
 
 public:
-	//! Sink interface, thread-local sink states
+	//! Sink interface, thread-local sink states. Contains an index for each state.
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
-	//! Sink interface, global sink state
+	//! Sink interface, global sink state. Contains the global index.
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
-	//! Sink for unsorted data: insert iteratively
+	//! Sink for unsorted data: insert iteratively.
 	SinkResultType SinkUnsorted(OperatorSinkInput &input) const;
-	//! Sink for sorted data: build + merge
+	//! Sink for sorted data: build + merge.
 	SinkResultType SinkSorted(OperatorSinkInput &input) const;
 
 	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp

@@ -52,10 +52,6 @@ public:
 		return false;
 	}
 
-	bool RequiresBatchIndex() const override {
-		return false;
-	}
-
 	InsertionOrderPreservingMap<string> ParamsToString() const override;
 
 public:

package/src/duckdb/src/include/duckdb/execution/partition_info.hpp

@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/partition_info.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/common/optional_idx.hpp"
+
+namespace duckdb {
+
+enum class PartitionInfo { NONE, REQUIRES_BATCH_INDEX };
+
+struct ColumnPartitionData {
+	explicit ColumnPartitionData(Value partition_val) : min_val(partition_val), max_val(std::move(partition_val)) {
+	}
+
+	Value min_val;
+	Value max_val;
+};
+
+struct SourcePartitionInfo {
+	//! The current batch index
+	//! This is only set in case RequiresBatchIndex() is true, and the source has support for it (SupportsBatchIndex())
+	//! Otherwise this is left on INVALID_INDEX
+	//! The batch index is a globally unique, increasing index that should be used to maintain insertion order
+	//! //! in conjunction with parallelism
+	optional_idx batch_index;
+	//! The minimum batch index that any thread is currently actively reading
+	optional_idx min_batch_index;
+	//! Column partition data
+	vector<ColumnPartitionData> partition_data;
+};
+
+struct OperatorPartitionInfo {
+	OperatorPartitionInfo() = default;
+	explicit OperatorPartitionInfo(bool batch_index) : batch_index(batch_index) {
+	}
+	explicit OperatorPartitionInfo(vector<column_t> partition_columns_p)
+	    : partition_columns(std::move(partition_columns_p)) {
+	}
+
+	bool batch_index = false;
+	vector<column_t> partition_columns;
+
+	static OperatorPartitionInfo NoPartitionInfo() {
+		return OperatorPartitionInfo(false);
+	}
+	static OperatorPartitionInfo BatchIndex() {
+		return OperatorPartitionInfo(true);
+	}
+	static OperatorPartitionInfo PartitionColumns(vector<column_t> columns) {
+		return OperatorPartitionInfo(std::move(columns));
+	}
+	bool RequiresPartitionColumns() const {
+		return !partition_columns.empty();
+	}
+	bool RequiresBatchIndex() const {
+		return batch_index;
+	}
+	bool AnyRequired() const {
+		return RequiresPartitionColumns() || RequiresBatchIndex();
+	}
+};
+
+struct OperatorPartitionData {
+	explicit OperatorPartitionData(idx_t batch_index) : batch_index(batch_index) {
+	}
+
+	idx_t batch_index;
+	vector<ColumnPartitionData> partition_data;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp

@@ -15,11 +15,13 @@
 #include "duckdb/common/enums/explain_format.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/execution/execution_context.hpp"
+#include "duckdb/execution/progress_data.hpp"
 #include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/common/optional_idx.hpp"
 #include "duckdb/execution/physical_operator_states.hpp"
 #include "duckdb/common/enums/order_preservation_type.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/execution/partition_info.hpp"
 
 namespace duckdb {
 class Event;
@@ -113,8 +115,9 @@ public:
 	virtual unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const;
 	virtual SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const;
 
-	virtual idx_t GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
-	                            LocalSourceState &lstate) const;
+	virtual OperatorPartitionData GetPartitionData(ExecutionContext &context, DataChunk &chunk,
+	                                               GlobalSourceState &gstate, LocalSourceState &lstate,
+	                                               const OperatorPartitionInfo &partition_info) const;
 
 	virtual bool IsSource() const {
 		return false;
@@ -124,8 +127,11 @@ public:
 		return false;
 	}
 
-	virtual bool SupportsBatchIndex() const {
-		return false;
+	virtual bool SupportsPartitioning(const OperatorPartitionInfo &partition_info) const {
+		if (partition_info.AnyRequired()) {
+			return false;
+		}
+		return true;
 	}
 
 	//! The type of order emitted by the operator (as a source)
@@ -134,10 +140,11 @@ public:
 	}
 
 	//! Returns the current progress percentage, or a negative value if progress bars are not supported
-	virtual double GetProgress(ClientContext &context, GlobalSourceState &gstate) const;
+	virtual ProgressData GetProgress(ClientContext &context, GlobalSourceState &gstate) const;
 
 	//! Returns the current progress percentage, or a negative value if progress bars are not supported
-	virtual double GetSinkProgress(ClientContext &context, GlobalSinkState &gstate, double source_progress) const {
+	virtual ProgressData GetSinkProgress(ClientContext &context, GlobalSinkState &gstate,
+	                                     const ProgressData source_progress) const {
 		return source_progress;
 	}
 
@@ -157,7 +164,7 @@ public:
 	virtual void PrepareFinalize(ClientContext &context, GlobalSinkState &sink_state) const;
 	//! The finalize is called when ALL threads are finished execution. It is called only once per pipeline, and is
 	//! entirely single threaded.
-	//! If Finalize returns SinkResultType::FINISHED, the sink is marked as finished
+	//! If Finalize returns SinkResultType::Finished, the sink is marked as finished
 	virtual SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
 	                                  OperatorSinkFinalizeInput &input) const;
 	//! For sinks with RequiresBatchIndex set to true, when a new batch starts being processed this method is called
@@ -181,8 +188,8 @@ public:
 		return false;
 	}
 
-	virtual bool RequiresBatchIndex() const {
-		return false;
+	virtual OperatorPartitionInfo RequiredPartitionInfo() const {
+		return OperatorPartitionInfo::NoPartitionInfo();
 	}
 
 	//! Whether or not the sink operator depends on the order of the input chunks
@@ -241,8 +248,12 @@ public:
 	bool caching_supported;
 
 public:
+	//! This Execute will prevent small chunks from entering the pipeline, buffering them until a bigger chunk is
+	//! created.
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const final;
+	//! FinalExecute is used here to send out the remainder of the chunk (< STANDARD_VECTOR_SIZE) that we still had
+	//! cached.
 	OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, GlobalOperatorState &gstate,
 	                                        OperatorState &state) const final;
 

package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/execution/execution_context.hpp"
 #include "duckdb/optimizer/join_order/join_node.hpp"
 #include "duckdb/parallel/interrupt.hpp"
+#include "duckdb/execution/partition_info.hpp"
 
 namespace duckdb {
 class Event;
@@ -26,17 +27,6 @@ class PipelineBuildState;
 class MetaPipeline;
 class InterruptState;
 
-struct SourcePartitionInfo {
-	//! The current batch index
-	//! This is only set in case RequiresBatchIndex() is true, and the source has support for it (SupportsBatchIndex())
-	//! Otherwise this is left on INVALID_INDEX
-	//! The batch index is a globally unique, increasing index that should be used to maintain insertion order
-	//! //! in conjunction with parallelism
-	optional_idx batch_index;
-	//! The minimum batch index that any thread is currently actively reading
-	optional_idx min_batch_index;
-};
-
 // LCOV_EXCL_START
 class OperatorState {
 public:

package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp

@@ -45,8 +45,6 @@ public:
 	//! Whether or not we should preserve insertion order for executing the given sink
 	static bool PreserveInsertionOrder(ClientContext &context, PhysicalOperator &plan);
 
-	static bool HasEquality(vector<JoinCondition> &conds, idx_t &range_count);
-
 protected:
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalOperator &op);