duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp

@@ -20,11 +20,12 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::IE_JOIN;
 
 public:
-	PhysicalIEJoin(LogicalComparisonJoin &op, PhysicalOperator &left, PhysicalOperator &right,
-	               vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality,
+	PhysicalIEJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, PhysicalOperator &left,
+	               PhysicalOperator &right, vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality,
 	               unique_ptr<JoinFilterPushdownInfo> pushdown_info);
-	PhysicalIEJoin(LogicalComparisonJoin &op, PhysicalOperator &left, PhysicalOperator &right,
-	               vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality);
+	PhysicalIEJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, PhysicalOperator &left,
+	               PhysicalOperator &right, vector<JoinCondition> cond, JoinType join_type,
+	               idx_t estimated_cardinality);
 
 	vector<LogicalType> join_key_types;
 	vector<BoundOrderByNode> lhs_orders;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_join.hpp

@@ -19,7 +19,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::INVALID;
 
 public:
-	PhysicalJoin(LogicalOperator &op, PhysicalOperatorType type, JoinType join_type, idx_t estimated_cardinality);
+	PhysicalJoin(PhysicalPlan &physical_plan, LogicalOperator &op, PhysicalOperatorType type, JoinType join_type,
+	             idx_t estimated_cardinality);
 
 	JoinType join_type;
 

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_left_delim_join.hpp

@@ -19,9 +19,10 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::LEFT_DELIM_JOIN;
 
 public:
-	PhysicalLeftDelimJoin(PhysicalPlanGenerator &planner, vector<LogicalType> types, PhysicalOperator &original_join,
-	                      PhysicalOperator &distinct, const vector<const_reference<PhysicalOperator>> &delim_scans,
-	                      idx_t estimated_cardinality, optional_idx delim_idx);
+	PhysicalLeftDelimJoin(PhysicalPlan &physical_plan, PhysicalPlanGenerator &planner, vector<LogicalType> types,
+	                      PhysicalOperator &original_join, PhysicalOperator &distinct,
+	                      const vector<const_reference<PhysicalOperator>> &delim_scans, idx_t estimated_cardinality,
+	                      optional_idx delim_idx);
 
 public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp

@@ -18,11 +18,12 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::NESTED_LOOP_JOIN;
 
 public:
-	PhysicalNestedLoopJoin(LogicalOperator &op, PhysicalOperator &left, PhysicalOperator &right,
-	                       vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality,
-	                       unique_ptr<JoinFilterPushdownInfo> pushdown_info);
-	PhysicalNestedLoopJoin(LogicalOperator &op, PhysicalOperator &left, PhysicalOperator &right,
-	                       vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality);
+	PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalOperator &op, PhysicalOperator &left,
+	                       PhysicalOperator &right, vector<JoinCondition> cond, JoinType join_type,
+	                       idx_t estimated_cardinality, unique_ptr<JoinFilterPushdownInfo> pushdown_info);
+	PhysicalNestedLoopJoin(PhysicalPlan &physical_plan, LogicalOperator &op, PhysicalOperator &left,
+	                       PhysicalOperator &right, vector<JoinCondition> cond, JoinType join_type,
+	                       idx_t estimated_cardinality);
 
 public:
 	// Operator Interface

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp

@@ -22,9 +22,9 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::PIECEWISE_MERGE_JOIN;
 
 public:
-	PhysicalPiecewiseMergeJoin(LogicalComparisonJoin &op, PhysicalOperator &left, PhysicalOperator &right,
-	                           vector<JoinCondition> cond, JoinType join_type, idx_t estimated_cardinality,
-	                           unique_ptr<JoinFilterPushdownInfo> pushdown_info);
+	PhysicalPiecewiseMergeJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, PhysicalOperator &left,
+	                           PhysicalOperator &right, vector<JoinCondition> cond, JoinType join_type,
+	                           idx_t estimated_cardinality, unique_ptr<JoinFilterPushdownInfo> pushdown_info);
 
 	vector<LogicalType> join_key_types;
 	vector<BoundOrderByNode> lhs_orders;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_positional_join.hpp

@@ -19,8 +19,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::POSITIONAL_JOIN;
 
 public:
-	PhysicalPositionalJoin(vector<LogicalType> types, PhysicalOperator &left, PhysicalOperator &right,
-	                       idx_t estimated_cardinality);
+	PhysicalPositionalJoin(PhysicalPlan &physical_plan, vector<LogicalType> types, PhysicalOperator &left,
+	                       PhysicalOperator &right, idx_t estimated_cardinality);
 
 public:
 	// Operator Interface

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp

@@ -92,9 +92,9 @@ public:
 	};
 
 public:
-	PhysicalRangeJoin(LogicalComparisonJoin &op, PhysicalOperatorType type, PhysicalOperator &left,
-	                  PhysicalOperator &right, vector<JoinCondition> cond, JoinType join_type,
-	                  idx_t estimated_cardinality);
+	PhysicalRangeJoin(PhysicalPlan &physical_plan, LogicalComparisonJoin &op, PhysicalOperatorType type,
+	                  PhysicalOperator &left, PhysicalOperator &right, vector<JoinCondition> cond, JoinType join_type,
+	                  idx_t estimated_cardinality, unique_ptr<JoinFilterPushdownInfo> pushdown_info);
 
 	// Projection mappings
 	using ProjectionMapping = vector<column_t>;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_right_delim_join.hpp

@@ -19,9 +19,10 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::RIGHT_DELIM_JOIN;
 
 public:
-	PhysicalRightDelimJoin(PhysicalPlanGenerator &planner, vector<LogicalType> types, PhysicalOperator &original_join,
-	                       PhysicalOperator &distinct, const vector<const_reference<PhysicalOperator>> &delim_scans,
-	                       idx_t estimated_cardinality, optional_idx delim_idx);
+	PhysicalRightDelimJoin(PhysicalPlan &physical_plan, PhysicalPlanGenerator &planner, vector<LogicalType> types,
+	                       PhysicalOperator &original_join, PhysicalOperator &distinct,
+	                       const vector<const_reference<PhysicalOperator>> &delim_scans, idx_t estimated_cardinality,
+	                       optional_idx delim_idx);
 
 public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp

@@ -16,21 +16,23 @@ namespace duckdb {
 
 class OrderGlobalSinkState;
 
-//! Physically re-orders the input data
+//! PhysicalOrder sorts the input data
 class PhysicalOrder : public PhysicalOperator {
 public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::ORDER_BY;
 
 public:
-	PhysicalOrder(vector<LogicalType> types, vector<BoundOrderByNode> orders, vector<idx_t> projections,
-	              idx_t estimated_cardinality);
+	PhysicalOrder(PhysicalPlan &physical_plan, vector<LogicalType> types, vector<BoundOrderByNode> orders,
+	              vector<idx_t> projections, idx_t estimated_cardinality, bool is_index_sort = false);
 
-	//! Input data
 	vector<BoundOrderByNode> orders;
 	vector<idx_t> projections;
+	bool is_index_sort;
 
 public:
-	// Source interface
+	//===--------------------------------------------------------------------===//
+	// Source Interface
+	//===--------------------------------------------------------------------===//
 	unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
 	                                                 GlobalSourceState &gstate) const override;
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
@@ -38,6 +40,7 @@ public:
 	OperatorPartitionData GetPartitionData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	                                       LocalSourceState &lstate,
 	                                       const OperatorPartitionInfo &partition_info) const override;
+	ProgressData GetProgress(ClientContext &context, GlobalSourceState &gstate) const override;
 
 	bool IsSource() const override {
 		return true;
@@ -59,13 +62,17 @@ public:
 	}
 
 public:
-	// Sink interface
+	//===--------------------------------------------------------------------===//
+	// Sink Interface
+	//===--------------------------------------------------------------------===//
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
 	SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
 	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
 	                          OperatorSinkFinalizeInput &input) const override;
+	ProgressData GetSinkProgress(ClientContext &context, GlobalSinkState &gstate,
+	                             const ProgressData source_progress) const override;
 
 	bool IsSink() const override {
 		return true;
@@ -79,9 +86,6 @@ public:
 
 public:
 	InsertionOrderPreservingMap<string> ParamsToString() const override;
-
-	//! Schedules tasks to merge the data during the Finalize phase
-	static void ScheduleMergeTasks(Pipeline &pipeline, Event &event, OrderGlobalSinkState &state);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp

@@ -21,8 +21,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::TOP_N;
 
 public:
-	PhysicalTopN(vector<LogicalType> types, vector<BoundOrderByNode> orders, idx_t limit, idx_t offset,
-	             shared_ptr<DynamicFilterData> dynamic_filter, idx_t estimated_cardinality);
+	PhysicalTopN(PhysicalPlan &physical_plan, vector<LogicalType> types, vector<BoundOrderByNode> orders, idx_t limit,
+	             idx_t offset, shared_ptr<DynamicFilterData> dynamic_filter, idx_t estimated_cardinality);
 	~PhysicalTopN() override;
 
 	vector<BoundOrderByNode> orders;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp

@@ -22,8 +22,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::BATCH_COPY_TO_FILE;
 
 public:
-	PhysicalBatchCopyToFile(vector<LogicalType> types, CopyFunction function, unique_ptr<FunctionData> bind_data,
-	                        idx_t estimated_cardinality);
+	PhysicalBatchCopyToFile(PhysicalPlan &physical_plan, vector<LogicalType> types, CopyFunction function,
+	                        unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality);
 
 	CopyFunction function;
 	unique_ptr<FunctionData> bind_data;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp

@@ -18,11 +18,11 @@ public:
 
 public:
 	//! INSERT INTO
-	PhysicalBatchInsert(vector<LogicalType> types, TableCatalogEntry &table,
+	PhysicalBatchInsert(PhysicalPlan &physical_plan, vector<LogicalType> types, TableCatalogEntry &table,
 	                    vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t estimated_cardinality);
 	//! CREATE TABLE AS
-	PhysicalBatchInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr<BoundCreateTableInfo> info,
-	                    idx_t estimated_cardinality);
+	PhysicalBatchInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema,
+	                    unique_ptr<BoundCreateTableInfo> info, idx_t estimated_cardinality);
 
 	//! The table to insert into
 	optional_ptr<TableCatalogEntry> insert_table;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_database.hpp

@@ -18,7 +18,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::COPY_DATABASE;
 
 public:
-	PhysicalCopyDatabase(vector<LogicalType> types, idx_t estimated_cardinality, unique_ptr<CopyDatabaseInfo> info_p);
+	PhysicalCopyDatabase(PhysicalPlan &physical_plan, vector<LogicalType> types, idx_t estimated_cardinality,
+	                     unique_ptr<CopyDatabaseInfo> info_p);
 	~PhysicalCopyDatabase() override;
 
 	unique_ptr<CopyDatabaseInfo> info;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp

@@ -33,8 +33,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::COPY_TO_FILE;
 
 public:
-	PhysicalCopyToFile(vector<LogicalType> types, CopyFunction function, unique_ptr<FunctionData> bind_data,
-	                   idx_t estimated_cardinality);
+	PhysicalCopyToFile(PhysicalPlan &physical_plan, vector<LogicalType> types, CopyFunction function,
+	                   unique_ptr<FunctionData> bind_data, idx_t estimated_cardinality);
 
 	CopyFunction function;
 	unique_ptr<FunctionData> bind_data;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_delete.hpp

@@ -20,8 +20,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::DELETE_OPERATOR;
 
 public:
-	PhysicalDelete(vector<LogicalType> types, TableCatalogEntry &tableref, DataTable &table,
-	               vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t row_id_index,
+	PhysicalDelete(PhysicalPlan &physical_plan, vector<LogicalType> types, TableCatalogEntry &tableref,
+	               DataTable &table, vector<unique_ptr<BoundConstraint>> bound_constraints, idx_t row_id_index,
 	               idx_t estimated_cardinality, bool return_chunk);
 
 	TableCatalogEntry &tableref;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_export.hpp

@@ -34,8 +34,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::EXPORT;
 
 public:
-	PhysicalExport(vector<LogicalType> types, CopyFunction function, unique_ptr<CopyInfo> info,
-	               idx_t estimated_cardinality, unique_ptr<BoundExportData> exported_tables);
+	PhysicalExport(PhysicalPlan &physical_plan, vector<LogicalType> types, CopyFunction function,
+	               unique_ptr<CopyInfo> info, idx_t estimated_cardinality, unique_ptr<BoundExportData> exported_tables);
 
 	//! The copy function to use to read the file
 	CopyFunction function;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp

@@ -69,7 +69,7 @@ public:
 
 public:
 	//! INSERT INTO
-	PhysicalInsert(vector<LogicalType> types, TableCatalogEntry &table,
+	PhysicalInsert(PhysicalPlan &physical_plan, vector<LogicalType> types, TableCatalogEntry &table,
 	               vector<unique_ptr<BoundConstraint>> bound_constraints,
 	               vector<unique_ptr<Expression>> set_expressions, vector<PhysicalIndex> set_columns,
 	               vector<LogicalType> set_types, idx_t estimated_cardinality, bool return_chunk, bool parallel,
@@ -77,8 +77,8 @@ public:
 	               unique_ptr<Expression> do_update_condition, unordered_set<column_t> on_conflict_filter,
 	               vector<column_t> columns_to_fetch, bool update_is_del_and_insert);
 	//! CREATE TABLE AS
-	PhysicalInsert(LogicalOperator &op, SchemaCatalogEntry &schema, unique_ptr<BoundCreateTableInfo> info,
-	               idx_t estimated_cardinality, bool parallel);
+	PhysicalInsert(PhysicalPlan &physical_plan, LogicalOperator &op, SchemaCatalogEntry &schema,
+	               unique_ptr<BoundCreateTableInfo> info, idx_t estimated_cardinality, bool parallel);
 
 	//! The table to insert into
 	optional_ptr<TableCatalogEntry> insert_table;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_merge_into.hpp

@@ -0,0 +1,87 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/persistent/physical_merge_into.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/physical_operator.hpp"
+#include "duckdb/common/enums/merge_action_type.hpp"
+
+namespace duckdb {
+class MergeIntoLocalState;
+
+class MergeIntoOperator {
+public:
+	// Merge action type
+	MergeActionType action_type;
+	//! Condition - or NULL if this should always be performed for the given action
+	unique_ptr<Expression> condition;
+	//! The operator to push data into for this action (if any)
+	optional_ptr<PhysicalOperator> op;
+	//! Expressions to execute (if any) prior to sinking
+	vector<unique_ptr<Expression>> expressions;
+};
+
+struct MergeActionRange {
+	MergeActionCondition condition;
+	idx_t start = 0;
+	idx_t end = 0;
+};
+
+class PhysicalMergeInto : public PhysicalOperator {
+public:
+	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::MERGE_INTO;
+
+public:
+	PhysicalMergeInto(PhysicalPlan &physical_plan, vector<LogicalType> types,
+	                  map<MergeActionCondition, vector<unique_ptr<MergeIntoOperator>>> actions, idx_t row_id_index,
+	                  optional_idx source_marker, bool parallel, bool return_chunk);
+
+	//! List of all actions
+	vector<unique_ptr<MergeIntoOperator>> actions;
+	//! Sequence of match actions
+	vector<MergeActionCondition> match_actions;
+	//! List of all actions that apply to a given action condition
+	vector<MergeActionRange> action_ranges;
+	idx_t row_id_index;
+	optional_idx source_marker;
+	bool parallel;
+	bool return_chunk;
+
+public:
+	// Source interface
+	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
+	unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context,
+	                                                 GlobalSourceState &gstate) const override;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
+
+public:
+	// Sink interface
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
+	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
+	SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
+	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+	                          OperatorSinkFinalizeInput &input) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+	bool ParallelSink() const override {
+		return parallel;
+	}
+
+private:
+	idx_t GetIndex(MergeActionCondition condition) const;
+	void ComputeMatches(MergeIntoLocalState &local_state, DataChunk &chunk) const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp

@@ -21,8 +21,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::UPDATE;
 
 public:
-	PhysicalUpdate(vector<LogicalType> types, TableCatalogEntry &tableref, DataTable &table,
-	               vector<PhysicalIndex> columns, vector<unique_ptr<Expression>> expressions,
+	PhysicalUpdate(PhysicalPlan &physical_plan, vector<LogicalType> types, TableCatalogEntry &tableref,
+	               DataTable &table, vector<PhysicalIndex> columns, vector<unique_ptr<Expression>> expressions,
 	               vector<unique_ptr<Expression>> bound_defaults, vector<unique_ptr<BoundConstraint>> bound_constraints,
 	               idx_t estimated_cardinality, bool return_chunk);
 

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_pivot.hpp

@@ -18,7 +18,8 @@ namespace duckdb {
 //! PhysicalPivot implements the physical PIVOT operation
 class PhysicalPivot : public PhysicalOperator {
 public:
-	PhysicalPivot(vector<LogicalType> types, PhysicalOperator &child, BoundPivotInfo bound_pivot);
+	PhysicalPivot(PhysicalPlan &physical_plan, vector<LogicalType> types, PhysicalOperator &child,
+	              BoundPivotInfo bound_pivot);
 
 	BoundPivotInfo bound_pivot;
 	//! The map for pivot value -> column index

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_projection.hpp

@@ -18,8 +18,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::PROJECTION;
 
 public:
-	PhysicalProjection(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
-	                   idx_t estimated_cardinality);
+	PhysicalProjection(PhysicalPlan &physical_plan, vector<LogicalType> types,
+	                   vector<unique_ptr<Expression>> select_list, idx_t estimated_cardinality);
 
 	vector<unique_ptr<Expression>> select_list;
 
@@ -33,11 +33,6 @@ public:
 	}
 
 	InsertionOrderPreservingMap<string> ParamsToString() const override;
-
-	static unique_ptr<PhysicalOperator>
-	CreateJoinProjection(vector<LogicalType> proj_types, const vector<LogicalType> &lhs_types,
-	                     const vector<LogicalType> &rhs_types, const vector<idx_t> &left_projection_map,
-	                     const vector<idx_t> &right_projection_map, const idx_t estimated_cardinality);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_tableinout_function.hpp

@@ -19,13 +19,15 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::INOUT_FUNCTION;
 
 public:
-	PhysicalTableInOutFunction(vector<LogicalType> types, TableFunction function_p,
+	PhysicalTableInOutFunction(PhysicalPlan &physical_plan, vector<LogicalType> types, TableFunction function_p,
 	                           unique_ptr<FunctionData> bind_data_p, vector<ColumnIndex> column_ids_p,
 	                           idx_t estimated_cardinality, vector<column_t> projected_input);
 
 public:
 	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
 	unique_ptr<GlobalOperatorState> GetGlobalOperatorState(ClientContext &context) const override;
+	static void SetOrdinality(DataChunk &chunk, const optional_idx &ordinality_column_idx, const idx_t &ordinality_idx,
+	                          const idx_t &ordinality);
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 	OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, GlobalOperatorState &gstate,
@@ -41,6 +43,9 @@ public:
 
 	InsertionOrderPreservingMap<string> ParamsToString() const override;
 
+	//! Information for WITH ORDINALITY
+	optional_idx ordinality_idx;
+
 private:
 	//! The table function
 	TableFunction function;

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_unnest.hpp

@@ -19,8 +19,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::UNNEST;
 
 public:
-	PhysicalUnnest(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list, idx_t estimated_cardinality,
-	               PhysicalOperatorType type = PhysicalOperatorType::UNNEST);
+	PhysicalUnnest(PhysicalPlan &physical_plan, vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
+	               idx_t estimated_cardinality, PhysicalOperatorType type = PhysicalOperatorType::UNNEST);
 
 	//! The projection list of the UNNEST
 	//! E.g. SELECT 1, UNNEST([1]), UNNEST([2, 3]); has two UNNESTs in its select_list

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp

@@ -20,11 +20,11 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::INVALID;
 
 public:
-	PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type, idx_t estimated_cardinality,
-	                       optionally_owned_ptr<ColumnDataCollection> collection);
+	PhysicalColumnDataScan(PhysicalPlan &physical_plan, vector<LogicalType> types, PhysicalOperatorType op_type,
+	                       idx_t estimated_cardinality, optionally_owned_ptr<ColumnDataCollection> collection);
 
-	PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type, idx_t estimated_cardinality,
-	                       idx_t cte_index);
+	PhysicalColumnDataScan(PhysicalPlan &physical_plan, vector<LogicalType> types, PhysicalOperatorType op_type,
+	                       idx_t estimated_cardinality, idx_t cte_index);
 
 	//! (optionally owned) column data collection to scan
 	optionally_owned_ptr<ColumnDataCollection> collection;

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_dummy_scan.hpp

@@ -17,8 +17,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::DUMMY_SCAN;
 
 public:
-	explicit PhysicalDummyScan(vector<LogicalType> types, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::DUMMY_SCAN, std::move(types), estimated_cardinality) {
+	explicit PhysicalDummyScan(PhysicalPlan &physical_plan, vector<LogicalType> types, idx_t estimated_cardinality)
+	    : PhysicalOperator(physical_plan, PhysicalOperatorType::DUMMY_SCAN, std::move(types), estimated_cardinality) {
 	}
 
 public:

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_empty_result.hpp

@@ -17,8 +17,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::EMPTY_RESULT;
 
 public:
-	explicit PhysicalEmptyResult(vector<LogicalType> types, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::EMPTY_RESULT, std::move(types), estimated_cardinality) {
+	explicit PhysicalEmptyResult(PhysicalPlan &physical_plan, vector<LogicalType> types, idx_t estimated_cardinality)
+	    : PhysicalOperator(physical_plan, PhysicalOperatorType::EMPTY_RESULT, std::move(types), estimated_cardinality) {
 	}
 
 public:

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_expression_scan.hpp

@@ -19,9 +19,10 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::EXPRESSION_SCAN;
 
 public:
-	PhysicalExpressionScan(vector<LogicalType> types, vector<vector<unique_ptr<Expression>>> expressions,
-	                       idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::EXPRESSION_SCAN, std::move(types), estimated_cardinality),
+	PhysicalExpressionScan(PhysicalPlan &physical_plan, vector<LogicalType> types,
+	                       vector<vector<unique_ptr<Expression>>> expressions, idx_t estimated_cardinality)
+	    : PhysicalOperator(physical_plan, PhysicalOperatorType::EXPRESSION_SCAN, std::move(types),
+	                       estimated_cardinality),
 	      expressions(std::move(expressions)) {
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_positional_scan.hpp

@@ -22,7 +22,8 @@ public:
 
 public:
 	//! Regular Table Scan
-	PhysicalPositionalScan(vector<LogicalType> types, PhysicalOperator &left, PhysicalOperator &right);
+	PhysicalPositionalScan(PhysicalPlan &physical_plan, vector<LogicalType> types, PhysicalOperator &left,
+	                       PhysicalOperator &right);
 
 	//! The child table functions
 	vector<reference<PhysicalOperator>> child_tables;

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp

@@ -24,9 +24,10 @@ public:
 
 public:
 	//! Table scan that immediately projects out filter columns that are unused in the remainder of the query plan
-	PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
-	                  vector<LogicalType> returned_types, vector<ColumnIndex> column_ids, vector<idx_t> projection_ids,
-	                  vector<string> names, unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality,
+	PhysicalTableScan(PhysicalPlan &physical_plan, vector<LogicalType> types, TableFunction function,
+	                  unique_ptr<FunctionData> bind_data, vector<LogicalType> returned_types,
+	                  vector<ColumnIndex> column_ids, vector<idx_t> projection_ids, vector<string> names,
+	                  unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality,
 	                  ExtraOperatorInfo extra_info, vector<Value> parameters, virtual_column_map_t virtual_columns);
 
 	//! The table function

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_alter.hpp

@@ -19,8 +19,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::ALTER;
 
 public:
-	explicit PhysicalAlter(unique_ptr<AlterInfo> info, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::ALTER, {LogicalType::BOOLEAN}, estimated_cardinality),
+	explicit PhysicalAlter(PhysicalPlan &physical_plan, unique_ptr<AlterInfo> info, idx_t estimated_cardinality)
+	    : PhysicalOperator(physical_plan, PhysicalOperatorType::ALTER, {LogicalType::BOOLEAN}, estimated_cardinality),
 	      info(std::move(info)) {
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_attach.hpp

@@ -19,8 +19,8 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::ATTACH;
 
 public:
-	explicit PhysicalAttach(unique_ptr<AttachInfo> info, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::ATTACH, {LogicalType::BOOLEAN}, estimated_cardinality),
+	explicit PhysicalAttach(PhysicalPlan &physical_plan, unique_ptr<AttachInfo> info, idx_t estimated_cardinality)
+	    : PhysicalOperator(physical_plan, PhysicalOperatorType::ATTACH, {LogicalType::BOOLEAN}, estimated_cardinality),
 	      info(std::move(info)) {
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_art_index.hpp

@@ -25,10 +25,10 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::CREATE_INDEX;
 
 public:
-	PhysicalCreateARTIndex(LogicalOperator &op, TableCatalogEntry &table, const vector<column_t> &column_ids,
-	                       unique_ptr<CreateIndexInfo> info, vector<unique_ptr<Expression>> unbound_expressions,
-	                       idx_t estimated_cardinality, const bool sorted,
-	                       unique_ptr<AlterTableInfo> alter_table_info = nullptr);
+	PhysicalCreateARTIndex(PhysicalPlan &physical_plan, LogicalOperator &op, TableCatalogEntry &table,
+	                       const vector<column_t> &column_ids, unique_ptr<CreateIndexInfo> info,
+	                       vector<unique_ptr<Expression>> unbound_expressions, idx_t estimated_cardinality,
+	                       const bool sorted, unique_ptr<AlterTableInfo> alter_table_info = nullptr);
 
 	//! The table to create the index for.
 	DuckTableEntry &table;

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_function.hpp

@@ -19,8 +19,10 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::CREATE_MACRO;
 
 public:
-	explicit PhysicalCreateFunction(unique_ptr<CreateMacroInfo> info, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::CREATE_MACRO, {LogicalType::BIGINT}, estimated_cardinality),
+	explicit PhysicalCreateFunction(PhysicalPlan &physical_plan, unique_ptr<CreateMacroInfo> info,
+	                                idx_t estimated_cardinality)
+	    : PhysicalOperator(physical_plan, PhysicalOperatorType::CREATE_MACRO, {LogicalType::BIGINT},
+	                       estimated_cardinality),
 	      info(std::move(info)) {
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_schema.hpp

@@ -19,8 +19,10 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::CREATE_SCHEMA;
 
 public:
-	explicit PhysicalCreateSchema(unique_ptr<CreateSchemaInfo> info, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::CREATE_SCHEMA, {LogicalType::BIGINT}, estimated_cardinality),
+	explicit PhysicalCreateSchema(PhysicalPlan &physical_plan, unique_ptr<CreateSchemaInfo> info,
+	                              idx_t estimated_cardinality)
+	    : PhysicalOperator(physical_plan, PhysicalOperatorType::CREATE_SCHEMA, {LogicalType::BIGINT},
+	                       estimated_cardinality),
 	      info(std::move(info)) {
 	}