duckdb 0.8.2-dev2673.0 → 0.8.2-dev2809.0

package/src/duckdb/src/include/duckdb/common/types/row/row_data_collection_scanner.hpp

@@ -50,6 +50,10 @@ public:
 	RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
 	                         bool flush = true);
 
+	//	Single block scan
+	RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
+	                         idx_t block_idx, bool flush);
+
 	//! The type layout of the payload
 	inline const vector<LogicalType> &GetTypes() const {
 		return layout.GetTypes();
@@ -93,7 +97,7 @@ private:
 	//! Read state
 	ScanState read_state;
 	//! The total count of sorted_data
-	const idx_t total_count;
+	idx_t total_count;
 	//! The number of rows scanned so far
 	idx_t total_scanned;
 	//! Addresses used to gather from the sorted data

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -36,7 +36,7 @@ public:
 	ART(const vector<column_t> &column_ids, TableIOManager &table_io_manager,
 	    const vector<unique_ptr<Expression>> &unbound_expressions, const IndexConstraintType constraint_type,
 	    AttachedDatabase &db, const shared_ptr<vector<FixedSizeAllocator>> &allocators_ptr = nullptr,
-	    const idx_t block_id = DConstants::INVALID_INDEX, const idx_t block_offset = DConstants::INVALID_INDEX);
+	    BlockPointer block = BlockPointer());
 	~ART() override;
 
 	//! Root of the tree
@@ -80,7 +80,7 @@ public:
 	void SearchEqualJoinNoFetch(ARTKey &key, idx_t &result_size);
 
 	//! Serializes the index and returns the pair of block_id offset positions
-	BlockPointer Serialize(MetaBlockWriter &writer) override;
+	BlockPointer Serialize(MetadataWriter &writer) override;
 
 	//! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
 	//! index must also be locked during the merge

package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp

@@ -15,8 +15,8 @@
 namespace duckdb {
 
 // classes
-class MetaBlockWriter;
-class MetaBlockReader;
+class MetadataWriter;
+class MetadataReader;
 
 // structs
 struct BlockPointer;
@@ -70,9 +70,9 @@ public:
 	static string VerifyAndToString(ART &art, Node &node);
 
 	//! Serialize the leaf (chain)
-	static BlockPointer Serialize(ART &art, Node &node, MetaBlockWriter &writer);
+	static BlockPointer Serialize(ART &art, Node &node, MetadataWriter &writer);
 	//! Deserialize the leaf (chain)
-	static void Deserialize(ART &art, Node &node, MetaBlockReader &reader);
+	static void Deserialize(ART &art, Node &node, MetadataReader &reader);
 
 	//! Vacuum the leaf (chain)
 	static void Vacuum(ART &art, Node &node);

package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/common/to_string.hpp"
 #include "duckdb/common/typedefs.hpp"
+#include "duckdb/common/limits.hpp"
 
 namespace duckdb {
 
@@ -28,12 +29,13 @@ enum class NType : uint8_t {
 class FixedSizeAllocator;
 class ART;
 class Prefix;
-class MetaBlockReader;
-class MetaBlockWriter;
+class MetadataReader;
+class MetadataWriter;
 
 // structs
 struct BlockPointer;
 struct ARTFlags;
+struct MetaBlockPointer;
 
 //! The Node is the pointer class of the ART index.
 //! If the node is serialized, then the pointer points to a storage address (and has no type),
@@ -70,7 +72,7 @@ public:
 	//! Constructs an empty Node
 	Node() : data(0) {};
 	//! Constructs a serialized Node pointer from a block ID and an offset
-	explicit Node(MetaBlockReader &reader);
+	explicit Node(MetadataReader &reader);
 	//! Constructs an in-memory Node from a buffer ID and an offset
 	Node(const uint32_t buffer_id, const uint32_t offset) : data(0) {
 		SetPtr(buffer_id, offset);
@@ -95,7 +97,7 @@ public:
 	optional_ptr<Node> GetNextChild(ART &art, uint8_t &byte, const bool deserialize = true) const;
 
 	//! Serialize the node
-	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
+	BlockPointer Serialize(ART &art, MetadataWriter &writer);
 	//! Deserialize the node
 	void Deserialize(ART &art);
 

package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp

@@ -56,9 +56,9 @@ public:
 	optional_ptr<Node> GetNextChild(uint8_t &byte);
 
 	//! Serialize this node
-	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
+	BlockPointer Serialize(ART &art, MetadataWriter &writer);
 	//! Deserialize this node
-	void Deserialize(MetaBlockReader &reader);
+	void Deserialize(MetadataReader &reader);
 
 	//! Vacuum the children of the node
 	void Vacuum(ART &art, const ARTFlags &flags);

package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp

@@ -59,9 +59,9 @@ public:
 	optional_ptr<Node> GetNextChild(uint8_t &byte);
 
 	//! Serialize this node
-	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
+	BlockPointer Serialize(ART &art, MetadataWriter &writer);
 	//! Deserialize this node
-	void Deserialize(MetaBlockReader &reader);
+	void Deserialize(MetadataReader &reader);
 
 	//! Vacuum the children of the node
 	void Vacuum(ART &art, const ARTFlags &flags);

package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp

@@ -54,9 +54,9 @@ public:
 	optional_ptr<Node> GetNextChild(uint8_t &byte);
 
 	//! Serialize this node
-	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
+	BlockPointer Serialize(ART &art, MetadataWriter &writer);
 	//! Deserialize this node
-	void Deserialize(MetaBlockReader &reader);
+	void Deserialize(MetadataReader &reader);
 
 	//! Vacuum the children of the node
 	void Vacuum(ART &art, const ARTFlags &flags);

package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp

@@ -66,9 +66,9 @@ public:
 	optional_ptr<Node> GetNextChild(uint8_t &byte);
 
 	//! Serialize this node
-	BlockPointer Serialize(ART &art, MetaBlockWriter &writer);
+	BlockPointer Serialize(ART &art, MetadataWriter &writer);
 	//! Deserialize this node
-	void Deserialize(MetaBlockReader &reader);
+	void Deserialize(MetadataReader &reader);
 
 	//! Vacuum the children of the node
 	void Vacuum(ART &art, const ARTFlags &flags);

package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp

@@ -74,9 +74,9 @@ public:
 	static string VerifyAndToString(ART &art, Node &node, const bool only_verify);
 
 	//! Serialize this node and all subsequent nodes
-	static BlockPointer Serialize(ART &art, Node &node, MetaBlockWriter &writer);
+	static BlockPointer Serialize(ART &art, Node &node, MetadataWriter &writer);
 	//! Deserialize this node and all subsequent prefix nodes
-	static void Deserialize(ART &art, Node &node, MetaBlockReader &reader);
+	static void Deserialize(ART &art, Node &node, MetadataReader &reader);
 
 	//! Vacuum the child of the node
 	static void Vacuum(ART &art, Node &node, const ARTFlags &flags);

package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp

@@ -36,7 +36,6 @@ public:
 public:
 	PhysicalOperator(PhysicalOperatorType type, vector<LogicalType> types, idx_t estimated_cardinality)
 	    : type(type), types(std::move(types)), estimated_cardinality(estimated_cardinality) {
-		estimated_props = make_uniq<EstimatedProperties>(estimated_cardinality, 0);
 	}
 
 	virtual ~PhysicalOperator() {
@@ -50,7 +49,6 @@ public:
 	vector<LogicalType> types;
 	//! The estimated cardinality of this physical operator
 	idx_t estimated_cardinality;
-	unique_ptr<EstimatedProperties> estimated_props;
 
 	//! The global sink state of this operator
 	unique_ptr<GlobalSinkState> sink_state;

package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp

@@ -34,6 +34,8 @@ public:
 	//! If this returns RESULT_READY, the Execute function can be called to obtain a pointer to the result.
 	//! If this returns RESULT_NOT_READY, the ExecuteTask function should be called again.
 	//! If this returns EXECUTION_ERROR, an error occurred during execution.
+	//! If this returns NO_TASKS_AVAILABLE, this means currently no meaningful work can be done by the current executor,
+	//!	    but tasks may become available in the future.
 	//! The error message can be obtained by calling GetError() on the PendingQueryResult.
 	DUCKDB_API PendingExecutionResult ExecuteTask();
 
@@ -43,6 +45,9 @@ public:
 
 	DUCKDB_API void Close();
 
+	//! Function to determine whether execution is considered finished
+	DUCKDB_API static bool IsFinished(PendingExecutionResult result);
+
 private:
 	shared_ptr<ClientContext> context;
 	bool allow_stream_result;

package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp

@@ -7,21 +7,14 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
-#include "duckdb/optimizer/join_order/join_node.hpp"
-#include "duckdb/planner/column_binding.hpp"
 #include "duckdb/planner/column_binding_map.hpp"
-#include "duckdb/planner/filter/conjunction_filter.hpp"
-#include "duckdb/planner/filter/constant_filter.hpp"
+#include "duckdb/optimizer/join_order/query_graph.hpp"
+
+#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp"
 
 namespace duckdb {
 
-struct RelationAttributes {
-	string original_name;
-	// the relation columns used in join filters
-	// Needed when iterating over columns and initializing total domain values.
-	unordered_set<idx_t> columns;
-	double cardinality;
-};
+struct FilterInfo;
 
 struct RelationsToTDom {
 	//! column binding sets that are equivalent in a join plan.
@@ -33,19 +26,13 @@ struct RelationsToTDom {
 	idx_t tdom_no_hll;
 	bool has_tdom_hll;
 	vector<FilterInfo *> filters;
+	vector<string> column_names;
 
 	RelationsToTDom(const column_binding_set_t &column_binding_set)
 	    : equivalent_relations(column_binding_set), tdom_hll(0), tdom_no_hll(NumericLimits<idx_t>::Maximum()),
 	      has_tdom_hll(false) {};
 };
 
-struct NodeOp {
-	unique_ptr<JoinNode> node;
-	LogicalOperator &op;
-
-	NodeOp(unique_ptr<JoinNode> node, LogicalOperator &op) : node(std::move(node)), op(op) {};
-};
-
 struct Subgraph2Denominator {
 	unordered_set<idx_t> relations;
 	double denom;
@@ -53,69 +40,56 @@ struct Subgraph2Denominator {
 	Subgraph2Denominator() : relations(), denom(1) {};
 };
 
-class CardinalityEstimator {
+class CardinalityHelper {
 public:
-	explicit CardinalityEstimator(ClientContext &context) : context(context) {
+	CardinalityHelper() {
 	}
+	CardinalityHelper(double cardinality_before_filters, double filter_string)
+	    : cardinality_before_filters(cardinality_before_filters), filter_strength(filter_string) {};
 
-private:
-	ClientContext &context;
+public:
+	double cardinality_before_filters;
+	double filter_strength;
+
+	vector<string> table_names_joined;
+	vector<string> column_names;
+};
 
-	//! A mapping of relation id -> RelationAttributes
-	unordered_map<idx_t, RelationAttributes> relation_attributes;
-	//! A mapping of (relation, bound_column) -> (actual table, actual column)
-	column_binding_map_t<ColumnBinding> relation_column_to_original_column;
+class CardinalityEstimator {
+public:
+	explicit CardinalityEstimator() {};
 
+private:
 	vector<RelationsToTDom> relations_to_tdoms;
+	unordered_map<string, CardinalityHelper> relation_set_2_cardinality;
+	JoinRelationSetManager set_manager;
+	vector<RelationStats> relation_stats;
 
 public:
-	static constexpr double DEFAULT_SELECTIVITY = 0.2;
-
-	static void VerifySymmetry(JoinNode &result, JoinNode &entry);
-
-	//! given a binding of (relation, column) used for DP, and a (table, column) in that catalog
-	//! Add the key value entry into the relation_column_to_original_column
-	void AddRelationToColumnMapping(ColumnBinding key, ColumnBinding value);
-	//! Add a column to the relation_to_columns map.
-	void AddColumnToRelationMap(idx_t table_index, idx_t column_index);
-	//! Dump all bindings in relation_column_to_original_column into the child_binding_map
-	// If you have a non-reorderable join, this function is used to keep track of bindings
-	// in the child join plan.
-	void CopyRelationMap(column_binding_map_t<ColumnBinding> &child_binding_map);
-	void MergeBindings(idx_t, idx_t relation_id, vector<column_binding_map_t<ColumnBinding>> &child_binding_maps);
-	void AddRelationColumnMapping(LogicalGet &get, idx_t relation_id);
-
-	void InitTotalDomains();
-	void UpdateTotalDomains(JoinNode &node, LogicalOperator &op);
-	void InitEquivalentRelations(vector<unique_ptr<FilterInfo>> &filter_infos);
-
-	void InitCardinalityEstimatorProps(vector<NodeOp> &node_ops, vector<unique_ptr<FilterInfo>> &filter_infos);
-	double EstimateCardinalityWithSet(JoinRelationSet &new_set);
-	void EstimateBaseTableCardinality(JoinNode &node, LogicalOperator &op);
-	double EstimateCrossProduct(const JoinNode &left, const JoinNode &right);
-	static double ComputeCost(JoinNode &left, JoinNode &right, double expected_cardinality);
+	void RemoveEmptyTotalDomains();
+	void UpdateTotalDomains(optional_ptr<JoinRelationSet> set, RelationStats &stats);
+	void InitEquivalentRelations(const vector<unique_ptr<FilterInfo>> &filter_infos);
+
+	void InitCardinalityEstimatorProps(optional_ptr<JoinRelationSet> set, RelationStats &stats);
+
+	//! cost model needs estimated cardinalities to the fraction since the formula captures
+	//! distinct count selectivities and multiplicities. Hence the template
+	template <class T>
+	T EstimateCardinalityWithSet(JoinRelationSet &new_set);
+
+	//! used for debugging.
+	void AddRelationNamesToTdoms(vector<RelationStats> &stats);
+	void PrintRelationToTdomInfo();
 
 private:
 	bool SingleColumnFilter(FilterInfo &filter_info);
-	//! Filter & bindings -> list of indexes into the equivalent_relations array.
-	// The column binding set at each index is an equivalence set.
 	vector<idx_t> DetermineMatchingEquivalentSets(FilterInfo *filter_info);
-
 	//! Given a filter, add the column bindings to the matching equivalent set at the index
 	//! given in matching equivalent sets.
 	//! If there are multiple equivalence sets, they are merged.
 	void AddToEquivalenceSets(FilterInfo *filter_info, vector<idx_t> matching_equivalent_sets);
-
-	optional_ptr<TableFilterSet> GetTableFilters(LogicalOperator &op, idx_t table_index);
-
 	void AddRelationTdom(FilterInfo &filter_info);
 	bool EmptyFilter(FilterInfo &filter_info);
-
-	idx_t InspectConjunctionAND(idx_t cardinality, idx_t column_index, ConjunctionAndFilter &fil,
-	                            unique_ptr<BaseStatistics> base_stats);
-	idx_t InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter &fil,
-	                           unique_ptr<BaseStatistics> base_stats);
-	idx_t InspectTableFilters(idx_t cardinality, LogicalOperator &op, TableFilterSet &table_filters, idx_t table_index);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/join_order/cost_model.hpp

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_order/cost_model.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
+
+namespace duckdb {
+
+class QueryGraphManager;
+
+class CostModel {
+public:
+	CostModel(QueryGraphManager &query_graph_manager);
+
+private:
+	//! query graph storing relation manager information
+	QueryGraphManager &query_graph_manager;
+
+public:
+	void InitCostModel();
+
+	//! Compute cost of a join relation set
+	double ComputeCost(JoinNode &left, JoinNode &right);
+
+	//! Cardinality Estimator used to calculate cost
+	CardinalityEstimator cardinality_estimator;
+
+private:
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/join_order/join_node.hpp

@@ -5,62 +5,47 @@
 //
 //
 //===----------------------------------------------------------------------===//
-
 #pragma once
 
-#include "duckdb/common/unordered_map.hpp"
-#include "duckdb/common/unordered_set.hpp"
-#include "duckdb/optimizer/join_order/estimated_properties.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
 #include "duckdb/optimizer/join_order/query_graph.hpp"
-#include "duckdb/parser/expression_map.hpp"
-#include "duckdb/planner/logical_operator_visitor.hpp"
-#include "duckdb/planner/table_filter.hpp"
-#include "duckdb/storage/statistics/distinct_statistics.hpp"
 
 namespace duckdb {
 
-class JoinOrderOptimizer;
+struct NeighborInfo;
 
 class JoinNode {
 public:
 	//! Represents a node in the join plan
 	JoinRelationSet &set;
+	//! information on how left and right are connected
 	optional_ptr<NeighborInfo> info;
-	//! If the JoinNode is a base table, then base_cardinality is the cardinality before filters
-	//! estimated_props.cardinality will be the cardinality after filters. With no filters, the two are equal
-	bool has_filter;
+	//! left and right plans
 	optional_ptr<JoinNode> left;
 	optional_ptr<JoinNode> right;
 
-	unique_ptr<EstimatedProperties> estimated_props;
+	//! The cost of the join node. The cost is stored here so that the cost of
+	//! a join node stays in sync with how the join node is constructed. Storing the cost in an unordered_set
+	//! in the cost model is error prone. If the plan enumerator join node is updated and not the cost model
+	//! the whole Join Order Optimizer can start exhibiting undesired behavior.
+	double cost;
+	//! used only to populate logical operators with estimated caridnalities after the best join plan has been found.
+	idx_t cardinality;
+
+	//! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality
+	JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right, double cost);
 
 	//! Create a leaf node in the join tree
 	//! set cost to 0 for leaf nodes
 	//! cost will be the cost to *produce* an intermediate table
-	JoinNode(JoinRelationSet &set, const double base_cardinality);
-
-	//! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality
-	JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right,
-	         const double base_cardinality, double cost);
+	JoinNode(JoinRelationSet &set);
 
 	bool operator==(const JoinNode &other) {
 		return other.set.ToString().compare(set.ToString()) == 0;
 	}
 
 private:
-	double base_cardinality;
-
 public:
-	template <class CARDINALITY_TYPE>
-	CARDINALITY_TYPE GetCardinality() const {
-		return estimated_props->GetCardinality<CARDINALITY_TYPE>();
-	}
-	double GetCost();
-	void SetCost(double cost);
-	double GetBaseTableCardinality();
-	void SetBaseTableCardinality(double base_card);
-	void SetEstimatedCardinality(double estimated_card);
 	void PrintJoinNode();
 	string ToString();
 };

package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
+#include "duckdb/optimizer/join_order/query_graph_manager.hpp"
 #include "duckdb/optimizer/join_order/join_relation.hpp"
 #include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
 #include "duckdb/optimizer/join_order/query_graph.hpp"
@@ -22,22 +23,13 @@
 
 namespace duckdb {
 
-struct GenerateJoinRelation {
-	GenerateJoinRelation(JoinRelationSet &set, unique_ptr<LogicalOperator> op_p) : set(set), op(std::move(op_p)) {
-	}
-
-	JoinRelationSet &set;
-	unique_ptr<LogicalOperator> op;
-};
-
 class JoinOrderOptimizer {
 public:
-	explicit JoinOrderOptimizer(ClientContext &context)
-	    : context(context), cardinality_estimator(context), full_plan_found(false), must_update_full_plan(false) {
+	explicit JoinOrderOptimizer(ClientContext &context) : context(context), query_graph_manager(context) {
 	}
 
 	//! Perform join reordering inside a plan
-	unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> plan);
+	unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> plan, optional_ptr<RelationStats> stats = nullptr);
 
 	unique_ptr<JoinNode> CreateJoinTree(JoinRelationSet &set,
 	                                    const vector<reference<NeighborInfo>> &possible_connections, JoinNode &left,
@@ -45,16 +37,10 @@ public:
 
 private:
 	ClientContext &context;
-	//! The total amount of join pairs that have been considered
-	idx_t pairs = 0;
-	//! Set of all relations considered in the join optimizer
-	vector<unique_ptr<SingleJoinRelation>> relations;
-	//! A mapping of base table index -> index into relations array (relation number)
-	unordered_map<idx_t, idx_t> relation_mapping;
-	//! A structure holding all the created JoinRelationSet objects
-	JoinRelationSetManager set_manager;
-	//! The set of edges used in the join optimizer
-	QueryGraph query_graph;
+
+	//! manages the query graph, relations, and edges between relations
+	QueryGraphManager query_graph_manager;
+
 	//! The optimal join plan found for the specific JoinRelationSet*
 	unordered_map<JoinRelationSet *, unique_ptr<JoinNode>> plans;
 

package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp

@@ -11,19 +11,8 @@
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/unordered_set.hpp"
-#include "duckdb/common/optional_ptr.hpp"
 
 namespace duckdb {
-class LogicalOperator;
-
-//! Represents a single relation and any metadata accompanying that relation
-struct SingleJoinRelation {
-	LogicalOperator &op;
-	optional_ptr<LogicalOperator> parent;
-
-	SingleJoinRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent) : op(op), parent(parent) {
-	}
-};
 
 //! Set of relations, used in the join graph.
 struct JoinRelationSet {

package/src/duckdb/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp

@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_order/plan_enumerator.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/optimizer/join_order/join_relation.hpp"
+#include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
+#include "duckdb/optimizer/join_order/query_graph.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/optimizer/join_order/cost_model.hpp"
+#include "duckdb/parser/expression_map.hpp"
+#include "duckdb/common/reference_map.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/logical_operator_visitor.hpp"
+
+#include <functional>
+
+namespace duckdb {
+
+class QueryGraphManager;
+
+class PlanEnumerator {
+public:
+	explicit PlanEnumerator(QueryGraphManager &query_graph_manager, CostModel &cost_model,
+	                        const QueryGraphEdges &query_graph)
+	    : query_graph(query_graph), query_graph_manager(query_graph_manager), cost_model(cost_model),
+	      full_plan_found(false), must_update_full_plan(false) {
+	}
+
+	//! Perform the join order solving
+	unique_ptr<JoinNode> SolveJoinOrder();
+	void InitLeafPlans();
+
+	static unique_ptr<LogicalOperator> BuildSideProbeSideSwaps(unique_ptr<LogicalOperator> plan);
+
+private:
+	QueryGraphEdges const &query_graph;
+	//! The total amount of join pairs that have been considered
+	idx_t pairs = 0;
+	//! The set of edges used in the join optimizer
+	QueryGraphManager &query_graph_manager;
+	//! Cost model to evaluate cost of joins
+	CostModel &cost_model;
+	//! A map to store the optimal join plan found for a specific JoinRelationSet*
+	reference_map_t<JoinRelationSet, unique_ptr<JoinNode>> plans;
+
+	bool full_plan_found;
+	bool must_update_full_plan;
+	unordered_set<string> join_nodes_in_full_plan;
+
+	unique_ptr<JoinNode> CreateJoinTree(JoinRelationSet &set,
+	                                    const vector<reference<NeighborInfo>> &possible_connections, JoinNode &left,
+	                                    JoinNode &right);
+
+	//! Emit a pair as a potential join candidate. Returns the best plan found for the (left, right) connection (either
+	//! the newly created plan, or an existing plan)
+	JoinNode &EmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector<reference<NeighborInfo>> &info);
+	//! Tries to emit a potential join candidate pair. Returns false if too many pairs have already been emitted,
+	//! cancelling the dynamic programming step.
+	bool TryEmitPair(JoinRelationSet &left, JoinRelationSet &right, const vector<reference<NeighborInfo>> &info);
+
+	bool EnumerateCmpRecursive(JoinRelationSet &left, JoinRelationSet &right, unordered_set<idx_t> &exclusion_set);
+	//! Emit a relation set node
+	bool EmitCSG(JoinRelationSet &node);
+	//! Enumerate the possible connected subgraphs that can be joined together in the join graph
+	bool EnumerateCSGRecursive(JoinRelationSet &node, unordered_set<idx_t> &exclusion_set);
+	//! Generate cross product edges inside the side
+	void GenerateCrossProducts();
+
+	//! Solve the join order exactly using dynamic programming. Returns true if it was completed successfully (i.e. did
+	//! not time-out)
+	bool SolveJoinOrderExactly();
+	//! Solve the join order approximately using a greedy algorithm
+	void SolveJoinOrderApproximately();
+
+	void UpdateDPTree(JoinNode &new_plan);
+
+	void UpdateJoinNodesInFullPlan(JoinNode &node);
+	bool NodeInFullPlan(JoinNode &node);
+};
+
+} // namespace duckdb