duckdb 0.6.2-dev1184.0 → 0.6.2-dev1192.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1184.0",
+  "version": "0.6.2-dev1192.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/execution/operator/filter/physical_filter.cpp

@@ -53,7 +53,10 @@ OperatorResultType PhysicalFilter::ExecuteInternal(ExecutionContext &context, Da
 }
 
 string PhysicalFilter::ParamsToString() const {
-	return expression->GetName();
+	auto result = expression->GetName();
+	result += "\n[INFOSEPARATOR]\n";
+	result += StringUtil::Format("EC: %llu", estimated_props->GetCardinality<idx_t>());
+	return result;
 }
 
 } // namespace duckdb

package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp

@@ -29,8 +29,9 @@ string PhysicalComparisonJoin::ParamsToString() const {
 		string op = ExpressionTypeToOperator(it.comparison);
 		extra_info += it.left->GetName() + " " + op + " " + it.right->GetName() + "\n";
 	}
-	extra_info += "\nEC = " + std::to_string(estimated_props->GetCardinality<double>()) + "\n";
-	extra_info += "COST = " + std::to_string(estimated_props->GetCost()) + "\n";
+	extra_info += "\n[INFOSEPARATOR]\n";
+	extra_info += StringUtil::Format("EC: %llu\n", estimated_props->GetCardinality<idx_t>());
+	extra_info += StringUtil::Format("Cost: %llu", (idx_t)estimated_props->GetCost());
 	return extra_info;
 }
 

package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp

@@ -135,7 +135,8 @@ string PhysicalTableScan::ParamsToString() const {
 			}
 		}
 	}
-	result += "\nEC=" + to_string(estimated_cardinality) + "\n";
+	result += "\n[INFOSEPARATOR]\n";
+	result += StringUtil::Format("EC: %llu", estimated_props->GetCardinality<idx_t>());
 	return result;
 }
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev1184"
+#define DUCKDB_VERSION "0.6.2-dev1192"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "b0ed39acb9"
+#define DUCKDB_SOURCE_ID "4b2d12b676"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp

@@ -21,7 +21,7 @@ class DeliminatorPlanUpdater;
 //! The Deliminator optimizer traverses the logical operator tree and removes any redundant DelimGets/DelimJoins
 class Deliminator {
 public:
-	Deliminator() {
+	explicit Deliminator(ClientContext &context) : context(context) {
 	}
 	//! Perform DelimJoin elimination
 	unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
@@ -35,6 +35,9 @@ private:
 	//! Try to remove an inequality Join with a DelimGet, returns true if it was successful
 	bool RemoveInequalityCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<LogicalOperator> *candidate,
 	                               DeliminatorPlanUpdater &updater);
+
+private:
+	ClientContext &context;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp

@@ -69,13 +69,11 @@ private:
 
 	vector<RelationsToTDom> relations_to_tdoms;
 
+public:
 	static constexpr double DEFAULT_SELECTIVITY = 0.2;
 
-public:
 	static void VerifySymmetry(JoinNode *result, JoinNode *entry);
 
-	void AssertEquivalentRelationSize();
-
 	//! given a binding of (relation, column) used for DP, and a (table, column) in that catalog
 	//! Add the key value entry into the relation_column_to_original_column
 	void AddRelationToColumnMapping(ColumnBinding key, ColumnBinding value);
@@ -92,7 +90,7 @@ public:
 	void UpdateTotalDomains(JoinNode *node, LogicalOperator *op);
 	void InitEquivalentRelations(vector<unique_ptr<FilterInfo>> *filter_infos);
 
-	void InitCardinalityEstimatorProps(vector<struct NodeOp> *node_ops, vector<unique_ptr<FilterInfo>> *filter_infos);
+	void InitCardinalityEstimatorProps(vector<NodeOp> *node_ops, vector<unique_ptr<FilterInfo>> *filter_infos);
 	double EstimateCardinalityWithSet(JoinRelationSet *new_set);
 	void EstimateBaseTableCardinality(JoinNode *node, LogicalOperator *op);
 	double EstimateCrossProduct(const JoinNode *left, const JoinNode *right);

package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp

@@ -54,8 +54,7 @@ private:
 	//! The set of filter infos created from the extracted filters
 	vector<unique_ptr<FilterInfo>> filter_infos;
 	//! A map of all expressions a given expression has to be equivalent to. This is used to add "implied join edges".
-	//! i.e. in the join A=B AND B=C, the equivalence set of {B} is {A, C}, thus we can add an implied join edge {A <->
-	//! C}
+	//! i.e. in the join A=B AND B=C, the equivalence set of {B} is {A, C}, thus we can add an implied join edge {A = C}
 	expression_map_t<vector<FilterInfo *>> equivalence_sets;
 
 	CardinalityEstimator cardinality_estimator;

package/src/duckdb/src/optimizer/deliminator.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/optimizer/deliminator.hpp"
 
+#include "duckdb/optimizer/join_order/join_order_optimizer.hpp"
 #include "duckdb/planner/expression/bound_cast_expression.hpp"
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
 #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
@@ -13,13 +14,15 @@ namespace duckdb {
 
 class DeliminatorPlanUpdater : LogicalOperatorVisitor {
 public:
-	DeliminatorPlanUpdater() {
+	explicit DeliminatorPlanUpdater(ClientContext &context) : context(context) {
 	}
 	//! Update the plan after a DelimGet has been removed
 	void VisitOperator(LogicalOperator &op) override;
 	void VisitExpression(unique_ptr<Expression> *expression) override;
 
 public:
+	ClientContext &context;
+
 	expression_map_t<Expression *> expr_map;
 	column_binding_map_t<bool> projection_map;
 	column_binding_map_t<Expression *> reverse_proj_or_agg_map;
@@ -94,6 +97,10 @@ void DeliminatorPlanUpdater::VisitOperator(LogicalOperator &op) {
 		// change type if there are no more duplicate-eliminated columns
 		if (decs->empty()) {
 			delim_join.type = LogicalOperatorType::LOGICAL_COMPARISON_JOIN;
+			// sub-plans with DelimGets are not re-orderable (yet), however, we removed all DelimGet of this DelimJoin
+			// the DelimGets are on the RHS of the DelimJoin, so we can call the JoinOrderOptimizer on the RHS now
+			JoinOrderOptimizer optimizer(context);
+			delim_join.children[1] = optimizer.Optimize(std::move(delim_join.children[1]));
 		}
 	}
 }
@@ -111,7 +118,7 @@ unique_ptr<LogicalOperator> Deliminator::Optimize(unique_ptr<LogicalOperator> op
 	FindCandidates(&op, candidates);
 
 	for (auto &candidate : candidates) {
-		DeliminatorPlanUpdater updater;
+		DeliminatorPlanUpdater updater(context);
 		if (RemoveCandidate(&op, candidate, updater)) {
 			updater.VisitOperator(*op);
 		}

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -358,7 +358,7 @@ bool SortTdoms(const RelationsToTDom &a, const RelationsToTDom &b) {
 	return a.tdom_no_hll > b.tdom_no_hll;
 }
 
-void CardinalityEstimator::InitCardinalityEstimatorProps(vector<struct NodeOp> *node_ops,
+void CardinalityEstimator::InitCardinalityEstimatorProps(vector<NodeOp> *node_ops,
                                                          vector<unique_ptr<FilterInfo>> *filter_infos) {
 	InitEquivalentRelations(filter_infos);
 	InitTotalDomains();
@@ -564,13 +564,13 @@ void CardinalityEstimator::EstimateBaseTableCardinality(JoinNode *node, LogicalO
 	auto table_filters = GetTableFilters(op);
 
 	auto card_after_filters = node->GetBaseTableCardinality();
-	// Logical Filter on a seq scan
-	if (has_logical_filter) {
-		card_after_filters *= DEFAULT_SELECTIVITY;
-	} else if (table_filters) {
+	if (table_filters) {
 		double inspect_result = (double)InspectTableFilters(card_after_filters, op, table_filters);
 		card_after_filters = MinValue(inspect_result, (double)card_after_filters);
 	}
+	if (has_logical_filter) {
+		card_after_filters *= DEFAULT_SELECTIVITY;
+	}
 	node->SetEstimatedCardinality(card_after_filters);
 }
 

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -182,13 +182,16 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 		relations.push_back(std::move(relation));
 		return true;
 	}
-	if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN ||
-	    op->type == LogicalOperatorType::LOGICAL_CROSS_PRODUCT) {
+
+	switch (op->type) {
+	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
+	case LogicalOperatorType::LOGICAL_CROSS_PRODUCT: {
 		// inner join or cross product
 		bool can_reorder_left = ExtractJoinRelations(*op->children[0], filter_operators, op);
 		bool can_reorder_right = ExtractJoinRelations(*op->children[1], filter_operators, op);
 		return can_reorder_left && can_reorder_right;
-	} else if (op->type == LogicalOperatorType::LOGICAL_GET) {
+	}
+	case LogicalOperatorType::LOGICAL_GET: {
 		// base table scan, add to set of relations
 		auto get = (LogicalGet *)op;
 		auto relation = make_unique<SingleJoinRelation>(&input_op, parent);
@@ -199,24 +202,25 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 		cardinality_estimator.AddRelationColumnMapping(get, relation_id);
 		relations.push_back(std::move(relation));
 		return true;
-	} else if (op->type == LogicalOperatorType::LOGICAL_EXPRESSION_GET) {
+	}
+	case LogicalOperatorType::LOGICAL_EXPRESSION_GET: {
 		// base table scan, add to set of relations
 		auto get = (LogicalExpressionGet *)op;
 		auto relation = make_unique<SingleJoinRelation>(&input_op, parent);
-		idx_t relation_id = relations.size();
 		//! make sure the optimizer has knowledge of the exact column bindings as well.
-		auto table_index = get->table_index;
-		relation_mapping[table_index] = relation_id;
+		relation_mapping[get->table_index] = relations.size();
 		relations.push_back(std::move(relation));
 		return true;
-	} else if (op->type == LogicalOperatorType::LOGICAL_DUMMY_SCAN) {
+	}
+	case LogicalOperatorType::LOGICAL_DUMMY_SCAN: {
 		// table function call, add to set of relations
 		auto dummy_scan = (LogicalDummyScan *)op;
 		auto relation = make_unique<SingleJoinRelation>(&input_op, parent);
 		relation_mapping[dummy_scan->table_index] = relations.size();
 		relations.push_back(std::move(relation));
 		return true;
-	} else if (op->type == LogicalOperatorType::LOGICAL_PROJECTION) {
+	}
+	case LogicalOperatorType::LOGICAL_PROJECTION: {
 		auto proj = (LogicalProjection *)op;
 		// we run the join order optimizer witin the subquery as well
 		JoinOrderOptimizer optimizer(context);
@@ -227,7 +231,9 @@ bool JoinOrderOptimizer::ExtractJoinRelations(LogicalOperator &input_op, vector<
 		relations.push_back(std::move(relation));
 		return true;
 	}
-	return false;
+	default:
+		return false;
+	}
 }
 
 //! Update the exclusion set with all entries in the subgraph
@@ -768,9 +774,19 @@ JoinOrderOptimizer::GenerateJoins(vector<unique_ptr<LogicalOperator>> &extracted
 		result_relation = node->set;
 		result_operator = std::move(extracted_relations[node->set->relations[0]]);
 	}
-	result_operator->estimated_cardinality = node->GetCardinality<idx_t>();
-	result_operator->has_estimated_cardinality = true;
 	result_operator->estimated_props = node->estimated_props->Copy();
+	result_operator->estimated_cardinality = result_operator->estimated_props->GetCardinality<idx_t>();
+	result_operator->has_estimated_cardinality = true;
+	if (result_operator->type == LogicalOperatorType::LOGICAL_FILTER &&
+	    result_operator->children[0]->type == LogicalOperatorType::LOGICAL_GET) {
+		// FILTER on top of GET, add estimated properties to both
+		auto &filter_props = *result_operator->estimated_props;
+		auto &child_operator = *result_operator->children[0];
+		child_operator.estimated_props = make_unique<EstimatedProperties>(
+		    filter_props.GetCardinality<double>() / CardinalityEstimator::DEFAULT_SELECTIVITY, filter_props.GetCost());
+		child_operator.estimated_cardinality = child_operator.estimated_props->GetCardinality<idx_t>();
+		child_operator.has_estimated_cardinality = true;
+	}
 	// check if we should do a pushdown on this node
 	// basically, any remaining filter that is a subset of the current relation will no longer be used in joins
 	// hence we should push it here
@@ -911,9 +927,9 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 	// now that we know we are going to perform join ordering we actually extract the filters, eliminating duplicate
 	// filters in the process
 	expression_set_t filter_set;
-	for (auto &op : filter_operators) {
-		if (op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
-			auto &join = (LogicalComparisonJoin &)*op;
+	for (auto &f_op : filter_operators) {
+		if (f_op->type == LogicalOperatorType::LOGICAL_COMPARISON_JOIN) {
+			auto &join = (LogicalComparisonJoin &)*f_op;
 			D_ASSERT(join.join_type == JoinType::INNER);
 			D_ASSERT(join.expressions.empty());
 			for (auto &cond : join.conditions) {
@@ -926,13 +942,13 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 			}
 			join.conditions.clear();
 		} else {
-			for (auto &expression : op->expressions) {
+			for (auto &expression : f_op->expressions) {
 				if (filter_set.find(expression.get()) == filter_set.end()) {
 					filter_set.insert(expression.get());
 					filters.push_back(std::move(expression));
 				}
 			}
-			op->expressions.clear();
+			f_op->expressions.clear();
 		}
 	}
 	// create potential edges from the comparisons

package/src/duckdb/src/optimizer/optimizer.cpp

@@ -107,7 +107,7 @@ unique_ptr<LogicalOperator> Optimizer::Optimize(unique_ptr<LogicalOperator> plan
 
 	// removes any redundant DelimGets/DelimJoins
 	RunOptimizer(OptimizerType::DELIMINATOR, [&]() {
-		Deliminator deliminator;
+		Deliminator deliminator(context);
 		plan = deliminator.Optimize(std::move(plan));
 	});