duckdb 0.8.2-dev2700.0 → 0.8.2-dev2809.0

package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp

@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_order/relation_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/optimizer/join_order/join_relation.hpp"
+#include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
+#include "duckdb/optimizer/join_order/relation_statistics_helper.hpp"
+#include "duckdb/optimizer/join_order/join_node.hpp"
+#include "duckdb/parser/expression_map.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/logical_operator_visitor.hpp"
+
+namespace duckdb {
+
+struct FilterInfo;
+
+//! Represents a single relation and any metadata accompanying that relation
+struct SingleJoinRelation {
+	LogicalOperator &op;
+	optional_ptr<LogicalOperator> parent;
+	RelationStats stats;
+
+	SingleJoinRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent) : op(op), parent(parent) {
+	}
+	SingleJoinRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent, RelationStats stats)
+	    : op(op), parent(parent), stats(stats) {
+	}
+};
+
+class RelationManager {
+public:
+	explicit RelationManager(ClientContext &context) : context(context) {
+	}
+
+	idx_t NumRelations();
+
+	bool ExtractJoinRelations(LogicalOperator &input_op, vector<reference<LogicalOperator>> &filter_operators,
+	                          optional_ptr<LogicalOperator> parent = nullptr);
+
+	//! for each join filter in the logical plan op, extract the relations that are referred to on
+	//! both sides of the join filter, along with the tables & indexes.
+	vector<unique_ptr<FilterInfo>> ExtractEdges(LogicalOperator &op,
+	                                            vector<reference<LogicalOperator>> &filter_operators,
+	                                            JoinRelationSetManager &set_manager);
+
+	//! Extract the set of relations referred to inside an expression
+	bool ExtractBindings(Expression &expression, unordered_set<idx_t> &bindings);
+	void AddRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent, const RelationStats &stats);
+
+	void AddAggregateRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent, const RelationStats &stats);
+	vector<unique_ptr<SingleJoinRelation>> GetRelations();
+
+	const vector<RelationStats> GetRelationStats();
+	//! A mapping of base table index -> index into relations array (relation number)
+	unordered_map<idx_t, idx_t> relation_mapping;
+
+	void PrintRelationStats();
+
+private:
+	ClientContext &context;
+	//! Set of all relations considered in the join optimizer
+	vector<unique_ptr<SingleJoinRelation>> relations;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp

@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_order/statistics_extractor.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+#pragma once
+
+#include "duckdb/planner/filter/conjunction_filter.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+
+namespace duckdb {
+
+class CardinalityEstimator;
+
+struct DistinctCount {
+	idx_t distinct_count;
+	bool from_hll;
+};
+
+struct ExpressionBinding {
+	bool found_expression = false;
+	ColumnBinding child_binding;
+	bool expression_is_constant = false;
+};
+
+struct RelationStats {
+	// column_id -> estimated distinct count for column
+	vector<DistinctCount> column_distinct_count;
+	idx_t cardinality;
+	double filter_strength = 1;
+	bool stats_initialized = false;
+
+	// for debug, column names and tables
+	vector<string> column_names;
+	string table_name;
+
+	RelationStats() : cardinality(1), filter_strength(1), stats_initialized(false) {
+	}
+};
+
+class RelationStatisticsHelper {
+public:
+	static constexpr double DEFAULT_SELECTIVITY = 0.2;
+
+public:
+	static idx_t InspectConjunctionAND(idx_t cardinality, idx_t column_index, ConjunctionAndFilter &filter,
+	                                   BaseStatistics &base_stats);
+	//	static idx_t InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter &filter,
+	//	                                  BaseStatistics &base_stats);
+	//! Extract Statistics from a LogicalGet.
+	static RelationStats ExtractGetStats(LogicalGet &get, ClientContext &context);
+	static RelationStats ExtractDelimGetStats(LogicalDelimGet &delim_get, ClientContext &context);
+	//! Create the statistics for a projection using the statistics of the operator that sits underneath the
+	//! projection. Then also create statistics for any extra columns the projection creates.
+	static RelationStats ExtractDummyScanStats(LogicalDummyScan &dummy_scan, ClientContext &context);
+	static RelationStats ExtractExpressionGetStats(LogicalExpressionGet &expression_get, ClientContext &context);
+	//! All relation extractors for blocking relations
+	static RelationStats ExtractProjectionStats(LogicalProjection &proj, RelationStats &child_stats);
+	static RelationStats ExtractAggregationStats(LogicalAggregate &aggr, RelationStats &child_stats);
+	static RelationStats ExtractWindowStats(LogicalWindow &window, RelationStats &child_stats);
+	//! Called after reordering a query plan with potentially 2+ relations.
+	static RelationStats CombineStatsOfReorderableOperator(vector<ColumnBinding> &bindings,
+	                                                       vector<RelationStats> relation_stats);
+	//! Called after reordering a query plan with potentially 2+ relations.
+	static RelationStats CombineStatsOfNonReorderableOperator(LogicalOperator &op, vector<RelationStats> child_stats);
+	static void CopyRelationStats(RelationStats &to, const RelationStats &from);
+
+private:
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -39,7 +39,7 @@ class TaskScheduler {
 	constexpr static int64_t TASK_TIMEOUT_USECS = 5000;
 
 public:
-	TaskScheduler(DatabaseInstance &db);
+	explicit TaskScheduler(DatabaseInstance &db);
 	~TaskScheduler();
 
 	DUCKDB_API static TaskScheduler &GetScheduler(ClientContext &context);
@@ -67,6 +67,9 @@ public:
 	//! Send signals to n threads, signalling for them to wake up and attempt to execute a task
 	void Signal(idx_t n);
 
+	//! Yield to other threads
+	void YieldThread();
+
 	//! Set the allocator flush threshold
 	void SetAllocatorFlushTreshold(idx_t threshold);
 

package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp

@@ -49,8 +49,6 @@ public:
 	idx_t estimated_cardinality;
 	bool has_estimated_cardinality;
 
-	unique_ptr<EstimatedProperties> estimated_props;
-
 public:
 	virtual vector<ColumnBinding> GetColumnBindings();
 	static vector<ColumnBinding> GenerateColumnBindings(idx_t table_idx, idx_t column_count);

package/src/duckdb/src/include/duckdb.h

@@ -308,7 +308,8 @@ typedef enum { DuckDBSuccess = 0, DuckDBError = 1 } duckdb_state;
 typedef enum {
 	DUCKDB_PENDING_RESULT_READY = 0,
 	DUCKDB_PENDING_RESULT_NOT_READY = 1,
-	DUCKDB_PENDING_ERROR = 2
+	DUCKDB_PENDING_ERROR = 2,
+	DUCKDB_PENDING_NO_TASKS_AVAILABLE = 3
 } duckdb_pending_state;
 
 //===--------------------------------------------------------------------===//
@@ -1260,6 +1261,15 @@ Otherwise, all remaining tasks must be executed first.
 */
 DUCKDB_API duckdb_state duckdb_execute_pending(duckdb_pending_result pending_result, duckdb_result *out_result);
 
+/*!
+Returns whether a duckdb_pending_state is finished executing. For example if `pending_state` is
+DUCKDB_PENDING_RESULT_READY, this function will return true.
+
+* pending_state: The pending state on which to decide whether to finish execution.
+* returns: Boolean indicating pending execution should be considered finished.
+*/
+DUCKDB_API bool duckdb_pending_execution_is_finished(duckdb_pending_state pending_state);
+
 //===--------------------------------------------------------------------===//
 // Value Interface
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/capi/pending-c.cpp

@@ -92,6 +92,8 @@ duckdb_pending_state duckdb_pending_execute_task(duckdb_pending_result pending_r
 	switch (return_value) {
 	case PendingExecutionResult::RESULT_READY:
 		return DUCKDB_PENDING_RESULT_READY;
+	case PendingExecutionResult::NO_TASKS_AVAILABLE:
+		return DUCKDB_PENDING_NO_TASKS_AVAILABLE;
 	case PendingExecutionResult::RESULT_NOT_READY:
 		return DUCKDB_PENDING_RESULT_NOT_READY;
 	default:
@@ -99,6 +101,21 @@ duckdb_pending_state duckdb_pending_execute_task(duckdb_pending_result pending_r
 	}
 }
 
+bool duckdb_pending_execution_is_finished(duckdb_pending_state pending_state) {
+	switch (pending_state) {
+	case DUCKDB_PENDING_RESULT_READY:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::RESULT_READY);
+	case DUCKDB_PENDING_NO_TASKS_AVAILABLE:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::NO_TASKS_AVAILABLE);
+	case DUCKDB_PENDING_RESULT_NOT_READY:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::RESULT_NOT_READY);
+	case DUCKDB_PENDING_ERROR:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::EXECUTION_ERROR);
+	default:
+		return PendingQueryResult::IsFinished(PendingExecutionResult::EXECUTION_ERROR);
+	}
+}
+
 duckdb_state duckdb_execute_pending(duckdb_pending_result pending_result, duckdb_result *out_result) {
 	if (!pending_result || !out_result) {
 		return DuckDBError;

package/src/duckdb/src/main/pending_query_result.cpp

@@ -55,7 +55,8 @@ PendingExecutionResult PendingQueryResult::ExecuteTaskInternal(ClientContextLock
 
 unique_ptr<QueryResult> PendingQueryResult::ExecuteInternal(ClientContextLock &lock) {
 	CheckExecutableInternal(lock);
-	while (ExecuteTaskInternal(lock) == PendingExecutionResult::RESULT_NOT_READY) {
+	// Busy wait while execution is not finished
+	while (!IsFinished(ExecuteTaskInternal(lock))) {
 	}
 	if (HasError()) {
 		return make_uniq<MaterializedQueryResult>(error);
@@ -74,4 +75,11 @@ void PendingQueryResult::Close() {
 	context.reset();
 }
 
+bool PendingQueryResult::IsFinished(PendingExecutionResult result) {
+	if (result == PendingExecutionResult::RESULT_READY || result == PendingExecutionResult::EXECUTION_ERROR) {
+		return true;
+	}
+	return false;
+}
+
 } // namespace duckdb