duckdb 0.10.3-dev0.0 → 0.10.3-dev13.0

package/src/duckdb/src/main/settings/settings.cpp

@@ -437,6 +437,38 @@ Value EnableExternalAccessSetting::GetSetting(const ClientContext &context) {
 	return Value::BOOLEAN(config.options.enable_external_access);
 }
 
+//===--------------------------------------------------------------------===//
+// Enable Macro Dependencies
+//===--------------------------------------------------------------------===//
+void EnableMacrosDependencies::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.options.enable_macro_dependencies = input.GetValue<bool>();
+}
+
+void EnableMacrosDependencies::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.enable_macro_dependencies = DBConfig().options.enable_macro_dependencies;
+}
+
+Value EnableMacrosDependencies::GetSetting(const ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.enable_macro_dependencies);
+}
+
+//===--------------------------------------------------------------------===//
+// Enable View Dependencies
+//===--------------------------------------------------------------------===//
+void EnableViewDependencies::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.options.enable_view_dependencies = input.GetValue<bool>();
+}
+
+void EnableViewDependencies::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.enable_view_dependencies = DBConfig().options.enable_view_dependencies;
+}
+
+Value EnableViewDependencies::GetSetting(const ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.enable_view_dependencies);
+}
+
 //===--------------------------------------------------------------------===//
 // Enable FSST Vectors
 //===--------------------------------------------------------------------===//
@@ -476,6 +508,36 @@ Value AllowUnsignedExtensionsSetting::GetSetting(const ClientContext &context) {
 	return Value::BOOLEAN(config.options.allow_unsigned_extensions);
 }
 
+//===--------------------------------------------------------------------===//
+// Allow Community Extensions
+//===--------------------------------------------------------------------===//
+void AllowCommunityExtensionsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	if (db && !config.options.allow_community_extensions) {
+		auto new_value = input.GetValue<bool>();
+		if (new_value) {
+			throw InvalidInputException("Cannot upgrade allow_community_extensions setting while database is running");
+		}
+		return;
+	}
+	auto new_value = input.GetValue<bool>();
+	config.options.allow_community_extensions = new_value;
+}
+
+void AllowCommunityExtensionsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	if (db && !config.options.allow_community_extensions) {
+		if (DBConfig().options.allow_community_extensions) {
+			throw InvalidInputException("Cannot upgrade allow_community_extensions setting while database is running");
+		}
+		return;
+	}
+	config.options.allow_community_extensions = DBConfig().options.allow_community_extensions;
+}
+
+Value AllowCommunityExtensionsSetting::GetSetting(const ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BOOLEAN(config.options.allow_community_extensions);
+}
+
 //===--------------------------------------------------------------------===//
 // Allow Extensions Metadata Mismatch
 //===--------------------------------------------------------------------===//
@@ -532,6 +594,26 @@ Value EnableObjectCacheSetting::GetSetting(const ClientContext &context) {
 	return Value::BOOLEAN(config.options.object_cache_enable);
 }
 
+//===--------------------------------------------------------------------===//
+// Storage Compatibility Version (for serialization)
+//===--------------------------------------------------------------------===//
+void StorageCompatibilityVersion::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	auto version_string = input.GetValue<string>();
+	auto serialization_compatibility = SerializationCompatibility::FromString(version_string);
+	config.options.serialization_compatibility = serialization_compatibility;
+}
+
+void StorageCompatibilityVersion::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.serialization_compatibility = DBConfig().options.serialization_compatibility;
+}
+
+Value StorageCompatibilityVersion::GetSetting(const ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+
+	auto &version_name = config.options.serialization_compatibility.duckdb_version;
+	return Value(version_name);
+}
+
 //===--------------------------------------------------------------------===//
 // Enable HTTP Metadata Cache
 //===--------------------------------------------------------------------===//
@@ -766,7 +848,7 @@ void ExternalThreadsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, c
 	if (new_val < 0) {
 		throw SyntaxException("Must have a non-negative number of external threads!");
 	}
-	idx_t new_external_threads = new_val;
+	auto new_external_threads = NumericCast<idx_t>(new_val);
 	if (db) {
 		TaskScheduler::GetScheduler(*db).SetThreads(config.options.maximum_threads, new_external_threads);
 	}
@@ -783,7 +865,7 @@ void ExternalThreadsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config)
 
 Value ExternalThreadsSetting::GetSetting(const ClientContext &context) {
 	auto &config = DBConfig::GetConfig(context);
-	return Value::BIGINT(config.options.external_threads);
+	return Value::BIGINT(NumericCast<int64_t>(config.options.external_threads));
 }
 
 //===--------------------------------------------------------------------===//
@@ -975,7 +1057,7 @@ Value MaximumExpressionDepthSetting::GetSetting(const ClientContext &context) {
 void MaximumMemorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
 	config.options.maximum_memory = DBConfig::ParseMemoryLimit(input.ToString());
 	if (db) {
-		BufferManager::GetBufferManager(*db).SetLimit(config.options.maximum_memory);
+		BufferManager::GetBufferManager(*db).SetMemoryLimit(config.options.maximum_memory);
 	}
 }
 
@@ -988,6 +1070,51 @@ Value MaximumMemorySetting::GetSetting(const ClientContext &context) {
 	return Value(StringUtil::BytesToHumanReadableString(config.options.maximum_memory));
 }
 
+//===--------------------------------------------------------------------===//
+// Maximum Temp Directory Size
+//===--------------------------------------------------------------------===//
+void MaximumTempDirectorySize::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	auto maximum_swap_space = DBConfig::ParseMemoryLimit(input.ToString());
+	if (maximum_swap_space == DConstants::INVALID_INDEX) {
+		// We use INVALID_INDEX to indicate that the value is not set by the user
+		// use one lower to indicate 'unlimited'
+		maximum_swap_space--;
+	}
+	if (!db) {
+		config.options.maximum_swap_space = maximum_swap_space;
+		return;
+	}
+	auto &buffer_manager = BufferManager::GetBufferManager(*db);
+	buffer_manager.SetSwapLimit(maximum_swap_space);
+	config.options.maximum_swap_space = maximum_swap_space;
+}
+
+void MaximumTempDirectorySize::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.maximum_swap_space = DConstants::INVALID_INDEX;
+	if (!db) {
+		return;
+	}
+	auto &buffer_manager = BufferManager::GetBufferManager(*db);
+	buffer_manager.SetSwapLimit();
+}
+
+Value MaximumTempDirectorySize::GetSetting(const ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	if (config.options.maximum_swap_space != DConstants::INVALID_INDEX) {
+		// Explicitly set by the user
+		return Value(StringUtil::BytesToHumanReadableString(config.options.maximum_swap_space));
+	}
+	auto &buffer_manager = BufferManager::GetBufferManager(context);
+	// Database is initialized, use the setting from the temporary directory
+	auto max_swap = buffer_manager.GetMaxSwap();
+	if (max_swap.IsValid()) {
+		return Value(StringUtil::BytesToHumanReadableString(max_swap.GetIndex()));
+	} else {
+		// The temp directory has not been used yet
+		return Value(StringUtil::BytesToHumanReadableString(0));
+	}
+}
+
 //===--------------------------------------------------------------------===//
 // Old Implicit Casting
 //===--------------------------------------------------------------------===//
@@ -1017,7 +1144,7 @@ void PartitionedWriteFlushThreshold::SetLocal(ClientContext &context, const Valu
 }
 
 Value PartitionedWriteFlushThreshold::GetSetting(const ClientContext &context) {
-	return Value::BIGINT(ClientConfig::GetConfig(context).partitioned_write_flush_threshold);
+	return Value::BIGINT(NumericCast<int64_t>(ClientConfig::GetConfig(context).partitioned_write_flush_threshold));
 }
 
 //===--------------------------------------------------------------------===//
@@ -1047,11 +1174,11 @@ void PerfectHashThresholdSetting::SetLocal(ClientContext &context, const Value &
 	if (bits < 0 || bits > 32) {
 		throw ParserException("Perfect HT threshold out of range: should be within range 0 - 32");
 	}
-	ClientConfig::GetConfig(context).perfect_ht_threshold = bits;
+	ClientConfig::GetConfig(context).perfect_ht_threshold = NumericCast<idx_t>(bits);
 }
 
 Value PerfectHashThresholdSetting::GetSetting(const ClientContext &context) {
-	return Value::BIGINT(ClientConfig::GetConfig(context).perfect_ht_threshold);
+	return Value::BIGINT(NumericCast<int64_t>(ClientConfig::GetConfig(context).perfect_ht_threshold));
 }
 
 //===--------------------------------------------------------------------===//
@@ -1066,7 +1193,7 @@ void PivotFilterThreshold::SetLocal(ClientContext &context, const Value &input)
 }
 
 Value PivotFilterThreshold::GetSetting(const ClientContext &context) {
-	return Value::BIGINT(ClientConfig::GetConfig(context).pivot_filter_threshold);
+	return Value::BIGINT(NumericCast<int64_t>(ClientConfig::GetConfig(context).pivot_filter_threshold));
 }
 
 //===--------------------------------------------------------------------===//
@@ -1081,7 +1208,7 @@ void PivotLimitSetting::SetLocal(ClientContext &context, const Value &input) {
 }
 
 Value PivotLimitSetting::GetSetting(const ClientContext &context) {
-	return Value::BIGINT(ClientConfig::GetConfig(context).pivot_limit);
+	return Value::BIGINT(NumericCast<int64_t>(ClientConfig::GetConfig(context).pivot_limit));
 }
 
 //===--------------------------------------------------------------------===//
@@ -1276,7 +1403,8 @@ void TempDirectorySetting::SetGlobal(DatabaseInstance *db, DBConfig &config, con
 }
 
 void TempDirectorySetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
-	config.options.temporary_directory = DBConfig().options.temporary_directory;
+	config.SetDefaultTempDirectory();
+
 	config.options.use_temporary_directory = DBConfig().options.use_temporary_directory;
 	if (db) {
 		auto &buffer_manager = BufferManager::GetBufferManager(*db);
@@ -1297,7 +1425,7 @@ void ThreadsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Val
 	if (new_val < 1) {
 		throw SyntaxException("Must have at least 1 thread!");
 	}
-	idx_t new_maximum_threads = new_val;
+	auto new_maximum_threads = NumericCast<idx_t>(new_val);
 	if (db) {
 		TaskScheduler::GetScheduler(*db).SetThreads(new_maximum_threads, config.options.external_threads);
 	}
@@ -1314,7 +1442,7 @@ void ThreadsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
 
 Value ThreadsSetting::GetSetting(const ClientContext &context) {
 	auto &config = DBConfig::GetConfig(context);
-	return Value::BIGINT(config.options.maximum_threads);
+	return Value::BIGINT(NumericCast<int64_t>(config.options.maximum_threads));
 }
 
 //===--------------------------------------------------------------------===//
@@ -1403,4 +1531,34 @@ Value CustomUserAgentSetting::GetSetting(const ClientContext &context) {
 	return Value(config.options.custom_user_agent);
 }
 
+//===--------------------------------------------------------------------===//
+// EnableHTTPLogging Setting
+//===--------------------------------------------------------------------===//
+void EnableHTTPLoggingSetting::ResetLocal(ClientContext &context) {
+	ClientConfig::GetConfig(context).enable_http_logging = ClientConfig().enable_http_logging;
+}
+
+void EnableHTTPLoggingSetting::SetLocal(ClientContext &context, const Value &input) {
+	ClientConfig::GetConfig(context).enable_http_logging = input.GetValue<bool>();
+}
+
+Value EnableHTTPLoggingSetting::GetSetting(const ClientContext &context) {
+	return Value(ClientConfig::GetConfig(context).enable_http_logging);
+}
+
+//===--------------------------------------------------------------------===//
+// HTTPLoggingOutput Setting
+//===--------------------------------------------------------------------===//
+void HTTPLoggingOutputSetting::ResetLocal(ClientContext &context) {
+	ClientConfig::GetConfig(context).http_logging_output = ClientConfig().http_logging_output;
+}
+
+void HTTPLoggingOutputSetting::SetLocal(ClientContext &context, const Value &input) {
+	ClientConfig::GetConfig(context).http_logging_output = input.GetValue<string>();
+}
+
+Value HTTPLoggingOutputSetting::GetSetting(const ClientContext &context) {
+	return Value(ClientConfig::GetConfig(context).http_logging_output);
+}
+
 } // namespace duckdb

package/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp

@@ -47,7 +47,7 @@ void CommonAggregateOptimizer::ExtractCommonAggregates(LogicalAggregate &aggr) {
 		} else {
 			// aggregate already exists! we can remove this entry
 			total_erased++;
-			aggr.expressions.erase(aggr.expressions.begin() + i);
+			aggr.expressions.erase_at(i);
 			i--;
 			// we need to remap any references to this aggregate so they point to the other aggregate
 			ColumnBinding original_binding(aggr.aggregate_index, original_index);

package/src/duckdb/src/optimizer/filter_combiner.cpp

@@ -67,7 +67,7 @@ FilterResult FilterCombiner::AddConstantComparison(vector<ExpressionValueInforma
 		switch (comparison) {
 		case ValueComparisonResult::PRUNE_LEFT:
 			// prune the entry from the info list
-			info_list.erase(info_list.begin() + i);
+			info_list.erase_at(i);
 			i--;
 			break;
 		case ValueComparisonResult::PRUNE_RIGHT:
@@ -607,7 +607,7 @@ TableFilterSet FilterCombiner::GenerateTableScanFilters(vector<idx_t> &column_id
 			table_filters.PushFilter(column_index, std::move(upper_bound));
 			table_filters.PushFilter(column_index, make_uniq<IsNotNullFilter>());
 
-			remaining_filters.erase(remaining_filters.begin() + rem_fil_idx);
+			remaining_filters.erase_at(rem_fil_idx);
 		}
 	}
 
@@ -971,7 +971,7 @@ unique_ptr<Expression> FilterCombiner::FindTransitiveFilter(Expression &expr) {
 			auto &comparison = remaining_filters[i]->Cast<BoundComparisonExpression>();
 			if (expr.Equals(*comparison.right) && comparison.type != ExpressionType::COMPARE_NOTEQUAL) {
 				auto filter = std::move(remaining_filters[i]);
-				remaining_filters.erase(remaining_filters.begin() + i);
+				remaining_filters.erase_at(i);
 				return filter;
 			}
 		}

package/src/duckdb/src/optimizer/filter_pushdown.cpp

@@ -9,7 +9,8 @@ namespace duckdb {
 
 using Filter = FilterPushdown::Filter;
 
-FilterPushdown::FilterPushdown(Optimizer &optimizer) : optimizer(optimizer), combiner(optimizer.context) {
+FilterPushdown::FilterPushdown(Optimizer &optimizer, bool convert_mark_joins)
+    : optimizer(optimizer), combiner(optimizer.context), convert_mark_joins(convert_mark_joins) {
 }
 
 unique_ptr<LogicalOperator> FilterPushdown::Rewrite(unique_ptr<LogicalOperator> op) {
@@ -144,7 +145,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalO
 unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
 	// unhandled type, first perform filter pushdown in its children
 	for (auto &child : op->children) {
-		FilterPushdown pushdown(optimizer);
+		FilterPushdown pushdown(optimizer, convert_mark_joins);
 		child = pushdown.Rewrite(std::move(child));
 	}
 	// now push any existing filters

package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp

@@ -50,7 +50,7 @@ bool CardinalityEstimator::SingleRelationFilter(FilterInfo &filter_info) {
 
 vector<idx_t> CardinalityEstimator::DetermineMatchingEquivalentSets(FilterInfo *filter_info) {
 	vector<idx_t> matching_equivalent_sets;
-	auto equivalent_relation_index = 0;
+	idx_t equivalent_relation_index = 0;
 
 	for (const RelationsToTDom &r2tdom : relations_to_tdoms) {
 		auto &i_set = r2tdom.equivalent_relations;
@@ -246,7 +246,7 @@ double CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set
 		denom *= match.denom;
 	}
 	// can happen if a table has cardinality 0, or a tdom is set to 0
-	if (denom == 0) {
+	if (denom <= 1) {
 		denom = 1;
 	}
 	auto result = numerator / denom;
@@ -259,7 +259,7 @@ template <>
 idx_t CardinalityEstimator::EstimateCardinalityWithSet(JoinRelationSet &new_set) {
 	auto cardinality_as_double = EstimateCardinalityWithSet<double>(new_set);
 	auto max = NumericLimits<idx_t>::Maximum();
-	if (cardinality_as_double > max) {
+	if (cardinality_as_double >= max) {
 		return max;
 	}
 	return (idx_t)cardinality_as_double;

package/src/duckdb/src/optimizer/join_order/cost_model.cpp

@@ -8,7 +8,7 @@ CostModel::CostModel(QueryGraphManager &query_graph_manager)
     : query_graph_manager(query_graph_manager), cardinality_estimator() {
 }
 
-double CostModel::ComputeCost(JoinNode &left, JoinNode &right) {
+double CostModel::ComputeCost(DPJoinNode &left, DPJoinNode &right) {
 	auto &combination = query_graph_manager.set_manager.Union(left.set, right.set);
 	auto join_card = cardinality_estimator.EstimateCardinalityWithSet<double>(combination);
 	auto join_cost = join_card;

package/src/duckdb/src/optimizer/join_order/join_node.cpp

@@ -6,35 +6,12 @@
 
 namespace duckdb {
 
-JoinNode::JoinNode(JoinRelationSet &set) : set(set) {
+DPJoinNode::DPJoinNode(JoinRelationSet &set) : set(set), info(nullptr), is_leaf(true), left_set(set), right_set(set) {
 }
 
-JoinNode::JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right, double cost)
-    : set(set), info(info), left(&left), right(&right), cost(cost) {
-}
-
-unique_ptr<EstimatedProperties> EstimatedProperties::Copy() {
-	auto result = make_uniq<EstimatedProperties>(cardinality, cost);
-	return result;
-}
-
-string JoinNode::ToString() {
-	string result = "-------------------------------\n";
-	result += set.ToString() + "\n";
-	result += "cost = " + to_string(cost) + "\n";
-	result += "left = \n";
-	if (left) {
-		result += left->ToString();
-	}
-	result += "right = \n";
-	if (right) {
-		result += right->ToString();
-	}
-	return result;
-}
-
-void JoinNode::Print() {
-	Printer::Print(ToString());
+DPJoinNode::DPJoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinRelationSet &left,
+                       JoinRelationSet &right, double cost)
+    : set(set), info(info), is_leaf(false), left_set(left), right_set(right), cost(cost) {
 }
 
 } // namespace duckdb

package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp

@@ -5,7 +5,6 @@
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/planner/expression/list.hpp"
-#include "duckdb/planner/expression_iterator.hpp"
 #include "duckdb/planner/operator/list.hpp"
 
 namespace duckdb {
@@ -32,7 +31,7 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 	// We optimize the children of any non-reorderable operations we come across.
 	bool reorderable = query_graph_manager.Build(*op);
 
-	// get relation_stats here since the reconstruction process will move all of the relations.
+	// get relation_stats here since the reconstruction process will move all relations.
 	auto relation_stats = query_graph_manager.relation_manager.GetRelationStats();
 	unique_ptr<LogicalOperator> new_logical_plan = nullptr;
 
@@ -46,13 +45,11 @@ unique_ptr<LogicalOperator> JoinOrderOptimizer::Optimize(unique_ptr<LogicalOpera
 
 		// Initialize the leaf/single node plans
 		plan_enumerator.InitLeafPlans();
-
-		// Ask the plan enumerator to enumerate a number of join orders
-		auto final_plan = plan_enumerator.SolveJoinOrder();
-		// TODO: add in the check that if no plan exists, you have to add a cross product.
-
+		plan_enumerator.SolveJoinOrder();
 		// now reconstruct a logical plan from the query graph plan
-		new_logical_plan = query_graph_manager.Reconstruct(std::move(plan), *final_plan);
+		query_graph_manager.plans = &plan_enumerator.GetPlans();
+
+		new_logical_plan = query_graph_manager.Reconstruct(std::move(plan));
 	} else {
 		new_logical_plan = std::move(plan);
 		if (relation_stats.size() == 1) {