duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

package/src/duckdb/src/main/settings/settings.cpp

@@ -2,18 +2,18 @@
 
 #include "duckdb/catalog/catalog_search_path.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/main/attached_database.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/main/config.hpp"
+#include "duckdb/main/database.hpp"
+#include "duckdb/main/database_manager.hpp"
 #include "duckdb/main/query_profiler.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/parser/parser.hpp"
 #include "duckdb/planner/expression_binder.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
 #include "duckdb/storage/storage_manager.hpp"
-#include "duckdb/main/database_manager.hpp"
-#include "duckdb/main/attached_database.hpp"
-#include "duckdb/main/database.hpp"
 
 namespace duckdb {
 
@@ -112,7 +112,6 @@ Value DebugCheckpointAbort::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Debug Force External
 //===--------------------------------------------------------------------===//
-
 void DebugForceExternal::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).force_external = ClientConfig().force_external;
 }
@@ -128,7 +127,6 @@ Value DebugForceExternal::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Debug Force NoCrossProduct
 //===--------------------------------------------------------------------===//
-
 void DebugForceNoCrossProduct::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).force_no_cross_product = ClientConfig().force_no_cross_product;
 }
@@ -144,7 +142,6 @@ Value DebugForceNoCrossProduct::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Ordered Aggregate Threshold
 //===--------------------------------------------------------------------===//
-
 void OrderedAggregateThreshold::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).ordered_aggregate_threshold = ClientConfig().ordered_aggregate_threshold;
 }
@@ -303,6 +300,30 @@ Value DefaultNullOrderSetting::GetSetting(ClientContext &context) {
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Disabled File Systems
+//===--------------------------------------------------------------------===//
+void DisabledFileSystemsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	if (!db) {
+		throw InternalException("disabled_filesystems can only be set in an active database");
+	}
+	auto &fs = FileSystem::GetFileSystem(*db);
+	auto list = StringUtil::Split(input.ToString(), ",");
+	fs.SetDisabledFileSystems(list);
+}
+
+void DisabledFileSystemsSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	if (!db) {
+		throw InternalException("disabled_filesystems can only be set in an active database");
+	}
+	auto &fs = FileSystem::GetFileSystem(*db);
+	fs.SetDisabledFileSystems(vector<string>());
+}
+
+Value DisabledFileSystemsSetting::GetSetting(ClientContext &context) {
+	return Value("");
+}
+
 //===--------------------------------------------------------------------===//
 // Disabled Optimizer
 //===--------------------------------------------------------------------===//
@@ -478,7 +499,6 @@ Value EnableProfilingSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Custom Extension Repository
 //===--------------------------------------------------------------------===//
-
 void CustomExtensionRepository::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).custom_extension_repo = ClientConfig().custom_extension_repo;
 }
@@ -494,7 +514,6 @@ Value CustomExtensionRepository::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Enable Progress Bar
 //===--------------------------------------------------------------------===//
-
 void EnableProgressBarSetting::ResetLocal(ClientContext &context) {
 	auto &config = ClientConfig::GetConfig(context);
 	ProgressBar::SystemOverrideCheck(config);
@@ -548,7 +567,6 @@ Value ExperimentalParallelCSVSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Explain Output
 //===--------------------------------------------------------------------===//
-
 void ExplainOutputSetting::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).explain_output_type = ClientConfig().explain_output_type;
 }
@@ -686,7 +704,6 @@ Value ForceBitpackingModeSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Home Directory
 //===--------------------------------------------------------------------===//
-
 void HomeDirectorySetting::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).home_directory = ClientConfig().home_directory;
 }
@@ -780,7 +797,6 @@ Value ImmediateTransactionModeSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Maximum Expression Depth
 //===--------------------------------------------------------------------===//
-
 void MaximumExpressionDepthSetting::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).max_expression_depth = ClientConfig().max_expression_depth;
 }
@@ -830,7 +846,6 @@ Value PasswordSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Perfect Hash Threshold
 //===--------------------------------------------------------------------===//
-
 void PerfectHashThresholdSetting::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).perfect_ht_threshold = ClientConfig().perfect_ht_threshold;
 }
@@ -848,9 +863,23 @@ Value PerfectHashThresholdSetting::GetSetting(ClientContext &context) {
 }
 
 //===--------------------------------------------------------------------===//
-// Pivot Limit
+// Pivot Filter Threshold
 //===--------------------------------------------------------------------===//
+void PivotFilterThreshold::ResetLocal(ClientContext &context) {
+	ClientConfig::GetConfig(context).pivot_filter_threshold = ClientConfig().pivot_filter_threshold;
+}
+
+void PivotFilterThreshold::SetLocal(ClientContext &context, const Value &input) {
+	ClientConfig::GetConfig(context).pivot_filter_threshold = input.GetValue<uint64_t>();
+}
 
+Value PivotFilterThreshold::GetSetting(ClientContext &context) {
+	return Value::BIGINT(ClientConfig::GetConfig(context).pivot_filter_threshold);
+}
+
+//===--------------------------------------------------------------------===//
+// Pivot Limit
+//===--------------------------------------------------------------------===//
 void PivotLimitSetting::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).pivot_limit = ClientConfig().pivot_limit;
 }
@@ -866,7 +895,6 @@ Value PivotLimitSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // PreserveIdentifierCase
 //===--------------------------------------------------------------------===//
-
 void PreserveIdentifierCase::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).preserve_identifier_case = ClientConfig().preserve_identifier_case;
 }
@@ -917,7 +945,6 @@ Value ExportLargeBufferArrow::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Profiler History Size
 //===--------------------------------------------------------------------===//
-
 void ProfilerHistorySize::ResetLocal(ClientContext &context) {
 	auto &client_data = ClientData::Get(context);
 	client_data.query_profiler_history->ResetProfilerHistorySize();
@@ -939,7 +966,6 @@ Value ProfilerHistorySize::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Profile Output
 //===--------------------------------------------------------------------===//
-
 void ProfileOutputSetting::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).profiler_save_location = ClientConfig().profiler_save_location;
 }
@@ -958,7 +984,6 @@ Value ProfileOutputSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Profiling Mode
 //===--------------------------------------------------------------------===//
-
 void ProfilingModeSetting::ResetLocal(ClientContext &context) {
 	ClientConfig::GetConfig(context).enable_profiler = ClientConfig().enable_profiler;
 	ClientConfig::GetConfig(context).enable_detailed_profiling = ClientConfig().enable_detailed_profiling;
@@ -992,7 +1017,6 @@ Value ProfilingModeSetting::GetSetting(ClientContext &context) {
 //===--------------------------------------------------------------------===//
 // Progress Bar Time
 //===--------------------------------------------------------------------===//
-
 void ProgressBarTimeSetting::ResetLocal(ClientContext &context) {
 	auto &config = ClientConfig::GetConfig(context);
 	ProgressBar::SystemOverrideCheck(config);
@@ -1112,4 +1136,26 @@ Value UsernameSetting::GetSetting(ClientContext &context) {
 	return Value();
 }
 
+//===--------------------------------------------------------------------===//
+// Allocator Flush Threshold
+//===--------------------------------------------------------------------===//
+void FlushAllocatorSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.options.allocator_flush_threshold = DBConfig::ParseMemoryLimit(input.ToString());
+	if (db) {
+		TaskScheduler::GetScheduler(*db).SetAllocatorFlushTreshold(config.options.allocator_flush_threshold);
+	}
+}
+
+void FlushAllocatorSetting::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.allocator_flush_threshold = DBConfig().options.allocator_flush_threshold;
+	if (db) {
+		TaskScheduler::GetScheduler(*db).SetAllocatorFlushTreshold(config.options.allocator_flush_threshold);
+	}
+}
+
+Value FlushAllocatorSetting::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value(StringUtil::BytesToHumanReadableString(config.options.allocator_flush_threshold));
+}
+
 } // namespace duckdb

package/src/duckdb/src/optimizer/column_binding_replacer.cpp

@@ -0,0 +1,43 @@
+#include "duckdb/optimizer/column_binding_replacer.hpp"
+
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+
+namespace duckdb {
+
+ReplacementBinding::ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding)
+    : old_binding(old_binding), new_binding(new_binding), replace_type(false) {
+}
+
+ReplacementBinding::ReplacementBinding(ColumnBinding old_binding, ColumnBinding new_binding, LogicalType new_type)
+    : old_binding(old_binding), new_binding(new_binding), replace_type(true), new_type(std::move(new_type)) {
+}
+
+ColumnBindingReplacer::ColumnBindingReplacer() {
+}
+
+void ColumnBindingReplacer::VisitOperator(LogicalOperator &op) {
+	if (stop_operator && stop_operator.get() == &op) {
+		return;
+	}
+	VisitOperatorChildren(op);
+	VisitOperatorExpressions(op);
+}
+
+void ColumnBindingReplacer::VisitExpression(unique_ptr<Expression> *expression) {
+	auto &expr = *expression;
+	if (expr->expression_class == ExpressionClass::BOUND_COLUMN_REF) {
+		auto &bound_column_ref = expr->Cast<BoundColumnRefExpression>();
+		for (const auto &replace_binding : replacement_bindings) {
+			if (bound_column_ref.binding == replace_binding.old_binding) {
+				bound_column_ref.binding = replace_binding.new_binding;
+				if (replace_binding.replace_type) {
+					bound_column_ref.return_type = replace_binding.new_type;
+				}
+			}
+		}
+	}
+
+	VisitExpressionChildren(**expression);
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp

@@ -1,7 +1,5 @@
 #include "duckdb/optimizer/column_lifetime_optimizer.hpp"
-
 #include "duckdb/planner/expression/bound_columnref_expression.hpp"
-
 #include "duckdb/planner/operator/logical_comparison_join.hpp"
 #include "duckdb/planner/operator/logical_delim_join.hpp"
 #include "duckdb/planner/operator/logical_filter.hpp"
@@ -20,6 +18,7 @@ void ColumnLifetimeAnalyzer::ExtractUnusedColumnBindings(vector<ColumnBinding> b
 void ColumnLifetimeAnalyzer::GenerateProjectionMap(vector<ColumnBinding> bindings,
                                                    column_binding_set_t &unused_bindings,
                                                    vector<idx_t> &projection_map) {
+	projection_map.clear();
 	if (unused_bindings.empty()) {
 		return;
 	}

package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp

@@ -0,0 +1,140 @@
+#include "duckdb/optimizer/compressed_materialization.hpp"
+#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/operator/logical_aggregate.hpp"
+
+namespace duckdb {
+
+void CompressedMaterialization::CompressAggregate(unique_ptr<LogicalOperator> &op) {
+	auto &aggregate = op->Cast<LogicalAggregate>();
+	auto &groups = aggregate.groups;
+	column_binding_set_t group_binding_set;
+	for (const auto &group : groups) {
+		if (group->type != ExpressionType::BOUND_COLUMN_REF) {
+			continue;
+		}
+		auto &colref = group->Cast<BoundColumnRefExpression>();
+		if (group_binding_set.find(colref.binding) != group_binding_set.end()) {
+			return; // Duplicate group - don't compress
+		}
+		group_binding_set.insert(colref.binding);
+	}
+	auto &group_stats = aggregate.group_stats;
+
+	// No need to compress if there are no groups/stats
+	if (groups.empty() || group_stats.empty()) {
+		return;
+	}
+	D_ASSERT(groups.size() == group_stats.size());
+
+	// Find all bindings referenced by non-colref expressions in the groups
+	// These are excluded from compression by projection
+	// But we can try to compress the expression directly
+	column_binding_set_t referenced_bindings;
+	vector<ColumnBinding> group_bindings(groups.size(), ColumnBinding());
+	vector<bool> needs_decompression(groups.size(), false);
+	vector<unique_ptr<BaseStatistics>> stored_group_stats;
+	stored_group_stats.resize(groups.size());
+	for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
+		auto &group_expr = *groups[group_idx];
+		if (group_expr.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
+			auto &colref = group_expr.Cast<BoundColumnRefExpression>();
+			group_bindings[group_idx] = colref.binding;
+			continue; // Will be compressed generically
+		}
+
+		// Mark the bindings referenced by the non-colref expression so they won't be modified
+		GetReferencedBindings(group_expr, referenced_bindings);
+
+		// The non-colref expression won't be compressed generically, so try to compress it here
+		if (!group_stats[group_idx]) {
+			continue; // Can't compress without stats
+		}
+
+		// Try to compress, if successful, replace the expression
+		auto compress_expr = GetCompressExpression(group_expr.Copy(), *group_stats[group_idx]);
+		if (compress_expr) {
+			needs_decompression[group_idx] = true;
+			stored_group_stats[group_idx] = std::move(group_stats[group_idx]);
+			groups[group_idx] = std::move(compress_expr->expression);
+			group_stats[group_idx] = std::move(compress_expr->stats);
+		}
+	}
+
+	// Anything referenced in the aggregate functions is also excluded
+	for (idx_t expr_idx = 0; expr_idx < aggregate.expressions.size(); expr_idx++) {
+		const auto &expr = *aggregate.expressions[expr_idx];
+		D_ASSERT(expr.type == ExpressionType::BOUND_AGGREGATE);
+		const auto &aggr_expr = expr.Cast<BoundAggregateExpression>();
+		for (const auto &child : aggr_expr.children) {
+			GetReferencedBindings(*child, referenced_bindings);
+		}
+		if (aggr_expr.filter) {
+			GetReferencedBindings(*aggr_expr.filter, referenced_bindings);
+		}
+		if (aggr_expr.order_bys) {
+			for (const auto &order : aggr_expr.order_bys->orders) {
+				const auto &order_expr = *order.expression;
+				if (order_expr.type != ExpressionType::BOUND_COLUMN_REF) {
+					GetReferencedBindings(order_expr, referenced_bindings);
+				}
+			}
+		}
+	}
+
+	// Create info for compression
+	CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
+
+	// Create binding mapping
+	const auto bindings_out = aggregate.GetColumnBindings();
+	const auto &types = aggregate.types;
+	for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
+		// Aggregate changes bindings as it has a table idx
+		CMBindingInfo binding_info(bindings_out[group_idx], types[group_idx]);
+		binding_info.needs_decompression = needs_decompression[group_idx];
+		if (needs_decompression[group_idx]) {
+			// Compressed non-generically
+			auto entry = info.binding_map.emplace(bindings_out[group_idx], std::move(binding_info));
+			entry.first->second.stats = std::move(stored_group_stats[group_idx]);
+		} else if (group_bindings[group_idx] != ColumnBinding()) {
+			info.binding_map.emplace(group_bindings[group_idx], std::move(binding_info));
+		}
+	}
+
+	// Now try to compress
+	CreateProjections(op, info);
+
+	// Update aggregate statistics
+	UpdateAggregateStats(op);
+}
+
+void CompressedMaterialization::UpdateAggregateStats(unique_ptr<LogicalOperator> &op) {
+	if (op->type != LogicalOperatorType::LOGICAL_PROJECTION) {
+		return;
+	}
+
+	// Update aggregate group stats if compressed
+	auto &compressed_aggregate = op->children[0]->Cast<LogicalAggregate>();
+	auto &groups = compressed_aggregate.groups;
+	auto &group_stats = compressed_aggregate.group_stats;
+
+	for (idx_t group_idx = 0; group_idx < groups.size(); group_idx++) {
+		auto &group_expr = *groups[group_idx];
+		if (group_expr.GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
+			continue;
+		}
+		auto &colref = group_expr.Cast<BoundColumnRefExpression>();
+		if (!group_stats[group_idx]) {
+			continue;
+		}
+		if (colref.return_type == group_stats[group_idx]->GetType()) {
+			continue;
+		}
+		auto it = statistics_map.find(colref.binding);
+		if (it != statistics_map.end() && it->second) {
+			group_stats[group_idx] = it->second->ToUnique();
+		}
+	}
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp

@@ -0,0 +1,42 @@
+#include "duckdb/optimizer/compressed_materialization.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/operator/logical_distinct.hpp"
+
+namespace duckdb {
+
+void CompressedMaterialization::CompressDistinct(unique_ptr<LogicalOperator> &op) {
+	auto &distinct = op->Cast<LogicalDistinct>();
+	auto &distinct_targets = distinct.distinct_targets;
+
+	column_binding_set_t referenced_bindings;
+	for (auto &target : distinct_targets) {
+		if (target->type != ExpressionType::BOUND_COLUMN_REF) { // LCOV_EXCL_START
+			GetReferencedBindings(*target, referenced_bindings);
+		} // LCOV_EXCL_STOP
+	}
+
+	if (distinct.order_by) {
+		for (auto &order : distinct.order_by->orders) {
+			if (order.expression->type != ExpressionType::BOUND_COLUMN_REF) { // LCOV_EXCL_START
+				GetReferencedBindings(*order.expression, referenced_bindings);
+			} // LCOV_EXCL_STOP
+		}
+	}
+
+	// Create info for compression
+	CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
+
+	// Create binding mapping
+	const auto bindings = distinct.GetColumnBindings();
+	const auto &types = distinct.types;
+	D_ASSERT(bindings.size() == types.size());
+	for (idx_t col_idx = 0; col_idx < bindings.size(); col_idx++) {
+		// Distinct does not change bindings, input binding is output binding
+		info.binding_map.emplace(bindings[col_idx], CMBindingInfo(bindings[col_idx], types[col_idx]));
+	}
+
+	// Now try to compress
+	CreateProjections(op, info);
+}
+
+} // namespace duckdb

package/src/duckdb/src/optimizer/compressed_materialization/compress_order.cpp

@@ -0,0 +1,65 @@
+#include "duckdb/optimizer/compressed_materialization.hpp"
+#include "duckdb/planner/expression/bound_columnref_expression.hpp"
+#include "duckdb/planner/operator/logical_order.hpp"
+
+namespace duckdb {
+
+void CompressedMaterialization::CompressOrder(unique_ptr<LogicalOperator> &op) {
+	auto &order = op->Cast<LogicalOrder>();
+
+	// Find all bindings referenced by non-colref expressions in the order nodes
+	// These are excluded from compression by projection
+	// But we can try to compress the expression directly
+	column_binding_set_t referenced_bindings;
+	for (idx_t order_node_idx = 0; order_node_idx < order.orders.size(); order_node_idx++) {
+		auto &bound_order = order.orders[order_node_idx];
+		auto &order_expression = *bound_order.expression;
+		if (order_expression.GetExpressionType() == ExpressionType::BOUND_COLUMN_REF) {
+			continue; // Will be compressed generically
+		}
+
+		// Mark the bindings referenced by the non-colref expression so they won't be modified
+		GetReferencedBindings(order_expression, referenced_bindings);
+	}
+
+	// Create info for compression
+	CompressedMaterializationInfo info(*op, {0}, referenced_bindings);
+
+	// Create binding mapping
+	const auto bindings = order.GetColumnBindings();
+	const auto &types = order.types;
+	D_ASSERT(bindings.size() == types.size());
+	for (idx_t col_idx = 0; col_idx < bindings.size(); col_idx++) {
+		// Order does not change bindings, input binding is output binding
+		info.binding_map.emplace(bindings[col_idx], CMBindingInfo(bindings[col_idx], types[col_idx]));
+	}
+
+	// Now try to compress
+	CreateProjections(op, info);
+
+	// Update order statistics
+	UpdateOrderStats(op);
+}
+
+void CompressedMaterialization::UpdateOrderStats(unique_ptr<LogicalOperator> &op) {
+	if (op->type != LogicalOperatorType::LOGICAL_PROJECTION) {
+		return;
+	}
+
+	// Update order stats if compressed
+	auto &compressed_order = op->children[0]->Cast<LogicalOrder>();
+	for (idx_t order_node_idx = 0; order_node_idx < compressed_order.orders.size(); order_node_idx++) {
+		auto &bound_order = compressed_order.orders[order_node_idx];
+		auto &order_expression = *bound_order.expression;
+		if (order_expression.GetExpressionType() != ExpressionType::BOUND_COLUMN_REF) {
+			continue;
+		}
+		auto &colref = order_expression.Cast<BoundColumnRefExpression>();
+		auto it = statistics_map.find(colref.binding);
+		if (it != statistics_map.end() && it->second) {
+			bound_order.stats = it->second->ToUnique();
+		}
+	}
+}
+
+} // namespace duckdb