duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/src/execution/operator/helper/physical_reservoir_sample.cpp

@@ -6,6 +6,7 @@ namespace duckdb {
 //===--------------------------------------------------------------------===//
 // Sink
 //===--------------------------------------------------------------------===//
+
 class SampleGlobalSinkState : public GlobalSinkState {
 public:
 	explicit SampleGlobalSinkState(Allocator &allocator, SampleOptions &options) {
@@ -24,7 +25,8 @@ public:
 		}
 	}
 
-	//! The lock for updating the global aggregate state
+	//! The lock for updating the global aggoregate state
+	//! Also used to update the global sample when percentages are used
 	mutex lock;
 	//! The reservoir sample
 	unique_ptr<BlockingSample> sample;
@@ -36,24 +38,47 @@ unique_ptr<GlobalSinkState> PhysicalReservoirSample::GetGlobalSinkState(ClientCo
 
 SinkResultType PhysicalReservoirSample::Sink(ExecutionContext &context, DataChunk &chunk,
                                              OperatorSinkInput &input) const {
-	auto &gstate = input.global_state.Cast<SampleGlobalSinkState>();
-	if (!gstate.sample) {
-		return SinkResultType::FINISHED;
+	auto &global_state = input.global_state.Cast<SampleGlobalSinkState>();
+	// Percentage only has a global sample.
+	lock_guard<mutex> glock(global_state.lock);
+	if (!global_state.sample) {
+		// always gather full thread percentage
+		auto &allocator = Allocator::Get(context.client);
+		if (options->is_percentage) {
+			double percentage = options->sample_size.GetValue<double>();
+			if (percentage == 0) {
+				return SinkResultType::FINISHED;
+			}
+			global_state.sample = make_uniq<ReservoirSamplePercentage>(allocator, percentage, options->seed);
+		} else {
+			idx_t num_samples = options->sample_size.GetValue<idx_t>();
+			if (num_samples == 0) {
+				return SinkResultType::FINISHED;
+			}
+			global_state.sample = make_uniq<ReservoirSample>(allocator, num_samples, options->seed);
+		}
 	}
-	// we implement reservoir sampling without replacement and exponential jumps here
-	// the algorithm is adopted from the paper Weighted random sampling with a reservoir by Pavlos S. Efraimidis et al.
-	// note that the original algorithm is about weighted sampling; this is a simplified approach for uniform sampling
-	lock_guard<mutex> glock(gstate.lock);
-	gstate.sample->AddToReservoir(chunk);
+	global_state.sample->AddToReservoir(chunk);
 	return SinkResultType::NEED_MORE_INPUT;
 }
 
+SinkCombineResultType PhysicalReservoirSample::Combine(ExecutionContext &context,
+                                                       OperatorSinkCombineInput &input) const {
+	return SinkCombineResultType::FINISHED;
+}
+
+SinkFinalizeType PhysicalReservoirSample::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
+                                                   OperatorSinkFinalizeInput &input) const {
+	return SinkFinalizeType::READY;
+}
+
 //===--------------------------------------------------------------------===//
 // Source
 //===--------------------------------------------------------------------===//
 SourceResultType PhysicalReservoirSample::GetData(ExecutionContext &context, DataChunk &chunk,
                                                   OperatorSourceInput &input) const {
 	auto &sink = this->sink_state->Cast<SampleGlobalSinkState>();
+	lock_guard<mutex> glock(sink.lock);
 	if (!sink.sample) {
 		return SourceResultType::FINISHED;
 	}

package/src/duckdb/src/execution/operator/helper/physical_result_collector.cpp

@@ -2,6 +2,7 @@
 
 #include "duckdb/execution/operator/helper/physical_batch_collector.hpp"
 #include "duckdb/execution/operator/helper/physical_materialized_collector.hpp"
+#include "duckdb/execution/operator/helper/physical_buffered_collector.hpp"
 #include "duckdb/execution/physical_plan_generator.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/main/prepared_statement_data.hpp"
@@ -20,13 +21,22 @@ unique_ptr<PhysicalResultCollector> PhysicalResultCollector::GetResultCollector(
                                                                                 PreparedStatementData &data) {
 	if (!PhysicalPlanGenerator::PreserveInsertionOrder(context, *data.plan)) {
 		// the plan is not order preserving, so we just use the parallel materialized collector
+		if (data.is_streaming) {
+			return make_uniq_base<PhysicalResultCollector, PhysicalBufferedCollector>(data, true);
+		}
 		return make_uniq_base<PhysicalResultCollector, PhysicalMaterializedCollector>(data, true);
 	} else if (!PhysicalPlanGenerator::UseBatchIndex(context, *data.plan)) {
 		// the plan is order preserving, but we cannot use the batch index: use a single-threaded result collector
+		if (data.is_streaming) {
+			return make_uniq_base<PhysicalResultCollector, PhysicalBufferedCollector>(data, false);
+		}
 		return make_uniq_base<PhysicalResultCollector, PhysicalMaterializedCollector>(data, false);
 	} else {
 		// we care about maintaining insertion order and the sources all support batch indexes
 		// use a batch collector
+		if (data.is_streaming) {
+			return make_uniq_base<PhysicalResultCollector, PhysicalBufferedCollector>(data, false);
+		}
 		return make_uniq_base<PhysicalResultCollector, PhysicalBatchCollector>(data);
 	}
 }

package/src/duckdb/src/execution/operator/helper/physical_transaction.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/execution/operator/helper/physical_transaction.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/valid_checker.hpp"
+#include "duckdb/common/exception/transaction_exception.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp

@@ -15,8 +15,13 @@ PhysicalVacuum::PhysicalVacuum(unique_ptr<VacuumInfo> info_p, idx_t estimated_ca
 class VacuumLocalSinkState : public LocalSinkState {
 public:
 	explicit VacuumLocalSinkState(VacuumInfo &info) {
-		for (idx_t col_idx = 0; col_idx < info.columns.size(); col_idx++) {
-			column_distinct_stats.push_back(make_uniq<DistinctStatistics>());
+		for (const auto &column_name : info.columns) {
+			auto &column = info.table->GetColumn(column_name);
+			if (DistinctStatistics::TypeIsSupported(column.GetType())) {
+				column_distinct_stats.push_back(make_uniq<DistinctStatistics>());
+			} else {
+				column_distinct_stats.push_back(nullptr);
+			}
 		}
 	};
 
@@ -30,8 +35,14 @@ unique_ptr<LocalSinkState> PhysicalVacuum::GetLocalSinkState(ExecutionContext &c
 class VacuumGlobalSinkState : public GlobalSinkState {
 public:
 	explicit VacuumGlobalSinkState(VacuumInfo &info) {
-		for (idx_t col_idx = 0; col_idx < info.columns.size(); col_idx++) {
-			column_distinct_stats.push_back(make_uniq<DistinctStatistics>());
+
+		for (const auto &column_name : info.columns) {
+			auto &column = info.table->GetColumn(column_name);
+			if (DistinctStatistics::TypeIsSupported(column.GetType())) {
+				column_distinct_stats.push_back(make_uniq<DistinctStatistics>());
+			} else {
+				column_distinct_stats.push_back(nullptr);
+			}
 		}
 	};
 
@@ -58,13 +69,17 @@ SinkResultType PhysicalVacuum::Sink(ExecutionContext &context, DataChunk &chunk,
 }
 
 SinkCombineResultType PhysicalVacuum::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
-	auto &gstate = input.global_state.Cast<VacuumGlobalSinkState>();
-	auto &lstate = input.local_state.Cast<VacuumLocalSinkState>();
+	auto &g_state = input.global_state.Cast<VacuumGlobalSinkState>();
+	auto &l_state = input.local_state.Cast<VacuumLocalSinkState>();
 
-	lock_guard<mutex> lock(gstate.stats_lock);
-	D_ASSERT(gstate.column_distinct_stats.size() == lstate.column_distinct_stats.size());
-	for (idx_t col_idx = 0; col_idx < gstate.column_distinct_stats.size(); col_idx++) {
-		gstate.column_distinct_stats[col_idx]->Merge(*lstate.column_distinct_stats[col_idx]);
+	lock_guard<mutex> lock(g_state.stats_lock);
+	D_ASSERT(g_state.column_distinct_stats.size() == l_state.column_distinct_stats.size());
+
+	for (idx_t col_idx = 0; col_idx < g_state.column_distinct_stats.size(); col_idx++) {
+		if (g_state.column_distinct_stats[col_idx]) {
+			D_ASSERT(l_state.column_distinct_stats[col_idx]);
+			g_state.column_distinct_stats[col_idx]->Merge(*l_state.column_distinct_stats[col_idx]);
+		}
 	}
 
 	return SinkCombineResultType::FINISHED;

package/src/duckdb/src/execution/operator/join/perfect_hash_join_executor.cpp

@@ -20,7 +20,7 @@ bool PerfectHashJoinExecutor::CanDoPerfectHashJoin() {
 bool PerfectHashJoinExecutor::BuildPerfectHashTable(LogicalType &key_type) {
 	// First, allocate memory for each build column
 	auto build_size = perfect_join_statistics.build_range + 1;
-	for (const auto &type : ht.build_types) {
+	for (const auto &type : join.rhs_output_types) {
 		perfect_hash_table.emplace_back(type, build_size);
 	}
 
@@ -69,16 +69,15 @@ bool PerfectHashJoinExecutor::FullScanHashTable(LogicalType &key_type) {
 
 	// Full scan the remaining build columns and fill the perfect hash table
 	const auto build_size = perfect_join_statistics.build_range + 1;
-	for (idx_t i = 0; i < ht.build_types.size(); i++) {
+	for (idx_t i = 0; i < join.rhs_output_types.size(); i++) {
 		auto &vector = perfect_hash_table[i];
-		D_ASSERT(vector.GetType() == ht.build_types[i]);
+		const auto output_col_idx = ht.output_columns[i];
+		D_ASSERT(vector.GetType() == ht.layout.GetTypes()[output_col_idx]);
 		if (build_size > STANDARD_VECTOR_SIZE) {
 			auto &col_mask = FlatVector::Validity(vector);
 			col_mask.Initialize(build_size);
 		}
-
-		const auto col_no = ht.condition_types.size() + i;
-		data_collection.Gather(tuples_addresses, sel_tuples, key_count, col_no, vector, sel_build);
+		data_collection.Gather(tuples_addresses, sel_tuples, key_count, output_col_idx, vector, sel_build, nullptr);
 	}
 
 	return true;
@@ -189,9 +188,9 @@ OperatorResultType PerfectHashJoinExecutor::ProbePerfectHashTable(ExecutionConte
 		result.Slice(input, state.probe_sel_vec, probe_sel_count, 0);
 	}
 	// on the build side, we need to fetch the data and build dictionary vectors with the sel_vec
-	for (idx_t i = 0; i < ht.build_types.size(); i++) {
+	for (idx_t i = 0; i < join.rhs_output_types.size(); i++) {
 		auto &result_vector = result.data[input.ColumnCount() + i];
-		D_ASSERT(result_vector.GetType() == ht.build_types[i]);
+		D_ASSERT(result_vector.GetType() == ht.layout.GetTypes()[ht.output_columns[i]]);
 		auto &build_vec = perfect_hash_table[i];
 		result_vector.Reference(build_vec);
 		result_vector.Slice(state.build_sel_vec, probe_sel_count);

package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp

@@ -48,7 +48,7 @@ PhysicalAsOfJoin::PhysicalAsOfJoin(LogicalComparisonJoin &op, unique_ptr<Physica
 			break;
 		case ExpressionType::COMPARE_EQUAL:
 			null_sensitive.emplace_back(lhs_orders.size());
-			// Fall through
+			DUCKDB_EXPLICIT_FALLTHROUGH;
 		case ExpressionType::COMPARE_NOT_DISTINCT_FROM:
 			lhs_partitions.emplace_back(std::move(left));
 			rhs_partitions.emplace_back(std::move(right));

package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp

@@ -34,7 +34,7 @@ public:
 class BlockwiseNLJoinGlobalState : public GlobalSinkState {
 public:
 	explicit BlockwiseNLJoinGlobalState(ClientContext &context, const PhysicalBlockwiseNLJoin &op)
-	    : right_chunks(context, op.children[1]->GetTypes()), right_outer(IsRightOuterJoin(op.join_type)) {
+	    : right_chunks(context, op.children[1]->GetTypes()), right_outer(PropagatesBuildSide(op.join_type)) {
 	}
 
 	mutex lock;
@@ -104,6 +104,9 @@ unique_ptr<OperatorState> PhysicalBlockwiseNLJoin::GetOperatorState(ExecutionCon
 		}
 		result->intermediate_chunk.Initialize(Allocator::DefaultAllocator(), intermediate_types);
 	}
+	if (join_type == JoinType::RIGHT_ANTI || join_type == JoinType::RIGHT_SEMI) {
+		throw NotImplementedException("physical blockwise RIGHT_SEMI/RIGHT_ANTI join not yet implemented");
+	}
 	return std::move(result);
 }
 
@@ -249,7 +252,7 @@ unique_ptr<LocalSourceState> PhysicalBlockwiseNLJoin::GetLocalSourceState(Execut
 
 SourceResultType PhysicalBlockwiseNLJoin::GetData(ExecutionContext &context, DataChunk &chunk,
                                                   OperatorSourceInput &input) const {
-	D_ASSERT(IsRightOuterJoin(join_type));
+	D_ASSERT(PropagatesBuildSide(join_type));
 	// check if we need to scan any unmatched tuples from the RHS for the full/right outer join
 	auto &sink = sink_state->Cast<BlockwiseNLJoinGlobalState>();
 	auto &gstate = input.global_state.Cast<BlockwiseNLJoinGlobalScanState>();

package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/execution/operator/join/physical_comparison_join.hpp"
-#include "duckdb/common/types/chunk_collection.hpp"
+
 #include "duckdb/common/enum_util.hpp"
 
 namespace duckdb {

package/src/duckdb/src/execution/operator/join/physical_delim_join.cpp

@@ -1,30 +1,15 @@
 #include "duckdb/execution/operator/join/physical_delim_join.hpp"
 
-#include "duckdb/common/types/column/column_data_collection.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
-#include "duckdb/execution/operator/scan/physical_column_data_scan.hpp"
-#include "duckdb/execution/operator/set/physical_recursive_cte.hpp"
-#include "duckdb/parallel/meta_pipeline.hpp"
-#include "duckdb/parallel/pipeline.hpp"
-#include "duckdb/parallel/thread_context.hpp"
 
 namespace duckdb {
 
-PhysicalDelimJoin::PhysicalDelimJoin(vector<LogicalType> types, unique_ptr<PhysicalOperator> original_join,
+PhysicalDelimJoin::PhysicalDelimJoin(PhysicalOperatorType type, vector<LogicalType> types,
+                                     unique_ptr<PhysicalOperator> original_join,
                                      vector<const_reference<PhysicalOperator>> delim_scans, idx_t estimated_cardinality)
-    : PhysicalOperator(PhysicalOperatorType::DELIM_JOIN, std::move(types), estimated_cardinality),
-      join(std::move(original_join)), delim_scans(std::move(delim_scans)) {
-	D_ASSERT(join->children.size() == 2);
-	// now for the original join
-	// we take its left child, this is the side that we will duplicate eliminate
-	children.push_back(std::move(join->children[0]));
-
-	// we replace it with a PhysicalColumnDataScan, that scans the ColumnDataCollection that we keep cached
-	// the actual chunk collection to scan will be created in the DelimJoinGlobalState
-	auto cached_chunk_scan = make_uniq<PhysicalColumnDataScan>(
-	    children[0]->GetTypes(), PhysicalOperatorType::COLUMN_DATA_SCAN, estimated_cardinality);
-	join->children[0] = std::move(cached_chunk_scan);
+    : PhysicalOperator(type, std::move(types), estimated_cardinality), join(std::move(original_join)),
+      delim_scans(std::move(delim_scans)) {
+	D_ASSERT(type == PhysicalOperatorType::LEFT_DELIM_JOIN || type == PhysicalOperatorType::RIGHT_DELIM_JOIN);
 }
 
 vector<const_reference<PhysicalOperator>> PhysicalDelimJoin::GetChildren() const {
@@ -37,115 +22,8 @@ vector<const_reference<PhysicalOperator>> PhysicalDelimJoin::GetChildren() const
 	return result;
 }
 
-//===--------------------------------------------------------------------===//
-// Sink
-//===--------------------------------------------------------------------===//
-class DelimJoinGlobalState : public GlobalSinkState {
-public:
-	explicit DelimJoinGlobalState(ClientContext &context, const PhysicalDelimJoin &delim_join)
-	    : lhs_data(context, delim_join.children[0]->GetTypes()) {
-		D_ASSERT(delim_join.delim_scans.size() > 0);
-		// set up the delim join chunk to scan in the original join
-		auto &cached_chunk_scan = delim_join.join->children[0]->Cast<PhysicalColumnDataScan>();
-		cached_chunk_scan.collection = &lhs_data;
-	}
-
-	ColumnDataCollection lhs_data;
-	mutex lhs_lock;
-
-	void Merge(ColumnDataCollection &input) {
-		lock_guard<mutex> guard(lhs_lock);
-		lhs_data.Combine(input);
-	}
-};
-
-class DelimJoinLocalState : public LocalSinkState {
-public:
-	explicit DelimJoinLocalState(ClientContext &context, const PhysicalDelimJoin &delim_join)
-	    : lhs_data(context, delim_join.children[0]->GetTypes()) {
-		lhs_data.InitializeAppend(append_state);
-	}
-
-	unique_ptr<LocalSinkState> distinct_state;
-	ColumnDataCollection lhs_data;
-	ColumnDataAppendState append_state;
-
-	void Append(DataChunk &input) {
-		lhs_data.Append(input);
-	}
-};
-
-unique_ptr<GlobalSinkState> PhysicalDelimJoin::GetGlobalSinkState(ClientContext &context) const {
-	auto state = make_uniq<DelimJoinGlobalState>(context, *this);
-	distinct->sink_state = distinct->GetGlobalSinkState(context);
-	if (delim_scans.size() > 1) {
-		PhysicalHashAggregate::SetMultiScan(*distinct->sink_state);
-	}
-	return std::move(state);
-}
-
-unique_ptr<LocalSinkState> PhysicalDelimJoin::GetLocalSinkState(ExecutionContext &context) const {
-	auto state = make_uniq<DelimJoinLocalState>(context.client, *this);
-	state->distinct_state = distinct->GetLocalSinkState(context);
-	return std::move(state);
-}
-
-SinkResultType PhysicalDelimJoin::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const {
-	auto &lstate = input.local_state.Cast<DelimJoinLocalState>();
-	lstate.lhs_data.Append(lstate.append_state, chunk);
-	OperatorSinkInput distinct_sink_input {*distinct->sink_state, *lstate.distinct_state, input.interrupt_state};
-	distinct->Sink(context, chunk, distinct_sink_input);
-	return SinkResultType::NEED_MORE_INPUT;
-}
-
-SinkCombineResultType PhysicalDelimJoin::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const {
-	auto &lstate = input.local_state.Cast<DelimJoinLocalState>();
-	auto &gstate = input.global_state.Cast<DelimJoinGlobalState>();
-	gstate.Merge(lstate.lhs_data);
-
-	OperatorSinkCombineInput distinct_combine_input {*distinct->sink_state, *lstate.distinct_state,
-	                                                 input.interrupt_state};
-	distinct->Combine(context, distinct_combine_input);
-
-	return SinkCombineResultType::FINISHED;
-}
-
-SinkFinalizeType PhysicalDelimJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &client,
-                                             OperatorSinkFinalizeInput &input) const {
-	// finalize the distinct HT
-	D_ASSERT(distinct);
-
-	OperatorSinkFinalizeInput finalize_input {*distinct->sink_state, input.interrupt_state};
-	distinct->Finalize(pipeline, event, client, finalize_input);
-	return SinkFinalizeType::READY;
-}
-
 string PhysicalDelimJoin::ParamsToString() const {
 	return join->ParamsToString();
 }
 
-//===--------------------------------------------------------------------===//
-// Pipeline Construction
-//===--------------------------------------------------------------------===//
-void PhysicalDelimJoin::BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) {
-	op_state.reset();
-	sink_state.reset();
-
-	auto &child_meta_pipeline = meta_pipeline.CreateChildMetaPipeline(current, *this);
-	child_meta_pipeline.Build(*children[0]);
-
-	if (type == PhysicalOperatorType::DELIM_JOIN) {
-		// recurse into the actual join
-		// any pipelines in there depend on the main pipeline
-		// any scan of the duplicate eliminated data on the RHS depends on this pipeline
-		// we add an entry to the mapping of (PhysicalOperator*) -> (Pipeline*)
-		auto &state = meta_pipeline.GetState();
-		for (auto &delim_scan : delim_scans) {
-			state.delim_join_dependencies.insert(
-			    make_pair(delim_scan, reference<Pipeline>(*child_meta_pipeline.GetBasePipeline())));
-		}
-		join->BuildPipelines(current, meta_pipeline);
-	}
-}
-
 } // namespace duckdb