duckdb 0.7.2-dev12.0 → 0.7.2-dev1244.0

package/src/duckdb/src/execution/window_segment_tree.cpp

@@ -6,6 +6,178 @@
 
 namespace duckdb {
 
+//===--------------------------------------------------------------------===//
+// WindowAggregateState
+//===--------------------------------------------------------------------===//
+
+WindowAggregateState::WindowAggregateState(AggregateFunction &aggregate, FunctionData *bind_info,
+                                           const LogicalType &result_type_p)
+    : aggregate(aggregate), bind_info(bind_info), result_type(result_type_p), state(aggregate.state_size()),
+      statev(Value::POINTER((idx_t)state.data())), statep(Value::POINTER((idx_t)state.data())) {
+	statev.SetVectorType(VectorType::FLAT_VECTOR); // Prevent conversion of results to constants
+}
+
+WindowAggregateState::~WindowAggregateState() {
+}
+
+void WindowAggregateState::AggregateInit() {
+	aggregate.initialize(state.data());
+}
+
+void WindowAggregateState::AggegateFinal(Vector &result, idx_t rid) {
+	AggregateInputData aggr_input_data(bind_info, Allocator::DefaultAllocator());
+	aggregate.finalize(statev, aggr_input_data, result, 1, rid);
+
+	if (aggregate.destructor) {
+		aggregate.destructor(statev, 1);
+	}
+}
+
+void WindowAggregateState::Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered) {
+}
+
+void WindowAggregateState::Finalize() {
+}
+
+void WindowAggregateState::Compute(Vector &result, idx_t rid, idx_t start, idx_t end) {
+}
+
+//===--------------------------------------------------------------------===//
+// WindowConstantAggregate
+//===--------------------------------------------------------------------===//
+
+WindowConstantAggregate::WindowConstantAggregate(AggregateFunction &aggregate, FunctionData *bind_info,
+                                                 const LogicalType &result_type, const ValidityMask &partition_mask,
+                                                 const idx_t count)
+    : WindowAggregateState(aggregate, bind_info, result_type), partition(0), row(0) {
+
+	// Locate the partition boundaries
+	idx_t start = 0;
+	if (partition_mask.AllValid()) {
+		partition_offsets.emplace_back(0);
+	} else {
+		idx_t entry_idx;
+		idx_t shift;
+		while (start < count) {
+			partition_mask.GetEntryIndex(start, entry_idx, shift);
+
+			//	If start is aligned with the start of a block,
+			//	and the block is blank, then skip forward one block.
+			const auto block = partition_mask.GetValidityEntry(entry_idx);
+			if (partition_mask.NoneValid(block) && !shift) {
+				start += ValidityMask::BITS_PER_VALUE;
+				continue;
+			}
+
+			// Loop over the block
+			for (; shift < ValidityMask::BITS_PER_VALUE && start < count; ++shift, ++start) {
+				if (partition_mask.RowIsValid(block, shift)) {
+					partition_offsets.emplace_back(start);
+				}
+			}
+		}
+	}
+
+	//	Initialise the vector for caching the results
+	results = make_unique<Vector>(result_type, partition_offsets.size());
+	partition_offsets.emplace_back(count);
+
+	//	Start the first aggregate
+	AggregateInit();
+}
+
+void WindowConstantAggregate::Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered) {
+	const auto chunk_begin = row;
+	const auto chunk_end = chunk_begin + payload_chunk.size();
+
+	if (!inputs.ColumnCount() && payload_chunk.ColumnCount()) {
+		inputs.Initialize(Allocator::DefaultAllocator(), payload_chunk.GetTypes());
+	}
+
+	AggregateInputData aggr_input_data(bind_info, Allocator::DefaultAllocator());
+	idx_t begin = 0;
+	idx_t filter_idx = 0;
+	auto partition_end = partition_offsets[partition + 1];
+	while (row < chunk_end) {
+		if (row == partition_end) {
+			AggegateFinal(*results, partition++);
+			AggregateInit();
+			partition_end = partition_offsets[partition + 1];
+		}
+		partition_end = MinValue(partition_end, chunk_end);
+		auto end = partition_end - chunk_begin;
+
+		inputs.Reset();
+		if (begin) {
+			for (idx_t c = 0; c < payload_chunk.ColumnCount(); ++c) {
+				inputs.data[c].Slice(payload_chunk.data[c], begin, end);
+			}
+		} else {
+			inputs.Reference(payload_chunk);
+		}
+		inputs.SetCardinality(end - begin);
+
+		// Slice to any filtered rows
+		SelectionVector sel;
+		if (filter_sel) {
+			//	Find the first value in [begin, end)
+			for (; filter_idx < filtered; ++filter_idx) {
+				auto idx = filter_sel->get_index(filter_idx);
+				if (idx >= begin) {
+					break;
+				}
+			}
+			sel.Initialize(filter_sel->data());
+			idx_t nsel = 0;
+			for (; filter_idx < filtered; ++filter_idx, ++nsel) {
+				auto idx = filter_sel->get_index(filter_idx);
+				if (idx >= end) {
+					break;
+				}
+			}
+
+			if (nsel != inputs.size()) {
+				inputs.Slice(sel, nsel);
+			}
+		}
+
+		//	Aggregate the filtered rows into a single state
+		const auto count = inputs.size();
+		if (aggregate.simple_update) {
+			aggregate.simple_update(inputs.data.data(), aggr_input_data, inputs.ColumnCount(), state.data(), count);
+		} else {
+			aggregate.update(inputs.data.data(), aggr_input_data, inputs.ColumnCount(), statep, count);
+		}
+
+		//	Skip filtered rows too!
+		row += end - begin;
+		begin = end;
+	}
+}
+
+void WindowConstantAggregate::Finalize() {
+	AggegateFinal(*results, partition++);
+
+	partition = 0;
+	row = 0;
+}
+
+void WindowConstantAggregate::Compute(Vector &target, idx_t rid, idx_t start, idx_t end) {
+	//	Find the partition containing [start, end)
+	while (start < partition_offsets[partition] || partition_offsets[partition + 1] <= start) {
+		++partition;
+	}
+	D_ASSERT(partition_offsets[partition] <= start);
+	D_ASSERT(partition + 1 < partition_offsets.size());
+	D_ASSERT(end <= partition_offsets[partition + 1]);
+
+	// Copy the value
+	VectorOperations::Copy(*results, target, partition + 1, partition, rid);
+}
+
+//===--------------------------------------------------------------------===//
+// WindowSegmentTree
+//===--------------------------------------------------------------------===//
 WindowSegmentTree::WindowSegmentTree(AggregateFunction &aggregate, FunctionData *bind_info,
                                      const LogicalType &result_type_p, DataChunk *input,
                                      const ValidityMask &filter_mask_p, WindowAggregationMode mode_p)
@@ -107,7 +279,7 @@ void WindowSegmentTree::WindowSegmentValue(idx_t l_idx, idx_t begin, idx_t end)
 	if (l_idx == 0) {
 		ExtractFrame(begin, end);
 		AggregateInputData aggr_input_data(bind_info, Allocator::DefaultAllocator());
-		D_ASSERT(inputs.data.size() > 0);
+		D_ASSERT(!inputs.data.empty());
 		aggregate.update(&inputs.data[0], aggr_input_data, input_ref->ColumnCount(), s, inputs.size());
 	} else {
 		// find out where the states begin

package/src/duckdb/src/function/aggregate/algebraic/avg.cpp

@@ -127,9 +127,6 @@ struct NumericAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			if (!Value::DoubleIsFinite(state->value)) {
-				throw OutOfRangeException("AVG is out of range!");
-			}
 			target[idx] = (state->value / state->count);
 		}
 	}
@@ -141,9 +138,6 @@ struct KahanAverageOperation : public BaseSumOperation<AverageSetOperation, Kaha
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			if (!Value::DoubleIsFinite(state->value)) {
-				throw OutOfRangeException("AVG is out of range!");
-			}
 			target[idx] = (state->value / state->count) + (state->err / state->count);
 		}
 	}

package/src/duckdb/src/function/aggregate/distributive/bitagg.cpp

@@ -3,6 +3,8 @@
 #include "duckdb/common/types/null_value.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/common/vector_operations/aggregate_executor.hpp"
+#include "duckdb/common/types/bit.hpp"
+#include "duckdb/common/types/cast_helpers.hpp"
 
 namespace duckdb {
 
@@ -38,37 +40,32 @@ static AggregateFunction GetBitfieldUnaryAggregate(LogicalType type) {
 	}
 }
 
-struct BitAndOperation {
+struct BitwiseOperation {
 	template <class STATE>
 	static void Initialize(STATE *state) {
-		//  If there are no matching rows, BIT_AND() returns a null value.
+		//  If there are no matching rows, returns a null value.
 		state->is_set = false;
 	}
 
 	template <class INPUT_TYPE, class STATE, class OP>
 	static void Operation(STATE *state, AggregateInputData &, INPUT_TYPE *input, ValidityMask &mask, idx_t idx) {
 		if (!state->is_set) {
+			OP::template Assign(state, input[idx]);
 			state->is_set = true;
-			state->value = input[idx];
 		} else {
-			state->value &= input[idx];
+			OP::template Execute(state, input[idx]);
 		}
 	}
 
 	template <class INPUT_TYPE, class STATE, class OP>
 	static void ConstantOperation(STATE *state, AggregateInputData &aggr_input_data, INPUT_TYPE *input,
 	                              ValidityMask &mask, idx_t count) {
-		//  count is not relevant
-		Operation<INPUT_TYPE, STATE, OP>(state, aggr_input_data, input, mask, 0);
+		OP::template Operation<INPUT_TYPE, STATE, OP>(state, aggr_input_data, input, mask, 0);
 	}
 
-	template <class T, class STATE>
-	static void Finalize(Vector &result, AggregateInputData &, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
-		if (!state->is_set) {
-			mask.SetInvalid(idx);
-		} else {
-			target[idx] = state->value;
-		}
+	template <class INPUT_TYPE, class STATE>
+	static void Assign(STATE *state, INPUT_TYPE input) {
+		state->value = input;
 	}
 
 	template <class STATE, class OP>
@@ -79,9 +76,19 @@ struct BitAndOperation {
 		}
 		if (!target->is_set) {
 			// target is NULL, use source value directly.
-			*target = source;
+			OP::template Assign(target, source.value);
+			target->is_set = true;
+		} else {
+			OP::template Execute(target, source.value);
+		}
+	}
+
+	template <class T, class STATE>
+	static void Finalize(Vector &result, AggregateInputData &, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
+		if (!state->is_set) {
+			mask.SetInvalid(idx);
 		} else {
-			target->value &= source.value;
+			target[idx] = state->value;
 		}
 	}
 
@@ -90,36 +97,55 @@ struct BitAndOperation {
 	}
 };
 
-void BitAndFun::RegisterFunction(BuiltinFunctions &set) {
-	AggregateFunctionSet bit_and("bit_and");
-	for (auto &type : LogicalType::Integral()) {
-		bit_and.AddFunction(GetBitfieldUnaryAggregate<BitAndOperation>(type));
+struct BitAndOperation : public BitwiseOperation {
+	template <class INPUT_TYPE, class STATE>
+	static void Execute(STATE *state, INPUT_TYPE input) {
+		state->value &= input;
 	}
-	set.AddFunction(bit_and);
-}
+};
 
-struct BitOrOperation {
-	template <class STATE>
-	static void Initialize(STATE *state) {
-		//  If there are no matching rows, BIT_OR() returns a null value.
-		state->is_set = false;
+struct BitOrOperation : public BitwiseOperation {
+	template <class INPUT_TYPE, class STATE>
+	static void Execute(STATE *state, INPUT_TYPE input) {
+		state->value |= input;
 	}
+};
 
-	template <class INPUT_TYPE, class STATE, class OP>
-	static void Operation(STATE *state, AggregateInputData &, INPUT_TYPE *input, ValidityMask &mask, idx_t idx) {
-		if (!state->is_set) {
-			state->is_set = true;
-			state->value = input[idx];
-		} else {
-			state->value |= input[idx];
-		}
+struct BitXorOperation : public BitwiseOperation {
+	template <class INPUT_TYPE, class STATE>
+	static void Execute(STATE *state, INPUT_TYPE input) {
+		state->value ^= input;
 	}
 
 	template <class INPUT_TYPE, class STATE, class OP>
 	static void ConstantOperation(STATE *state, AggregateInputData &aggr_input_data, INPUT_TYPE *input,
 	                              ValidityMask &mask, idx_t count) {
-		//  count is irrelevant
-		Operation<INPUT_TYPE, STATE, OP>(state, aggr_input_data, input, mask, 0);
+		for (idx_t i = 0; i < count; i++) {
+			Operation<INPUT_TYPE, STATE, OP>(state, aggr_input_data, input, mask, 0);
+		}
+	}
+};
+
+struct BitStringBitwiseOperation : public BitwiseOperation {
+	template <class STATE>
+	static void Destroy(STATE *state) {
+		if (state->is_set && !state->value.IsInlined()) {
+			delete[] state->value.GetDataUnsafe();
+		}
+	}
+
+	template <class INPUT_TYPE, class STATE>
+	static void Assign(STATE *state, INPUT_TYPE input) {
+		D_ASSERT(state->is_set == false);
+		if (input.IsInlined()) {
+			state->value = input;
+		} else { // non-inlined string, need to allocate space for it
+			auto len = input.GetSize();
+			auto ptr = new char[len];
+			memcpy(ptr, input.GetDataUnsafe(), len);
+
+			state->value = string_t(ptr, len);
+		}
 	}
 
 	template <class T, class STATE>
@@ -127,52 +153,31 @@ struct BitOrOperation {
 		if (!state->is_set) {
 			mask.SetInvalid(idx);
 		} else {
-			target[idx] = state->value;
+			target[idx] = StringVector::AddStringOrBlob(result, state->value);
 		}
 	}
+};
 
-	template <class STATE, class OP>
-	static void Combine(const STATE &source, STATE *target, AggregateInputData &) {
-		if (!source.is_set) {
-			// source is NULL, nothing to do.
-			return;
-		}
-		if (!target->is_set) {
-			// target is NULL, use source value directly.
-			*target = source;
-		} else {
-			target->value |= source.value;
-		}
-	}
+struct BitStringAndOperation : public BitStringBitwiseOperation {
 
-	static bool IgnoreNull() {
-		return true;
+	template <class INPUT_TYPE, class STATE>
+	static void Execute(STATE *state, INPUT_TYPE input) {
+		Bit::BitwiseAnd(input, state->value, state->value);
 	}
 };
 
-void BitOrFun::RegisterFunction(BuiltinFunctions &set) {
-	AggregateFunctionSet bit_or("bit_or");
-	for (auto &type : LogicalType::Integral()) {
-		bit_or.AddFunction(GetBitfieldUnaryAggregate<BitOrOperation>(type));
-	}
-	set.AddFunction(bit_or);
-}
+struct BitStringOrOperation : public BitStringBitwiseOperation {
 
-struct BitXorOperation {
-	template <class STATE>
-	static void Initialize(STATE *state) {
-		//  If there are no matching rows, BIT_XOR() returns a null value.
-		state->is_set = false;
+	template <class INPUT_TYPE, class STATE>
+	static void Execute(STATE *state, INPUT_TYPE input) {
+		Bit::BitwiseOr(input, state->value, state->value);
 	}
+};
 
-	template <class INPUT_TYPE, class STATE, class OP>
-	static void Operation(STATE *state, AggregateInputData &, INPUT_TYPE *input, ValidityMask &mask, idx_t idx) {
-		if (!state->is_set) {
-			state->is_set = true;
-			state->value = input[idx];
-		} else {
-			state->value ^= input[idx];
-		}
+struct BitStringXorOperation : public BitStringBitwiseOperation {
+	template <class INPUT_TYPE, class STATE>
+	static void Execute(STATE *state, INPUT_TYPE input) {
+		Bit::BitwiseXor(input, state->value, state->value);
 	}
 
 	template <class INPUT_TYPE, class STATE, class OP>
@@ -182,40 +187,39 @@ struct BitXorOperation {
 			Operation<INPUT_TYPE, STATE, OP>(state, aggr_input_data, input, mask, 0);
 		}
 	}
+};
 
-	template <class T, class STATE>
-	static void Finalize(Vector &result, AggregateInputData &, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
-		if (!state->is_set) {
-			mask.SetInvalid(idx);
-		} else {
-			target[idx] = state->value;
-		}
+void BitAndFun::RegisterFunction(BuiltinFunctions &set) {
+	AggregateFunctionSet bit_and("bit_and");
+	for (auto &type : LogicalType::Integral()) {
+		bit_and.AddFunction(GetBitfieldUnaryAggregate<BitAndOperation>(type));
 	}
 
-	template <class STATE, class OP>
-	static void Combine(const STATE &source, STATE *target, AggregateInputData &) {
-		if (!source.is_set) {
-			// source is NULL, nothing to do.
-			return;
-		}
-		if (!target->is_set) {
-			// target is NULL, use source value directly.
-			*target = source;
-		} else {
-			target->value ^= source.value;
-		}
-	}
+	bit_and.AddFunction(
+	    AggregateFunction::UnaryAggregateDestructor<BitState<string_t>, string_t, string_t, BitStringAndOperation>(
+	        LogicalType::BIT, LogicalType::BIT));
+	set.AddFunction(bit_and);
+}
 
-	static bool IgnoreNull() {
-		return true;
+void BitOrFun::RegisterFunction(BuiltinFunctions &set) {
+	AggregateFunctionSet bit_or("bit_or");
+	for (auto &type : LogicalType::Integral()) {
+		bit_or.AddFunction(GetBitfieldUnaryAggregate<BitOrOperation>(type));
 	}
-};
+	bit_or.AddFunction(
+	    AggregateFunction::UnaryAggregateDestructor<BitState<string_t>, string_t, string_t, BitStringOrOperation>(
+	        LogicalType::BIT, LogicalType::BIT));
+	set.AddFunction(bit_or);
+}
 
 void BitXorFun::RegisterFunction(BuiltinFunctions &set) {
 	AggregateFunctionSet bit_xor("bit_xor");
 	for (auto &type : LogicalType::Integral()) {
 		bit_xor.AddFunction(GetBitfieldUnaryAggregate<BitXorOperation>(type));
 	}
+	bit_xor.AddFunction(
+	    AggregateFunction::UnaryAggregateDestructor<BitState<string_t>, string_t, string_t, BitStringXorOperation>(
+	        LogicalType::BIT, LogicalType::BIT));
 	set.AddFunction(bit_xor);
 }