duckdb 0.8.2-dev2356.0 → 0.8.2-dev2509.0

package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp

@@ -37,8 +37,6 @@ inline interval_t operator-(const interval_t &lhs, const interval_t &rhs) {
 	return Interval::FromMicro(Interval::GetMicro(lhs) - Interval::GetMicro(rhs));
 }
 
-using FrameBounds = std::pair<idx_t, idx_t>;
-
 template <typename SAVE_TYPE>
 struct QuantileState {
 	using SaveType = SAVE_TYPE;
@@ -89,7 +87,7 @@ void ReuseIndexes(idx_t *index, const FrameBounds &frame, const FrameBounds &pre
 	idx_t j = 0;
 
 	//  Copy overlapping indices
-	for (idx_t p = 0; p < (prev.second - prev.first); ++p) {
+	for (idx_t p = 0; p < (prev.end - prev.start); ++p) {
 		auto idx = index[p];
 
 		//  Shift down into any hole
@@ -98,7 +96,7 @@ void ReuseIndexes(idx_t *index, const FrameBounds &frame, const FrameBounds &pre
 		}
 
 		//  Skip overlapping values
-		if (frame.first <= idx && idx < frame.second) {
+		if (frame.start <= idx && idx < frame.end) {
 			++j;
 		}
 	}
@@ -106,15 +104,15 @@ void ReuseIndexes(idx_t *index, const FrameBounds &frame, const FrameBounds &pre
 	//  Insert new indices
 	if (j > 0) {
 		// Overlap: append the new ends
-		for (auto f = frame.first; f < prev.first; ++f, ++j) {
+		for (auto f = frame.start; f < prev.start; ++f, ++j) {
 			index[j] = f;
 		}
-		for (auto f = prev.second; f < frame.second; ++f, ++j) {
+		for (auto f = prev.end; f < frame.end; ++f, ++j) {
 			index[j] = f;
 		}
 	} else {
 		//  No overlap: overwrite with new values
-		for (auto f = frame.first; f < frame.second; ++f, ++j) {
+		for (auto f = frame.start; f < frame.end; ++f, ++j) {
 			index[j] = f;
 		}
 	}
@@ -124,17 +122,17 @@ static idx_t ReplaceIndex(idx_t *index, const FrameBounds &frame, const FrameBou
 	D_ASSERT(index);
 
 	idx_t j = 0;
-	for (idx_t p = 0; p < (prev.second - prev.first); ++p) {
+	for (idx_t p = 0; p < (prev.end - prev.start); ++p) {
 		auto idx = index[p];
 		if (j != p) {
 			break;
 		}
 
-		if (frame.first <= idx && idx < frame.second) {
+		if (frame.start <= idx && idx < frame.end) {
 			++j;
 		}
 	}
-	index[j] = frame.second - 1;
+	index[j] = frame.end - 1;
 
 	return j;
 }
@@ -560,7 +558,7 @@ struct QuantileScalarOperation : public QuantileOperation {
 
 		//  Lazily initialise frame state
 		auto prev_pos = state.pos;
-		state.SetPos(frame.second - frame.first);
+		state.SetPos(frame.end - frame.start);
 
 		auto index = state.w.data();
 		D_ASSERT(index);
@@ -572,11 +570,11 @@ struct QuantileScalarOperation : public QuantileOperation {
 		const auto q = bind_data.quantiles[0];
 
 		bool replace = false;
-		if (frame.first == prev.first + 1 && frame.second == prev.second + 1) {
+		if (frame.start == prev.start + 1 && frame.end == prev.end + 1) {
 			//  Fixed frame size
 			const auto j = ReplaceIndex(index, frame, prev);
 			//	We can only replace if the number of NULLs has not changed
-			if (included.AllValid() || included(prev.first) == included(prev.second)) {
+			if (included.AllValid() || included(prev.start) == included(prev.end)) {
 				Interpolator<DISCRETE> interp(q, prev_pos, false);
 				replace = CanReplace(index, data, j, interp.FRN, interp.CRN, included);
 				if (replace) {
@@ -720,7 +718,7 @@ struct QuantileListOperation : public QuantileOperation {
 
 		//  Lazily initialise frame state
 		auto prev_pos = state.pos;
-		state.SetPos(frame.second - frame.first);
+		state.SetPos(frame.end - frame.start);
 
 		auto index = state.w.data();
 
@@ -731,11 +729,11 @@ struct QuantileListOperation : public QuantileOperation {
 		// then Q25 must be recomputed, but Q50 and Q75 are unaffected.
 		// For a single element list, this reduces to the scalar case.
 		std::pair<idx_t, idx_t> replaceable {state.pos, 0};
-		if (frame.first == prev.first + 1 && frame.second == prev.second + 1) {
+		if (frame.start == prev.start + 1 && frame.end == prev.end + 1) {
 			//  Fixed frame size
 			const auto j = ReplaceIndex(index, frame, prev);
 			//	We can only replace if the number of NULLs has not changed
-			if (included.AllValid() || included(prev.first) == included(prev.second)) {
+			if (included.AllValid() || included(prev.start) == included(prev.end)) {
 				for (const auto &q : bind_data.order) {
 					const auto &quantile = bind_data.quantiles[q];
 					Interpolator<DISCRETE> interp(quantile, prev_pos, false);
@@ -1062,7 +1060,7 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 
 		//  Lazily initialise frame state
 		auto prev_pos = state.pos;
-		state.SetPos(frame.second - frame.first);
+		state.SetPos(frame.end - frame.start);
 
 		auto index = state.w.data();
 		D_ASSERT(index);
@@ -1085,11 +1083,11 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 		const float q = 0.5;
 
 		bool replace = false;
-		if (frame.first == prev.first + 1 && frame.second == prev.second + 1) {
+		if (frame.start == prev.start + 1 && frame.end == prev.end + 1) {
 			//  Fixed frame size
 			const auto j = ReplaceIndex(index, frame, prev);
 			//	We can only replace if the number of NULLs has not changed
-			if (included.AllValid() || included(prev.first) == included(prev.second)) {
+			if (included.AllValid() || included(prev.start) == included(prev.end)) {
 				Interpolator<false> interp(q, prev_pos, false);
 				replace = CanReplace(index, data, j, interp.FRN, interp.CRN, included);
 				if (replace) {

package/src/duckdb/src/core_functions/aggregate/nested/list.cpp

@@ -5,24 +5,6 @@
 
 namespace duckdb {
 
-static void RecursiveFlatten(Vector &vector, idx_t &count) {
-	if (vector.GetVectorType() != VectorType::FLAT_VECTOR) {
-		vector.Flatten(count);
-	}
-
-	auto internal_type = vector.GetType().InternalType();
-	if (internal_type == PhysicalType::LIST) {
-		auto &child_vector = ListVector::GetEntry(vector);
-		auto child_vector_count = ListVector::GetListSize(vector);
-		RecursiveFlatten(child_vector, child_vector_count);
-	} else if (internal_type == PhysicalType::STRUCT) {
-		auto &children = StructVector::GetEntries(vector);
-		for (auto &child : children) {
-			RecursiveFlatten(*child, count);
-		}
-	}
-}
-
 struct ListBindData : public FunctionData {
 	explicit ListBindData(const LogicalType &stype_p);
 	~ListBindData() override;
@@ -60,12 +42,6 @@ struct ListFunction {
 		state.linked_list.first_segment = nullptr;
 		state.linked_list.last_segment = nullptr;
 	}
-
-	template <class STATE>
-	static void Destroy(STATE &state, AggregateInputData &aggr_input_data) {
-		// nop
-	}
-
 	static bool IgnoreNull() {
 		return false;
 	}
@@ -73,58 +49,54 @@ struct ListFunction {
 
 static void ListUpdateFunction(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count,
                                Vector &state_vector, idx_t count) {
-	D_ASSERT(input_count == 1);
 
+	D_ASSERT(input_count == 1);
 	auto &input = inputs[0];
-	UnifiedVectorFormat sdata;
-	state_vector.ToUnifiedFormat(count, sdata);
+	RecursiveUnifiedVectorFormat input_data;
+	Vector::RecursiveToUnifiedFormat(input, count, input_data);
 
-	auto states = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
-	RecursiveFlatten(input, count);
+	UnifiedVectorFormat states_data;
+	state_vector.ToUnifiedFormat(count, states_data);
+	auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
 
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
-		list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list, input, i, count);
+		auto &state = *states[states_data.sel->get_index(i)];
+		list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list, input_data, i);
 	}
 }
 
-static void ListCombineFunction(Vector &state, Vector &combined, AggregateInputData &aggr_input_data, idx_t count) {
-	UnifiedVectorFormat sdata;
-	state.ToUnifiedFormat(count, sdata);
-	auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
+static void ListCombineFunction(Vector &states_vector, Vector &combined, AggregateInputData &, idx_t count) {
 
-	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
+	UnifiedVectorFormat states_data;
+	states_vector.ToUnifiedFormat(count, states_data);
+	auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	auto combined_ptr = FlatVector::GetData<ListAggState *>(combined);
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states_ptr[sdata.sel->get_index(i)];
-		if (state.linked_list.total_capacity == 0) {
-			// NULL, no need to append.
-			continue;
-		}
 
-		// copy the linked list of the state
-		auto copied_linked_list = LinkedList(state.linked_list.total_capacity, nullptr, nullptr);
-		list_bind_data.functions.CopyLinkedList(state.linked_list, copied_linked_list, aggr_input_data.allocator);
+		auto &state = *states_ptr[states_data.sel->get_index(i)];
+		D_ASSERT(state.linked_list.total_capacity != 0);
 
-		// append the copied linked list to the combined state
-		if (combined_ptr[i]->linked_list.last_segment) {
-			combined_ptr[i]->linked_list.last_segment->next = copied_linked_list.first_segment;
-		} else {
-			combined_ptr[i]->linked_list.first_segment = copied_linked_list.first_segment;
+		if (combined_ptr[i]->linked_list.total_capacity == 0) {
+			combined_ptr[i]->linked_list = state.linked_list;
+			continue;
 		}
-		combined_ptr[i]->linked_list.last_segment = copied_linked_list.last_segment;
-		combined_ptr[i]->linked_list.total_capacity += copied_linked_list.total_capacity;
+
+		// append the linked list
+		combined_ptr[i]->linked_list.last_segment->next = state.linked_list.first_segment;
+		combined_ptr[i]->linked_list.last_segment = state.linked_list.last_segment;
+		combined_ptr[i]->linked_list.total_capacity += state.linked_list.total_capacity;
 	}
 }
 
-static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_data, Vector &result, idx_t count,
+static void ListFinalize(Vector &states_vector, AggregateInputData &aggr_input_data, Vector &result, idx_t count,
                          idx_t offset) {
-	UnifiedVectorFormat sdata;
-	state_vector.ToUnifiedFormat(count, sdata);
-	auto states = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
+
+	UnifiedVectorFormat states_data;
+	states_vector.ToUnifiedFormat(count, states_data);
+	auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
 
@@ -133,9 +105,11 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 	size_t total_len = ListVector::GetListSize(result);
 
 	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
-	// first iterate over all of the entries and set up the list entries, plus get the newly required total length
+
+	// first iterate over all entries and set up the list entries, and get the newly required total length
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
+
+		auto &state = *states[states_data.sel->get_index(i)];
 		const auto rid = i + offset;
 		result_data[rid].offset = total_len;
 		if (state.linked_list.total_capacity == 0) {
@@ -143,16 +117,19 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 			result_data[rid].length = 0;
 			continue;
 		}
+
 		// set the length and offset of this list in the result vector
 		auto total_capacity = state.linked_list.total_capacity;
 		result_data[rid].length = total_capacity;
 		total_len += total_capacity;
 	}
-	// reserve capacity, then iterate over all of the entries again and copy over the data tot he child vector
+
+	// reserve capacity, then iterate over all entries again and copy over the data to the child vector
 	ListVector::Reserve(result, total_len);
 	auto &result_child = ListVector::GetEntry(result);
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
+
+		auto &state = *states[states_data.sel->get_index(i)];
 		const auto rid = i + offset;
 		if (state.linked_list.total_capacity == 0) {
 			continue;
@@ -161,6 +138,48 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 		idx_t current_offset = result_data[rid].offset;
 		list_bind_data.functions.BuildListVector(state.linked_list, result_child, current_offset);
 	}
+
+	ListVector::SetListSize(result, total_len);
+}
+
+static void ListWindow(Vector inputs[], const ValidityMask &filter_mask, AggregateInputData &aggr_input_data,
+                       idx_t input_count, data_ptr_t state, const FrameBounds &frame, const FrameBounds &prev,
+                       Vector &result, idx_t rid, idx_t bias) {
+
+	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
+	LinkedList linked_list;
+
+	// UPDATE step
+
+	D_ASSERT(input_count == 1);
+	auto &input = inputs[0];
+
+	// FIXME: we unify more values than necessary (count is frame.end)
+	RecursiveUnifiedVectorFormat input_data;
+	Vector::RecursiveToUnifiedFormat(input, frame.end, input_data);
+
+	for (idx_t i = frame.start; i < frame.end; i++) {
+		list_bind_data.functions.AppendRow(aggr_input_data.allocator, linked_list, input_data, i);
+	}
+
+	// FINALIZE step
+
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	size_t total_len = ListVector::GetListSize(result);
+
+	// set the length and offset of this list in the result vector
+	result_data[rid].offset = total_len;
+	result_data[rid].length = linked_list.total_capacity;
+	D_ASSERT(linked_list.total_capacity != 0);
+	total_len += linked_list.total_capacity;
+
+	// reserve capacity, then copy over the data to the child vector
+	ListVector::Reserve(result, total_len);
+	auto &result_child = ListVector::GetEntry(result);
+	idx_t offset = result_data[rid].offset;
+	list_bind_data.functions.BuildListVector(linked_list, result_child, offset);
+
 	ListVector::SetListSize(result, total_len);
 }
 
@@ -182,8 +201,8 @@ unique_ptr<FunctionData> ListBindFunction(ClientContext &context, AggregateFunct
 AggregateFunction ListFun::GetFunction() {
 	return AggregateFunction({LogicalType::ANY}, LogicalTypeId::LIST, AggregateFunction::StateSize<ListAggState>,
 	                         AggregateFunction::StateInitialize<ListAggState, ListFunction>, ListUpdateFunction,
-	                         ListCombineFunction, ListFinalize, nullptr, ListBindFunction,
-	                         AggregateFunction::StateDestroy<ListAggState, ListFunction>, nullptr, nullptr);
+	                         ListCombineFunction, ListFinalize, nullptr, ListBindFunction, nullptr, nullptr,
+	                         ListWindow);
 }
 
 } // namespace duckdb

package/src/duckdb/src/core_functions/function_list.cpp

@@ -73,7 +73,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayDistinctFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayFilterFun),
 	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ArrayReverseSortFun),
-	DUCKDB_SCALAR_FUNCTION_ALIAS(ArraySliceFun),
+	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ArraySliceFun),
 	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ArraySortFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayTransformFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayUniqueFun),
@@ -200,7 +200,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(ListFilterFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ListPackFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ListReverseSortFun),
-	DUCKDB_SCALAR_FUNCTION(ListSliceFun),
+	DUCKDB_SCALAR_FUNCTION_SET(ListSliceFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ListSortFun),
 	DUCKDB_SCALAR_FUNCTION(ListTransformFun),
 	DUCKDB_SCALAR_FUNCTION(ListUniqueFun),