duckdb 0.8.2-dev37.0 → 0.8.2-dev3949.0

package/src/duckdb/src/core_functions/aggregate/nested/list.cpp

@@ -5,24 +5,6 @@
 
 namespace duckdb {
 
-static void RecursiveFlatten(Vector &vector, idx_t &count) {
-	if (vector.GetVectorType() != VectorType::FLAT_VECTOR) {
-		vector.Flatten(count);
-	}
-
-	auto internal_type = vector.GetType().InternalType();
-	if (internal_type == PhysicalType::LIST) {
-		auto &child_vector = ListVector::GetEntry(vector);
-		auto child_vector_count = ListVector::GetListSize(vector);
-		RecursiveFlatten(child_vector, child_vector_count);
-	} else if (internal_type == PhysicalType::STRUCT) {
-		auto &children = StructVector::GetEntries(vector);
-		for (auto &child : children) {
-			RecursiveFlatten(*child, count);
-		}
-	}
-}
-
 struct ListBindData : public FunctionData {
 	explicit ListBindData(const LogicalType &stype_p);
 	~ListBindData() override;
@@ -60,12 +42,6 @@ struct ListFunction {
 		state.linked_list.first_segment = nullptr;
 		state.linked_list.last_segment = nullptr;
 	}
-
-	template <class STATE>
-	static void Destroy(STATE &state, AggregateInputData &aggr_input_data) {
-		auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
-		list_bind_data.functions.Destroy(aggr_input_data.allocator, state.linked_list);
-	}
 	static bool IgnoreNull() {
 		return false;
 	}
@@ -73,58 +49,59 @@ struct ListFunction {
 
 static void ListUpdateFunction(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count,
                                Vector &state_vector, idx_t count) {
-	D_ASSERT(input_count == 1);
 
+	D_ASSERT(input_count == 1);
 	auto &input = inputs[0];
-	UnifiedVectorFormat sdata;
-	state_vector.ToUnifiedFormat(count, sdata);
+	RecursiveUnifiedVectorFormat input_data;
+	Vector::RecursiveToUnifiedFormat(input, count, input_data);
 
-	auto states = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
-	RecursiveFlatten(input, count);
+	UnifiedVectorFormat states_data;
+	state_vector.ToUnifiedFormat(count, states_data);
+	auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
 
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
-		list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list, input, i, count);
+		auto &state = *states[states_data.sel->get_index(i)];
+		list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list, input_data, i);
 	}
 }
 
-static void ListCombineFunction(Vector &state, Vector &combined, AggregateInputData &aggr_input_data, idx_t count) {
-	UnifiedVectorFormat sdata;
-	state.ToUnifiedFormat(count, sdata);
-	auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
+static void ListCombineFunction(Vector &states_vector, Vector &combined, AggregateInputData &, idx_t count) {
 
-	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
+	UnifiedVectorFormat states_data;
+	states_vector.ToUnifiedFormat(count, states_data);
+	auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	auto combined_ptr = FlatVector::GetData<ListAggState *>(combined);
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states_ptr[sdata.sel->get_index(i)];
+
+		auto &state = *states_ptr[states_data.sel->get_index(i)];
 		if (state.linked_list.total_capacity == 0) {
-			// NULL, no need to append.
+			// NULL, no need to append
+			// this can happen when adding a FILTER to the grouping, e.g.,
+			// LIST(i) FILTER (WHERE i <> 3)
 			continue;
 		}
 
-		// copy the linked list of the state
-		auto copied_linked_list = LinkedList(state.linked_list.total_capacity, nullptr, nullptr);
-		list_bind_data.functions.CopyLinkedList(state.linked_list, copied_linked_list, aggr_input_data.allocator);
-
-		// append the copied linked list to the combined state
-		if (combined_ptr[i]->linked_list.last_segment) {
-			combined_ptr[i]->linked_list.last_segment->next = copied_linked_list.first_segment;
-		} else {
-			combined_ptr[i]->linked_list.first_segment = copied_linked_list.first_segment;
+		if (combined_ptr[i]->linked_list.total_capacity == 0) {
+			combined_ptr[i]->linked_list = state.linked_list;
+			continue;
 		}
-		combined_ptr[i]->linked_list.last_segment = copied_linked_list.last_segment;
-		combined_ptr[i]->linked_list.total_capacity += copied_linked_list.total_capacity;
+
+		// append the linked list
+		combined_ptr[i]->linked_list.last_segment->next = state.linked_list.first_segment;
+		combined_ptr[i]->linked_list.last_segment = state.linked_list.last_segment;
+		combined_ptr[i]->linked_list.total_capacity += state.linked_list.total_capacity;
 	}
 }
 
-static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_data, Vector &result, idx_t count,
+static void ListFinalize(Vector &states_vector, AggregateInputData &aggr_input_data, Vector &result, idx_t count,
                          idx_t offset) {
-	UnifiedVectorFormat sdata;
-	state_vector.ToUnifiedFormat(count, sdata);
-	auto states = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
+
+	UnifiedVectorFormat states_data;
+	states_vector.ToUnifiedFormat(count, states_data);
+	auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
 
 	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
 
@@ -133,9 +110,11 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 	size_t total_len = ListVector::GetListSize(result);
 
 	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
-	// first iterate over all of the entries and set up the list entries, plus get the newly required total length
+
+	// first iterate over all entries and set up the list entries, and get the newly required total length
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
+
+		auto &state = *states[states_data.sel->get_index(i)];
 		const auto rid = i + offset;
 		result_data[rid].offset = total_len;
 		if (state.linked_list.total_capacity == 0) {
@@ -143,16 +122,19 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 			result_data[rid].length = 0;
 			continue;
 		}
+
 		// set the length and offset of this list in the result vector
 		auto total_capacity = state.linked_list.total_capacity;
 		result_data[rid].length = total_capacity;
 		total_len += total_capacity;
 	}
-	// reserve capacity, then iterate over all of the entries again and copy over the data tot he child vector
+
+	// reserve capacity, then iterate over all entries again and copy over the data to the child vector
 	ListVector::Reserve(result, total_len);
 	auto &result_child = ListVector::GetEntry(result);
 	for (idx_t i = 0; i < count; i++) {
-		auto &state = *states[sdata.sel->get_index(i)];
+
+		auto &state = *states[states_data.sel->get_index(i)];
 		const auto rid = i + offset;
 		if (state.linked_list.total_capacity == 0) {
 			continue;
@@ -161,6 +143,48 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
 		idx_t current_offset = result_data[rid].offset;
 		list_bind_data.functions.BuildListVector(state.linked_list, result_child, current_offset);
 	}
+
+	ListVector::SetListSize(result, total_len);
+}
+
+static void ListWindow(Vector inputs[], const ValidityMask &filter_mask, AggregateInputData &aggr_input_data,
+                       idx_t input_count, data_ptr_t state, const FrameBounds &frame, const FrameBounds &prev,
+                       Vector &result, idx_t rid, idx_t bias) {
+
+	auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
+	LinkedList linked_list;
+
+	// UPDATE step
+
+	D_ASSERT(input_count == 1);
+	auto &input = inputs[0];
+
+	// FIXME: we unify more values than necessary (count is frame.end)
+	RecursiveUnifiedVectorFormat input_data;
+	Vector::RecursiveToUnifiedFormat(input, frame.end, input_data);
+
+	for (idx_t i = frame.start; i < frame.end; i++) {
+		list_bind_data.functions.AppendRow(aggr_input_data.allocator, linked_list, input_data, i);
+	}
+
+	// FINALIZE step
+
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	size_t total_len = ListVector::GetListSize(result);
+
+	// set the length and offset of this list in the result vector
+	result_data[rid].offset = total_len;
+	result_data[rid].length = linked_list.total_capacity;
+	D_ASSERT(linked_list.total_capacity != 0);
+	total_len += linked_list.total_capacity;
+
+	// reserve capacity, then copy over the data to the child vector
+	ListVector::Reserve(result, total_len);
+	auto &result_child = ListVector::GetEntry(result);
+	idx_t offset = result_data[rid].offset;
+	list_bind_data.functions.BuildListVector(linked_list, result_child, offset);
+
 	ListVector::SetListSize(result, total_len);
 }
 
@@ -182,8 +206,8 @@ unique_ptr<FunctionData> ListBindFunction(ClientContext &context, AggregateFunct
 AggregateFunction ListFun::GetFunction() {
 	return AggregateFunction({LogicalType::ANY}, LogicalTypeId::LIST, AggregateFunction::StateSize<ListAggState>,
 	                         AggregateFunction::StateInitialize<ListAggState, ListFunction>, ListUpdateFunction,
-	                         ListCombineFunction, ListFinalize, nullptr, ListBindFunction,
-	                         AggregateFunction::StateDestroy<ListAggState, ListFunction>, nullptr, nullptr);
+	                         ListCombineFunction, ListFinalize, nullptr, ListBindFunction, nullptr, nullptr,
+	                         ListWindow);
 }
 
 } // namespace duckdb

package/src/duckdb/src/core_functions/aggregate/regression/regr_avg.cpp

@@ -37,16 +37,16 @@ struct RegrAvgFunction {
 };
 struct RegrAvgXFunction : RegrAvgFunction {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		state.sum += y;
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		state.sum += x;
 		state.count++;
 	}
 };
 
 struct RegrAvgYFunction : RegrAvgFunction {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		state.sum += x;
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		state.sum += y;
 		state.count++;
 	}
 };

package/src/duckdb/src/core_functions/aggregate/regression/regr_intercept.cpp

@@ -23,11 +23,11 @@ struct RegrInterceptOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
 		state.count++;
-		state.sum_x += y;
-		state.sum_y += x;
-		RegrSlopeOperation::Operation<A_TYPE, B_TYPE, RegrSlopeState, OP>(state.slope, x, y, idata);
+		state.sum_x += x;
+		state.sum_y += y;
+		RegrSlopeOperation::Operation<A_TYPE, B_TYPE, RegrSlopeState, OP>(state.slope, y, x, idata);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/aggregate/regression/regr_r2.cpp

@@ -1,3 +1,4 @@
+// REGR_R2(y, x)
 // Returns the coefficient of determination for non-null pairs in a group.
 // It is computed for non-null pairs using the following formula:
 // null                 if var_pop(x) = 0, else
@@ -24,10 +25,10 @@ struct RegrR2Operation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		CorrOperation::Operation<A_TYPE, B_TYPE, CorrState, OP>(state.corr, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_x, y);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_y, x);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		CorrOperation::Operation<A_TYPE, B_TYPE, CorrState, OP>(state.corr, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_x, x);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop_y, y);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/aggregate/regression/regr_sxx_syy.cpp

@@ -1,6 +1,6 @@
-// regr_sxx
+// REGR_SXX(y, x)
 // Returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs.
-// regrsyy
+// REGR_SYY(y, x)
 // Returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs.
 
 #include "duckdb/core_functions/aggregate/regression/regr_count.hpp"
@@ -48,17 +48,17 @@ struct RegrBaseOperation {
 
 struct RegrSXXOperation : RegrBaseOperation {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, y);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, x);
 	}
 };
 
 struct RegrSYYOperation : RegrBaseOperation {
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, x);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.var_pop, y);
 	}
 };
 

package/src/duckdb/src/core_functions/aggregate/regression/regr_sxy.cpp

@@ -1,3 +1,4 @@
+// REGR_SXY(y, x)
 // Returns REGR_COUNT(expr1, expr2) * COVAR_POP(expr1, expr2) for non-null pairs.
 
 #include "duckdb/core_functions/aggregate/regression/regr_count.hpp"
@@ -20,9 +21,9 @@ struct RegrSXYOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
-		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, x, y, idata);
-		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, x, y, idata);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		RegrCountFunction::Operation<A_TYPE, B_TYPE, size_t, OP>(state.count, y, x, idata);
+		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, y, x, idata);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/core_functions/function_list.cpp

@@ -17,6 +17,7 @@
 #include "duckdb/core_functions/scalar/string_functions.hpp"
 #include "duckdb/core_functions/scalar/struct_functions.hpp"
 #include "duckdb/core_functions/scalar/union_functions.hpp"
+#include "duckdb/core_functions/scalar/debug_functions.hpp"
 
 namespace duckdb {
 
@@ -72,7 +73,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayDistinctFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayFilterFun),
 	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ArrayReverseSortFun),
-	DUCKDB_SCALAR_FUNCTION_ALIAS(ArraySliceFun),
+	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ArraySliceFun),
 	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ArraySortFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayTransformFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ArrayUniqueFun),
@@ -137,7 +138,9 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(EnumRangeFun),
 	DUCKDB_SCALAR_FUNCTION(EnumRangeBoundaryFun),
 	DUCKDB_SCALAR_FUNCTION_SET(EpochFun),
-	DUCKDB_SCALAR_FUNCTION(EpochMsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochMsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochNsFun),
+	DUCKDB_SCALAR_FUNCTION_SET(EpochUsFun),
 	DUCKDB_SCALAR_FUNCTION_SET(EraFun),
 	DUCKDB_SCALAR_FUNCTION(ErrorFun),
 	DUCKDB_SCALAR_FUNCTION(EvenFun),
@@ -179,6 +182,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(JaccardFun),
 	DUCKDB_SCALAR_FUNCTION(JaroSimilarityFun),
 	DUCKDB_SCALAR_FUNCTION(JaroWinklerSimilarityFun),
+	DUCKDB_SCALAR_FUNCTION_SET(JulianDayFun),
 	DUCKDB_AGGREGATE_FUNCTION(KahanSumFun),
 	DUCKDB_AGGREGATE_FUNCTION(KurtosisFun),
 	DUCKDB_SCALAR_FUNCTION_SET(LastDayFun),
@@ -197,7 +201,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION(ListFilterFun),
 	DUCKDB_SCALAR_FUNCTION_ALIAS(ListPackFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ListReverseSortFun),
-	DUCKDB_SCALAR_FUNCTION(ListSliceFun),
+	DUCKDB_SCALAR_FUNCTION_SET(ListSliceFun),
 	DUCKDB_SCALAR_FUNCTION_SET(ListSortFun),
 	DUCKDB_SCALAR_FUNCTION(ListTransformFun),
 	DUCKDB_SCALAR_FUNCTION(ListUniqueFun),
@@ -211,7 +215,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_AGGREGATE_FUNCTION_SET(MadFun),
 	DUCKDB_SCALAR_FUNCTION_SET(MakeDateFun),
 	DUCKDB_SCALAR_FUNCTION(MakeTimeFun),
-	DUCKDB_SCALAR_FUNCTION(MakeTimestampFun),
+	DUCKDB_SCALAR_FUNCTION_SET(MakeTimestampFun),
 	DUCKDB_SCALAR_FUNCTION(MapFun),
 	DUCKDB_SCALAR_FUNCTION(MapConcatFun),
 	DUCKDB_SCALAR_FUNCTION(MapEntriesFun),
@@ -307,6 +311,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_SCALAR_FUNCTION_SET(TimezoneFun),
 	DUCKDB_SCALAR_FUNCTION_SET(TimezoneHourFun),
 	DUCKDB_SCALAR_FUNCTION_SET(TimezoneMinuteFun),
+	DUCKDB_SCALAR_FUNCTION_SET(ToBaseFun),
 	DUCKDB_SCALAR_FUNCTION(ToBase64Fun),
 	DUCKDB_SCALAR_FUNCTION_SET_ALIAS(ToBinaryFun),
 	DUCKDB_SCALAR_FUNCTION(ToDaysFun),
@@ -337,6 +342,7 @@ static StaticFunctionDefinition internal_functions[] = {
 	DUCKDB_AGGREGATE_FUNCTION(VarPopFun),
 	DUCKDB_AGGREGATE_FUNCTION(VarSampFun),
 	DUCKDB_AGGREGATE_FUNCTION_ALIAS(VarianceFun),
+	DUCKDB_SCALAR_FUNCTION(VectorTypeFun),
 	DUCKDB_SCALAR_FUNCTION(VersionFun),
 	DUCKDB_SCALAR_FUNCTION_SET(WeekFun),
 	DUCKDB_SCALAR_FUNCTION_SET(WeekDayFun),

package/src/duckdb/src/core_functions/scalar/date/date_diff.cpp

@@ -304,6 +304,7 @@ static int64_t DifferenceDates(DatePartSpecifier type, TA startdate, TB enddate)
 	case DatePartSpecifier::DOW:
 	case DatePartSpecifier::ISODOW:
 	case DatePartSpecifier::DOY:
+	case DatePartSpecifier::JULIAN_DAY:
 		return DateDiff::DayOperator::template Operation<TA, TB, TR>(startdate, enddate);
 	case DatePartSpecifier::DECADE:
 		return DateDiff::DecadeOperator::template Operation<TA, TB, TR>(startdate, enddate);
@@ -359,6 +360,7 @@ static void DateDiffBinaryExecutor(DatePartSpecifier type, Vector &left, Vector
 	case DatePartSpecifier::DOW:
 	case DatePartSpecifier::ISODOW:
 	case DatePartSpecifier::DOY:
+	case DatePartSpecifier::JULIAN_DAY:
 		DateDiff::BinaryExecute<TA, TB, TR, DateDiff::DayOperator>(left, right, result, count);
 		break;
 	case DatePartSpecifier::DECADE: