duckdb 0.3.1-dev501.0 → 0.3.1-dev517.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.1-dev501.0",
+  "version": "0.3.1-dev517.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -9435,7 +9435,11 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "round_even", {"x", "n", nullptr}, "CASE ((abs(x) * power(10, n+1)) % 10) WHEN 5 THEN round(x/2, n) * 2 ELSE round(x, n) END"},
 	{DEFAULT_SCHEMA, "roundbankers", {"x", "n", nullptr}, "round_even(x, n)"},
 	{DEFAULT_SCHEMA, "nullif", {"a", "b", nullptr}, "CASE WHEN a=b THEN NULL ELSE a END"},
-    {nullptr, nullptr, {nullptr}, nullptr}};
+	{DEFAULT_SCHEMA, "list_append", {"l", "e", nullptr}, "list_concat(l, list_value(e))"},
+	{DEFAULT_SCHEMA, "array_append", {"arr", "el", nullptr}, "list_append(arr, el)"},
+	{DEFAULT_SCHEMA, "list_prepend", {"e", "l", nullptr}, "list_concat(list_value(e), l)"},
+	{DEFAULT_SCHEMA, "array_prepend", {"el", "arr", nullptr}, "list_prepend(el, arr)"},
+	{nullptr, nullptr, {nullptr}, nullptr}};
 
 static unique_ptr<CreateFunctionInfo> GetDefaultFunction(const string &schema, const string &name) {
 	for (idx_t index = 0; internal_macros[index].name != nullptr; index++) {
@@ -35946,20 +35950,30 @@ bool Hugeint::TryCast(hugeint_t input, float &result) {
 	return true;
 }
 
-template <>
-bool Hugeint::TryCast(hugeint_t input, double &result) {
+template <class REAL_T>
+bool CastBigintToFloating(hugeint_t input, REAL_T &result) {
 	switch (input.upper) {
 	case -1:
 		// special case for upper = -1 to avoid rounding issues in small negative numbers
-		result = -double(NumericLimits<uint64_t>::Maximum() - input.lower) - 1;
+		result = -REAL_T(NumericLimits<uint64_t>::Maximum() - input.lower) - 1;
 		break;
 	default:
-		result = double(input.lower) + double(input.upper) * double(NumericLimits<uint64_t>::Maximum());
+		result = REAL_T(input.lower) + REAL_T(input.upper) * REAL_T(NumericLimits<uint64_t>::Maximum());
 		break;
 	}
 	return true;
 }
 
+template <>
+bool Hugeint::TryCast(hugeint_t input, double &result) {
+	return CastBigintToFloating<double>(input, result);
+}
+
+template <>
+bool Hugeint::TryCast(hugeint_t input, long double &result) {
+	return CastBigintToFloating<long double>(input, result);
+}
+
 template <class DST>
 hugeint_t HugeintConvertInteger(DST input) {
 	hugeint_t result;
@@ -36017,8 +36031,8 @@ bool Hugeint::TryConvert(float value, hugeint_t &result) {
 	return Hugeint::TryConvert(double(value), result);
 }
 
-template <>
-bool Hugeint::TryConvert(double value, hugeint_t &result) {
+template <class REAL_T>
+bool ConvertFloatingToBigint(REAL_T value, hugeint_t &result) {
 	if (value <= -170141183460469231731687303715884105728.0 || value >= 170141183460469231731687303715884105727.0) {
 		return false;
 	}
@@ -36026,14 +36040,24 @@ bool Hugeint::TryConvert(double value, hugeint_t &result) {
 	if (negative) {
 		value = -value;
 	}
-	result.lower = (uint64_t)fmod(value, double(NumericLimits<uint64_t>::Maximum()));
-	result.upper = (uint64_t)(value / double(NumericLimits<uint64_t>::Maximum()));
+	result.lower = (uint64_t)fmod(value, REAL_T(NumericLimits<uint64_t>::Maximum()));
+	result.upper = (uint64_t)(value / REAL_T(NumericLimits<uint64_t>::Maximum()));
 	if (negative) {
-		NegateInPlace(result);
+		Hugeint::NegateInPlace(result);
 	}
 	return true;
 }
 
+template <>
+bool Hugeint::TryConvert(double value, hugeint_t &result) {
+	return ConvertFloatingToBigint<double>(value, result);
+}
+
+template <>
+bool Hugeint::TryConvert(long double value, hugeint_t &result) {
+	return ConvertFloatingToBigint<long double>(value, result);
+}
+
 //===--------------------------------------------------------------------===//
 // hugeint_t operators
 //===--------------------------------------------------------------------===//
@@ -70552,6 +70576,45 @@ struct VarianceFun {
 
 namespace duckdb {
 
+static inline void KahanAddInternal(double input, double &summed, double &err) {
+	double diff = input - err;
+	double newval = summed + diff;
+	err = (newval - summed) - diff;
+	summed = newval;
+}
+
+template <class T>
+struct SumState {
+	bool isset;
+	T value;
+
+	void Initialize() {
+		this->isset = false;
+	}
+
+	void Combine(const SumState<T> &other) {
+		this->isset = other.isset || this->isset;
+		this->value += other.value;
+	}
+};
+
+struct KahanSumState {
+	bool isset;
+	double value;
+	double err;
+
+	void Initialize() {
+		this->isset = false;
+		this->err = 0.0;
+	}
+
+	void Combine(const KahanSumState &other) {
+		this->isset = other.isset || this->isset;
+		KahanAddInternal(other.value, this->value, this->err);
+		KahanAddInternal(other.err, this->value, this->err);
+	}
+};
+
 struct RegularAdd {
 	template <class STATE, class T>
 	static void AddNumber(STATE &state, T input) {
@@ -70564,6 +70627,18 @@ struct RegularAdd {
 	}
 };
 
+struct KahanAdd {
+	template <class STATE, class T>
+	static void AddNumber(STATE &state, T input) {
+		KahanAddInternal(input, state.value, state.err);
+	}
+
+	template <class STATE, class T>
+	static void AddConstant(STATE &state, T input, idx_t count) {
+		KahanAddInternal(input * count, state.value, state.err);
+	}
+};
+
 struct HugeintAdd {
 	static void AddValue(hugeint_t &result, uint64_t value, int positive) {
 		// integer summation taken from Tim Gubner et al. - Efficient Query Processing
@@ -70659,8 +70734,34 @@ namespace duckdb {
 
 template <class T>
 struct AvgState {
+	uint64_t count;
 	T value;
+
+	void Initialize() {
+		this->count = 0;
+	}
+
+	void Combine(const AvgState<T> &other) {
+		this->count += other.count;
+		this->value += other.value;
+	}
+};
+
+struct KahanAvgState {
 	uint64_t count;
+	double value;
+	double err;
+
+	void Initialize() {
+		this->count = 0;
+		this->err = 0.0;
+	}
+
+	void Combine(const KahanAvgState &other) {
+		this->count += other.count;
+		KahanAddInternal(other.value, this->value, this->err);
+		KahanAddInternal(other.err, this->value, this->err);
+	}
 };
 
 struct AverageDecimalBindData : public FunctionData {
@@ -70678,12 +70779,11 @@ public:
 struct AverageSetOperation {
 	template <class STATE>
 	static void Initialize(STATE *state) {
-		state->count = 0;
+		state->Initialize();
 	}
 	template <class STATE>
 	static void Combine(const STATE &source, STATE *target) {
-		target->count += source.count;
-		target->value += source.value;
+		target->Combine(source);
 	}
 	template <class STATE>
 	static void AddValues(STATE *state, idx_t count) {
@@ -70691,8 +70791,9 @@ struct AverageSetOperation {
 	}
 };
 
-static double GetAverageDivident(uint64_t count, FunctionData *bind_data) {
-	double divident = double(count);
+template <class T>
+static T GetAverageDivident(uint64_t count, FunctionData *bind_data) {
+	T divident = T(count);
 	if (bind_data) {
 		auto &avg_bind_data = (AverageDecimalBindData &)*bind_data;
 		divident *= avg_bind_data.scale;
@@ -70707,7 +70808,7 @@ struct IntegerAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			double divident = GetAverageDivident(state->count, bind_data);
+			double divident = GetAverageDivident<double>(state->count, bind_data);
 			target[idx] = double(state->value) / divident;
 		}
 	}
@@ -70720,8 +70821,8 @@ struct IntegerAverageOperationHugeint : public BaseSumOperation<AverageSetOperat
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			double divident = GetAverageDivident(state->count, bind_data);
-			target[idx] = Hugeint::Cast<double>(state->value) / divident;
+			long double divident = GetAverageDivident<long double>(state->count, bind_data);
+			target[idx] = Hugeint::Cast<long double>(state->value) / divident;
 		}
 	}
 };
@@ -70733,8 +70834,8 @@ struct HugeintAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			double divident = GetAverageDivident(state->count, bind_data);
-			target[idx] = Hugeint::Cast<double>(state->value) / divident;
+			long double divident = GetAverageDivident<long double>(state->count, bind_data);
+			target[idx] = Hugeint::Cast<long double>(state->value) / divident;
 		}
 	}
 };
@@ -70748,7 +70849,21 @@ struct NumericAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 			if (!Value::DoubleIsValid(state->value)) {
 				throw OutOfRangeException("AVG is out of range!");
 			}
-			target[idx] = state->value / state->count;
+			target[idx] = (state->value / state->count);
+		}
+	}
+};
+
+struct KahanAverageOperation : public BaseSumOperation<AverageSetOperation, KahanAdd> {
+	template <class T, class STATE>
+	static void Finalize(Vector &result, FunctionData *, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
+		if (state->count == 0) {
+			mask.SetInvalid(idx);
+		} else {
+			if (!Value::DoubleIsValid(state->value)) {
+				throw OutOfRangeException("AVG is out of range!");
+			}
+			target[idx] = (state->value / state->count) + (state->err / state->count);
 		}
 	}
 };
@@ -70800,6 +70915,11 @@ void AvgFun::RegisterFunction(BuiltinFunctions &set) {
 	avg.AddFunction(AggregateFunction::UnaryAggregate<AvgState<double>, double, double, NumericAverageOperation>(
 	    LogicalType::DOUBLE, LogicalType::DOUBLE));
 	set.AddFunction(avg);
+
+	AggregateFunctionSet favg("favg");
+	favg.AddFunction(AggregateFunction::UnaryAggregate<KahanAvgState, double, double, KahanAverageOperation>(
+	    LogicalType::DOUBLE, LogicalType::DOUBLE));
+	set.AddFunction(favg);
 }
 
 } // namespace duckdb
@@ -73480,21 +73600,14 @@ void StringAggFun::RegisterFunction(BuiltinFunctions &set) {
 
 namespace duckdb {
 
-template <class T>
-struct SumState {
-	T value;
-	bool isset;
-};
-
 struct SumSetOperation {
 	template <class STATE>
 	static void Initialize(STATE *state) {
-		state->isset = false;
+		state->Initialize();
 	}
 	template <class STATE>
 	static void Combine(const STATE &source, STATE *target) {
-		target->isset = source.isset || target->isset;
-		target->value += source.value;
+		target->Combine(source);
 	}
 	template <class STATE>
 	static void AddValues(STATE *state, idx_t count) {
@@ -73524,7 +73637,8 @@ struct SumToHugeintOperation : public BaseSumOperation<SumSetOperation, HugeintA
 	}
 };
 
-struct NumericSumOperation : public BaseSumOperation<SumSetOperation, RegularAdd> {
+template <class ADD_OPERATOR>
+struct DoubleSumOperation : public BaseSumOperation<SumSetOperation, ADD_OPERATOR> {
 	template <class T, class STATE>
 	static void Finalize(Vector &result, FunctionData *, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
 		if (!state->isset) {
@@ -73538,6 +73652,9 @@ struct NumericSumOperation : public BaseSumOperation<SumSetOperation, RegularAdd
 	}
 };
 
+using NumericSumOperation = DoubleSumOperation<RegularAdd>;
+using KahanSumOperation = DoubleSumOperation<KahanAdd>;
+
 struct HugeintSumOperation : public BaseSumOperation<SumSetOperation, RegularAdd> {
 	template <class T, class STATE>
 	static void Finalize(Vector &result, FunctionData *, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
@@ -73646,12 +73763,23 @@ void SumFun::RegisterFunction(BuiltinFunctions &set) {
 	sum.AddFunction(GetSumAggregate(PhysicalType::INT32));
 	sum.AddFunction(GetSumAggregate(PhysicalType::INT64));
 	sum.AddFunction(GetSumAggregate(PhysicalType::INT128));
-	// float sums to float
-	// FIXME: implement http://ic.ese.upenn.edu/pdf/parallel_fpaccum_tc2016.pdf for parallel FP sums
 	sum.AddFunction(AggregateFunction::UnaryAggregate<SumState<double>, double, double, NumericSumOperation>(
 	    LogicalType::DOUBLE, LogicalType::DOUBLE));
 
 	set.AddFunction(sum);
+
+	// fsum
+	AggregateFunctionSet fsum("fsum");
+	fsum.AddFunction(AggregateFunction::UnaryAggregate<KahanSumState, double, double, KahanSumOperation>(
+	    LogicalType::DOUBLE, LogicalType::DOUBLE));
+
+	set.AddFunction(fsum);
+
+	fsum.name = "kahan_sum";
+	set.AddFunction(fsum);
+
+	fsum.name = "sumKahan";
+	set.AddFunction(fsum);
 }
 
 } // namespace duckdb
@@ -74963,14 +75091,12 @@ struct QuantileState {
 	// Regular aggregation
 	std::vector<SaveType> v;
 
-	// Windowed Aggregation
+	// Windowed Quantile indirection
 	std::vector<idx_t> w;
 	idx_t pos;
 
-	// List temporaries
-	std::vector<idx_t> disturbed;
-	std::vector<idx_t> lower;
-	std::vector<idx_t> upper;
+	// Windowed MAD indirection
+	std::vector<idx_t> m;
 
 	QuantileState() : pos(0) {
 	}
@@ -75052,33 +75178,28 @@ static idx_t ReplaceIndex(idx_t *index, const FrameBounds &frame, const FrameBou
 }
 
 template <class INPUT_TYPE>
-static inline bool CanReplace(const idx_t *index, const INPUT_TYPE *fdata, const idx_t j, const idx_t k0,
-                              const idx_t k1, const QuantileNotNull &validity) {
-	auto replace = false;
-
+static inline int CanReplace(const idx_t *index, const INPUT_TYPE *fdata, const idx_t j, const idx_t k0, const idx_t k1,
+                             const QuantileNotNull &validity) {
 	D_ASSERT(index);
 
-	// NULLs can replace NULLs
-	// and NULLs are at the end, so we only have to check the first one
+	// NULLs sort to the end, so if we have inserted a NULL,
+	// it must be past the end of the quantile to be replaceable.
+	// Note that the quantile values are never NULL.
 	const auto ij = index[j];
-	const auto ik0 = index[k0];
-	const auto ik1 = index[k1];
-	const auto nullj = !validity(ij);
-	const auto nullk0 = !validity(ik0);
-	if (nullj || nullk0) {
-		return nullj == nullk0;
+	if (!validity(ij)) {
+		return k1 < j ? 1 : 0;
 	}
 
 	auto curr = fdata[ij];
 	if (k1 < j) {
-		auto hi = fdata[ik0];
-		replace = hi < curr;
+		auto hi = fdata[index[k0]];
+		return hi < curr ? 1 : 0;
 	} else if (j < k0) {
-		auto lo = fdata[ik1];
-		replace = curr < lo;
+		auto lo = fdata[index[k1]];
+		return curr < lo ? -1 : 0;
 	}
 
-	return replace;
+	return 0;
 }
 
 template <class INPUT_TYPE>
@@ -75195,7 +75316,8 @@ struct QuantileLess {
 // Continuous interpolation
 template <bool DISCRETE>
 struct Interpolator {
-	Interpolator(const double q, const idx_t n_p) : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(ceil(RN)) {
+	Interpolator(const double q, const idx_t n_p)
+	    : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(ceil(RN)), begin(0), end(n_p) {
 	}
 
 	template <class INPUT_TYPE, class TARGET_TYPE, typename ACCESSOR = QuantileDirect<INPUT_TYPE>>
@@ -75203,11 +75325,11 @@ struct Interpolator {
 		using ACCESS_TYPE = typename ACCESSOR::RESULT_TYPE;
 		QuantileLess<ACCESSOR> comp(accessor);
 		if (CRN == FRN) {
-			std::nth_element(v_t, v_t + FRN, v_t + n, comp);
+			std::nth_element(v_t + begin, v_t + FRN, v_t + end, comp);
 			return CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[FRN]), result);
 		} else {
-			std::nth_element(v_t, v_t + FRN, v_t + n, comp);
-			std::nth_element(v_t + FRN, v_t + CRN, v_t + n, comp);
+			std::nth_element(v_t + begin, v_t + FRN, v_t + end, comp);
+			std::nth_element(v_t + FRN, v_t + CRN, v_t + end, comp);
 			auto lo = CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[FRN]), result);
 			auto hi = CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[CRN]), result);
 			return CastInterpolation::Interpolate<TARGET_TYPE>(lo, RN - FRN, hi);
@@ -75230,19 +75352,23 @@ struct Interpolator {
 	const double RN;
 	const idx_t FRN;
 	const idx_t CRN;
+
+	idx_t begin;
+	idx_t end;
 };
 
 // Discrete "interpolation"
 template <>
 struct Interpolator<true> {
-	Interpolator(const double q, const idx_t n_p) : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(FRN) {
+	Interpolator(const double q, const idx_t n_p)
+	    : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(FRN), begin(0), end(n_p) {
 	}
 
 	template <class INPUT_TYPE, class TARGET_TYPE, typename ACCESSOR = QuantileDirect<INPUT_TYPE>>
 	TARGET_TYPE Operation(INPUT_TYPE *v_t, Vector &result, const ACCESSOR &accessor = ACCESSOR()) const {
 		using ACCESS_TYPE = typename ACCESSOR::RESULT_TYPE;
 		QuantileLess<ACCESSOR> comp(accessor);
-		std::nth_element(v_t, v_t + FRN, v_t + n, comp);
+		std::nth_element(v_t + begin, v_t + FRN, v_t + end, comp);
 		return CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[FRN]), result);
 	}
 
@@ -75256,6 +75382,9 @@ struct Interpolator<true> {
 	const double RN;
 	const idx_t FRN;
 	const idx_t CRN;
+
+	idx_t begin;
+	idx_t end;
 };
 
 struct QuantileBindData : public FunctionData {
@@ -75328,7 +75457,9 @@ static void ExecuteListFinalize(Vector &states, FunctionData *bind_data_p, Vecto
 
 	D_ASSERT(bind_data_p);
 	auto bind_data = (QuantileBindData *)bind_data_p;
-	ListVector::SetListSize(result, 0);
+	if (offset == 0) {
+		ListVector::SetListSize(result, 0);
+	}
 
 	if (states.GetVectorType() == VectorType::CONSTANT_VECTOR) {
 		result.SetVectorType(VectorType::CONSTANT_VECTOR);
@@ -75406,7 +75537,7 @@ struct QuantileScalarOperation : public QuantileOperation {
 		if (frame.first == prev.first + 1 && frame.second == prev.second + 1) {
 			//  Fixed frame size
 			const auto j = ReplaceIndex(index, frame, prev);
-			//	We can only replace if the number of NULls has not changed
+			//	We can only replace if the number of NULLs has not changed
 			if (dmask.AllValid() || not_null(prev.first) == not_null(prev.second)) {
 				Interpolator<DISCRETE> interp(q, prev_pos);
 				replace = CanReplace(index, data, j, interp.FRN, interp.CRN, not_null);
@@ -75515,15 +75646,19 @@ struct QuantileListOperation : public QuantileOperation {
 		auto v_t = state->v.data();
 		D_ASSERT(v_t);
 
-		target[idx].offset = ridx;
-		for (const auto &quantile : bind_data->quantiles) {
+		auto &entry = target[idx];
+		entry.offset = ridx;
+		idx_t lower = 0;
+		for (const auto &q : bind_data->order) {
+			const auto &quantile = bind_data->quantiles[q];
 			Interpolator<DISCRETE> interp(quantile, state->v.size());
-			rdata[ridx] = interp.template Operation<typename STATE::SaveType, CHILD_TYPE>(v_t, result);
-			++ridx;
+			interp.begin = lower;
+			rdata[ridx + q] = interp.template Operation<typename STATE::SaveType, CHILD_TYPE>(v_t, result);
+			lower = interp.FRN;
 		}
-		target[idx].length = bind_data->quantiles.size();
+		entry.length = bind_data->quantiles.size();
 
-		ListVector::SetListSize(result_list, ridx);
+		ListVector::SetListSize(result_list, entry.offset + entry.length);
 	}
 
 	template <class STATE, class INPUT_TYPE, class RESULT_TYPE>
@@ -75547,57 +75682,75 @@ struct QuantileListOperation : public QuantileOperation {
 		auto rdata = FlatVector::GetData<CHILD_TYPE>(result);
 
 		//  Lazily initialise frame state
-		const auto prev_valid = state->w.size() == (prev.second - prev.first);
+		auto prev_pos = state->pos;
 		state->SetPos(frame.second - frame.first);
 
 		auto index = state->w.data();
 
-		bool fixed = false;
-		auto j = state->pos;
-		if (prev_valid && dmask.AllValid() && frame.first == prev.first + 1 && frame.second == prev.second + 1) {
+		// We can generalise replacement for quantile lists by observing that when a replacement is
+		// valid for a single quantile, it is valid for all quantiles greater/less than that quantile
+		// based on whether the insertion is below/above the quantile location.
+		// So if a replaced index in an IQR is located between Q25 and Q50, but has a value below Q25,
+		// then Q25 must be recomputed, but Q50 and Q75 are unaffected.
+		// For a single element list, this reduces to the scalar case.
+		std::pair<idx_t, idx_t> replaceable {state->pos, 0};
+		if (frame.first == prev.first + 1 && frame.second == prev.second + 1) {
 			//  Fixed frame size
-			j = ReplaceIndex(index, frame, prev);
-			fixed = true;
+			const auto j = ReplaceIndex(index, frame, prev);
+			//	We can only replace if the number of NULLs has not changed
+			if (dmask.AllValid() || not_null(prev.first) == not_null(prev.second)) {
+				for (const auto &q : bind_data->order) {
+					const auto &quantile = bind_data->quantiles[q];
+					Interpolator<DISCRETE> interp(quantile, prev_pos);
+					const auto replace = CanReplace(index, data, j, interp.FRN, interp.CRN, not_null);
+					if (replace < 0) {
+						//	Replacement is before this quantile, so the rest will be replaceable too.
+						replaceable.first = MinValue(replaceable.first, interp.FRN);
+						replaceable.second = prev_pos;
+						break;
+					} else if (replace > 0) {
+						//	Replacement is after this quantile, so everything before it is replaceable too.
+						replaceable.first = 0;
+						replaceable.second = MaxValue(replaceable.second, interp.CRN);
+					}
+				}
+				if (replaceable.first < replaceable.second) {
+					state->pos = prev_pos;
+				}
+			}
 		} else {
 			ReuseIndexes(index, frame, prev);
-			if (!dmask.AllValid()) {
-				state->pos = std::partition(index, index + state->pos, not_null) - index;
-			}
 		}
 
-		if (!state->pos) {
-			lmask.Set(lidx, false);
-			return;
+		if (replaceable.first >= replaceable.second && !dmask.AllValid()) {
+			// Remove the NULLs
+			state->pos = std::partition(index, index + state->pos, not_null) - index;
 		}
 
-		// First pass: Fill in the undisturbed values and find the islands of stability.
-		QuantileIndirect<INPUT_TYPE> indirect(data);
-		state->disturbed.clear();
-		state->lower.clear();
-		state->upper.clear();
-		idx_t lb = 0;
-		for (idx_t i = 0; i < bind_data->order.size(); ++i) {
-			const auto q = bind_data->order[i];
-			const auto &quantile = bind_data->quantiles[q];
-			Interpolator<DISCRETE> interp(quantile, state->pos);
-
-			if (fixed && CanReplace(index, data, j, interp.FRN, interp.CRN, not_null)) {
-				rdata[lentry.offset + q] = interp.template Replace<idx_t, CHILD_TYPE>(index, result, indirect);
-				state->upper.resize(state->lower.size(), interp.FRN);
-			} else {
-				state->disturbed.push_back(q);
-				state->lower.push_back(MinValue(lb, interp.FRN));
+		if (state->pos) {
+			using ID = QuantileIndirect<INPUT_TYPE>;
+			ID indirect(data);
+			for (const auto &q : bind_data->order) {
+				const auto &quantile = bind_data->quantiles[q];
+				Interpolator<DISCRETE> interp(quantile, state->pos);
+				if (replaceable.first <= interp.FRN && interp.CRN <= replaceable.second) {
+					rdata[lentry.offset + q] = interp.template Replace<idx_t, CHILD_TYPE, ID>(index, result, indirect);
+				} else {
+					// Make sure we don't disturb any replacements
+					if (replaceable.first < replaceable.second) {
+						if (interp.FRN < replaceable.first) {
+							interp.end = replaceable.first;
+						}
+						if (replaceable.second < interp.CRN) {
+							interp.begin = replaceable.second;
+						}
+					}
+					rdata[lentry.offset + q] =
+					    interp.template Operation<idx_t, CHILD_TYPE, ID>(index, result, indirect);
+				}
 			}
-			lb = interp.CRN + 1;
-		}
-		state->upper.resize(state->lower.size(), state->pos);
-
-		// Second pass: select the disturbed values
-		for (idx_t i = 0; i < state->disturbed.size(); ++i) {
-			const auto &q = state->disturbed[i];
-			const auto &quantile = bind_data->quantiles[q];
-			Interpolator<DISCRETE> interp(quantile, state->pos);
-			rdata[lentry.offset + q] = interp.template Operation<idx_t, CHILD_TYPE>(index, result, indirect);
+		} else {
+			lmask.Set(lidx, false);
 		}
 	}
 };
@@ -75877,12 +76030,11 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 		D_ASSERT(index);
 
 		// We need a second index for the second pass.
-		// so repurpose disturbed as the second index.
-		if (state->pos > state->disturbed.size()) {
-			state->disturbed.resize(state->pos);
+		if (state->pos > state->m.size()) {
+			state->m.resize(state->pos);
 		}
 
-		auto index2 = state->disturbed.data();
+		auto index2 = state->m.data();
 		D_ASSERT(index2);
 
 		// The replacement trick does not work on the second index because if
@@ -76475,9 +76627,16 @@ struct MapFun {
 struct MapExtractFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
+
 struct ListExtractFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
+
+struct ListConcatFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct CardinalityFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -83453,11 +83612,6 @@ void ArraySliceFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
-
-
-
-
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -83497,6 +83651,147 @@ public:
 
 
 
+namespace duckdb {
+
+static void ListConcatFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 2);
+	auto count = args.size();
+
+	Vector &lhs = args.data[0];
+	Vector &rhs = args.data[1];
+	if (lhs.GetType().id() == LogicalTypeId::SQLNULL) {
+		result.Reference(rhs);
+		return;
+	}
+	if (rhs.GetType().id() == LogicalTypeId::SQLNULL) {
+		result.Reference(lhs);
+		return;
+	}
+
+	VectorData lhs_data;
+	VectorData rhs_data;
+	lhs.Orrify(count, lhs_data);
+	rhs.Orrify(count, rhs_data);
+	auto lhs_entries = (list_entry_t *)lhs_data.data;
+	auto rhs_entries = (list_entry_t *)rhs_data.data;
+
+	auto lhs_list_size = ListVector::GetListSize(lhs);
+	auto rhs_list_size = ListVector::GetListSize(rhs);
+	auto &lhs_child = ListVector::GetEntry(lhs);
+	auto &rhs_child = ListVector::GetEntry(rhs);
+	VectorData lhs_child_data;
+	VectorData rhs_child_data;
+	lhs_child.Orrify(lhs_list_size, lhs_child_data);
+	rhs_child.Orrify(rhs_list_size, rhs_child_data);
+
+	result.SetVectorType(VectorType::FLAT_VECTOR);
+	auto result_entries = FlatVector::GetData<list_entry_t>(result);
+	auto &result_validity = FlatVector::Validity(result);
+
+	idx_t offset = 0;
+	for (idx_t i = 0; i < count; i++) {
+		auto lhs_list_index = lhs_data.sel->get_index(i);
+		auto rhs_list_index = rhs_data.sel->get_index(i);
+		if (!lhs_data.validity.RowIsValid(lhs_list_index) && !rhs_data.validity.RowIsValid(rhs_list_index)) {
+			result_validity.SetInvalid(i);
+			continue;
+		}
+		result_entries[i].offset = offset;
+		result_entries[i].length = 0;
+		if (lhs_data.validity.RowIsValid(lhs_list_index)) {
+			const auto &lhs_entry = lhs_entries[lhs_list_index];
+			result_entries[i].length += lhs_entry.length;
+			ListVector::Append(result, lhs_child, *lhs_child_data.sel, lhs_entry.offset + lhs_entry.length,
+			                   lhs_entry.offset);
+		}
+		if (rhs_data.validity.RowIsValid(rhs_list_index)) {
+			const auto &rhs_entry = rhs_entries[rhs_list_index];
+			result_entries[i].length += rhs_entry.length;
+			ListVector::Append(result, rhs_child, *rhs_child_data.sel, rhs_entry.offset + rhs_entry.length,
+			                   rhs_entry.offset);
+		}
+		offset += result_entries[i].length;
+	}
+	D_ASSERT(ListVector::GetListSize(result) == offset);
+
+	if (lhs.GetVectorType() == VectorType::CONSTANT_VECTOR && rhs.GetVectorType() == VectorType::CONSTANT_VECTOR) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+static unique_ptr<FunctionData> ListConcatBind(ClientContext &context, ScalarFunction &bound_function,
+                                               vector<unique_ptr<Expression>> &arguments) {
+	D_ASSERT(bound_function.arguments.size() == 2);
+
+	auto &lhs = arguments[0]->return_type;
+	auto &rhs = arguments[1]->return_type;
+	if (lhs.id() == LogicalTypeId::SQLNULL && rhs.id() == LogicalTypeId::SQLNULL) {
+		bound_function.return_type = LogicalType::SQLNULL;
+	} else if (lhs.id() == LogicalTypeId::SQLNULL || rhs.id() == LogicalTypeId::SQLNULL) {
+		// we mimic postgres behaviour: list_concat(NULL, my_list) = my_list
+		bound_function.arguments[0] = lhs;
+		bound_function.arguments[1] = rhs;
+		bound_function.return_type = rhs.id() == LogicalTypeId::SQLNULL ? lhs : rhs;
+	} else {
+		D_ASSERT(lhs.id() == LogicalTypeId::LIST);
+		D_ASSERT(rhs.id() == LogicalTypeId::LIST);
+
+		// Resolve list type
+		auto child_type = LogicalType::SQLNULL;
+		for (const auto &argument : arguments) {
+			child_type = LogicalType::MaxLogicalType(child_type, ListType::GetChildType(argument->return_type));
+		}
+		ExpressionBinder::ResolveParameterType(child_type);
+		auto list_type = LogicalType::LIST(move(child_type));
+
+		bound_function.arguments[0] = list_type;
+		bound_function.arguments[1] = list_type;
+		bound_function.return_type = list_type;
+	}
+	return make_unique<VariableReturnBindData>(bound_function.return_type);
+}
+
+static unique_ptr<BaseStatistics> ListConcatStats(ClientContext &context, BoundFunctionExpression &expr,
+                                                  FunctionData *bind_data,
+                                                  vector<unique_ptr<BaseStatistics>> &child_stats) {
+	D_ASSERT(child_stats.size() == 2);
+	if (!child_stats[0] || !child_stats[1]) {
+		return nullptr;
+	}
+
+	auto &left_stats = (ListStatistics &)*child_stats[0];
+	auto &right_stats = (ListStatistics &)*child_stats[1];
+
+	auto stats = left_stats.Copy();
+	stats->Merge(right_stats);
+
+	return stats;
+}
+
+ScalarFunction ListConcatFun::GetFunction() {
+	// the arguments and return types are actually set in the binder function
+	return ScalarFunction({LogicalType::LIST(LogicalType::ANY), LogicalType::LIST(LogicalType::ANY)},
+	                      LogicalType::LIST(LogicalType::ANY), ListConcatFunction, false, ListConcatBind, nullptr,
+	                      ListConcatStats);
+}
+
+void ListConcatFun::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction({"list_concat", "list_cat", "array_concat", "array_cat"}, GetFunction());
+}
+
+} // namespace duckdb
+
+
+
+
+
+
+
+
+
+
+
+
 namespace duckdb {
 
 template <class T, bool HEAP_REF = false, bool VALIDITY_ONLY = false>
@@ -85465,6 +85760,7 @@ void BuiltinFunctions::RegisterNestedFunctions() {
 	Register<ArraySliceFun>();
 	Register<StructPackFun>();
 	Register<StructExtractFun>();
+	Register<ListConcatFun>();
 	Register<ListValueFun>();
 	Register<ListExtractFun>();
 	Register<ListRangeFun>();
@@ -85811,6 +86107,7 @@ struct BitwiseNotFun {
 
 
 
+
 #include <limits>
 
 namespace duckdb {
@@ -86126,6 +86423,8 @@ void AddFun::RegisterFunction(BuiltinFunctions &set) {
 
 	functions.AddFunction(GetFunction(LogicalType::TIMESTAMP, LogicalType::INTERVAL));
 	functions.AddFunction(GetFunction(LogicalType::INTERVAL, LogicalType::TIMESTAMP));
+	// we can add lists together
+	functions.AddFunction(ListConcatFun::GetFunction());
 
 	set.AddFunction(functions);
 }
@@ -87708,6 +88007,7 @@ void CHR::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 #include <string.h>
 
 namespace duckdb {
@@ -87953,6 +88253,7 @@ void ConcatFun::RegisterFunction(BuiltinFunctions &set) {
 	concat_op.AddFunction(
 	    ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, ConcatOperator));
 	concat_op.AddFunction(ScalarFunction({LogicalType::BLOB, LogicalType::BLOB}, LogicalType::BLOB, ConcatOperator));
+	concat_op.AddFunction(ListConcatFun::GetFunction());
 	set.AddFunction(concat_op);
 
 	ScalarFunction concat_ws = ScalarFunction("concat_ws", {LogicalType::VARCHAR, LogicalType::VARCHAR},

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "fbeb407d4"
-#define DUCKDB_VERSION "0.3.1-dev501"
+#define DUCKDB_SOURCE_ID "9fd34ad70"
+#define DUCKDB_VERSION "0.3.1-dev517"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -5957,6 +5957,8 @@ template <>
 bool Hugeint::TryCast(hugeint_t input, float &result);
 template <>
 bool Hugeint::TryCast(hugeint_t input, double &result);
+template <>
+bool Hugeint::TryCast(hugeint_t input, long double &result);
 
 template <>
 bool Hugeint::TryConvert(int8_t value, hugeint_t &result);
@@ -5978,6 +5980,8 @@ template <>
 bool Hugeint::TryConvert(float value, hugeint_t &result);
 template <>
 bool Hugeint::TryConvert(double value, hugeint_t &result);
+template <>
+bool Hugeint::TryConvert(long double value, hugeint_t &result);
 
 } // namespace duckdb