duckdb 0.3.1-dev501.0 → 0.3.1-dev532.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.1-dev501.0",
+  "version": "0.3.1-dev532.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -9435,7 +9435,11 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "round_even", {"x", "n", nullptr}, "CASE ((abs(x) * power(10, n+1)) % 10) WHEN 5 THEN round(x/2, n) * 2 ELSE round(x, n) END"},
 	{DEFAULT_SCHEMA, "roundbankers", {"x", "n", nullptr}, "round_even(x, n)"},
 	{DEFAULT_SCHEMA, "nullif", {"a", "b", nullptr}, "CASE WHEN a=b THEN NULL ELSE a END"},
-    {nullptr, nullptr, {nullptr}, nullptr}};
+	{DEFAULT_SCHEMA, "list_append", {"l", "e", nullptr}, "list_concat(l, list_value(e))"},
+	{DEFAULT_SCHEMA, "array_append", {"arr", "el", nullptr}, "list_append(arr, el)"},
+	{DEFAULT_SCHEMA, "list_prepend", {"e", "l", nullptr}, "list_concat(list_value(e), l)"},
+	{DEFAULT_SCHEMA, "array_prepend", {"el", "arr", nullptr}, "list_prepend(el, arr)"},
+	{nullptr, nullptr, {nullptr}, nullptr}};
 
 static unique_ptr<CreateFunctionInfo> GetDefaultFunction(const string &schema, const string &name) {
 	for (idx_t index = 0; internal_macros[index].name != nullptr; index++) {
@@ -9897,10 +9901,11 @@ public:
 
 class ResultArrowArrayStreamWrapper {
 public:
-	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result);
+	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result, idx_t approx_batch_size);
 	ArrowArrayStream stream;
 	unique_ptr<QueryResult> result;
 	std::string last_error;
+	idx_t vectors_per_chunk;
 
 private:
 	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);
@@ -10017,13 +10022,21 @@ int ResultArrowArrayStreamWrapper::MyStreamGetNext(struct ArrowArrayStream *stre
 			return -1;
 		}
 	}
-	auto data_chunk = result.Fetch();
-	if (!data_chunk) {
-		//! Nothing to output
+	unique_ptr<DataChunk> chunk_result = result.Fetch();
+	if (!chunk_result) {
+		// Nothing to output
 		out->release = nullptr;
 		return 0;
 	}
-	data_chunk->ToArrowArray(out);
+	for (idx_t i = 1; i < my_stream->vectors_per_chunk; i++) {
+		auto new_chunk = result.Fetch();
+		if (!new_chunk) {
+			break;
+		} else {
+			chunk_result->Append(*new_chunk, true);
+		}
+	}
+	chunk_result->ToArrowArray(out);
 	return 0;
 }
 
@@ -10043,11 +10056,15 @@ const char *ResultArrowArrayStreamWrapper::MyStreamGetLastError(struct ArrowArra
 	auto my_stream = (ResultArrowArrayStreamWrapper *)stream->private_data;
 	return my_stream->last_error.c_str();
 }
-ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result_p)
+ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result_p, idx_t approx_batch_size)
     : result(move(result_p)) {
 	//! We first initialize the private data of the stream
 	stream.private_data = this;
-
+	//! Ceil Approx_Batch_Size/STANDARD_VECTOR_SIZE
+	if (approx_batch_size == 0) {
+		throw std::runtime_error("Approximate Batch Size of Record Batch MUST be higher than 0");
+	}
+	vectors_per_chunk = (approx_batch_size + STANDARD_VECTOR_SIZE - 1) / STANDARD_VECTOR_SIZE;
 	//! We initialize the stream functions
 	stream.get_schema = ResultArrowArrayStreamWrapper::MyStreamGetSchema;
 	stream.get_next = ResultArrowArrayStreamWrapper::MyStreamGetNext;
@@ -29102,6 +29119,13 @@ namespace duckdb {
 class RowLayout;
 struct LocalSortState;
 
+struct SortConstants {
+	static constexpr idx_t VALUES_PER_RADIX = 256;
+	static constexpr idx_t MSD_RADIX_LOCATIONS = VALUES_PER_RADIX + 1;
+	static constexpr idx_t INSERTION_SORT_THRESHOLD = 24;
+	static constexpr idx_t MSD_RADIX_SORT_SIZE_THRESHOLD = 4;
+};
+
 struct SortLayout {
 public:
 	explicit SortLayout(const vector<BoundOrderByNode> &orders);
@@ -30403,12 +30427,12 @@ static void ComputeTies(data_ptr_t dataptr, const idx_t &count, const idx_t &col
 }
 
 //! Textbook LSD radix sort
-static void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count,
-                      const idx_t &col_offset, const idx_t &sorting_size, const SortLayout &sort_layout) {
-	auto temp_block = buffer_manager.Allocate(MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
+void RadixSortLSD(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count, const idx_t &col_offset,
+                  const idx_t &row_width, const idx_t &sorting_size) {
+	auto temp_block = buffer_manager.Allocate(MaxValue(count * row_width, (idx_t)Storage::BLOCK_SIZE));
 	bool swap = false;
 
-	idx_t counts[256];
+	idx_t counts[SortConstants::VALUES_PER_RADIX];
 	for (idx_t r = 1; r <= sorting_size; r++) {
 		// Init counts to 0
 		memset(counts, 0, sizeof(counts));
@@ -30420,24 +30444,129 @@ static void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr,
 		data_ptr_t offset_ptr = source_ptr + offset;
 		for (idx_t i = 0; i < count; i++) {
 			counts[*offset_ptr]++;
-			offset_ptr += sort_layout.entry_size;
+			offset_ptr += row_width;
 		}
 		// Compute offsets from counts
-		for (idx_t val = 1; val < 256; val++) {
+		idx_t max_count = counts[0];
+		for (idx_t val = 1; val < SortConstants::VALUES_PER_RADIX; val++) {
+			max_count = MaxValue<idx_t>(max_count, counts[val]);
 			counts[val] = counts[val] + counts[val - 1];
 		}
+		if (max_count == count) {
+			continue;
+		}
 		// Re-order the data in temporary array
-		data_ptr_t row_ptr = source_ptr + (count - 1) * sort_layout.entry_size;
+		data_ptr_t row_ptr = source_ptr + (count - 1) * row_width;
 		for (idx_t i = 0; i < count; i++) {
 			idx_t &radix_offset = --counts[*(row_ptr + offset)];
-			memcpy(target_ptr + radix_offset * sort_layout.entry_size, row_ptr, sort_layout.entry_size);
-			row_ptr -= sort_layout.entry_size;
+			memcpy(target_ptr + radix_offset * row_width, row_ptr, row_width);
+			row_ptr -= row_width;
 		}
 		swap = !swap;
 	}
 	// Move data back to original buffer (if it was swapped)
 	if (swap) {
-		memcpy(dataptr, temp_block->Ptr(), count * sort_layout.entry_size);
+		memcpy(dataptr, temp_block->Ptr(), count * row_width);
+	}
+}
+
+//! Insertion sort, used when count of values is low
+inline void InsertionSort(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr, const idx_t &count,
+                          const idx_t &col_offset, const idx_t &row_width, const idx_t &total_comp_width,
+                          const idx_t &offset, bool swap) {
+	const data_ptr_t source_ptr = swap ? temp_ptr : orig_ptr;
+	const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr;
+	if (count > 1) {
+		const idx_t total_offset = col_offset + offset;
+		auto temp_val = unique_ptr<data_t[]>(new data_t[row_width]);
+		const data_ptr_t val = temp_val.get();
+		const auto comp_width = total_comp_width - offset;
+		for (idx_t i = 1; i < count; i++) {
+			memcpy(val, source_ptr + i * row_width, row_width);
+			idx_t j = i;
+			while (j > 0 &&
+			       memcmp(source_ptr + (j - 1) * row_width + total_offset, val + total_offset, comp_width) > 0) {
+				memcpy(source_ptr + j * row_width, source_ptr + (j - 1) * row_width, row_width);
+				j--;
+			}
+			memcpy(source_ptr + j * row_width, val, row_width);
+		}
+	}
+	if (swap) {
+		memcpy(target_ptr, source_ptr, count * row_width);
+	}
+}
+
+//! MSD radix sort that switches to insertion sort with low bucket sizes
+void RadixSortMSD(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr, const idx_t &count, const idx_t &col_offset,
+                  const idx_t &row_width, const idx_t &comp_width, const idx_t &offset, idx_t locations[], bool swap) {
+	const data_ptr_t source_ptr = swap ? temp_ptr : orig_ptr;
+	const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr;
+	// Init counts to 0
+	memset(locations, 0, SortConstants::MSD_RADIX_LOCATIONS * sizeof(idx_t));
+	idx_t *counts = locations + 1;
+	// Collect counts
+	const idx_t total_offset = col_offset + offset;
+	data_ptr_t offset_ptr = source_ptr + total_offset;
+	for (idx_t i = 0; i < count; i++) {
+		counts[*offset_ptr]++;
+		offset_ptr += row_width;
+	}
+	// Compute locations from counts
+	idx_t max_count = 0;
+	for (idx_t radix = 0; radix < SortConstants::VALUES_PER_RADIX; radix++) {
+		max_count = MaxValue<idx_t>(max_count, counts[radix]);
+		counts[radix] += locations[radix];
+	}
+	if (max_count != count) {
+		// Re-order the data in temporary array
+		data_ptr_t row_ptr = source_ptr;
+		for (idx_t i = 0; i < count; i++) {
+			const idx_t &radix_offset = locations[*(row_ptr + total_offset)]++;
+			memcpy(target_ptr + radix_offset * row_width, row_ptr, row_width);
+			row_ptr += row_width;
+		}
+		swap = !swap;
+	}
+	// Check if done
+	if (offset == comp_width - 1) {
+		if (swap) {
+			memcpy(orig_ptr, temp_ptr, count * row_width);
+		}
+		return;
+	}
+	if (max_count == count) {
+		RadixSortMSD(orig_ptr, temp_ptr, count, col_offset, row_width, comp_width, offset + 1,
+		             locations + SortConstants::MSD_RADIX_LOCATIONS, swap);
+		return;
+	}
+	// Recurse
+	idx_t radix_count = locations[0];
+	for (idx_t radix = 0; radix < SortConstants::VALUES_PER_RADIX; radix++) {
+		const idx_t loc = (locations[radix] - radix_count) * row_width;
+		if (radix_count > SortConstants::INSERTION_SORT_THRESHOLD) {
+			RadixSortMSD(orig_ptr + loc, temp_ptr + loc, radix_count, col_offset, row_width, comp_width, offset + 1,
+			             locations + SortConstants::MSD_RADIX_LOCATIONS, swap);
+		} else if (radix_count != 0) {
+			InsertionSort(orig_ptr + loc, temp_ptr + loc, radix_count, col_offset, row_width, comp_width, offset + 1,
+			              swap);
+		}
+		radix_count = locations[radix + 1] - locations[radix];
+	}
+}
+
+//! Calls different sort functions, depending on the count and sorting sizes
+void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count, const idx_t &col_offset,
+               const idx_t &sorting_size, const SortLayout &sort_layout) {
+	if (count <= SortConstants::INSERTION_SORT_THRESHOLD) {
+		InsertionSort(dataptr, nullptr, count, 0, sort_layout.entry_size, sort_layout.comparison_size, 0, false);
+	} else if (sorting_size <= SortConstants::MSD_RADIX_SORT_SIZE_THRESHOLD) {
+		RadixSortLSD(buffer_manager, dataptr, count, col_offset, sort_layout.entry_size, sorting_size);
+	} else {
+		auto temp_block = buffer_manager.Allocate(MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
+		auto preallocated_array = unique_ptr<idx_t[]>(new idx_t[sorting_size * SortConstants::MSD_RADIX_LOCATIONS]);
+		RadixSortMSD(dataptr, temp_block->Ptr(), count, col_offset, sort_layout.entry_size, sorting_size, 0,
+		             preallocated_array.get(), false);
 	}
 }
 
@@ -30478,11 +30607,10 @@ void LocalSortState::SortInMemory() {
 	idx_t sorting_size = 0;
 	idx_t col_offset = 0;
 	unique_ptr<bool[]> ties_ptr;
-	unique_ptr<BufferHandle> ties_handle;
 	bool *ties = nullptr;
 	for (idx_t i = 0; i < sort_layout->column_count; i++) {
 		sorting_size += sort_layout->column_sizes[i];
-		if (sort_layout->constant_size[i] && i < sort_layout->column_count - 1 && sorting_size < 32) {
+		if (sort_layout->constant_size[i] && i < sort_layout->column_count - 1) {
 			// Add columns to the sorting size until we reach a variable size column, or the last column
 			continue;
 		}
@@ -34222,13 +34350,14 @@ public:
 
 namespace duckdb {
 
-DataChunk::DataChunk() : count(0) {
+DataChunk::DataChunk() : count(0), capacity(STANDARD_VECTOR_SIZE) {
 }
 
 DataChunk::~DataChunk() {
 }
 
 void DataChunk::InitializeEmpty(const vector<LogicalType> &types) {
+	capacity = STANDARD_VECTOR_SIZE;
 	D_ASSERT(data.empty());   // can only be initialized once
 	D_ASSERT(!types.empty()); // empty chunk not allowed
 	for (idx_t i = 0; i < types.size(); i++) {
@@ -34239,6 +34368,7 @@ void DataChunk::InitializeEmpty(const vector<LogicalType> &types) {
 void DataChunk::Initialize(const vector<LogicalType> &types) {
 	D_ASSERT(data.empty());   // can only be initialized once
 	D_ASSERT(!types.empty()); // empty chunk not allowed
+	capacity = STANDARD_VECTOR_SIZE;
 	for (idx_t i = 0; i < types.size(); i++) {
 		VectorCache cache(types[i]);
 		data.emplace_back(cache);
@@ -34256,12 +34386,14 @@ void DataChunk::Reset() {
 	for (idx_t i = 0; i < ColumnCount(); i++) {
 		data[i].ResetFromCache(vector_caches[i]);
 	}
+	capacity = STANDARD_VECTOR_SIZE;
 	SetCardinality(0);
 }
 
 void DataChunk::Destroy() {
 	data.clear();
 	vector_caches.clear();
+	capacity = 0;
 	SetCardinality(0);
 }
 
@@ -34277,6 +34409,7 @@ void DataChunk::SetValue(idx_t col_idx, idx_t index, const Value &val) {
 void DataChunk::Reference(DataChunk &chunk) {
 	D_ASSERT(chunk.ColumnCount() <= ColumnCount());
 	SetCardinality(chunk);
+	SetCapacity(chunk);
 	for (idx_t i = 0; i < chunk.ColumnCount(); i++) {
 		data[i].Reference(chunk.data[i]);
 	}
@@ -34284,6 +34417,7 @@ void DataChunk::Reference(DataChunk &chunk) {
 
 void DataChunk::Move(DataChunk &chunk) {
 	SetCardinality(chunk);
+	SetCapacity(chunk);
 	data = move(chunk.data);
 	vector_caches = move(chunk.vector_caches);
 
@@ -34329,32 +34463,33 @@ void DataChunk::Split(DataChunk &other, idx_t split_idx) {
 	other.SetCardinality(*this);
 }
 
-void DataChunk::Append(const DataChunk &other) {
+void DataChunk::Append(const DataChunk &other, bool resize, SelectionVector *sel, idx_t sel_count) {
+	idx_t new_size = sel ? size() + sel_count : size() + other.size();
 	if (other.size() == 0) {
 		return;
 	}
 	if (ColumnCount() != other.ColumnCount()) {
 		throw InternalException("Column counts of appending chunk doesn't match!");
 	}
-	for (idx_t i = 0; i < ColumnCount(); i++) {
-		D_ASSERT(data[i].GetVectorType() == VectorType::FLAT_VECTOR);
-		VectorOperations::Copy(other.data[i], data[i], other.size(), 0, size());
-	}
-	SetCardinality(size() + other.size());
-}
-
-void DataChunk::Append(const DataChunk &other, const SelectionVector &sel, idx_t count) {
-	if (other.size() == 0) {
-		return;
-	}
-	if (ColumnCount() != other.ColumnCount()) {
-		throw InternalException("Column counts of appending chunk doesn't match!");
+	if (new_size > capacity) {
+		if (resize) {
+			for (idx_t i = 0; i < ColumnCount(); i++) {
+				data[i].Resize(size(), new_size);
+			}
+			capacity = new_size;
+		} else {
+			throw InternalException("Can't append chunk to other chunk without resizing");
+		}
 	}
 	for (idx_t i = 0; i < ColumnCount(); i++) {
 		D_ASSERT(data[i].GetVectorType() == VectorType::FLAT_VECTOR);
-		VectorOperations::Copy(other.data[i], data[i], sel, count, 0, size());
+		if (sel) {
+			VectorOperations::Copy(other.data[i], data[i], *sel, sel_count, 0, size());
+		} else {
+			VectorOperations::Copy(other.data[i], data[i], other.size(), 0, size());
+		}
 	}
-	SetCardinality(size() + count);
+	SetCardinality(new_size);
 }
 
 void DataChunk::Normalify() {
@@ -34451,7 +34586,7 @@ void DataChunk::Hash(Vector &result) {
 
 void DataChunk::Verify() {
 #ifdef DEBUG
-	D_ASSERT(size() <= STANDARD_VECTOR_SIZE);
+	D_ASSERT(size() <= capacity);
 	// verify that all vectors in this chunk have the chunk selection vector
 	for (idx_t i = 0; i < ColumnCount(); i++) {
 		data[i].Verify(size());
@@ -35946,20 +36081,30 @@ bool Hugeint::TryCast(hugeint_t input, float &result) {
 	return true;
 }
 
-template <>
-bool Hugeint::TryCast(hugeint_t input, double &result) {
+template <class REAL_T>
+bool CastBigintToFloating(hugeint_t input, REAL_T &result) {
 	switch (input.upper) {
 	case -1:
 		// special case for upper = -1 to avoid rounding issues in small negative numbers
-		result = -double(NumericLimits<uint64_t>::Maximum() - input.lower) - 1;
+		result = -REAL_T(NumericLimits<uint64_t>::Maximum() - input.lower) - 1;
 		break;
 	default:
-		result = double(input.lower) + double(input.upper) * double(NumericLimits<uint64_t>::Maximum());
+		result = REAL_T(input.lower) + REAL_T(input.upper) * REAL_T(NumericLimits<uint64_t>::Maximum());
 		break;
 	}
 	return true;
 }
 
+template <>
+bool Hugeint::TryCast(hugeint_t input, double &result) {
+	return CastBigintToFloating<double>(input, result);
+}
+
+template <>
+bool Hugeint::TryCast(hugeint_t input, long double &result) {
+	return CastBigintToFloating<long double>(input, result);
+}
+
 template <class DST>
 hugeint_t HugeintConvertInteger(DST input) {
 	hugeint_t result;
@@ -36017,8 +36162,8 @@ bool Hugeint::TryConvert(float value, hugeint_t &result) {
 	return Hugeint::TryConvert(double(value), result);
 }
 
-template <>
-bool Hugeint::TryConvert(double value, hugeint_t &result) {
+template <class REAL_T>
+bool ConvertFloatingToBigint(REAL_T value, hugeint_t &result) {
 	if (value <= -170141183460469231731687303715884105728.0 || value >= 170141183460469231731687303715884105727.0) {
 		return false;
 	}
@@ -36026,14 +36171,24 @@ bool Hugeint::TryConvert(double value, hugeint_t &result) {
 	if (negative) {
 		value = -value;
 	}
-	result.lower = (uint64_t)fmod(value, double(NumericLimits<uint64_t>::Maximum()));
-	result.upper = (uint64_t)(value / double(NumericLimits<uint64_t>::Maximum()));
+	result.lower = (uint64_t)fmod(value, REAL_T(NumericLimits<uint64_t>::Maximum()));
+	result.upper = (uint64_t)(value / REAL_T(NumericLimits<uint64_t>::Maximum()));
 	if (negative) {
-		NegateInPlace(result);
+		Hugeint::NegateInPlace(result);
 	}
 	return true;
 }
 
+template <>
+bool Hugeint::TryConvert(double value, hugeint_t &result) {
+	return ConvertFloatingToBigint<double>(value, result);
+}
+
+template <>
+bool Hugeint::TryConvert(long double value, hugeint_t &result) {
+	return ConvertFloatingToBigint<long double>(value, result);
+}
+
 //===--------------------------------------------------------------------===//
 // hugeint_t operators
 //===--------------------------------------------------------------------===//
@@ -54656,8 +54811,8 @@ static void SortCollectionForPartition(WindowOperatorState &state, BoundWindowEx
 
 			// Copy the data for each collection.
 			if (bin_size) {
-				over_partition.Append(over_chunk, sel, bin_size);
-				payload_partition.Append(payload_chunk, sel, bin_size);
+				over_partition.Append(over_chunk, false, &sel, bin_size);
+				payload_partition.Append(payload_chunk, false, &sel, bin_size);
 			}
 		} else {
 			local_sort_state.SinkChunk(over_chunk, payload_chunk);
@@ -70552,6 +70707,45 @@ struct VarianceFun {
 
 namespace duckdb {
 
+static inline void KahanAddInternal(double input, double &summed, double &err) {
+	double diff = input - err;
+	double newval = summed + diff;
+	err = (newval - summed) - diff;
+	summed = newval;
+}
+
+template <class T>
+struct SumState {
+	bool isset;
+	T value;
+
+	void Initialize() {
+		this->isset = false;
+	}
+
+	void Combine(const SumState<T> &other) {
+		this->isset = other.isset || this->isset;
+		this->value += other.value;
+	}
+};
+
+struct KahanSumState {
+	bool isset;
+	double value;
+	double err;
+
+	void Initialize() {
+		this->isset = false;
+		this->err = 0.0;
+	}
+
+	void Combine(const KahanSumState &other) {
+		this->isset = other.isset || this->isset;
+		KahanAddInternal(other.value, this->value, this->err);
+		KahanAddInternal(other.err, this->value, this->err);
+	}
+};
+
 struct RegularAdd {
 	template <class STATE, class T>
 	static void AddNumber(STATE &state, T input) {
@@ -70564,6 +70758,18 @@ struct RegularAdd {
 	}
 };
 
+struct KahanAdd {
+	template <class STATE, class T>
+	static void AddNumber(STATE &state, T input) {
+		KahanAddInternal(input, state.value, state.err);
+	}
+
+	template <class STATE, class T>
+	static void AddConstant(STATE &state, T input, idx_t count) {
+		KahanAddInternal(input * count, state.value, state.err);
+	}
+};
+
 struct HugeintAdd {
 	static void AddValue(hugeint_t &result, uint64_t value, int positive) {
 		// integer summation taken from Tim Gubner et al. - Efficient Query Processing
@@ -70659,8 +70865,34 @@ namespace duckdb {
 
 template <class T>
 struct AvgState {
+	uint64_t count;
 	T value;
+
+	void Initialize() {
+		this->count = 0;
+	}
+
+	void Combine(const AvgState<T> &other) {
+		this->count += other.count;
+		this->value += other.value;
+	}
+};
+
+struct KahanAvgState {
 	uint64_t count;
+	double value;
+	double err;
+
+	void Initialize() {
+		this->count = 0;
+		this->err = 0.0;
+	}
+
+	void Combine(const KahanAvgState &other) {
+		this->count += other.count;
+		KahanAddInternal(other.value, this->value, this->err);
+		KahanAddInternal(other.err, this->value, this->err);
+	}
 };
 
 struct AverageDecimalBindData : public FunctionData {
@@ -70678,12 +70910,11 @@ public:
 struct AverageSetOperation {
 	template <class STATE>
 	static void Initialize(STATE *state) {
-		state->count = 0;
+		state->Initialize();
 	}
 	template <class STATE>
 	static void Combine(const STATE &source, STATE *target) {
-		target->count += source.count;
-		target->value += source.value;
+		target->Combine(source);
 	}
 	template <class STATE>
 	static void AddValues(STATE *state, idx_t count) {
@@ -70691,8 +70922,9 @@ struct AverageSetOperation {
 	}
 };
 
-static double GetAverageDivident(uint64_t count, FunctionData *bind_data) {
-	double divident = double(count);
+template <class T>
+static T GetAverageDivident(uint64_t count, FunctionData *bind_data) {
+	T divident = T(count);
 	if (bind_data) {
 		auto &avg_bind_data = (AverageDecimalBindData &)*bind_data;
 		divident *= avg_bind_data.scale;
@@ -70707,7 +70939,7 @@ struct IntegerAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			double divident = GetAverageDivident(state->count, bind_data);
+			double divident = GetAverageDivident<double>(state->count, bind_data);
 			target[idx] = double(state->value) / divident;
 		}
 	}
@@ -70720,8 +70952,8 @@ struct IntegerAverageOperationHugeint : public BaseSumOperation<AverageSetOperat
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			double divident = GetAverageDivident(state->count, bind_data);
-			target[idx] = Hugeint::Cast<double>(state->value) / divident;
+			long double divident = GetAverageDivident<long double>(state->count, bind_data);
+			target[idx] = Hugeint::Cast<long double>(state->value) / divident;
 		}
 	}
 };
@@ -70733,8 +70965,8 @@ struct HugeintAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 		if (state->count == 0) {
 			mask.SetInvalid(idx);
 		} else {
-			double divident = GetAverageDivident(state->count, bind_data);
-			target[idx] = Hugeint::Cast<double>(state->value) / divident;
+			long double divident = GetAverageDivident<long double>(state->count, bind_data);
+			target[idx] = Hugeint::Cast<long double>(state->value) / divident;
 		}
 	}
 };
@@ -70748,7 +70980,21 @@ struct NumericAverageOperation : public BaseSumOperation<AverageSetOperation, Re
 			if (!Value::DoubleIsValid(state->value)) {
 				throw OutOfRangeException("AVG is out of range!");
 			}
-			target[idx] = state->value / state->count;
+			target[idx] = (state->value / state->count);
+		}
+	}
+};
+
+struct KahanAverageOperation : public BaseSumOperation<AverageSetOperation, KahanAdd> {
+	template <class T, class STATE>
+	static void Finalize(Vector &result, FunctionData *, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
+		if (state->count == 0) {
+			mask.SetInvalid(idx);
+		} else {
+			if (!Value::DoubleIsValid(state->value)) {
+				throw OutOfRangeException("AVG is out of range!");
+			}
+			target[idx] = (state->value / state->count) + (state->err / state->count);
 		}
 	}
 };
@@ -70800,6 +71046,11 @@ void AvgFun::RegisterFunction(BuiltinFunctions &set) {
 	avg.AddFunction(AggregateFunction::UnaryAggregate<AvgState<double>, double, double, NumericAverageOperation>(
 	    LogicalType::DOUBLE, LogicalType::DOUBLE));
 	set.AddFunction(avg);
+
+	AggregateFunctionSet favg("favg");
+	favg.AddFunction(AggregateFunction::UnaryAggregate<KahanAvgState, double, double, KahanAverageOperation>(
+	    LogicalType::DOUBLE, LogicalType::DOUBLE));
+	set.AddFunction(favg);
 }
 
 } // namespace duckdb
@@ -73480,21 +73731,14 @@ void StringAggFun::RegisterFunction(BuiltinFunctions &set) {
 
 namespace duckdb {
 
-template <class T>
-struct SumState {
-	T value;
-	bool isset;
-};
-
 struct SumSetOperation {
 	template <class STATE>
 	static void Initialize(STATE *state) {
-		state->isset = false;
+		state->Initialize();
 	}
 	template <class STATE>
 	static void Combine(const STATE &source, STATE *target) {
-		target->isset = source.isset || target->isset;
-		target->value += source.value;
+		target->Combine(source);
 	}
 	template <class STATE>
 	static void AddValues(STATE *state, idx_t count) {
@@ -73524,7 +73768,8 @@ struct SumToHugeintOperation : public BaseSumOperation<SumSetOperation, HugeintA
 	}
 };
 
-struct NumericSumOperation : public BaseSumOperation<SumSetOperation, RegularAdd> {
+template <class ADD_OPERATOR>
+struct DoubleSumOperation : public BaseSumOperation<SumSetOperation, ADD_OPERATOR> {
 	template <class T, class STATE>
 	static void Finalize(Vector &result, FunctionData *, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
 		if (!state->isset) {
@@ -73538,6 +73783,9 @@ struct NumericSumOperation : public BaseSumOperation<SumSetOperation, RegularAdd
 	}
 };
 
+using NumericSumOperation = DoubleSumOperation<RegularAdd>;
+using KahanSumOperation = DoubleSumOperation<KahanAdd>;
+
 struct HugeintSumOperation : public BaseSumOperation<SumSetOperation, RegularAdd> {
 	template <class T, class STATE>
 	static void Finalize(Vector &result, FunctionData *, STATE *state, T *target, ValidityMask &mask, idx_t idx) {
@@ -73646,12 +73894,23 @@ void SumFun::RegisterFunction(BuiltinFunctions &set) {
 	sum.AddFunction(GetSumAggregate(PhysicalType::INT32));
 	sum.AddFunction(GetSumAggregate(PhysicalType::INT64));
 	sum.AddFunction(GetSumAggregate(PhysicalType::INT128));
-	// float sums to float
-	// FIXME: implement http://ic.ese.upenn.edu/pdf/parallel_fpaccum_tc2016.pdf for parallel FP sums
 	sum.AddFunction(AggregateFunction::UnaryAggregate<SumState<double>, double, double, NumericSumOperation>(
 	    LogicalType::DOUBLE, LogicalType::DOUBLE));
 
 	set.AddFunction(sum);
+
+	// fsum
+	AggregateFunctionSet fsum("fsum");
+	fsum.AddFunction(AggregateFunction::UnaryAggregate<KahanSumState, double, double, KahanSumOperation>(
+	    LogicalType::DOUBLE, LogicalType::DOUBLE));
+
+	set.AddFunction(fsum);
+
+	fsum.name = "kahan_sum";
+	set.AddFunction(fsum);
+
+	fsum.name = "sumKahan";
+	set.AddFunction(fsum);
 }
 
 } // namespace duckdb
@@ -74963,14 +75222,12 @@ struct QuantileState {
 	// Regular aggregation
 	std::vector<SaveType> v;
 
-	// Windowed Aggregation
+	// Windowed Quantile indirection
 	std::vector<idx_t> w;
 	idx_t pos;
 
-	// List temporaries
-	std::vector<idx_t> disturbed;
-	std::vector<idx_t> lower;
-	std::vector<idx_t> upper;
+	// Windowed MAD indirection
+	std::vector<idx_t> m;
 
 	QuantileState() : pos(0) {
 	}
@@ -75052,33 +75309,28 @@ static idx_t ReplaceIndex(idx_t *index, const FrameBounds &frame, const FrameBou
 }
 
 template <class INPUT_TYPE>
-static inline bool CanReplace(const idx_t *index, const INPUT_TYPE *fdata, const idx_t j, const idx_t k0,
-                              const idx_t k1, const QuantileNotNull &validity) {
-	auto replace = false;
-
+static inline int CanReplace(const idx_t *index, const INPUT_TYPE *fdata, const idx_t j, const idx_t k0, const idx_t k1,
+                             const QuantileNotNull &validity) {
 	D_ASSERT(index);
 
-	// NULLs can replace NULLs
-	// and NULLs are at the end, so we only have to check the first one
+	// NULLs sort to the end, so if we have inserted a NULL,
+	// it must be past the end of the quantile to be replaceable.
+	// Note that the quantile values are never NULL.
 	const auto ij = index[j];
-	const auto ik0 = index[k0];
-	const auto ik1 = index[k1];
-	const auto nullj = !validity(ij);
-	const auto nullk0 = !validity(ik0);
-	if (nullj || nullk0) {
-		return nullj == nullk0;
+	if (!validity(ij)) {
+		return k1 < j ? 1 : 0;
 	}
 
 	auto curr = fdata[ij];
 	if (k1 < j) {
-		auto hi = fdata[ik0];
-		replace = hi < curr;
+		auto hi = fdata[index[k0]];
+		return hi < curr ? 1 : 0;
 	} else if (j < k0) {
-		auto lo = fdata[ik1];
-		replace = curr < lo;
+		auto lo = fdata[index[k1]];
+		return curr < lo ? -1 : 0;
 	}
 
-	return replace;
+	return 0;
 }
 
 template <class INPUT_TYPE>
@@ -75195,7 +75447,8 @@ struct QuantileLess {
 // Continuous interpolation
 template <bool DISCRETE>
 struct Interpolator {
-	Interpolator(const double q, const idx_t n_p) : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(ceil(RN)) {
+	Interpolator(const double q, const idx_t n_p)
+	    : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(ceil(RN)), begin(0), end(n_p) {
 	}
 
 	template <class INPUT_TYPE, class TARGET_TYPE, typename ACCESSOR = QuantileDirect<INPUT_TYPE>>
@@ -75203,11 +75456,11 @@ struct Interpolator {
 		using ACCESS_TYPE = typename ACCESSOR::RESULT_TYPE;
 		QuantileLess<ACCESSOR> comp(accessor);
 		if (CRN == FRN) {
-			std::nth_element(v_t, v_t + FRN, v_t + n, comp);
+			std::nth_element(v_t + begin, v_t + FRN, v_t + end, comp);
 			return CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[FRN]), result);
 		} else {
-			std::nth_element(v_t, v_t + FRN, v_t + n, comp);
-			std::nth_element(v_t + FRN, v_t + CRN, v_t + n, comp);
+			std::nth_element(v_t + begin, v_t + FRN, v_t + end, comp);
+			std::nth_element(v_t + FRN, v_t + CRN, v_t + end, comp);
 			auto lo = CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[FRN]), result);
 			auto hi = CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[CRN]), result);
 			return CastInterpolation::Interpolate<TARGET_TYPE>(lo, RN - FRN, hi);
@@ -75230,19 +75483,23 @@ struct Interpolator {
 	const double RN;
 	const idx_t FRN;
 	const idx_t CRN;
+
+	idx_t begin;
+	idx_t end;
 };
 
 // Discrete "interpolation"
 template <>
 struct Interpolator<true> {
-	Interpolator(const double q, const idx_t n_p) : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(FRN) {
+	Interpolator(const double q, const idx_t n_p)
+	    : n(n_p), RN((double)(n_p - 1) * q), FRN(floor(RN)), CRN(FRN), begin(0), end(n_p) {
 	}
 
 	template <class INPUT_TYPE, class TARGET_TYPE, typename ACCESSOR = QuantileDirect<INPUT_TYPE>>
 	TARGET_TYPE Operation(INPUT_TYPE *v_t, Vector &result, const ACCESSOR &accessor = ACCESSOR()) const {
 		using ACCESS_TYPE = typename ACCESSOR::RESULT_TYPE;
 		QuantileLess<ACCESSOR> comp(accessor);
-		std::nth_element(v_t, v_t + FRN, v_t + n, comp);
+		std::nth_element(v_t + begin, v_t + FRN, v_t + end, comp);
 		return CastInterpolation::Cast<ACCESS_TYPE, TARGET_TYPE>(accessor(v_t[FRN]), result);
 	}
 
@@ -75256,6 +75513,9 @@ struct Interpolator<true> {
 	const double RN;
 	const idx_t FRN;
 	const idx_t CRN;
+
+	idx_t begin;
+	idx_t end;
 };
 
 struct QuantileBindData : public FunctionData {
@@ -75328,7 +75588,9 @@ static void ExecuteListFinalize(Vector &states, FunctionData *bind_data_p, Vecto
 
 	D_ASSERT(bind_data_p);
 	auto bind_data = (QuantileBindData *)bind_data_p;
-	ListVector::SetListSize(result, 0);
+	if (offset == 0) {
+		ListVector::SetListSize(result, 0);
+	}
 
 	if (states.GetVectorType() == VectorType::CONSTANT_VECTOR) {
 		result.SetVectorType(VectorType::CONSTANT_VECTOR);
@@ -75406,7 +75668,7 @@ struct QuantileScalarOperation : public QuantileOperation {
 		if (frame.first == prev.first + 1 && frame.second == prev.second + 1) {
 			//  Fixed frame size
 			const auto j = ReplaceIndex(index, frame, prev);
-			//	We can only replace if the number of NULls has not changed
+			//	We can only replace if the number of NULLs has not changed
 			if (dmask.AllValid() || not_null(prev.first) == not_null(prev.second)) {
 				Interpolator<DISCRETE> interp(q, prev_pos);
 				replace = CanReplace(index, data, j, interp.FRN, interp.CRN, not_null);
@@ -75515,15 +75777,19 @@ struct QuantileListOperation : public QuantileOperation {
 		auto v_t = state->v.data();
 		D_ASSERT(v_t);
 
-		target[idx].offset = ridx;
-		for (const auto &quantile : bind_data->quantiles) {
+		auto &entry = target[idx];
+		entry.offset = ridx;
+		idx_t lower = 0;
+		for (const auto &q : bind_data->order) {
+			const auto &quantile = bind_data->quantiles[q];
 			Interpolator<DISCRETE> interp(quantile, state->v.size());
-			rdata[ridx] = interp.template Operation<typename STATE::SaveType, CHILD_TYPE>(v_t, result);
-			++ridx;
+			interp.begin = lower;
+			rdata[ridx + q] = interp.template Operation<typename STATE::SaveType, CHILD_TYPE>(v_t, result);
+			lower = interp.FRN;
 		}
-		target[idx].length = bind_data->quantiles.size();
+		entry.length = bind_data->quantiles.size();
 
-		ListVector::SetListSize(result_list, ridx);
+		ListVector::SetListSize(result_list, entry.offset + entry.length);
 	}
 
 	template <class STATE, class INPUT_TYPE, class RESULT_TYPE>
@@ -75547,57 +75813,75 @@ struct QuantileListOperation : public QuantileOperation {
 		auto rdata = FlatVector::GetData<CHILD_TYPE>(result);
 
 		//  Lazily initialise frame state
-		const auto prev_valid = state->w.size() == (prev.second - prev.first);
+		auto prev_pos = state->pos;
 		state->SetPos(frame.second - frame.first);
 
 		auto index = state->w.data();
 
-		bool fixed = false;
-		auto j = state->pos;
-		if (prev_valid && dmask.AllValid() && frame.first == prev.first + 1 && frame.second == prev.second + 1) {
+		// We can generalise replacement for quantile lists by observing that when a replacement is
+		// valid for a single quantile, it is valid for all quantiles greater/less than that quantile
+		// based on whether the insertion is below/above the quantile location.
+		// So if a replaced index in an IQR is located between Q25 and Q50, but has a value below Q25,
+		// then Q25 must be recomputed, but Q50 and Q75 are unaffected.
+		// For a single element list, this reduces to the scalar case.
+		std::pair<idx_t, idx_t> replaceable {state->pos, 0};
+		if (frame.first == prev.first + 1 && frame.second == prev.second + 1) {
 			//  Fixed frame size
-			j = ReplaceIndex(index, frame, prev);
-			fixed = true;
+			const auto j = ReplaceIndex(index, frame, prev);
+			//	We can only replace if the number of NULLs has not changed
+			if (dmask.AllValid() || not_null(prev.first) == not_null(prev.second)) {
+				for (const auto &q : bind_data->order) {
+					const auto &quantile = bind_data->quantiles[q];
+					Interpolator<DISCRETE> interp(quantile, prev_pos);
+					const auto replace = CanReplace(index, data, j, interp.FRN, interp.CRN, not_null);
+					if (replace < 0) {
+						//	Replacement is before this quantile, so the rest will be replaceable too.
+						replaceable.first = MinValue(replaceable.first, interp.FRN);
+						replaceable.second = prev_pos;
+						break;
+					} else if (replace > 0) {
+						//	Replacement is after this quantile, so everything before it is replaceable too.
+						replaceable.first = 0;
+						replaceable.second = MaxValue(replaceable.second, interp.CRN);
+					}
+				}
+				if (replaceable.first < replaceable.second) {
+					state->pos = prev_pos;
+				}
+			}
 		} else {
 			ReuseIndexes(index, frame, prev);
-			if (!dmask.AllValid()) {
-				state->pos = std::partition(index, index + state->pos, not_null) - index;
-			}
 		}
 
-		if (!state->pos) {
-			lmask.Set(lidx, false);
-			return;
+		if (replaceable.first >= replaceable.second && !dmask.AllValid()) {
+			// Remove the NULLs
+			state->pos = std::partition(index, index + state->pos, not_null) - index;
 		}
 
-		// First pass: Fill in the undisturbed values and find the islands of stability.
-		QuantileIndirect<INPUT_TYPE> indirect(data);
-		state->disturbed.clear();
-		state->lower.clear();
-		state->upper.clear();
-		idx_t lb = 0;
-		for (idx_t i = 0; i < bind_data->order.size(); ++i) {
-			const auto q = bind_data->order[i];
-			const auto &quantile = bind_data->quantiles[q];
-			Interpolator<DISCRETE> interp(quantile, state->pos);
-
-			if (fixed && CanReplace(index, data, j, interp.FRN, interp.CRN, not_null)) {
-				rdata[lentry.offset + q] = interp.template Replace<idx_t, CHILD_TYPE>(index, result, indirect);
-				state->upper.resize(state->lower.size(), interp.FRN);
-			} else {
-				state->disturbed.push_back(q);
-				state->lower.push_back(MinValue(lb, interp.FRN));
+		if (state->pos) {
+			using ID = QuantileIndirect<INPUT_TYPE>;
+			ID indirect(data);
+			for (const auto &q : bind_data->order) {
+				const auto &quantile = bind_data->quantiles[q];
+				Interpolator<DISCRETE> interp(quantile, state->pos);
+				if (replaceable.first <= interp.FRN && interp.CRN <= replaceable.second) {
+					rdata[lentry.offset + q] = interp.template Replace<idx_t, CHILD_TYPE, ID>(index, result, indirect);
+				} else {
+					// Make sure we don't disturb any replacements
+					if (replaceable.first < replaceable.second) {
+						if (interp.FRN < replaceable.first) {
+							interp.end = replaceable.first;
+						}
+						if (replaceable.second < interp.CRN) {
+							interp.begin = replaceable.second;
+						}
+					}
+					rdata[lentry.offset + q] =
+					    interp.template Operation<idx_t, CHILD_TYPE, ID>(index, result, indirect);
+				}
 			}
-			lb = interp.CRN + 1;
-		}
-		state->upper.resize(state->lower.size(), state->pos);
-
-		// Second pass: select the disturbed values
-		for (idx_t i = 0; i < state->disturbed.size(); ++i) {
-			const auto &q = state->disturbed[i];
-			const auto &quantile = bind_data->quantiles[q];
-			Interpolator<DISCRETE> interp(quantile, state->pos);
-			rdata[lentry.offset + q] = interp.template Operation<idx_t, CHILD_TYPE>(index, result, indirect);
+		} else {
+			lmask.Set(lidx, false);
 		}
 	}
 };
@@ -75877,12 +76161,11 @@ struct MedianAbsoluteDeviationOperation : public QuantileOperation {
 		D_ASSERT(index);
 
 		// We need a second index for the second pass.
-		// so repurpose disturbed as the second index.
-		if (state->pos > state->disturbed.size()) {
-			state->disturbed.resize(state->pos);
+		if (state->pos > state->m.size()) {
+			state->m.resize(state->pos);
 		}
 
-		auto index2 = state->disturbed.data();
+		auto index2 = state->m.data();
 		D_ASSERT(index2);
 
 		// The replacement trick does not work on the second index because if
@@ -76475,9 +76758,16 @@ struct MapFun {
 struct MapExtractFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
+
 struct ListExtractFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
+
+struct ListConcatFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct CardinalityFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -83453,11 +83743,6 @@ void ArraySliceFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
-
-
-
-
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -83497,6 +83782,147 @@ public:
 
 
 
+namespace duckdb {
+
+static void ListConcatFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 2);
+	auto count = args.size();
+
+	Vector &lhs = args.data[0];
+	Vector &rhs = args.data[1];
+	if (lhs.GetType().id() == LogicalTypeId::SQLNULL) {
+		result.Reference(rhs);
+		return;
+	}
+	if (rhs.GetType().id() == LogicalTypeId::SQLNULL) {
+		result.Reference(lhs);
+		return;
+	}
+
+	VectorData lhs_data;
+	VectorData rhs_data;
+	lhs.Orrify(count, lhs_data);
+	rhs.Orrify(count, rhs_data);
+	auto lhs_entries = (list_entry_t *)lhs_data.data;
+	auto rhs_entries = (list_entry_t *)rhs_data.data;
+
+	auto lhs_list_size = ListVector::GetListSize(lhs);
+	auto rhs_list_size = ListVector::GetListSize(rhs);
+	auto &lhs_child = ListVector::GetEntry(lhs);
+	auto &rhs_child = ListVector::GetEntry(rhs);
+	VectorData lhs_child_data;
+	VectorData rhs_child_data;
+	lhs_child.Orrify(lhs_list_size, lhs_child_data);
+	rhs_child.Orrify(rhs_list_size, rhs_child_data);
+
+	result.SetVectorType(VectorType::FLAT_VECTOR);
+	auto result_entries = FlatVector::GetData<list_entry_t>(result);
+	auto &result_validity = FlatVector::Validity(result);
+
+	idx_t offset = 0;
+	for (idx_t i = 0; i < count; i++) {
+		auto lhs_list_index = lhs_data.sel->get_index(i);
+		auto rhs_list_index = rhs_data.sel->get_index(i);
+		if (!lhs_data.validity.RowIsValid(lhs_list_index) && !rhs_data.validity.RowIsValid(rhs_list_index)) {
+			result_validity.SetInvalid(i);
+			continue;
+		}
+		result_entries[i].offset = offset;
+		result_entries[i].length = 0;
+		if (lhs_data.validity.RowIsValid(lhs_list_index)) {
+			const auto &lhs_entry = lhs_entries[lhs_list_index];
+			result_entries[i].length += lhs_entry.length;
+			ListVector::Append(result, lhs_child, *lhs_child_data.sel, lhs_entry.offset + lhs_entry.length,
+			                   lhs_entry.offset);
+		}
+		if (rhs_data.validity.RowIsValid(rhs_list_index)) {
+			const auto &rhs_entry = rhs_entries[rhs_list_index];
+			result_entries[i].length += rhs_entry.length;
+			ListVector::Append(result, rhs_child, *rhs_child_data.sel, rhs_entry.offset + rhs_entry.length,
+			                   rhs_entry.offset);
+		}
+		offset += result_entries[i].length;
+	}
+	D_ASSERT(ListVector::GetListSize(result) == offset);
+
+	if (lhs.GetVectorType() == VectorType::CONSTANT_VECTOR && rhs.GetVectorType() == VectorType::CONSTANT_VECTOR) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+static unique_ptr<FunctionData> ListConcatBind(ClientContext &context, ScalarFunction &bound_function,
+                                               vector<unique_ptr<Expression>> &arguments) {
+	D_ASSERT(bound_function.arguments.size() == 2);
+
+	auto &lhs = arguments[0]->return_type;
+	auto &rhs = arguments[1]->return_type;
+	if (lhs.id() == LogicalTypeId::SQLNULL && rhs.id() == LogicalTypeId::SQLNULL) {
+		bound_function.return_type = LogicalType::SQLNULL;
+	} else if (lhs.id() == LogicalTypeId::SQLNULL || rhs.id() == LogicalTypeId::SQLNULL) {
+		// we mimic postgres behaviour: list_concat(NULL, my_list) = my_list
+		bound_function.arguments[0] = lhs;
+		bound_function.arguments[1] = rhs;
+		bound_function.return_type = rhs.id() == LogicalTypeId::SQLNULL ? lhs : rhs;
+	} else {
+		D_ASSERT(lhs.id() == LogicalTypeId::LIST);
+		D_ASSERT(rhs.id() == LogicalTypeId::LIST);
+
+		// Resolve list type
+		auto child_type = LogicalType::SQLNULL;
+		for (const auto &argument : arguments) {
+			child_type = LogicalType::MaxLogicalType(child_type, ListType::GetChildType(argument->return_type));
+		}
+		ExpressionBinder::ResolveParameterType(child_type);
+		auto list_type = LogicalType::LIST(move(child_type));
+
+		bound_function.arguments[0] = list_type;
+		bound_function.arguments[1] = list_type;
+		bound_function.return_type = list_type;
+	}
+	return make_unique<VariableReturnBindData>(bound_function.return_type);
+}
+
+static unique_ptr<BaseStatistics> ListConcatStats(ClientContext &context, BoundFunctionExpression &expr,
+                                                  FunctionData *bind_data,
+                                                  vector<unique_ptr<BaseStatistics>> &child_stats) {
+	D_ASSERT(child_stats.size() == 2);
+	if (!child_stats[0] || !child_stats[1]) {
+		return nullptr;
+	}
+
+	auto &left_stats = (ListStatistics &)*child_stats[0];
+	auto &right_stats = (ListStatistics &)*child_stats[1];
+
+	auto stats = left_stats.Copy();
+	stats->Merge(right_stats);
+
+	return stats;
+}
+
+ScalarFunction ListConcatFun::GetFunction() {
+	// the arguments and return types are actually set in the binder function
+	return ScalarFunction({LogicalType::LIST(LogicalType::ANY), LogicalType::LIST(LogicalType::ANY)},
+	                      LogicalType::LIST(LogicalType::ANY), ListConcatFunction, false, ListConcatBind, nullptr,
+	                      ListConcatStats);
+}
+
+void ListConcatFun::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction({"list_concat", "list_cat", "array_concat", "array_cat"}, GetFunction());
+}
+
+} // namespace duckdb
+
+
+
+
+
+
+
+
+
+
+
+
 namespace duckdb {
 
 template <class T, bool HEAP_REF = false, bool VALIDITY_ONLY = false>
@@ -85465,6 +85891,7 @@ void BuiltinFunctions::RegisterNestedFunctions() {
 	Register<ArraySliceFun>();
 	Register<StructPackFun>();
 	Register<StructExtractFun>();
+	Register<ListConcatFun>();
 	Register<ListValueFun>();
 	Register<ListExtractFun>();
 	Register<ListRangeFun>();
@@ -85811,6 +86238,7 @@ struct BitwiseNotFun {
 
 
 
+
 #include <limits>
 
 namespace duckdb {
@@ -86126,6 +86554,8 @@ void AddFun::RegisterFunction(BuiltinFunctions &set) {
 
 	functions.AddFunction(GetFunction(LogicalType::TIMESTAMP, LogicalType::INTERVAL));
 	functions.AddFunction(GetFunction(LogicalType::INTERVAL, LogicalType::TIMESTAMP));
+	// we can add lists together
+	functions.AddFunction(ListConcatFun::GetFunction());
 
 	set.AddFunction(functions);
 }
@@ -87708,6 +88138,7 @@ void CHR::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 #include <string.h>
 
 namespace duckdb {
@@ -87953,6 +88384,7 @@ void ConcatFun::RegisterFunction(BuiltinFunctions &set) {
 	concat_op.AddFunction(
 	    ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, ConcatOperator));
 	concat_op.AddFunction(ScalarFunction({LogicalType::BLOB, LogicalType::BLOB}, LogicalType::BLOB, ConcatOperator));
+	concat_op.AddFunction(ListConcatFun::GetFunction());
 	set.AddFunction(concat_op);
 
 	ScalarFunction concat_ws = ScalarFunction("concat_ws", {LogicalType::VARCHAR, LogicalType::VARCHAR},

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "fbeb407d4"
-#define DUCKDB_VERSION "0.3.1-dev501"
+#define DUCKDB_SOURCE_ID "b36732a57"
+#define DUCKDB_VERSION "0.3.1-dev532"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -3660,12 +3660,15 @@ public:
 		return data.size();
 	}
 	void SetCardinality(idx_t count_p) {
-		D_ASSERT(count <= STANDARD_VECTOR_SIZE);
+		D_ASSERT(count_p <= capacity);
 		this->count = count_p;
 	}
 	void SetCardinality(const DataChunk &other) {
 		this->count = other.size();
 	}
+	void SetCapacity(const DataChunk &other) {
+		this->capacity = other.capacity;
+	}
 
 	DUCKDB_API Value GetValue(idx_t col_idx, idx_t index) const;
 	DUCKDB_API void SetValue(idx_t col_idx, idx_t index, const Value &val);
@@ -3684,10 +3687,10 @@ public:
 	void InitializeEmpty(const vector<LogicalType> &types);
 	//! Append the other DataChunk to this one. The column count and types of
 	//! the two DataChunks have to match exactly. Throws an exception if there
-	//! is not enough space in the chunk.
-	DUCKDB_API void Append(const DataChunk &other);
-	//! Append a selection of the other DataChunk to this one
-	DUCKDB_API void Append(const DataChunk &other, const SelectionVector &sel, idx_t count);
+	//! is not enough space in the chunk and resize is not allowed.
+	DUCKDB_API void Append(const DataChunk &other, bool resize = false, SelectionVector *sel = nullptr,
+	                       idx_t count = 0);
+
 	//! Destroy all data and columns owned by this DataChunk
 	DUCKDB_API void Destroy();
 
@@ -3739,6 +3742,8 @@ public:
 private:
 	//! The amount of tuples stored in the data chunk
 	idx_t count;
+	//! The amount of tuples that can be stored in the data chunk
+	idx_t capacity;
 	//! Vector caches, used to store data when ::Initialize is called
 	vector<VectorCache> vector_caches;
 };
@@ -5957,6 +5962,8 @@ template <>
 bool Hugeint::TryCast(hugeint_t input, float &result);
 template <>
 bool Hugeint::TryCast(hugeint_t input, double &result);
+template <>
+bool Hugeint::TryCast(hugeint_t input, long double &result);
 
 template <>
 bool Hugeint::TryConvert(int8_t value, hugeint_t &result);
@@ -5978,6 +5985,8 @@ template <>
 bool Hugeint::TryConvert(float value, hugeint_t &result);
 template <>
 bool Hugeint::TryConvert(double value, hugeint_t &result);
+template <>
+bool Hugeint::TryConvert(long double value, hugeint_t &result);
 
 } // namespace duckdb