duckdb 0.3.1-dev511.0 → 0.3.1-dev532.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.1-dev511.0",
+  "version": "0.3.1-dev532.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -9435,7 +9435,11 @@ static DefaultMacro internal_macros[] = {
 	{DEFAULT_SCHEMA, "round_even", {"x", "n", nullptr}, "CASE ((abs(x) * power(10, n+1)) % 10) WHEN 5 THEN round(x/2, n) * 2 ELSE round(x, n) END"},
 	{DEFAULT_SCHEMA, "roundbankers", {"x", "n", nullptr}, "round_even(x, n)"},
 	{DEFAULT_SCHEMA, "nullif", {"a", "b", nullptr}, "CASE WHEN a=b THEN NULL ELSE a END"},
-    {nullptr, nullptr, {nullptr}, nullptr}};
+	{DEFAULT_SCHEMA, "list_append", {"l", "e", nullptr}, "list_concat(l, list_value(e))"},
+	{DEFAULT_SCHEMA, "array_append", {"arr", "el", nullptr}, "list_append(arr, el)"},
+	{DEFAULT_SCHEMA, "list_prepend", {"e", "l", nullptr}, "list_concat(list_value(e), l)"},
+	{DEFAULT_SCHEMA, "array_prepend", {"el", "arr", nullptr}, "list_prepend(el, arr)"},
+	{nullptr, nullptr, {nullptr}, nullptr}};
 
 static unique_ptr<CreateFunctionInfo> GetDefaultFunction(const string &schema, const string &name) {
 	for (idx_t index = 0; internal_macros[index].name != nullptr; index++) {
@@ -9897,10 +9901,11 @@ public:
 
 class ResultArrowArrayStreamWrapper {
 public:
-	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result);
+	explicit ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result, idx_t approx_batch_size);
 	ArrowArrayStream stream;
 	unique_ptr<QueryResult> result;
 	std::string last_error;
+	idx_t vectors_per_chunk;
 
 private:
 	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);
@@ -10017,13 +10022,21 @@ int ResultArrowArrayStreamWrapper::MyStreamGetNext(struct ArrowArrayStream *stre
 			return -1;
 		}
 	}
-	auto data_chunk = result.Fetch();
-	if (!data_chunk) {
-		//! Nothing to output
+	unique_ptr<DataChunk> chunk_result = result.Fetch();
+	if (!chunk_result) {
+		// Nothing to output
 		out->release = nullptr;
 		return 0;
 	}
-	data_chunk->ToArrowArray(out);
+	for (idx_t i = 1; i < my_stream->vectors_per_chunk; i++) {
+		auto new_chunk = result.Fetch();
+		if (!new_chunk) {
+			break;
+		} else {
+			chunk_result->Append(*new_chunk, true);
+		}
+	}
+	chunk_result->ToArrowArray(out);
 	return 0;
 }
 
@@ -10043,11 +10056,15 @@ const char *ResultArrowArrayStreamWrapper::MyStreamGetLastError(struct ArrowArra
 	auto my_stream = (ResultArrowArrayStreamWrapper *)stream->private_data;
 	return my_stream->last_error.c_str();
 }
-ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result_p)
+ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryResult> result_p, idx_t approx_batch_size)
     : result(move(result_p)) {
 	//! We first initialize the private data of the stream
 	stream.private_data = this;
-
+	//! Ceil Approx_Batch_Size/STANDARD_VECTOR_SIZE
+	if (approx_batch_size == 0) {
+		throw std::runtime_error("Approximate Batch Size of Record Batch MUST be higher than 0");
+	}
+	vectors_per_chunk = (approx_batch_size + STANDARD_VECTOR_SIZE - 1) / STANDARD_VECTOR_SIZE;
 	//! We initialize the stream functions
 	stream.get_schema = ResultArrowArrayStreamWrapper::MyStreamGetSchema;
 	stream.get_next = ResultArrowArrayStreamWrapper::MyStreamGetNext;
@@ -29102,6 +29119,13 @@ namespace duckdb {
 class RowLayout;
 struct LocalSortState;
 
+struct SortConstants {
+	static constexpr idx_t VALUES_PER_RADIX = 256;
+	static constexpr idx_t MSD_RADIX_LOCATIONS = VALUES_PER_RADIX + 1;
+	static constexpr idx_t INSERTION_SORT_THRESHOLD = 24;
+	static constexpr idx_t MSD_RADIX_SORT_SIZE_THRESHOLD = 4;
+};
+
 struct SortLayout {
 public:
 	explicit SortLayout(const vector<BoundOrderByNode> &orders);
@@ -30403,12 +30427,12 @@ static void ComputeTies(data_ptr_t dataptr, const idx_t &count, const idx_t &col
 }
 
 //! Textbook LSD radix sort
-static void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count,
-                      const idx_t &col_offset, const idx_t &sorting_size, const SortLayout &sort_layout) {
-	auto temp_block = buffer_manager.Allocate(MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
+void RadixSortLSD(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count, const idx_t &col_offset,
+                  const idx_t &row_width, const idx_t &sorting_size) {
+	auto temp_block = buffer_manager.Allocate(MaxValue(count * row_width, (idx_t)Storage::BLOCK_SIZE));
 	bool swap = false;
 
-	idx_t counts[256];
+	idx_t counts[SortConstants::VALUES_PER_RADIX];
 	for (idx_t r = 1; r <= sorting_size; r++) {
 		// Init counts to 0
 		memset(counts, 0, sizeof(counts));
@@ -30420,24 +30444,129 @@ static void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr,
 		data_ptr_t offset_ptr = source_ptr + offset;
 		for (idx_t i = 0; i < count; i++) {
 			counts[*offset_ptr]++;
-			offset_ptr += sort_layout.entry_size;
+			offset_ptr += row_width;
 		}
 		// Compute offsets from counts
-		for (idx_t val = 1; val < 256; val++) {
+		idx_t max_count = counts[0];
+		for (idx_t val = 1; val < SortConstants::VALUES_PER_RADIX; val++) {
+			max_count = MaxValue<idx_t>(max_count, counts[val]);
 			counts[val] = counts[val] + counts[val - 1];
 		}
+		if (max_count == count) {
+			continue;
+		}
 		// Re-order the data in temporary array
-		data_ptr_t row_ptr = source_ptr + (count - 1) * sort_layout.entry_size;
+		data_ptr_t row_ptr = source_ptr + (count - 1) * row_width;
 		for (idx_t i = 0; i < count; i++) {
 			idx_t &radix_offset = --counts[*(row_ptr + offset)];
-			memcpy(target_ptr + radix_offset * sort_layout.entry_size, row_ptr, sort_layout.entry_size);
-			row_ptr -= sort_layout.entry_size;
+			memcpy(target_ptr + radix_offset * row_width, row_ptr, row_width);
+			row_ptr -= row_width;
 		}
 		swap = !swap;
 	}
 	// Move data back to original buffer (if it was swapped)
 	if (swap) {
-		memcpy(dataptr, temp_block->Ptr(), count * sort_layout.entry_size);
+		memcpy(dataptr, temp_block->Ptr(), count * row_width);
+	}
+}
+
+//! Insertion sort, used when count of values is low
+inline void InsertionSort(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr, const idx_t &count,
+                          const idx_t &col_offset, const idx_t &row_width, const idx_t &total_comp_width,
+                          const idx_t &offset, bool swap) {
+	const data_ptr_t source_ptr = swap ? temp_ptr : orig_ptr;
+	const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr;
+	if (count > 1) {
+		const idx_t total_offset = col_offset + offset;
+		auto temp_val = unique_ptr<data_t[]>(new data_t[row_width]);
+		const data_ptr_t val = temp_val.get();
+		const auto comp_width = total_comp_width - offset;
+		for (idx_t i = 1; i < count; i++) {
+			memcpy(val, source_ptr + i * row_width, row_width);
+			idx_t j = i;
+			while (j > 0 &&
+			       memcmp(source_ptr + (j - 1) * row_width + total_offset, val + total_offset, comp_width) > 0) {
+				memcpy(source_ptr + j * row_width, source_ptr + (j - 1) * row_width, row_width);
+				j--;
+			}
+			memcpy(source_ptr + j * row_width, val, row_width);
+		}
+	}
+	if (swap) {
+		memcpy(target_ptr, source_ptr, count * row_width);
+	}
+}
+
+//! MSD radix sort that switches to insertion sort with low bucket sizes
+void RadixSortMSD(const data_ptr_t orig_ptr, const data_ptr_t temp_ptr, const idx_t &count, const idx_t &col_offset,
+                  const idx_t &row_width, const idx_t &comp_width, const idx_t &offset, idx_t locations[], bool swap) {
+	const data_ptr_t source_ptr = swap ? temp_ptr : orig_ptr;
+	const data_ptr_t target_ptr = swap ? orig_ptr : temp_ptr;
+	// Init counts to 0
+	memset(locations, 0, SortConstants::MSD_RADIX_LOCATIONS * sizeof(idx_t));
+	idx_t *counts = locations + 1;
+	// Collect counts
+	const idx_t total_offset = col_offset + offset;
+	data_ptr_t offset_ptr = source_ptr + total_offset;
+	for (idx_t i = 0; i < count; i++) {
+		counts[*offset_ptr]++;
+		offset_ptr += row_width;
+	}
+	// Compute locations from counts
+	idx_t max_count = 0;
+	for (idx_t radix = 0; radix < SortConstants::VALUES_PER_RADIX; radix++) {
+		max_count = MaxValue<idx_t>(max_count, counts[radix]);
+		counts[radix] += locations[radix];
+	}
+	if (max_count != count) {
+		// Re-order the data in temporary array
+		data_ptr_t row_ptr = source_ptr;
+		for (idx_t i = 0; i < count; i++) {
+			const idx_t &radix_offset = locations[*(row_ptr + total_offset)]++;
+			memcpy(target_ptr + radix_offset * row_width, row_ptr, row_width);
+			row_ptr += row_width;
+		}
+		swap = !swap;
+	}
+	// Check if done
+	if (offset == comp_width - 1) {
+		if (swap) {
+			memcpy(orig_ptr, temp_ptr, count * row_width);
+		}
+		return;
+	}
+	if (max_count == count) {
+		RadixSortMSD(orig_ptr, temp_ptr, count, col_offset, row_width, comp_width, offset + 1,
+		             locations + SortConstants::MSD_RADIX_LOCATIONS, swap);
+		return;
+	}
+	// Recurse
+	idx_t radix_count = locations[0];
+	for (idx_t radix = 0; radix < SortConstants::VALUES_PER_RADIX; radix++) {
+		const idx_t loc = (locations[radix] - radix_count) * row_width;
+		if (radix_count > SortConstants::INSERTION_SORT_THRESHOLD) {
+			RadixSortMSD(orig_ptr + loc, temp_ptr + loc, radix_count, col_offset, row_width, comp_width, offset + 1,
+			             locations + SortConstants::MSD_RADIX_LOCATIONS, swap);
+		} else if (radix_count != 0) {
+			InsertionSort(orig_ptr + loc, temp_ptr + loc, radix_count, col_offset, row_width, comp_width, offset + 1,
+			              swap);
+		}
+		radix_count = locations[radix + 1] - locations[radix];
+	}
+}
+
+//! Calls different sort functions, depending on the count and sorting sizes
+void RadixSort(BufferManager &buffer_manager, const data_ptr_t &dataptr, const idx_t &count, const idx_t &col_offset,
+               const idx_t &sorting_size, const SortLayout &sort_layout) {
+	if (count <= SortConstants::INSERTION_SORT_THRESHOLD) {
+		InsertionSort(dataptr, nullptr, count, 0, sort_layout.entry_size, sort_layout.comparison_size, 0, false);
+	} else if (sorting_size <= SortConstants::MSD_RADIX_SORT_SIZE_THRESHOLD) {
+		RadixSortLSD(buffer_manager, dataptr, count, col_offset, sort_layout.entry_size, sorting_size);
+	} else {
+		auto temp_block = buffer_manager.Allocate(MaxValue(count * sort_layout.entry_size, (idx_t)Storage::BLOCK_SIZE));
+		auto preallocated_array = unique_ptr<idx_t[]>(new idx_t[sorting_size * SortConstants::MSD_RADIX_LOCATIONS]);
+		RadixSortMSD(dataptr, temp_block->Ptr(), count, col_offset, sort_layout.entry_size, sorting_size, 0,
+		             preallocated_array.get(), false);
 	}
 }
 
@@ -30478,11 +30607,10 @@ void LocalSortState::SortInMemory() {
 	idx_t sorting_size = 0;
 	idx_t col_offset = 0;
 	unique_ptr<bool[]> ties_ptr;
-	unique_ptr<BufferHandle> ties_handle;
 	bool *ties = nullptr;
 	for (idx_t i = 0; i < sort_layout->column_count; i++) {
 		sorting_size += sort_layout->column_sizes[i];
-		if (sort_layout->constant_size[i] && i < sort_layout->column_count - 1 && sorting_size < 32) {
+		if (sort_layout->constant_size[i] && i < sort_layout->column_count - 1) {
 			// Add columns to the sorting size until we reach a variable size column, or the last column
 			continue;
 		}
@@ -34222,13 +34350,14 @@ public:
 
 namespace duckdb {
 
-DataChunk::DataChunk() : count(0) {
+DataChunk::DataChunk() : count(0), capacity(STANDARD_VECTOR_SIZE) {
 }
 
 DataChunk::~DataChunk() {
 }
 
 void DataChunk::InitializeEmpty(const vector<LogicalType> &types) {
+	capacity = STANDARD_VECTOR_SIZE;
 	D_ASSERT(data.empty());   // can only be initialized once
 	D_ASSERT(!types.empty()); // empty chunk not allowed
 	for (idx_t i = 0; i < types.size(); i++) {
@@ -34239,6 +34368,7 @@ void DataChunk::InitializeEmpty(const vector<LogicalType> &types) {
 void DataChunk::Initialize(const vector<LogicalType> &types) {
 	D_ASSERT(data.empty());   // can only be initialized once
 	D_ASSERT(!types.empty()); // empty chunk not allowed
+	capacity = STANDARD_VECTOR_SIZE;
 	for (idx_t i = 0; i < types.size(); i++) {
 		VectorCache cache(types[i]);
 		data.emplace_back(cache);
@@ -34256,12 +34386,14 @@ void DataChunk::Reset() {
 	for (idx_t i = 0; i < ColumnCount(); i++) {
 		data[i].ResetFromCache(vector_caches[i]);
 	}
+	capacity = STANDARD_VECTOR_SIZE;
 	SetCardinality(0);
 }
 
 void DataChunk::Destroy() {
 	data.clear();
 	vector_caches.clear();
+	capacity = 0;
 	SetCardinality(0);
 }
 
@@ -34277,6 +34409,7 @@ void DataChunk::SetValue(idx_t col_idx, idx_t index, const Value &val) {
 void DataChunk::Reference(DataChunk &chunk) {
 	D_ASSERT(chunk.ColumnCount() <= ColumnCount());
 	SetCardinality(chunk);
+	SetCapacity(chunk);
 	for (idx_t i = 0; i < chunk.ColumnCount(); i++) {
 		data[i].Reference(chunk.data[i]);
 	}
@@ -34284,6 +34417,7 @@ void DataChunk::Reference(DataChunk &chunk) {
 
 void DataChunk::Move(DataChunk &chunk) {
 	SetCardinality(chunk);
+	SetCapacity(chunk);
 	data = move(chunk.data);
 	vector_caches = move(chunk.vector_caches);
 
@@ -34329,32 +34463,33 @@ void DataChunk::Split(DataChunk &other, idx_t split_idx) {
 	other.SetCardinality(*this);
 }
 
-void DataChunk::Append(const DataChunk &other) {
+void DataChunk::Append(const DataChunk &other, bool resize, SelectionVector *sel, idx_t sel_count) {
+	idx_t new_size = sel ? size() + sel_count : size() + other.size();
 	if (other.size() == 0) {
 		return;
 	}
 	if (ColumnCount() != other.ColumnCount()) {
 		throw InternalException("Column counts of appending chunk doesn't match!");
 	}
-	for (idx_t i = 0; i < ColumnCount(); i++) {
-		D_ASSERT(data[i].GetVectorType() == VectorType::FLAT_VECTOR);
-		VectorOperations::Copy(other.data[i], data[i], other.size(), 0, size());
-	}
-	SetCardinality(size() + other.size());
-}
-
-void DataChunk::Append(const DataChunk &other, const SelectionVector &sel, idx_t count) {
-	if (other.size() == 0) {
-		return;
-	}
-	if (ColumnCount() != other.ColumnCount()) {
-		throw InternalException("Column counts of appending chunk doesn't match!");
+	if (new_size > capacity) {
+		if (resize) {
+			for (idx_t i = 0; i < ColumnCount(); i++) {
+				data[i].Resize(size(), new_size);
+			}
+			capacity = new_size;
+		} else {
+			throw InternalException("Can't append chunk to other chunk without resizing");
+		}
 	}
 	for (idx_t i = 0; i < ColumnCount(); i++) {
 		D_ASSERT(data[i].GetVectorType() == VectorType::FLAT_VECTOR);
-		VectorOperations::Copy(other.data[i], data[i], sel, count, 0, size());
+		if (sel) {
+			VectorOperations::Copy(other.data[i], data[i], *sel, sel_count, 0, size());
+		} else {
+			VectorOperations::Copy(other.data[i], data[i], other.size(), 0, size());
+		}
 	}
-	SetCardinality(size() + count);
+	SetCardinality(new_size);
 }
 
 void DataChunk::Normalify() {
@@ -34451,7 +34586,7 @@ void DataChunk::Hash(Vector &result) {
 
 void DataChunk::Verify() {
 #ifdef DEBUG
-	D_ASSERT(size() <= STANDARD_VECTOR_SIZE);
+	D_ASSERT(size() <= capacity);
 	// verify that all vectors in this chunk have the chunk selection vector
 	for (idx_t i = 0; i < ColumnCount(); i++) {
 		data[i].Verify(size());
@@ -54676,8 +54811,8 @@ static void SortCollectionForPartition(WindowOperatorState &state, BoundWindowEx
 
 			// Copy the data for each collection.
 			if (bin_size) {
-				over_partition.Append(over_chunk, sel, bin_size);
-				payload_partition.Append(payload_chunk, sel, bin_size);
+				over_partition.Append(over_chunk, false, &sel, bin_size);
+				payload_partition.Append(payload_chunk, false, &sel, bin_size);
 			}
 		} else {
 			local_sort_state.SinkChunk(over_chunk, payload_chunk);
@@ -76623,9 +76758,16 @@ struct MapFun {
 struct MapExtractFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
+
 struct ListExtractFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
+
+struct ListConcatFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct CardinalityFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -83601,11 +83743,6 @@ void ArraySliceFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
-
-
-
-
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -83645,6 +83782,147 @@ public:
 
 
 
+namespace duckdb {
+
+static void ListConcatFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 2);
+	auto count = args.size();
+
+	Vector &lhs = args.data[0];
+	Vector &rhs = args.data[1];
+	if (lhs.GetType().id() == LogicalTypeId::SQLNULL) {
+		result.Reference(rhs);
+		return;
+	}
+	if (rhs.GetType().id() == LogicalTypeId::SQLNULL) {
+		result.Reference(lhs);
+		return;
+	}
+
+	VectorData lhs_data;
+	VectorData rhs_data;
+	lhs.Orrify(count, lhs_data);
+	rhs.Orrify(count, rhs_data);
+	auto lhs_entries = (list_entry_t *)lhs_data.data;
+	auto rhs_entries = (list_entry_t *)rhs_data.data;
+
+	auto lhs_list_size = ListVector::GetListSize(lhs);
+	auto rhs_list_size = ListVector::GetListSize(rhs);
+	auto &lhs_child = ListVector::GetEntry(lhs);
+	auto &rhs_child = ListVector::GetEntry(rhs);
+	VectorData lhs_child_data;
+	VectorData rhs_child_data;
+	lhs_child.Orrify(lhs_list_size, lhs_child_data);
+	rhs_child.Orrify(rhs_list_size, rhs_child_data);
+
+	result.SetVectorType(VectorType::FLAT_VECTOR);
+	auto result_entries = FlatVector::GetData<list_entry_t>(result);
+	auto &result_validity = FlatVector::Validity(result);
+
+	idx_t offset = 0;
+	for (idx_t i = 0; i < count; i++) {
+		auto lhs_list_index = lhs_data.sel->get_index(i);
+		auto rhs_list_index = rhs_data.sel->get_index(i);
+		if (!lhs_data.validity.RowIsValid(lhs_list_index) && !rhs_data.validity.RowIsValid(rhs_list_index)) {
+			result_validity.SetInvalid(i);
+			continue;
+		}
+		result_entries[i].offset = offset;
+		result_entries[i].length = 0;
+		if (lhs_data.validity.RowIsValid(lhs_list_index)) {
+			const auto &lhs_entry = lhs_entries[lhs_list_index];
+			result_entries[i].length += lhs_entry.length;
+			ListVector::Append(result, lhs_child, *lhs_child_data.sel, lhs_entry.offset + lhs_entry.length,
+			                   lhs_entry.offset);
+		}
+		if (rhs_data.validity.RowIsValid(rhs_list_index)) {
+			const auto &rhs_entry = rhs_entries[rhs_list_index];
+			result_entries[i].length += rhs_entry.length;
+			ListVector::Append(result, rhs_child, *rhs_child_data.sel, rhs_entry.offset + rhs_entry.length,
+			                   rhs_entry.offset);
+		}
+		offset += result_entries[i].length;
+	}
+	D_ASSERT(ListVector::GetListSize(result) == offset);
+
+	if (lhs.GetVectorType() == VectorType::CONSTANT_VECTOR && rhs.GetVectorType() == VectorType::CONSTANT_VECTOR) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+static unique_ptr<FunctionData> ListConcatBind(ClientContext &context, ScalarFunction &bound_function,
+                                               vector<unique_ptr<Expression>> &arguments) {
+	D_ASSERT(bound_function.arguments.size() == 2);
+
+	auto &lhs = arguments[0]->return_type;
+	auto &rhs = arguments[1]->return_type;
+	if (lhs.id() == LogicalTypeId::SQLNULL && rhs.id() == LogicalTypeId::SQLNULL) {
+		bound_function.return_type = LogicalType::SQLNULL;
+	} else if (lhs.id() == LogicalTypeId::SQLNULL || rhs.id() == LogicalTypeId::SQLNULL) {
+		// we mimic postgres behaviour: list_concat(NULL, my_list) = my_list
+		bound_function.arguments[0] = lhs;
+		bound_function.arguments[1] = rhs;
+		bound_function.return_type = rhs.id() == LogicalTypeId::SQLNULL ? lhs : rhs;
+	} else {
+		D_ASSERT(lhs.id() == LogicalTypeId::LIST);
+		D_ASSERT(rhs.id() == LogicalTypeId::LIST);
+
+		// Resolve list type
+		auto child_type = LogicalType::SQLNULL;
+		for (const auto &argument : arguments) {
+			child_type = LogicalType::MaxLogicalType(child_type, ListType::GetChildType(argument->return_type));
+		}
+		ExpressionBinder::ResolveParameterType(child_type);
+		auto list_type = LogicalType::LIST(move(child_type));
+
+		bound_function.arguments[0] = list_type;
+		bound_function.arguments[1] = list_type;
+		bound_function.return_type = list_type;
+	}
+	return make_unique<VariableReturnBindData>(bound_function.return_type);
+}
+
+static unique_ptr<BaseStatistics> ListConcatStats(ClientContext &context, BoundFunctionExpression &expr,
+                                                  FunctionData *bind_data,
+                                                  vector<unique_ptr<BaseStatistics>> &child_stats) {
+	D_ASSERT(child_stats.size() == 2);
+	if (!child_stats[0] || !child_stats[1]) {
+		return nullptr;
+	}
+
+	auto &left_stats = (ListStatistics &)*child_stats[0];
+	auto &right_stats = (ListStatistics &)*child_stats[1];
+
+	auto stats = left_stats.Copy();
+	stats->Merge(right_stats);
+
+	return stats;
+}
+
+ScalarFunction ListConcatFun::GetFunction() {
+	// the arguments and return types are actually set in the binder function
+	return ScalarFunction({LogicalType::LIST(LogicalType::ANY), LogicalType::LIST(LogicalType::ANY)},
+	                      LogicalType::LIST(LogicalType::ANY), ListConcatFunction, false, ListConcatBind, nullptr,
+	                      ListConcatStats);
+}
+
+void ListConcatFun::RegisterFunction(BuiltinFunctions &set) {
+	set.AddFunction({"list_concat", "list_cat", "array_concat", "array_cat"}, GetFunction());
+}
+
+} // namespace duckdb
+
+
+
+
+
+
+
+
+
+
+
+
 namespace duckdb {
 
 template <class T, bool HEAP_REF = false, bool VALIDITY_ONLY = false>
@@ -85613,6 +85891,7 @@ void BuiltinFunctions::RegisterNestedFunctions() {
 	Register<ArraySliceFun>();
 	Register<StructPackFun>();
 	Register<StructExtractFun>();
+	Register<ListConcatFun>();
 	Register<ListValueFun>();
 	Register<ListExtractFun>();
 	Register<ListRangeFun>();
@@ -85959,6 +86238,7 @@ struct BitwiseNotFun {
 
 
 
+
 #include <limits>
 
 namespace duckdb {
@@ -86274,6 +86554,8 @@ void AddFun::RegisterFunction(BuiltinFunctions &set) {
 
 	functions.AddFunction(GetFunction(LogicalType::TIMESTAMP, LogicalType::INTERVAL));
 	functions.AddFunction(GetFunction(LogicalType::INTERVAL, LogicalType::TIMESTAMP));
+	// we can add lists together
+	functions.AddFunction(ListConcatFun::GetFunction());
 
 	set.AddFunction(functions);
 }
@@ -87856,6 +88138,7 @@ void CHR::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+
 #include <string.h>
 
 namespace duckdb {
@@ -88101,6 +88384,7 @@ void ConcatFun::RegisterFunction(BuiltinFunctions &set) {
 	concat_op.AddFunction(
 	    ScalarFunction({LogicalType::VARCHAR, LogicalType::VARCHAR}, LogicalType::VARCHAR, ConcatOperator));
 	concat_op.AddFunction(ScalarFunction({LogicalType::BLOB, LogicalType::BLOB}, LogicalType::BLOB, ConcatOperator));
+	concat_op.AddFunction(ListConcatFun::GetFunction());
 	set.AddFunction(concat_op);
 
 	ScalarFunction concat_ws = ScalarFunction("concat_ws", {LogicalType::VARCHAR, LogicalType::VARCHAR},

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "c0d4ec2d7"
-#define DUCKDB_VERSION "0.3.1-dev511"
+#define DUCKDB_SOURCE_ID "b36732a57"
+#define DUCKDB_VERSION "0.3.1-dev532"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -3660,12 +3660,15 @@ public:
 		return data.size();
 	}
 	void SetCardinality(idx_t count_p) {
-		D_ASSERT(count <= STANDARD_VECTOR_SIZE);
+		D_ASSERT(count_p <= capacity);
 		this->count = count_p;
 	}
 	void SetCardinality(const DataChunk &other) {
 		this->count = other.size();
 	}
+	void SetCapacity(const DataChunk &other) {
+		this->capacity = other.capacity;
+	}
 
 	DUCKDB_API Value GetValue(idx_t col_idx, idx_t index) const;
 	DUCKDB_API void SetValue(idx_t col_idx, idx_t index, const Value &val);
@@ -3684,10 +3687,10 @@ public:
 	void InitializeEmpty(const vector<LogicalType> &types);
 	//! Append the other DataChunk to this one. The column count and types of
 	//! the two DataChunks have to match exactly. Throws an exception if there
-	//! is not enough space in the chunk.
-	DUCKDB_API void Append(const DataChunk &other);
-	//! Append a selection of the other DataChunk to this one
-	DUCKDB_API void Append(const DataChunk &other, const SelectionVector &sel, idx_t count);
+	//! is not enough space in the chunk and resize is not allowed.
+	DUCKDB_API void Append(const DataChunk &other, bool resize = false, SelectionVector *sel = nullptr,
+	                       idx_t count = 0);
+
 	//! Destroy all data and columns owned by this DataChunk
 	DUCKDB_API void Destroy();
 
@@ -3739,6 +3742,8 @@ public:
 private:
 	//! The amount of tuples stored in the data chunk
 	idx_t count;
+	//! The amount of tuples that can be stored in the data chunk
+	idx_t capacity;
 	//! Vector caches, used to store data when ::Initialize is called
 	vector<VectorCache> vector_caches;
 };