duckdb 0.3.5-dev705.0 → 0.3.5-dev725.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.5-dev705.0",
+  "version": "0.3.5-dev725.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -45325,7 +45325,7 @@ void Vector::Initialize(bool zero_data, idx_t capacity) {
 		auto struct_buffer = make_unique<VectorStructBuffer>(type, capacity);
 		auxiliary = move(struct_buffer);
 	} else if (internal_type == PhysicalType::LIST) {
-		auto list_buffer = make_unique<VectorListBuffer>(type);
+		auto list_buffer = make_unique<VectorListBuffer>(type, capacity);
 		auxiliary = move(list_buffer);
 	}
 	auto type_size = GetTypeIdSize(internal_type);
@@ -46675,18 +46675,14 @@ VectorListBuffer::VectorListBuffer(unique_ptr<Vector> vector, idx_t initial_capa
 }
 
 VectorListBuffer::VectorListBuffer(const LogicalType &list_type, idx_t initial_capacity)
-    : VectorBuffer(VectorBufferType::LIST_BUFFER) {
-	// FIXME: directly construct vector of correct size
-	child = make_unique<Vector>(ListType::GetChildType(list_type));
-	capacity = STANDARD_VECTOR_SIZE;
-	Reserve(initial_capacity);
+    : VectorBuffer(VectorBufferType::LIST_BUFFER), capacity(initial_capacity),
+      child(make_unique<Vector>(ListType::GetChildType(list_type), initial_capacity)) {
 }
 
 void VectorListBuffer::Reserve(idx_t to_reserve) {
 	if (to_reserve > capacity) {
-		idx_t new_capacity = (to_reserve + STANDARD_VECTOR_SIZE - 1) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE;
+		idx_t new_capacity = NextPowerOfTwo(to_reserve);
 		D_ASSERT(new_capacity >= to_reserve);
-		D_ASSERT(new_capacity % STANDARD_VECTOR_SIZE == 0);
 		child->Resize(capacity, new_capacity);
 		capacity = new_capacity;
 	}
@@ -85972,7 +85968,9 @@ static void ListUpdateFunction(Vector inputs[], FunctionData *, idx_t input_coun
 	for (idx_t i = 0; i < count; i++) {
 		auto state = states[sdata.sel->get_index(i)];
 		if (!state->list_vector) {
-			state->list_vector = new Vector(list_vector_type);
+			// NOTE: any number bigger than 1 can cause DuckDB to run out of memory for specific queries
+			// consisting of millions of groups in the group by and complex (nested) vectors
+			state->list_vector = new Vector(list_vector_type, 1);
 		}
 		ListVector::Append(*state->list_vector, input, i + 1, i);
 	}
@@ -85991,7 +85989,10 @@ static void ListCombineFunction(Vector &state, Vector &combined, FunctionData *b
 			continue;
 		}
 		if (!combined_ptr[i]->list_vector) {
-			combined_ptr[i]->list_vector = new Vector(state->list_vector->GetType());
+			// NOTE: initializing this with a capacity of ListVector::GetListSize(*state->list_vector) causes
+			// DuckDB to run out of memory for multiple threads with millions of groups in the group by and complex
+			// (nested) vectors
+			combined_ptr[i]->list_vector = new Vector(state->list_vector->GetType(), 1);
 		}
 		ListVector::Append(*combined_ptr[i]->list_vector, ListVector::GetEntry(*state->list_vector),
 		                   ListVector::GetListSize(*state->list_vector));
@@ -86006,6 +86007,7 @@ static void ListFinalize(Vector &state_vector, FunctionData *, Vector &result, i
 	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
 
 	auto &mask = FlatVector::Validity(result);
+	auto list_struct_data = FlatVector::GetData<list_entry_t>(result);
 	size_t total_len = ListVector::GetListSize(result);
 
 	for (idx_t i = 0; i < count; i++) {
@@ -86016,22 +86018,14 @@ static void ListFinalize(Vector &state_vector, FunctionData *, Vector &result, i
 			continue;
 		}
 
-		auto list_struct_data = FlatVector::GetData<list_entry_t>(result);
 		auto &state_lv = *state->list_vector;
 		auto state_lv_count = ListVector::GetListSize(state_lv);
 		list_struct_data[rid].length = state_lv_count;
 		list_struct_data[rid].offset = total_len;
 		total_len += state_lv_count;
-	}
 
-	for (idx_t i = 0; i < count; i++) {
-		auto state = states[sdata.sel->get_index(i)];
-		if (!state->list_vector) {
-			continue;
-		}
-		auto &list_vec = *state->list_vector;
-		auto &list_vec_to_append = ListVector::GetEntry(list_vec);
-		ListVector::Append(result, list_vec_to_append, ListVector::GetListSize(list_vec));
+		auto &list_vec_to_append = ListVector::GetEntry(state_lv);
+		ListVector::Append(result, list_vec_to_append, state_lv_count);
 	}
 }
 

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "d7fb3035e"
-#define DUCKDB_VERSION "v0.3.5-dev705"
+#define DUCKDB_SOURCE_ID "463e45ba9"
+#define DUCKDB_VERSION "v0.3.5-dev725"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //