duckdb 0.3.5-dev699.0 → 0.3.5-dev715.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.5-dev699.0",
+  "version": "0.3.5-dev715.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/connection.cpp

@@ -363,6 +363,13 @@ struct ExecTask : public Task {
 			}
 		}
 	}
+
+	void Callback() override {
+		auto env = object.Env();
+		Napi::HandleScope scope(env);
+		callback.Value().MakeCallback(object.Value(), {success ? env.Null() : Napi::String::New(env, error)});
+	};
+
 	std::string sql;
 	bool success;
 	std::string error;

package/src/duckdb.cpp

@@ -45325,7 +45325,7 @@ void Vector::Initialize(bool zero_data, idx_t capacity) {
 		auto struct_buffer = make_unique<VectorStructBuffer>(type, capacity);
 		auxiliary = move(struct_buffer);
 	} else if (internal_type == PhysicalType::LIST) {
-		auto list_buffer = make_unique<VectorListBuffer>(type);
+		auto list_buffer = make_unique<VectorListBuffer>(type, capacity);
 		auxiliary = move(list_buffer);
 	}
 	auto type_size = GetTypeIdSize(internal_type);
@@ -46260,7 +46260,6 @@ const SelectionVector *ConstantVector::ZeroSelectionVector(idx_t count, Selectio
 }
 
 void ConstantVector::Reference(Vector &vector, Vector &source, idx_t position, idx_t count) {
-	D_ASSERT(position < count);
 	auto &source_type = source.GetType();
 	switch (source_type.InternalType()) {
 	case PhysicalType::LIST: {
@@ -46308,7 +46307,7 @@ void ConstantVector::Reference(Vector &vector, Vector &source, idx_t position, i
 		auto &source_entries = StructVector::GetEntries(source);
 		auto &target_entries = StructVector::GetEntries(vector);
 		for (idx_t i = 0; i < source_entries.size(); i++) {
-			ConstantVector::Reference(*target_entries[i], *source_entries[i], position, count);
+			ConstantVector::Reference(*target_entries[i], *source_entries[i], struct_index, count);
 		}
 		vector.SetVectorType(VectorType::CONSTANT_VECTOR);
 		break;
@@ -46676,18 +46675,14 @@ VectorListBuffer::VectorListBuffer(unique_ptr<Vector> vector, idx_t initial_capa
 }
 
 VectorListBuffer::VectorListBuffer(const LogicalType &list_type, idx_t initial_capacity)
-    : VectorBuffer(VectorBufferType::LIST_BUFFER) {
-	// FIXME: directly construct vector of correct size
-	child = make_unique<Vector>(ListType::GetChildType(list_type));
-	capacity = STANDARD_VECTOR_SIZE;
-	Reserve(initial_capacity);
+    : VectorBuffer(VectorBufferType::LIST_BUFFER), capacity(initial_capacity),
+      child(make_unique<Vector>(ListType::GetChildType(list_type), initial_capacity)) {
 }
 
 void VectorListBuffer::Reserve(idx_t to_reserve) {
 	if (to_reserve > capacity) {
-		idx_t new_capacity = (to_reserve + STANDARD_VECTOR_SIZE - 1) / STANDARD_VECTOR_SIZE * STANDARD_VECTOR_SIZE;
+		idx_t new_capacity = NextPowerOfTwo(to_reserve);
 		D_ASSERT(new_capacity >= to_reserve);
-		D_ASSERT(new_capacity % STANDARD_VECTOR_SIZE == 0);
 		child->Resize(capacity, new_capacity);
 		capacity = new_capacity;
 	}
@@ -51740,6 +51735,7 @@ static inline void TemplatedLoopHash(Vector &input, Vector &result, const Select
 
 template <bool HAS_RSEL, bool FIRST_HASH>
 static inline void StructLoopHash(Vector &input, Vector &hashes, const SelectionVector *rsel, idx_t count) {
+	input.Normalify(count);
 	auto &children = StructVector::GetEntries(input);
 
 	D_ASSERT(!children.empty());
@@ -85972,7 +85968,9 @@ static void ListUpdateFunction(Vector inputs[], FunctionData *, idx_t input_coun
 	for (idx_t i = 0; i < count; i++) {
 		auto state = states[sdata.sel->get_index(i)];
 		if (!state->list_vector) {
-			state->list_vector = new Vector(list_vector_type);
+			// NOTE: any number bigger than 1 can cause DuckDB to run out of memory for specific queries
+			// consisting of millions of groups in the group by and complex (nested) vectors
+			state->list_vector = new Vector(list_vector_type, 1);
 		}
 		ListVector::Append(*state->list_vector, input, i + 1, i);
 	}
@@ -85991,7 +85989,10 @@ static void ListCombineFunction(Vector &state, Vector &combined, FunctionData *b
 			continue;
 		}
 		if (!combined_ptr[i]->list_vector) {
-			combined_ptr[i]->list_vector = new Vector(state->list_vector->GetType());
+			// NOTE: initializing this with a capacity of ListVector::GetListSize(*state->list_vector) causes
+			// DuckDB to run out of memory for multiple threads with millions of groups in the group by and complex
+			// (nested) vectors
+			combined_ptr[i]->list_vector = new Vector(state->list_vector->GetType(), 1);
 		}
 		ListVector::Append(*combined_ptr[i]->list_vector, ListVector::GetEntry(*state->list_vector),
 		                   ListVector::GetListSize(*state->list_vector));
@@ -86006,6 +86007,7 @@ static void ListFinalize(Vector &state_vector, FunctionData *, Vector &result, i
 	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
 
 	auto &mask = FlatVector::Validity(result);
+	auto list_struct_data = FlatVector::GetData<list_entry_t>(result);
 	size_t total_len = ListVector::GetListSize(result);
 
 	for (idx_t i = 0; i < count; i++) {
@@ -86016,22 +86018,14 @@ static void ListFinalize(Vector &state_vector, FunctionData *, Vector &result, i
 			continue;
 		}
 
-		auto list_struct_data = FlatVector::GetData<list_entry_t>(result);
 		auto &state_lv = *state->list_vector;
 		auto state_lv_count = ListVector::GetListSize(state_lv);
 		list_struct_data[rid].length = state_lv_count;
 		list_struct_data[rid].offset = total_len;
 		total_len += state_lv_count;
-	}
 
-	for (idx_t i = 0; i < count; i++) {
-		auto state = states[sdata.sel->get_index(i)];
-		if (!state->list_vector) {
-			continue;
-		}
-		auto &list_vec = *state->list_vector;
-		auto &list_vec_to_append = ListVector::GetEntry(list_vec);
-		ListVector::Append(result, list_vec_to_append, ListVector::GetListSize(list_vec));
+		auto &list_vec_to_append = ListVector::GetEntry(state_lv);
+		ListVector::Append(result, list_vec_to_append, state_lv_count);
 	}
 }
 
@@ -114875,6 +114869,9 @@ void ClientContext::LogQueryInternal(ClientContextLock &, const string &query) {
 
 unique_ptr<QueryResult> ClientContext::Query(unique_ptr<SQLStatement> statement, bool allow_stream_result) {
 	auto pending_query = PendingQuery(move(statement), allow_stream_result);
+	if (!pending_query->success) {
+		return make_unique<MaterializedQueryResult>(pending_query->error);
+	}
 	return pending_query->Execute();
 }