duckdb 0.5.2-dev1468.0 → 0.5.2-dev1479.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.5.2-dev1468.0",
+  "version": "0.5.2-dev1479.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -27954,6 +27954,16 @@ struct IntegerCastOperation {
 		return true;
 	}
 
+	template <class T, bool NEGATIVE>
+	static bool HandleHexDigit(T &state, uint8_t digit) {
+		using result_t = typename T::Result;
+		if (state.result > (NumericLimits<result_t>::Maximum() - digit) / 16) {
+			return false;
+		}
+		state.result = state.result * 16 + digit;
+		return true;
+	}
+
 	template <class T, bool NEGATIVE>
 	static bool HandleExponent(T &state, int32_t exponent) {
 		using result_t = typename T::Result;
@@ -28078,6 +28088,36 @@ static bool IntegerCastLoop(const char *buf, idx_t len, T &result, bool strict)
 	return pos > start_pos;
 }
 
+template <class T, bool NEGATIVE, bool ALLOW_EXPONENT, class OP = IntegerCastOperation>
+static bool IntegerHexCastLoop(const char *buf, idx_t len, T &result, bool strict) {
+	if (ALLOW_EXPONENT || NEGATIVE) {
+		return false;
+	}
+	idx_t start_pos = 1;
+	idx_t pos = start_pos;
+	char current_char;
+	while (pos < len) {
+		current_char = StringUtil::CharacterToLower(buf[pos]);
+		if (!StringUtil::CharacterIsHex(current_char)) {
+			return false;
+		}
+		uint8_t digit;
+		if (current_char >= 'a') {
+			digit = current_char - 'a' + 10;
+		} else {
+			digit = current_char - '0';
+		}
+		pos++;
+		if (!OP::template HandleHexDigit<T, NEGATIVE>(result, digit)) {
+			return false;
+		}
+	}
+	if (!OP::template Finalize<T, NEGATIVE>(result)) {
+		return false;
+	}
+	return pos > start_pos;
+}
+
 template <class T, bool IS_SIGNED = true, bool ALLOW_EXPONENT = true, class OP = IntegerCastOperation,
           bool ZERO_INITIALIZE = true>
 static bool TryIntegerCast(const char *buf, idx_t len, T &result, bool strict) {
@@ -28091,11 +28131,21 @@ static bool TryIntegerCast(const char *buf, idx_t len, T &result, bool strict) {
 	}
 	int negative = *buf == '-';
 
+	// If it starts with 0x or 0X, we parse it as a hex value
+	int hex = len > 1 && *buf == '0' && (buf[1] == 'x' || buf[1] == 'X');
+
 	if (ZERO_INITIALIZE) {
 		memset(&result, 0, sizeof(T));
 	}
 	if (!negative) {
-		return IntegerCastLoop<T, false, ALLOW_EXPONENT, OP>(buf, len, result, strict);
+		if (hex) {
+			// Skip the 0x
+			buf++;
+			len--;
+			return IntegerHexCastLoop<T, false, false, OP>(buf, len, result, strict);
+		} else {
+			return IntegerCastLoop<T, false, ALLOW_EXPONENT, OP>(buf, len, result, strict);
+		}
 	} else {
 		if (!IS_SIGNED) {
 			// Need to check if its not -0
@@ -28601,6 +28651,11 @@ struct HugeIntegerCastOperation {
 		return true;
 	}
 
+	template <class T, bool NEGATIVE>
+	static bool HandleHexDigit(T &result, uint8_t digit) {
+		return false;
+	}
+
 	template <class T, bool NEGATIVE>
 	static bool HandleExponent(T &result, int32_t exponent) {
 		if (!result.Flush()) {
@@ -28696,6 +28751,11 @@ struct DecimalCastOperation {
 		return true;
 	}
 
+	template <class T, bool NEGATIVE>
+	static bool HandleHexDigit(T &state, uint8_t digit) {
+		return false;
+	}
+
 	template <class T, bool NEGATIVE>
 	static void RoundUpResult(T &state) {
 		if (NEGATIVE) {
@@ -64951,10 +65011,9 @@ void PhysicalHashAggregate::Combine(ExecutionContext &context, GlobalSinkState &
 
 //! REGULAR FINALIZE EVENT
 
-class HashAggregateFinalizeEvent : public BasePipelineEvent {
+class HashAggregateMergeEvent : public BasePipelineEvent {
 public:
-	HashAggregateFinalizeEvent(const PhysicalHashAggregate &op_p, HashAggregateGlobalState &gstate_p,
-	                           Pipeline *pipeline_p)
+	HashAggregateMergeEvent(const PhysicalHashAggregate &op_p, HashAggregateGlobalState &gstate_p, Pipeline *pipeline_p)
 	    : BasePipelineEvent(*pipeline_p), op(op_p), gstate(gstate_p) {
 	}
 
@@ -64976,14 +65035,61 @@ public:
 	}
 };
 
+//! REGULAR FINALIZE FROM DISTINCT FINALIZE
+
+class HashAggregateFinalizeTask : public ExecutorTask {
+public:
+	HashAggregateFinalizeTask(Pipeline &pipeline, shared_ptr<Event> event_p, HashAggregateGlobalState &state_p,
+	                          ClientContext &context, const PhysicalHashAggregate &op)
+	    : ExecutorTask(pipeline.executor), pipeline(pipeline), event(move(event_p)), gstate(state_p), context(context),
+	      op(op) {
+	}
+
+	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
+		op.FinalizeInternal(pipeline, *event, context, gstate, false);
+		D_ASSERT(!gstate.finished);
+		gstate.finished = true;
+		event->FinishTask();
+		return TaskExecutionResult::TASK_FINISHED;
+	}
+
+private:
+	Pipeline &pipeline;
+	shared_ptr<Event> event;
+	HashAggregateGlobalState &gstate;
+	ClientContext &context;
+	const PhysicalHashAggregate &op;
+};
+
+class HashAggregateFinalizeEvent : public BasePipelineEvent {
+public:
+	HashAggregateFinalizeEvent(const PhysicalHashAggregate &op_p, HashAggregateGlobalState &gstate_p,
+	                           Pipeline *pipeline_p, ClientContext &context)
+	    : BasePipelineEvent(*pipeline_p), op(op_p), gstate(gstate_p), context(context) {
+	}
+
+	const PhysicalHashAggregate &op;
+	HashAggregateGlobalState &gstate;
+	ClientContext &context;
+
+public:
+	void Schedule() override {
+		vector<unique_ptr<Task>> tasks;
+		tasks.push_back(make_unique<HashAggregateFinalizeTask>(*pipeline, shared_from_this(), gstate, context, op));
+		D_ASSERT(!tasks.empty());
+		SetTasks(move(tasks));
+	}
+};
+
 //! DISTINCT FINALIZE TASK
 
 class HashDistinctAggregateFinalizeTask : public ExecutorTask {
 public:
 	HashDistinctAggregateFinalizeTask(Pipeline &pipeline, shared_ptr<Event> event_p, HashAggregateGlobalState &state_p,
-	                                  ClientContext &context, const PhysicalHashAggregate &op)
+	                                  ClientContext &context, const PhysicalHashAggregate &op,
+	                                  vector<vector<unique_ptr<GlobalSourceState>>> &global_sources_p)
 	    : ExecutorTask(pipeline.executor), pipeline(pipeline), event(move(event_p)), gstate(state_p), context(context),
-	      op(op) {
+	      op(op), global_sources(global_sources_p) {
 	}
 
 	void AggregateDistinctGrouping(DistinctAggregateCollectionInfo &info,
@@ -64992,6 +65098,7 @@ public:
 		auto &aggregates = info.aggregates;
 		auto &data = *grouping_data.distinct_data;
 		auto &state = *grouping_state.distinct_state;
+		auto &table_state = *grouping_state.table_state;
 
 		ThreadContext temp_thread_context(context);
 		ExecutionContext temp_exec_context(context, temp_thread_context, &pipeline);
@@ -65029,9 +65136,12 @@ public:
 			D_ASSERT(data.info.table_map.count(i));
 			auto table_idx = data.info.table_map.at(i);
 			auto &radix_table_p = data.radix_tables[table_idx];
-			auto &output_chunk = *state.distinct_output_chunks[table_idx];
 
-			auto global_source = radix_table_p->GetGlobalSourceState(context);
+			// Create a duplicate of the output_chunk, because of multi-threading we cant alter the original
+			DataChunk output_chunk;
+			output_chunk.Initialize(context, state.distinct_output_chunks[table_idx]->GetTypes());
+
+			auto &global_source = global_sources[grouping_idx][i];
 			auto local_source = radix_table_p->GetLocalSourceState(temp_exec_context);
 
 			// Fetch all the data from the aggregate ht, and Sink it into the main ht
@@ -65063,11 +65173,11 @@ public:
 				aggregate_input_chunk.SetCardinality(output_chunk);
 
 				// Sink it into the main ht
-				grouping_data.table_data.Sink(temp_exec_context, *grouping_state.table_state, *temp_local_state,
-				                              group_chunk, aggregate_input_chunk, {i});
+				grouping_data.table_data.Sink(temp_exec_context, table_state, *temp_local_state, group_chunk,
+				                              aggregate_input_chunk, {i});
 			}
 		}
-		grouping_data.table_data.Combine(temp_exec_context, *grouping_state.table_state, *temp_local_state);
+		grouping_data.table_data.Combine(temp_exec_context, table_state, *temp_local_state);
 	}
 
 	TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
@@ -65078,9 +65188,6 @@ public:
 			auto &grouping_state = gstate.grouping_states[i];
 			AggregateDistinctGrouping(info, grouping, grouping_state, i);
 		}
-		op.FinalizeInternal(pipeline, *event, context, gstate, false);
-		D_ASSERT(!gstate.finished);
-		gstate.finished = true;
 		event->FinishTask();
 		return TaskExecutionResult::TASK_FINISHED;
 	}
@@ -65091,6 +65198,7 @@ private:
 	HashAggregateGlobalState &gstate;
 	ClientContext &context;
 	const PhysicalHashAggregate &op;
+	vector<vector<unique_ptr<GlobalSourceState>>> &global_sources;
 };
 
 //! DISTINCT FINALIZE EVENT
@@ -65105,15 +65213,60 @@ public:
 	const PhysicalHashAggregate &op;
 	HashAggregateGlobalState &gstate;
 	ClientContext &context;
+	//! The GlobalSourceStates for all the radix tables of the distinct aggregates
+	vector<vector<unique_ptr<GlobalSourceState>>> global_sources;
 
 public:
 	void Schedule() override {
+		global_sources = CreateGlobalSources();
+
 		vector<unique_ptr<Task>> tasks;
-		tasks.push_back(
-		    make_unique<HashDistinctAggregateFinalizeTask>(*pipeline, shared_from_this(), gstate, context, op));
+		auto &scheduler = TaskScheduler::GetScheduler(context);
+		auto number_of_threads = scheduler.NumberOfThreads();
+		tasks.reserve(number_of_threads);
+		for (int32_t i = 0; i < number_of_threads; i++) {
+			tasks.push_back(make_unique<HashDistinctAggregateFinalizeTask>(*pipeline, shared_from_this(), gstate,
+			                                                               context, op, global_sources));
+		}
 		D_ASSERT(!tasks.empty());
 		SetTasks(move(tasks));
 	}
+
+	void FinishEvent() override {
+		//! Now that everything is added to the main ht, we can actually finalize
+		auto new_event = make_shared<HashAggregateFinalizeEvent>(op, gstate, pipeline.get(), context);
+		this->InsertEvent(move(new_event));
+	}
+
+private:
+	vector<vector<unique_ptr<GlobalSourceState>>> CreateGlobalSources() {
+		vector<vector<unique_ptr<GlobalSourceState>>> grouping_sources;
+		grouping_sources.reserve(op.groupings.size());
+		for (idx_t grouping_idx = 0; grouping_idx < op.groupings.size(); grouping_idx++) {
+			auto &grouping = op.groupings[grouping_idx];
+			auto &data = *grouping.distinct_data;
+
+			vector<unique_ptr<GlobalSourceState>> aggregate_sources;
+			aggregate_sources.reserve(op.grouped_aggregate_data.aggregates.size());
+
+			for (idx_t i = 0; i < op.grouped_aggregate_data.aggregates.size(); i++) {
+				auto &aggregate = op.grouped_aggregate_data.aggregates[i];
+				auto &aggr = (BoundAggregateExpression &)*aggregate;
+
+				if (!aggr.IsDistinct()) {
+					aggregate_sources.push_back(nullptr);
+					continue;
+				}
+
+				D_ASSERT(data.info.table_map.count(i));
+				auto table_idx = data.info.table_map.at(i);
+				auto &radix_table_p = data.radix_tables[table_idx];
+				aggregate_sources.push_back(radix_table_p->GetGlobalSourceState(context));
+			}
+			grouping_sources.push_back(move(aggregate_sources));
+		}
+		return grouping_sources;
+	}
 };
 
 //! DISTINCT COMBINE EVENT
@@ -65145,12 +65298,12 @@ public:
 			}
 		}
 
+		D_ASSERT(!tasks.empty());
+		SetTasks(move(tasks));
+
 		//! Now that all tables are combined, it's time to do the distinct aggregations
 		auto new_event = make_shared<HashDistinctAggregateFinalizeEvent>(op, gstate, *pipeline, client);
 		this->InsertEvent(move(new_event));
-
-		D_ASSERT(!tasks.empty());
-		SetTasks(move(tasks));
 	}
 };
 
@@ -65214,7 +65367,7 @@ SinkFinalizeType PhysicalHashAggregate::FinalizeInternal(Pipeline &pipeline, Eve
 		}
 	}
 	if (any_partitioned) {
-		auto new_event = make_shared<HashAggregateFinalizeEvent>(*this, gstate, &pipeline);
+		auto new_event = make_shared<HashAggregateMergeEvent>(*this, gstate, &pipeline);
 		event.InsertEvent(move(new_event));
 	}
 	return SinkFinalizeType::READY;

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "4f495e8eea"
-#define DUCKDB_VERSION "v0.5.2-dev1468"
+#define DUCKDB_SOURCE_ID "d95b0d50d3"
+#define DUCKDB_VERSION "v0.5.2-dev1479"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -6279,6 +6279,9 @@ public:
 	DUCKDB_API static bool CharacterIsDigit(char c) {
 		return c >= '0' && c <= '9';
 	}
+	DUCKDB_API static bool CharacterIsHex(char c) {
+		return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+	}
 	DUCKDB_API static char CharacterToLower(char c) {
 		if (c >= 'A' && c <= 'Z') {
 			return c - ('A' - 'a');