duckdb 0.4.1-dev1530.0 → 0.4.1-dev1544.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.4.1-dev1530.0",
+  "version": "0.4.1-dev1544.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -68473,7 +68473,7 @@ public:
 
 class HashJoinGlobalState : public GlobalSinkState {
 public:
-	HashJoinGlobalState() {
+	HashJoinGlobalState() : scanned_data(false) {
 	}
 
 	//! The HT used by the join
@@ -68482,6 +68482,8 @@ public:
 	unique_ptr<PerfectHashJoinExecutor> perfect_join_executor;
 	//! Whether or not the hash table has been finalized
 	bool finalized = false;
+	//! Whether or not we have started scanning data using GetData
+	atomic<bool> scanned_data;
 };
 
 unique_ptr<GlobalSinkState> PhysicalHashJoin::GetGlobalSinkState(ClientContext &context) const {
@@ -68640,6 +68642,7 @@ OperatorResultType PhysicalHashJoin::Execute(ExecutionContext &context, DataChun
 	auto &state = (PhysicalHashJoinState &)state_p;
 	auto &sink = (HashJoinGlobalState &)*sink_state;
 	D_ASSERT(sink.finalized);
+	D_ASSERT(!sink.scanned_data);
 
 	if (sink.hash_table->Count() == 0 && EmptyResultIfRHSIsEmpty()) {
 		return OperatorResultType::FINISHED;
@@ -68702,6 +68705,7 @@ void PhysicalHashJoin::GetData(ExecutionContext &context, DataChunk &chunk, Glob
 	// check if we need to scan any unmatched tuples from the RHS for the full/right outer join
 	auto &sink = (HashJoinGlobalState &)*sink_state;
 	auto &state = (HashJoinScanState &)gstate;
+	sink.scanned_data = true;
 	sink.hash_table->ScanFullOuter(chunk, state.ht_scan_state);
 }
 
@@ -147624,52 +147628,96 @@ struct PipelineEventStack {
 	Event *pipeline_complete_event;
 };
 
-Pipeline *Executor::ScheduleUnionPipeline(const shared_ptr<Pipeline> &pipeline, const Pipeline *parent,
-                                          event_map_t &event_map, vector<shared_ptr<Event>> &events) {
-	pipeline->Ready();
+using event_map_t = unordered_map<const Pipeline *, PipelineEventStack>;
+
+struct ScheduleEventData {
+	ScheduleEventData(const vector<shared_ptr<Pipeline>> &pipelines,
+	                  unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &child_pipelines,
+	                  unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &union_pipelines,
+	                  unordered_map<Pipeline *, vector<Pipeline *>> &child_dependencies,
+	                  vector<shared_ptr<Event>> &events, bool initial_schedule)
+	    : pipelines(pipelines), child_pipelines(child_pipelines), union_pipelines(union_pipelines),
+	      child_dependencies(child_dependencies), events(events), initial_schedule(initial_schedule) {
+	}
+
+	const vector<shared_ptr<Pipeline>> &pipelines;
+	unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &child_pipelines;
+	unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &union_pipelines;
+	unordered_map<Pipeline *, vector<Pipeline *>> &child_dependencies;
+	unordered_map<Pipeline *, vector<Pipeline *>> scheduled_pipelines;
+	vector<shared_ptr<Event>> &events;
+	bool initial_schedule;
+	event_map_t event_map;
+};
 
+void Executor::SchedulePipeline(const shared_ptr<Pipeline> &pipeline, ScheduleEventData &event_data,
+                                vector<Pipeline *> &scheduled_pipelines) {
 	D_ASSERT(pipeline);
-	auto pipeline_event = make_shared<PipelineEvent>(pipeline);
 
-	auto parent_stack_entry = event_map.find(parent);
-	D_ASSERT(parent_stack_entry != event_map.end());
+	auto &event_map = event_data.event_map;
+	auto &events = event_data.events;
+	auto &union_pipelines = event_data.union_pipelines;
+	pipeline->Ready();
 
-	auto &parent_stack = parent_stack_entry->second;
+	auto pipeline_event = make_shared<PipelineEvent>(pipeline);
 
 	PipelineEventStack stack;
 	stack.pipeline_event = pipeline_event.get();
-	stack.pipeline_finish_event = parent_stack.pipeline_finish_event;
-	stack.pipeline_complete_event = parent_stack.pipeline_complete_event;
+	if (!scheduled_pipelines.empty()) {
+		// this pipeline has a parent pipeline - i.e. it is scheduled as part of a `UNION`
+		// set up the events
+		auto parent = scheduled_pipelines.back();
+		auto parent_stack_entry = event_map.find(parent);
+		D_ASSERT(parent_stack_entry != event_map.end());
+
+		auto &parent_stack = parent_stack_entry->second;
+		stack.pipeline_finish_event = parent_stack.pipeline_finish_event;
+		stack.pipeline_complete_event = parent_stack.pipeline_complete_event;
+
+		stack.pipeline_event->AddDependency(*parent_stack.pipeline_event);
+		parent_stack.pipeline_finish_event->AddDependency(*pipeline_event);
+	} else {
+		// stand-alone pipeline
+		auto pipeline_finish_event = make_shared<PipelineFinishEvent>(pipeline);
+		auto pipeline_complete_event =
+		    make_shared<PipelineCompleteEvent>(pipeline->executor, event_data.initial_schedule);
+
+		pipeline_finish_event->AddDependency(*pipeline_event);
+		pipeline_complete_event->AddDependency(*pipeline_finish_event);
 
-	stack.pipeline_event->AddDependency(*parent_stack.pipeline_event);
-	parent_stack.pipeline_finish_event->AddDependency(*pipeline_event);
+		stack.pipeline_finish_event = pipeline_finish_event.get();
+		stack.pipeline_complete_event = pipeline_complete_event.get();
+
+		events.push_back(move(pipeline_finish_event));
+		events.push_back(move(pipeline_complete_event));
+	}
 
 	events.push_back(move(pipeline_event));
 	event_map.insert(make_pair(pipeline.get(), stack));
 
-	auto parent_pipeline = pipeline.get();
-
+	scheduled_pipelines.push_back(pipeline.get());
 	auto union_entry = union_pipelines.find(pipeline.get());
 	if (union_entry != union_pipelines.end()) {
 		for (auto &entry : union_entry->second) {
-			parent_pipeline = ScheduleUnionPipeline(entry, parent_pipeline, event_map, events);
+			SchedulePipeline(entry, event_data, scheduled_pipelines);
 		}
 	}
-
-	return parent_pipeline;
 }
 
-void Executor::ScheduleChildPipeline(Pipeline *parent, const shared_ptr<Pipeline> &pipeline, event_map_t &event_map,
-                                     vector<shared_ptr<Event>> &events) {
+void Executor::ScheduleChildPipeline(Pipeline *parent, const shared_ptr<Pipeline> &pipeline,
+                                     ScheduleEventData &event_data) {
+	auto &events = event_data.events;
+	auto &child_dependencies = event_data.child_dependencies;
 	pipeline->Ready();
 
 	auto child_ptr = pipeline.get();
 	auto dependencies = child_dependencies.find(child_ptr);
-	D_ASSERT(union_pipelines.find(child_ptr) == union_pipelines.end());
+	D_ASSERT(event_data.union_pipelines.find(child_ptr) == event_data.union_pipelines.end());
 	D_ASSERT(dependencies != child_dependencies.end());
 	// create the pipeline event and the event stack
 	auto pipeline_event = make_shared<PipelineEvent>(pipeline);
 
+	auto &event_map = event_data.event_map;
 	auto parent_entry = event_map.find(parent);
 	PipelineEventStack stack;
 	stack.pipeline_event = pipeline_event.get();
@@ -147678,17 +147726,26 @@ void Executor::ScheduleChildPipeline(Pipeline *parent, const shared_ptr<Pipeline
 
 	// set up the dependencies for this child pipeline
 	unordered_set<Event *> finish_events;
-	for (auto &dep : dependencies->second) {
-		auto dep_entry = event_map.find(dep);
-		D_ASSERT(dep_entry != event_map.end());
-		D_ASSERT(dep_entry->second.pipeline_event);
-		D_ASSERT(dep_entry->second.pipeline_finish_event);
+	for (auto &main_dep : dependencies->second) {
+		vector<Pipeline *> pipeline_dependencies;
+		auto dep_scheduled = event_data.scheduled_pipelines.find(main_dep);
+		if (dep_scheduled == event_data.scheduled_pipelines.end()) {
+			pipeline_dependencies.push_back(main_dep);
+		} else {
+			pipeline_dependencies = dep_scheduled->second;
+		}
+		for (auto &dep : pipeline_dependencies) {
+			auto dep_entry = event_map.find(dep);
+			D_ASSERT(dep_entry != event_map.end());
+			D_ASSERT(dep_entry->second.pipeline_event);
+			D_ASSERT(dep_entry->second.pipeline_finish_event);
 
-		auto finish_event = dep_entry->second.pipeline_finish_event;
-		stack.pipeline_event->AddDependency(*dep_entry->second.pipeline_event);
-		if (finish_events.find(finish_event) == finish_events.end()) {
-			finish_event->AddDependency(*stack.pipeline_event);
-			finish_events.insert(finish_event);
+			auto finish_event = dep_entry->second.pipeline_finish_event;
+			stack.pipeline_event->AddDependency(*dep_entry->second.pipeline_event);
+			if (finish_events.find(finish_event) == finish_events.end()) {
+				finish_event->AddDependency(*stack.pipeline_event);
+				finish_events.insert(finish_event);
+			}
 		}
 	}
 
@@ -147696,56 +147753,25 @@ void Executor::ScheduleChildPipeline(Pipeline *parent, const shared_ptr<Pipeline
 	event_map.insert(make_pair(child_ptr, stack));
 }
 
-void Executor::SchedulePipeline(const shared_ptr<Pipeline> &pipeline, event_map_t &event_map,
-                                vector<shared_ptr<Event>> &events, bool complete_pipeline) {
-	D_ASSERT(pipeline);
-
-	pipeline->Ready();
-
-	auto pipeline_event = make_shared<PipelineEvent>(pipeline);
-	auto pipeline_finish_event = make_shared<PipelineFinishEvent>(pipeline);
-	auto pipeline_complete_event = make_shared<PipelineCompleteEvent>(pipeline->executor, complete_pipeline);
-
-	PipelineEventStack stack;
-	stack.pipeline_event = pipeline_event.get();
-	stack.pipeline_finish_event = pipeline_finish_event.get();
-	stack.pipeline_complete_event = pipeline_complete_event.get();
-
-	pipeline_finish_event->AddDependency(*pipeline_event);
-	pipeline_complete_event->AddDependency(*pipeline_finish_event);
-
-	events.push_back(move(pipeline_event));
-	events.push_back(move(pipeline_finish_event));
-	events.push_back(move(pipeline_complete_event));
-
-	event_map.insert(make_pair(pipeline.get(), stack));
-
-	auto union_entry = union_pipelines.find(pipeline.get());
-	if (union_entry != union_pipelines.end()) {
-		auto parent_pipeline = pipeline.get();
-		for (auto &entry : union_entry->second) {
-			parent_pipeline = ScheduleUnionPipeline(entry, parent_pipeline, event_map, events);
-		}
-	}
-}
-
-void Executor::ScheduleEventsInternal(const vector<shared_ptr<Pipeline>> &pipelines,
-                                      unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &child_pipelines,
-                                      vector<shared_ptr<Event>> &events, bool main_schedule) {
+void Executor::ScheduleEventsInternal(ScheduleEventData &event_data) {
+	auto &events = event_data.events;
 	D_ASSERT(events.empty());
 	// create all the required pipeline events
-	event_map_t event_map;
-	for (auto &pipeline : pipelines) {
-		SchedulePipeline(pipeline, event_map, events, main_schedule);
+	auto &event_map = event_data.event_map;
+	for (auto &pipeline : event_data.pipelines) {
+		vector<Pipeline *> scheduled_pipelines;
+		SchedulePipeline(pipeline, event_data, scheduled_pipelines);
+
+		event_data.scheduled_pipelines[pipeline.get()] = move(scheduled_pipelines);
 	}
 	// schedule child pipelines
-	for (auto &entry : child_pipelines) {
+	for (auto &entry : event_data.child_pipelines) {
 		// iterate in reverse order
 		// since child entries are added from top to bottom
 		// dependencies are in reverse order (bottom to top)
 		for (idx_t i = entry.second.size(); i > 0; i--) {
 			auto &child_entry = entry.second[i - 1];
-			ScheduleChildPipeline(entry.first, child_entry, event_map, events);
+			ScheduleChildPipeline(entry.first, child_entry, event_data);
 		}
 	}
 	// set up the dependencies between pipeline events
@@ -147770,12 +147796,16 @@ void Executor::ScheduleEventsInternal(const vector<shared_ptr<Pipeline>> &pipeli
 }
 
 void Executor::ScheduleEvents() {
-	ScheduleEventsInternal(pipelines, child_pipelines, events);
+	ScheduleEventData event_data(pipelines, child_pipelines, union_pipelines, child_dependencies, events, true);
+	ScheduleEventsInternal(event_data);
 }
 
 void Executor::ReschedulePipelines(const vector<shared_ptr<Pipeline>> &pipelines, vector<shared_ptr<Event>> &events) {
 	unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> child_pipelines;
-	ScheduleEventsInternal(pipelines, child_pipelines, events, false);
+	unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> union_pipelines;
+	unordered_map<Pipeline *, vector<Pipeline *>> child_dependencies;
+	ScheduleEventData event_data(pipelines, child_pipelines, union_pipelines, child_dependencies, events, false);
+	ScheduleEventsInternal(event_data);
 }
 
 void Executor::ExtractPipelines(shared_ptr<Pipeline> &pipeline, vector<shared_ptr<Pipeline>> &result) {

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "cadbb22d0"
-#define DUCKDB_VERSION "v0.4.1-dev1530"
+#define DUCKDB_SOURCE_ID "cc3979656"
+#define DUCKDB_VERSION "v0.4.1-dev1544"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -11612,8 +11612,7 @@ class Task;
 
 struct PipelineEventStack;
 struct ProducerToken;
-
-using event_map_t = unordered_map<const Pipeline *, PipelineEventStack>;
+struct ScheduleEventData;
 
 class Executor {
 	friend class Pipeline;
@@ -11677,16 +11676,12 @@ private:
 	void InitializeInternal(PhysicalOperator *physical_plan);
 
 	void ScheduleEvents();
-	void ScheduleEventsInternal(const vector<shared_ptr<Pipeline>> &pipelines,
-	                            unordered_map<Pipeline *, vector<shared_ptr<Pipeline>>> &child_pipelines,
-	                            vector<shared_ptr<Event>> &events, bool main_schedule = true);
-
-	void SchedulePipeline(const shared_ptr<Pipeline> &pipeline, event_map_t &event_map,
-	                      vector<shared_ptr<Event>> &events, bool complete_pipeline);
-	Pipeline *ScheduleUnionPipeline(const shared_ptr<Pipeline> &pipeline, const Pipeline *parent,
-	                                event_map_t &event_map, vector<shared_ptr<Event>> &events);
-	void ScheduleChildPipeline(Pipeline *parent, const shared_ptr<Pipeline> &pipeline, event_map_t &event_map,
-	                           vector<shared_ptr<Event>> &events);
+	static void ScheduleEventsInternal(ScheduleEventData &event_data);
+
+	static void SchedulePipeline(const shared_ptr<Pipeline> &pipeline, ScheduleEventData &event_data,
+	                             vector<Pipeline *> &scheduled_pipelines);
+	static void ScheduleChildPipeline(Pipeline *parent, const shared_ptr<Pipeline> &pipeline,
+	                                  ScheduleEventData &event_data);
 	void ExtractPipelines(shared_ptr<Pipeline> &pipeline, vector<shared_ptr<Pipeline>> &result);
 	bool NextExecutor();