duckdb 0.3.5-dev242.0 → 0.3.5-dev256.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.5-dev242.0",
+  "version": "0.3.5-dev256.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -58827,10 +58827,10 @@ static void SortCollectionForPartition(WindowOperatorState &state, BoundWindowEx
 
 	// fuse input and sort collection into one
 	// (sorting columns are not decoded, and we need them later)
-	ChunkCollection payload;
-	payload.Fuse(input);
-	payload.Fuse(over);
-	auto payload_types = payload.Types();
+	auto payload_types = input.Types();
+	payload_types.insert(payload_types.end(), over.Types().begin(), over.Types().end());
+	DataChunk payload_chunk;
+	payload_chunk.InitializeEmpty(payload_types);
 
 	// initialise partitioning memory
 	// to minimise copying, we fill up a chunk and then sink it.
@@ -58856,8 +58856,15 @@ static void SortCollectionForPartition(WindowOperatorState &state, BoundWindowEx
 	// sink collection chunks into row format
 	const idx_t chunk_count = over.ChunkCount();
 	for (idx_t i = 0; i < chunk_count; i++) {
+		auto &input_chunk = *input.Chunks()[i];
+		for (idx_t col_idx = 0; col_idx < input_chunk.ColumnCount(); ++col_idx) {
+			payload_chunk.data[col_idx].Reference(input_chunk.data[col_idx]);
+		}
 		auto &over_chunk = *over.Chunks()[i];
-		auto &payload_chunk = *payload.Chunks()[i];
+		for (idx_t col_idx = 0; col_idx < over_chunk.ColumnCount(); ++col_idx) {
+			payload_chunk.data[input_chunk.ColumnCount() + col_idx].Reference(over_chunk.data[col_idx]);
+		}
+		payload_chunk.SetCardinality(input_chunk);
 
 		// Extract the hash partition, if any
 		if (hashes) {
@@ -83735,7 +83742,7 @@ static void HistogramFinalize(Vector &state_vector, FunctionData *, Vector &resu
 		list_struct_data = FlatVector::GetData<list_entry_t>(*count_list);
 		list_struct_data[rid].length = ListVector::GetListSize(*count_list) - old_len;
 		list_struct_data[rid].offset = old_len;
-		old_len = list_struct_data[rid].length;
+		old_len += list_struct_data[rid].length;
 	}
 }
 
@@ -110790,6 +110797,19 @@ void duckdb_function_set_error(duckdb_function_info info, const char *error) {
 
 
 
+using duckdb::DatabaseData;
+
+void duckdb_execute_tasks(duckdb_database database, idx_t max_tasks) {
+	if (!database) {
+		return;
+	}
+	auto wrapper = (DatabaseData *)database;
+	auto &scheduler = duckdb::TaskScheduler::GetScheduler(*wrapper->database->instance);
+	scheduler.ExecuteTasks(max_tasks);
+}
+
+
+
 
 
 
@@ -112850,6 +112870,14 @@ struct ExplainOutputSetting {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct ExternalThreadsSetting {
+	static constexpr const char *Name = "external_threads";
+	static constexpr const char *Description = "The number of external threads that work on DuckDB tasks.";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BIGINT;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct ForceCompressionSetting {
 	static constexpr const char *Name = "force_compression";
 	static constexpr const char *Description = "DEBUG SETTING: forces a specific compression method to be used";
@@ -112997,6 +113025,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(EnableProfilingSetting),
                                                  DUCKDB_LOCAL(EnableProgressBarSetting),
                                                  DUCKDB_LOCAL(ExplainOutputSetting),
+                                                 DUCKDB_GLOBAL(ExternalThreadsSetting),
                                                  DUCKDB_GLOBAL(ForceCompressionSetting),
                                                  DUCKDB_LOCAL(LogQueryPathSetting),
                                                  DUCKDB_GLOBAL(MaximumMemorySetting),
@@ -113772,7 +113801,7 @@ void DatabaseInstance::Initialize(const char *path, DBConfig *new_config) {
 	    make_unique<StorageManager>(*this, path ? string(path) : string(), config.access_mode == AccessMode::READ_ONLY);
 	catalog = make_unique<Catalog>(*this);
 	transaction_manager = make_unique<TransactionManager>(*this);
-	scheduler = make_unique<TaskScheduler>();
+	scheduler = make_unique<TaskScheduler>(*this);
 	object_cache = make_unique<ObjectCache>();
 	connection_manager = make_unique<ConnectionManager>();
 
@@ -113862,6 +113891,7 @@ void DatabaseInstance::Configure(DBConfig &new_config) {
 	} else {
 		config.maximum_threads = new_config.maximum_threads;
 	}
+	config.external_threads = new_config.external_threads;
 	config.load_extensions = new_config.load_extensions;
 	config.force_compression = new_config.force_compression;
 	config.allocator = move(new_config.allocator);
@@ -126514,6 +126544,18 @@ Value ExplainOutputSetting::GetSetting(ClientContext &context) {
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// External Threads Setting
+//===--------------------------------------------------------------------===//
+void ExternalThreadsSetting::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	config.external_threads = input.GetValue<int64_t>();
+}
+
+Value ExternalThreadsSetting::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	return Value::BIGINT(config.external_threads);
+}
+
 //===--------------------------------------------------------------------===//
 // Force Compression
 //===--------------------------------------------------------------------===//
@@ -141519,7 +141561,7 @@ ProducerToken::ProducerToken(TaskScheduler &scheduler, unique_ptr<QueueProducerT
 ProducerToken::~ProducerToken() {
 }
 
-TaskScheduler::TaskScheduler() : queue(make_unique<ConcurrentQueue>()) {
+TaskScheduler::TaskScheduler(DatabaseInstance &db) : db(db), queue(make_unique<ConcurrentQueue>()) {
 }
 
 TaskScheduler::~TaskScheduler() {
@@ -141556,7 +141598,7 @@ void TaskScheduler::ExecuteForever(atomic<bool> *marker) {
 	// loop until the marker is set to false
 	while (*marker) {
 		// wait for a signal with a timeout; the timeout allows us to periodically check
-		queue->semaphore.wait(TASK_TIMEOUT_USECS);
+		queue->semaphore.wait();
 		if (queue->q.try_dequeue(task)) {
 			task->Execute(TaskExecutionMode::PROCESS_ALL);
 			task.reset();
@@ -141567,6 +141609,26 @@ void TaskScheduler::ExecuteForever(atomic<bool> *marker) {
 #endif
 }
 
+void TaskScheduler::ExecuteTasks(idx_t max_tasks) {
+#ifndef DUCKDB_NO_THREADS
+	unique_ptr<Task> task;
+	for (idx_t i = 0; i < max_tasks; i++) {
+		queue->semaphore.wait(TASK_TIMEOUT_USECS);
+		if (!queue->q.try_dequeue(task)) {
+			return;
+		}
+		try {
+			task->Execute(TaskExecutionMode::PROCESS_ALL);
+			task.reset();
+		} catch (...) {
+			return;
+		}
+	}
+#else
+	throw NotImplementedException("DuckDB was compiled without threads! Background thread loop is not allowed.");
+#endif
+}
+
 #ifndef DUCKDB_NO_THREADS
 static void ThreadExecuteTasks(TaskScheduler *scheduler, atomic<bool> *marker) {
 	scheduler->ExecuteForever(marker);
@@ -141574,7 +141636,8 @@ static void ThreadExecuteTasks(TaskScheduler *scheduler, atomic<bool> *marker) {
 #endif
 
 int32_t TaskScheduler::NumberOfThreads() {
-	return threads.size() + 1;
+	auto &config = DBConfig::GetConfig(db);
+	return threads.size() + config.external_threads + 1;
 }
 
 void TaskScheduler::SetThreads(int32_t n) {
@@ -141596,6 +141659,20 @@ void TaskScheduler::SetThreadsInternal(int32_t n) {
 		return;
 	}
 	idx_t new_thread_count = n - 1;
+	if (threads.size() > new_thread_count) {
+		// we are reducing the number of threads: clear all threads first
+		for (idx_t i = 0; i < threads.size(); i++) {
+			*markers[i] = false;
+		}
+		queue->semaphore.signal(threads.size());
+		// now join the threads to ensure they are fully stopped before erasing them
+		for (idx_t i = 0; i < threads.size(); i++) {
+			threads[i]->internal_thread->join();
+		}
+		// erase the threads/markers
+		threads.clear();
+		markers.clear();
+	}
 	if (threads.size() < new_thread_count) {
 		// we are increasing the number of threads: launch them and run tasks on them
 		idx_t create_new_threads = new_thread_count - threads.size();
@@ -141608,18 +141685,6 @@ void TaskScheduler::SetThreadsInternal(int32_t n) {
 			threads.push_back(move(thread_wrapper));
 			markers.push_back(move(marker));
 		}
-	} else if (threads.size() > new_thread_count) {
-		// we are reducing the number of threads: cancel any threads exceeding new_thread_count
-		for (idx_t i = new_thread_count; i < threads.size(); i++) {
-			*markers[i] = false;
-		}
-		// now join the threads to ensure they are fully stopped before erasing them
-		for (idx_t i = new_thread_count; i < threads.size(); i++) {
-			threads[i]->internal_thread->join();
-		}
-		// erase the threads/markers
-		threads.resize(new_thread_count);
-		markers.resize(new_thread_count);
 	}
 #endif
 }

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "a90b3be6d"
-#define DUCKDB_VERSION "v0.3.5-dev242"
+#define DUCKDB_SOURCE_ID "6962a87a9"
+#define DUCKDB_VERSION "v0.3.5-dev256"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -10593,11 +10593,11 @@ struct ProducerToken {
 
 //! The TaskScheduler is responsible for managing tasks and threads
 class TaskScheduler {
-	// timeout for semaphore wait, default 50ms
-	constexpr static int64_t TASK_TIMEOUT_USECS = 50000;
+	// timeout for semaphore wait, default 5ms
+	constexpr static int64_t TASK_TIMEOUT_USECS = 5000;
 
 public:
-	TaskScheduler();
+	TaskScheduler(DatabaseInstance &db);
 	~TaskScheduler();
 
 	static TaskScheduler &GetScheduler(ClientContext &context);
@@ -10610,6 +10610,8 @@ public:
 	bool GetTaskFromProducer(ProducerToken &token, unique_ptr<Task> &task);
 	//! Run tasks forever until "marker" is set to false, "marker" must remain valid until the thread is joined
 	void ExecuteForever(atomic<bool> *marker);
+	//! Run tasks until `max_tasks` have been completed, or until there are no more tasks available
+	void ExecuteTasks(idx_t max_tasks);
 
 	//! Sets the amount of active threads executing tasks for the system; n-1 background threads will be launched.
 	//! The main thread will also be used for execution
@@ -10620,6 +10622,8 @@ public:
 private:
 	void SetThreadsInternal(int32_t n);
 
+private:
+	DatabaseInstance &db;
 	//! The task queue
 	unique_ptr<ConcurrentQueue> queue;
 	//! The active background threads of the task scheduler
@@ -16893,6 +16897,19 @@ Closes the result and de-allocates all memory allocated for the arrow result.
 */
 DUCKDB_API void duckdb_destroy_arrow(duckdb_arrow *result);
 
+//===--------------------------------------------------------------------===//
+// Threading Information
+//===--------------------------------------------------------------------===//
+/*!
+Execute DuckDB tasks on this thread.
+
+Will return after `max_tasks` have been executed, or if there are no more tasks present.
+
+* database: The database object to execute tasks for
+* max_tasks: The maximum amount of tasks to execute
+*/
+DUCKDB_API void duckdb_execute_tasks(duckdb_database database, idx_t max_tasks);
+
 #ifdef __cplusplus
 }
 #endif
@@ -18141,6 +18158,8 @@ public:
 	idx_t maximum_memory = (idx_t)-1;
 	//! The maximum amount of CPU threads used by the database system. Default: all available.
 	idx_t maximum_threads = (idx_t)-1;
+	//! The number of external threads that work on DuckDB tasks. Default: none.
+	idx_t external_threads = 0;
 	//! Whether or not to create and use a temporary directory to store intermediates that do not fit in memory
 	bool use_temporary_directory = true;
 	//! Directory to store temporary structures that do not fit in memory