duckdb 1.3.3-dev0.0 → 1.3.3-dev4.0

package/src/duckdb/src/include/duckdb_extension.h

@@ -199,7 +199,7 @@ typedef struct {
 	duckdb_value (*duckdb_create_timestamp)(duckdb_timestamp input);
 	duckdb_value (*duckdb_create_interval)(duckdb_interval input);
 	duckdb_value (*duckdb_create_blob)(const uint8_t *data, idx_t length);
-	duckdb_value (*duckdb_create_varint)(duckdb_varint input);
+	duckdb_value (*duckdb_create_bignum)(duckdb_bignum input);
 	duckdb_value (*duckdb_create_decimal)(duckdb_decimal input);
 	duckdb_value (*duckdb_create_bit)(duckdb_bit input);
 	duckdb_value (*duckdb_create_uuid)(duckdb_uhugeint input);
@@ -223,7 +223,7 @@ typedef struct {
 	duckdb_interval (*duckdb_get_interval)(duckdb_value val);
 	duckdb_logical_type (*duckdb_get_value_type)(duckdb_value val);
 	duckdb_blob (*duckdb_get_blob)(duckdb_value val);
-	duckdb_varint (*duckdb_get_varint)(duckdb_value val);
+	duckdb_bignum (*duckdb_get_bignum)(duckdb_value val);
 	duckdb_decimal (*duckdb_get_decimal)(duckdb_value val);
 	duckdb_bit (*duckdb_get_bit)(duckdb_value val);
 	duckdb_uhugeint (*duckdb_get_uuid)(duckdb_value val);
@@ -540,6 +540,42 @@ typedef struct {
 #ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
 	duckdb_state (*duckdb_append_default_to_chunk)(duckdb_appender appender, duckdb_data_chunk chunk, idx_t col,
 	                                               idx_t row);
+	duckdb_error_data (*duckdb_appender_error_data)(duckdb_appender appender);
+	duckdb_state (*duckdb_appender_create_query)(duckdb_connection connection, const char *query, idx_t column_count,
+	                                             duckdb_logical_type *types, const char *table_name,
+	                                             const char **column_names, duckdb_appender *out_appender);
+#endif
+
+// New arrow interface functions
+#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
+	duckdb_error_data (*duckdb_to_arrow_schema)(duckdb_arrow_options arrow_options, duckdb_logical_type *types,
+	                                            const char **names, idx_t column_count, struct ArrowSchema *out_schema);
+	duckdb_error_data (*duckdb_data_chunk_to_arrow)(duckdb_arrow_options arrow_options, duckdb_data_chunk chunk,
+	                                                struct ArrowArray *out_arrow_array);
+	duckdb_error_data (*duckdb_schema_from_arrow)(duckdb_connection connection, struct ArrowSchema *schema,
+	                                              duckdb_arrow_converted_schema *out_types);
+	duckdb_error_data (*duckdb_data_chunk_from_arrow)(duckdb_connection connection, struct ArrowArray *arrow_array,
+	                                                  duckdb_arrow_converted_schema converted_schema,
+	                                                  duckdb_data_chunk *out_chunk);
+	void (*duckdb_destroy_arrow_converted_schema)(duckdb_arrow_converted_schema *arrow_converted_schema);
+#endif
+
+// New functions for duckdb error data
+#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
+	duckdb_error_data (*duckdb_create_error_data)(duckdb_error_type type, const char *message);
+	void (*duckdb_destroy_error_data)(duckdb_error_data *error_data);
+	duckdb_error_type (*duckdb_error_data_error_type)(duckdb_error_data error_data);
+	const char *(*duckdb_error_data_message)(duckdb_error_data error_data);
+	bool (*duckdb_error_data_has_error)(duckdb_error_data error_data);
+#endif
+
+// API to create and manipulate expressions
+#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
+	void (*duckdb_destroy_expression)(duckdb_expression *expr);
+	duckdb_logical_type (*duckdb_expression_return_type)(duckdb_expression expr);
+	bool (*duckdb_expression_is_foldable)(duckdb_expression expr);
+	duckdb_error_data (*duckdb_expression_fold)(duckdb_client_context context, duckdb_expression expr,
+	                                            duckdb_value *out_value);
 #endif
 
 // New functions around the client context
@@ -548,6 +584,22 @@ typedef struct {
 	void (*duckdb_destroy_client_context)(duckdb_client_context *context);
 	void (*duckdb_connection_get_client_context)(duckdb_connection connection, duckdb_client_context *out_context);
 	duckdb_value (*duckdb_get_table_names)(duckdb_connection connection, const char *query, bool qualified);
+	void (*duckdb_connection_get_arrow_options)(duckdb_connection connection, duckdb_arrow_options *out_arrow_options);
+	void (*duckdb_destroy_arrow_options)(duckdb_arrow_options *arrow_options);
+#endif
+
+// API to get information about the results of a prepared statement
+#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
+	idx_t (*duckdb_prepared_statement_column_count)(duckdb_prepared_statement prepared_statement);
+	const char *(*duckdb_prepared_statement_column_name)(duckdb_prepared_statement prepared_statement, idx_t col_idx);
+	duckdb_logical_type (*duckdb_prepared_statement_column_logical_type)(duckdb_prepared_statement prepared_statement,
+	                                                                     idx_t col_idx);
+	duckdb_type (*duckdb_prepared_statement_column_type)(duckdb_prepared_statement prepared_statement, idx_t col_idx);
+#endif
+
+// New query execution functions
+#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
+	duckdb_arrow_options (*duckdb_result_get_arrow_options)(duckdb_result *result);
 #endif
 
 // New functions around scalar function binding
@@ -558,6 +610,10 @@ typedef struct {
 	void (*duckdb_scalar_function_set_bind_data)(duckdb_bind_info info, void *bind_data,
 	                                             duckdb_delete_callback_t destroy);
 	void *(*duckdb_scalar_function_get_bind_data)(duckdb_function_info info);
+	void *(*duckdb_scalar_function_bind_get_extra_info)(duckdb_bind_info info);
+	idx_t (*duckdb_scalar_function_bind_get_argument_count)(duckdb_bind_info info);
+	duckdb_expression (*duckdb_scalar_function_bind_get_argument)(duckdb_bind_info info, idx_t index);
+	void (*duckdb_scalar_function_set_bind_data_copy)(duckdb_bind_info info, duckdb_copy_callback_t copy);
 #endif
 
 // New string functions that are added
@@ -565,23 +621,32 @@ typedef struct {
 	char *(*duckdb_value_to_string)(duckdb_value value);
 #endif
 
+// New functions around table function binding
+#ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
+	void (*duckdb_table_function_get_client_context)(duckdb_bind_info info, duckdb_client_context *out_context);
+#endif
+
 // New value functions that are added
 #ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
 	duckdb_value (*duckdb_create_map_value)(duckdb_logical_type map_type, duckdb_value *keys, duckdb_value *values,
 	                                        idx_t entry_count);
 	duckdb_value (*duckdb_create_union_value)(duckdb_logical_type union_type, idx_t tag_index, duckdb_value value);
+	duckdb_value (*duckdb_create_time_ns)(duckdb_time_ns input);
+	duckdb_time_ns (*duckdb_get_time_ns)(duckdb_value val);
 #endif
 
-// An API to create new vector types
+// API to create and manipulate vector types
 #ifdef DUCKDB_EXTENSION_API_VERSION_UNSTABLE
 	duckdb_vector (*duckdb_create_vector)(duckdb_logical_type type, idx_t capacity);
 	void (*duckdb_destroy_vector)(duckdb_vector *vector);
-	void (*duckdb_slice_vector)(duckdb_vector vector, duckdb_selection_vector selection, idx_t len);
+	void (*duckdb_slice_vector)(duckdb_vector vector, duckdb_selection_vector sel, idx_t len);
 	void (*duckdb_vector_reference_value)(duckdb_vector vector, duckdb_value value);
 	void (*duckdb_vector_reference_vector)(duckdb_vector to_vector, duckdb_vector from_vector);
 	duckdb_selection_vector (*duckdb_create_selection_vector)(idx_t size);
-	void (*duckdb_destroy_selection_vector)(duckdb_selection_vector vector);
-	sel_t *(*duckdb_selection_vector_get_data_ptr)(duckdb_selection_vector vector);
+	void (*duckdb_destroy_selection_vector)(duckdb_selection_vector sel);
+	sel_t *(*duckdb_selection_vector_get_data_ptr)(duckdb_selection_vector sel);
+	void (*duckdb_vector_copy_sel)(duckdb_vector src, duckdb_vector dst, duckdb_selection_vector sel, idx_t src_count,
+	                               idx_t src_offset, idx_t dst_offset);
 #endif
 
 } duckdb_ext_api_v1;
@@ -699,7 +764,7 @@ typedef struct {
 #define duckdb_create_int64                            duckdb_ext_api.duckdb_create_int64
 #define duckdb_create_hugeint                          duckdb_ext_api.duckdb_create_hugeint
 #define duckdb_create_uhugeint                         duckdb_ext_api.duckdb_create_uhugeint
-#define duckdb_create_varint                           duckdb_ext_api.duckdb_create_varint
+#define duckdb_create_bignum                           duckdb_ext_api.duckdb_create_bignum
 #define duckdb_create_decimal                          duckdb_ext_api.duckdb_create_decimal
 #define duckdb_create_float                            duckdb_ext_api.duckdb_create_float
 #define duckdb_create_double                           duckdb_ext_api.duckdb_create_double
@@ -726,7 +791,7 @@ typedef struct {
 #define duckdb_get_uint64                              duckdb_ext_api.duckdb_get_uint64
 #define duckdb_get_hugeint                             duckdb_ext_api.duckdb_get_hugeint
 #define duckdb_get_uhugeint                            duckdb_ext_api.duckdb_get_uhugeint
-#define duckdb_get_varint                              duckdb_ext_api.duckdb_get_varint
+#define duckdb_get_bignum                              duckdb_ext_api.duckdb_get_bignum
 #define duckdb_get_decimal                             duckdb_ext_api.duckdb_get_decimal
 #define duckdb_get_float                               duckdb_ext_api.duckdb_get_float
 #define duckdb_get_double                              duckdb_ext_api.duckdb_get_double
@@ -1004,32 +1069,75 @@ typedef struct {
 #define duckdb_destroy_instance_cache   duckdb_ext_api.duckdb_destroy_instance_cache
 
 // Version unstable_new_append_functions
+#define duckdb_appender_create_query   duckdb_ext_api.duckdb_appender_create_query
+#define duckdb_appender_error_data     duckdb_ext_api.duckdb_appender_error_data
 #define duckdb_append_default_to_chunk duckdb_ext_api.duckdb_append_default_to_chunk
 
+// Version unstable_new_arrow_functions
+#define duckdb_to_arrow_schema                duckdb_ext_api.duckdb_to_arrow_schema
+#define duckdb_data_chunk_to_arrow            duckdb_ext_api.duckdb_data_chunk_to_arrow
+#define duckdb_schema_from_arrow              duckdb_ext_api.duckdb_schema_from_arrow
+#define duckdb_data_chunk_from_arrow          duckdb_ext_api.duckdb_data_chunk_from_arrow
+#define duckdb_destroy_arrow_converted_schema duckdb_ext_api.duckdb_destroy_arrow_converted_schema
+
+// Version unstable_new_error_data_functions
+#define duckdb_create_error_data     duckdb_ext_api.duckdb_create_error_data
+#define duckdb_destroy_error_data    duckdb_ext_api.duckdb_destroy_error_data
+#define duckdb_error_data_error_type duckdb_ext_api.duckdb_error_data_error_type
+#define duckdb_error_data_message    duckdb_ext_api.duckdb_error_data_message
+#define duckdb_error_data_has_error  duckdb_ext_api.duckdb_error_data_has_error
+
+// Version unstable_new_expression_functions
+#define duckdb_destroy_expression     duckdb_ext_api.duckdb_destroy_expression
+#define duckdb_expression_return_type duckdb_ext_api.duckdb_expression_return_type
+#define duckdb_expression_is_foldable duckdb_ext_api.duckdb_expression_is_foldable
+#define duckdb_expression_fold        duckdb_ext_api.duckdb_expression_fold
+
 // Version unstable_new_open_connect_functions
 #define duckdb_connection_get_client_context    duckdb_ext_api.duckdb_connection_get_client_context
+#define duckdb_connection_get_arrow_options     duckdb_ext_api.duckdb_connection_get_arrow_options
 #define duckdb_client_context_get_connection_id duckdb_ext_api.duckdb_client_context_get_connection_id
 #define duckdb_destroy_client_context           duckdb_ext_api.duckdb_destroy_client_context
+#define duckdb_destroy_arrow_options            duckdb_ext_api.duckdb_destroy_arrow_options
 #define duckdb_get_table_names                  duckdb_ext_api.duckdb_get_table_names
 
+// Version unstable_new_prepared_statement_functions
+#define duckdb_prepared_statement_column_count        duckdb_ext_api.duckdb_prepared_statement_column_count
+#define duckdb_prepared_statement_column_name         duckdb_ext_api.duckdb_prepared_statement_column_name
+#define duckdb_prepared_statement_column_logical_type duckdb_ext_api.duckdb_prepared_statement_column_logical_type
+#define duckdb_prepared_statement_column_type         duckdb_ext_api.duckdb_prepared_statement_column_type
+
+// Version unstable_new_query_execution_functions
+#define duckdb_result_get_arrow_options duckdb_ext_api.duckdb_result_get_arrow_options
+
 // Version unstable_new_scalar_function_functions
-#define duckdb_scalar_function_set_bind           duckdb_ext_api.duckdb_scalar_function_set_bind
-#define duckdb_scalar_function_set_bind_data      duckdb_ext_api.duckdb_scalar_function_set_bind_data
-#define duckdb_scalar_function_bind_set_error     duckdb_ext_api.duckdb_scalar_function_bind_set_error
-#define duckdb_scalar_function_get_bind_data      duckdb_ext_api.duckdb_scalar_function_get_bind_data
-#define duckdb_scalar_function_get_client_context duckdb_ext_api.duckdb_scalar_function_get_client_context
+#define duckdb_scalar_function_set_bind                duckdb_ext_api.duckdb_scalar_function_set_bind
+#define duckdb_scalar_function_set_bind_data           duckdb_ext_api.duckdb_scalar_function_set_bind_data
+#define duckdb_scalar_function_set_bind_data_copy      duckdb_ext_api.duckdb_scalar_function_set_bind_data_copy
+#define duckdb_scalar_function_bind_set_error          duckdb_ext_api.duckdb_scalar_function_bind_set_error
+#define duckdb_scalar_function_bind_get_extra_info     duckdb_ext_api.duckdb_scalar_function_bind_get_extra_info
+#define duckdb_scalar_function_get_bind_data           duckdb_ext_api.duckdb_scalar_function_get_bind_data
+#define duckdb_scalar_function_get_client_context      duckdb_ext_api.duckdb_scalar_function_get_client_context
+#define duckdb_scalar_function_bind_get_argument_count duckdb_ext_api.duckdb_scalar_function_bind_get_argument_count
+#define duckdb_scalar_function_bind_get_argument       duckdb_ext_api.duckdb_scalar_function_bind_get_argument
 
 // Version unstable_new_string_functions
 #define duckdb_value_to_string duckdb_ext_api.duckdb_value_to_string
 
+// Version unstable_new_table_function_functions
+#define duckdb_table_function_get_client_context duckdb_ext_api.duckdb_table_function_get_client_context
+
 // Version unstable_new_value_functions
+#define duckdb_create_time_ns     duckdb_ext_api.duckdb_create_time_ns
+#define duckdb_get_time_ns        duckdb_ext_api.duckdb_get_time_ns
 #define duckdb_create_map_value   duckdb_ext_api.duckdb_create_map_value
 #define duckdb_create_union_value duckdb_ext_api.duckdb_create_union_value
 
-// Version unstable_new_vector_types
+// Version unstable_new_vector_functions
 #define duckdb_create_vector                 duckdb_ext_api.duckdb_create_vector
 #define duckdb_destroy_vector                duckdb_ext_api.duckdb_destroy_vector
 #define duckdb_slice_vector                  duckdb_ext_api.duckdb_slice_vector
+#define duckdb_vector_copy_sel               duckdb_ext_api.duckdb_vector_copy_sel
 #define duckdb_vector_reference_value        duckdb_ext_api.duckdb_vector_reference_value
 #define duckdb_vector_reference_vector       duckdb_ext_api.duckdb_vector_reference_vector
 #define duckdb_create_selection_vector       duckdb_ext_api.duckdb_create_selection_vector

package/src/duckdb/src/logging/log_manager.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/client_data.hpp"
+#include "duckdb/common/local_file_system.hpp"
 
 namespace duckdb {
 
@@ -49,7 +50,7 @@ shared_ptr<Logger> LogManager::GlobalLoggerReference() {
 
 void LogManager::Flush() {
 	unique_lock<mutex> lck(lock);
-	log_storage->Flush();
+	log_storage->FlushAll();
 }
 
 shared_ptr<LogStorage> LogManager::GetLogStorage() {
@@ -57,9 +58,9 @@ shared_ptr<LogStorage> LogManager::GetLogStorage() {
 	return log_storage;
 }
 
-bool LogManager::CanScan() {
+bool LogManager::CanScan(LoggingTargetTable table) {
 	unique_lock<mutex> lck(lock);
-	return log_storage->CanScan();
+	return log_storage->CanScan(table);
 }
 
 LogManager::LogManager(DatabaseInstance &db, LogConfig config_p) : config(std::move(config_p)) {
@@ -102,6 +103,15 @@ void LogManager::FlushCachedLogEntries(DataChunk &chunk, const RegisteredLogging
 	throw NotImplementedException("FlushCachedLogEntries");
 }
 
+void LogManager::SetConfig(DatabaseInstance &db, const LogConfig &config_p) {
+	unique_lock<mutex> lck(lock);
+
+	// We need extra handling for switching storage
+	SetLogStorageInternal(db, config_p.storage);
+
+	SetConfigInternal(config_p);
+}
+
 void LogManager::SetEnableLogging(bool enable) {
 	unique_lock<mutex> lck(lock);
 	config.enabled = enable;
@@ -120,35 +130,54 @@ void LogManager::SetLogLevel(LogLevel level) {
 	global_logger->UpdateConfig(config);
 }
 
-void LogManager::SetEnabledLogTypes(unordered_set<string> &enabled_log_types) {
+void LogManager::SetEnabledLogTypes(optional_ptr<unordered_set<string>> enabled_log_types) {
 	unique_lock<mutex> lck(lock);
-	config.enabled_log_types = enabled_log_types;
+	if (enabled_log_types) {
+		config.enabled_log_types = *enabled_log_types;
+	} else {
+		config.enabled_log_types = {};
+	}
 	global_logger->UpdateConfig(config);
 }
 
-void LogManager::SetDisabledLogTypes(unordered_set<string> &disabled_log_types) {
+void LogManager::SetDisabledLogTypes(optional_ptr<unordered_set<string>> disabled_log_types) {
 	unique_lock<mutex> lck(lock);
-	config.disabled_log_types = disabled_log_types;
+	if (disabled_log_types) {
+		config.disabled_log_types = *disabled_log_types;
+	} else {
+		config.disabled_log_types = {};
+	}
 	global_logger->UpdateConfig(config);
 }
 
 void LogManager::SetLogStorage(DatabaseInstance &db, const string &storage_name) {
 	unique_lock<mutex> lck(lock);
+	SetLogStorageInternal(db, storage_name);
+}
+
+void LogManager::SetLogStorageInternal(DatabaseInstance &db, const string &storage_name) {
 	auto storage_name_to_lower = StringUtil::Lower(storage_name);
 
 	if (config.storage == storage_name_to_lower) {
 		return;
 	}
 
+	if (storage_name_to_lower == LogConfig::FILE_STORAGE_NAME) {
+		auto &fs = FileSystem::GetFileSystem(db);
+		if (fs.SubSystemIsDisabled(LocalFileSystem().GetName())) {
+			throw InvalidConfigurationException("Can not enable file logging with the LocalFileSystem disabled");
+		}
+	}
+
 	// Flush the old storage, we are going to replace it.
-	log_storage->Flush();
+	log_storage->FlushAll();
 
 	if (storage_name_to_lower == LogConfig::IN_MEMORY_STORAGE_NAME) {
 		log_storage = make_shared_ptr<InMemoryLogStorage>(db);
 	} else if (storage_name_to_lower == LogConfig::STDOUT_STORAGE_NAME) {
-		log_storage = make_shared_ptr<StdOutLogStorage>();
+		log_storage = make_shared_ptr<StdOutLogStorage>(db);
 	} else if (storage_name_to_lower == LogConfig::FILE_STORAGE_NAME) {
-		throw NotImplementedException("File log storage is not yet implemented");
+		log_storage = make_shared_ptr<FileLogStorage>(db);
 	} else if (registered_log_storages.find(storage_name_to_lower) != registered_log_storages.end()) {
 		log_storage = registered_log_storages[storage_name_to_lower];
 	} else {
@@ -157,9 +186,16 @@ void LogManager::SetLogStorage(DatabaseInstance &db, const string &storage_name)
 	config.storage = storage_name_to_lower;
 }
 
+void LogManager::UpdateLogStorageConfig(DatabaseInstance &db, case_insensitive_map_t<Value> &config_value) {
+	unique_lock<mutex> lck(lock);
+	log_storage->UpdateConfig(db, config_value);
+}
+
 void LogManager::SetEnableStructuredLoggers(vector<string> &enabled_logger_types) {
 	unique_lock<mutex> lck(lock);
-	config.enabled_log_types.clear();
+
+	LogConfig new_config = config;
+	new_config.enabled_log_types.clear();
 
 	LogLevel min_log_level = LogLevel::LOG_FATAL;
 
@@ -169,14 +205,16 @@ void LogManager::SetEnableStructuredLoggers(vector<string> &enabled_logger_types
 			throw InvalidInputException("Unknown log type: '%s'", enabled_logger_type);
 		}
 
-		config.enabled_log_types.insert(enabled_logger_type);
+		new_config.enabled_log_types.insert(enabled_logger_type);
 
 		min_log_level = MinValue(min_log_level, lookup->level);
 	}
 
-	config.level = min_log_level;
-	config.mode = LogMode::ENABLE_SELECTED;
-	config.enabled = true;
+	new_config.level = min_log_level;
+	new_config.mode = LogMode::ENABLE_SELECTED;
+	new_config.enabled = true;
+
+	SetConfigInternal(new_config);
 }
 
 void LogManager::TruncateLogStorage() {
@@ -205,6 +243,12 @@ optional_ptr<const LogType> LogManager::LookupLogTypeInternal(const string &type
 	return nullptr;
 }
 
+void LogManager::SetConfigInternal(LogConfig config_p) {
+	// Apply the remainder of the config
+	config = std::move(config_p);
+	global_logger->UpdateConfig(config);
+}
+
 void LogManager::RegisterLogType(unique_ptr<LogType> type) {
 	unique_lock<mutex> lck(lock);