duckdb 1.1.2-dev4.0 → 1.1.2-dev6.0

package/src/duckdb/src/include/duckdb/planner/operator/logical_export.hpp

@@ -20,26 +20,23 @@ public:
 	static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_EXPORT;
 
 public:
-	LogicalExport(CopyFunction function, unique_ptr<CopyInfo> copy_info, BoundExportData exported_tables)
-	    : LogicalOperator(LogicalOperatorType::LOGICAL_EXPORT), function(std::move(function)),
-	      copy_info(std::move(copy_info)), exported_tables(std::move(exported_tables)) {
-	}
-	CopyFunction function;
+	LogicalExport(CopyFunction function, unique_ptr<CopyInfo> copy_info, unique_ptr<BoundExportData> exported_tables);
+
 	unique_ptr<CopyInfo> copy_info;
-	BoundExportData exported_tables;
+	CopyFunction function;
+	unique_ptr<BoundExportData> exported_tables;
 
-	void Serialize(Serializer &serializer) const override {
-		throw NotImplementedException("FIXME: Serialize LogicalExport statement");
-	}
-	unique_ptr<LogicalOperator> Deserialize(Deserializer &deserializer) {
-		throw NotImplementedException("FIXME: Deserialize LogicalExport statement");
-	}
+	void Serialize(Serializer &serializer) const override;
+	static unique_ptr<LogicalOperator> Deserialize(Deserializer &deserializer);
 
-public:
 protected:
+	LogicalExport(ClientContext &context, unique_ptr<ParseInfo> copy_info, unique_ptr<ParseInfo> exported_tables);
+
 	void ResolveTypes() override {
 		types.emplace_back(LogicalType::BOOLEAN);
 	}
+
+	CopyFunction GetCopyFunction(ClientContext &context, CopyInfo &info);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/table_filter.hpp

@@ -45,6 +45,7 @@ public:
 	//! Returns true if the statistics indicate that the segment can contain values that satisfy that filter
 	virtual FilterPropagateResult CheckStatistics(BaseStatistics &stats) = 0;
 	virtual string ToString(const string &column_name) = 0;
+	string DebugToString();
 	virtual unique_ptr<TableFilter> Copy() const = 0;
 	virtual bool Equals(const TableFilter &other) const {
 		return filter_type != other.filter_type;

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -50,9 +50,9 @@ public:
 	~MetadataManager();
 
 	MetadataHandle AllocateHandle();
-	MetadataHandle Pin(MetadataPointer pointer);
+	MetadataHandle Pin(const MetadataPointer &pointer);
 
-	MetaBlockPointer GetDiskPointer(MetadataPointer pointer, uint32_t offset = 0);
+	MetaBlockPointer GetDiskPointer(const MetadataPointer &pointer, uint32_t offset = 0);
 	MetadataPointer FromDiskPointer(MetaBlockPointer pointer);
 	MetadataPointer RegisterDiskPointer(MetaBlockPointer pointer);
 

package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp

@@ -76,7 +76,7 @@ public:
 	void SetMemoryLimit(idx_t limit = (idx_t)-1) final;
 	void SetSwapLimit(optional_idx limit = optional_idx()) final;
 
-	//! Returns informaton about memory usage
+	//! Returns information about memory usage
 	vector<MemoryInformation> GetMemoryUsageInfo() const override;
 
 	//! Returns a list of all temporary files

package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp

@@ -52,8 +52,6 @@ private:
 	static constexpr double INTEGRAL_SAMPLE_RATE = 0.3;
 	//! For concurrent access
 	mutable mutex lock;
-	//! Preallocated vector for hashes
-	Vector hash_vec;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp

@@ -145,6 +145,7 @@ public:
 		}
 		SegmentNode<T> node;
 		segment->index = nodes.size();
+		segment->next = nullptr;
 		node.row_start = segment->start;
 		node.node = std::move(segment);
 		nodes.push_back(std::move(node));

package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp

@@ -102,8 +102,12 @@ private:
 	reference_map_t<RowGroupCollection, shared_ptr<RowGroupCollection>> updated_collections;
 	//! Lock for the active_locks map
 	mutex active_locks_lock;
+	struct ActiveTableLock {
+		mutex checkpoint_lock_mutex; // protects access to the checkpoint_lock field in this class
+		weak_ptr<CheckpointLock> checkpoint_lock;
+	};
 	//! Active locks on tables
-	reference_map_t<DataTableInfo, weak_ptr<CheckpointLock>> active_locks;
+	reference_map_t<DataTableInfo, unique_ptr<ActiveTableLock>> active_locks;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb.h

@@ -626,12 +626,12 @@ typedef struct _duckdb_arrow_array {
 //===--------------------------------------------------------------------===//
 //! Passed to C API extension as parameter to the entrypoint
 struct duckdb_extension_access {
-	//! Indicate that an error has occured
+	//! Indicate that an error has occurred
 	void (*set_error)(duckdb_extension_info info, const char *error);
 	//! Fetch the database from duckdb to register extensions to
 	duckdb_database *(*get_database)(duckdb_extension_info info);
 	//! Fetch the API
-	void *(*get_api)(duckdb_extension_info info, const char *version);
+	const void *(*get_api)(duckdb_extension_info info, const char *version);
 };
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/appender.cpp

@@ -62,6 +62,9 @@ Appender::Appender(Connection &con, const string &schema_name, const string &tab
 	}
 	vector<optional_ptr<const ParsedExpression>> defaults;
 	for (auto &column : description->columns) {
+		if (column.Generated()) {
+			continue;
+		}
 		types.push_back(column.Type());
 		defaults.push_back(column.HasDefaultValue() ? &column.DefaultValue() : nullptr);
 	}

package/src/duckdb/src/main/capi/profiling_info-c.cpp

@@ -30,7 +30,7 @@ duckdb_value duckdb_profiling_info_get_value(duckdb_profiling_info info, const c
 	auto &node = *reinterpret_cast<duckdb::ProfilingNode *>(info);
 	auto &profiling_info = node.GetProfilingInfo();
 	auto key_enum = EnumUtil::FromString<MetricsType>(duckdb::StringUtil::Upper(key));
-	if (!profiling_info.Enabled(key_enum)) {
+	if (!profiling_info.Enabled(profiling_info.settings, key_enum)) {
 		return nullptr;
 	}
 
@@ -46,10 +46,13 @@ duckdb_value duckdb_profiling_info_get_metrics(duckdb_profiling_info info) {
 	auto &node = *reinterpret_cast<duckdb::ProfilingNode *>(info);
 	auto &profiling_info = node.GetProfilingInfo();
 
-	// FIXME: filter between operator metrics and query node metrics.
 	duckdb::unordered_map<duckdb::string, duckdb::string> metrics_map;
 	for (const auto &metric : profiling_info.metrics) {
 		auto key = EnumUtil::ToString(metric.first);
+		if (!profiling_info.Enabled(profiling_info.settings, metric.first)) {
+			continue;
+		}
+
 		if (key == EnumUtil::ToString(MetricsType::OPERATOR_TYPE)) {
 			auto type = duckdb::PhysicalOperatorType(metric.second.GetValue<uint8_t>());
 			metrics_map[key] = EnumUtil::ToString(type);

package/src/duckdb/src/main/client_context.cpp

@@ -1132,13 +1132,19 @@ void ClientContext::Append(TableDescription &description, ColumnDataCollection &
 		auto &table_entry =
 		    Catalog::GetEntry<TableCatalogEntry>(*this, INVALID_CATALOG, description.schema, description.table);
 		// verify that the table columns and types match up
-		if (description.columns.size() != table_entry.GetColumns().PhysicalColumnCount()) {
+		if (description.PhysicalColumnCount() != table_entry.GetColumns().PhysicalColumnCount()) {
 			throw InvalidInputException("Failed to append: table entry has different number of columns!");
 		}
+		idx_t table_entry_col_idx = 0;
 		for (idx_t i = 0; i < description.columns.size(); i++) {
-			if (description.columns[i].Type() != table_entry.GetColumns().GetColumn(PhysicalIndex(i)).Type()) {
+			auto &column = description.columns[i];
+			if (column.Generated()) {
+				continue;
+			}
+			if (column.Type() != table_entry.GetColumns().GetColumn(PhysicalIndex(table_entry_col_idx)).Type()) {
 				throw InvalidInputException("Failed to append: table entry has different number of columns!");
 			}
+			table_entry_col_idx++;
 		}
 		auto binder = Binder::CreateBinder(*this);
 		auto bound_constraints = binder->BindConstraints(table_entry);

package/src/duckdb/src/main/connection.cpp

@@ -19,7 +19,7 @@
 namespace duckdb {
 
 Connection::Connection(DatabaseInstance &database)
-    : context(make_shared_ptr<ClientContext>(database.shared_from_this())) {
+    : context(make_shared_ptr<ClientContext>(database.shared_from_this())), warning_cb(nullptr) {
 	ConnectionManager::Get(database).AddConnection(*context);
 #ifdef DEBUG
 	EnableProfiling();

package/src/duckdb/src/main/database.cpp

@@ -25,6 +25,7 @@
 #include "duckdb/storage/storage_extension.hpp"
 #include "duckdb/storage/storage_manager.hpp"
 #include "duckdb/transaction/transaction_manager.hpp"
+#include "duckdb/main/capi/extension_api.hpp"
 
 #ifndef DUCKDB_NO_THREADS
 #include "duckdb/common/thread.hpp"
@@ -56,6 +57,7 @@ DBConfig::~DBConfig() {
 
 DatabaseInstance::DatabaseInstance() {
 	config.is_user_config = false;
+	create_api_v0 = nullptr;
 }
 
 DatabaseInstance::~DatabaseInstance() {
@@ -258,6 +260,10 @@ void DatabaseInstance::LoadExtensionSettings() {
 	}
 }
 
+static duckdb_ext_api_v0 CreateAPIv0Wrapper() {
+	return CreateAPIv0();
+}
+
 void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_config) {
 	DBConfig default_config;
 	DBConfig *config_ptr = &default_config;
@@ -267,6 +273,8 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
 
 	Configure(*config_ptr, database_path);
 
+	create_api_v0 = CreateAPIv0Wrapper;
+
 	if (user_config && !user_config->options.use_temporary_directory) {
 		// temporary directories explicitly disabled
 		config.options.temporary_directory = string();
@@ -504,6 +512,11 @@ ValidChecker &DatabaseInstance::GetValidChecker() {
 	return db_validity;
 }
 
+const duckdb_ext_api_v0 DatabaseInstance::GetExtensionAPIV0() {
+	D_ASSERT(create_api_v0);
+	return create_api_v0();
+}
+
 ValidChecker &ValidChecker::Get(DatabaseInstance &db) {
 	return db.GetValidChecker();
 }

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -363,7 +363,7 @@ ExtensionUpdateResult ExtensionHelper::UpdateExtension(ClientContext &context, c
 		throw InvalidInputException("Failed to update the extension '%s', the extension is not installed!",
 		                            extension_name);
 	} else if (update_result.tag == ExtensionUpdateResultTag::UNKNOWN) {
-		throw InternalException("Failed to update extension '%s', an unknown error ocurred", extension_name);
+		throw InternalException("Failed to update extension '%s', an unknown error occurred", extension_name);
 	}
 	return update_result;
 }

package/src/duckdb/src/main/extension/extension_install.cpp

@@ -358,7 +358,15 @@ static unique_ptr<ExtensionInstallInfo> InstallFromHttpUrl(DatabaseInstance &db,
 	{
 		auto fs = FileSystem::CreateLocal();
 		if (fs->FileExists(local_extension_path + ".info")) {
-			install_info = ExtensionInstallInfo::TryReadInfoFile(*fs, local_extension_path + ".info", extension_name);
+			try {
+				install_info =
+				    ExtensionInstallInfo::TryReadInfoFile(*fs, local_extension_path + ".info", extension_name);
+			} catch (...) {
+				if (!options.force_install) {
+					// We are going to rewrite the file anyhow, so this is fine
+					throw;
+				}
+			}
 		}
 	}
 

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -91,7 +91,7 @@ struct ExtensionAccess {
 	}
 
 	//! Called by the extension get a pointer the correctly versioned extension C API struct.
-	static void *GetAPI(duckdb_extension_info info, const char *version) {
+	static const void *GetAPI(duckdb_extension_info info, const char *version) {
 
 		string version_string = version;
 		idx_t major, minor, patch;
@@ -106,7 +106,8 @@ struct ExtensionAccess {
 			              "Unsupported C CAPI version detected during extension initialization: " + string(version));
 			return nullptr;
 		}
-		load_state.api_struct = CreateAPIv0();
+
+		load_state.api_struct = load_state.db.GetExtensionAPIV0();
 		return &load_state.api_struct;
 	}
 };

package/src/duckdb/src/main/extension_install_info.cpp

@@ -100,7 +100,7 @@ unique_ptr<ExtensionInstallInfo> ExtensionInstallInfo::TryReadInfoFile(FileSyste
 		} catch (std::exception &ex) {
 			ErrorData error(ex);
 			throw IOException(
-			    "Failed to read info file for '%s' extension: '%s'.\nA serialization error occured: '%s'\n%s",
+			    "Failed to read info file for '%s' extension: '%s'.\nA serialization error occurred: '%s'\n%s",
 			    extension_name, info_file_path, error.RawMessage(), hint);
 		}
 	}

package/src/duckdb/src/main/profiling_info.cpp

@@ -9,44 +9,53 @@ using namespace duckdb_yyjson; // NOLINT
 
 namespace duckdb {
 
+ProfilingInfo::ProfilingInfo(const profiler_settings_t &n_settings, const idx_t depth) : settings(n_settings) {
+	// Expand.
+	if (depth == 0) {
+		settings.insert(MetricsType::QUERY_NAME);
+	} else {
+		settings.insert(MetricsType::OPERATOR_TYPE);
+	}
+	for (const auto &metric : settings) {
+		Expand(expanded_settings, metric);
+	}
+
+	// Reduce.
+	if (depth == 0) {
+		auto op_metrics = DefaultOperatorSettings();
+		for (const auto metric : op_metrics) {
+			settings.erase(metric);
+		}
+	} else {
+		auto root_metrics = DefaultRootSettings();
+		for (const auto metric : root_metrics) {
+			settings.erase(metric);
+		}
+	}
+	ResetMetrics();
+}
+
 profiler_settings_t ProfilingInfo::DefaultSettings() {
 	return {MetricsType::QUERY_NAME,           MetricsType::BLOCKED_THREAD_TIME,     MetricsType::CPU_TIME,
 	        MetricsType::EXTRA_INFO,           MetricsType::CUMULATIVE_CARDINALITY,  MetricsType::OPERATOR_TYPE,
 	        MetricsType::OPERATOR_CARDINALITY, MetricsType::CUMULATIVE_ROWS_SCANNED, MetricsType::OPERATOR_ROWS_SCANNED,
-	        MetricsType::OPERATOR_TIMING,      MetricsType::RESULT_SET_SIZE};
+	        MetricsType::OPERATOR_TIMING,      MetricsType::RESULT_SET_SIZE,         MetricsType::LATENCY,
+	        MetricsType::ROWS_RETURNED};
 }
 
-profiler_settings_t ProfilingInfo::DefaultOperatorSettings() {
-	return {MetricsType::OPERATOR_CARDINALITY, MetricsType::OPERATOR_ROWS_SCANNED, MetricsType::OPERATOR_TIMING,
-	        MetricsType::RESULT_SET_SIZE};
+profiler_settings_t ProfilingInfo::DefaultRootSettings() {
+	return {MetricsType::QUERY_NAME, MetricsType::BLOCKED_THREAD_TIME, MetricsType::LATENCY,
+	        MetricsType::ROWS_RETURNED};
 }
 
-profiler_settings_t ProfilingInfo::AllSettings() {
-	auto all_settings = DefaultSettings();
-	auto optimizer_settings = MetricsUtils::GetOptimizerMetrics();
-	auto phase_timings = MetricsUtils::GetPhaseTimingMetrics();
-
-	for (auto &setting : optimizer_settings) {
-		all_settings.insert(setting);
-	}
-
-	for (auto &setting : phase_timings) {
-		all_settings.insert(setting);
-	}
-
-	return all_settings;
+profiler_settings_t ProfilingInfo::DefaultOperatorSettings() {
+	return {MetricsType::OPERATOR_CARDINALITY, MetricsType::OPERATOR_ROWS_SCANNED, MetricsType::OPERATOR_TIMING,
+	        MetricsType::OPERATOR_TYPE};
 }
 
 void ProfilingInfo::ResetMetrics() {
 	metrics.clear();
-
-	auto all_settings = AllSettings();
-
-	for (auto &metric : all_settings) {
-		if (!Enabled(metric)) {
-			continue;
-		}
-
+	for (auto &metric : expanded_settings) {
 		if (MetricsUtils::IsOptimizerMetric(metric) || MetricsUtils::IsPhaseTimingMetric(metric)) {
 			metrics[metric] = Value::CreateValue(0.0);
 			continue;
@@ -54,24 +63,25 @@ void ProfilingInfo::ResetMetrics() {
 
 		switch (metric) {
 		case MetricsType::QUERY_NAME:
+			metrics[metric] = Value::CreateValue("");
+			break;
+		case MetricsType::LATENCY:
 		case MetricsType::BLOCKED_THREAD_TIME:
 		case MetricsType::CPU_TIME:
-		case MetricsType::OPERATOR_TIMING: {
+		case MetricsType::OPERATOR_TIMING:
 			metrics[metric] = Value::CreateValue(0.0);
 			break;
-		}
-		case MetricsType::OPERATOR_TYPE: {
+		case MetricsType::OPERATOR_TYPE:
 			metrics[metric] = Value::CreateValue<uint8_t>(0);
 			break;
-		}
+		case MetricsType::ROWS_RETURNED:
 		case MetricsType::RESULT_SET_SIZE:
 		case MetricsType::CUMULATIVE_CARDINALITY:
 		case MetricsType::OPERATOR_CARDINALITY:
 		case MetricsType::CUMULATIVE_ROWS_SCANNED:
-		case MetricsType::OPERATOR_ROWS_SCANNED: {
+		case MetricsType::OPERATOR_ROWS_SCANNED:
 			metrics[metric] = Value::CreateValue<uint64_t>(0);
 			break;
-		}
 		case MetricsType::EXTRA_INFO:
 			break;
 		default:
@@ -80,35 +90,45 @@ void ProfilingInfo::ResetMetrics() {
 	}
 }
 
-bool ProfilingInfo::Enabled(const MetricsType setting) const {
-	if (settings.find(setting) != settings.end()) {
+bool ProfilingInfo::Enabled(const profiler_settings_t &settings, const MetricsType metric) {
+	if (settings.find(metric) != settings.end()) {
 		return true;
 	}
+	return false;
+}
 
-	switch (setting) {
-	case MetricsType::OPERATOR_TIMING:
-		return Enabled(MetricsType::CPU_TIME);
-	case MetricsType::OPERATOR_CARDINALITY:
-		return Enabled(MetricsType::CUMULATIVE_CARDINALITY);
-	case MetricsType::OPERATOR_ROWS_SCANNED:
-		return Enabled(MetricsType::CUMULATIVE_ROWS_SCANNED);
-	default:
-		break;
+void ProfilingInfo::Expand(profiler_settings_t &settings, const MetricsType metric) {
+	settings.insert(metric);
+
+	switch (metric) {
+	case MetricsType::CPU_TIME:
+		settings.insert(MetricsType::OPERATOR_TIMING);
+		return;
+	case MetricsType::CUMULATIVE_CARDINALITY:
+		settings.insert(MetricsType::OPERATOR_CARDINALITY);
+		return;
+	case MetricsType::CUMULATIVE_ROWS_SCANNED:
+		settings.insert(MetricsType::OPERATOR_ROWS_SCANNED);
+		return;
+	case MetricsType::CUMULATIVE_OPTIMIZER_TIMING:
+	case MetricsType::ALL_OPTIMIZERS: {
+		auto optimizer_metrics = MetricsUtils::GetOptimizerMetrics();
+		for (const auto optimizer_metric : optimizer_metrics) {
+			settings.insert(optimizer_metric);
+		}
+		return;
 	}
-
-	if (MetricsUtils::IsOptimizerMetric(setting)) {
-		return Enabled(MetricsType::CUMULATIVE_OPTIMIZER_TIMING);
+	default:
+		return;
 	}
-
-	return false;
 }
 
-string ProfilingInfo::GetMetricAsString(MetricsType setting) const {
-	if (!Enabled(setting)) {
-		throw InternalException("Metric %s not enabled", EnumUtil::ToString(setting));
+string ProfilingInfo::GetMetricAsString(const MetricsType metric) const {
+	if (!Enabled(settings, metric)) {
+		throw InternalException("Metric %s not enabled", EnumUtil::ToString(metric));
 	}
 
-	if (setting == MetricsType::EXTRA_INFO) {
+	if (metric == MetricsType::EXTRA_INFO) {
 		string result;
 		for (auto &it : extra_info) {
 			if (!result.empty()) {
@@ -120,14 +140,12 @@ string ProfilingInfo::GetMetricAsString(MetricsType setting) const {
 	}
 
 	// The metric cannot be NULL and must be initialized.
-	D_ASSERT(!metrics.at(setting).IsNull());
-
-	if (setting == MetricsType::OPERATOR_TYPE) {
-		auto type = PhysicalOperatorType(metrics.at(setting).GetValue<uint8_t>());
+	D_ASSERT(!metrics.at(metric).IsNull());
+	if (metric == MetricsType::OPERATOR_TYPE) {
+		auto type = PhysicalOperatorType(metrics.at(metric).GetValue<uint8_t>());
 		return EnumUtil::ToString(type);
 	}
-
-	return metrics.at(setting).ToString();
+	return metrics.at(metric).ToString();
 }
 
 void ProfilingInfo::WriteMetricsToJSON(yyjson_mut_doc *doc, yyjson_mut_val *dest) {
@@ -169,6 +187,7 @@ void ProfilingInfo::WriteMetricsToJSON(yyjson_mut_doc *doc, yyjson_mut_val *dest
 		case MetricsType::QUERY_NAME:
 			yyjson_mut_obj_add_strcpy(doc, dest, key_ptr, metrics[metric].GetValue<string>().c_str());
 			break;
+		case MetricsType::LATENCY:
 		case MetricsType::BLOCKED_THREAD_TIME:
 		case MetricsType::CPU_TIME:
 		case MetricsType::OPERATOR_TIMING: {
@@ -179,6 +198,7 @@ void ProfilingInfo::WriteMetricsToJSON(yyjson_mut_doc *doc, yyjson_mut_val *dest
 			yyjson_mut_obj_add_strcpy(doc, dest, key_ptr, GetMetricAsString(metric).c_str());
 			break;
 		}
+		case MetricsType::ROWS_RETURNED:
 		case MetricsType::RESULT_SET_SIZE:
 		case MetricsType::CUMULATIVE_CARDINALITY:
 		case MetricsType::OPERATOR_CARDINALITY: