duckdb 1.2.1-dev4.0 → 1.2.1-dev8.0

package/src/duckdb/src/include/duckdb/main/extension_entries.hpp

@@ -489,6 +489,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"read_xlsx", "excel", CatalogType::TABLE_FUNCTION_ENTRY},
     {"reduce", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"reduce_sql_statement", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"register_geoarrow_extensions", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
     {"regr_avgx", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"regr_avgy", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"regr_count", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
@@ -538,6 +539,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_centroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_collect", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_collectionextract", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_concavehull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_contains", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_containsproperly", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_convexhull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -588,14 +590,17 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_isvalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_length", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_length_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_lineinterpolatepoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_lineinterpolatepoints", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_linemerge", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_linestring2dfromwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
-    {"st_list_proj_crs", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"st_linesubstring", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_m", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makeenvelope", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makeline", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makepolygon", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makevalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_minimumrotatedrectangle", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_multi", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -634,6 +639,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_transform", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_union", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_union_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
+    {"st_voronoidiagram", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_within", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_x", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_xmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -645,13 +651,15 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_zmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_zmflag", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_zmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
-    {"start_ui", "motherduck", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"start_ui", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"start_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
     {"starts_with", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"stats", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"stddev", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"stddev_pop", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"stddev_samp", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"stem", "fts", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"stop_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
     {"string_agg", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"strpos", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"struct_insert", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -695,6 +703,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"trunc", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"txid_current", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"typeof", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"ui_is_started", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
     {"unbin", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"unhex", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"unicode", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -983,6 +992,9 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
     {"sqlite_all_varchar", "sqlite_scanner"},
     {"sqlite_debug_show_queries", "sqlite_scanner"},
     {"timezone", "icu"},
+    {"ui_local_port", "ui"},
+    {"ui_polling_interval", "ui"},
+    {"ui_remote_url", "ui"},
     {"unsafe_enable_version_guessing", "iceberg"},
 }; // END_OF_EXTENSION_SETTINGS
 
@@ -1070,10 +1082,16 @@ static constexpr ExtensionEntry EXTENSION_SECRET_PROVIDERS[] = {
     {"mysql/config", "mysql_scanner"},
     {"postgres/config", "postgres_scanner"}}; // EXTENSION_SECRET_PROVIDERS
 
-static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {
-    "aws",        "azure",         "autocomplete", "core_functions", "delta",    "excel",
-    "fts",        "httpfs",        "iceberg",      "inet",           "icu",      "json",
-    "motherduck", "mysql_scanner", "parquet",      "sqlite_scanner", "sqlsmith", "postgres_scanner",
-    "tpcds",      "tpch",          "uc_catalog"}; // END_OF_AUTOLOADABLE_EXTENSIONS
+static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {"aws",          "azure",
+                                                          "autocomplete", "core_functions",
+                                                          "delta",        "excel",
+                                                          "fts",          "httpfs",
+                                                          "iceberg",      "inet",
+                                                          "icu",          "json",
+                                                          "motherduck",   "mysql_scanner",
+                                                          "parquet",      "sqlite_scanner",
+                                                          "sqlsmith",     "postgres_scanner",
+                                                          "tpcds",        "tpch",
+                                                          "uc_catalog",   "ui"}; // END_OF_AUTOLOADABLE_EXTENSIONS
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp

@@ -29,6 +29,8 @@ public:
 	DUCKDB_API explicit PendingQueryResult(ErrorData error_message);
 	DUCKDB_API ~PendingQueryResult() override;
 	DUCKDB_API bool AllowStreamResult() const;
+	PendingQueryResult(const PendingQueryResult &) = delete;
+	PendingQueryResult &operator=(const PendingQueryResult &) = delete;
 
 public:
 	//! Executes a single task within the query, returning whether or not the query is ready.

package/src/duckdb/src/include/duckdb/main/query_profiler.hpp

@@ -166,6 +166,13 @@ public:
 		return root.get();
 	}
 
+	//! Provides access to the root of the query tree, but ensures there are no concurrent modifications
+	//! This can be useful when implementing continuous profiling or making customizations
+	DUCKDB_API void GetRootUnderLock(const std::function<void(optional_ptr<ProfilingNode>)> &callback) {
+		lock_guard<std::mutex> guard(lock);
+		callback(GetRoot());
+	}
+
 private:
 	ClientContext &context;
 

package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp

@@ -103,8 +103,8 @@ private:
 
 	expression_map_t<unique_ptr<Expression>> stored_expressions;
 	expression_map_t<idx_t> equivalence_set_map;
-	unordered_map<idx_t, vector<ExpressionValueInformation>> constant_values;
-	unordered_map<idx_t, vector<reference<Expression>>> equivalence_map;
+	map<idx_t, vector<ExpressionValueInformation>> constant_values;
+	map<idx_t, vector<reference<Expression>>> equivalence_map;
 	idx_t set_index = 0;
 	//
 	//	//! Structures used for OR Filters

package/src/duckdb/src/include/duckdb/optimizer/late_materialization.hpp

@@ -14,6 +14,7 @@
 namespace duckdb {
 class LogicalOperator;
 class LogicalGet;
+class LogicalLimit;
 class Optimizer;
 
 //! Transform
@@ -34,7 +35,7 @@ private:
 	void ReplaceTableReferences(Expression &expr, idx_t new_table_index);
 	unique_ptr<Expression> GetExpression(LogicalOperator &op, idx_t column_index);
 	void ReplaceExpressionReferences(LogicalOperator &next_op, unique_ptr<Expression> &expr);
-	bool OptimizeLargeLimit(LogicalOperator &child);
+	bool OptimizeLargeLimit(LogicalLimit &limit, idx_t limit_val, bool has_offset);
 
 private:
 	Optimizer &optimizer;

package/src/duckdb/src/include/duckdb/optimizer/optimizer_extension.hpp

@@ -29,14 +29,20 @@ struct OptimizerExtensionInput {
 };
 
 typedef void (*optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
+typedef void (*pre_optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
 
 class OptimizerExtension {
 public:
-	//! The parse function of the parser extension.
-	//! Takes a query string as input and returns ParserExtensionParseData (on success) or an error
-	optimize_function_t optimize_function;
-
-	//! Additional parser info passed to the parse function
+	//! The optimize function of the optimizer extension.
+	//! Takes a logical query plan as an input, which it can modify in place
+	//! This runs, after the DuckDB optimizers have run
+	optimize_function_t optimize_function = nullptr;
+	//! The pre-optimize function of the optimizer extension.
+	//! Takes a logical query plan as an input, which it can modify in place
+	//! This runs, before the DuckDB optimizers have run
+	pre_optimize_function_t pre_optimize_function = nullptr;
+
+	//! Additional optimizer info passed to the optimize functions
 	shared_ptr<OptimizerExtensionInfo> optimizer_info;
 };
 

package/src/duckdb/src/include/duckdb/parallel/executor_task.hpp

@@ -9,10 +9,10 @@
 #pragma once
 
 #include "duckdb/parallel/task.hpp"
-#include "duckdb/parallel/event.hpp"
 #include "duckdb/common/optional_ptr.hpp"
 
 namespace duckdb {
+class Event;
 class PhysicalOperator;
 class ThreadContext;
 
@@ -34,6 +34,9 @@ public:
 	unique_ptr<ThreadContext> thread_context;
 	optional_ptr<const PhysicalOperator> op;
 
+private:
+	ClientContext &context;
+
 public:
 	virtual TaskExecutionResult ExecuteTask(TaskExecutionMode mode) = 0;
 	TaskExecutionResult Execute(TaskExecutionMode mode) override;

package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp

@@ -20,7 +20,6 @@
 namespace duckdb {
 
 class Executor;
-class Event;
 class MetaPipeline;
 class PipelineExecutor;
 class Pipeline;

package/src/duckdb/src/include/duckdb/parallel/task_executor.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/parallel/task.hpp"
 #include "duckdb/execution/task_error_manager.hpp"
 
@@ -47,6 +48,8 @@ private:
 	unique_ptr<ProducerToken> token;
 	atomic<idx_t> completed_tasks;
 	atomic<idx_t> total_tasks;
+	friend class BaseExecutorTask;
+	optional_ptr<ClientContext> context;
 };
 
 class BaseExecutorTask : public Task {

package/src/duckdb/src/include/duckdb/parallel/task_notifier.hpp

@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parallel/task_notifier.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/optional_ptr.hpp"
+
+namespace duckdb {
+class ClientContext;
+
+//! The TaskNotifier notifies ClientContextState listener about started / stopped tasks
+class TaskNotifier {
+public:
+	explicit TaskNotifier(optional_ptr<ClientContext> context_p);
+
+	~TaskNotifier();
+
+private:
+	optional_ptr<ClientContext> context;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -68,6 +68,10 @@ public:
 	//! Returns the number of threads
 	DUCKDB_API int32_t NumberOfThreads();
 
+	idx_t GetNumberOfTasks() const;
+	idx_t GetProducerCount() const;
+	idx_t GetTaskCountForProducer(ProducerToken &token) const;
+
 	//! Send signals to n threads, signalling for them to wake up and attempt to execute a task
 	void Signal(idx_t n);
 

package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp

@@ -40,7 +40,7 @@ public:
 	vector<LogicalType> child_types;
 	//! The target LogicalType of the subquery result (i.e. to which type it should be casted, if child_type <>
 	//! child_target). Only used for ANY expressions.
-	LogicalType child_target;
+	vector<LogicalType> child_targets;
 
 public:
 	bool HasSubquery() const override {

package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/bound_tableref.hpp"
+#include "duckdb/common/enums/cte_materialize.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -23,7 +23,7 @@ class TableStatistics;
 //! Abstraction will support, for example: tiering, versioning, or splitting into multiple block managers.
 class TableDataWriter {
 public:
-	explicit TableDataWriter(TableCatalogEntry &table);
+	explicit TableDataWriter(TableCatalogEntry &table, optional_ptr<ClientContext> client_context);
 	virtual ~TableDataWriter();
 
 public:
@@ -39,9 +39,11 @@ public:
 
 	TaskScheduler &GetScheduler();
 	DatabaseInstance &GetDatabase();
+	optional_ptr<ClientContext> GetClientContext();
 
 protected:
 	DuckTableEntry &table;
+	optional_ptr<ClientContext> client_context;
 	//! Pointers to the start of each row group.
 	vector<RowGroupPointer> row_group_pointers;
 };

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -97,7 +97,8 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
 	friend class SingleFileTableDataWriter;
 
 public:
-	SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager, CheckpointType checkpoint_type);
+	SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context, AttachedDatabase &db,
+	                           BlockManager &block_manager, CheckpointType checkpoint_type);
 
 	//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
 	//! connection is available because right now the checkpointing cannot be done online. (TODO)
@@ -112,10 +113,15 @@ public:
 		return checkpoint_type;
 	}
 
+	optional_ptr<ClientContext> GetClientContext() const {
+		return client_context;
+	}
+
 public:
 	void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
 
 private:
+	optional_ptr<ClientContext> client_context;
 	//! The metadata writer is responsible for writing schema information
 	unique_ptr<MetadataWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -100,7 +100,8 @@ public:
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
-	virtual void CreateCheckpoint(CheckpointOptions options = CheckpointOptions()) = 0;
+	virtual void CreateCheckpoint(optional_ptr<ClientContext> client_context,
+	                              CheckpointOptions options = CheckpointOptions()) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -159,7 +160,7 @@ public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
-	void CreateCheckpoint(CheckpointOptions options) override;
+	void CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) override;
 	DatabaseSize GetDatabaseSize() override;
 	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;