duckdb 1.2.1-dev6.0 → 1.2.1

package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp

@@ -48,20 +48,21 @@ public:
 	DUCKDB_API void Set(vector<CatalogSearchEntry> new_paths, CatalogSetPathType set_type);
 	DUCKDB_API void Reset();
 
-	DUCKDB_API const vector<CatalogSearchEntry> &Get();
-	const vector<CatalogSearchEntry> &GetSetPaths() {
+	DUCKDB_API const vector<CatalogSearchEntry> &Get() const;
+	const vector<CatalogSearchEntry> &GetSetPaths() const {
 		return set_paths;
 	}
-	DUCKDB_API const CatalogSearchEntry &GetDefault();
+	DUCKDB_API const CatalogSearchEntry &GetDefault() const;
 	//! FIXME: this method is deprecated
-	DUCKDB_API string GetDefaultSchema(const string &catalog);
-	DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog);
-	DUCKDB_API string GetDefaultCatalog(const string &schema);
+	DUCKDB_API string GetDefaultSchema(const string &catalog) const;
+	DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog) const;
+	DUCKDB_API string GetDefaultCatalog(const string &schema) const;
 
-	DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog);
-	DUCKDB_API vector<string> GetCatalogsForSchema(const string &schema);
+	DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog) const;
+	DUCKDB_API vector<string> GetCatalogsForSchema(const string &schema) const;
 
-	DUCKDB_API bool SchemaInSearchPath(ClientContext &context, const string &catalog_name, const string &schema_name);
+	DUCKDB_API bool SchemaInSearchPath(ClientContext &context, const string &catalog_name,
+	                                   const string &schema_name) const;
 
 private:
 	//! Set paths without checking if they exist

package/src/duckdb/src/include/duckdb/common/adbc/adbc-init.hpp

@@ -30,7 +30,7 @@ extern "C" {
 typedef uint8_t AdbcStatusCode;
 
 //! We gotta leak the symbols of the init function
-DUCKDB_API AdbcStatusCode duckdb_adbc_init(int version, void *driver, struct AdbcError *error);
+DUCKDB_C_API AdbcStatusCode duckdb_adbc_init(int version, void *driver, struct AdbcError *error);
 
 #ifdef __cplusplus
 }

package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp

@@ -168,7 +168,7 @@ struct MultiFileFilterEntry {
 struct MultiFileConstantEntry {
 	MultiFileConstantEntry(idx_t column_id, Value value_p) : column_id(column_id), value(std::move(value_p)) {
 	}
-	//! The column id to apply the constant value to
+	//! The (global) column id to apply the constant value to
 	idx_t column_id;
 	//! The constant value
 	Value value;
@@ -273,7 +273,7 @@ struct MultiFileReader {
 	                                      const string &initial_file, const MultiFileReaderBindData &options,
 	                                      optional_ptr<MultiFileReaderGlobalState> global_state);
 	//! Populated the filter_map
-	DUCKDB_API virtual void CreateFilterMap(const vector<MultiFileReaderColumnDefinition> &global_columns,
+	DUCKDB_API virtual void CreateFilterMap(const vector<ColumnIndex> &global_column_ids,
 	                                        optional_ptr<TableFilterSet> filters, MultiFileReaderData &reader_data,
 	                                        optional_ptr<MultiFileReaderGlobalState> global_state);
 

package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp

@@ -90,6 +90,8 @@ private:
 	GateStatus status;
 	//! Depth in a nested leaf.
 	uint8_t nested_depth = 0;
+	//! True, if we entered a nested leaf to retrieve the next node.
+	bool entered_nested_leaf = false;
 
 private:
 	//! Goes to the next leaf in the ART and sets it as last_leaf,

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp

@@ -116,7 +116,7 @@ class BaseScanner {
 public:
 	explicit BaseScanner(shared_ptr<CSVBufferManager> buffer_manager, shared_ptr<CSVStateMachine> state_machine,
 	                     shared_ptr<CSVErrorHandler> error_handler, bool sniffing = false,
-	                     shared_ptr<CSVFileScan> csv_file_scan = nullptr, CSVIterator iterator = {});
+	                     shared_ptr<CSVFileScan> csv_file_scan = nullptr, const CSVIterator &iterator = {});
 
 	virtual ~BaseScanner() = default;
 

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp

@@ -45,14 +45,15 @@ class CSVBuffer {
 public:
 	//! Constructor for Initial Buffer
 	CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
-	          idx_t &global_csv_current_position, idx_t file_number);
+	          const idx_t &global_csv_current_position, idx_t file_number);
 
 	//! Constructor for `Next()` Buffers
 	CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position,
 	          idx_t file_number_p, idx_t buffer_idx);
 
 	//! Creates a new buffer with the next part of the CSV File
-	shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number, bool &has_seaked);
+	shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number,
+	                           bool &has_seaked) const;
 
 	//! Gets the buffer actual size
 	idx_t GetBufferSize() const;
@@ -67,12 +68,12 @@ public:
 	//! Wrapper for the Pin Function, if it can seek, it means that the buffer might have been destroyed, hence we must
 	//! Scan it from the disk file again.
 	shared_ptr<CSVBufferHandle> Pin(CSVFileHandle &file_handle, bool &has_seeked);
-	//! Wrapper for the unpin
+	//! Wrapper for unpin
 	void Unpin();
 	char *Ptr() {
 		return char_ptr_cast(handle.Ptr());
 	}
-	bool IsUnloaded() {
+	bool IsUnloaded() const {
 		return block->IsUnloaded();
 	}
 

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp

@@ -30,7 +30,7 @@ struct CSVOption { // NOLINT: work-around bug in clang-tidy
 public:
 	CSVOption(T value_p) : value(value_p) { // NOLINT: allow implicit conversion from value
 	}
-	CSVOption(T value_p, bool set_by_user_p) : value(value_p), set_by_user(set_by_user_p) {
+	CSVOption(T value_p, bool set_by_user_p) : set_by_user(set_by_user_p), value(value_p) {
 	}
 
 	CSVOption() {};

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp

@@ -14,7 +14,7 @@
 namespace duckdb {
 //! Basic CSV Column Info
 struct CSVColumnInfo {
-	CSVColumnInfo(string &name_p, LogicalType &type_p) : name(name_p), type(type_p) {
+	CSVColumnInfo(const string &name_p, const LogicalType &type_p) : name(name_p), type(type_p) {
 	}
 	string name;
 	LogicalType type;
@@ -25,7 +25,7 @@ struct CSVSchema {
 	explicit CSVSchema(const bool empty = false) : empty(empty) {
 	}
 
-	CSVSchema(vector<string> &names, vector<LogicalType> &types, const string &file_path, idx_t rows_read,
+	CSVSchema(const vector<string> &names, const vector<LogicalType> &types, const string &file_path, idx_t rows_read,
 	          const bool empty = false);
 
 	//! Initializes the schema based on names and types

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/encode/csv_encoder.hpp

@@ -46,7 +46,7 @@ private:
 class CSVEncoder {
 public:
 	//! Constructor, basically takes an encoding and the output buffer size
-	CSVEncoder(DBConfig &config, const string &encoding_name, idx_t buffer_size);
+	CSVEncoder(const DBConfig &config, const string &encoding_name, idx_t buffer_size);
 	//! Main encode function, it reads the file into an encoded buffer and converts it to the output buffer
 	idx_t Encode(FileHandle &file_handle_input, char *output_buffer, const idx_t decoded_buffer_size);
 	string encoding_name;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp

@@ -222,7 +222,7 @@ private:
 	//! ------------------ Type Refinement ------------------ //
 	//! ------------------------------------------------------//
 	void RefineTypes();
-	bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type);
+	bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type) const;
 	vector<LogicalType> detected_types;
 	//! If when finding a SQLNULL type in type detection we default it to varchar
 	const bool default_null_to_varchar;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp

@@ -121,8 +121,8 @@ private:
 };
 
 struct ParseTypeInfo {
-	ParseTypeInfo() {};
-	ParseTypeInfo(const LogicalType &type, bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
+	ParseTypeInfo() : validate_utf8(false), type_id(), internal_type(), scale(0), width(0) {};
+	ParseTypeInfo(const LogicalType &type, const bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
 		type_id = type.id();
 		internal_type = type.InternalType();
 		if (type.id() == LogicalTypeId::DECIMAL) {

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_streaming_sample.hpp

@@ -19,12 +19,10 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::STREAMING_SAMPLE;
 
 public:
-	PhysicalStreamingSample(vector<LogicalType> types, SampleMethod method, double percentage, int64_t seed,
-	                        idx_t estimated_cardinality);
+	PhysicalStreamingSample(vector<LogicalType> types, unique_ptr<SampleOptions> options, idx_t estimated_cardinality);
 
-	SampleMethod method;
+	unique_ptr<SampleOptions> sample_options;
 	double percentage;
-	int64_t seed;
 
 public:
 	// Operator interface
@@ -32,9 +30,7 @@ public:
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
-	bool ParallelOperator() const override {
-		return true;
-	}
+	bool ParallelOperator() const override;
 
 	InsertionOrderPreservingMap<string> ParamsToString() const override;
 

package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp

@@ -201,7 +201,8 @@ public:
 	void ConvertToReservoirSample();
 
 	//! Get the capactiy of the data chunk reserved for storing samples
-	idx_t GetReservoirChunkCapacity() const;
+	template <typename T>
+	T GetReservoirChunkCapacity() const;
 
 	//! If for_serialization=true then the sample_chunk is not padded with extra spaces for
 	//! future sampling values

package/src/duckdb/src/include/duckdb/function/lambda_functions.hpp

@@ -17,7 +17,7 @@
 
 namespace duckdb {
 
-struct ListLambdaBindData : public FunctionData {
+struct ListLambdaBindData final : public FunctionData {
 public:
 	ListLambdaBindData(const LogicalType &return_type, unique_ptr<Expression> lambda_expr, const bool has_index = false)
 	    : return_type(return_type), lambda_expr(std::move(lambda_expr)), has_index(has_index) {};
@@ -30,8 +30,16 @@ public:
 	bool has_index;
 
 public:
-	bool Equals(const FunctionData &other_p) const override;
-	unique_ptr<FunctionData> Copy() const override;
+	unique_ptr<FunctionData> Copy() const override {
+		auto lambda_expr_copy = lambda_expr ? lambda_expr->Copy() : nullptr;
+		return make_uniq<ListLambdaBindData>(return_type, std::move(lambda_expr_copy), has_index);
+	}
+
+	bool Equals(const FunctionData &other_p) const override {
+		auto &other = other_p.Cast<ListLambdaBindData>();
+		return Expression::Equals(lambda_expr, other.lambda_expr) && return_type == other.return_type &&
+		       has_index == other.has_index;
+	}
 
 	//! Serializes a lambda function's bind data
 	static void Serialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data_p,

package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp

@@ -148,6 +148,10 @@ struct WindowBoundariesState {
 	idx_t valid_end = 0;
 
 	FrameBounds prev;
+
+	// Extra range cursor
+	optional_ptr<WindowCursor> range_lo;
+	unique_ptr<WindowCursor> range_hi;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/client_context_state.hpp

@@ -80,6 +80,10 @@ public:
 	}
 	virtual void WriteProfilingInformation(std::ostream &ss) {
 	}
+	virtual void OnTaskStart(ClientContext &context) {
+	}
+	virtual void OnTaskStop(ClientContext &context) {
+	}
 
 public:
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/main/extension_entries.hpp

@@ -489,6 +489,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"read_xlsx", "excel", CatalogType::TABLE_FUNCTION_ENTRY},
     {"reduce", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"reduce_sql_statement", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"register_geoarrow_extensions", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
     {"regr_avgx", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"regr_avgy", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"regr_count", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
@@ -538,6 +539,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_centroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_collect", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_collectionextract", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_concavehull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_contains", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_containsproperly", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_convexhull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -588,14 +590,17 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_isvalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_length", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_length_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_lineinterpolatepoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_lineinterpolatepoints", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_linemerge", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_linestring2dfromwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
-    {"st_list_proj_crs", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"st_linesubstring", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_m", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makeenvelope", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makeline", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makepolygon", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_makevalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_minimumrotatedrectangle", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_mmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_multi", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -634,6 +639,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_transform", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_union", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_union_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
+    {"st_voronoidiagram", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_within", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_x", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_xmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -645,13 +651,15 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_zmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_zmflag", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_zmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
-    {"start_ui", "motherduck", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"start_ui", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"start_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
     {"starts_with", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"stats", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"stddev", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"stddev_pop", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"stddev_samp", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"stem", "fts", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"stop_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
     {"string_agg", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"strpos", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"struct_insert", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -695,6 +703,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"trunc", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"txid_current", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"typeof", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"ui_is_started", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
     {"unbin", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"unhex", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"unicode", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -983,6 +992,9 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
     {"sqlite_all_varchar", "sqlite_scanner"},
     {"sqlite_debug_show_queries", "sqlite_scanner"},
     {"timezone", "icu"},
+    {"ui_local_port", "ui"},
+    {"ui_polling_interval", "ui"},
+    {"ui_remote_url", "ui"},
     {"unsafe_enable_version_guessing", "iceberg"},
 }; // END_OF_EXTENSION_SETTINGS
 
@@ -1070,10 +1082,16 @@ static constexpr ExtensionEntry EXTENSION_SECRET_PROVIDERS[] = {
     {"mysql/config", "mysql_scanner"},
     {"postgres/config", "postgres_scanner"}}; // EXTENSION_SECRET_PROVIDERS
 
-static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {
-    "aws",        "azure",         "autocomplete", "core_functions", "delta",    "excel",
-    "fts",        "httpfs",        "iceberg",      "inet",           "icu",      "json",
-    "motherduck", "mysql_scanner", "parquet",      "sqlite_scanner", "sqlsmith", "postgres_scanner",
-    "tpcds",      "tpch",          "uc_catalog"}; // END_OF_AUTOLOADABLE_EXTENSIONS
+static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {"aws",          "azure",
+                                                          "autocomplete", "core_functions",
+                                                          "delta",        "excel",
+                                                          "fts",          "httpfs",
+                                                          "iceberg",      "inet",
+                                                          "icu",          "json",
+                                                          "motherduck",   "mysql_scanner",
+                                                          "parquet",      "sqlite_scanner",
+                                                          "sqlsmith",     "postgres_scanner",
+                                                          "tpcds",        "tpch",
+                                                          "uc_catalog",   "ui"}; // END_OF_AUTOLOADABLE_EXTENSIONS
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp

@@ -29,6 +29,8 @@ public:
 	DUCKDB_API explicit PendingQueryResult(ErrorData error_message);
 	DUCKDB_API ~PendingQueryResult() override;
 	DUCKDB_API bool AllowStreamResult() const;
+	PendingQueryResult(const PendingQueryResult &) = delete;
+	PendingQueryResult &operator=(const PendingQueryResult &) = delete;
 
 public:
 	//! Executes a single task within the query, returning whether or not the query is ready.

package/src/duckdb/src/include/duckdb/main/query_profiler.hpp

@@ -166,6 +166,13 @@ public:
 		return root.get();
 	}
 
+	//! Provides access to the root of the query tree, but ensures there are no concurrent modifications
+	//! This can be useful when implementing continuous profiling or making customizations
+	DUCKDB_API void GetRootUnderLock(const std::function<void(optional_ptr<ProfilingNode>)> &callback) {
+		lock_guard<std::mutex> guard(lock);
+		callback(GetRoot());
+	}
+
 private:
 	ClientContext &context;
 

package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp

@@ -103,8 +103,8 @@ private:
 
 	expression_map_t<unique_ptr<Expression>> stored_expressions;
 	expression_map_t<idx_t> equivalence_set_map;
-	unordered_map<idx_t, vector<ExpressionValueInformation>> constant_values;
-	unordered_map<idx_t, vector<reference<Expression>>> equivalence_map;
+	map<idx_t, vector<ExpressionValueInformation>> constant_values;
+	map<idx_t, vector<reference<Expression>>> equivalence_map;
 	idx_t set_index = 0;
 	//
 	//	//! Structures used for OR Filters

package/src/duckdb/src/include/duckdb/optimizer/late_materialization.hpp

@@ -14,6 +14,7 @@
 namespace duckdb {
 class LogicalOperator;
 class LogicalGet;
+class LogicalLimit;
 class Optimizer;
 
 //! Transform
@@ -34,7 +35,7 @@ private:
 	void ReplaceTableReferences(Expression &expr, idx_t new_table_index);
 	unique_ptr<Expression> GetExpression(LogicalOperator &op, idx_t column_index);
 	void ReplaceExpressionReferences(LogicalOperator &next_op, unique_ptr<Expression> &expr);
-	bool OptimizeLargeLimit(LogicalOperator &child);
+	bool OptimizeLargeLimit(LogicalLimit &limit, idx_t limit_val, bool has_offset);
 
 private:
 	Optimizer &optimizer;

package/src/duckdb/src/include/duckdb/optimizer/optimizer_extension.hpp

@@ -29,14 +29,20 @@ struct OptimizerExtensionInput {
 };
 
 typedef void (*optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
+typedef void (*pre_optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
 
 class OptimizerExtension {
 public:
-	//! The parse function of the parser extension.
-	//! Takes a query string as input and returns ParserExtensionParseData (on success) or an error
-	optimize_function_t optimize_function;
-
-	//! Additional parser info passed to the parse function
+	//! The optimize function of the optimizer extension.
+	//! Takes a logical query plan as an input, which it can modify in place
+	//! This runs, after the DuckDB optimizers have run
+	optimize_function_t optimize_function = nullptr;
+	//! The pre-optimize function of the optimizer extension.
+	//! Takes a logical query plan as an input, which it can modify in place
+	//! This runs, before the DuckDB optimizers have run
+	pre_optimize_function_t pre_optimize_function = nullptr;
+
+	//! Additional optimizer info passed to the optimize functions
 	shared_ptr<OptimizerExtensionInfo> optimizer_info;
 };
 

package/src/duckdb/src/include/duckdb/parallel/executor_task.hpp

@@ -9,10 +9,10 @@
 #pragma once
 
 #include "duckdb/parallel/task.hpp"
-#include "duckdb/parallel/event.hpp"
 #include "duckdb/common/optional_ptr.hpp"
 
 namespace duckdb {
+class Event;
 class PhysicalOperator;
 class ThreadContext;
 
@@ -34,6 +34,9 @@ public:
 	unique_ptr<ThreadContext> thread_context;
 	optional_ptr<const PhysicalOperator> op;
 
+private:
+	ClientContext &context;
+
 public:
 	virtual TaskExecutionResult ExecuteTask(TaskExecutionMode mode) = 0;
 	TaskExecutionResult Execute(TaskExecutionMode mode) override;

package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp

@@ -20,7 +20,6 @@
 namespace duckdb {
 
 class Executor;
-class Event;
 class MetaPipeline;
 class PipelineExecutor;
 class Pipeline;

package/src/duckdb/src/include/duckdb/parallel/task_executor.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/atomic.hpp"
+#include "duckdb/common/optional_ptr.hpp"
 #include "duckdb/parallel/task.hpp"
 #include "duckdb/execution/task_error_manager.hpp"
 
@@ -47,6 +48,8 @@ private:
 	unique_ptr<ProducerToken> token;
 	atomic<idx_t> completed_tasks;
 	atomic<idx_t> total_tasks;
+	friend class BaseExecutorTask;
+	optional_ptr<ClientContext> context;
 };
 
 class BaseExecutorTask : public Task {

package/src/duckdb/src/include/duckdb/parallel/task_notifier.hpp

@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parallel/task_notifier.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/optional_ptr.hpp"
+
+namespace duckdb {
+class ClientContext;
+
+//! The TaskNotifier notifies ClientContextState listener about started / stopped tasks
+class TaskNotifier {
+public:
+	explicit TaskNotifier(optional_ptr<ClientContext> context_p);
+
+	~TaskNotifier();
+
+private:
+	optional_ptr<ClientContext> context;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp

@@ -68,6 +68,10 @@ public:
 	//! Returns the number of threads
 	DUCKDB_API int32_t NumberOfThreads();
 
+	idx_t GetNumberOfTasks() const;
+	idx_t GetProducerCount() const;
+	idx_t GetTaskCountForProducer(ProducerToken &token) const;
+
 	//! Send signals to n threads, signalling for them to wake up and attempt to execute a task
 	void Signal(idx_t n);
 

package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp

@@ -40,7 +40,7 @@ public:
 	vector<LogicalType> child_types;
 	//! The target LogicalType of the subquery result (i.e. to which type it should be casted, if child_type <>
 	//! child_target). Only used for ANY expressions.
-	LogicalType child_target;
+	vector<LogicalType> child_targets;
 
 public:
 	bool HasSubquery() const override {

package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/planner/bound_tableref.hpp"
+#include "duckdb/common/enums/cte_materialize.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp

@@ -23,7 +23,7 @@ class TableStatistics;
 //! Abstraction will support, for example: tiering, versioning, or splitting into multiple block managers.
 class TableDataWriter {
 public:
-	explicit TableDataWriter(TableCatalogEntry &table);
+	explicit TableDataWriter(TableCatalogEntry &table, optional_ptr<ClientContext> client_context);
 	virtual ~TableDataWriter();
 
 public:
@@ -39,9 +39,11 @@ public:
 
 	TaskScheduler &GetScheduler();
 	DatabaseInstance &GetDatabase();
+	optional_ptr<ClientContext> GetClientContext();
 
 protected:
 	DuckTableEntry &table;
+	optional_ptr<ClientContext> client_context;
 	//! Pointers to the start of each row group.
 	vector<RowGroupPointer> row_group_pointers;
 };

package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp

@@ -97,7 +97,8 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
 	friend class SingleFileTableDataWriter;
 
 public:
-	SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager, CheckpointType checkpoint_type);
+	SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context, AttachedDatabase &db,
+	                           BlockManager &block_manager, CheckpointType checkpoint_type);
 
 	//! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
 	//! connection is available because right now the checkpointing cannot be done online. (TODO)
@@ -112,10 +113,15 @@ public:
 		return checkpoint_type;
 	}
 
+	optional_ptr<ClientContext> GetClientContext() const {
+		return client_context;
+	}
+
 public:
 	void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
 
 private:
+	optional_ptr<ClientContext> client_context;
 	//! The metadata writer is responsible for writing schema information
 	unique_ptr<MetadataWriter> metadata_writer;
 	//! The table data writer is responsible for writing the DataPointers used by the table chunks

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -100,7 +100,8 @@ public:
 	virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
 	virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
 	virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
-	virtual void CreateCheckpoint(CheckpointOptions options = CheckpointOptions()) = 0;
+	virtual void CreateCheckpoint(optional_ptr<ClientContext> client_context,
+	                              CheckpointOptions options = CheckpointOptions()) = 0;
 	virtual DatabaseSize GetDatabaseSize() = 0;
 	virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -159,7 +160,7 @@ public:
 	bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
 	unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
 	bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
-	void CreateCheckpoint(CheckpointOptions options) override;
+	void CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) override;
 	DatabaseSize GetDatabaseSize() override;
 	vector<MetadataBlockInfo> GetMetadataInfo() override;
 	shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;