duckdb 0.8.2-dev145.0 → 0.8.2-dev1493.0

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp

@@ -18,11 +18,12 @@ public:
 	bool CanItGetLine(idx_t file_idx, idx_t batch_idx);
 
 	//! Return the 1-indexed line number
-	idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true);
+	idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true,
+	              bool stop_at_first = true);
 	//! Verify if the CSV File was read correctly from [0,batch_idx] batches.
 	void Verify(idx_t file_idx, idx_t batch_idx, idx_t cur_first_pos);
-	//! Lines read per batch, <batch_index,count>
-	unordered_map<idx_t, idx_t> lines_read;
+	//! Lines read per batch, <file_index, <batch_index, count>>
+	vector<unordered_map<idx_t, idx_t>> lines_read;
 	//! Set of batches that have been initialized but are not yet finished.
 	vector<set<idx_t>> current_batches;
 	//! Pointer to CSV Reader Mutex

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp

@@ -55,6 +55,14 @@ struct BufferedCSVReaderOptions {
 	bool header = false;
 	//! Whether or not we should ignore InvalidInput errors
 	bool ignore_errors = false;
+	//! Rejects table name
+	string rejects_table_name;
+	//! Rejects table entry limit (0 = no limit)
+	idx_t rejects_limit = 0;
+	//! Columns to use as recovery key for rejected rows when reading with ignore_errors = true
+	vector<string> rejects_recovery_columns;
+	//! Index of the recovery columns
+	vector<idx_t> rejects_recovery_column_ids;
 	//! Expected number of columns
 	idx_t num_cols = 0;
 	//! Number of samples to buffer
@@ -155,7 +163,6 @@ struct BufferedCSVReaderOptions {
 	//! set - argument(s) to the option
 	//! expected_names - names expected if the option is "columns"
 	void SetReadOption(const string &loption, const Value &value, vector<string> &expected_names);
-
 	void SetWriteOption(const string &loption, const Value &value);
 	void SetDateFormat(LogicalTypeId type, const string &format, bool read_format);
 

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_rejects_table.hpp

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "duckdb.hpp"
+#ifndef DUCKDB_AMALGAMATION
+#include "duckdb/storage/object_cache.hpp"
+#endif
+
+namespace duckdb {
+
+struct ReadCSVData;
+
+class CSVRejectsTable : public ObjectCacheEntry {
+public:
+	CSVRejectsTable(string name) : name(name), count(0) {
+	}
+	~CSVRejectsTable() override = default;
+	mutex write_lock;
+	string name;
+	idx_t count;
+
+	static shared_ptr<CSVRejectsTable> GetOrCreate(ClientContext &context, const string &name);
+
+	void InitializeTable(ClientContext &context, const ReadCSVData &options);
+	TableCatalogEntry &GetTable(ClientContext &context);
+
+public:
+	static string ObjectType() {
+		return "csv_rejects_table_cache";
+	}
+
+	string GetObjectType() override {
+		return ObjectType();
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp

@@ -137,7 +137,7 @@ public:
 	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
 	void ParseCSV(DataChunk &insert_chunk);
 
-	idx_t GetLineError(idx_t line_error, idx_t buffer_idx) override;
+	idx_t GetLineError(idx_t line_error, idx_t buffer_idx, bool stop_at_first = true) override;
 
 private:
 	//! Initialize Parser

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp

@@ -22,11 +22,19 @@ public:
 	PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type, idx_t estimated_cardinality,
 	                       unique_ptr<ColumnDataCollection> owned_collection = nullptr);
 
+	PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type, idx_t estimated_cardinality,
+	                       idx_t cte_index)
+	    : PhysicalOperator(op_type, std::move(types), estimated_cardinality), collection(nullptr),
+	      cte_index(cte_index) {
+	}
+
 	// the column data collection to scan
 	optional_ptr<ColumnDataCollection> collection;
 	//! Owned column data collection, if any
 	unique_ptr<ColumnDataCollection> owned_collection;
 
+	idx_t cte_index;
+
 public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
@@ -35,6 +43,8 @@ public:
 		return true;
 	}
 
+	string ParamsToString() const override;
+
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 };

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/data_table.hpp"
+#include "duckdb/common/extra_operator_info.hpp"
 
 namespace duckdb {
 
@@ -28,7 +29,8 @@ public:
 	//! Table scan that immediately projects out filter columns that are unused in the remainder of the query plan
 	PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
 	                  vector<LogicalType> returned_types, vector<column_t> column_ids, vector<idx_t> projection_ids,
-	                  vector<string> names, unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality);
+	                  vector<string> names, unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality,
+	                  ExtraOperatorInfo extra_info);
 
 	//! The table function
 	TableFunction function;
@@ -44,6 +46,8 @@ public:
 	vector<string> names;
 	//! The table filters
 	unique_ptr<TableFilterSet> table_filters;
+	//! Currently stores any filters applied to file names (as strings)
+	ExtraOperatorInfo extra_info;
 
 public:
 	string GetName() const override;

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/set/physical_cte.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/execution/physical_operator.hpp"
+
+namespace duckdb {
+
+class RecursiveCTEState;
+
+class PhysicalCTE : public PhysicalOperator {
+public:
+	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::CTE;
+
+public:
+	PhysicalCTE(string ctename, idx_t table_index, vector<LogicalType> types, unique_ptr<PhysicalOperator> top,
+	            unique_ptr<PhysicalOperator> bottom, idx_t estimated_cardinality);
+	~PhysicalCTE() override;
+
+	std::shared_ptr<ColumnDataCollection> working_table;
+	shared_ptr<MetaPipeline> recursive_meta_pipeline;
+
+	idx_t table_index;
+	string ctename;
+
+public:
+	// Source interface
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
+
+public:
+	// Sink interface
+	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
+
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+
+	string ParamsToString() const override;
+
+public:
+	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
+
+	vector<const_reference<PhysicalOperator>> GetSources() const override;
+
+private:
+	void ExecuteRecursivePipelines(ExecutionContext &context) const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp

@@ -20,10 +20,14 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::RECURSIVE_CTE;
 
 public:
-	PhysicalRecursiveCTE(vector<LogicalType> types, bool union_all, unique_ptr<PhysicalOperator> top,
-	                     unique_ptr<PhysicalOperator> bottom, idx_t estimated_cardinality);
+	PhysicalRecursiveCTE(string ctename, idx_t table_index, vector<LogicalType> types, bool union_all,
+	                     unique_ptr<PhysicalOperator> top, unique_ptr<PhysicalOperator> bottom,
+	                     idx_t estimated_cardinality);
 	~PhysicalRecursiveCTE() override;
 
+	string ctename;
+	idx_t table_index;
+
 	bool union_all;
 	std::shared_ptr<ColumnDataCollection> working_table;
 	shared_ptr<MetaPipeline> recursive_meta_pipeline;
@@ -46,6 +50,8 @@ public:
 		return true;
 	}
 
+	string ParamsToString() const override;
+
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 

package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp

@@ -33,14 +33,18 @@ public:
 	                       vector<BoundAggregateExpression *> bindings_p);
 
 	idx_t AddChunk(DataChunk &groups, DataChunk &payload, bool do_partition, const unsafe_vector<idx_t> &filter);
-	void Partition();
+	void Partition(bool sink_done);
 	bool IsPartitioned();
 
 	HashTableList GetPartition(idx_t partition);
 	HashTableList GetUnpartitioned();
+	idx_t GetPartitionCount(idx_t partition) const;
+	idx_t GetPartitionSize(idx_t partition) const;
 
 	void Finalize();
 
+	void Append(GroupedAggregateHashTable &ht);
+
 private:
 	ClientContext &context;
 	Allocator &allocator;

package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp

@@ -32,6 +32,8 @@ public:
 	//! Recursive CTEs require at least one ChunkScan, referencing the working_table.
 	//! This data structure is used to establish it.
 	unordered_map<idx_t, std::shared_ptr<ColumnDataCollection>> recursive_cte_tables;
+	//! Materialized CTE ids must be collected.
+	unordered_set<idx_t> materialized_ctes;
 
 public:
 	//! Creates a plan from the logical operator. This involves resolving column bindings and generating physical
@@ -87,6 +89,7 @@ protected:
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalSimple &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalUnnest &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalRecursiveCTE &op);
+	unique_ptr<PhysicalOperator> CreatePlan(LogicalMaterializedCTE &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalCTERef &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalPivot &op);
 

package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp

@@ -8,10 +8,10 @@
 
 #pragma once
 
+#include "duckdb/execution/operator/aggregate/grouped_aggregate_data.hpp"
 #include "duckdb/execution/partitionable_hashtable.hpp"
-#include "duckdb/parser/group_by_node.hpp"
 #include "duckdb/execution/physical_operator.hpp"
-#include "duckdb/execution/operator/aggregate/grouped_aggregate_data.hpp"
+#include "duckdb/parser/group_by_node.hpp"
 
 namespace duckdb {
 class BufferManager;
@@ -57,11 +57,18 @@ public:
 	                         OperatorSourceInput &input) const;
 
 	static void SetMultiScan(GlobalSinkState &state);
-	bool ForceSingleHT(GlobalSinkState &state) const;
+	static bool ForceSingleHT(GlobalSinkState &state);
+	static bool AnyPartitioned(GlobalSinkState &state);
+	static void GetRepartitionInfo(ClientContext &context, GlobalSinkState &state, idx_t &repartition_radix_bits,
+	                               idx_t &concurrent_repartitions, idx_t &tasks_per_partition);
 
 private:
 	void SetGroupingValues();
 	void PopulateGroupChunk(DataChunk &group_chunk, DataChunk &input_chunk) const;
+	void InitializeFinalizedHTs(ClientContext &context, GlobalSinkState &state) const;
+	void ScheduleRepartitionTasks(Executor &executor, const shared_ptr<Event> &event, GlobalSinkState &state,
+	                              vector<shared_ptr<Task>> &tasks, const idx_t repartition_radix_bits,
+	                              const idx_t concurrent_repartitions, const idx_t tasks_per_partition) const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp

@@ -18,12 +18,13 @@ namespace duckdb {
 
 class WindowAggregateState {
 public:
-	WindowAggregateState(AggregateObject aggr, const LogicalType &result_type_p);
+	WindowAggregateState(AggregateObject aggr, const LogicalType &result_type_p, idx_t partition_count);
 	virtual ~WindowAggregateState();
 
 	virtual void Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered);
 	virtual void Finalize();
 	virtual void Compute(Vector &result, idx_t rid, idx_t start, idx_t end);
+	virtual void Evaluate(const idx_t *begins, const idx_t *ends, Vector &result, idx_t count);
 
 protected:
 	void AggregateInit();
@@ -33,20 +34,28 @@ protected:
 	//! The result type of the window function
 	LogicalType result_type;
 
+	//! The cardinality of the partition
+	const idx_t partition_count;
+	//! The size of a single aggregate state
+	const idx_t state_size;
 	//! Data pointer that contains a single state, used for intermediate window segment aggregation
 	vector<data_t> state;
 	//! Reused result state container for the window functions
-	Vector statev;
-	//! A vector of pointers to "state", used for intermediate window segment aggregation
-	Vector statep;
-	//! Input data chunk, used for intermediate window segment aggregation
+	Vector statef;
+	//! Partition data chunk
 	DataChunk inputs;
+
+	//! The filtered rows in inputs.
+	vector<validity_t> filter_bits;
+	ValidityMask filter_mask;
+	idx_t filter_pos;
+
+	//! Allocator for aggregates
+	ArenaAllocator allocator;
 };
 
 class WindowConstantAggregate : public WindowAggregateState {
 public:
-	static bool IsConstantAggregate(const BoundWindowExpression &wexpr);
-
 	WindowConstantAggregate(AggregateObject aggr, const LogicalType &result_type_p, const ValidityMask &partition_mask,
 	                        const idx_t count);
 	~WindowConstantAggregate() override {
@@ -54,7 +63,7 @@ public:
 
 	void Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered) override;
 	void Finalize() override;
-	void Compute(Vector &result, idx_t rid, idx_t start, idx_t end) override;
+	void Evaluate(const idx_t *begins, const idx_t *ends, Vector &result, idx_t count) override;
 
 private:
 	//! Partition starts
@@ -65,52 +74,57 @@ private:
 	idx_t partition;
 	//! The current input row being built/read
 	idx_t row;
+	//! A vector of pointers to "state", used for intermediate window segment aggregation
+	Vector statep;
+	//! Shared SV for evaluation
+	SelectionVector matches;
+};
+
+class WindowCustomAggregate : public WindowAggregateState {
+public:
+	WindowCustomAggregate(AggregateObject aggr, const LogicalType &result_type_p, idx_t partition_count);
+	~WindowCustomAggregate() override;
+
+	void Compute(Vector &result, idx_t rid, idx_t start, idx_t end) override;
+
+private:
+	//! The frame boundaries, used for the window functions
+	FrameBounds frame;
 };
 
-class WindowSegmentTree {
+class WindowSegmentTree : public WindowAggregateState {
 public:
 	using FrameBounds = std::pair<idx_t, idx_t>;
 
-	WindowSegmentTree(AggregateObject aggr, const LogicalType &result_type, DataChunk *input,
-	                  const ValidityMask &filter_mask, WindowAggregationMode mode);
-	~WindowSegmentTree();
+	WindowSegmentTree(AggregateObject aggr, const LogicalType &result_type, idx_t count, WindowAggregationMode mode_p);
+	~WindowSegmentTree() override;
 
-	//! First row contains the result.
-	void Compute(Vector &result, idx_t rid, idx_t start, idx_t end);
+	void Finalize() override;
+	void Evaluate(const idx_t *begins, const idx_t *ends, Vector &result, idx_t count) override;
 
 private:
 	void ConstructTree();
-	void ExtractFrame(idx_t begin, idx_t end);
-	void WindowSegmentValue(idx_t l_idx, idx_t begin, idx_t end);
-	void AggregateInit();
-	void AggegateFinal(Vector &result, idx_t rid);
+	void ExtractFrame(idx_t begin, idx_t end, data_ptr_t current_state);
+	void FlushStates(bool combining);
+	void WindowSegmentValue(idx_t l_idx, idx_t begin, idx_t end, data_ptr_t current_state);
 
-	//! Use the window API, if available
-	inline bool UseWindowAPI() const {
-		return mode < WindowAggregationMode::COMBINE;
-	}
 	//! Use the combine API, if available
 	inline bool UseCombineAPI() const {
 		return mode < WindowAggregationMode::SEPARATE;
 	}
 
-	//! The aggregate that the window function is computed over
-	AggregateObject aggr;
-	//! The result type of the window function
-	LogicalType result_type;
-
-	//! Data pointer that contains a single state, used for intermediate window segment aggregation
-	vector<data_t> state;
-	//! Input data chunk, used for intermediate window segment aggregation
-	DataChunk inputs;
+	//! Input data chunk, used for leaf segment aggregation
+	DataChunk leaves;
 	//! The filtered rows in inputs.
 	SelectionVector filter_sel;
 	//! A vector of pointers to "state", used for intermediate window segment aggregation
 	Vector statep;
 	//! The frame boundaries, used for the window functions
 	FrameBounds frame;
-	//! Reused result state container for the window functions
-	Vector statev;
+	//! Reused state pointers for combining segment tree levels
+	Vector statel;
+	//! Count of buffered values
+	idx_t flush_count;
 
 	//! The actual window segment tree: an array of aggregate states that represent all the intermediate nodes
 	unsafe_unique_array<data_t> levels_flat_native;
@@ -120,17 +134,14 @@ private:
 	//! The total number of internal nodes of the tree, stored in levels_flat_native
 	idx_t internal_nodes;
 
-	//! The (sorted) input chunk collection on which the tree is built
-	DataChunk *input_ref;
-
-	//! The filtered rows in input_ref.
-	const ValidityMask &filter_mask;
-
 	//! Use the window API, if available
 	WindowAggregationMode mode;
 
+	//! Aggregate allocator
+	ArenaAllocator allocator;
+
 	// TREE_FANOUT needs to cleanly divide STANDARD_VECTOR_SIZE
-	static constexpr idx_t TREE_FANOUT = 64;
+	static constexpr idx_t TREE_FANOUT = 16;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp

@@ -8,10 +8,10 @@
 
 #pragma once
 
+#include "duckdb/common/vector_operations/aggregate_executor.hpp"
 #include "duckdb/function/aggregate_state.hpp"
 #include "duckdb/planner/bound_result_modifier.hpp"
 #include "duckdb/planner/expression.hpp"
-#include "duckdb/common/vector_operations/aggregate_executor.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp

@@ -21,11 +21,11 @@ enum class AggregateOrderDependent : uint8_t { ORDER_DEPENDENT = 1, NOT_ORDER_DE
 class BoundAggregateExpression;
 
 struct AggregateInputData {
-	AggregateInputData(optional_ptr<FunctionData> bind_data_p, Allocator &allocator_p)
+	AggregateInputData(optional_ptr<FunctionData> bind_data_p, ArenaAllocator &allocator_p)
 	    : bind_data(bind_data_p), allocator(allocator_p) {
 	}
 	optional_ptr<FunctionData> bind_data;
-	Allocator &allocator;
+	ArenaAllocator &allocator;
 };
 
 struct AggregateUnaryInput {

package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp

@@ -57,6 +57,7 @@ private:
 	void RegisterDistributiveAggregates();
 
 	// scalar functions
+	void RegisterCompressedMaterializationFunctions();
 	void RegisterGenericFunctions();
 	void RegisterOperators();
 	void RegisterStringFunctions();

package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp

@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/scalar/compressed_materialization_functions.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/function/built_in_functions.hpp"
+#include "duckdb/function/function_set.hpp"
+
+namespace duckdb {
+
+struct CompressedMaterializationFunctions {
+	//! The types we compress integral types to
+	static const vector<LogicalType> IntegralTypes();
+	//! The types we compress strings to
+	static const vector<LogicalType> StringTypes();
+
+	static unique_ptr<FunctionData> Bind(ClientContext &context, ScalarFunction &bound_function,
+	                                     vector<unique_ptr<Expression>> &arguments);
+};
+
+//! Needed for (de)serialization without binding
+enum class CompressedMaterializationDirection : uint8_t { INVALID = 0, COMPRESS = 1, DECOMPRESS = 2 };
+
+struct CMIntegralCompressFun {
+	static ScalarFunction GetFunction(const LogicalType &input_type, const LogicalType &result_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct CMIntegralDecompressFun {
+	static ScalarFunction GetFunction(const LogicalType &input_type, const LogicalType &result_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct CMStringCompressFun {
+	static ScalarFunction GetFunction(const LogicalType &result_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct CMStringDecompressFun {
+	static ScalarFunction GetFunction(const LogicalType &input_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp

@@ -130,7 +130,7 @@ void ListContainsOrPosition(DataChunk &args, Vector &result) {
 		TemplatedContainsOrPosition<int8_t, T, OP, LIST_ACCESSOR>(args, result, true);
 		break;
 	default:
-		throw NotImplementedException("This function has not been implemented for physical type %s",
+		throw NotImplementedException("This function has not been implemented for logical type %s",
 		                              TypeIdToString(physical_type));
 	}
 }

package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp

@@ -99,6 +99,11 @@ struct ListPositionFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct ListResizeFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct StructExtractFun {
 	static ScalarFunction GetFunction();
 	static void RegisterFunction(BuiltinFunctions &set);

package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp

@@ -83,12 +83,14 @@ struct LengthFun {
 };
 
 struct LikeFun {
+	static ScalarFunction GetLikeFunction();
 	static void RegisterFunction(BuiltinFunctions &set);
 	DUCKDB_API static bool Glob(const char *s, idx_t slen, const char *pattern, idx_t plen,
 	                            bool allow_question_mark = true);
 };
 
 struct LikeEscapeFun {
+	static ScalarFunction GetLikeEscapeFun();
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 

package/src/duckdb/src/include/duckdb/main/client_config.hpp

@@ -96,6 +96,9 @@ struct ClientConfig {
 	//! The maximum amount of pivot columns
 	idx_t pivot_limit = 100000;
 
+	//! The threshold at which we switch from using filtered aggregates to LIST with a dedicated pivot operator
+	idx_t pivot_filter_threshold = 10;
+
 	//! Whether or not the "/" division operator defaults to integer division or floating point division
 	bool integer_division = false;
 

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -158,6 +158,8 @@ struct DBConfigOptions {
 	bool lock_configuration = false;
 	//! Whether to print bindings when printing the plan (debug mode only)
 	static bool debug_print_bindings;
+	//! The peak allocation threshold at which to flush the allocator after completing a task (1 << 27, ~128MB)
+	idx_t allocator_flush_threshold = 134217728;
 
 	bool operator==(const DBConfigOptions &other) const;
 };

package/src/duckdb/src/include/duckdb/main/connection.hpp

@@ -169,8 +169,7 @@ public:
 	template <typename TR, typename... Args>
 	void CreateScalarFunction(const string &name, vector<LogicalType> args, LogicalType ret_type,
 	                          TR (*udf_func)(Args...)) {
-		scalar_function_t function =
-		    UDFWrapper::CreateScalarFunction<TR, Args...>(name, args, std::move(ret_type), udf_func);
+		scalar_function_t function = UDFWrapper::CreateScalarFunction<TR, Args...>(name, args, ret_type, udf_func);
 		UDFWrapper::RegisterFunction(name, args, ret_type, function, *context);
 	}
 

package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp

@@ -9,15 +9,18 @@
 #pragma once
 
 #include "duckdb/main/relation.hpp"
+#include "duckdb/common/enums/joinref_type.hpp"
 
 namespace duckdb {
 
 class CrossProductRelation : public Relation {
 public:
-	DUCKDB_API CrossProductRelation(shared_ptr<Relation> left, shared_ptr<Relation> right);
+	DUCKDB_API CrossProductRelation(shared_ptr<Relation> left, shared_ptr<Relation> right,
+	                                JoinRefType join_ref_type = JoinRefType::CROSS);
 
 	shared_ptr<Relation> left;
 	shared_ptr<Relation> right;
+	JoinRefType ref_type;
 	vector<ColumnDefinition> columns;
 
 public: