duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine.hpp

@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_state_machine.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp"
+
+namespace duckdb {
+
+//! All States of CSV Parsing
+enum class CSVState : uint8_t {
+	STANDARD = 0,         //! Regular unquoted field state
+	DELIMITER = 1,        //! State after encountering a field separator (e.g., ;)
+	RECORD_SEPARATOR = 2, //! State after encountering a record separator (i.e., \n)
+	CARRIAGE_RETURN = 3,  //! State after encountering a carriage return(i.e., \r)
+	QUOTED = 4,           //! State when inside a quoted field
+	UNQUOTED = 5,         //! State when leaving a quoted field
+	ESCAPE = 6,           //! State when encountering an escape character (e.g., \)
+	EMPTY_LINE = 7,       //! State when encountering an empty line (i.e., \r\r \n\n, \n\r)
+	INVALID = 8           //! Got to an Invalid State, this should error.
+};
+
+//! The CSV State Machine comprises a state transition array (STA).
+//! The STA indicates the current state of parsing based on both the current and preceding characters.
+//! This reveals whether we are dealing with a Field, a New Line, a Delimiter, and so forth.
+//! The STA's creation depends on the provided quote, character, and delimiter options for that state machine.
+//! The motivation behind implementing an STA is to remove branching in regular CSV Parsing by predicting and detecting
+//! the states. Note: The State Machine is currently utilized solely in the CSV Sniffer.
+class CSVStateMachine {
+public:
+	explicit CSVStateMachine(CSVReaderOptions &options_p, const CSVStateMachineOptions &state_machine_options,
+	                         shared_ptr<CSVBufferManager> buffer_manager_p,
+	                         CSVStateMachineCache &csv_state_machine_cache_p);
+	//! Resets the state machine, so it can be used again
+	void Reset();
+
+	//! Aux Function for string UTF8 Verification
+	void VerifyUTF8();
+
+	CSVStateMachineCache &csv_state_machine_cache;
+
+	const CSVReaderOptions &options;
+	CSVBufferIterator csv_buffer_iterator;
+	//! Stores identified start row for this file (e.g., a file can start with garbage like notes, before the header)
+	idx_t start_row = 0;
+	//! The Transition Array is a Finite State Machine
+	//! It holds the transitions of all states, on all 256 possible different characters
+	const state_machine_t &transition_array;
+
+	//! Both these variables are used for new line identifier detection
+	bool single_record_separator = false;
+	bool carry_on_separator = false;
+
+	//! Variables Used for Sniffing
+	CSVState state;
+	CSVState previous_state;
+	CSVState pre_previous_state;
+	idx_t cur_rows;
+	idx_t column_count;
+	string value;
+	idx_t rows_read;
+	idx_t line_start_pos = 0;
+
+	//! Dialect options resulting from sniffing
+	DialectOptions dialect_options;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/scan/csv/quote_rules.hpp"
+
+namespace duckdb {
+static constexpr uint32_t NUM_STATES = 9;
+static constexpr uint32_t NUM_TRANSITIONS = 256;
+typedef uint8_t state_machine_t[NUM_STATES][NUM_TRANSITIONS];
+
+//! Hash function used in out state machine cache, it hashes and combines all options used to generate a state machine
+struct HashCSVStateMachineConfig {
+	size_t operator()(CSVStateMachineOptions const &config) const noexcept {
+		auto h_delimiter = Hash(config.delimiter);
+		auto h_quote = Hash(config.quote);
+		auto h_escape = Hash(config.escape);
+		return CombineHash(h_delimiter, CombineHash(h_quote, h_escape));
+	}
+};
+
+//! The CSVStateMachineCache caches state machines, although small ~2kb, the actual creation of multiple State Machines
+//! can become a bottleneck on sniffing, when reading very small csv files.
+//! Hence the cache stores State Machines based on their different delimiter|quote|escape options.
+class CSVStateMachineCache {
+public:
+	CSVStateMachineCache();
+	~CSVStateMachineCache() {};
+	//! Gets a state machine from the cache, if it's not from one the default options
+	//! It first caches it, then returns it.
+	const state_machine_t &Get(const CSVStateMachineOptions &state_machine_options);
+
+private:
+	void Insert(const CSVStateMachineOptions &state_machine_options);
+	//! Cache on delimiter|quote|escape
+	unordered_map<CSVStateMachineOptions, state_machine_t, HashCSVStateMachineConfig> state_machine_cache;
+	//! Default value for options used to intialize CSV State Machine Cache
+	const vector<char> default_delimiter = {',', '|', ';', '\t'};
+	const vector<vector<char>> default_quote = {{'\"'}, {'\"', '\''}, {'\0'}};
+	const vector<QuoteRule> default_quote_rule = {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
+	const vector<vector<char>> default_escape = {{'\0', '\"', '\''}, {'\\'}, {'\0'}};
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/parallel_csv_reader.hpp

@@ -1,18 +1,18 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/parallel_csv_reader.hpp
+// duckdb/execution/operator/scan/csv/parallel_csv_reader.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/execution/operator/persistent/base_csv_reader.hpp"
-#include "duckdb/execution/operator/persistent/csv_reader_options.hpp"
-#include "duckdb/execution/operator/persistent/csv_file_handle.hpp"
-#include "duckdb/execution/operator/persistent/csv_buffer.hpp"
-#include "duckdb/execution/operator/persistent/csv_line_info.hpp"
+#include "duckdb/execution/operator/scan/csv/base_csv_reader.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_file_handle.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_buffer.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_line_info.hpp"
 
 #include <sstream>
 #include <utility>
@@ -20,21 +20,17 @@
 namespace duckdb {
 
 struct CSVBufferRead {
-	CSVBufferRead(shared_ptr<CSVBuffer> buffer_p, idx_t buffer_start_p, idx_t buffer_end_p, idx_t batch_index,
+	CSVBufferRead(unique_ptr<CSVBufferHandle> buffer_p, idx_t buffer_start_p, idx_t buffer_end_p, idx_t batch_index,
 	              idx_t local_batch_index_p, optional_ptr<LineInfo> line_info_p)
 	    : buffer(std::move(buffer_p)), line_info(line_info_p), buffer_start(buffer_start_p), buffer_end(buffer_end_p),
 	      batch_index(batch_index), local_batch_index(local_batch_index_p) {
-		if (buffer) {
-			if (buffer_end > buffer->GetBufferSize()) {
-				buffer_end = buffer->GetBufferSize();
-			}
-		} else {
-			buffer_start = 0;
-			buffer_end = 0;
+		D_ASSERT(buffer);
+		if (buffer_end > buffer->actual_size) {
+			buffer_end = buffer->actual_size;
 		}
 	}
 
-	CSVBufferRead(shared_ptr<CSVBuffer> buffer_p, shared_ptr<CSVBuffer> nxt_buffer_p, idx_t buffer_start_p,
+	CSVBufferRead(unique_ptr<CSVBufferHandle> buffer_p, unique_ptr<CSVBufferHandle> nxt_buffer_p, idx_t buffer_start_p,
 	              idx_t buffer_end_p, idx_t batch_index, idx_t local_batch_index, optional_ptr<LineInfo> line_info_p)
 	    : CSVBufferRead(std::move(buffer_p), buffer_start_p, buffer_end_p, batch_index, local_batch_index,
 	                    line_info_p) {
@@ -44,33 +40,33 @@ struct CSVBufferRead {
 	CSVBufferRead() : buffer_start(0), buffer_end(NumericLimits<idx_t>::Maximum()) {};
 
 	const char &operator[](size_t i) const {
-		if (i < buffer->GetBufferSize()) {
+		if (i < buffer->actual_size) {
 			auto buffer_ptr = buffer->Ptr();
 			return buffer_ptr[i];
 		}
 		auto next_ptr = next_buffer->Ptr();
-		return next_ptr[i - buffer->GetBufferSize()];
+		return next_ptr[i - buffer->actual_size];
 	}
 
 	string_t GetValue(idx_t start_buffer, idx_t position_buffer, idx_t offset) {
 		idx_t length = position_buffer - start_buffer - offset;
 		// 1) It's all in the current buffer
-		if (start_buffer + length <= buffer->GetBufferSize()) {
+		if (start_buffer + length <= buffer->actual_size) {
 			auto buffer_ptr = buffer->Ptr();
 			return string_t(buffer_ptr + start_buffer, length);
-		} else if (start_buffer >= buffer->GetBufferSize()) {
+		} else if (start_buffer >= buffer->actual_size) {
 			// 2) It's all in the next buffer
 			D_ASSERT(next_buffer);
-			D_ASSERT(next_buffer->GetBufferSize() >= length + (start_buffer - buffer->GetBufferSize()));
+			D_ASSERT(next_buffer->actual_size >= length + (start_buffer - buffer->actual_size));
 			auto buffer_ptr = next_buffer->Ptr();
-			return string_t(buffer_ptr + (start_buffer - buffer->GetBufferSize()), length);
+			return string_t(buffer_ptr + (start_buffer - buffer->actual_size), length);
 		} else {
 			// 3) It starts in the current buffer and ends in the next buffer
 			D_ASSERT(next_buffer);
 			auto intersection = make_unsafe_uniq_array<char>(length);
 			idx_t cur_pos = 0;
 			auto buffer_ptr = buffer->Ptr();
-			for (idx_t i = start_buffer; i < buffer->GetBufferSize(); i++) {
+			for (idx_t i = start_buffer; i < buffer->actual_size; i++) {
 				intersection[cur_pos++] = buffer_ptr[i];
 			}
 			idx_t nxt_buffer_pos = 0;
@@ -83,8 +79,8 @@ struct CSVBufferRead {
 		}
 	}
 
-	shared_ptr<CSVBuffer> buffer;
-	shared_ptr<CSVBuffer> next_buffer;
+	unique_ptr<CSVBufferHandle> buffer;
+	unique_ptr<CSVBufferHandle> next_buffer;
 	vector<unsafe_unique_array<char>> intersections;
 	optional_ptr<LineInfo> line_info;
 
@@ -103,7 +99,7 @@ struct VerificationPositions {
 //! CSV Reader for Parallel Reading
 class ParallelCSVReader : public BaseCSVReader {
 public:
-	ParallelCSVReader(ClientContext &context, BufferedCSVReaderOptions options, unique_ptr<CSVBufferRead> buffer,
+	ParallelCSVReader(ClientContext &context, CSVReaderOptions options, unique_ptr<CSVBufferRead> buffer,
 	                  idx_t first_pos_first_buffer, const vector<LogicalType> &requested_types, idx_t file_idx_p);
 	virtual ~ParallelCSVReader() {
 	}
@@ -137,7 +133,8 @@ public:
 	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
 	void ParseCSV(DataChunk &insert_chunk);
 
-	idx_t GetLineError(idx_t line_error, idx_t buffer_idx) override;
+	idx_t GetLineError(idx_t line_error, idx_t buffer_idx, bool stop_at_first = true) override;
+	void Increment(idx_t buffer_idx) override;
 
 private:
 	//! Initialize Parser
@@ -162,8 +159,6 @@ private:
 
 	//! Parses a CSV file with a one-byte delimiter, escape and quote character
 	bool TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message, bool try_add_line = false);
-	//! Verifies that the line length did not go over a pre-defined limit.
-	void VerifyLineLength(idx_t line_size);
 
 	//! First Position of First Buffer
 	idx_t first_pos_first_buffer = 0;

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/quote_rules.hpp

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/quote_rules.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/vector.hpp"
+
+namespace duckdb {
+//! Different Rules regarding possible combinations of Quote and Escape Values for CSV Dialects.
+//! Each rule has a comment on the possible combinations.
+enum class QuoteRule : uint8_t {
+	QUOTES_RFC = 0,   //! quote = " escape = (\0 || " || ')
+	QUOTES_OTHER = 1, //! quote = ( " || ' ) escape = '\\'
+	NO_QUOTES = 2     //! quote = \0 escape = \0
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_column_data_scan.hpp

@@ -22,11 +22,19 @@ public:
 	PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type, idx_t estimated_cardinality,
 	                       unique_ptr<ColumnDataCollection> owned_collection = nullptr);
 
+	PhysicalColumnDataScan(vector<LogicalType> types, PhysicalOperatorType op_type, idx_t estimated_cardinality,
+	                       idx_t cte_index)
+	    : PhysicalOperator(op_type, std::move(types), estimated_cardinality), collection(nullptr),
+	      cte_index(cte_index) {
+	}
+
 	// the column data collection to scan
 	optional_ptr<ColumnDataCollection> collection;
 	//! Owned column data collection, if any
 	unique_ptr<ColumnDataCollection> owned_collection;
 
+	idx_t cte_index;
+
 public:
 	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const override;
 	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
@@ -35,6 +43,8 @@ public:
 		return true;
 	}
 
+	string ParamsToString() const override;
+
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 };

package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "duckdb/storage/data_table.hpp"
+#include "duckdb/common/extra_operator_info.hpp"
 
 namespace duckdb {
 
@@ -21,14 +22,11 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::TABLE_SCAN;
 
 public:
-	//! Regular Table Scan
-	PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
-	                  vector<column_t> column_ids, vector<string> names, unique_ptr<TableFilterSet> table_filters,
-	                  idx_t estimated_cardinality);
 	//! Table scan that immediately projects out filter columns that are unused in the remainder of the query plan
 	PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
 	                  vector<LogicalType> returned_types, vector<column_t> column_ids, vector<idx_t> projection_ids,
-	                  vector<string> names, unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality);
+	                  vector<string> names, unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality,
+	                  ExtraOperatorInfo extra_info);
 
 	//! The table function
 	TableFunction function;
@@ -44,6 +42,8 @@ public:
 	vector<string> names;
 	//! The table filters
 	unique_ptr<TableFilterSet> table_filters;
+	//! Currently stores any filters applied to file names (as strings)
+	ExtraOperatorInfo extra_info;
 
 public:
 	string GetName() const override;

package/src/duckdb/src/include/duckdb/execution/operator/schema/{physical_create_index.hpp → physical_create_art_index.hpp}

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/schema/physical_create_index.hpp
+// duckdb/execution/operator/schema/physical_create_art_index.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -20,14 +20,14 @@ namespace duckdb {
 class DuckTableEntry;
 
 //! Physical CREATE (UNIQUE) INDEX statement
-class PhysicalCreateIndex : public PhysicalOperator {
+class PhysicalCreateARTIndex : public PhysicalOperator {
 public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::CREATE_INDEX;
 
 public:
-	PhysicalCreateIndex(LogicalOperator &op, TableCatalogEntry &table, const vector<column_t> &column_ids,
-	                    unique_ptr<CreateIndexInfo> info, vector<unique_ptr<Expression>> unbound_expressions,
-	                    idx_t estimated_cardinality);
+	PhysicalCreateARTIndex(LogicalOperator &op, TableCatalogEntry &table, const vector<column_t> &column_ids,
+	                       unique_ptr<CreateIndexInfo> info, vector<unique_ptr<Expression>> unbound_expressions,
+	                       idx_t estimated_cardinality, const bool sorted);
 
 	//! The table to create the index for
 	DuckTableEntry &table;
@@ -37,6 +37,8 @@ public:
 	unique_ptr<CreateIndexInfo> info;
 	//! Unbound expressions to be used in the optimizer
 	vector<unique_ptr<Expression>> unbound_expressions;
+	//! Whether the pipeline sorts the data prior to index creation
+	const bool sorted;
 
 public:
 	//! Source interface, NOP for this operator
@@ -52,10 +54,15 @@ public:
 	//! Sink interface, global sink state
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
+	//! Sink for unsorted data: insert iteratively
+	SinkResultType SinkUnsorted(Vector &row_identifiers, OperatorSinkInput &input) const;
+	//! Sink for sorted data: build + merge
+	SinkResultType SinkSorted(Vector &row_identifiers, OperatorSinkInput &input) const;
+
 	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
-	void Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const override;
+	SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
 	SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-	                          GlobalSinkState &gstate) const override;
+	                          OperatorSinkFinalizeInput &input) const override;
 
 	bool IsSink() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/set/physical_cte.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/execution/physical_operator.hpp"
+
+namespace duckdb {
+
+class RecursiveCTEState;
+
+class PhysicalCTE : public PhysicalOperator {
+public:
+	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::CTE;
+
+public:
+	PhysicalCTE(string ctename, idx_t table_index, vector<LogicalType> types, unique_ptr<PhysicalOperator> top,
+	            unique_ptr<PhysicalOperator> bottom, idx_t estimated_cardinality);
+	~PhysicalCTE() override;
+
+	std::shared_ptr<ColumnDataCollection> working_table;
+	shared_ptr<MetaPipeline> recursive_meta_pipeline;
+
+	idx_t table_index;
+	string ctename;
+
+public:
+	// Source interface
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
+
+public:
+	// Sink interface
+	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
+
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+
+	string ParamsToString() const override;
+
+public:
+	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
+
+	vector<const_reference<PhysicalOperator>> GetSources() const override;
+
+private:
+	void ExecuteRecursivePipelines(ExecutionContext &context) const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/set/physical_recursive_cte.hpp

@@ -20,10 +20,14 @@ public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::RECURSIVE_CTE;
 
 public:
-	PhysicalRecursiveCTE(vector<LogicalType> types, bool union_all, unique_ptr<PhysicalOperator> top,
-	                     unique_ptr<PhysicalOperator> bottom, idx_t estimated_cardinality);
+	PhysicalRecursiveCTE(string ctename, idx_t table_index, vector<LogicalType> types, bool union_all,
+	                     unique_ptr<PhysicalOperator> top, unique_ptr<PhysicalOperator> bottom,
+	                     idx_t estimated_cardinality);
 	~PhysicalRecursiveCTE() override;
 
+	string ctename;
+	idx_t table_index;
+
 	bool union_all;
 	std::shared_ptr<ColumnDataCollection> working_table;
 	shared_ptr<MetaPipeline> recursive_meta_pipeline;
@@ -46,6 +50,8 @@ public:
 		return true;
 	}
 
+	string ParamsToString() const override;
+
 public:
 	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
 

package/src/duckdb/src/include/duckdb/execution/perfect_aggregate_hashtable.hpp

@@ -56,8 +56,10 @@ protected:
 	//! Reused selection vector
 	SelectionVector sel;
 
-	//! The arena allocator used by the aggregates for their internal state
-	ArenaAllocator aggregate_allocator;
+	//! The active arena allocator used by the aggregates for their internal state
+	unique_ptr<ArenaAllocator> aggregate_allocator;
+	//! Owning arena allocators that this HT has data from
+	vector<unique_ptr<ArenaAllocator>> stored_allocators;
 
 private:
 	//! Destroy the perfect aggregate HT (called automatically by the destructor)

package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp

@@ -36,7 +36,6 @@ public:
 public:
 	PhysicalOperator(PhysicalOperatorType type, vector<LogicalType> types, idx_t estimated_cardinality)
 	    : type(type), types(std::move(types)), estimated_cardinality(estimated_cardinality) {
-		estimated_props = make_uniq<EstimatedProperties>(estimated_cardinality, 0);
 	}
 
 	virtual ~PhysicalOperator() {
@@ -50,7 +49,6 @@ public:
 	vector<LogicalType> types;
 	//! The estimated cardinality of this physical operator
 	idx_t estimated_cardinality;
-	unique_ptr<EstimatedProperties> estimated_props;
 
 	//! The global sink state of this operator
 	unique_ptr<GlobalSinkState> sink_state;
@@ -140,14 +138,14 @@ public:
 	// The combine is called when a single thread has completed execution of its part of the pipeline, it is the final
 	// time that a specific LocalSinkState is accessible. This method can be called in parallel while other Sink() or
 	// Combine() calls are active on the same GlobalSinkState.
-	virtual void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const;
+	virtual SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const;
 	//! The finalize is called when ALL threads are finished execution. It is called only once per pipeline, and is
 	//! entirely single threaded.
 	//! If Finalize returns SinkResultType::FINISHED, the sink is marked as finished
 	virtual SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
-	                                  GlobalSinkState &gstate) const;
+	                                  OperatorSinkFinalizeInput &input) const;
 	//! For sinks with RequiresBatchIndex set to true, when a new batch starts being processed this method is called
-	//! This allows flushing of the current batch (e.g. to disk)
+	//! This allows flushing of the current batch (e.g. to disk) TODO: should this be able to block too?
 	virtual void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const;
 
 	virtual unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const;
@@ -156,6 +154,9 @@ public:
 	//! The maximum amount of memory the operator should use per thread.
 	static idx_t GetMaxThreadMemory(ClientContext &context);
 
+	//! Whether operator caching is allowed in the current execution context
+	static bool OperatorCachingAllowed(ExecutionContext &context);
+
 	virtual bool IsSink() const {
 		return false;
 	}

package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp

@@ -165,6 +165,17 @@ struct OperatorSourceInput {
 	InterruptState &interrupt_state;
 };
 
+struct OperatorSinkCombineInput {
+	GlobalSinkState &global_state;
+	LocalSinkState &local_state;
+	InterruptState &interrupt_state;
+};
+
+struct OperatorSinkFinalizeInput {
+	GlobalSinkState &global_state;
+	InterruptState &interrupt_state;
+};
+
 // LCOV_EXCL_STOP
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp

@@ -32,6 +32,8 @@ public:
 	//! Recursive CTEs require at least one ChunkScan, referencing the working_table.
 	//! This data structure is used to establish it.
 	unordered_map<idx_t, std::shared_ptr<ColumnDataCollection>> recursive_cte_tables;
+	//! Materialized CTE ids must be collected.
+	unordered_set<idx_t> materialized_ctes;
 
 public:
 	//! Creates a plan from the logical operator. This involves resolving column bindings and generating physical
@@ -48,7 +50,6 @@ protected:
 
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalAggregate &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalAnyJoin &op);
-	unique_ptr<PhysicalOperator> CreatePlan(LogicalAsOfJoin &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalColumnDataGet &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalComparisonJoin &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalCreate &op);
@@ -57,7 +58,6 @@ protected:
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalCrossProduct &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalDelete &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalDelimGet &op);
-	unique_ptr<PhysicalOperator> CreatePlan(LogicalDelimJoin &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalDistinct &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalDummyScan &expr);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalEmptyResult &op);
@@ -87,9 +87,13 @@ protected:
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalSimple &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalUnnest &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalRecursiveCTE &op);
+	unique_ptr<PhysicalOperator> CreatePlan(LogicalMaterializedCTE &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalCTERef &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalPivot &op);
 
+	unique_ptr<PhysicalOperator> PlanAsOfJoin(LogicalComparisonJoin &op);
+	unique_ptr<PhysicalOperator> PlanComparisonJoin(LogicalComparisonJoin &op);
+	unique_ptr<PhysicalOperator> PlanDelimJoin(LogicalComparisonJoin &op);
 	unique_ptr<PhysicalOperator> ExtractAggregateExpressions(unique_ptr<PhysicalOperator> child,
 	                                                         vector<unique_ptr<Expression>> &expressions,
 	                                                         vector<unique_ptr<Expression>> &groups);