duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp

@@ -105,8 +105,9 @@ public:
 	//! Load a secret storage
 	DUCKDB_API void LoadSecretStorage(unique_ptr<SecretStorage> storage);
 
-	//! Deserialize a secret by automatically selecting the correct deserializer
-	DUCKDB_API unique_ptr<BaseSecret> DeserializeSecret(Deserializer &deserializer);
+	//! Deserialize a secret by automatically selecting the correct deserializer, secret_path can be set to improve
+	//! error hints
+	DUCKDB_API unique_ptr<BaseSecret> DeserializeSecret(Deserializer &deserializer, const string &secret_path = "");
 	//! Register a new SecretType
 	DUCKDB_API void RegisterSecretType(SecretType &type);
 	//! Lookup a SecretType
@@ -153,8 +154,14 @@ public:
 	                                 const string &storage = "");
 
 private:
-	//! Lookup a SecretType
+	//! Register a secret type
+	void RegisterSecretTypeInternal(SecretType &type);
+	//! Lookup a SecretType, throws if not found
 	SecretType LookupTypeInternal(const string &type);
+	//! Try to lookup a SecretType
+	bool TryLookupTypeInternal(const string &type, SecretType &type_out);
+	//! Register a secret provider
+	void RegisterSecretFunctionInternal(CreateSecretFunction function, OnCreateConflict on_conflict);
 	//! Lookup a CreateSecretFunction
 	optional_ptr<CreateSecretFunction> LookupFunctionInternal(const string &type, const string &provider);
 	//! Register a new Secret
@@ -172,7 +179,7 @@ private:
 	void AutoloadExtensionForFunction(const string &type, const string &provider);
 
 	//! Will throw appropriate error message when type not found
-	[[noreturn]] void ThrowTypeNotFoundError(const string &type);
+	[[noreturn]] void ThrowTypeNotFoundError(const string &type, const string &secret_path = "");
 	[[noreturn]] void ThrowProviderNotFoundError(const string &type, const string &provider, bool was_default = false);
 
 	//! Thread-safe accessors for secret_storages
@@ -204,10 +211,13 @@ public:
 	DefaultSecretGenerator(Catalog &catalog, SecretManager &secret_manager, case_insensitive_set_t &persistent_secrets);
 
 public:
+	unique_ptr<CatalogEntry> CreateDefaultEntry(CatalogTransaction transaction, const string &entry_name) override;
 	unique_ptr<CatalogEntry> CreateDefaultEntry(ClientContext &context, const string &entry_name) override;
 	vector<string> GetDefaultEntries() override;
 
 protected:
+	unique_ptr<CatalogEntry> CreateDefaultEntryInternal(const string &entry_name);
+
 	SecretManager &secret_manager;
 	case_insensitive_set_t persistent_secrets;
 };

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -18,7 +18,16 @@ struct DBConfig;
 
 const string GetDefaultUserAgent();
 
-enum class SettingScope : uint8_t { GLOBAL, LOCAL, INVALID };
+enum class SettingScope : uint8_t {
+	//! Setting is from the global Setting scope
+	GLOBAL,
+	//! Setting is from the local Setting scope
+	LOCAL,
+	//! Setting was not feteched from settings, but it was fetched from a secret instead
+	SECRET,
+	//! The setting was not found or invalid in some other way
+	INVALID
+};
 
 struct SettingLookupResult {
 public:
@@ -62,6 +71,16 @@ struct AllowPersistentSecrets {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct CatalogErrorMaxSchema {
+	static constexpr const char *Name = "catalog_error_max_schemas";
+	static constexpr const char *Description =
+	    "The maximum number of schemas the system will scan for \"did you mean...\" style errors in the catalog";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct CheckpointThresholdSetting {
 	static constexpr const char *Name = "checkpoint_threshold";
 	static constexpr const char *Description =
@@ -102,6 +121,15 @@ struct DebugForceNoCrossProduct {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct DebugSkipCheckpointOnCommit {
+	static constexpr const char *Name = "debug_skip_checkpoint_on_commit";
+	static constexpr const char *Description = "DEBUG SETTING: skip checkpointing on commit";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct OrderedAggregateThreshold {
 	static constexpr const char *Name = "ordered_aggregate_threshold"; // NOLINT
 	static constexpr const char *Description =                         // NOLINT
@@ -348,6 +376,15 @@ struct EnableProfilingSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct CustomProfilingSettings {
+	static constexpr const char *Name = "custom_profiling_settings";
+	static constexpr const char *Description = "Accepts a JSON enabling custom metrics";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct EnableProgressBarSetting {
 	static constexpr const char *Name = "enable_progress_bar";
 	static constexpr const char *Description =
@@ -450,6 +487,33 @@ struct HomeDirectorySetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct HTTPProxy {
+	static constexpr const char *Name = "http_proxy";
+	static constexpr const char *Description = "HTTP proxy host";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct HTTPProxyUsername {
+	static constexpr const char *Name = "http_proxy_username";
+	static constexpr const char *Description = "Username for HTTP proxy";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct HTTPProxyPassword {
+	static constexpr const char *Name = "http_proxy";
+	static constexpr const char *Description = "Password for HTTP proxy";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct IntegerDivisionSetting {
 	static constexpr const char *Name = "integer_division";
 	static constexpr const char *Description =
@@ -479,6 +543,16 @@ struct LockConfigurationSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct IEEEFloatingPointOpsSetting {
+	static constexpr const char *Name = "ieee_floating_point_ops";
+	static constexpr const char *Description =
+	    "Use IEE754-compliant floating point operations (returning NAN instead of errors/NULL)";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct ImmediateTransactionModeSetting {
 	static constexpr const char *Name = "immediate_transaction_mode";
 	static constexpr const char *Description =
@@ -509,6 +583,16 @@ struct MaximumMemorySetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct StreamingBufferSize {
+	static constexpr const char *Name = "streaming_buffer_size";
+	static constexpr const char *Description =
+	    "The maximum memory to buffer between fetching from a streaming result (e.g. 1GB)";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct MaximumTempDirectorySize {
 	static constexpr const char *Name = "max_temp_directory_size";
 	static constexpr const char *Description =
@@ -519,6 +603,34 @@ struct MaximumTempDirectorySize {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct MaximumVacuumTasks {
+	static constexpr const char *Name = "max_vacuum_tasks";
+	static constexpr const char *Description = "The maximum vacuum tasks to schedule during a checkpoint";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct MergeJoinThreshold {
+	static constexpr const char *Name = "merge_join_threshold";
+	static constexpr const char *Description = "The number of rows we need on either table to choose a merge join";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct NestedLoopJoinThreshold {
+	static constexpr const char *Name = "nested_loop_join_threshold";
+	static constexpr const char *Description =
+	    "The number of rows we need on either table to choose a nested loop join";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct OldImplicitCasting {
 	static constexpr const char *Name = "old_implicit_casting";
 	static constexpr const char *Description = "Allow implicit casting to/from VARCHAR";
@@ -528,16 +640,68 @@ struct OldImplicitCasting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct OrderByNonIntegerLiteral {
+	static constexpr const char *Name = "order_by_non_integer_literal";
+	static constexpr const char *Description =
+	    "Allow ordering by non-integer literals - ordering by such literals has no effect";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct PartitionedWriteFlushThreshold {
 	static constexpr const char *Name = "partitioned_write_flush_threshold";
 	static constexpr const char *Description =
 	    "The threshold in number of rows after which we flush a thread state when writing using PARTITION_BY";
-	static constexpr const LogicalTypeId InputType = LogicalTypeId::BIGINT;
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
 	static void SetLocal(ClientContext &context, const Value &parameter);
 	static void ResetLocal(ClientContext &context);
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct PartitionedWriteMaxOpenFiles {
+	static constexpr const char *Name = "partitioned_write_max_open_files";
+	static constexpr const char *Description =
+	    "The maximum amount of files the system can keep open before flushing to disk when writing using PARTITION_BY";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct DefaultBlockAllocSize {
+	static constexpr const char *Name = "default_block_size";
+	static constexpr const char *Description =
+	    "The default block size for new duckdb database files (new as-in, they do not yet exist).";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct IndexScanPercentage {
+	static constexpr const char *Name = "index_scan_percentage";
+	static constexpr const char *Description =
+	    "The index scan percentage sets a threshold for index scans. If fewer than MAX(index_scan_max_count, "
+	    "index_scan_percentage * total_row_count) rows match, we perform an index scan instead of a table scan.";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::DOUBLE;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct IndexScanMaxCount {
+	static constexpr const char *Name = "index_scan_max_count";
+	static constexpr const char *Description =
+	    "The maximum index scan count sets a threshold for index scans. If fewer than MAX(index_scan_max_count, "
+	    "index_scan_percentage * total_row_count) rows match, we perform an index scan instead of a table scan.";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::UBIGINT;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct PasswordSetting {
 	static constexpr const char *Name = "password";
 	static constexpr const char *Description = "The password to use. Ignored for legacy compatibility.";
@@ -596,6 +760,37 @@ struct PreserveInsertionOrder {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct ArrowOutputListView {
+	static constexpr const char *Name = "arrow_output_list_view";
+	static constexpr const char *Description =
+	    "If export to arrow format should use ListView as the physical layout for LIST columns";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct LosslessConversionArrow {
+	static constexpr const char *Name = "arrow_lossless_conversion";
+	static constexpr const char *Description =
+	    "Whenever a DuckDB type does not have a clear native or canonical extension match in Arrow, export the types "
+	    "with a duckdb.type_name extension name.";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct ProduceArrowStringView {
+	static constexpr const char *Name = "produce_arrow_string_view";
+	static constexpr const char *Description =
+	    "If strings should be produced by DuckDB in Utf8View format instead of Utf8";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct ProfileOutputSetting {
 	static constexpr const char *Name = "profile_output";
 	static constexpr const char *Description =
@@ -625,6 +820,16 @@ struct ProgressBarTimeSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct ScalarSubqueryErrorOnMultipleRows {
+	static constexpr const char *Name = "scalar_subquery_error_on_multiple_rows";
+	static constexpr const char *Description =
+	    "When a scalar subquery returns multiple rows - return a random row instead of returning an error";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetLocal(ClientContext &context, const Value &parameter);
+	static void ResetLocal(ClientContext &context);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct SchemaSetting {
 	static constexpr const char *Name = "schema";
 	static constexpr const char *Description =
@@ -681,7 +886,7 @@ struct UsernameSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
-struct FlushAllocatorSetting {
+struct AllocatorFlushThreshold {
 	static constexpr const char *Name = "allocator_flush_threshold";
 	static constexpr const char *Description =
 	    "Peak allocation threshold at which to flush the allocator after completing a task.";
@@ -691,6 +896,25 @@ struct FlushAllocatorSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct AllocatorBulkDeallocationFlushThreshold {
+	static constexpr const char *Name = "allocator_bulk_deallocation_flush_threshold";
+	static constexpr const char *Description =
+	    "If a bulk deallocation larger than this occurs, flush outstanding allocations.";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::VARCHAR;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
+struct AllocatorBackgroundThreadsSetting {
+	static constexpr const char *Name = "allocator_background_threads";
+	static constexpr const char *Description = "Whether to enable the allocator background thread.";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct DuckDBApiSetting {
 	static constexpr const char *Name = "duckdb_api";
 	static constexpr const char *Description = "DuckDB API surface";

package/src/duckdb/src/include/duckdb/main/stream_query_result.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/main/query_result.hpp"
 #include "duckdb/parallel/interrupt.hpp"
 #include "duckdb/common/queue.hpp"
+#include "duckdb/common/enums/stream_execution_result.hpp"
 #include "duckdb/main/buffered_data/simple_buffered_data.hpp"
 
 namespace duckdb {
@@ -34,9 +35,15 @@ public:
 	DUCKDB_API StreamQueryResult(StatementType statement_type, StatementProperties properties,
 	                             vector<LogicalType> types, vector<string> names, ClientProperties client_properties,
 	                             shared_ptr<BufferedData> buffered_data);
+	DUCKDB_API explicit StreamQueryResult(ErrorData error);
 	DUCKDB_API ~StreamQueryResult() override;
 
 public:
+	static bool IsChunkReady(StreamExecutionResult result);
+	//! Reschedules the tasks that work on producing a result chunk, returning when at least one task can be executed
+	DUCKDB_API void WaitForTask();
+	//! Executes a single task within the final pipeline, returning whether or not a chunk is ready to be fetched
+	DUCKDB_API StreamExecutionResult ExecuteTask();
 	//! Fetches a DataChunk from the query result.
 	DUCKDB_API unique_ptr<DataChunk> FetchRaw() override;
 	//! Converts the QueryResult to a string
@@ -53,6 +60,7 @@ public:
 	shared_ptr<ClientContext> context;
 
 private:
+	StreamExecutionResult ExecuteTaskInternal(ClientContextLock &lock);
 	unique_ptr<DataChunk> FetchInternal(ClientContextLock &lock);
 	unique_ptr<ClientContextLock> LockContext();
 	void CheckExecutableInternal(ClientContextLock &lock);

package/src/duckdb/src/include/duckdb/optimizer/build_probe_side_optimizer.hpp

@@ -0,0 +1,51 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/build_side_probe_side_optimizer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/common/vector.hpp"
+#include "duckdb/planner/logical_operator.hpp"
+#include "duckdb/planner/operator/logical_filter.hpp"
+
+namespace duckdb {
+
+struct BuildSize {
+	double left_side;
+	double right_side;
+
+	// Initialize with 1 so the build side is just the cardinality if types aren't
+	// known.
+	BuildSize() : left_side(1), right_side(1) {
+	}
+};
+
+class BuildProbeSideOptimizer : LogicalOperatorVisitor {
+private:
+	static constexpr idx_t COLUMN_COUNT_PENALTY = 2;
+	static constexpr double PREFER_RIGHT_DEEP_PENALTY = 0.15;
+
+public:
+	explicit BuildProbeSideOptimizer(ClientContext &context, LogicalOperator &op);
+
+	void VisitOperator(LogicalOperator &op) override;
+	void VisitExpression(unique_ptr<Expression> *expression) override {};
+
+private:
+	void TryFlipJoinChildren(LogicalOperator &op);
+	static idx_t ChildHasJoins(LogicalOperator &op);
+
+	static BuildSize GetBuildSizes(const LogicalOperator &op, idx_t lhs_cardinality, idx_t rhs_cardinality);
+	static double GetBuildSize(vector<LogicalType> types, idx_t cardinality);
+
+private:
+	ClientContext &context;
+	vector<ColumnBinding> preferred_on_probe_side;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/compressed_materialization.hpp

@@ -74,6 +74,11 @@ typedef column_binding_map_t<unique_ptr<BaseStatistics>> statistics_map_t;
 //! The CompressedMaterialization optimizer compressed columns using projections, based on available statistics,
 //! but only if the data enters a materializing operator
 class CompressedMaterialization {
+private:
+	//! Somewhat defensive constants that try to limit when compressed materialization is triggered for joins
+	static constexpr idx_t JOIN_BUILD_CARDINALITY_THRESHOLD = 1048576;
+	static constexpr double JOIN_CARDINALITY_RATIO_THRESHOLD = 8;
+
 public:
 	CompressedMaterialization(Optimizer &optimizer, LogicalOperator &root, statistics_map_t &statistics_map);
 
@@ -82,11 +87,13 @@ public:
 private:
 	//! Compress materializing operators
 	void CompressAggregate(unique_ptr<LogicalOperator> &op);
+	void CompressComparisonJoin(unique_ptr<LogicalOperator> &op);
 	void CompressDistinct(unique_ptr<LogicalOperator> &op);
 	void CompressOrder(unique_ptr<LogicalOperator> &op);
 
 	//! Update statistics after compressing
 	void UpdateAggregateStats(unique_ptr<LogicalOperator> &op);
+	void UpdateComparisonJoinStats(unique_ptr<LogicalOperator> &op);
 	void UpdateOrderStats(unique_ptr<LogicalOperator> &op);
 
 	//! Adds bindings referenced in expression to referenced_bindings

package/src/duckdb/src/include/duckdb/optimizer/cte_filter_pusher.hpp

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/cte_filter_pusher.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/insertion_order_preserving_map.hpp"
+
+namespace duckdb {
+
+class LogicalOperator;
+class Optimizer;
+
+class CTEFilterPusher {
+public:
+	explicit CTEFilterPusher(Optimizer &optimizer);
+	//! Finds all materialized CTEs and pushes OR filters into them (if applicable)
+	unique_ptr<LogicalOperator> Optimize(unique_ptr<LogicalOperator> op);
+
+private:
+	//! CTE info needed for creating OR filters that can be pushed down
+	struct MaterializedCTEInfo {
+		explicit MaterializedCTEInfo(LogicalOperator &materialized_cte);
+		LogicalOperator &materialized_cte;
+		vector<reference<LogicalOperator>> filters;
+		bool all_cte_refs_are_filtered;
+	};
+
+private:
+	//! Find all materialized CTEs and their refs
+	void FindCandidates(LogicalOperator &op);
+	//! Creates an OR filter and pushes it into a materialized CTE
+	void PushFilterIntoCTE(MaterializedCTEInfo &info);
+
+private:
+	//! The optimizer
+	Optimizer &optimizer;
+	//! Mapping from CTE index to CTE info, order preserving so deepest CTEs are done first
+	InsertionOrderPreservingMap<unique_ptr<MaterializedCTEInfo>> cte_info_map;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp

@@ -47,7 +47,7 @@ public:
 
 	void GenerateFilters(const std::function<void(unique_ptr<Expression> filter)> &callback);
 	bool HasFilters();
-	TableFilterSet GenerateTableScanFilters(vector<idx_t> &column_ids);
+	TableFilterSet GenerateTableScanFilters(const vector<idx_t> &column_ids);
 	// vector<unique_ptr<TableFilter>> GenerateZonemapChecks(vector<idx_t> &column_ids, vector<unique_ptr<TableFilter>>
 	// &pushed_filters);
 

package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp

@@ -25,6 +25,8 @@ public:
 	//! Return a reference to the client context (from the optimizer)
 	ClientContext &GetContext();
 
+	void CheckMarkToSemi(LogicalOperator &op, unordered_set<idx_t> &table_bindings);
+
 	struct Filter {
 		unordered_set<idx_t> bindings;
 		unique_ptr<Expression> filter;
@@ -61,6 +63,8 @@ private:
 	unique_ptr<LogicalOperator> PushdownGet(unique_ptr<LogicalOperator> op);
 	//! Push down a LogicalLimit op
 	unique_ptr<LogicalOperator> PushdownLimit(unique_ptr<LogicalOperator> op);
+	//! Push down a LogicalWindow op
+	unique_ptr<LogicalOperator> PushdownWindow(unique_ptr<LogicalOperator> op);
 	// Pushdown an inner join
 	unique_ptr<LogicalOperator> PushdownInnerJoin(unique_ptr<LogicalOperator> op, unordered_set<idx_t> &left_bindings,
 	                                              unordered_set<idx_t> &right_bindings);
@@ -88,6 +92,9 @@ private:
 	//! Adds a filter to the set of filters. Returns FilterResult::UNSATISFIABLE if the subtree should be stripped, or
 	//! FilterResult::SUCCESS otherwise
 	FilterResult AddFilter(unique_ptr<Expression> expr);
+	//! Extract filter bindings to compare them with expressions in an operator and determine if the filter
+	//! can be pushed down
+	void ExtractFilterBindings(Expression &expr, vector<ColumnBinding> &bindings);
 	//! Generate filters from the current set of filters stored in the FilterCombiner
 	void GenerateFilters();
 	//! if there are filters in this FilterPushdown node, push them into the combiner

package/src/duckdb/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/optimizer/join_filter_pushdown_optimizer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/planner/logical_operator_visitor.hpp"
+#include "duckdb/planner/column_binding_map.hpp"
+
+namespace duckdb {
+class Optimizer;
+
+//! The JoinFilterPushdownOptimizer links comparison joins to data sources to enable dynamic execution-time filter
+//! pushdown
+class JoinFilterPushdownOptimizer : public LogicalOperatorVisitor {
+public:
+	explicit JoinFilterPushdownOptimizer(Optimizer &optimizer);
+
+	void VisitOperator(LogicalOperator &op) override;
+
+private:
+	void GenerateJoinFilters(LogicalComparisonJoin &join);
+
+private:
+	Optimizer &optimizer;
+};
+} // namespace duckdb