duckdb 1.4.2-dev4.0 → 1.4.2

package/src/duckdb/src/include/duckdb/execution/index/unbound_index.hpp

@@ -16,15 +16,28 @@ namespace duckdb {
 
 class ColumnDataCollection;
 
+enum class BufferedIndexReplay : uint8_t { INSERT_ENTRY = 0, DEL_ENTRY = 1 };
+
+struct BufferedIndexData {
+	BufferedIndexReplay type;
+	unique_ptr<ColumnDataCollection> data;
+
+	BufferedIndexData(BufferedIndexReplay replay_type, unique_ptr<ColumnDataCollection> data_p);
+};
+
 class UnboundIndex final : public Index {
 private:
 	//! The CreateInfo of the index.
 	unique_ptr<CreateInfo> create_info;
 	//! The serialized storage information of the index.
 	IndexStorageInfo storage_info;
-	//! Buffer for WAL replay appends.
-	unique_ptr<ColumnDataCollection> buffered_appends;
-	//! Maps the column IDs in the buffered appends to the table columns.
+	//! Buffer for WAL replays.
+	vector<BufferedIndexData> buffered_replays;
+
+	//! Maps the column IDs in the buffered replays to a physical table offset.
+	//! For example, column [i] in a buffered ColumnDataCollection is the data for an Indexed column with
+	//! physical table index mapped_column_ids[i].
+	//! This is in sorted order of physical column IDs.
 	vector<StorageIndex> mapped_column_ids;
 
 public:
@@ -59,12 +72,17 @@ public:
 
 	void CommitDrop() override;
 
-	void BufferChunk(DataChunk &chunk, Vector &row_ids, const vector<StorageIndex> &mapped_column_ids_p);
-	bool HasBufferedAppends() const {
-		return buffered_appends != nullptr;
+	//! Buffer Index delete or insert (replay_type) data chunk.
+	//! See note above on mapped_column_ids, this function assumes that index_column_chunk maps into
+	//! mapped_column_ids_p to get the physical column index for each Indexed column in the chunk.
+	void BufferChunk(DataChunk &index_column_chunk, Vector &row_ids, const vector<StorageIndex> &mapped_column_ids_p,
+	                 BufferedIndexReplay replay_type);
+	bool HasBufferedReplays() const {
+		return !buffered_replays.empty();
 	}
-	ColumnDataCollection &GetBufferedAppends() const {
-		return *buffered_appends;
+
+	vector<BufferedIndexData> &GetBufferedReplays() {
+		return buffered_replays;
 	}
 	const vector<StorageIndex> &GetMappedColumnIds() const {
 		return mapped_column_ids;

package/src/duckdb/src/include/duckdb/execution/join_hashtable.hpp

@@ -277,6 +277,8 @@ public:
 	uint64_t bitmask = DConstants::INVALID_INDEX;
 	//! Whether or not we error on multiple rows found per match in a SINGLE join
 	bool single_join_error_on_multiple_rows = true;
+	//! Number of probe matches
+	atomic<idx_t> total_probe_matches {0};
 
 	struct {
 		mutex mj_lock;

package/src/duckdb/src/include/duckdb/function/table/read_file.hpp

@@ -31,53 +31,4 @@ struct ReadFileGlobalState : public GlobalTableFunctionState {
 	bool requires_file_open = false;
 };
 
-struct ReadBlobOperation {
-	static constexpr const char *NAME = "read_blob";
-	static constexpr const char *FILE_TYPE = "blob";
-
-	static inline LogicalType TYPE() {
-		return LogicalType::BLOB;
-	}
-};
-
-struct ReadTextOperation {
-	static constexpr const char *NAME = "read_text";
-	static constexpr const char *FILE_TYPE = "text";
-
-	static inline LogicalType TYPE() {
-		return LogicalType::VARCHAR;
-	}
-};
-
-template <class OP>
-struct DirectMultiFileInfo : MultiFileReaderInterface {
-	static unique_ptr<MultiFileReaderInterface> CreateInterface(ClientContext &context);
-	unique_ptr<BaseFileReaderOptions> InitializeOptions(ClientContext &context,
-	                                                    optional_ptr<TableFunctionInfo> info) override;
-	bool ParseCopyOption(ClientContext &context, const string &key, const vector<Value> &values,
-	                     BaseFileReaderOptions &options, vector<string> &expected_names,
-	                     vector<LogicalType> &expected_types) override;
-	bool ParseOption(ClientContext &context, const string &key, const Value &val, MultiFileOptions &file_options,
-	                 BaseFileReaderOptions &options) override;
-	unique_ptr<TableFunctionData> InitializeBindData(MultiFileBindData &multi_file_data,
-	                                                 unique_ptr<BaseFileReaderOptions> options) override;
-	void BindReader(ClientContext &context, vector<LogicalType> &return_types, vector<string> &names,
-	                MultiFileBindData &bind_data) override;
-	optional_idx MaxThreads(const MultiFileBindData &bind_data_p, const MultiFileGlobalState &global_state,
-	                        FileExpandResult expand_result) override;
-	unique_ptr<GlobalTableFunctionState> InitializeGlobalState(ClientContext &context, MultiFileBindData &bind_data,
-	                                                           MultiFileGlobalState &global_state) override;
-	unique_ptr<LocalTableFunctionState> InitializeLocalState(ExecutionContext &, GlobalTableFunctionState &) override;
-	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, GlobalTableFunctionState &gstate,
-	                                        BaseUnionData &union_data, const MultiFileBindData &bind_data_p) override;
-	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, GlobalTableFunctionState &gstate,
-	                                        const OpenFileInfo &file, idx_t file_idx,
-	                                        const MultiFileBindData &bind_data) override;
-	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, const OpenFileInfo &file,
-	                                        BaseFileReaderOptions &options,
-	                                        const MultiFileOptions &file_options) override;
-	unique_ptr<NodeStatistics> GetCardinality(const MultiFileBindData &bind_data, idx_t file_count) override;
-	FileGlobInput GetGlobInput() override;
-};
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/logging/log_manager.hpp

@@ -21,7 +21,7 @@ class LogType;
 // - Creates Loggers with cached configuration
 // - Main sink for logs (either by logging directly into this, or by syncing a pre-cached set of log entries)
 // - Holds the log storage
-class LogManager : public enable_shared_from_this<LogManager> {
+class LogManager {
 	friend class ThreadSafeLogger;
 	friend class ThreadLocalLogger;
 	friend class MutableLogger;

package/src/duckdb/src/include/duckdb/logging/log_type.hpp

@@ -20,6 +20,7 @@ class PhysicalOperator;
 class AttachedDatabase;
 class RowGroup;
 struct DataTableInfo;
+enum class MetricsType : uint8_t;
 
 //! Log types provide some structure to the formats that the different log messages can have
 //! For now, this holds a type that the VARCHAR value will be auto-cast into.
@@ -106,6 +107,19 @@ public:
 	                                  const vector<pair<string, string>> &info);
 };
 
+class MetricsLogType : public LogType {
+public:
+	static constexpr const char *NAME = "Metrics";
+	static constexpr LogLevel LEVEL = LogLevel::LOG_INFO;
+
+	//! Construct the log type
+	MetricsLogType();
+
+	static LogicalType GetLogType();
+
+	static string ConstructLogMessage(const MetricsType &type, const Value &value);
+};
+
 class CheckpointLogType : public LogType {
 public:
 	static constexpr const char *NAME = "Checkpoint";

package/src/duckdb/src/include/duckdb/main/attached_database.hpp

@@ -35,9 +35,10 @@ enum class AttachedDatabaseType {
 class DatabaseFilePathManager;
 
 struct StoredDatabasePath {
-	StoredDatabasePath(DatabaseFilePathManager &manager, string path, const string &name);
+	StoredDatabasePath(DatabaseManager &db_manager, DatabaseFilePathManager &manager, string path, const string &name);
 	~StoredDatabasePath();
 
+	DatabaseManager &db_manager;
 	DatabaseFilePathManager &manager;
 	string path;
 

package/src/duckdb/src/include/duckdb/main/buffered_data/batched_buffered_data.hpp

@@ -32,7 +32,7 @@ public:
 	static constexpr const BufferedData::Type TYPE = BufferedData::Type::BATCHED;
 
 public:
-	explicit BatchedBufferedData(weak_ptr<ClientContext> context);
+	explicit BatchedBufferedData(ClientContext &context);
 
 public:
 	void Append(const DataChunk &chunk, idx_t batch);

package/src/duckdb/src/include/duckdb/main/buffered_data/buffered_data.hpp

@@ -28,7 +28,7 @@ protected:
 	enum class Type { SIMPLE, BATCHED };
 
 public:
-	BufferedData(Type type, weak_ptr<ClientContext> context_p);
+	BufferedData(Type type, ClientContext &context);
 	virtual ~BufferedData();
 
 public:

package/src/duckdb/src/include/duckdb/main/buffered_data/simple_buffered_data.hpp

@@ -24,7 +24,7 @@ public:
 	static constexpr const BufferedData::Type TYPE = BufferedData::Type::SIMPLE;
 
 public:
-	explicit SimpleBufferedData(weak_ptr<ClientContext> context);
+	explicit SimpleBufferedData(ClientContext &context);
 	~SimpleBufferedData() override;
 
 public:

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -51,6 +51,8 @@ struct PreparedStatementWrapper {
 	//! Map of name -> values
 	case_insensitive_map_t<BoundParameterData> values;
 	unique_ptr<PreparedStatement> statement;
+	bool success = true;
+	ErrorData error_data;
 };
 
 struct ExtractStatementsWrapper {

package/src/duckdb/src/include/duckdb/main/database.hpp

@@ -69,7 +69,7 @@ public:
 
 	DUCKDB_API SettingLookupResult TryGetCurrentSetting(const string &key, Value &result) const;
 
-	DUCKDB_API shared_ptr<EncryptionUtil> GetEncryptionUtil() const;
+	DUCKDB_API shared_ptr<EncryptionUtil> GetEncryptionUtil();
 
 	shared_ptr<AttachedDatabase> CreateAttachedDatabase(ClientContext &context, AttachInfo &info,
 	                                                    AttachOptions &options);
@@ -90,7 +90,7 @@ private:
 	unique_ptr<ExtensionManager> extension_manager;
 	ValidChecker db_validity;
 	unique_ptr<DatabaseFileSystem> db_file_system;
-	shared_ptr<LogManager> log_manager;
+	unique_ptr<LogManager> log_manager;
 	unique_ptr<ExternalFileCache> external_file_cache;
 
 	duckdb_ext_api_v1 (*create_api_v1)();

package/src/duckdb/src/include/duckdb/main/database_file_path_manager.hpp

@@ -12,31 +12,35 @@
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/enums/on_create_conflict.hpp"
+#include "duckdb/common/enums/access_mode.hpp"
+#include "duckdb/common/reference_map.hpp"
 
 namespace duckdb {
 struct AttachInfo;
 struct AttachOptions;
+class DatabaseManager;
 
 enum class InsertDatabasePathResult { SUCCESS, ALREADY_EXISTS };
 
 struct DatabasePathInfo {
-	explicit DatabasePathInfo(string name_p) : name(std::move(name_p)), is_attached(true) {
-	}
+	DatabasePathInfo(DatabaseManager &manager, string name_p, AccessMode access_mode);
 
 	string name;
-	bool is_attached;
+	AccessMode access_mode;
+	reference_set_t<DatabaseManager> attached_databases;
+	idx_t reference_count = 1;
 };
 
 //! The DatabaseFilePathManager is used to ensure we only ever open a single database file once
 class DatabaseFilePathManager {
 public:
 	idx_t ApproxDatabaseCount() const;
-	InsertDatabasePathResult InsertDatabasePath(const string &path, const string &name, OnCreateConflict on_conflict,
-	                                            AttachOptions &options);
+	InsertDatabasePathResult InsertDatabasePath(DatabaseManager &manager, const string &path, const string &name,
+	                                            OnCreateConflict on_conflict, AttachOptions &options);
 	//! Erase a database path - indicating we are done with using it
 	void EraseDatabasePath(const string &path);
 	//! Called when a database is detached, but before it is fully finished being used
-	void DetachDatabase(const string &path);
+	void DetachDatabase(DatabaseManager &manager, const string &path);
 
 private:
 	//! The lock to add entries to the database path map

package/src/duckdb/src/include/duckdb/main/extension_entries.hpp

@@ -227,6 +227,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"iceberg_metadata", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY},
     {"iceberg_scan", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY},
     {"iceberg_snapshots", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"iceberg_table_properties", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY},
     {"iceberg_to_ducklake", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY},
     {"icu_calendar_names", "icu", CatalogType::TABLE_FUNCTION_ENTRY},
     {"icu_collate_af", "icu", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -525,6 +526,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"regr_sxx", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"regr_sxy", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"regr_syy", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
+    {"remove_iceberg_table_properties", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY},
     {"repeat", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"replace", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"replace_type", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -540,6 +542,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"rtrim", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"sem", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"set_bit", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"set_iceberg_table_properties", "iceberg", CatalogType::TABLE_FUNCTION_ENTRY},
     {"setseed", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"shapefile_meta", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
     {"sign", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -599,6 +602,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"st_envelope", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_envelope_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"st_equals", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"st_expand", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_extent", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"st_extent_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"st_extent_approx", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},

package/src/duckdb/src/include/duckdb/main/profiling_info.hpp

@@ -56,6 +56,7 @@ public:
 
 public:
 	string GetMetricAsString(const MetricsType metric) const;
+	void WriteMetricsToLog(ClientContext &context);
 	void WriteMetricsToJSON(duckdb_yyjson::yyjson_mut_doc *doc, duckdb_yyjson::yyjson_mut_val *destination);
 
 public:

package/src/duckdb/src/include/duckdb/main/query_profiler.hpp

@@ -183,6 +183,7 @@ public:
 	static InsertionOrderPreservingMap<string> JSONSanitize(const InsertionOrderPreservingMap<string> &input);
 	static string JSONSanitize(const string &text);
 	static string DrawPadded(const string &str, idx_t width);
+	DUCKDB_API void ToLog() const;
 	DUCKDB_API string ToJSON() const;
 	DUCKDB_API void WriteToFile(const char *path, string &info) const;
 

package/src/duckdb/src/include/duckdb/main/relation/create_table_relation.hpp

@@ -16,8 +16,11 @@ class CreateTableRelation : public Relation {
 public:
 	CreateTableRelation(shared_ptr<Relation> child, string schema_name, string table_name, bool temporary,
 	                    OnCreateConflict on_conflict);
+	CreateTableRelation(shared_ptr<Relation> child, string catalog_name, string schema_name, string table_name,
+	                    bool temporary, OnCreateConflict on_conflict);
 
 	shared_ptr<Relation> child;
+	string catalog_name;
 	string schema_name;
 	string table_name;
 	vector<ColumnDefinition> columns;

package/src/duckdb/src/include/duckdb/main/relation/insert_relation.hpp

@@ -15,8 +15,10 @@ namespace duckdb {
 class InsertRelation : public Relation {
 public:
 	InsertRelation(shared_ptr<Relation> child, string schema_name, string table_name);
+	InsertRelation(shared_ptr<Relation> child, string catalog_name, string schema_name, string table_name);
 
 	shared_ptr<Relation> child;
+	string catalog_name;
 	string schema_name;
 	string table_name;
 	vector<ColumnDefinition> columns;

package/src/duckdb/src/include/duckdb/main/relation/table_relation.hpp

@@ -29,6 +29,8 @@ public:
 
 	unique_ptr<TableRef> GetTableRef() override;
 
+	void Insert(const vector<vector<Value>> &values) override;
+	void Insert(vector<vector<unique_ptr<ParsedExpression>>> &&expressions) override;
 	void Update(const string &update, const string &condition = string()) override;
 	void Update(vector<string> column_names, vector<unique_ptr<ParsedExpression>> &&update,
 	            unique_ptr<ParsedExpression> condition = nullptr) override;

package/src/duckdb/src/include/duckdb/main/relation.hpp

@@ -162,19 +162,27 @@ public:
 
 	//! Insert the data from this relation into a table
 	DUCKDB_API shared_ptr<Relation> InsertRel(const string &schema_name, const string &table_name);
+	DUCKDB_API shared_ptr<Relation> InsertRel(const string &catalog_name, const string &schema_name,
+	                                          const string &table_name);
 	DUCKDB_API void Insert(const string &table_name);
 	DUCKDB_API void Insert(const string &schema_name, const string &table_name);
+	DUCKDB_API void Insert(const string &catalog_name, const string &schema_name, const string &table_name);
 	//! Insert a row (i.e.,list of values) into a table
-	DUCKDB_API void Insert(const vector<vector<Value>> &values);
-	DUCKDB_API void Insert(vector<vector<unique_ptr<ParsedExpression>>> &&expressions);
+	DUCKDB_API virtual void Insert(const vector<vector<Value>> &values);
+	DUCKDB_API virtual void Insert(vector<vector<unique_ptr<ParsedExpression>>> &&expressions);
 	//! Create a table and insert the data from this relation into that table
 	DUCKDB_API shared_ptr<Relation> CreateRel(const string &schema_name, const string &table_name,
 	                                          bool temporary = false,
 	                                          OnCreateConflict on_conflict = OnCreateConflict::ERROR_ON_CONFLICT);
+	DUCKDB_API shared_ptr<Relation> CreateRel(const string &catalog_name, const string &schema_name,
+	                                          const string &table_name, bool temporary = false,
+	                                          OnCreateConflict on_conflict = OnCreateConflict::ERROR_ON_CONFLICT);
 	DUCKDB_API void Create(const string &table_name, bool temporary = false,
 	                       OnCreateConflict on_conflict = OnCreateConflict::ERROR_ON_CONFLICT);
 	DUCKDB_API void Create(const string &schema_name, const string &table_name, bool temporary = false,
 	                       OnCreateConflict on_conflict = OnCreateConflict::ERROR_ON_CONFLICT);
+	DUCKDB_API void Create(const string &catalog_name, const string &schema_name, const string &table_name,
+	                       bool temporary = false, OnCreateConflict on_conflict = OnCreateConflict::ERROR_ON_CONFLICT);
 
 	//! Write a relation to a CSV file
 	DUCKDB_API shared_ptr<Relation>

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -337,6 +337,15 @@ struct DebugSkipCheckpointOnCommitSetting {
 	static constexpr SetScope DefaultScope = SetScope::GLOBAL;
 };
 
+struct DebugVerifyBlocksSetting {
+	using RETURN_TYPE = bool;
+	static constexpr const char *Name = "debug_verify_blocks";
+	static constexpr const char *Description = "DEBUG SETTING: verify block metadata during checkpointing";
+	static constexpr const char *InputType = "BOOLEAN";
+	static constexpr const char *DefaultValue = "false";
+	static constexpr SetScope DefaultScope = SetScope::GLOBAL;
+};
+
 struct DebugVerifyVectorSetting {
 	using RETURN_TYPE = DebugVectorVerification;
 	static constexpr const char *Name = "debug_verify_vector";

package/src/duckdb/src/include/duckdb/optimizer/filter_pullup.hpp

@@ -30,7 +30,7 @@ private:
 	// only pull up filters when there is a fork
 	bool can_pullup = false;
 
-	// identifiy case the branch is a set operation (INTERSECT or EXCEPT)
+	// identify case the branch is a set operation (INTERSECT or EXCEPT)
 	bool can_add_column = false;
 
 private:
@@ -40,30 +40,26 @@ private:
 
 	//! Pull up a LogicalFilter op
 	unique_ptr<LogicalOperator> PullupFilter(unique_ptr<LogicalOperator> op);
-
 	//! Pull up filter in a LogicalProjection op
 	unique_ptr<LogicalOperator> PullupProjection(unique_ptr<LogicalOperator> op);
-
 	//! Pull up filter in a LogicalCrossProduct op
 	unique_ptr<LogicalOperator> PullupCrossProduct(unique_ptr<LogicalOperator> op);
-
+	//! Pullup a filter in a LogicalJoin
 	unique_ptr<LogicalOperator> PullupJoin(unique_ptr<LogicalOperator> op);
-
-	// PPullup filter in a left join
+	//! Pullup filter in a left join
 	unique_ptr<LogicalOperator> PullupFromLeft(unique_ptr<LogicalOperator> op);
-
-	// Pullup filter in a inner join
+	//! Pullup filter in an inner join
 	unique_ptr<LogicalOperator> PullupInnerJoin(unique_ptr<LogicalOperator> op);
-
-	// Pullup filter in LogicalIntersect or LogicalExcept op
+	//! Pullup filter through a distinct
+	unique_ptr<LogicalOperator> PullupDistinct(unique_ptr<LogicalOperator> op);
+	//! Pullup filter in LogicalIntersect or LogicalExcept op
 	unique_ptr<LogicalOperator> PullupSetOperation(unique_ptr<LogicalOperator> op);
-
+	//! Pullup filter in both sides of a join
 	unique_ptr<LogicalOperator> PullupBothSide(unique_ptr<LogicalOperator> op);
 
-	// Finish pull up at this operator
+	//! Finish pull up at this operator
 	unique_ptr<LogicalOperator> FinishPullup(unique_ptr<LogicalOperator> op);
-
-	// special treatment for SetOperations and projections
+	//! special treatment for SetOperations and projections
 	void ProjectSetOperation(LogicalProjection &proj);
 
 }; // end FilterPullup

package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp

@@ -56,7 +56,11 @@ public:
 	//! Extract the set of relations referred to inside an expression
 	bool ExtractBindings(Expression &expression, unordered_set<idx_t> &bindings);
 	void AddRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent, const RelationStats &stats);
-
+	//! Add an unnest relation which can come from a logical unnest or a logical get which has an unnest function
+	void AddUnnestRelation(JoinOrderOptimizer &optimizer, LogicalOperator &op, LogicalOperator &input_op,
+	                       optional_ptr<LogicalOperator> parent, RelationStats &child_stats,
+	                       optional_ptr<LogicalOperator> limit_op,
+	                       vector<reference<LogicalOperator>> &datasource_filters);
 	void AddAggregateOrWindowRelation(LogicalOperator &op, optional_ptr<LogicalOperator> parent,
 	                                  const RelationStats &stats, LogicalOperatorType op_type);
 	vector<unique_ptr<SingleJoinRelation>> GetRelations();

package/src/duckdb/src/include/duckdb/parser/query_node.hpp

@@ -78,6 +78,9 @@ public:
 	virtual void Serialize(Serializer &serializer) const;
 	static unique_ptr<QueryNode> Deserialize(Deserializer &deserializer);
 
+	//! TEMPORARY BUG FIX WORKAROUND: extract elements from the CommonTableExpressionMap and construct CTENodes
+	static void ExtractCTENodes(unique_ptr<QueryNode> &query_node);
+
 protected:
 	//! Copy base QueryNode properties from another expression to this one,
 	//! used in Copy method

package/src/duckdb/src/include/duckdb/planner/bound_statement.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/string.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 #include "duckdb/common/vector.hpp"
 
 namespace duckdb {

package/src/duckdb/src/include/duckdb/storage/block.hpp

@@ -61,6 +61,15 @@ struct MetaBlockPointer {
 	block_id_t GetBlockId() const;
 	uint32_t GetBlockIndex() const;
 
+	bool operator==(const MetaBlockPointer &rhs) const {
+		return block_pointer == rhs.block_pointer && offset == rhs.offset;
+	}
+
+	friend std::ostream &operator<<(std::ostream &os, const MetaBlockPointer &obj) {
+		return os << "{block_id: " << obj.GetBlockId() << " index: " << obj.GetBlockIndex() << " offset: " << obj.offset
+		          << "}";
+	}
+
 	void Serialize(Serializer &serializer) const;
 	static MetaBlockPointer Deserialize(Deserializer &source);
 };

package/src/duckdb/src/include/duckdb/storage/block_manager.hpp

@@ -24,6 +24,8 @@ class ClientContext;
 class DatabaseInstance;
 class MetadataManager;
 
+enum class ConvertToPersistentMode { DESTRUCTIVE, THREAD_SAFE };
+
 //! BlockManager is an abstract representation to manage blocks on DuckDB. When writing or reading blocks, the
 //! BlockManager creates and accesses blocks. The concrete types implement specific block storage strategies.
 class BlockManager {
@@ -91,10 +93,15 @@ public:
 	//! Register a block with the given block id in the base file
 	shared_ptr<BlockHandle> RegisterBlock(block_id_t block_id);
 	//! Convert an existing in-memory buffer into a persistent disk-backed block
+	//! If mode is set to destructive (default) - the old_block will be destroyed as part of this method
+	//! This can only be safely used when there is no other (lingering) usage of old_block
+	//! If there is concurrent usage of the block elsewhere - use the THREAD_SAFE mode which creates an extra copy
 	shared_ptr<BlockHandle> ConvertToPersistent(QueryContext context, block_id_t block_id,
-	                                            shared_ptr<BlockHandle> old_block, BufferHandle old_handle);
+	                                            shared_ptr<BlockHandle> old_block, BufferHandle old_handle,
+	                                            ConvertToPersistentMode mode = ConvertToPersistentMode::DESTRUCTIVE);
 	shared_ptr<BlockHandle> ConvertToPersistent(QueryContext context, block_id_t block_id,
-	                                            shared_ptr<BlockHandle> old_block);
+	                                            shared_ptr<BlockHandle> old_block,
+	                                            ConvertToPersistentMode mode = ConvertToPersistentMode::DESTRUCTIVE);
 
 	void UnregisterBlock(BlockHandle &block);
 	//! UnregisterBlock, only accepts non-temporary block ids

package/src/duckdb/src/include/duckdb/storage/index.hpp

@@ -31,9 +31,15 @@ class Index {
 protected:
 	Index(const vector<column_t> &column_ids, TableIOManager &table_io_manager, AttachedDatabase &db);
 
-	//! The logical column ids of the indexed table
+	//! The physical column ids of the indexed columns.
+	//! For example, given a table with the following columns:
+	//! (a INT, gen AS (2 * a), b INT, c VARCHAR), an index on columns (a,c) would have physical
+	//! column_ids [0,2] (since the virtual column is skipped in the physical representation).
+	//! Also see comments in bound_index.hpp to see how these column IDs are used in the context of
+	//! bound/unbound expressions.
+	//! Note that these are the columns for this Index, not all Indexes on the table.
 	vector<column_t> column_ids;
-	//! Unordered set of column_ids used by the index
+	//! Unordered set of column_ids used by the Index
 	unordered_set<column_t> column_id_set;
 
 public:

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp

@@ -77,6 +77,8 @@ public:
 	//! Flush all blocks to disk
 	void Flush();
 
+	bool BlockHasBeenCleared(const MetaBlockPointer &ptr);
+
 	void MarkBlocksAsModified();
 	void ClearModifiedBlocks(const vector<MetaBlockPointer> &pointers);
 

package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp

@@ -52,7 +52,7 @@ private:
 	MetadataManager &manager;
 	BlockReaderType type;
 	MetadataHandle block;
-	MetadataPointer next_pointer;
+	MetaBlockPointer next_pointer;
 	bool has_next_block;
 	optional_ptr<vector<MetaBlockPointer>> read_pointers;
 	idx_t index;

package/src/duckdb/src/include/duckdb/storage/storage_options.hpp

@@ -40,11 +40,4 @@ struct StorageOptions {
 	void Initialize(const unordered_map<string, Value> &options);
 };
 
-inline void ClearUserKey(shared_ptr<string> const &encryption_key) {
-	if (encryption_key && !encryption_key->empty()) {
-		memset(&(*encryption_key)[0], 0, encryption_key->size());
-		encryption_key->clear();
-	}
-}
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -65,7 +65,8 @@ struct RowGroupWriteInfo {
 struct RowGroupWriteData {
 	vector<unique_ptr<ColumnCheckpointState>> states;
 	vector<BaseStatistics> statistics;
-	vector<MetaBlockPointer> existing_pointers;
+	bool reuse_existing_metadata_blocks = false;
+	vector<idx_t> existing_extra_metadata_blocks;
 };
 
 class RowGroup : public SegmentBase<RowGroup> {
@@ -94,7 +95,10 @@ public:
 		return collection.get();
 	}
 	//! Returns the list of meta block pointers used by the columns
-	vector<MetaBlockPointer> GetColumnPointers();
+	vector<idx_t> GetOrComputeExtraMetadataBlocks(bool force_compute = false);
+
+	const vector<MetaBlockPointer> &GetColumnStartPointers() const;
+
 	//! Returns the list of meta block pointers used by the deletes
 	const vector<MetaBlockPointer> &GetDeletesPointers() const {
 		return deletes_pointers;

package/src/duckdb/src/include/duckdb/verification/deserialized_statement_verifier.hpp

@@ -18,6 +18,12 @@ public:
 	                                       optional_ptr<case_insensitive_map_t<BoundParameterData>> parameters);
 	static unique_ptr<StatementVerifier> Create(const SQLStatement &statement,
 	                                            optional_ptr<case_insensitive_map_t<BoundParameterData>> parameters);
+
+public:
+	// TEMPORARY FIX: work-around for CTE serialization for v1.4.X
+	bool RequireEquality() const override {
+		return false;
+	}
 };
 
 } // namespace duckdb