duckdb 1.1.4-dev9.0 → 1.2.0

package/src/duckdb/src/include/duckdb/main/extension_entries.hpp

@@ -135,6 +135,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"covar_samp", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"create_fts_index", "fts", CatalogType::PRAGMA_FUNCTION_ENTRY},
     {"current_database", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"current_date", "icu", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"current_localtime", "icu", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"current_localtimestamp", "icu", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"current_query", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -179,8 +180,6 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"from_hex", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"from_json", "json", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"from_json_strict", "json", CatalogType::SCALAR_FUNCTION_ENTRY},
-    {"from_substrait", "substrait", CatalogType::TABLE_FUNCTION_ENTRY},
-    {"from_substrait_json", "substrait", CatalogType::TABLE_FUNCTION_ENTRY},
     {"fsum", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"fuzz_all_functions", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY},
     {"fuzzyduck", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY},
@@ -188,9 +187,8 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"gcd", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"gen_random_uuid", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"get_bit", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"get_current_time", "icu", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"get_current_timestamp", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
-    {"get_substrait", "substrait", CatalogType::TABLE_FUNCTION_ENTRY},
-    {"get_substrait_json", "substrait", CatalogType::TABLE_FUNCTION_ENTRY},
     {"grade_up", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"greatest", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"greatest_common_divisor", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -430,6 +428,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"map_concat", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"map_entries", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"map_extract", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"map_extract_value", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"map_from_entries", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"map_keys", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"map_values", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -523,6 +522,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"skewness", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
     {"sql_auto_complete", "autocomplete", CatalogType::TABLE_FUNCTION_ENTRY},
     {"sqlite_attach", "sqlite_scanner", CatalogType::TABLE_FUNCTION_ENTRY},
+    {"sqlite_query", "sqlite_scanner", CatalogType::TABLE_FUNCTION_ENTRY},
     {"sqlite_scan", "sqlite_scanner", CatalogType::TABLE_FUNCTION_ENTRY},
     {"sqlsmith", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY},
     {"sqrt", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -682,6 +682,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
     {"to_timestamp", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"to_weeks", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"to_years", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
+    {"today", "icu", CatalogType::SCALAR_FUNCTION_ENTRY},
     {"tpcds", "tpcds", CatalogType::PRAGMA_FUNCTION_ENTRY},
     {"tpcds_answers", "tpcds", CatalogType::TABLE_FUNCTION_ENTRY},
     {"tpcds_queries", "tpcds", CatalogType::TABLE_FUNCTION_ENTRY},
@@ -980,7 +981,9 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
     {"s3_url_style", "httpfs"},
     {"s3_use_ssl", "httpfs"},
     {"sqlite_all_varchar", "sqlite_scanner"},
+    {"sqlite_debug_show_queries", "sqlite_scanner"},
     {"timezone", "icu"},
+    {"unsafe_enable_version_guessing", "iceberg"},
 }; // END_OF_EXTENSION_SETTINGS
 
 // Note: these are currently hardcoded in scripts/generate_extensions_function.py
@@ -1068,8 +1071,9 @@ static constexpr ExtensionEntry EXTENSION_SECRET_PROVIDERS[] = {
     {"postgres/config", "postgres_scanner"}}; // EXTENSION_SECRET_PROVIDERS
 
 static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {
-    "aws",   "azure", "autocomplete", "core_functions", "delta",   "excel",          "fts",      "httpfs",
-    "inet",  "icu",   "json",         "mysql_scanner",  "parquet", "sqlite_scanner", "sqlsmith", "postgres_scanner",
-    "tpcds", "tpch"}; // END_OF_AUTOLOADABLE_EXTENSIONS
+    "aws",        "azure",         "autocomplete", "core_functions", "delta",    "excel",
+    "fts",        "httpfs",        "iceberg",      "inet",           "icu",      "json",
+    "motherduck", "mysql_scanner", "parquet",      "sqlite_scanner", "sqlsmith", "postgres_scanner",
+    "tpcds",      "tpch",          "uc_catalog"}; // END_OF_AUTOLOADABLE_EXTENSIONS
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -236,6 +236,7 @@ public:
 
 	static bool IsRelease(const string &version_tag);
 	static bool CreateSuggestions(const string &extension_name, string &message);
+	static string ExtensionInstallDocumentationLink(const string &extension_name);
 
 private:
 	static unique_ptr<ExtensionInstallInfo> InstallExtensionInternal(DatabaseInstance &db, FileSystem &fs,

package/src/duckdb/src/include/duckdb/main/secret/secret.hpp

@@ -78,6 +78,8 @@ struct SecretType {
 	secret_deserializer_t deserializer;
 	//! Provider to use when non is specified
 	string default_provider;
+	//! The extension that registered this secret type
+	string extension;
 };
 
 enum class SecretSerializationType : uint8_t {

package/src/duckdb/src/include/duckdb/main/secret/secret_manager.hpp

@@ -135,6 +135,9 @@ public:
 	//! List all secrets from all secret storages
 	DUCKDB_API vector<SecretEntry> AllSecrets(CatalogTransaction transaction);
 
+	//! List all secret types
+	DUCKDB_API vector<SecretType> AllSecretTypes();
+
 	//! Secret Manager settings
 	DUCKDB_API virtual void SetEnablePersistentSecrets(bool enabled);
 	DUCKDB_API virtual void ResetEnablePersistentSecrets();

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -419,6 +419,16 @@ struct DefaultSecretStorageSetting {
 	static Value GetSetting(const ClientContext &context);
 };
 
+struct DisabledCompressionMethodsSetting {
+	using RETURN_TYPE = string;
+	static constexpr const char *Name = "disabled_compression_methods";
+	static constexpr const char *Description = "Disable a specific set of compression methods (comma separated)";
+	static constexpr const char *InputType = "VARCHAR";
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(const ClientContext &context);
+};
+
 struct DisabledFilesystemsSetting {
 	using RETURN_TYPE = string;
 	static constexpr const char *Name = "disabled_filesystems";

package/src/duckdb/src/include/duckdb/parser/constraint.hpp

@@ -44,6 +44,15 @@ struct ForeignKeyInfo {
 	vector<PhysicalIndex> pk_keys;
 	//! The set of foreign key table's column's index
 	vector<PhysicalIndex> fk_keys;
+
+	bool IsDeleteConstraint() const {
+		return type == ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE ||
+		       type == ForeignKeyType::FK_TYPE_SELF_REFERENCE_TABLE;
+	}
+	bool IsAppendConstraint() const {
+		return type == ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE ||
+		       type == ForeignKeyType::FK_TYPE_SELF_REFERENCE_TABLE;
+	}
 };
 
 //! Constraint is the base class of any type of table constraint.

package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp

@@ -22,7 +22,10 @@ enum class WindowBoundary : uint8_t {
 	EXPR_PRECEDING_ROWS = 5,
 	EXPR_FOLLOWING_ROWS = 6,
 	EXPR_PRECEDING_RANGE = 7,
-	EXPR_FOLLOWING_RANGE = 8
+	EXPR_FOLLOWING_RANGE = 8,
+	CURRENT_ROW_GROUPS = 9,
+	EXPR_PRECEDING_GROUPS = 10,
+	EXPR_FOLLOWING_GROUPS = 11
 };
 
 //! Represents the window exclusion mode
@@ -92,6 +95,17 @@ public:
 	static ExpressionType WindowToExpressionType(string &fun_name);
 
 public:
+	static inline string ToUnits(const WindowBoundary boundary, const WindowBoundary rows, const WindowBoundary range,
+	                             const WindowBoundary groups) {
+		if (boundary == rows) {
+			return "ROWS";
+		} else if (boundary == range) {
+			return "RANGE";
+		} else {
+			return "GROUPS";
+		}
+	}
+
 	template <class T, class BASE, class ORDER_NODE>
 	static string ToString(const T &entry, const string &schema, const string &function_name) {
 		// Start with function call
@@ -156,8 +170,10 @@ public:
 		switch (entry.start) {
 		case WindowBoundary::CURRENT_ROW_RANGE:
 		case WindowBoundary::CURRENT_ROW_ROWS:
+		case WindowBoundary::CURRENT_ROW_GROUPS:
 			from = "CURRENT ROW";
-			units = (entry.start == WindowBoundary::CURRENT_ROW_RANGE) ? "RANGE" : "ROWS";
+			units = ToUnits(entry.start, WindowBoundary::CURRENT_ROW_ROWS, WindowBoundary::CURRENT_ROW_RANGE,
+			                WindowBoundary::CURRENT_ROW_GROUPS);
 			break;
 		case WindowBoundary::UNBOUNDED_PRECEDING:
 			if (entry.end != WindowBoundary::CURRENT_ROW_RANGE) {
@@ -166,15 +182,20 @@ public:
 			break;
 		case WindowBoundary::EXPR_PRECEDING_ROWS:
 		case WindowBoundary::EXPR_PRECEDING_RANGE:
+		case WindowBoundary::EXPR_PRECEDING_GROUPS:
 			from = entry.start_expr->ToString() + " PRECEDING";
-			units = (entry.start == WindowBoundary::EXPR_PRECEDING_RANGE) ? "RANGE" : "ROWS";
+			units = ToUnits(entry.start, WindowBoundary::EXPR_PRECEDING_ROWS, WindowBoundary::EXPR_PRECEDING_RANGE,
+			                WindowBoundary::EXPR_PRECEDING_GROUPS);
 			break;
 		case WindowBoundary::EXPR_FOLLOWING_ROWS:
 		case WindowBoundary::EXPR_FOLLOWING_RANGE:
+		case WindowBoundary::EXPR_FOLLOWING_GROUPS:
 			from = entry.start_expr->ToString() + " FOLLOWING";
-			units = (entry.start == WindowBoundary::EXPR_FOLLOWING_RANGE) ? "RANGE" : "ROWS";
+			units = ToUnits(entry.start, WindowBoundary::EXPR_FOLLOWING_ROWS, WindowBoundary::EXPR_FOLLOWING_RANGE,
+			                WindowBoundary::EXPR_FOLLOWING_GROUPS);
 			break;
-		default:
+		case WindowBoundary::UNBOUNDED_FOLLOWING:
+		case WindowBoundary::INVALID:
 			throw InternalException("Unrecognized FROM in WindowExpression");
 		}
 
@@ -187,8 +208,10 @@ public:
 			}
 			break;
 		case WindowBoundary::CURRENT_ROW_ROWS:
+		case WindowBoundary::CURRENT_ROW_GROUPS:
 			to = "CURRENT ROW";
-			units = "ROWS";
+			units = ToUnits(entry.end, WindowBoundary::CURRENT_ROW_ROWS, WindowBoundary::CURRENT_ROW_RANGE,
+			                WindowBoundary::CURRENT_ROW_GROUPS);
 			break;
 		case WindowBoundary::UNBOUNDED_PRECEDING:
 			to = "UNBOUNDED PRECEDING";
@@ -198,15 +221,19 @@ public:
 			break;
 		case WindowBoundary::EXPR_PRECEDING_ROWS:
 		case WindowBoundary::EXPR_PRECEDING_RANGE:
+		case WindowBoundary::EXPR_PRECEDING_GROUPS:
 			to = entry.end_expr->ToString() + " PRECEDING";
-			units = (entry.end == WindowBoundary::EXPR_PRECEDING_RANGE) ? "RANGE" : "ROWS";
+			units = ToUnits(entry.end, WindowBoundary::EXPR_PRECEDING_ROWS, WindowBoundary::EXPR_PRECEDING_RANGE,
+			                WindowBoundary::EXPR_PRECEDING_GROUPS);
 			break;
 		case WindowBoundary::EXPR_FOLLOWING_ROWS:
 		case WindowBoundary::EXPR_FOLLOWING_RANGE:
+		case WindowBoundary::EXPR_FOLLOWING_GROUPS:
 			to = entry.end_expr->ToString() + " FOLLOWING";
-			units = (entry.end == WindowBoundary::EXPR_FOLLOWING_RANGE) ? "RANGE" : "ROWS";
+			units = ToUnits(entry.end, WindowBoundary::EXPR_FOLLOWING_ROWS, WindowBoundary::EXPR_FOLLOWING_RANGE,
+			                WindowBoundary::EXPR_FOLLOWING_GROUPS);
 			break;
-		default:
+		case WindowBoundary::INVALID:
 			throw InternalException("Unrecognized TO in WindowExpression");
 		}
 		if (entry.exclude_clause != WindowExcludeMode::NO_OTHER) {

package/src/duckdb/src/include/duckdb/parser/parsed_data/create_view_info.hpp

@@ -40,7 +40,8 @@ public:
 	//! Gets a bound CreateViewInfo object from a SELECT statement and a view name, schema name, etc
 	DUCKDB_API static unique_ptr<CreateViewInfo> FromSelect(ClientContext &context, unique_ptr<CreateViewInfo> info);
 	//! Gets a bound CreateViewInfo object from a CREATE VIEW statement
-	DUCKDB_API static unique_ptr<CreateViewInfo> FromCreateView(ClientContext &context, const string &sql);
+	DUCKDB_API static unique_ptr<CreateViewInfo> FromCreateView(ClientContext &context, SchemaCatalogEntry &schema,
+	                                                            const string &sql);
 	//! Parse a SELECT statement from a SQL string
 	DUCKDB_API static unique_ptr<SelectStatement> ParseSelect(const string &sql);
 

package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp

@@ -20,8 +20,7 @@ public:
 	static constexpr const QueryNodeType TYPE = QueryNodeType::SET_OPERATION_NODE;
 
 public:
-	SetOperationNode() : QueryNode(QueryNodeType::SET_OPERATION_NODE) {
-	}
+	SetOperationNode();
 
 	//! The type of set operation
 	SetOperationType setop_type = SetOperationType::NONE;
@@ -49,6 +48,13 @@ public:
 
 	void Serialize(Serializer &serializer) const override;
 	static unique_ptr<QueryNode> Deserialize(Deserializer &source);
+
+public:
+	// these methods exist for forwards/backwards compatibility of (de)serialization
+	SetOperationNode(SetOperationType setop_type, unique_ptr<QueryNode> left, unique_ptr<QueryNode> right,
+	                 vector<unique_ptr<QueryNode>> children, bool setop_all);
+
+	vector<unique_ptr<QueryNode>> SerializeChildNodes() const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/binder.hpp

@@ -419,6 +419,10 @@ private:
 	const string BindCatalog(string &catalog_name);
 	SchemaCatalogEntry &BindCreateSchema(CreateInfo &info);
 
+	vector<CatalogSearchEntry> GetSearchPath(Catalog &catalog, const string &schema_name);
+
+	LogicalType BindLogicalTypeInternal(const LogicalType &type, optional_ptr<Catalog> catalog, const string &schema);
+
 	unique_ptr<BoundQueryNode> BindSelectNode(SelectNode &statement, unique_ptr<BoundTableRef> from_table);
 
 	unique_ptr<LogicalOperator> BindCopyDatabaseSchema(Catalog &source_catalog, const string &target_database_name);

package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp

@@ -17,7 +17,7 @@ struct BoundParameterData {
 public:
 	BoundParameterData() {
 	}
-	explicit BoundParameterData(Value val) : value(std::move(val)), return_type(value.type()) {
+	explicit BoundParameterData(Value val) : value(std::move(val)), return_type(GetDefaultType(value.type())) {
 	}
 	BoundParameterData(Value val, LogicalType type_p) : value(std::move(val)), return_type(std::move(type_p)) {
 	}
@@ -39,6 +39,14 @@ public:
 
 	void Serialize(Serializer &serializer) const;
 	static shared_ptr<BoundParameterData> Deserialize(Deserializer &deserializer);
+
+private:
+	LogicalType GetDefaultType(const LogicalType &type) {
+		if (value.type().id() == LogicalTypeId::VARCHAR && StringType::GetCollation(type).empty()) {
+			return LogicalTypeId::STRING_LITERAL;
+		}
+		return value.type();
+	}
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp

@@ -27,6 +27,7 @@ public:
 	Value constant;
 
 public:
+	bool Compare(const Value &value) const;
 	FilterPropagateResult CheckStatistics(BaseStatistics &stats) override;
 	string ToString(const string &column_name) override;
 	bool Equals(const TableFilter &other) const override;

package/src/duckdb/src/include/duckdb/planner/filter/in_filter.hpp

@@ -19,10 +19,8 @@ public:
 
 public:
 	explicit InFilter(vector<Value> values);
-	InFilter(vector<Value> values, bool origin_is_hash_join);
 
 	vector<Value> values;
-	bool origin_is_hash_join;
 
 public:
 	FilterPropagateResult CheckStatistics(BaseStatistics &stats) override;

package/src/duckdb/src/include/duckdb/planner/filter/optional_filter.hpp

@@ -20,16 +20,16 @@ public:
 public:
 	explicit OptionalFilter(unique_ptr<TableFilter> filter = nullptr);
 
+	//! Optional child filters.
+	unique_ptr<TableFilter> child_filter;
+
+public:
 	string ToString(const string &column_name) override;
 	unique_ptr<TableFilter> Copy() const override;
 	unique_ptr<Expression> ToExpression(const Expression &column) const override;
 	FilterPropagateResult CheckStatistics(BaseStatistics &stats) override;
 	void Serialize(Serializer &serializer) const override;
 	static unique_ptr<TableFilter> Deserialize(Deserializer &deserializer);
-
-public:
-	// optional child filters
-	unique_ptr<TableFilter> child_filter;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/planner/table_filter.hpp

@@ -52,7 +52,7 @@ public:
 	string DebugToString();
 	virtual unique_ptr<TableFilter> Copy() const = 0;
 	virtual bool Equals(const TableFilter &other) const {
-		return filter_type != other.filter_type;
+		return filter_type == other.filter_type;
 	}
 	virtual unique_ptr<Expression> ToExpression(const Expression &column) const = 0;
 

package/src/duckdb/src/include/duckdb/storage/data_table.hpp

@@ -107,8 +107,8 @@ public:
 	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints, Vector &row_ids,
 	                 DataChunk &delete_chunk);
 	//! Append a chunk to the transaction-local storage of this table.
-	void LocalAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
-	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints);
+	void LocalWALAppend(TableCatalogEntry &table, ClientContext &context, DataChunk &chunk,
+	                    const vector<unique_ptr<BoundConstraint>> &bound_constraints);
 	//! Append a column data collection with default values to the transaction-local storage of this table.
 	void LocalAppend(TableCatalogEntry &table, ClientContext &context, ColumnDataCollection &collection,
 	                 const vector<unique_ptr<BoundConstraint>> &bound_constraints,
@@ -167,9 +167,10 @@ public:
 
 	//! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table.
 	//! Returns empty ErrorData, if the append was successful.
-	ErrorData AppendToIndexes(optional_ptr<TableIndexList> delete_indexes, DataChunk &chunk, row_t row_start);
+	ErrorData AppendToIndexes(optional_ptr<TableIndexList> delete_indexes, DataChunk &chunk, row_t row_start,
+	                          const IndexAppendMode index_append_mode);
 	static ErrorData AppendToIndexes(TableIndexList &indexes, optional_ptr<TableIndexList> delete_indexes,
-	                                 DataChunk &chunk, row_t row_start);
+	                                 DataChunk &chunk, row_t row_start, const IndexAppendMode index_append_mode);
 	//! Remove a chunk with the row ids [row_start, ..., row_start + chunk.size()] from all indexes of the table
 	void RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, row_t row_start);
 	//! Remove the chunk with the specified set of row identifiers from all indexes of the table
@@ -206,7 +207,6 @@ public:
 	idx_t GetTotalRows() const;
 
 	vector<ColumnSegmentInfo> GetColumnSegmentInfo();
-	static bool IsForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, Index &index, ForeignKeyType fk_type);
 
 	//! Scans the next chunk for the CREATE INDEX operator
 	bool CreateIndexScan(TableScanState &state, DataChunk &result, TableScanType type);
@@ -259,16 +259,20 @@ private:
 	void VerifyUpdateConstraints(ConstraintState &state, ClientContext &context, DataChunk &chunk,
 	                             const vector<PhysicalIndex> &column_ids);
 	//! Verify constraints with a chunk from the Delete containing all columns of the table
-	void VerifyDeleteConstraints(TableDeleteState &state, ClientContext &context, DataChunk &chunk);
+	void VerifyDeleteConstraints(optional_ptr<LocalTableStorage> storage, TableDeleteState &state,
+	                             ClientContext &context, DataChunk &chunk);
 
 	void InitializeScanWithOffset(DuckTransaction &transaction, TableScanState &state,
 	                              const vector<StorageIndex> &column_ids, idx_t start_row, idx_t end_row);
 
-	void VerifyForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, ClientContext &context, DataChunk &chunk,
-	                                VerifyExistenceType verify_type);
-	void VerifyAppendForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, ClientContext &context,
+	void VerifyForeignKeyConstraint(optional_ptr<LocalTableStorage> storage,
+	                                const BoundForeignKeyConstraint &bound_foreign_key, ClientContext &context,
+	                                DataChunk &chunk, VerifyExistenceType type);
+	void VerifyAppendForeignKeyConstraint(optional_ptr<LocalTableStorage> storage,
+	                                      const BoundForeignKeyConstraint &bound_foreign_key, ClientContext &context,
 	                                      DataChunk &chunk);
-	void VerifyDeleteForeignKeyConstraint(const BoundForeignKeyConstraint &bfk, ClientContext &context,
+	void VerifyDeleteForeignKeyConstraint(optional_ptr<LocalTableStorage> storage,
+	                                      const BoundForeignKeyConstraint &bound_foreign_key, ClientContext &context,
 	                                      DataChunk &chunk);
 
 private:

package/src/duckdb/src/include/duckdb/storage/index_storage_info.hpp

@@ -62,6 +62,10 @@ struct IndexStorageInfo {
 	BlockPointer root_block_ptr;
 
 	//! Returns true, if IndexStorageInfo holds information to deserialize an index.
+	//! Note that the name can be misleading - any index that is empty (no nodes, etc.) might
+	//! also have neither a root_block_ptr nor allocator_infos.
+	//! Ensure that your index constructor initializes an empty index correctly without the
+	//! need for these fields.
 	bool IsValid() const {
 		return root_block_ptr.IsValid() || !allocator_infos.empty();
 	}

package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp

@@ -26,7 +26,9 @@ struct StorageManagerOptions {
 	bool read_only = false;
 	bool use_direct_io = false;
 	DebugInitialize debug_initialize = DebugInitialize::NO_INITIALIZE;
-	optional_idx block_alloc_size = optional_idx();
+	optional_idx block_alloc_size;
+	optional_idx storage_version;
+	optional_idx version_number;
 };
 
 //! SingleFileBlockManager is an implementation for a BlockManager which manages blocks in a single file
@@ -107,6 +109,9 @@ private:
 	//! Verify the block usage count
 	void VerifyBlocks(const unordered_map<block_id_t, idx_t> &block_usage_count) override;
 
+	void AddStorageVersionTag();
+	uint64_t GetVersionNumber();
+
 private:
 	AttachedDatabase &db;
 	//! The active DatabaseHeader, either 0 (h1) or 1 (h2)

package/src/duckdb/src/include/duckdb/storage/storage_info.hpp

@@ -54,10 +54,13 @@ struct Storage {
 	static void VerifyBlockAllocSize(const idx_t block_alloc_size);
 };
 
-//! The version number of the database storage format
+//! The version number default, lower and upper bounds of the database storage format
 extern const uint64_t VERSION_NUMBER;
+extern const uint64_t VERSION_NUMBER_LOWER;
+extern const uint64_t VERSION_NUMBER_UPPER;
 string GetDuckDBVersion(idx_t version_number);
 optional_idx GetStorageVersion(const char *version_string);
+string GetStorageVersionName(idx_t serialization_version);
 optional_idx GetSerializationVersion(const char *version_string);
 vector<string> GetSerializationCandidates();
 
@@ -109,9 +112,11 @@ struct DatabaseHeader {
 	idx_t block_alloc_size;
 	//! The vector size of the database file
 	idx_t vector_size;
+	//! The serialization compatibility version
+	idx_t serialization_compatibility;
 
 	void Write(WriteStream &ser);
-	static DatabaseHeader Read(ReadStream &source);
+	static DatabaseHeader Read(const MainHeader &header, ReadStream &source);
 };
 
 //! Detect mismatching constant values when compiling

package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp

@@ -106,6 +106,13 @@ public:
 	virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
 	virtual BlockManager &GetBlockManager() = 0;
 
+	void SetStorageVersion(idx_t version) {
+		storage_version = version;
+	}
+	idx_t GetStorageVersion() const {
+		return storage_version.GetIndex();
+	}
+
 protected:
 	virtual void LoadDatabase(StorageOptions options) = 0;
 
@@ -121,6 +128,8 @@ protected:
 	//! When loading a database, we do not yet set the wal-field. Therefore, GetWriteAheadLog must
 	//! return nullptr when loading a database
 	bool load_complete = false;
+	//! The serialization compatibility version when reading and writing from this database
+	optional_idx storage_version;
 
 public:
 	template <class TARGET>

package/src/duckdb/src/include/duckdb/storage/storage_options.hpp

@@ -18,6 +18,8 @@ struct StorageOptions {
 	optional_idx block_alloc_size;
 	//! The row group size for this attached database (if any)
 	optional_idx row_group_size;
+	//! Target storage version (if any)
+	optional_idx storage_version;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp

@@ -207,6 +207,7 @@ public:
 
 	static void SetDictionary(ColumnSegment &segment, BufferHandle &handle, StringDictionaryContainer dict);
 	static StringDictionaryContainer GetDictionary(ColumnSegment &segment, BufferHandle &handle);
+	static uint32_t GetDictionaryEnd(ColumnSegment &segment, BufferHandle &handle);
 	static idx_t RemainingSpace(ColumnSegment &segment, BufferHandle &handle);
 	static void WriteString(ColumnSegment &segment, string_t string, block_id_t &result_block, int32_t &result_offset);
 	static void WriteStringMemory(ColumnSegment &segment, string_t string, block_id_t &result_block,
@@ -217,12 +218,12 @@ public:
 	static void WriteStringMarker(data_ptr_t target, block_id_t block_id, int32_t offset);
 	static void ReadStringMarker(data_ptr_t target, block_id_t &block_id, int32_t &offset);
 
-	inline static string_t FetchStringFromDict(ColumnSegment &segment, StringDictionaryContainer dict, Vector &result,
+	inline static string_t FetchStringFromDict(ColumnSegment &segment, uint32_t dict_end_offset, Vector &result,
 	                                           data_ptr_t base_ptr, int32_t dict_offset, uint32_t string_length) {
 		D_ASSERT(dict_offset <= NumericCast<int32_t>(segment.GetBlockManager().GetBlockSize()));
 		if (DUCKDB_LIKELY(dict_offset >= 0)) {
 			// regular string - fetch from dictionary
-			auto dict_end = base_ptr + dict.end;
+			auto dict_end = base_ptr + dict_end_offset;
 			auto dict_pos = dict_end - dict_offset;
 
 			auto str_ptr = char_ptr_cast(dict_pos);
@@ -231,7 +232,7 @@ public:
 			// read overflow string
 			block_id_t block_id;
 			int32_t offset;
-			ReadStringMarker(base_ptr + dict.end - AbsValue<int32_t>(dict_offset), block_id, offset);
+			ReadStringMarker(base_ptr + dict_end_offset - AbsValue<int32_t>(dict_offset), block_id, offset);
 
 			return ReadOverflowString(segment, result, block_id, offset);
 		}

package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp

@@ -26,6 +26,7 @@ class ColumnSegment;
 class DatabaseInstance;
 class RowGroup;
 class RowGroupWriter;
+class StorageManager;
 class TableDataWriter;
 class TableStorageInfo;
 struct DataTableInfo;
@@ -77,6 +78,7 @@ public:
 	}
 	DatabaseInstance &GetDatabase() const;
 	DataTableInfo &GetTableInfo() const;
+	StorageManager &GetStorageManager() const;
 	virtual idx_t GetMaxEntry();
 
 	idx_t GetAllocationSize() const {

package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp

@@ -16,7 +16,7 @@
 namespace duckdb {
 
 class ConflictManager;
-
+class LocalTableStorage;
 struct IndexStorageInfo;
 struct DataTableInfo;
 
@@ -78,17 +78,19 @@ public:
 	//! Overwrite this list with the other list.
 	void Move(TableIndexList &other);
 	//! Find the foreign key matching the keys.
-	Index *FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, const ForeignKeyType fk_type);
+	optional_ptr<Index> FindForeignKeyIndex(const vector<PhysicalIndex> &fk_keys, const ForeignKeyType fk_type);
 	//! Verify a foreign key constraint.
-	void VerifyForeignKey(const vector<PhysicalIndex> &fk_keys, DataChunk &chunk, ConflictManager &conflict_manager);
+	void VerifyForeignKey(optional_ptr<LocalTableStorage> storage, const vector<PhysicalIndex> &fk_keys,
+	                      DataChunk &chunk, ConflictManager &conflict_manager);
 	//! Get the combined column ids of the indexes in this list.
 	unordered_set<column_t> GetRequiredColumns();
 	//! Serialize all indexes of this table.
 	vector<IndexStorageInfo> GetStorageInfos(const case_insensitive_map_t<Value> &options);
 
 private:
+	//! A lock to prevent any concurrent changes to the indexes.
 	mutex indexes_lock;
-	// Indexes associated with the table.
+	//! Indexes associated with the table.
 	vector<unique_ptr<Index>> indexes;
 };
 

package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp

@@ -49,7 +49,7 @@ public:
 	//! The reference can only be safely accessed while the lock is held
 	ColumnStatistics &GetStats(TableStatisticsLock &lock, idx_t i);
 	//! Get a reference to the table sample - this requires us to hold the lock.
-	BlockingSample &GetTableSampleRef(TableStatisticsLock &lock);
+	// BlockingSample &GetTableSampleRef(TableStatisticsLock &lock);
 	//! Take ownership of the sample, needed for merging. Requires the lock
 	unique_ptr<BlockingSample> GetTableSample(TableStatisticsLock &lock);
 	void SetTableSample(TableStatisticsLock &lock, unique_ptr<BlockingSample> sample);

package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp

@@ -55,6 +55,8 @@ public:
 	//! Replay and initialize the WAL
 	static unique_ptr<WriteAheadLog> Replay(FileSystem &fs, AttachedDatabase &database, const string &wal_path);
 
+	AttachedDatabase &GetDatabase();
+
 	//! Gets the total bytes written to the WAL since startup
 	idx_t GetWALSize() const;
 	//! Gets the total bytes written to the WAL since startup

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -48,6 +48,8 @@ public:
 	TableIndexList append_indexes;
 	//! The set of delete indexes.
 	TableIndexList delete_indexes;
+	//! Set to INSERT_DUPLICATES, if we are skipping constraint checking during, e.g., WAL replay.
+	IndexAppendMode index_append_mode = IndexAppendMode::DEFAULT;
 	//! The number of deleted rows
 	idx_t deleted_rows;
 	//! The main optimistic data writer

package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp

@@ -33,7 +33,7 @@ public:
 	//! The validity checker of the transaction
 	ValidChecker transaction_validity;
 	//! The active query number
-	transaction_t active_query;
+	atomic<transaction_t> active_query;
 
 public:
 	DUCKDB_API static MetaTransaction &Get(ClientContext &context);