duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/function/table/system/duckdb_columns.cpp

@@ -131,7 +131,9 @@ public:
 	}
 	const Value ColumnDefault(idx_t col) override {
 		auto &column = entry.GetColumn(LogicalIndex(col));
-		if (column.DefaultValue()) {
+		if (column.Generated()) {
+			return Value(column.GeneratedExpression().ToString());
+		} else if (column.DefaultValue()) {
 			return Value(column.DefaultValue()->ToString());
 		}
 		return Value();

package/src/duckdb/src/function/table/system/pragma_metadata_info.cpp

@@ -0,0 +1,83 @@
+#include "duckdb/function/table/system_functions.hpp"
+
+#include "duckdb/catalog/catalog.hpp"
+#include "duckdb/storage/database_size.hpp"
+#include "duckdb/main/database_manager.hpp"
+#include "duckdb/function/function_set.hpp"
+namespace duckdb {
+
+struct PragmaMetadataFunctionData : public TableFunctionData {
+	explicit PragmaMetadataFunctionData() {
+	}
+
+	vector<MetadataBlockInfo> metadata_info;
+};
+
+struct PragmaMetadataOperatorData : public GlobalTableFunctionState {
+	PragmaMetadataOperatorData() : offset(0) {
+	}
+
+	idx_t offset;
+};
+
+static unique_ptr<FunctionData> PragmaMetadataInfoBind(ClientContext &context, TableFunctionBindInput &input,
+                                                       vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("block_id");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("total_blocks");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("free_blocks");
+	return_types.emplace_back(LogicalType::BIGINT);
+
+	names.emplace_back("free_list");
+	return_types.emplace_back(LogicalType::LIST(LogicalType::BIGINT));
+
+	string db_name =
+	    input.inputs.empty() ? DatabaseManager::GetDefaultDatabase(context) : StringValue::Get(input.inputs[0]);
+	auto &catalog = Catalog::GetCatalog(context, db_name);
+	auto result = make_uniq<PragmaMetadataFunctionData>();
+	result->metadata_info = catalog.GetMetadataInfo(context);
+	return std::move(result);
+}
+
+unique_ptr<GlobalTableFunctionState> PragmaMetadataInfoInit(ClientContext &context, TableFunctionInitInput &input) {
+	return make_uniq<PragmaMetadataOperatorData>();
+}
+
+static void PragmaMetadataInfoFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+	auto &bind_data = data_p.bind_data->Cast<PragmaMetadataFunctionData>();
+	auto &data = data_p.global_state->Cast<PragmaMetadataOperatorData>();
+	idx_t count = 0;
+	while (data.offset < bind_data.metadata_info.size() && count < STANDARD_VECTOR_SIZE) {
+		auto &entry = bind_data.metadata_info[data.offset++];
+
+		idx_t col_idx = 0;
+		// block_id
+		output.SetValue(col_idx++, count, Value::BIGINT(entry.block_id));
+		// total_blocks
+		output.SetValue(col_idx++, count, Value::BIGINT(entry.total_blocks));
+		// free_blocks
+		output.SetValue(col_idx++, count, Value::BIGINT(entry.free_list.size()));
+		// free_list
+		vector<Value> list_values;
+		for (auto &free_id : entry.free_list) {
+			list_values.push_back(Value::BIGINT(free_id));
+		}
+		output.SetValue(col_idx++, count, Value::LIST(LogicalType::BIGINT, std::move(list_values)));
+		count++;
+	}
+	output.SetCardinality(count);
+}
+
+void PragmaMetadataInfo::RegisterFunction(BuiltinFunctions &set) {
+	TableFunctionSet metadata_info("pragma_metadata_info");
+	metadata_info.AddFunction(
+	    TableFunction({}, PragmaMetadataInfoFunction, PragmaMetadataInfoBind, PragmaMetadataInfoInit));
+	metadata_info.AddFunction(TableFunction({LogicalType::VARCHAR}, PragmaMetadataInfoFunction, PragmaMetadataInfoBind,
+	                                        PragmaMetadataInfoInit));
+	set.AddFunction(metadata_info);
+}
+
+} // namespace duckdb

package/src/duckdb/src/function/table/system/pragma_storage_info.cpp

@@ -76,6 +76,9 @@ static unique_ptr<FunctionData> PragmaStorageInfoBind(ClientContext &context, Ta
 	names.emplace_back("block_offset");
 	return_types.emplace_back(LogicalType::BIGINT);
 
+	names.emplace_back("segment_info");
+	return_types.emplace_back(LogicalType::VARCHAR);
+
 	auto qname = QualifiedName::Parse(input.inputs[0].GetValue<string>());
 
 	// look up the table name in the catalog
@@ -133,6 +136,8 @@ static void PragmaStorageInfoFunction(ClientContext &context, TableFunctionInput
 			output.SetValue(col_idx++, count, Value());
 			output.SetValue(col_idx++, count, Value());
 		}
+		// segment_info
+		output.SetValue(col_idx++, count, Value(entry.segment_info));
 		count++;
 	}
 	output.SetCardinality(count);

package/src/duckdb/src/function/table/system/test_all_types.cpp

@@ -17,7 +17,7 @@ struct TestAllTypesData : public GlobalTableFunctionState {
 	idx_t offset;
 };
 
-vector<TestType> TestAllTypesFun::GetTestTypes() {
+vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
 	vector<TestType> result;
 	// scalar types/numerics
 	result.emplace_back(LogicalType::BOOLEAN, "bool");
@@ -70,22 +70,30 @@ vector<TestType> TestAllTypesFun::GetTestTypes() {
 	auto small_enum_ptr = FlatVector::GetData<string_t>(small_enum);
 	small_enum_ptr[0] = StringVector::AddStringOrBlob(small_enum, "DUCK_DUCK_ENUM");
 	small_enum_ptr[1] = StringVector::AddStringOrBlob(small_enum, "GOOSE");
-	result.emplace_back(LogicalType::ENUM("small_enum", small_enum, 2), "small_enum");
+	result.emplace_back(LogicalType::ENUM(small_enum, 2), "small_enum");
 
 	Vector medium_enum(LogicalType::VARCHAR, 300);
 	auto medium_enum_ptr = FlatVector::GetData<string_t>(medium_enum);
 	for (idx_t i = 0; i < 300; i++) {
 		medium_enum_ptr[i] = StringVector::AddStringOrBlob(medium_enum, string("enum_") + to_string(i));
 	}
-	result.emplace_back(LogicalType::ENUM("medium_enum", medium_enum, 300), "medium_enum");
-
-	// this is a big one... not sure if we should push this one here, but it's required for completeness
-	Vector large_enum(LogicalType::VARCHAR, 70000);
-	auto large_enum_ptr = FlatVector::GetData<string_t>(large_enum);
-	for (idx_t i = 0; i < 70000; i++) {
-		large_enum_ptr[i] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(i));
+	result.emplace_back(LogicalType::ENUM(medium_enum, 300), "medium_enum");
+
+	if (use_large_enum) {
+		// this is a big one... not sure if we should push this one here, but it's required for completeness
+		Vector large_enum(LogicalType::VARCHAR, 70000);
+		auto large_enum_ptr = FlatVector::GetData<string_t>(large_enum);
+		for (idx_t i = 0; i < 70000; i++) {
+			large_enum_ptr[i] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(i));
+		}
+		result.emplace_back(LogicalType::ENUM(large_enum, 70000), "large_enum");
+	} else {
+		Vector large_enum(LogicalType::VARCHAR, 2);
+		auto large_enum_ptr = FlatVector::GetData<string_t>(large_enum);
+		large_enum_ptr[0] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(0));
+		large_enum_ptr[1] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(69999));
+		result.emplace_back(LogicalType::ENUM(large_enum, 2), "large_enum");
 	}
-	result.emplace_back(LogicalType::ENUM("large_enum", large_enum, 70000), "large_enum");
 
 	// arrays
 	auto int_list_type = LogicalType::LIST(LogicalType::INTEGER);
@@ -196,29 +204,46 @@ vector<TestType> TestAllTypesFun::GetTestTypes() {
 	auto max_map_value = Value::MAP(ListType::GetChildType(map_type), map_values);
 	result.emplace_back(map_type, "map", std::move(min_map_value), std::move(max_map_value));
 
+	// union
+	child_list_t<LogicalType> members = {{"name", LogicalType::VARCHAR}, {"age", LogicalType::SMALLINT}};
+	auto union_type = LogicalType::UNION(members);
+	const Value &min = Value::UNION(members, 0, Value("Frank"));
+	const Value &max = Value::UNION(members, 1, Value::SMALLINT(5));
+	result.emplace_back(union_type, "union", min, max);
+
 	return result;
 }
 
+struct TestAllTypesBindData : public TableFunctionData {
+	vector<TestType> test_types;
+};
+
 static unique_ptr<FunctionData> TestAllTypesBind(ClientContext &context, TableFunctionBindInput &input,
                                                  vector<LogicalType> &return_types, vector<string> &names) {
-	auto test_types = TestAllTypesFun::GetTestTypes();
-	for (auto &test_type : test_types) {
-		return_types.push_back(std::move(test_type.type));
-		names.push_back(std::move(test_type.name));
+	auto result = make_uniq<TestAllTypesBindData>();
+	bool use_large_enum = false;
+	auto entry = input.named_parameters.find("use_large_enum");
+	if (entry != input.named_parameters.end()) {
+		use_large_enum = BooleanValue::Get(entry->second);
+	}
+	result->test_types = TestAllTypesFun::GetTestTypes(use_large_enum);
+	for (auto &test_type : result->test_types) {
+		return_types.push_back(test_type.type);
+		names.push_back(test_type.name);
 	}
-	return nullptr;
+	return std::move(result);
 }
 
 unique_ptr<GlobalTableFunctionState> TestAllTypesInit(ClientContext &context, TableFunctionInitInput &input) {
+	auto &bind_data = input.bind_data->Cast<TestAllTypesBindData>();
 	auto result = make_uniq<TestAllTypesData>();
-	auto test_types = TestAllTypesFun::GetTestTypes();
 	// 3 rows: min, max and NULL
 	result->entries.resize(3);
 	// initialize the values
-	for (auto &test_type : test_types) {
-		result->entries[0].push_back(std::move(test_type.min_value));
-		result->entries[1].push_back(std::move(test_type.max_value));
-		result->entries[2].emplace_back(std::move(test_type.type));
+	for (auto &test_type : bind_data.test_types) {
+		result->entries[0].push_back(test_type.min_value);
+		result->entries[1].push_back(test_type.max_value);
+		result->entries[2].emplace_back(test_type.type);
 	}
 	return std::move(result);
 }
@@ -243,7 +268,9 @@ void TestAllTypesFunction(ClientContext &context, TableFunctionInput &data_p, Da
 }
 
 void TestAllTypesFun::RegisterFunction(BuiltinFunctions &set) {
-	set.AddFunction(TableFunction("test_all_types", {}, TestAllTypesFunction, TestAllTypesBind, TestAllTypesInit));
+	TableFunction test_all_types("test_all_types", {}, TestAllTypesFunction, TestAllTypesBind, TestAllTypesInit);
+	test_all_types.named_parameters["use_large_enum"] = LogicalType::BOOLEAN;
+	set.AddFunction(test_all_types);
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/table/system_functions.cpp

@@ -10,9 +10,11 @@ namespace duckdb {
 
 void BuiltinFunctions::RegisterSQLiteFunctions() {
 	PragmaVersion::RegisterFunction(*this);
+	PragmaPlatform::RegisterFunction(*this);
 	PragmaCollations::RegisterFunction(*this);
 	PragmaTableInfo::RegisterFunction(*this);
 	PragmaStorageInfo::RegisterFunction(*this);
+	PragmaMetadataInfo::RegisterFunction(*this);
 	PragmaDatabaseSize::RegisterFunction(*this);
 	PragmaLastProfilingOutput::RegisterFunction(*this);
 	PragmaDetailedProfilingOutput::RegisterFunction(*this);

package/src/duckdb/src/function/table/table_scan.cpp

@@ -1,7 +1,6 @@
 #include "duckdb/function/table/table_scan.hpp"
 
 #include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/main/client_config.hpp"
 #include "duckdb/optimizer/matcher/expression_matcher.hpp"
@@ -15,6 +14,8 @@
 #include "duckdb/catalog/dependency_list.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -281,6 +282,15 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 	if (bind_data.is_index_scan) {
 		return;
 	}
+	if (!get.table_filters.filters.empty()) {
+		// if there were filters before we can't convert this to an index scan
+		return;
+	}
+	if (!get.projection_ids.empty()) {
+		// if columns were pruned by RemoveUnusedColumns we can't convert this to an index scan,
+		// because index scan does not support filter_prune (yet)
+		return;
+	}
 	if (filters.empty()) {
 		// no indexes or no filters: skip the pushdown
 		return;
@@ -406,35 +416,30 @@ string TableScanToString(const FunctionData *bind_data_p) {
 	return result;
 }
 
-static void TableScanSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {
+static void TableScanSerialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data_p,
+                               const TableFunction &function) {
 	auto &bind_data = bind_data_p->Cast<TableScanBindData>();
-
-	writer.WriteString(bind_data.table.schema.name);
-	writer.WriteString(bind_data.table.name);
-	writer.WriteField<bool>(bind_data.is_index_scan);
-	writer.WriteField<bool>(bind_data.is_create_index);
-	writer.WriteList<row_t>(bind_data.result_ids);
-	writer.WriteString(bind_data.table.schema.catalog.GetName());
+	serializer.WriteProperty(100, "catalog", bind_data.table.schema.catalog.GetName());
+	serializer.WriteProperty(101, "schema", bind_data.table.schema.name);
+	serializer.WriteProperty(102, "table", bind_data.table.name);
+	serializer.WriteProperty(103, "is_index_scan", bind_data.is_index_scan);
+	serializer.WriteProperty(104, "is_create_index", bind_data.is_create_index);
+	serializer.WriteProperty(105, "result_ids", bind_data.result_ids);
 }
 
-static unique_ptr<FunctionData> TableScanDeserialize(PlanDeserializationState &state, FieldReader &reader,
-                                                     TableFunction &function) {
-	auto schema_name = reader.ReadRequired<string>();
-	auto table_name = reader.ReadRequired<string>();
-	auto is_index_scan = reader.ReadRequired<bool>();
-	auto is_create_index = reader.ReadRequired<bool>();
-	auto result_ids = reader.ReadRequiredList<row_t>();
-	auto catalog_name = reader.ReadField<string>(INVALID_CATALOG);
-
-	auto &catalog_entry = Catalog::GetEntry<TableCatalogEntry>(state.context, catalog_name, schema_name, table_name);
+static unique_ptr<FunctionData> TableScanDeserialize(Deserializer &deserializer, TableFunction &function) {
+	auto catalog = deserializer.ReadProperty<string>(100, "catalog");
+	auto schema = deserializer.ReadProperty<string>(101, "schema");
+	auto table = deserializer.ReadProperty<string>(102, "table");
+	auto &catalog_entry =
+	    Catalog::GetEntry<TableCatalogEntry>(deserializer.Get<ClientContext &>(), catalog, schema, table);
 	if (catalog_entry.type != CatalogType::TABLE_ENTRY) {
-		throw SerializationException("Cant find table for %s.%s", schema_name, table_name);
+		throw SerializationException("Cant find table for %s.%s", schema, table);
 	}
-
 	auto result = make_uniq<TableScanBindData>(catalog_entry.Cast<DuckTableEntry>());
-	result->is_index_scan = is_index_scan;
-	result->is_create_index = is_create_index;
-	result->result_ids = std::move(result_ids);
+	deserializer.ReadProperty(103, "is_index_scan", result->is_index_scan);
+	deserializer.ReadProperty(104, "is_create_index", result->is_create_index);
+	deserializer.ReadProperty(105, "result_ids", result->result_ids);
 	return std::move(result);
 }
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,11 +1,12 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev87"
+#define DUCKDB_VERSION "0.9.0"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "8a17511028"
+#define DUCKDB_SOURCE_ID "0d84ccf478"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/common/string_util.hpp"
 
 #include <cstdint>
 
@@ -63,6 +64,13 @@ const char *DuckDB::LibraryVersion() {
 }
 
 string DuckDB::Platform() {
+#if defined(DUCKDB_CUSTOM_PLATFORM)
+	return DUCKDB_QUOTE_DEFINE(DUCKDB_CUSTOM_PLATFORM);
+#endif
+#if defined(DUCKDB_WASM_VERSION)
+	// DuckDB-Wasm requires CUSTOM_PLATFORM to be defined
+	static_assert(0, "DUCKDB_WASM_VERSION should rely on CUSTOM_PLATFORM being provided");
+#endif
 	string os = "linux";
 #if INTPTR_MAX == INT64_MAX
 	string arch = "amd64";
@@ -87,6 +95,9 @@ string DuckDB::Platform() {
 		postfix = "_gcc4";
 	}
 #endif
+#if defined(__ANDROID__)
+	postfix += "_android"; // using + because it may also be gcc4
+#endif
 #ifdef __MINGW32__
 	postfix = "_mingw";
 #endif
@@ -97,4 +108,40 @@ string DuckDB::Platform() {
 	return os + "_" + arch + postfix;
 }
 
+struct PragmaPlatformData : public GlobalTableFunctionState {
+	PragmaPlatformData() : finished(false) {
+	}
+
+	bool finished;
+};
+
+static unique_ptr<FunctionData> PragmaPlatformBind(ClientContext &context, TableFunctionBindInput &input,
+                                                   vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("platform");
+	return_types.emplace_back(LogicalType::VARCHAR);
+	return nullptr;
+}
+
+static unique_ptr<GlobalTableFunctionState> PragmaPlatformInit(ClientContext &context, TableFunctionInitInput &input) {
+	return make_uniq<PragmaPlatformData>();
+}
+
+static void PragmaPlatformFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+	auto &data = data_p.global_state->Cast<PragmaPlatformData>();
+	if (data.finished) {
+		// finished returning values
+		return;
+	}
+	output.SetCardinality(1);
+	output.SetValue(0, 0, DuckDB::Platform());
+	data.finished = true;
+}
+
+void PragmaPlatform::RegisterFunction(BuiltinFunctions &set) {
+	TableFunction pragma_platform("pragma_platform", {}, PragmaPlatformFunction);
+	pragma_platform.bind = PragmaPlatformBind;
+	pragma_platform.init_global = PragmaPlatformInit;
+	set.AddFunction(pragma_platform);
+}
+
 } // namespace duckdb

package/src/duckdb/src/function/table_macro_function.cpp

@@ -31,14 +31,4 @@ string TableMacroFunction::ToSQL(const string &schema, const string &name) const
 	return MacroFunction::ToSQL(schema, name) + StringUtil::Format("TABLE (%s);", query_node->ToString());
 }
 
-void TableMacroFunction::SerializeInternal(FieldWriter &writer) const {
-	writer.WriteSerializable(*query_node);
-}
-
-unique_ptr<MacroFunction> TableMacroFunction::Deserialize(FieldReader &reader) {
-	auto result = make_uniq<TableMacroFunction>();
-	result->query_node = reader.ReadRequiredSerializable<QueryNode>();
-	return std::move(result);
-}
-
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/catalog/catalog.hpp

@@ -34,6 +34,7 @@ struct CreateIndexInfo;
 struct CreateTypeInfo;
 struct CreateTableInfo;
 struct DatabaseSize;
+struct MetadataBlockInfo;
 
 class AttachedDatabase;
 class ClientContext;
@@ -266,6 +267,7 @@ public:
 	                                                    unique_ptr<LogicalOperator> plan) = 0;
 
 	virtual DatabaseSize GetDatabaseSize(ClientContext &context) = 0;
+	virtual vector<MetadataBlockInfo> GetMetadataInfo(ClientContext &context);
 
 	virtual bool InMemory() = 0;
 	virtual string GetDBPath() = 0;
@@ -301,18 +303,34 @@ public:
 
 	static CatalogException UnrecognizedConfigurationError(ClientContext &context, const string &name);
 
+	//! Autoload the extension required for `configuration_name` or throw a CatalogException
+	static void AutoloadExtensionByConfigName(ClientContext &context, const string &configuration_name);
+	//! Autoload the extension required for `function_name` or throw a CatalogException
+	static bool AutoLoadExtensionByCatalogEntry(ClientContext &context, CatalogType type, const string &entry_name);
+	DUCKDB_API static bool TryAutoLoad(ClientContext &context, const string &extension_name) noexcept;
+
 protected:
 	//! Reference to the database
 	AttachedDatabase &db;
 
 private:
-	CatalogEntryLookup LookupEntryInternal(CatalogTransaction transaction, CatalogType type, const string &schema,
-	                                       const string &name);
+	//! Lookup an entry in the schema, returning a lookup with the entry and schema if they exist
+	CatalogEntryLookup TryLookupEntryInternal(CatalogTransaction transaction, CatalogType type, const string &schema,
+	                                          const string &name);
+	//! Calls LookupEntryInternal on the schema, trying other schemas if the schema is invalid. Sets
+	//! CatalogEntryLookup->error depending on if_not_found when no entry is found
+	CatalogEntryLookup TryLookupEntry(ClientContext &context, CatalogType type, const string &schema,
+	                                  const string &name, OnEntryNotFound if_not_found,
+	                                  QueryErrorContext error_context = QueryErrorContext());
+	//! Lookup an entry using TryLookupEntry, throws if entry not found and if_not_found == THROW_EXCEPTION
 	CatalogEntryLookup LookupEntry(ClientContext &context, CatalogType type, const string &schema, const string &name,
 	                               OnEntryNotFound if_not_found, QueryErrorContext error_context = QueryErrorContext());
-	static CatalogEntryLookup LookupEntry(ClientContext &context, vector<CatalogLookup> &lookups, CatalogType type,
-	                                      const string &name, OnEntryNotFound if_not_found,
-	                                      QueryErrorContext error_context = QueryErrorContext());
+	static CatalogEntryLookup TryLookupEntry(ClientContext &context, vector<CatalogLookup> &lookups, CatalogType type,
+	                                         const string &name, OnEntryNotFound if_not_found,
+	                                         QueryErrorContext error_context = QueryErrorContext());
+	static CatalogEntryLookup TryLookupEntry(ClientContext &context, CatalogType type, const string &catalog,
+	                                         const string &schema, const string &name, OnEntryNotFound if_not_found,
+	                                         QueryErrorContext error_context);
 
 	//! Return an exception with did-you-mean suggestion.
 	static CatalogException CreateMissingEntryException(ClientContext &context, const string &entry_name,

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp

@@ -24,6 +24,8 @@ public:
 public:
 	string GetSchemaName() const override;
 	string GetTableName() const override;
+	//! This drops in-memory index data and marks all blocks on disk as free blocks, allowing to reclaim them
+	void CommitDrop();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp

@@ -10,7 +10,7 @@
 
 #include "duckdb/catalog/standard_entry.hpp"
 #include "duckdb/parser/parsed_data/create_index_info.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 
 namespace duckdb {
 
@@ -31,11 +31,11 @@ public:
 	string sql;
 	vector<unique_ptr<ParsedExpression>> expressions;
 	vector<unique_ptr<ParsedExpression>> parsed_expressions;
+	case_insensitive_map_t<Value> options;
 
 public:
+	unique_ptr<CreateInfo> GetInfo() const override;
 	string ToSQL() const override;
-	void Serialize(Serializer &serializer) const;
-	static unique_ptr<CreateIndexInfo> Deserialize(Deserializer &source, ClientContext &context);
 
 	virtual string GetSchemaName() const = 0;
 	virtual string GetTableName() const = 0;

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/macro_catalog_entry.hpp

@@ -24,10 +24,7 @@ public:
 	unique_ptr<MacroFunction> function;
 
 public:
-	virtual unique_ptr<CreateMacroInfo> GetInfoForSerialization() const;
-	//! Serialize the meta information
-	virtual void Serialize(Serializer &serializer) const;
-	static unique_ptr<CreateMacroInfo> Deserialize(Deserializer &main_source, ClientContext &context);
+	unique_ptr<CreateInfo> GetInfo() const override;
 
 	string ToSQL() const override {
 		return function->ToSQL(schema.name, name);

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/schema_catalog_entry.hpp

@@ -19,8 +19,6 @@ class StandardEntry;
 class TableCatalogEntry;
 class TableFunctionCatalogEntry;
 class SequenceCatalogEntry;
-class Serializer;
-class Deserializer;
 
 enum class OnCreateConflict : uint8_t;
 
@@ -49,17 +47,14 @@ public:
 	SchemaCatalogEntry(Catalog &catalog, string name, bool is_internal);
 
 public:
+	unique_ptr<CreateInfo> GetInfo() const override;
+
 	//! Scan the specified catalog set, invoking the callback method for every entry
 	virtual void Scan(ClientContext &context, CatalogType type,
 	                  const std::function<void(CatalogEntry &)> &callback) = 0;
 	//! Scan the specified catalog set, invoking the callback method for every committed entry
 	virtual void Scan(CatalogType type, const std::function<void(CatalogEntry &)> &callback) = 0;
 
-	//! Serialize the meta information of the SchemaCatalogEntry a serializer
-	virtual void Serialize(Serializer &serializer) const;
-	//! Deserializes to a CreateSchemaInfo
-	static unique_ptr<CreateSchemaInfo> Deserialize(Deserializer &source);
-
 	string ToSQL() const override;
 
 	//! Creates an index with the given name in the schema

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp

@@ -14,8 +14,6 @@
 #include "duckdb/parser/parsed_data/alter_table_info.hpp"
 
 namespace duckdb {
-class Serializer;
-class Deserializer;
 
 struct SequenceValue {
 	SequenceValue() : usage_count(0), counter(-1) {
@@ -57,13 +55,8 @@ public:
 	bool cycle;
 
 public:
-	//! Serialize the meta information of the SequenceCatalogEntry a serializer
-	virtual void Serialize(Serializer &serializer) const;
-	//! Deserializes to a CreateTableInfo
-	static unique_ptr<CreateSequenceInfo> Deserialize(Deserializer &source);
+	unique_ptr<CreateInfo> GetInfo() const override;
 
 	string ToSQL() const override;
-
-	CatalogEntry *AlterOwnership(ClientContext &context, AlterInfo *info);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp

@@ -55,6 +55,8 @@ public:
 	DUCKDB_API TableCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateTableInfo &info);
 
 public:
+	DUCKDB_API unique_ptr<CreateInfo> GetInfo() const override;
+
 	DUCKDB_API bool HasGeneratedColumns() const;
 
 	//! Returns whether or not a column with the given name exists
@@ -69,8 +71,6 @@ public:
 	DUCKDB_API vector<LogicalType> GetTypes();
 	//! Returns a list of the columns of the table
 	DUCKDB_API const ColumnList &GetColumns() const;
-	//! Returns a mutable list of the columns of the table
-	DUCKDB_API ColumnList &GetColumnsMutable();
 	//! Returns the underlying storage of the table
 	virtual DataTable &GetStorage();
 	//! Returns a list of the bound constraints of the table
@@ -83,11 +83,6 @@ public:
 	//! Get statistics of a column (physical or virtual) within the table
 	virtual unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, column_t column_id) = 0;
 
-	//! Serialize the meta information of the TableCatalogEntry a serializer
-	void Serialize(Serializer &serializer) const;
-	//! Deserializes to a CreateTableInfo
-	static unique_ptr<CreateTableInfo> Deserialize(Deserializer &source, ClientContext &context);
-
 	//! Returns the column index of the specified column name.
 	//! If the column does not exist:
 	//! If if_column_exists is true, returns DConstants::INVALID_INDEX
@@ -113,12 +108,6 @@ public:
 	                                   ClientContext &context);
 
 protected:
-	// This is used to serialize the entry by #Serialize(Serializer& ). It is virtual to allow
-	// Custom catalog implementations to override the default implementation. We can not make
-	// The Serialize method itself virtual as the logic is tightly coupled to the static
-	// Deserialize method.
-	virtual CreateTableInfo GetTableInfoForSerialization() const;
-
 	//! A list of columns that are part of this table
 	ColumnList columns;
 	//! A list of constraints that are part of this table

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp

@@ -13,8 +13,6 @@
 #include "duckdb/parser/parsed_data/create_type_info.hpp"
 
 namespace duckdb {
-class Serializer;
-class Deserializer;
 
 //! A type catalog entry
 class TypeCatalogEntry : public StandardEntry {
@@ -29,10 +27,7 @@ public:
 	LogicalType user_type;
 
 public:
-	//! Serialize the meta information of the TypeCatalogEntry a serializer
-	virtual void Serialize(Serializer &serializer) const;
-	//! Deserializes to a TypeCatalogEntry
-	static unique_ptr<CreateTypeInfo> Deserialize(Deserializer &source);
+	unique_ptr<CreateInfo> GetInfo() const override;
 
 	string ToSQL() const override;
 };