duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -15,13 +15,8 @@ namespace duckdb {
 
 struct ParquetMetaDataBindData : public TableFunctionData {
 	vector<LogicalType> return_types;
-	vector<string> files;
-
-public:
-	bool Equals(const FunctionData &other_p) const override {
-		auto &other = other_p.Cast<ParquetMetaDataBindData>();
-		return other.return_types == return_types && files == other.files;
-	}
+	unique_ptr<MultiFileList> file_list;
+	unique_ptr<MultiFileReader> multi_file_reader;
 };
 
 enum class ParquetMetadataOperatorType : uint8_t { META_DATA, SCHEMA, KEY_VALUE_META_DATA, FILE_META_DATA };
@@ -31,10 +26,12 @@ struct ParquetMetaDataOperatorData : public GlobalTableFunctionState {
 	    : collection(context, types) {
 	}
 
-	idx_t file_index;
 	ColumnDataCollection collection;
 	ColumnDataScanState scan_state;
 
+	MultiFileListScanData file_list_scan;
+	string current_file;
+
 public:
 	static void BindMetaData(vector<LogicalType> &return_types, vector<string> &names);
 	static void BindSchema(vector<LogicalType> &return_types, vector<string> &names);
@@ -588,33 +585,39 @@ unique_ptr<FunctionData> ParquetMetaDataBind(ClientContext &context, TableFuncti
 
 	auto result = make_uniq<ParquetMetaDataBindData>();
 	result->return_types = return_types;
-	result->files = MultiFileReader::GetFileList(context, input.inputs[0], "Parquet");
+	result->multi_file_reader = MultiFileReader::Create(input.table_function);
+	result->file_list = result->multi_file_reader->CreateFileList(context, input.inputs[0]);
 	return std::move(result);
 }
 
 template <ParquetMetadataOperatorType TYPE>
 unique_ptr<GlobalTableFunctionState> ParquetMetaDataInit(ClientContext &context, TableFunctionInitInput &input) {
 	auto &bind_data = input.bind_data->Cast<ParquetMetaDataBindData>();
-	D_ASSERT(!bind_data.files.empty());
 
 	auto result = make_uniq<ParquetMetaDataOperatorData>(context, bind_data.return_types);
+
+	bind_data.file_list->InitializeScan(result->file_list_scan);
+	bind_data.file_list->Scan(result->file_list_scan, result->current_file);
+
+	D_ASSERT(!bind_data.file_list->IsEmpty());
+
 	switch (TYPE) {
 	case ParquetMetadataOperatorType::SCHEMA:
-		result->LoadSchemaData(context, bind_data.return_types, bind_data.files[0]);
+		result->LoadSchemaData(context, bind_data.return_types, bind_data.file_list->GetFirstFile());
 		break;
 	case ParquetMetadataOperatorType::META_DATA:
-		result->LoadRowGroupMetadata(context, bind_data.return_types, bind_data.files[0]);
+		result->LoadRowGroupMetadata(context, bind_data.return_types, bind_data.file_list->GetFirstFile());
 		break;
 	case ParquetMetadataOperatorType::KEY_VALUE_META_DATA:
-		result->LoadKeyValueMetaData(context, bind_data.return_types, bind_data.files[0]);
+		result->LoadKeyValueMetaData(context, bind_data.return_types, bind_data.file_list->GetFirstFile());
 		break;
 	case ParquetMetadataOperatorType::FILE_META_DATA:
-		result->LoadFileMetaData(context, bind_data.return_types, bind_data.files[0]);
+		result->LoadFileMetaData(context, bind_data.return_types, bind_data.file_list->GetFirstFile());
 		break;
 	default:
 		throw InternalException("Unsupported ParquetMetadataOperatorType");
 	}
-	result->file_index = 0;
+
 	return std::move(result);
 }
 
@@ -625,30 +628,29 @@ void ParquetMetaDataImplementation(ClientContext &context, TableFunctionInput &d
 
 	while (true) {
 		if (!data.collection.Scan(data.scan_state, output)) {
-			if (data.file_index + 1 < bind_data.files.size()) {
-				// load the metadata for the next file
-				data.file_index++;
-				switch (TYPE) {
-				case ParquetMetadataOperatorType::SCHEMA:
-					data.LoadSchemaData(context, bind_data.return_types, bind_data.files[data.file_index]);
-					break;
-				case ParquetMetadataOperatorType::META_DATA:
-					data.LoadRowGroupMetadata(context, bind_data.return_types, bind_data.files[data.file_index]);
-					break;
-				case ParquetMetadataOperatorType::KEY_VALUE_META_DATA:
-					data.LoadKeyValueMetaData(context, bind_data.return_types, bind_data.files[data.file_index]);
-					break;
-				case ParquetMetadataOperatorType::FILE_META_DATA:
-					data.LoadFileMetaData(context, bind_data.return_types, bind_data.files[data.file_index]);
-					break;
-				default:
-					throw InternalException("Unsupported ParquetMetadataOperatorType");
-				}
-				continue;
-			} else {
-				// no files remaining: done
+
+			// Try get next file
+			if (!bind_data.file_list->Scan(data.file_list_scan, data.current_file)) {
 				return;
 			}
+
+			switch (TYPE) {
+			case ParquetMetadataOperatorType::SCHEMA:
+				data.LoadSchemaData(context, bind_data.return_types, data.current_file);
+				break;
+			case ParquetMetadataOperatorType::META_DATA:
+				data.LoadRowGroupMetadata(context, bind_data.return_types, data.current_file);
+				break;
+			case ParquetMetadataOperatorType::KEY_VALUE_META_DATA:
+				data.LoadKeyValueMetaData(context, bind_data.return_types, data.current_file);
+				break;
+			case ParquetMetadataOperatorType::FILE_META_DATA:
+				data.LoadFileMetaData(context, bind_data.return_types, data.current_file);
+				break;
+			default:
+				throw InternalException("Unsupported ParquetMetadataOperatorType");
+			}
+			continue;
 		}
 		if (output.size() != 0) {
 			return;

package/src/duckdb/extension/parquet/parquet_reader.cpp

@@ -19,6 +19,7 @@
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/common/pair.hpp"
+#include "duckdb/common/helper.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/types/date.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
@@ -49,7 +50,7 @@ using duckdb_parquet::format::Type;
 
 static unique_ptr<duckdb_apache::thrift::protocol::TProtocol>
 CreateThriftFileProtocol(Allocator &allocator, FileHandle &file_handle, bool prefetch_mode) {
-	auto transport = make_shared<ThriftFileTransport>(allocator, file_handle, prefetch_mode);
+	auto transport = std::make_shared<ThriftFileTransport>(allocator, file_handle, prefetch_mode);
 	return make_uniq<duckdb_apache::thrift::protocol::TCompactProtocolT<ThriftFileTransport>>(std::move(transport));
 }
 
@@ -112,7 +113,7 @@ LoadMetadata(Allocator &allocator, FileHandle &file_handle,
 		metadata->read(file_proto.get());
 	}
 
-	return make_shared<ParquetFileMetadataCache>(std::move(metadata), current_time);
+	return make_shared_ptr<ParquetFileMetadataCache>(std::move(metadata), current_time);
 }
 
 LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool binary_as_string) {
@@ -242,12 +243,14 @@ LogicalType ParquetReader::DeriveLogicalType(const SchemaElement &s_ele, bool bi
 			return LogicalType::INTERVAL;
 		case ConvertedType::JSON:
 			return LogicalType::VARCHAR;
+		case ConvertedType::NULL_TYPE:
+			return LogicalTypeId::SQLNULL;
 		case ConvertedType::MAP:
 		case ConvertedType::MAP_KEY_VALUE:
 		case ConvertedType::LIST:
 		case ConvertedType::BSON:
 		default:
-			throw IOException("Unsupported converted type");
+			throw IOException("Unsupported converted type (%d)", (int32_t)s_ele.converted_type);
 		}
 	} else {
 		// no converted type set
@@ -446,7 +449,7 @@ void ParquetReader::InitializeSchema() {
 
 	// Add generated constant column for row number
 	if (parquet_options.file_row_number) {
-		if (std::find(names.begin(), names.end(), "file_row_number") != names.end()) {
+		if (StringUtil::CIFind(names, "file_row_number") != DConstants::INVALID_INDEX) {
 			throw BinderException(
 			    "Using file_row_number option on file with column named file_row_number is not supported");
 		}

package/src/duckdb/extension/parquet/parquet_statistics.cpp

@@ -48,9 +48,6 @@ static unique_ptr<BaseStatistics> CreateNumericStats(const LogicalType &type,
 Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
                                            const duckdb_parquet::format::SchemaElement &schema_ele,
                                            const std::string &stats) {
-	if (stats.empty()) {
-		return Value();
-	}
 	auto stats_data = const_data_ptr_cast(stats.c_str());
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN: {
@@ -198,8 +195,12 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 		}
 		if (schema_ele.__isset.logicalType && schema_ele.logicalType.__isset.TIME) {
 			// logical type
-			if (schema_ele.logicalType.TIME.unit.__isset.MICROS) {
+			if (schema_ele.logicalType.TIME.unit.__isset.MILLIS) {
+				return Value::TIMETZ(ParquetIntToTimeMsTZ(val));
+			} else if (schema_ele.logicalType.TIME.unit.__isset.MICROS) {
 				return Value::TIMETZ(ParquetIntToTimeTZ(val));
+			} else if (schema_ele.logicalType.TIME.unit.__isset.NANOS) {
+				return Value::TIMETZ(ParquetIntToTimeNsTZ(val));
 			} else {
 				throw InternalException("Time With Time Zone logicalType is set but unit is not defined");
 			}

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -15,6 +15,8 @@
 #include "duckdb/main/connection.hpp"
 #include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 #endif
 
 namespace duckdb {
@@ -351,7 +353,7 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
                              CompressionCodec::type codec, ChildFieldIDs field_ids_p,
                              const vector<pair<string, string>> &kv_metadata,
                              shared_ptr<ParquetEncryptionConfig> encryption_config_p,
-                             double dictionary_compression_ratio_threshold_p)
+                             double dictionary_compression_ratio_threshold_p, optional_idx compression_level_p)
     : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
       field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)),
       dictionary_compression_ratio_threshold(dictionary_compression_ratio_threshold_p) {
@@ -368,7 +370,7 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
 		writer->WriteData(const_data_ptr_cast("PAR1"), 4);
 	}
 	TCompactProtocolFactoryT<MyTransport> tproto_factory;
-	protocol = tproto_factory.getProtocol(make_shared<MyTransport>(*writer));
+	protocol = tproto_factory.getProtocol(std::make_shared<MyTransport>(*writer));
 
 	file_meta_data.num_rows = 0;
 	file_meta_data.version = 1;
@@ -385,6 +387,19 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
 		file_meta_data.key_value_metadata.push_back(kv);
 		file_meta_data.__isset.key_value_metadata = true;
 	}
+	if (compression_level_p.IsValid()) {
+		idx_t level = compression_level_p.GetIndex();
+		switch (codec) {
+		case CompressionCodec::ZSTD:
+			if (level < 1 || level > 22) {
+				throw BinderException("Compression level for ZSTD must be between 1 and 22");
+			}
+			break;
+		default:
+			throw NotImplementedException("Compression level is only supported for the ZSTD compression codec");
+		}
+		compression_level = level;
+	}
 
 	// populate root schema object
 	file_meta_data.schema[0].name = "duckdb_schema";
@@ -468,6 +483,42 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 	result.heaps = buffer.GetHeapReferences();
 }
 
+// Validation code adapted from Impala
+static void ValidateOffsetInFile(const string &filename, idx_t col_idx, idx_t file_length, idx_t offset,
+                                 const string &offset_name) {
+	if (offset < 0 || offset >= file_length) {
+		throw IOException("File '%s': metadata is corrupt. Column %d has invalid "
+		                  "%s (offset=%llu file_size=%llu).",
+		                  filename, col_idx, offset_name, offset, file_length);
+	}
+}
+
+static void ValidateColumnOffsets(const string &filename, idx_t file_length, const ParquetRowGroup &row_group) {
+	for (idx_t i = 0; i < row_group.columns.size(); ++i) {
+		const auto &col_chunk = row_group.columns[i];
+		ValidateOffsetInFile(filename, i, file_length, col_chunk.meta_data.data_page_offset, "data page offset");
+		auto col_start = NumericCast<idx_t>(col_chunk.meta_data.data_page_offset);
+		// The file format requires that if a dictionary page exists, it be before data pages.
+		if (col_chunk.meta_data.__isset.dictionary_page_offset) {
+			ValidateOffsetInFile(filename, i, file_length, col_chunk.meta_data.dictionary_page_offset,
+			                     "dictionary page offset");
+			if (NumericCast<idx_t>(col_chunk.meta_data.dictionary_page_offset) >= col_start) {
+				throw IOException("Parquet file '%s': metadata is corrupt. Dictionary "
+				                  "page (offset=%llu) must come before any data pages (offset=%llu).",
+				                  filename, col_chunk.meta_data.dictionary_page_offset, col_start);
+			}
+			col_start = col_chunk.meta_data.dictionary_page_offset;
+		}
+		auto col_len = NumericCast<idx_t>(col_chunk.meta_data.total_compressed_size);
+		auto col_end = col_start + col_len;
+		if (col_end <= 0 || col_end > file_length) {
+			throw IOException("Parquet file '%s': metadata is corrupt. Column %llu has "
+			                  "invalid column offsets (offset=%llu, size=%llu, file_size=%llu).",
+			                  filename, i, col_start, col_len, file_length);
+		}
+	}
+}
+
 void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
 	lock_guard<mutex> glock(lock);
 	auto &row_group = prepared.row_group;
@@ -481,6 +532,8 @@ void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
 		auto write_state = std::move(states[col_idx]);
 		col_writer->FinalizeWrite(*write_state);
 	}
+	// let's make sure all offsets are ay-okay
+	ValidateColumnOffsets(file_name, writer->GetTotalWritten(), row_group);
 
 	// append the row group to the file meta data
 	file_meta_data.row_groups.push_back(row_group);

package/src/duckdb/extension/parquet/serialize_parquet.cpp

@@ -14,12 +14,12 @@
 namespace duckdb {
 
 void ChildFieldIDs::Serialize(Serializer &serializer) const {
-	serializer.WritePropertyWithDefault<unique_ptr<case_insensitive_map_t<FieldID>>>(100, "ids", ids);
+	serializer.WritePropertyWithDefault<case_insensitive_map_t<FieldID>>(100, "ids", ids.operator*());
 }
 
 ChildFieldIDs ChildFieldIDs::Deserialize(Deserializer &deserializer) {
 	ChildFieldIDs result;
-	deserializer.ReadPropertyWithDefault<unique_ptr<case_insensitive_map_t<FieldID>>>(100, "ids", result.ids);
+	deserializer.ReadPropertyWithDefault<case_insensitive_map_t<FieldID>>(100, "ids", result.ids.operator*());
 	return result;
 }
 

package/src/duckdb/src/catalog/catalog.cpp

@@ -280,14 +280,13 @@ optional_ptr<CatalogEntry> Catalog::CreateCollation(CatalogTransaction transacti
 // Index
 //===--------------------------------------------------------------------===//
 optional_ptr<CatalogEntry> Catalog::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info) {
-	auto &context = transaction.GetContext();
-	return CreateIndex(context, info);
+	auto &schema = GetSchema(transaction, info.schema);
+	auto &table = schema.GetEntry(transaction, CatalogType::TABLE_ENTRY, info.table)->Cast<TableCatalogEntry>();
+	return schema.CreateIndex(transaction, info, table);
 }
 
 optional_ptr<CatalogEntry> Catalog::CreateIndex(ClientContext &context, CreateIndexInfo &info) {
-	auto &schema = GetSchema(context, info.schema);
-	auto &table = GetEntry<TableCatalogEntry>(context, schema.name, info.table);
-	return schema.CreateIndex(context, info, table);
+	return CreateIndex(GetCatalogTransaction(context), info);
 }
 
 //===--------------------------------------------------------------------===//
@@ -760,19 +759,6 @@ CatalogEntryLookup Catalog::TryLookupEntry(ClientContext &context, CatalogType t
 	return Catalog::TryLookupEntry(context, lookups, type, name, if_not_found, error_context);
 }
 
-CatalogEntry &Catalog::GetEntry(ClientContext &context, const string &schema, const string &name) {
-	vector<CatalogType> entry_types {CatalogType::TABLE_ENTRY, CatalogType::SEQUENCE_ENTRY};
-
-	for (auto entry_type : entry_types) {
-		auto result = GetEntry(context, entry_type, schema, name, OnEntryNotFound::RETURN_NULL);
-		if (result) {
-			return *result;
-		}
-	}
-
-	throw CatalogException("CatalogElement \"%s.%s\" does not exist!", schema, name);
-}
-
 optional_ptr<CatalogEntry> Catalog::GetEntry(ClientContext &context, CatalogType type, const string &schema_name,
                                              const string &name, OnEntryNotFound if_not_found,
                                              QueryErrorContext error_context) {
@@ -840,21 +826,6 @@ optional_ptr<SchemaCatalogEntry> Catalog::GetSchema(ClientContext &context, cons
 	return nullptr;
 }
 
-LogicalType Catalog::GetType(ClientContext &context, const string &schema, const string &name,
-                             OnEntryNotFound if_not_found) {
-	auto type_entry = GetEntry<TypeCatalogEntry>(context, schema, name, if_not_found);
-	if (!type_entry) {
-		return LogicalType::INVALID;
-	}
-	return type_entry->user_type;
-}
-
-LogicalType Catalog::GetType(ClientContext &context, const string &catalog_name, const string &schema,
-                             const string &name) {
-	auto &type_entry = Catalog::GetEntry<TypeCatalogEntry>(context, catalog_name, schema, name);
-	return type_entry.user_type;
-}
-
 vector<reference<SchemaCatalogEntry>> Catalog::GetSchemas(ClientContext &context) {
 	vector<reference<SchemaCatalogEntry>> schemas;
 	ScanSchemas(context, [&](SchemaCatalogEntry &entry) { schemas.push_back(entry); });
@@ -912,15 +883,24 @@ vector<reference<SchemaCatalogEntry>> Catalog::GetAllSchemas(ClientContext &cont
 	return result;
 }
 
-void Catalog::Alter(ClientContext &context, AlterInfo &info) {
+void Catalog::Alter(CatalogTransaction transaction, AlterInfo &info) {
 	ModifyCatalog();
 
-	auto lookup = LookupEntry(context, info.GetCatalogType(), info.schema, info.name, info.if_not_found);
-
-	if (!lookup.Found()) {
-		return;
+	if (transaction.HasContext()) {
+		auto lookup =
+		    LookupEntry(transaction.GetContext(), info.GetCatalogType(), info.schema, info.name, info.if_not_found);
+		if (!lookup.Found()) {
+			return;
+		}
+		return lookup.schema->Alter(transaction, info);
 	}
-	return lookup.schema->Alter(context, info);
+	D_ASSERT(info.if_not_found == OnEntryNotFound::THROW_EXCEPTION);
+	auto &schema = GetSchema(transaction, info.schema);
+	return schema.Alter(transaction, info);
+}
+
+void Catalog::Alter(ClientContext &context, AlterInfo &info) {
+	Alter(GetCatalogTransaction(context), info);
 }
 
 vector<MetadataBlockInfo> Catalog::GetMetadataInfo(ClientContext &context) {

package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp

@@ -4,15 +4,15 @@
 
 namespace duckdb {
 
-IndexDataTableInfo::IndexDataTableInfo(shared_ptr<DataTableInfo> &info_p, const string &index_name_p)
-    : info(info_p), index_name(index_name_p) {
+IndexDataTableInfo::IndexDataTableInfo(shared_ptr<DataTableInfo> info_p, const string &index_name_p)
+    : info(std::move(info_p)), index_name(index_name_p) {
 }
 
 IndexDataTableInfo::~IndexDataTableInfo() {
 	if (!info) {
 		return;
 	}
-	info->indexes.RemoveIndex(index_name);
+	info->GetIndexes().RemoveIndex(index_name);
 }
 
 DuckIndexEntry::DuckIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &info)
@@ -38,11 +38,11 @@ unique_ptr<CatalogEntry> DuckIndexEntry::Copy(ClientContext &context) const {
 }
 
 string DuckIndexEntry::GetSchemaName() const {
-	return GetDataTableInfo().schema;
+	return GetDataTableInfo().GetSchemaName();
 }
 
 string DuckIndexEntry::GetTableName() const {
-	return GetDataTableInfo().table;
+	return GetDataTableInfo().GetTableName();
 }
 
 DataTableInfo &DuckIndexEntry::GetDataTableInfo() const {
@@ -51,7 +51,7 @@ DataTableInfo &DuckIndexEntry::GetDataTableInfo() const {
 
 void DuckIndexEntry::CommitDrop() {
 	D_ASSERT(info);
-	GetDataTableInfo().indexes.CommitDrop(name);
+	GetDataTableInfo().GetIndexes().CommitDrop(name);
 }
 
 } // namespace duckdb

package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp

@@ -33,16 +33,16 @@
 #include "duckdb/parser/parsed_data/create_type_info.hpp"
 #include "duckdb/parser/parsed_data/create_view_info.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
+#include "duckdb/transaction/meta_transaction.hpp"
+#include "duckdb/main/attached_database.hpp"
 
 namespace duckdb {
 
-static void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType alter_fk_type,
+static void FindForeignKeyInformation(TableCatalogEntry &table, AlterForeignKeyType alter_fk_type,
                                       vector<unique_ptr<AlterForeignKeyInfo>> &fk_arrays) {
-	if (entry.type != CatalogType::TABLE_ENTRY) {
-		return;
-	}
-	auto &table_entry = entry.Cast<TableCatalogEntry>();
-	auto &constraints = table_entry.GetConstraints();
+	auto &constraints = table.GetConstraints();
+	auto &catalog = table.ParentCatalog();
+	auto &name = table.name;
 	for (idx_t i = 0; i < constraints.size(); i++) {
 		auto &cond = constraints[i];
 		if (cond->type != ConstraintType::FOREIGN_KEY) {
@@ -50,9 +50,9 @@ static void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType a
 		}
 		auto &fk = cond->Cast<ForeignKeyConstraint>();
 		if (fk.info.type == ForeignKeyType::FK_TYPE_FOREIGN_KEY_TABLE) {
-			AlterEntryData alter_data(entry.ParentCatalog().GetName(), fk.info.schema, fk.info.table,
+			AlterEntryData alter_data(catalog.GetName(), fk.info.schema, fk.info.table,
 			                          OnEntryNotFound::THROW_EXCEPTION);
-			fk_arrays.push_back(make_uniq<AlterForeignKeyInfo>(std::move(alter_data), entry.name, fk.pk_columns,
+			fk_arrays.push_back(make_uniq<AlterForeignKeyInfo>(std::move(alter_data), name, fk.pk_columns,
 			                                                   fk.fk_columns, fk.info.pk_keys, fk.info.fk_keys,
 			                                                   alter_fk_type));
 		} else if (fk.info.type == ForeignKeyType::FK_TYPE_PRIMARY_KEY_TABLE &&
@@ -66,13 +66,13 @@ static void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType a
 static void LazyLoadIndexes(ClientContext &context, CatalogEntry &entry) {
 	if (entry.type == CatalogType::TABLE_ENTRY) {
 		auto &table_entry = entry.Cast<TableCatalogEntry>();
-		table_entry.GetStorage().info->InitializeIndexes(context);
+		table_entry.GetStorage().InitializeIndexes(context);
 	} else if (entry.type == CatalogType::INDEX_ENTRY) {
 		auto &index_entry = entry.Cast<IndexCatalogEntry>();
 		auto &table_entry = Catalog::GetEntry(context, CatalogType::TABLE_ENTRY, index_entry.catalog.GetName(),
 		                                      index_entry.GetSchemaName(), index_entry.GetTableName())
 		                        .Cast<TableCatalogEntry>();
-		table_entry.GetStorage().info->InitializeIndexes(context);
+		table_entry.GetStorage().InitializeIndexes(context);
 	}
 }
 
@@ -100,6 +100,17 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::AddEntryInternal(CatalogTransaction
 	auto entry_type = entry->type;
 	auto result = entry.get();
 
+	if (transaction.context) {
+		auto &meta = MetaTransaction::Get(transaction.GetContext());
+		auto modified_database = meta.ModifiedDatabase();
+		auto &db = ParentCatalog().GetAttached();
+		if (!db.IsTemporary() && !db.IsSystem()) {
+			if (!modified_database || !RefersToSameObject(*modified_database, ParentCatalog().GetAttached())) {
+				throw InternalException(
+				    "DuckSchemaEntry::AddEntryInternal called but this database is not marked as modified");
+			}
+		}
+	}
 	// first find the set for this entry
 	auto &set = GetCatalogSet(entry_type);
 	dependencies.AddDependency(*this);
@@ -107,6 +118,9 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::AddEntryInternal(CatalogTransaction
 		// CREATE OR REPLACE: first try to drop the entry
 		auto old_entry = set.GetEntry(transaction, entry_name);
 		if (old_entry) {
+			if (dependencies.Contains(*old_entry)) {
+				throw CatalogException("CREATE OR REPLACE is not allowed to depend on itself");
+			}
 			if (old_entry->type != entry_type) {
 				throw CatalogException("Existing object %s is of type %s, trying to replace with type %s", entry_name,
 				                       CatalogTypeToString(old_entry->type), CatalogTypeToString(entry_type));
@@ -128,26 +142,25 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::AddEntryInternal(CatalogTransaction
 
 optional_ptr<CatalogEntry> DuckSchemaEntry::CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) {
 	auto table = make_uniq<DuckTableEntry>(catalog, *this, info);
-	auto &storage = table->GetStorage();
-	storage.info->cardinality = storage.GetTotalRows();
-
-	auto entry = AddEntryInternal(transaction, std::move(table), info.Base().on_conflict, info.dependencies);
-	if (!entry) {
-		return nullptr;
-	}
 
 	// add a foreign key constraint in main key table if there is a foreign key constraint
 	vector<unique_ptr<AlterForeignKeyInfo>> fk_arrays;
-	FindForeignKeyInformation(*entry, AlterForeignKeyType::AFT_ADD, fk_arrays);
+	FindForeignKeyInformation(*table, AlterForeignKeyType::AFT_ADD, fk_arrays);
 	for (idx_t i = 0; i < fk_arrays.size(); i++) {
 		// alter primary key table
 		auto &fk_info = *fk_arrays[i];
-		catalog.Alter(transaction.GetContext(), fk_info);
+		Alter(transaction, fk_info);
 
 		// make a dependency between this table and referenced table
 		auto &set = GetCatalogSet(CatalogType::TABLE_ENTRY);
 		info.dependencies.AddDependency(*set.GetEntry(transaction, fk_info.name));
 	}
+
+	auto entry = AddEntryInternal(transaction, std::move(table), info.Base().on_conflict, info.dependencies);
+	if (!entry) {
+		return nullptr;
+	}
+
 	return entry;
 }
 
@@ -159,7 +172,7 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::CreateFunction(CatalogTransaction tr
 		if (current_entry) {
 			// the current entry exists - alter it instead
 			auto alter_info = info.GetAlterInfo();
-			Alter(transaction.GetContext(), *alter_info);
+			Alter(transaction, *alter_info);
 			return nullptr;
 		}
 	}
@@ -197,7 +210,7 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::CreateFunction(CatalogTransaction tr
 
 optional_ptr<CatalogEntry> DuckSchemaEntry::AddEntry(CatalogTransaction transaction, unique_ptr<StandardEntry> entry,
                                                      OnCreateConflict on_conflict) {
-	LogicalDependencyList dependencies;
+	LogicalDependencyList dependencies = entry->dependencies;
 	return AddEntryInternal(transaction, std::move(entry), on_conflict, dependencies);
 }
 
@@ -216,19 +229,20 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::CreateView(CatalogTransaction transa
 	return AddEntry(transaction, std::move(view), info.on_conflict);
 }
 
-optional_ptr<CatalogEntry> DuckSchemaEntry::CreateIndex(ClientContext &context, CreateIndexInfo &info,
+optional_ptr<CatalogEntry> DuckSchemaEntry::CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info,
                                                         TableCatalogEntry &table) {
-	LogicalDependencyList dependencies;
-	dependencies.AddDependency(table);
+	info.dependencies.AddDependency(table);
 
 	// currently, we can not alter PK/FK/UNIQUE constraints
 	// concurrency-safe name checks against other INDEX catalog entries happens in the catalog
-	if (!table.GetStorage().IndexNameIsUnique(info.index_name)) {
-		throw CatalogException("An index with the name " + info.index_name + "already exists!");
+	if (info.on_conflict != OnCreateConflict::IGNORE_ON_CONFLICT &&
+	    !table.GetStorage().IndexNameIsUnique(info.index_name)) {
+		throw CatalogException("An index with the name " + info.index_name + " already exists!");
 	}
 
 	auto index = make_uniq<DuckIndexEntry>(catalog, *this, info);
-	return AddEntryInternal(GetCatalogTransaction(context), std::move(index), info.on_conflict, dependencies);
+	auto dependencies = index->dependencies;
+	return AddEntryInternal(transaction, std::move(index), info.on_conflict, dependencies);
 }
 
 optional_ptr<CatalogEntry> DuckSchemaEntry::CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) {
@@ -258,11 +272,10 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::CreatePragmaFunction(CatalogTransact
 	return AddEntry(transaction, std::move(pragma_function), info.on_conflict);
 }
 
-void DuckSchemaEntry::Alter(ClientContext &context, AlterInfo &info) {
+void DuckSchemaEntry::Alter(CatalogTransaction transaction, AlterInfo &info) {
 	CatalogType type = info.GetCatalogType();
 
 	auto &set = GetCatalogSet(type);
-	auto transaction = GetCatalogTransaction(context);
 	if (info.type == AlterType::CHANGE_OWNERSHIP) {
 		if (!set.AlterOwnership(transaction, info.Cast<ChangeOwnershipInfo>())) {
 			throw CatalogException("Couldn't change ownership!");
@@ -305,7 +318,10 @@ void DuckSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) {
 
 	// if there is a foreign key constraint, get that information
 	vector<unique_ptr<AlterForeignKeyInfo>> fk_arrays;
-	FindForeignKeyInformation(*existing_entry, AlterForeignKeyType::AFT_DELETE, fk_arrays);
+	if (existing_entry->type == CatalogType::TABLE_ENTRY) {
+		FindForeignKeyInformation(existing_entry->Cast<TableCatalogEntry>(), AlterForeignKeyType::AFT_DELETE,
+		                          fk_arrays);
+	}
 
 	if (!set.DropEntry(transaction, info.name, info.cascade, info.allow_drop_internal)) {
 		throw InternalException("Could not drop element because of an internal error");
@@ -314,7 +330,7 @@ void DuckSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) {
 	// remove the foreign key constraint in main key table if main key table's name is valid
 	for (idx_t i = 0; i < fk_arrays.size(); i++) {
 		// alter primary key table
-		catalog.Alter(context, *fk_arrays[i]);
+		Alter(transaction, *fk_arrays[i]);
 	}
 }