duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -8,6 +8,8 @@
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/write_stream.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/function/table_function.hpp"
@@ -15,8 +17,6 @@
 #include "duckdb/main/connection.hpp"
 #include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
-#include "duckdb/common/serializer/serializer.hpp"
-#include "duckdb/common/serializer/deserializer.hpp"
 #endif
 
 namespace duckdb {
@@ -82,8 +82,8 @@ private:
 	WriteStream &serializer;
 };
 
-CopyTypeSupport ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type,
-                                                               Type::type &parquet_type) {
+bool ParquetWriter::TryGetParquetType(const LogicalType &duckdb_type, optional_ptr<Type::type> parquet_type_ptr) {
+	Type::type parquet_type;
 	switch (duckdb_type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		parquet_type = Type::BOOLEAN;
@@ -106,7 +106,7 @@ CopyTypeSupport ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType
 	case LogicalTypeId::UHUGEINT:
 	case LogicalTypeId::HUGEINT:
 		parquet_type = Type::DOUBLE;
-		return CopyTypeSupport::LOSSY;
+		break;
 	case LogicalTypeId::ENUM:
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::VARCHAR:
@@ -151,71 +151,29 @@ CopyTypeSupport ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType
 		break;
 	default:
 		// Anything that is not supported
-		return CopyTypeSupport::UNSUPPORTED;
+		return false;
 	}
-	return CopyTypeSupport::SUPPORTED;
+	if (parquet_type_ptr) {
+		*parquet_type_ptr = parquet_type;
+	}
+	return true;
 }
 
 Type::type ParquetWriter::DuckDBTypeToParquetType(const LogicalType &duckdb_type) {
 	Type::type result;
-	auto type_supports = DuckDBTypeToParquetTypeInternal(duckdb_type, result);
-	if (type_supports == CopyTypeSupport::UNSUPPORTED) {
-		throw NotImplementedException("Unimplemented type for Parquet \"%s\"", duckdb_type.ToString());
-	}
-	return result;
-}
-
-CopyTypeSupport ParquetWriter::TypeIsSupported(const LogicalType &type) {
-	Type::type unused;
-	auto id = type.id();
-	if (id == LogicalTypeId::LIST) {
-		auto &child_type = ListType::GetChildType(type);
-		return TypeIsSupported(child_type);
-	}
-	if (id == LogicalTypeId::ARRAY) {
-		auto &child_type = ArrayType::GetChildType(type);
-		return TypeIsSupported(child_type);
-	}
-	if (id == LogicalTypeId::UNION) {
-		auto count = UnionType::GetMemberCount(type);
-		for (idx_t i = 0; i < count; i++) {
-			auto &member_type = UnionType::GetMemberType(type, i);
-			auto type_support = TypeIsSupported(member_type);
-			if (type_support != CopyTypeSupport::SUPPORTED) {
-				return type_support;
-			}
-		}
-		return CopyTypeSupport::SUPPORTED;
-	}
-	if (id == LogicalTypeId::STRUCT) {
-		auto &children = StructType::GetChildTypes(type);
-		for (auto &child : children) {
-			auto &child_type = child.second;
-			auto type_support = TypeIsSupported(child_type);
-			if (type_support != CopyTypeSupport::SUPPORTED) {
-				return type_support;
-			}
-		}
-		return CopyTypeSupport::SUPPORTED;
-	}
-	if (id == LogicalTypeId::MAP) {
-		auto &key_type = MapType::KeyType(type);
-		auto &value_type = MapType::ValueType(type);
-		auto key_type_support = TypeIsSupported(key_type);
-		if (key_type_support != CopyTypeSupport::SUPPORTED) {
-			return key_type_support;
-		}
-		auto value_type_support = TypeIsSupported(value_type);
-		if (value_type_support != CopyTypeSupport::SUPPORTED) {
-			return value_type_support;
-		}
-		return CopyTypeSupport::SUPPORTED;
+	if (TryGetParquetType(duckdb_type, &result)) {
+		return result;
 	}
-	return DuckDBTypeToParquetTypeInternal(type, unused);
+	throw NotImplementedException("Unimplemented type for Parquet \"%s\"", duckdb_type.ToString());
 }
 
 void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type,
                                         duckdb_parquet::format::SchemaElement &schema_ele) {
+	if (duckdb_type.IsJSONType()) {
+		schema_ele.converted_type = ConvertedType::JSON;
+		schema_ele.__isset.converted_type = true;
+		return;
+	}
 	switch (duckdb_type.id()) {
 	case LogicalTypeId::TINYINT:
 		schema_ele.converted_type = ConvertedType::INT_8;
@@ -264,7 +222,6 @@ void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type,
 		break;
 	case LogicalTypeId::TIMESTAMP_TZ:
 	case LogicalTypeId::TIMESTAMP:
-	case LogicalTypeId::TIMESTAMP_NS:
 	case LogicalTypeId::TIMESTAMP_SEC:
 		schema_ele.converted_type = ConvertedType::TIMESTAMP_MICROS;
 		schema_ele.__isset.converted_type = true;
@@ -273,6 +230,13 @@ void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type,
 		schema_ele.logicalType.TIMESTAMP.isAdjustedToUTC = (duckdb_type.id() == LogicalTypeId::TIMESTAMP_TZ);
 		schema_ele.logicalType.TIMESTAMP.unit.__isset.MICROS = true;
 		break;
+	case LogicalTypeId::TIMESTAMP_NS:
+		schema_ele.__isset.converted_type = false;
+		schema_ele.__isset.logicalType = true;
+		schema_ele.logicalType.__isset.TIMESTAMP = true;
+		schema_ele.logicalType.TIMESTAMP.isAdjustedToUTC = false;
+		schema_ele.logicalType.TIMESTAMP.unit.__isset.NANOS = true;
+		break;
 	case LogicalTypeId::TIMESTAMP_MS:
 		schema_ele.converted_type = ConvertedType::TIMESTAMP_MILLIS;
 		schema_ele.__isset.converted_type = true;
@@ -321,7 +285,7 @@ void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type,
 
 uint32_t ParquetWriter::Write(const duckdb_apache::thrift::TBase &object) {
 	if (encryption_config) {
-		return ParquetCrypto::Write(object, *protocol, encryption_config->GetFooterKey());
+		return ParquetCrypto::Write(object, *protocol, encryption_config->GetFooterKey(), *encryption_util);
 	} else {
 		return object.write(protocol.get());
 	}
@@ -329,7 +293,8 @@ uint32_t ParquetWriter::Write(const duckdb_apache::thrift::TBase &object) {
 
 uint32_t ParquetWriter::WriteData(const const_data_ptr_t buffer, const uint32_t buffer_size) {
 	if (encryption_config) {
-		return ParquetCrypto::WriteData(*protocol, buffer, buffer_size, encryption_config->GetFooterKey());
+		return ParquetCrypto::WriteData(*protocol, buffer, buffer_size, encryption_config->GetFooterKey(),
+		                                *encryption_util);
 	} else {
 		protocol->getTransport()->write(buffer, buffer_size);
 		return buffer_size;
@@ -349,18 +314,27 @@ void VerifyUniqueNames(const vector<string> &names) {
 #endif
 }
 
-ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalType> types_p, vector<string> names_p,
-                             CompressionCodec::type codec, ChildFieldIDs field_ids_p,
+ParquetWriter::ParquetWriter(ClientContext &context, FileSystem &fs, string file_name_p, vector<LogicalType> types_p,
+                             vector<string> names_p, CompressionCodec::type codec, ChildFieldIDs field_ids_p,
                              const vector<pair<string, string>> &kv_metadata,
                              shared_ptr<ParquetEncryptionConfig> encryption_config_p,
-                             double dictionary_compression_ratio_threshold_p, optional_idx compression_level_p)
+                             double dictionary_compression_ratio_threshold_p, optional_idx compression_level_p,
+                             bool debug_use_openssl_p)
     : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
       field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)),
-      dictionary_compression_ratio_threshold(dictionary_compression_ratio_threshold_p) {
+      dictionary_compression_ratio_threshold(dictionary_compression_ratio_threshold_p),
+      debug_use_openssl(debug_use_openssl_p) {
 	// initialize the file writer
 	writer = make_uniq<BufferedFileWriter>(fs, file_name.c_str(),
 	                                       FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
 	if (encryption_config) {
+		auto &config = DBConfig::GetConfig(context);
+		if (config.encryption_util && debug_use_openssl) {
+			// Use OpenSSL
+			encryption_util = config.encryption_util;
+		} else {
+			encryption_util = make_shared_ptr<duckdb_mbedtls::MbedTlsWrapper::AESGCMStateMBEDTLSFactory>();
+		}
 		// encrypted parquet files start with the string "PARE"
 		writer->WriteData(const_data_ptr_cast("PARE"), 4);
 		// we only support this one for now, not "AES_GCM_CTR_V1"
@@ -403,7 +377,7 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
 
 	// populate root schema object
 	file_meta_data.schema[0].name = "duckdb_schema";
-	file_meta_data.schema[0].num_children = sql_types.size();
+	file_meta_data.schema[0].num_children = NumericCast<int32_t>(sql_types.size());
 	file_meta_data.schema[0].__isset.num_children = true;
 	file_meta_data.schema[0].repetition_type = duckdb_parquet::format::FieldRepetitionType::REQUIRED;
 	file_meta_data.schema[0].__isset.repetition_type = true;
@@ -413,8 +387,8 @@ ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalT
 
 	vector<string> schema_path;
 	for (idx_t i = 0; i < sql_types.size(); i++) {
-		column_writers.push_back(ColumnWriter::CreateWriterRecursive(file_meta_data.schema, *this, sql_types[i],
-		                                                             unique_names[i], schema_path, &field_ids));
+		column_writers.push_back(ColumnWriter::CreateWriterRecursive(
+		    context, file_meta_data.schema, *this, sql_types[i], unique_names[i], schema_path, &field_ids));
 	}
 }
 
@@ -428,8 +402,8 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 
 	// set up a new row group for this chunk collection
 	auto &row_group = result.row_group;
-	row_group.num_rows = buffer.Count();
-	row_group.total_byte_size = buffer.SizeInBytes();
+	row_group.num_rows = NumericCast<int64_t>(buffer.Count());
+	row_group.total_byte_size = NumericCast<int64_t>(buffer.SizeInBytes());
 	row_group.__isset.file_offset = true;
 
 	auto &states = result.states;
@@ -460,6 +434,11 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 			}
 		}
 
+		// Reserving these once at the start really pays off
+		for (auto &write_state : write_states) {
+			write_state->definition_levels.reserve(buffer.Count());
+		}
+
 		for (auto &chunk : buffer.Chunks({column_ids})) {
 			for (idx_t i = 0; i < next; i++) {
 				col_writers[i].get().Prepare(*write_states[i], nullptr, chunk.data[i], chunk.size());
@@ -526,7 +505,7 @@ void ParquetWriter::FlushRowGroup(PreparedRowGroup &prepared) {
 	if (states.empty()) {
 		throw InternalException("Attempting to flush a row group with no rows");
 	}
-	row_group.file_offset = writer->GetTotalWritten();
+	row_group.file_offset = NumericCast<int64_t>(writer->GetTotalWritten());
 	for (idx_t col_idx = 0; col_idx < states.size(); col_idx++) {
 		const auto &col_writer = column_writers[col_idx];
 		auto write_state = std::move(states[col_idx]);
@@ -555,7 +534,7 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 }
 
 void ParquetWriter::Finalize() {
-	auto start_offset = writer->GetTotalWritten();
+	const auto start_offset = writer->GetTotalWritten();
 	if (encryption_config) {
 		// Crypto metadata is written unencrypted
 		FileCryptoMetaData crypto_metadata;
@@ -565,6 +544,12 @@ void ParquetWriter::Finalize() {
 		crypto_metadata.__set_encryption_algorithm(alg);
 		crypto_metadata.write(protocol.get());
 	}
+
+	// Add geoparquet metadata to the file metadata
+	if (geoparquet_data) {
+		geoparquet_data->Write(file_meta_data);
+	}
+
 	Write(file_meta_data);
 
 	writer->Write<uint32_t>(writer->GetTotalWritten() - start_offset);
@@ -578,8 +563,15 @@ void ParquetWriter::Finalize() {
 	}
 
 	// flush to disk
-	writer->Sync();
+	writer->Close();
 	writer.reset();
 }
 
+GeoParquetFileMetadata &ParquetWriter::GetGeoParquetData() {
+	if (!geoparquet_data) {
+		geoparquet_data = make_uniq<GeoParquetFileMetadata>();
+	}
+	return *geoparquet_data;
+}
+
 } // namespace duckdb

package/src/duckdb/extension/parquet/serialize_parquet.cpp

@@ -71,6 +71,7 @@ void ParquetOptions::Serialize(Serializer &serializer) const {
 	serializer.WriteProperty<MultiFileReaderOptions>(102, "file_options", file_options);
 	serializer.WritePropertyWithDefault<vector<ParquetColumnDefinition>>(103, "schema", schema);
 	serializer.WritePropertyWithDefault<shared_ptr<ParquetEncryptionConfig>>(104, "encryption_config", encryption_config, nullptr);
+	serializer.WritePropertyWithDefault<bool>(105, "debug_use_openssl", debug_use_openssl, true);
 }
 
 ParquetOptions ParquetOptions::Deserialize(Deserializer &deserializer) {
@@ -79,7 +80,8 @@ ParquetOptions ParquetOptions::Deserialize(Deserializer &deserializer) {
 	deserializer.ReadPropertyWithDefault<bool>(101, "file_row_number", result.file_row_number);
 	deserializer.ReadProperty<MultiFileReaderOptions>(102, "file_options", result.file_options);
 	deserializer.ReadPropertyWithDefault<vector<ParquetColumnDefinition>>(103, "schema", result.schema);
-	deserializer.ReadPropertyWithDefault<shared_ptr<ParquetEncryptionConfig>>(104, "encryption_config", result.encryption_config, nullptr);
+	deserializer.ReadPropertyWithExplicitDefault<shared_ptr<ParquetEncryptionConfig>>(104, "encryption_config", result.encryption_config, nullptr);
+	deserializer.ReadPropertyWithExplicitDefault<bool>(105, "debug_use_openssl", result.debug_use_openssl, true);
 	return result;
 }
 

package/src/duckdb/extension/parquet/zstd_file_system.cpp

@@ -100,7 +100,7 @@ void ZstdStreamWrapper::Write(CompressedFile &file, StreamData &sd, data_ptr_t u
 			sd.out_buff_start = sd.out_buff.get();
 		}
 		uncompressed_data += input_consumed;
-		remaining -= input_consumed;
+		remaining -= UnsafeNumericCast<int64_t>(input_consumed);
 	}
 }
 
@@ -160,6 +160,10 @@ public:
 		Initialize(write);
 	}
 
+	FileCompressionType GetFileCompressionType() override {
+		return FileCompressionType::ZSTD;
+	}
+
 	ZStdFileSystem zstd_fs;
 };
 

package/src/duckdb/src/catalog/catalog.cpp

@@ -315,7 +315,6 @@ struct CatalogEntryLookup {
 // Generic
 //===--------------------------------------------------------------------===//
 void Catalog::DropEntry(ClientContext &context, DropInfo &info) {
-	ModifyCatalog();
 	if (info.type == CatalogType::SCHEMA_ENTRY) {
 		// DROP SCHEMA
 		DropSchema(context, info);
@@ -364,7 +363,7 @@ SimilarCatalogEntry Catalog::SimilarEntryInSchemas(ClientContext &context, const
 			// no similar entry found
 			continue;
 		}
-		if (!result.Found() || result.distance > entry.distance) {
+		if (!result.Found() || result.score < entry.score) {
 			result = entry;
 			result.schema = &schema;
 		}
@@ -563,10 +562,19 @@ CatalogException Catalog::CreateMissingEntryException(ClientContext &context, co
 	reference_set_t<SchemaCatalogEntry> unseen_schemas;
 	auto &db_manager = DatabaseManager::Get(context);
 	auto databases = db_manager.GetDatabases(context);
+	auto &config = DBConfig::GetConfig(context);
+
+	auto max_schema_count = config.options.catalog_error_max_schemas;
 	for (auto database : databases) {
+		if (unseen_schemas.size() >= max_schema_count) {
+			break;
+		}
 		auto &catalog = database.get().GetCatalog();
 		auto current_schemas = catalog.GetAllSchemas(context);
 		for (auto &current_schema : current_schemas) {
+			if (unseen_schemas.size() >= max_schema_count) {
+				break;
+			}
 			unseen_schemas.insert(current_schema.get());
 		}
 	}
@@ -626,9 +634,12 @@ CatalogException Catalog::CreateMissingEntryException(ClientContext &context, co
 		return CatalogException(error_message);
 	}
 
+	// entries in other schemas get a penalty
+	// however, if there is an exact match in another schema, we will always show it
+	static constexpr const double UNSEEN_PENALTY = 0.2;
 	auto unseen_entry = SimilarEntryInSchemas(context, entry_name, type, unseen_schemas);
 	string did_you_mean;
-	if (unseen_entry.Found() && unseen_entry.distance < entry.distance) {
+	if (unseen_entry.Found() && (unseen_entry.score == 1.0 || unseen_entry.score - UNSEEN_PENALTY > entry.score)) {
 		// the closest matching entry requires qualification as it is not in the default search path
 		// check how to minimally qualify this entry
 		auto catalog_name = unseen_entry.schema->catalog.GetName();
@@ -884,8 +895,6 @@ vector<reference<SchemaCatalogEntry>> Catalog::GetAllSchemas(ClientContext &cont
 }
 
 void Catalog::Alter(CatalogTransaction transaction, AlterInfo &info) {
-	ModifyCatalog();
-
 	if (transaction.HasContext()) {
 		auto lookup =
 		    LookupEntry(transaction.GetContext(), info.GetCatalogType(), info.schema, info.name, info.if_not_found);
@@ -910,17 +919,6 @@ vector<MetadataBlockInfo> Catalog::GetMetadataInfo(ClientContext &context) {
 void Catalog::Verify() {
 }
 
-//===--------------------------------------------------------------------===//
-// Catalog Version
-//===--------------------------------------------------------------------===//
-idx_t Catalog::GetCatalogVersion() {
-	return GetDatabase().GetDatabaseManager().catalog_version;
-}
-
-idx_t Catalog::ModifyCatalog() {
-	return GetDatabase().GetDatabaseManager().ModifyCatalog();
-}
-
 bool Catalog::IsSystemCatalog() const {
 	return db.IsSystem();
 }

package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp

@@ -1,6 +1,7 @@
 #include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
 
 #include "duckdb/storage/data_table.hpp"
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 
 namespace duckdb {
 
@@ -12,28 +13,30 @@ IndexDataTableInfo::~IndexDataTableInfo() {
 	if (!info) {
 		return;
 	}
+	// FIXME: this should happen differently.
 	info->GetIndexes().RemoveIndex(index_name);
 }
 
-DuckIndexEntry::DuckIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &info)
-    : IndexCatalogEntry(catalog, schema, info) {
+DuckIndexEntry::DuckIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &create_info,
+                               TableCatalogEntry &table_p)
+    : IndexCatalogEntry(catalog, schema, create_info), initial_index_size(0) {
+	auto &table = table_p.Cast<DuckTableEntry>();
+	auto &storage = table.GetStorage();
+	info = make_shared_ptr<IndexDataTableInfo>(storage.GetDataTableInfo(), name);
+}
+
+DuckIndexEntry::DuckIndexEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateIndexInfo &create_info,
+                               shared_ptr<IndexDataTableInfo> storage_info)
+    : IndexCatalogEntry(catalog, schema, create_info), info(std::move(storage_info)), initial_index_size(0) {
 }
 
 unique_ptr<CatalogEntry> DuckIndexEntry::Copy(ClientContext &context) const {
 	auto info_copy = GetInfo();
 	auto &cast_info = info_copy->Cast<CreateIndexInfo>();
 
-	auto result = make_uniq<DuckIndexEntry>(catalog, schema, cast_info);
-	result->info = info;
+	auto result = make_uniq<DuckIndexEntry>(catalog, schema, cast_info, info);
 	result->initial_index_size = initial_index_size;
 
-	for (auto &expr : expressions) {
-		result->expressions.push_back(expr->Copy());
-	}
-	for (auto &expr : parsed_expressions) {
-		result->parsed_expressions.push_back(expr->Copy());
-	}
-
 	return std::move(result);
 }
 

package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp

@@ -1,40 +1,44 @@
 #include "duckdb/catalog/catalog_entry/duck_schema_entry.hpp"
-#include "duckdb/catalog/default/default_functions.hpp"
-#include "duckdb/catalog/default/default_types.hpp"
-#include "duckdb/catalog/default/default_views.hpp"
+
+#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/collate_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/copy_function_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/duck_index_entry.hpp"
+#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
 #include "duckdb/catalog/catalog_entry/pragma_function_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/sequence_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
 #include "duckdb/catalog/catalog_entry/view_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
-#include "duckdb/catalog/catalog_entry/duck_table_entry.hpp"
-#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
+#include "duckdb/catalog/default/default_functions.hpp"
+#include "duckdb/catalog/default/default_table_functions.hpp"
+#include "duckdb/catalog/default/default_types.hpp"
+#include "duckdb/catalog/default/default_views.hpp"
 #include "duckdb/catalog/dependency_list.hpp"
-#include "duckdb/planner/constraints/bound_foreign_key_constraint.hpp"
+#include "duckdb/main/attached_database.hpp"
+#include "duckdb/main/database.hpp"
 #include "duckdb/parser/constraints/foreign_key_constraint.hpp"
 #include "duckdb/parser/parsed_data/alter_table_info.hpp"
-#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp"
-#include "duckdb/storage/data_table.hpp"
-#include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
 #include "duckdb/parser/parsed_data/create_collation_info.hpp"
 #include "duckdb/parser/parsed_data/create_copy_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_index_info.hpp"
 #include "duckdb/parser/parsed_data/create_pragma_function_info.hpp"
+#include "duckdb/parser/parsed_data/create_scalar_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_schema_info.hpp"
 #include "duckdb/parser/parsed_data/create_sequence_info.hpp"
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
 #include "duckdb/parser/parsed_data/create_type_info.hpp"
 #include "duckdb/parser/parsed_data/create_view_info.hpp"
 #include "duckdb/parser/parsed_data/drop_info.hpp"
+#include "duckdb/planner/constraints/bound_foreign_key_constraint.hpp"
+#include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
+#include "duckdb/storage/data_table.hpp"
+#include "duckdb/transaction/duck_transaction.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
-#include "duckdb/main/attached_database.hpp"
 
 namespace duckdb {
 
@@ -78,7 +82,8 @@ static void LazyLoadIndexes(ClientContext &context, CatalogEntry &entry) {
 
 DuckSchemaEntry::DuckSchemaEntry(Catalog &catalog, CreateSchemaInfo &info)
     : SchemaCatalogEntry(catalog, info), tables(catalog, make_uniq<DefaultViewGenerator>(catalog, *this)),
-      indexes(catalog), table_functions(catalog), copy_functions(catalog), pragma_functions(catalog),
+      indexes(catalog), table_functions(catalog, make_uniq<DefaultTableFunctionGenerator>(catalog, *this)),
+      copy_functions(catalog), pragma_functions(catalog),
       functions(catalog, make_uniq<DefaultFunctionGenerator>(catalog, *this)), sequences(catalog), collations(catalog),
       types(catalog, make_uniq<DefaultTypeGenerator>(catalog, *this)) {
 }
@@ -125,6 +130,7 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::AddEntryInternal(CatalogTransaction
 				throw CatalogException("Existing object %s is of type %s, trying to replace with type %s", entry_name,
 				                       CatalogTypeToString(old_entry->type), CatalogTypeToString(entry_type));
 			}
+			OnDropEntry(transaction, *old_entry);
 			(void)set.DropEntry(transaction, entry_name, false, entry->internal);
 		}
 	}
@@ -240,7 +246,7 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::CreateIndex(CatalogTransaction trans
 		throw CatalogException("An index with the name " + info.index_name + " already exists!");
 	}
 
-	auto index = make_uniq<DuckIndexEntry>(catalog, *this, info);
+	auto index = make_uniq<DuckIndexEntry>(catalog, *this, info, table);
 	auto dependencies = index->dependencies;
 	return AddEntryInternal(transaction, std::move(index), info.on_conflict, dependencies);
 }
@@ -316,13 +322,14 @@ void DuckSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) {
 	// if this is a index or table with indexes, initialize any unknown index instances
 	LazyLoadIndexes(context, *existing_entry);
 
-	// if there is a foreign key constraint, get that information
 	vector<unique_ptr<AlterForeignKeyInfo>> fk_arrays;
 	if (existing_entry->type == CatalogType::TABLE_ENTRY) {
-		FindForeignKeyInformation(existing_entry->Cast<TableCatalogEntry>(), AlterForeignKeyType::AFT_DELETE,
-		                          fk_arrays);
+		// if there is a foreign key constraint, get that information
+		auto &table_entry = existing_entry->Cast<TableCatalogEntry>();
+		FindForeignKeyInformation(table_entry, AlterForeignKeyType::AFT_DELETE, fk_arrays);
 	}
 
+	OnDropEntry(transaction, *existing_entry);
 	if (!set.DropEntry(transaction, info.name, info.cascade, info.allow_drop_internal)) {
 		throw InternalException("Could not drop element because of an internal error");
 	}
@@ -334,6 +341,19 @@ void DuckSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) {
 	}
 }
 
+void DuckSchemaEntry::OnDropEntry(CatalogTransaction transaction, CatalogEntry &entry) {
+	if (!transaction.transaction) {
+		return;
+	}
+	if (entry.type != CatalogType::TABLE_ENTRY) {
+		return;
+	}
+	// if we have transaction local insertions for this table - clear them
+	auto &table_entry = entry.Cast<TableCatalogEntry>();
+	auto &local_storage = LocalStorage::Get(transaction.transaction->Cast<DuckTransaction>());
+	local_storage.DropTable(table_entry.GetStorage());
+}
+
 optional_ptr<CatalogEntry> DuckSchemaEntry::GetEntry(CatalogTransaction transaction, CatalogType type,
                                                      const string &name) {
 	return GetCatalogSet(type).GetEntry(transaction, name);