duckdb 1.1.4-dev13.0 → 1.1.4-dev14.0

package/src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -662,7 +662,9 @@ static double CalculateTypeSimilarity(const LogicalType &merged, const LogicalTy
 		}
 
 		// Only maps and structs can be merged into a map
-		D_ASSERT(type.id() == LogicalTypeId::STRUCT);
+		if (type.id() != LogicalTypeId::STRUCT) {
+			return -1;
+		}
 		return CalculateMapAndStructSimilarity(merged, type, false, max_depth, depth);
 	}
 	case LogicalTypeId::LIST: {

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -498,6 +498,7 @@ void BasicColumnWriter::BeginWrite(ColumnWriterState &state_p) {
 		hdr.data_page_header.repetition_level_encoding = Encoding::RLE;
 
 		write_info.temp_writer = make_uniq<MemoryStream>(
+		    Allocator::Get(writer.GetContext()),
 		    MaxValue<idx_t>(NextPowerOfTwo(page_info.estimated_page_size), MemoryStream::DEFAULT_INITIAL_CAPACITY));
 		write_info.write_count = page_info.empty_count;
 		write_info.max_write_count = page_info.row_count;
@@ -717,6 +718,7 @@ void BasicColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 	column_chunk.meta_data.total_compressed_size =
 	    UnsafeNumericCast<int64_t>(column_writer.GetTotalWritten() - start_offset);
 	column_chunk.meta_data.total_uncompressed_size = UnsafeNumericCast<int64_t>(total_uncompressed_size);
+	state.row_group.total_byte_size += column_chunk.meta_data.total_uncompressed_size;
 
 	if (state.bloom_filter) {
 		writer.BufferBloomFilter(state.col_idx, std::move(state.bloom_filter));
@@ -1173,7 +1175,7 @@ void WriteValue(DlbaEncoder &encoder, WriteStream &writer, const string_t &value
 
 // helpers to get size from strings
 template <class SRC>
-static constexpr idx_t GetDlbaStringSize(const SRC &src_value) {
+static idx_t GetDlbaStringSize(const SRC &src_value) {
 	return 0;
 }
 
@@ -1311,21 +1313,26 @@ public:
 
 		auto &state = state_p.Cast<StandardColumnWriterState<SRC>>();
 		if (state.dictionary.size() == 0 || state.dictionary.size() > writer.DictionarySizeLimit()) {
-			// If we aren't doing dictionary encoding, the following encodings are virtually always better than PLAIN
-			switch (type) {
-			case Type::type::INT32:
-			case Type::type::INT64:
-				state.encoding = Encoding::DELTA_BINARY_PACKED;
-				break;
-			case Type::type::BYTE_ARRAY:
-				state.encoding = Encoding::DELTA_LENGTH_BYTE_ARRAY;
-				break;
-			case Type::type::FLOAT:
-			case Type::type::DOUBLE:
-				state.encoding = Encoding::BYTE_STREAM_SPLIT;
-				break;
-			default:
+			if (writer.GetParquetVersion() == ParquetVersion::V1) {
+				// Can't do the cool stuff for V1
 				state.encoding = Encoding::PLAIN;
+			} else {
+				// If we aren't doing dictionary encoding, these encodings are virtually always better than PLAIN
+				switch (type) {
+				case Type::type::INT32:
+				case Type::type::INT64:
+					state.encoding = Encoding::DELTA_BINARY_PACKED;
+					break;
+				case Type::type::BYTE_ARRAY:
+					state.encoding = Encoding::DELTA_LENGTH_BYTE_ARRAY;
+					break;
+				case Type::type::FLOAT:
+				case Type::type::DOUBLE:
+					state.encoding = Encoding::BYTE_STREAM_SPLIT;
+					break;
+				default:
+					state.encoding = Encoding::PLAIN;
+				}
 			}
 			state.dictionary.clear();
 		}
@@ -1463,8 +1470,9 @@ public:
 		    make_uniq<ParquetBloomFilter>(state.dictionary.size(), writer.BloomFilterFalsePositiveRatio());
 
 		// first write the contents of the dictionary page to a temporary buffer
-		auto temp_writer = make_uniq<MemoryStream>(MaxValue<idx_t>(
-		    NextPowerOfTwo(state.dictionary.size() * sizeof(TGT)), MemoryStream::DEFAULT_INITIAL_CAPACITY));
+		auto temp_writer = make_uniq<MemoryStream>(
+		    Allocator::Get(writer.GetContext()), MaxValue<idx_t>(NextPowerOfTwo(state.dictionary.size() * sizeof(TGT)),
+		                                                         MemoryStream::DEFAULT_INITIAL_CAPACITY));
 		for (idx_t r = 0; r < values.size(); r++) {
 			const TGT target_value = OP::template Operation<SRC, TGT>(values[r]);
 			// update the statistics
@@ -1838,7 +1846,7 @@ public:
 		auto enum_count = EnumType::GetSize(enum_type);
 		auto string_values = FlatVector::GetData<string_t>(enum_values);
 		// first write the contents of the dictionary page to a temporary buffer
-		auto temp_writer = make_uniq<MemoryStream>();
+		auto temp_writer = make_uniq<MemoryStream>(Allocator::Get(writer.GetContext()));
 		for (idx_t r = 0; r < enum_count; r++) {
 			D_ASSERT(!FlatVector::IsNull(enum_values, r));
 			// update the statistics

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -216,12 +216,6 @@ private:
 	void PrepareRowGroupBuffer(ParquetReaderScanState &state, idx_t out_col_idx);
 	LogicalType DeriveLogicalType(const SchemaElement &s_ele);
 
-	template <typename... Args>
-	std::runtime_error FormatException(const string fmt_str, Args... params) {
-		return std::runtime_error("Failed to read Parquet file \"" + file_name +
-		                          "\": " + StringUtil::Format(fmt_str, params...));
-	}
-
 private:
 	unique_ptr<FileHandle> file_handle;
 };

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -68,13 +68,19 @@ struct ParquetBloomFilterEntry {
 	idx_t column_idx;
 };
 
+enum class ParquetVersion : uint8_t {
+	V1 = 1, //! Excludes DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, BYTE_STREAM_SPLIT
+	V2 = 2, //! Includes the encodings above
+};
+
 class ParquetWriter {
 public:
 	ParquetWriter(ClientContext &context, FileSystem &fs, string file_name, vector<LogicalType> types,
 	              vector<string> names, duckdb_parquet::CompressionCodec::type codec, ChildFieldIDs field_ids,
 	              const vector<pair<string, string>> &kv_metadata,
 	              shared_ptr<ParquetEncryptionConfig> encryption_config, idx_t dictionary_size_limit,
-	              double bloom_filter_false_positive_ratio, int64_t compression_level, bool debug_use_openssl);
+	              double bloom_filter_false_positive_ratio, int64_t compression_level, bool debug_use_openssl,
+	              ParquetVersion parquet_version);
 
 public:
 	void PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result);
@@ -85,6 +91,9 @@ public:
 	static duckdb_parquet::Type::type DuckDBTypeToParquetType(const LogicalType &duckdb_type);
 	static void SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::SchemaElement &schema_ele);
 
+	ClientContext &GetContext() {
+		return context;
+	}
 	duckdb_apache::thrift::protocol::TProtocol *GetProtocol() {
 		return protocol.get();
 	}
@@ -117,6 +126,9 @@ public:
 		lock_guard<mutex> glock(lock);
 		return file_meta_data.row_groups.size();
 	}
+	ParquetVersion GetParquetVersion() const {
+		return parquet_version;
+	}
 
 	uint32_t Write(const duckdb_apache::thrift::TBase &object);
 	uint32_t WriteData(const const_data_ptr_t buffer, const uint32_t buffer_size);
@@ -129,6 +141,7 @@ public:
 	void BufferBloomFilter(idx_t col_idx, unique_ptr<ParquetBloomFilter> bloom_filter);
 
 private:
+	ClientContext &context;
 	string file_name;
 	vector<LogicalType> sql_types;
 	vector<string> column_names;
@@ -140,6 +153,7 @@ private:
 	int64_t compression_level;
 	bool debug_use_openssl;
 	shared_ptr<EncryptionUtil> encryption_util;
+	ParquetVersion parquet_version;
 
 	unique_ptr<BufferedFileWriter> writer;
 	std::shared_ptr<duckdb_apache::thrift::protocol::TProtocol> protocol;

package/src/duckdb/extension/parquet/include/resizable_buffer.hpp

@@ -98,6 +98,7 @@ public:
 		}
 		if (new_size > alloc_len) {
 			alloc_len = NextPowerOfTwo(new_size);
+			allocated_data.Reset(); // Have to reset before allocating new buffer (otherwise we use ~2x the memory)
 			allocated_data = allocator.Allocate(alloc_len);
 			ptr = allocated_data.get();
 		}

package/src/duckdb/extension/parquet/parquet_extension.cpp

@@ -203,6 +203,9 @@ struct ParquetWriteBindData : public TableFunctionData {
 	ChildFieldIDs field_ids;
 	//! The compression level, higher value is more
 	int64_t compression_level = ZStdFileSystem::DefaultCompressionLevel();
+
+	//! Which encodings to include when writing
+	ParquetVersion parquet_version = ParquetVersion::V1;
 };
 
 struct ParquetWriteGlobalState : public GlobalFunctionData {
@@ -371,6 +374,7 @@ public:
 		table_function.named_parameters["explicit_cardinality"] = LogicalType::UBIGINT;
 		table_function.named_parameters["schema"] = LogicalTypeId::ANY;
 		table_function.named_parameters["encryption_config"] = LogicalTypeId::ANY;
+		table_function.named_parameters["parquet_version"] = LogicalType::VARCHAR;
 		table_function.get_partition_data = ParquetScanGetPartitionData;
 		table_function.serialize = ParquetScanSerialize;
 		table_function.deserialize = ParquetScanDeserialize;
@@ -1289,6 +1293,15 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
 			}
 			bind_data->compression_level = val;
 			compression_level_set = true;
+		} else if (loption == "parquet_version") {
+			const auto roption = StringUtil::Upper(option.second[0].ToString());
+			if (roption == "V1") {
+				bind_data->parquet_version = ParquetVersion::V1;
+			} else if (roption == "V2") {
+				bind_data->parquet_version = ParquetVersion::V2;
+			} else {
+				throw BinderException("Expected parquet_version 'V1' or 'V2'");
+			}
 		} else {
 			throw NotImplementedException("Unrecognized option for PARQUET: %s", option.first.c_str());
 		}
@@ -1319,7 +1332,7 @@ unique_ptr<GlobalFunctionData> ParquetWriteInitializeGlobal(ClientContext &conte
 	    context, fs, file_path, parquet_bind.sql_types, parquet_bind.column_names, parquet_bind.codec,
 	    parquet_bind.field_ids.Copy(), parquet_bind.kv_metadata, parquet_bind.encryption_config,
 	    parquet_bind.dictionary_size_limit, parquet_bind.bloom_filter_false_positive_ratio,
-	    parquet_bind.compression_level, parquet_bind.debug_use_openssl);
+	    parquet_bind.compression_level, parquet_bind.debug_use_openssl, parquet_bind.parquet_version);
 	return std::move(global_state);
 }
 
@@ -1424,6 +1437,29 @@ duckdb_parquet::CompressionCodec::type EnumUtil::FromString<duckdb_parquet::Comp
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 
+template <>
+const char *EnumUtil::ToChars<ParquetVersion>(ParquetVersion value) {
+	switch (value) {
+	case ParquetVersion::V1:
+		return "V1";
+	case ParquetVersion::V2:
+		return "V2";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
+	}
+}
+
+template <>
+ParquetVersion EnumUtil::FromString<ParquetVersion>(const char *value) {
+	if (StringUtil::Equals(value, "V1")) {
+		return ParquetVersion::V1;
+	}
+	if (StringUtil::Equals(value, "V2")) {
+		return ParquetVersion::V2;
+	}
+	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
+}
+
 static optional_idx SerializeCompressionLevel(const int64_t compression_level) {
 	return compression_level < 0 ? NumericLimits<idx_t>::Maximum() - NumericCast<idx_t>(AbsValue(compression_level))
 	                             : NumericCast<idx_t>(compression_level);
@@ -1455,13 +1491,25 @@ static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bin
 	                                                                         bind_data.encryption_config, nullptr);
 
 	// 108 was dictionary_compression_ratio_threshold, but was deleted
+
+	// To avoid doubly defining the default values in both ParquetWriteBindData and here,
+	// and possibly making a mistake, we just get the values from ParquetWriteBindData.
+	// We have to std::move them, otherwise MSVC will complain that it's not a "const T &&"
 	const auto compression_level = SerializeCompressionLevel(bind_data.compression_level);
 	D_ASSERT(DeserializeCompressionLevel(compression_level) == bind_data.compression_level);
-	serializer.WritePropertyWithDefault<optional_idx>(109, "compression_level", compression_level);
-	serializer.WriteProperty(110, "row_groups_per_file", bind_data.row_groups_per_file);
-	serializer.WriteProperty(111, "debug_use_openssl", bind_data.debug_use_openssl);
-	serializer.WriteProperty(112, "dictionary_size_limit", bind_data.dictionary_size_limit);
-	serializer.WriteProperty(113, "bloom_filter_false_positive_ratio", bind_data.bloom_filter_false_positive_ratio);
+	ParquetWriteBindData default_value;
+	serializer.WritePropertyWithDefault(109, "compression_level", compression_level);
+	serializer.WritePropertyWithDefault(110, "row_groups_per_file", bind_data.row_groups_per_file,
+	                                    default_value.row_groups_per_file);
+	serializer.WritePropertyWithDefault(111, "debug_use_openssl", bind_data.debug_use_openssl,
+	                                    default_value.debug_use_openssl);
+	serializer.WritePropertyWithDefault(112, "dictionary_size_limit", bind_data.dictionary_size_limit,
+	                                    default_value.dictionary_size_limit);
+	serializer.WritePropertyWithDefault(113, "bloom_filter_false_positive_ratio",
+	                                    bind_data.bloom_filter_false_positive_ratio,
+	                                    default_value.bloom_filter_false_positive_ratio);
+	serializer.WritePropertyWithDefault(114, "parquet_version", bind_data.parquet_version,
+	                                    default_value.parquet_version);
 }
 
 static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserializer, CopyFunction &function) {
@@ -1473,21 +1521,25 @@ static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserialize
 	data->row_group_size_bytes = deserializer.ReadProperty<idx_t>(104, "row_group_size_bytes");
 	data->kv_metadata = deserializer.ReadProperty<vector<pair<string, string>>>(105, "kv_metadata");
 	data->field_ids = deserializer.ReadProperty<ChildFieldIDs>(106, "field_ids");
-	deserializer.ReadPropertyWithExplicitDefault<shared_ptr<ParquetEncryptionConfig>>(107, "encryption_config",
-	                                                                                  data->encryption_config, nullptr);
+	deserializer.ReadPropertyWithExplicitDefault<shared_ptr<ParquetEncryptionConfig>>(
+	    107, "encryption_config", data->encryption_config, std::move(ParquetWriteBindData().encryption_config));
 	deserializer.ReadDeletedProperty<double>(108, "dictionary_compression_ratio_threshold");
 
 	optional_idx compression_level;
 	deserializer.ReadPropertyWithDefault<optional_idx>(109, "compression_level", compression_level);
 	data->compression_level = DeserializeCompressionLevel(compression_level);
 	D_ASSERT(SerializeCompressionLevel(data->compression_level) == compression_level);
-	data->row_groups_per_file =
-	    deserializer.ReadPropertyWithExplicitDefault<optional_idx>(110, "row_groups_per_file", optional_idx::Invalid());
-	data->debug_use_openssl = deserializer.ReadPropertyWithExplicitDefault<bool>(111, "debug_use_openssl", true);
-	data->dictionary_size_limit =
-	    deserializer.ReadPropertyWithExplicitDefault<idx_t>(112, "dictionary_size_limit", data->row_group_size / 10);
-	data->bloom_filter_false_positive_ratio =
-	    deserializer.ReadPropertyWithExplicitDefault<double>(113, "bloom_filter_false_positive_ratio", 0.01);
+	ParquetWriteBindData default_value;
+	data->row_groups_per_file = deserializer.ReadPropertyWithExplicitDefault<optional_idx>(
+	    110, "row_groups_per_file", default_value.row_groups_per_file);
+	data->debug_use_openssl =
+	    deserializer.ReadPropertyWithExplicitDefault<bool>(111, "debug_use_openssl", default_value.debug_use_openssl);
+	data->dictionary_size_limit = deserializer.ReadPropertyWithExplicitDefault<idx_t>(
+	    112, "dictionary_size_limit", default_value.dictionary_size_limit);
+	data->bloom_filter_false_positive_ratio = deserializer.ReadPropertyWithExplicitDefault<double>(
+	    113, "bloom_filter_false_positive_ratio", default_value.bloom_filter_false_positive_ratio);
+	data->parquet_version =
+	    deserializer.ReadPropertyWithExplicitDefault(114, "parquet_version", default_value.parquet_version);
 
 	return std::move(data);
 }

package/src/duckdb/extension/parquet/parquet_reader.cpp

@@ -492,7 +492,8 @@ void ParquetReader::InitializeSchema(ClientContext &context) {
 	}
 	// check if we like this schema
 	if (file_meta_data->schema.size() < 2) {
-		throw FormatException("Need at least one non-root column in the file");
+		throw InvalidInputException("Failed to read Parquet file '%s': Need at least one non-root column in the file",
+		                            file_name);
 	}
 	root_reader = CreateReader(context);
 	auto &root_type = root_reader->Type();
@@ -778,12 +779,13 @@ void ParquetReader::PrepareRowGroupBuffer(ParquetReaderScanState &state, idx_t c
 			FilterPropagateResult prune_result;
 			// TODO we might not have stats but STILL a bloom filter so move this up
 			// check the bloom filter if present
-			if (!column_reader.Type().IsNested() &&
+			bool is_generated_column = column_reader.FileIdx() >= group.columns.size();
+			if (!column_reader.Type().IsNested() && !is_generated_column &&
 			    ParquetStatisticsUtils::BloomFilterSupported(column_reader.Type().id()) &&
 			    ParquetStatisticsUtils::BloomFilterExcludes(filter, group.columns[column_reader.FileIdx()].meta_data,
 			                                                *state.thrift_file_proto, allocator)) {
 				prune_result = FilterPropagateResult::FILTER_ALWAYS_FALSE;
-			} else if (column_reader.Type().id() == LogicalTypeId::VARCHAR &&
+			} else if (column_reader.Type().id() == LogicalTypeId::VARCHAR && !is_generated_column &&
 			           group.columns[column_reader.FileIdx()].meta_data.statistics.__isset.min_value &&
 			           group.columns[column_reader.FileIdx()].meta_data.statistics.__isset.max_value) {
 

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -321,12 +321,12 @@ ParquetWriter::ParquetWriter(ClientContext &context, FileSystem &fs, string file
                              const vector<pair<string, string>> &kv_metadata,
                              shared_ptr<ParquetEncryptionConfig> encryption_config_p, idx_t dictionary_size_limit_p,
                              double bloom_filter_false_positive_ratio_p, int64_t compression_level_p,
-                             bool debug_use_openssl_p)
-    : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
-      field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)),
-      dictionary_size_limit(dictionary_size_limit_p),
+                             bool debug_use_openssl_p, ParquetVersion parquet_version)
+    : context(context), file_name(std::move(file_name_p)), sql_types(std::move(types_p)),
+      column_names(std::move(names_p)), codec(codec), field_ids(std::move(field_ids_p)),
+      encryption_config(std::move(encryption_config_p)), dictionary_size_limit(dictionary_size_limit_p),
       bloom_filter_false_positive_ratio(bloom_filter_false_positive_ratio_p), compression_level(compression_level_p),
-      debug_use_openssl(debug_use_openssl_p) {
+      debug_use_openssl(debug_use_openssl_p), parquet_version(parquet_version) {
 
 	// initialize the file writer
 	writer = make_uniq<BufferedFileWriter>(fs, file_name.c_str(),
@@ -395,7 +395,6 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 	// set up a new row group for this chunk collection
 	auto &row_group = result.row_group;
 	row_group.num_rows = NumericCast<int64_t>(buffer.Count());
-	row_group.total_byte_size = NumericCast<int64_t>(buffer.SizeInBytes());
 	row_group.__isset.file_offset = true;
 
 	auto &states = result.states;

package/src/duckdb/src/catalog/catalog.cpp

@@ -426,7 +426,12 @@ vector<CatalogSearchEntry> GetCatalogEntries(CatalogEntryRetriever &retriever, c
 			entries.emplace_back(catalog, schema_name);
 		}
 		if (entries.empty()) {
-			entries.emplace_back(catalog, DEFAULT_SCHEMA);
+			auto catalog_entry = Catalog::GetCatalogEntry(context, catalog);
+			if (catalog_entry) {
+				entries.emplace_back(catalog, catalog_entry->GetDefaultSchema());
+			} else {
+				entries.emplace_back(catalog, DEFAULT_SCHEMA);
+			}
 		}
 	} else {
 		// specific catalog and schema provided
@@ -687,7 +692,7 @@ CatalogException Catalog::CreateMissingEntryException(CatalogEntryRetriever &ret
 	// however, if there is an exact match in another schema, we will always show it
 	static constexpr const double UNSEEN_PENALTY = 0.2;
 	auto unseen_entries = SimilarEntriesInSchemas(context, entry_name, type, unseen_schemas);
-	vector<string> suggestions;
+	set<string> suggestions;
 	if (!unseen_entries.empty() && (unseen_entries[0].score == 1.0 || unseen_entries[0].score - UNSEEN_PENALTY >
 	                                                                      (entries.empty() ? 0.0 : entries[0].score))) {
 		// the closest matching entry requires qualification as it is not in the default search path
@@ -698,19 +703,19 @@ CatalogException Catalog::CreateMissingEntryException(CatalogEntryRetriever &ret
 			bool qualify_database;
 			bool qualify_schema;
 			FindMinimalQualification(retriever, catalog_name, schema_name, qualify_database, qualify_schema);
-			suggestions.push_back(unseen_entry.GetQualifiedName(qualify_database, qualify_schema));
+			auto qualified_name = unseen_entry.GetQualifiedName(qualify_database, qualify_schema);
+			suggestions.insert(qualified_name);
 		}
 	} else if (!entries.empty()) {
 		for (auto &entry : entries) {
-			suggestions.push_back(entry.name);
+			suggestions.insert(entry.name);
 		}
 	}
 
 	string did_you_mean;
-	std::sort(suggestions.begin(), suggestions.end());
 	if (suggestions.size() > 2) {
-		auto last = suggestions.back();
-		suggestions.pop_back();
+		string last = *suggestions.rbegin();
+		suggestions.erase(last);
 		did_you_mean = StringUtil::Join(suggestions, ", ") + ", or " + last;
 	} else {
 		did_you_mean = StringUtil::Join(suggestions, " or ");
@@ -968,12 +973,16 @@ optional_ptr<SchemaCatalogEntry> Catalog::GetSchema(CatalogEntryRetriever &retri
 			// skip if it is not an attached database
 			continue;
 		}
-		auto on_not_found = i + 1 == entries.size() ? if_not_found : OnEntryNotFound::RETURN_NULL;
+		const auto on_not_found = i + 1 == entries.size() ? if_not_found : OnEntryNotFound::RETURN_NULL;
 		auto result = catalog->GetSchema(retriever.GetContext(), schema_name, on_not_found, error_context);
 		if (result) {
 			return result;
 		}
 	}
+	// Catalog has not been found.
+	if (if_not_found == OnEntryNotFound::THROW_EXCEPTION) {
+		throw CatalogException(error_context, "Catalog with name %s does not exist!", catalog_name);
+	}
 	return nullptr;
 }
 
@@ -1073,6 +1082,10 @@ optional_ptr<DependencyManager> Catalog::GetDependencyManager() {
 	return nullptr;
 }
 
+string Catalog::GetDefaultSchema() const {
+	return DEFAULT_SCHEMA;
+}
+
 //! Whether this catalog has a default table. Catalogs with a default table can be queries by their catalog name
 bool Catalog::HasDefaultTable() const {
 	return !default_table.empty();

package/src/duckdb/src/catalog/catalog_search_path.cpp

@@ -165,7 +165,7 @@ void CatalogSearchPath::Set(vector<CatalogSearchEntry> new_paths, CatalogSetPath
 		if (path.catalog.empty()) {
 			auto catalog = Catalog::GetCatalogEntry(context, path.schema);
 			if (catalog) {
-				auto schema = catalog->GetSchema(context, DEFAULT_SCHEMA, OnEntryNotFound::RETURN_NULL);
+				auto schema = catalog->GetSchema(context, catalog->GetDefaultSchema(), OnEntryNotFound::RETURN_NULL);
 				if (schema) {
 					path.catalog = std::move(path.schema);
 					path.schema = schema->name;
@@ -205,6 +205,22 @@ string CatalogSearchPath::GetDefaultSchema(const string &catalog) {
 	return DEFAULT_SCHEMA;
 }
 
+string CatalogSearchPath::GetDefaultSchema(ClientContext &context, const string &catalog) {
+	for (auto &path : paths) {
+		if (path.catalog == TEMP_CATALOG) {
+			continue;
+		}
+		if (StringUtil::CIEquals(path.catalog, catalog)) {
+			return path.schema;
+		}
+	}
+	auto catalog_entry = Catalog::GetCatalogEntry(context, catalog);
+	if (catalog_entry) {
+		return catalog_entry->GetDefaultSchema();
+	}
+	return DEFAULT_SCHEMA;
+}
+
 string CatalogSearchPath::GetDefaultCatalog(const string &schema) {
 	if (DefaultSchemaGenerator::IsDefaultSchema(schema)) {
 		return SYSTEM_CATALOG;

package/src/duckdb/src/catalog/catalog_set.cpp

@@ -353,7 +353,7 @@ bool CatalogSet::AlterEntry(CatalogTransaction transaction, const string &name,
 	// push the old entry in the undo buffer for this transaction
 	if (transaction.transaction) {
 		// serialize the AlterInfo into a temporary buffer
-		MemoryStream stream;
+		MemoryStream stream(Allocator::Get(*transaction.db));
 		BinarySerializer serializer(stream);
 		serializer.Begin();
 		serializer.WriteProperty(100, "column_name", alter_info.GetColumnName());

package/src/duckdb/src/catalog/default/default_functions.cpp

@@ -163,9 +163,6 @@ static const DefaultMacro internal_macros[] = {
 
 	// date functions
 	{DEFAULT_SCHEMA, "date_add", {"date", "interval", nullptr}, {{nullptr, nullptr}}, "date + interval"},
-	{DEFAULT_SCHEMA, "current_date", {nullptr}, {{nullptr, nullptr}}, "current_timestamp::DATE"},
-	{DEFAULT_SCHEMA, "today", {nullptr}, {{nullptr, nullptr}}, "current_timestamp::DATE"},
-	{DEFAULT_SCHEMA, "get_current_time", {nullptr}, {{nullptr, nullptr}}, "current_timestamp::TIMETZ"},
 
 	// regexp functions
 	{DEFAULT_SCHEMA, "regexp_split_to_table", {"text", "pattern", nullptr}, {{nullptr, nullptr}}, "unnest(string_split_regex(text, pattern))"},

package/src/duckdb/src/catalog/dependency_list.cpp

@@ -61,6 +61,13 @@ LogicalDependency::LogicalDependency(CatalogEntry &entry) {
 	}
 }
 
+LogicalDependency::LogicalDependency(optional_ptr<Catalog> catalog_p, CatalogEntryInfo entry_p, string catalog_str)
+    : entry(std::move(entry_p)), catalog(std::move(catalog_str)) {
+	if (catalog_p) {
+		catalog = catalog_p->GetName();
+	}
+}
+
 bool LogicalDependency::operator==(const LogicalDependency &other) const {
 	return other.entry.name == entry.name && other.entry.schema == entry.schema && other.entry.type == entry.type;
 }

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -53,7 +53,6 @@ AdbcStatusCode duckdb_adbc_init(int version, void *driver, struct AdbcError *err
 	adbc_driver->ConnectionGetInfo = duckdb_adbc::ConnectionGetInfo;
 	adbc_driver->StatementGetParameterSchema = duckdb_adbc::StatementGetParameterSchema;
 	adbc_driver->ConnectionGetTableSchema = duckdb_adbc::ConnectionGetTableSchema;
-	adbc_driver->StatementSetSubstraitPlan = duckdb_adbc::StatementSetSubstraitPlan;
 	return ADBC_STATUS_OK;
 }
 
@@ -70,7 +69,6 @@ struct DuckDBAdbcStatementWrapper {
 	ArrowArrayStream ingestion_stream;
 	IngestionMode ingestion_mode = IngestionMode::CREATE;
 	bool temporary_table = false;
-	uint8_t *substrait_plan;
 	uint64_t plan_length;
 };
 
@@ -157,36 +155,6 @@ AdbcStatusCode DatabaseNew(struct AdbcDatabase *database, struct AdbcError *erro
 	return CheckResult(res, error, "Failed to allocate");
 }
 
-AdbcStatusCode StatementSetSubstraitPlan(struct AdbcStatement *statement, const uint8_t *plan, size_t length,
-                                         struct AdbcError *error) {
-	if (!statement) {
-		SetError(error, "Statement is not set");
-		return ADBC_STATUS_INVALID_ARGUMENT;
-	}
-	if (!plan) {
-		SetError(error, "Substrait Plan is not set");
-		return ADBC_STATUS_INVALID_ARGUMENT;
-	}
-	if (length == 0) {
-		SetError(error, "Can't execute plan with size = 0");
-		return ADBC_STATUS_INVALID_ARGUMENT;
-	}
-	auto wrapper = static_cast<DuckDBAdbcStatementWrapper *>(statement->private_data);
-	if (wrapper->ingestion_stream.release) {
-		// Release any resources currently held by the ingestion stream before we overwrite it
-		wrapper->ingestion_stream.release(&wrapper->ingestion_stream);
-		wrapper->ingestion_stream.release = nullptr;
-	}
-	if (wrapper->statement) {
-		duckdb_destroy_prepare(&wrapper->statement);
-		wrapper->statement = nullptr;
-	}
-	wrapper->substrait_plan = static_cast<uint8_t *>(malloc(sizeof(uint8_t) * length));
-	wrapper->plan_length = length;
-	memcpy(wrapper->substrait_plan, plan, length);
-	return ADBC_STATUS_OK;
-}
-
 AdbcStatusCode DatabaseSetOption(struct AdbcDatabase *database, const char *key, const char *value,
                                  struct AdbcError *error) {
 	if (!database) {
@@ -677,7 +645,6 @@ AdbcStatusCode StatementNew(struct AdbcConnection *connection, struct AdbcStatem
 	statement_wrapper->ingestion_stream.release = nullptr;
 	statement_wrapper->ingestion_table_name = nullptr;
 	statement_wrapper->db_schema = nullptr;
-	statement_wrapper->substrait_plan = nullptr;
 	statement_wrapper->temporary_table = false;
 
 	statement_wrapper->ingestion_mode = IngestionMode::CREATE;
@@ -709,10 +676,6 @@ AdbcStatusCode StatementRelease(struct AdbcStatement *statement, struct AdbcErro
 		free(wrapper->db_schema);
 		wrapper->db_schema = nullptr;
 	}
-	if (wrapper->substrait_plan) {
-		free(wrapper->substrait_plan);
-		wrapper->substrait_plan = nullptr;
-	}
 	free(statement->private_data);
 	statement->private_data = nullptr;
 	return ADBC_STATUS_OK;
@@ -805,25 +768,7 @@ AdbcStatusCode StatementExecuteQuery(struct AdbcStatement *statement, struct Arr
 	if (has_stream && to_table) {
 		return IngestToTableFromBoundStream(wrapper, error);
 	}
-	if (wrapper->substrait_plan != nullptr) {
-		auto plan_str = std::string(reinterpret_cast<const char *>(wrapper->substrait_plan), wrapper->plan_length);
-		duckdb::vector<duckdb::Value> params;
-		params.emplace_back(duckdb::Value::BLOB_RAW(plan_str));
-		duckdb::unique_ptr<duckdb::QueryResult> query_result;
-		try {
-			query_result = reinterpret_cast<duckdb::Connection *>(wrapper->connection)
-			                   ->TableFunction("from_substrait", params)
-			                   ->Execute();
-		} catch (duckdb::Exception &e) {
-			std::string error_msg = "It was not possible to execute substrait query. " + std::string(e.what());
-			SetError(error, error_msg);
-			return ADBC_STATUS_INVALID_ARGUMENT;
-		}
-		auto arrow_wrapper = new duckdb::ArrowResultWrapper();
-		arrow_wrapper->result =
-		    duckdb::unique_ptr_cast<duckdb::QueryResult, duckdb::MaterializedQueryResult>(std::move(query_result));
-		wrapper->result = reinterpret_cast<duckdb_arrow>(arrow_wrapper);
-	} else if (has_stream) {
+	if (has_stream) {
 		// A stream was bound to the statement, use that to bind parameters
 		duckdb::unique_ptr<duckdb::QueryResult> result;
 		ArrowArrayStream stream = wrapper->ingestion_stream;

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -73,14 +73,15 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
 	InitializeChild(root_holder.nested_children.back()[0], root_holder);
 	child.children = &root_holder.nested_children_ptr.back()[0];
 	child.children[0]->name = "entries";
+	child.children[0]->flags = 0; // Set the 'entries' field to non-nullable
 	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options, context);
 }
 
 bool SetArrowExtension(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
                        ClientContext &context) {
 	auto &config = DBConfig::GetConfig(context);
-	if (config.HasArrowExtension(type.id())) {
-		auto arrow_extension = config.GetArrowExtension(type.id());
+	if (config.HasArrowExtension(type)) {
+		auto arrow_extension = config.GetArrowExtension(type);
 		arrow_extension.PopulateArrowSchema(root_holder, child, type, context, arrow_extension);
 		return true;
 	}