duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/common/enums/date_part_specifier.cpp

@@ -65,6 +65,8 @@ bool TryGetDatePartSpecifier(const string &specifier_p, DatePartSpecifier &resul
 		result = DatePartSpecifier::TIMEZONE_HOUR;
 	} else if (specifier == "timezone_minute") {
 		result = DatePartSpecifier::TIMEZONE_MINUTE;
+	} else if (specifier == "julian" || specifier == "jd") {
+		result = DatePartSpecifier::JULIAN_DAY;
 	} else {
 		return false;
 	}

package/src/duckdb/src/common/enums/logical_operator_type.cpp

@@ -22,6 +22,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "ANY_JOIN";
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 		return "ASOF_JOIN";
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
+		return "DEPENDENT_JOIN";
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		return "COMPARISON_JOIN";
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
@@ -86,6 +88,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "VACUUM";
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
 		return "REC_CTE";
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+		return "CTE";
 	case LogicalOperatorType::LOGICAL_CTE_REF:
 		return "CTE_SCAN";
 	case LogicalOperatorType::LOGICAL_SHOW:

package/src/duckdb/src/common/enums/optimizer_type.cpp

@@ -25,6 +25,8 @@ static DefaultOptimizerType internal_optimizer_types[] = {
     {"common_aggregate", OptimizerType::COMMON_AGGREGATE},
     {"column_lifetime", OptimizerType::COLUMN_LIFETIME},
     {"top_n", OptimizerType::TOP_N},
+    {"compressed_materialization", OptimizerType::COMPRESSED_MATERIALIZATION},
+    {"duplicate_groups", OptimizerType::DUPLICATE_GROUPS},
     {"reorder_filter", OptimizerType::REORDER_FILTER},
     {"extension", OptimizerType::EXTENSION},
     {nullptr, OptimizerType::INVALID}};

package/src/duckdb/src/common/enums/physical_operator_type.cpp

@@ -103,8 +103,12 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "VACUUM";
 	case PhysicalOperatorType::RECURSIVE_CTE:
 		return "REC_CTE";
+	case PhysicalOperatorType::CTE:
+		return "CTE";
 	case PhysicalOperatorType::RECURSIVE_CTE_SCAN:
 		return "REC_CTE_SCAN";
+	case PhysicalOperatorType::CTE_SCAN:
+		return "CTE_SCAN";
 	case PhysicalOperatorType::EXPRESSION_SCAN:
 		return "EXPRESSION_SCAN";
 	case PhysicalOperatorType::ALTER:

package/src/duckdb/src/common/exception.cpp

@@ -62,11 +62,11 @@ string Exception::ConstructMessageRecursive(const string &msg, std::vector<Excep
 #ifdef DEBUG
 	// Verify that we have the required amount of values for the message
 	idx_t parameter_count = 0;
-	for (idx_t i = 0; i < msg.size(); i++) {
+	for (idx_t i = 0; i + 1 < msg.size(); i++) {
 		if (msg[i] != '%') {
 			continue;
 		}
-		if (i < msg.size() && msg[i + 1] == '%') {
+		if (msg[i + 1] == '%') {
 			i++;
 			continue;
 		}
@@ -158,8 +158,12 @@ string Exception::ExceptionTypeToString(ExceptionType type) {
 		return "Parameter Not Allowed";
 	case ExceptionType::DEPENDENCY:
 		return "Dependency";
+	case ExceptionType::MISSING_EXTENSION:
+		return "Missing Extension";
 	case ExceptionType::HTTP:
 		return "HTTP";
+	case ExceptionType::AUTOLOAD:
+		return "Extension Autoloading";
 	default:
 		return "Unknown";
 	}
@@ -225,6 +229,8 @@ void Exception::ThrowAsTypeWithMessage(ExceptionType type, const string &message
 	case ExceptionType::HTTP: {
 		original->AsHTTPException().Throw();
 	}
+	case ExceptionType::MISSING_EXTENSION:
+		throw MissingExtensionException(message);
 	default:
 		throw Exception(type, message);
 	}
@@ -347,6 +353,13 @@ MissingExtensionException::MissingExtensionException(const string &msg)
     : Exception(ExceptionType::MISSING_EXTENSION, msg) {
 }
 
+AutoloadException::AutoloadException(const string &extension_name, Exception &e)
+    : Exception(ExceptionType::AUTOLOAD,
+                "An error occurred while trying to automatically install the required extension '" + extension_name +
+                    "':\n" + e.RawMessage()),
+      wrapped_exception(e) {
+}
+
 SerializationException::SerializationException(const string &msg) : Exception(ExceptionType::SERIALIZATION, msg) {
 }
 

package/src/duckdb/src/common/extra_type_info.cpp

@@ -0,0 +1,315 @@
+#include "duckdb/common/extra_type_info.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/enum_util.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp"
+#include "duckdb/common/string_map_set.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Extra Type Info
+//===--------------------------------------------------------------------===//
+ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type) : type(type) {
+}
+ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type, string alias) : type(type), alias(std::move(alias)) {
+}
+ExtraTypeInfo::~ExtraTypeInfo() {
+}
+
+bool ExtraTypeInfo::Equals(ExtraTypeInfo *other_p) const {
+	if (type == ExtraTypeInfoType::INVALID_TYPE_INFO || type == ExtraTypeInfoType::STRING_TYPE_INFO ||
+	    type == ExtraTypeInfoType::GENERIC_TYPE_INFO) {
+		if (!other_p) {
+			if (!alias.empty()) {
+				return false;
+			}
+			//! We only need to compare aliases when both types have them in this case
+			return true;
+		}
+		if (alias != other_p->alias) {
+			return false;
+		}
+		return true;
+	}
+	if (!other_p) {
+		return false;
+	}
+	if (type != other_p->type) {
+		return false;
+	}
+	return alias == other_p->alias && EqualsInternal(other_p);
+}
+
+bool ExtraTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	// Do nothing
+	return true;
+}
+
+//===--------------------------------------------------------------------===//
+// Decimal Type Info
+//===--------------------------------------------------------------------===//
+DecimalTypeInfo::DecimalTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::DECIMAL_TYPE_INFO) {
+}
+
+DecimalTypeInfo::DecimalTypeInfo(uint8_t width_p, uint8_t scale_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::DECIMAL_TYPE_INFO), width(width_p), scale(scale_p) {
+	D_ASSERT(width_p >= scale_p);
+}
+
+bool DecimalTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<DecimalTypeInfo>();
+	return width == other.width && scale == other.scale;
+}
+
+//===--------------------------------------------------------------------===//
+// String Type Info
+//===--------------------------------------------------------------------===//
+StringTypeInfo::StringTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::STRING_TYPE_INFO) {
+}
+
+StringTypeInfo::StringTypeInfo(string collation_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::STRING_TYPE_INFO), collation(std::move(collation_p)) {
+}
+
+bool StringTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	// collation info has no impact on equality
+	return true;
+}
+
+//===--------------------------------------------------------------------===//
+// List Type Info
+//===--------------------------------------------------------------------===//
+ListTypeInfo::ListTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::LIST_TYPE_INFO) {
+}
+
+ListTypeInfo::ListTypeInfo(LogicalType child_type_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::LIST_TYPE_INFO), child_type(std::move(child_type_p)) {
+}
+
+bool ListTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<ListTypeInfo>();
+	return child_type == other.child_type;
+}
+
+//===--------------------------------------------------------------------===//
+// Struct Type Info
+//===--------------------------------------------------------------------===//
+StructTypeInfo::StructTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::STRUCT_TYPE_INFO) {
+}
+
+StructTypeInfo::StructTypeInfo(child_list_t<LogicalType> child_types_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::STRUCT_TYPE_INFO), child_types(std::move(child_types_p)) {
+}
+
+bool StructTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<StructTypeInfo>();
+	return child_types == other.child_types;
+}
+
+//===--------------------------------------------------------------------===//
+// Aggregate State Type Info
+//===--------------------------------------------------------------------===//
+AggregateStateTypeInfo::AggregateStateTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO) {
+}
+
+AggregateStateTypeInfo::AggregateStateTypeInfo(aggregate_state_t state_type_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO), state_type(std::move(state_type_p)) {
+}
+
+bool AggregateStateTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<AggregateStateTypeInfo>();
+	return state_type.function_name == other.state_type.function_name &&
+	       state_type.return_type == other.state_type.return_type &&
+	       state_type.bound_argument_types == other.state_type.bound_argument_types;
+}
+
+//===--------------------------------------------------------------------===//
+// User Type Info
+//===--------------------------------------------------------------------===//
+UserTypeInfo::UserTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::USER_TYPE_INFO) {
+}
+
+UserTypeInfo::UserTypeInfo(string name_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::USER_TYPE_INFO), user_type_name(std::move(name_p)) {
+}
+
+bool UserTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<UserTypeInfo>();
+	return other.user_type_name == user_type_name;
+}
+
+//===--------------------------------------------------------------------===//
+// Enum Type Info
+//===--------------------------------------------------------------------===//
+PhysicalType EnumTypeInfo::DictType(idx_t size) {
+	if (size <= NumericLimits<uint8_t>::Maximum()) {
+		return PhysicalType::UINT8;
+	} else if (size <= NumericLimits<uint16_t>::Maximum()) {
+		return PhysicalType::UINT16;
+	} else if (size <= NumericLimits<uint32_t>::Maximum()) {
+		return PhysicalType::UINT32;
+	} else {
+		throw InternalException("Enum size must be lower than " + std::to_string(NumericLimits<uint32_t>::Maximum()));
+	}
+}
+
+template <class T>
+struct EnumTypeInfoTemplated : public EnumTypeInfo {
+	explicit EnumTypeInfoTemplated(Vector &values_insert_order_p, idx_t size_p)
+	    : EnumTypeInfo(values_insert_order_p, size_p) {
+		D_ASSERT(values_insert_order_p.GetType().InternalType() == PhysicalType::VARCHAR);
+
+		UnifiedVectorFormat vdata;
+		values_insert_order.ToUnifiedFormat(size_p, vdata);
+
+		auto data = UnifiedVectorFormat::GetData<string_t>(vdata);
+		for (idx_t i = 0; i < size_p; i++) {
+			auto idx = vdata.sel->get_index(i);
+			if (!vdata.validity.RowIsValid(idx)) {
+				throw InternalException("Attempted to create ENUM type with NULL value");
+			}
+			if (values.count(data[idx]) > 0) {
+				throw InvalidInputException("Attempted to create ENUM type with duplicate value %s",
+				                            data[idx].GetString());
+			}
+			values[data[idx]] = i;
+		}
+	}
+
+	static shared_ptr<EnumTypeInfoTemplated> Deserialize(Deserializer &deserializer, uint32_t size) {
+		Vector values_insert_order(LogicalType::VARCHAR, size);
+		auto strings = FlatVector::GetData<string_t>(values_insert_order);
+
+		deserializer.ReadList(201, "values", [&](Deserializer::List &list, idx_t i) {
+			strings[i] = StringVector::AddStringOrBlob(values_insert_order, list.ReadElement<string>());
+		});
+		return make_shared<EnumTypeInfoTemplated>(values_insert_order, size);
+	}
+
+	const string_map_t<T> &GetValues() const {
+		return values;
+	}
+
+	EnumTypeInfoTemplated(const EnumTypeInfoTemplated &) = delete;
+	EnumTypeInfoTemplated &operator=(const EnumTypeInfoTemplated &) = delete;
+
+private:
+	string_map_t<T> values;
+};
+
+EnumTypeInfo::EnumTypeInfo(Vector &values_insert_order_p, idx_t dict_size_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::ENUM_TYPE_INFO), values_insert_order(values_insert_order_p),
+      dict_type(EnumDictType::VECTOR_DICT), dict_size(dict_size_p) {
+}
+
+const EnumDictType &EnumTypeInfo::GetEnumDictType() const {
+	return dict_type;
+}
+
+const Vector &EnumTypeInfo::GetValuesInsertOrder() const {
+	return values_insert_order;
+}
+
+const idx_t &EnumTypeInfo::GetDictSize() const {
+	return dict_size;
+}
+
+LogicalType EnumTypeInfo::CreateType(Vector &ordered_data, idx_t size) {
+	// Generate EnumTypeInfo
+	shared_ptr<ExtraTypeInfo> info;
+	auto enum_internal_type = EnumTypeInfo::DictType(size);
+	switch (enum_internal_type) {
+	case PhysicalType::UINT8:
+		info = make_shared<EnumTypeInfoTemplated<uint8_t>>(ordered_data, size);
+		break;
+	case PhysicalType::UINT16:
+		info = make_shared<EnumTypeInfoTemplated<uint16_t>>(ordered_data, size);
+		break;
+	case PhysicalType::UINT32:
+		info = make_shared<EnumTypeInfoTemplated<uint32_t>>(ordered_data, size);
+		break;
+	default:
+		throw InternalException("Invalid Physical Type for ENUMs");
+	}
+	// Generate Actual Enum Type
+	return LogicalType(LogicalTypeId::ENUM, info);
+}
+
+template <class T>
+int64_t TemplatedGetPos(const string_map_t<T> &map, const string_t &key) {
+	auto it = map.find(key);
+	if (it == map.end()) {
+		return -1;
+	}
+	return it->second;
+}
+
+int64_t EnumType::GetPos(const LogicalType &type, const string_t &key) {
+	auto info = type.AuxInfo();
+	switch (type.InternalType()) {
+	case PhysicalType::UINT8:
+		return TemplatedGetPos(info->Cast<EnumTypeInfoTemplated<uint8_t>>().GetValues(), key);
+	case PhysicalType::UINT16:
+		return TemplatedGetPos(info->Cast<EnumTypeInfoTemplated<uint16_t>>().GetValues(), key);
+	case PhysicalType::UINT32:
+		return TemplatedGetPos(info->Cast<EnumTypeInfoTemplated<uint32_t>>().GetValues(), key);
+	default:
+		throw InternalException("ENUM can only have unsigned integers (except UINT64) as physical types");
+	}
+}
+
+string_t EnumType::GetString(const LogicalType &type, idx_t pos) {
+	D_ASSERT(pos < EnumType::GetSize(type));
+	return FlatVector::GetData<string_t>(EnumType::GetValuesInsertOrder(type))[pos];
+}
+
+shared_ptr<ExtraTypeInfo> EnumTypeInfo::Deserialize(Deserializer &deserializer) {
+	auto values_count = deserializer.ReadProperty<idx_t>(200, "values_count");
+	auto enum_internal_type = EnumTypeInfo::DictType(values_count);
+	switch (enum_internal_type) {
+	case PhysicalType::UINT8:
+		return EnumTypeInfoTemplated<uint8_t>::Deserialize(deserializer, values_count);
+	case PhysicalType::UINT16:
+		return EnumTypeInfoTemplated<uint16_t>::Deserialize(deserializer, values_count);
+	case PhysicalType::UINT32:
+		return EnumTypeInfoTemplated<uint32_t>::Deserialize(deserializer, values_count);
+	default:
+		throw InternalException("Invalid Physical Type for ENUMs");
+	}
+}
+
+// Equalities are only used in enums with different catalog entries
+bool EnumTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<EnumTypeInfo>();
+	if (dict_type != other.dict_type) {
+		return false;
+	}
+	D_ASSERT(dict_type == EnumDictType::VECTOR_DICT);
+	// We must check if both enums have the same size
+	if (other.dict_size != dict_size) {
+		return false;
+	}
+	auto other_vector_ptr = FlatVector::GetData<string_t>(other.values_insert_order);
+	auto this_vector_ptr = FlatVector::GetData<string_t>(values_insert_order);
+
+	// Now we must check if all strings are the same
+	for (idx_t i = 0; i < dict_size; i++) {
+		if (!Equals::Operation(other_vector_ptr[i], this_vector_ptr[i])) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void EnumTypeInfo::Serialize(Serializer &serializer) const {
+	ExtraTypeInfo::Serialize(serializer);
+
+	// Enums are special in that we serialize their values as a list instead of dumping the whole vector
+	auto strings = FlatVector::GetData<string_t>(values_insert_order);
+	serializer.WriteProperty(200, "values_count", dict_size);
+	serializer.WriteList(201, "values", dict_size,
+	                     [&](Serializer::List &list, idx_t i) { list.WriteElement(strings[i]); });
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/file_buffer.cpp

@@ -5,7 +5,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/helper.hpp"
-
+#include "duckdb/storage/storage_info.hpp"
 #include <cstring>
 
 namespace duckdb {
@@ -68,7 +68,7 @@ FileBuffer::MemoryRequirement FileBuffer::CalculateMemory(uint64_t user_size) {
 		result.alloc_size = user_size;
 	} else {
 		result.header_size = Storage::BLOCK_HEADER_SIZE;
-		result.alloc_size = AlignValue<uint32_t, Storage::SECTOR_SIZE>(result.header_size + user_size);
+		result.alloc_size = AlignValue<idx_t, Storage::SECTOR_SIZE>(result.header_size + user_size);
 	}
 	return result;
 }

package/src/duckdb/src/common/file_system.cpp

@@ -23,6 +23,14 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+
+#ifdef __MVS__
+#define _XOPEN_SOURCE_EXTENDED 1
+#include <sys/resource.h>
+// enjoy - https://reviews.llvm.org/D92110
+#define PATH_MAX _XOPEN_PATH_MAX
+#endif
+
 #else
 #include <string>
 #include <sysinfoapi.h>
@@ -63,11 +71,11 @@ string FileSystem::GetEnvVariable(const string &name) {
 }
 
 bool FileSystem::IsPathAbsolute(const string &path) {
-	auto path_separator = FileSystem::PathSeparator();
+	auto path_separator = PathSeparator(path);
 	return PathMatched(path, path_separator);
 }
 
-string FileSystem::PathSeparator() {
+string FileSystem::PathSeparator(const string &path) {
 	return "/";
 }
 
@@ -79,7 +87,14 @@ void FileSystem::SetWorkingDirectory(const string &path) {
 
 idx_t FileSystem::GetAvailableMemory() {
 	errno = 0;
+
+#ifdef __MVS__
+	struct rlimit limit;
+	int rlim_rc = getrlimit(RLIMIT_AS, &limit);
+	idx_t max_memory = MinValue<idx_t>(limit.rlim_max, UINTPTR_MAX);
+#else
 	idx_t max_memory = MinValue<idx_t>((idx_t)sysconf(_SC_PHYS_PAGES) * (idx_t)sysconf(_SC_PAGESIZE), UINTPTR_MAX);
+#endif
 	if (errno != 0) {
 		return DConstants::INVALID_INDEX;
 	}
@@ -152,7 +167,7 @@ string FileSystem::NormalizeAbsolutePath(const string &path) {
 	return result;
 }
 
-string FileSystem::PathSeparator() {
+string FileSystem::PathSeparator(const string &path) {
 	return "\\";
 }
 
@@ -195,11 +210,11 @@ string FileSystem::GetWorkingDirectory() {
 
 string FileSystem::JoinPath(const string &a, const string &b) {
 	// FIXME: sanitize paths
-	return a + PathSeparator() + b;
+	return a + PathSeparator(a) + b;
 }
 
 string FileSystem::ConvertSeparators(const string &path) {
-	auto separator_str = PathSeparator();
+	auto separator_str = PathSeparator(path);
 	char separator = separator_str[0];
 	if (separator == '/') {
 		// on unix-based systems we only accept / as a separator
@@ -214,7 +229,7 @@ string FileSystem::ExtractName(const string &path) {
 		return string();
 	}
 	auto normalized_path = ConvertSeparators(path);
-	auto sep = PathSeparator();
+	auto sep = PathSeparator(path);
 	auto splits = StringUtil::Split(normalized_path, sep);
 	D_ASSERT(!splits.empty());
 	return splits.back();
@@ -370,6 +385,10 @@ void FileSystem::UnregisterSubSystem(const string &name) {
 	throw NotImplementedException("%s: Can't unregister a sub system on a non-virtual file system", GetName());
 }
 
+void FileSystem::SetDisabledFileSystems(const vector<string> &names) {
+	throw NotImplementedException("%s: Can't disable file systems on a non-virtual file system", GetName());
+}
+
 vector<string> FileSystem::ListSubSystems() {
 	throw NotImplementedException("%s: Can't list sub systems on a non-virtual file system", GetName());
 }
@@ -378,16 +397,31 @@ bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
 
+static string LookupExtensionForPattern(const string &pattern) {
+	for (const auto &entry : EXTENSION_FILE_PREFIXES) {
+		if (StringUtil::StartsWith(pattern, entry.name)) {
+			return entry.extension;
+		}
+	}
+	return "";
+}
+
 vector<string> FileSystem::GlobFiles(const string &pattern, ClientContext &context, FileGlobOptions options) {
 	auto result = Glob(pattern);
 	if (result.empty()) {
-		string required_extension;
-		if (FileSystem::IsRemoteFile(pattern)) {
-			required_extension = "httpfs";
-		}
+		string required_extension = LookupExtensionForPattern(pattern);
 		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
-			// an extension is required to read this file but it is not loaded - try to load it
-			ExtensionHelper::LoadExternalExtension(context, required_extension);
+			auto &dbconfig = DBConfig::GetConfig(context);
+			if (!ExtensionHelper::CanAutoloadExtension(required_extension) ||
+			    !dbconfig.options.autoload_known_extensions) {
+				auto error_message =
+				    "File " + pattern + " requires the extension " + required_extension + " to be loaded";
+				error_message =
+				    ExtensionHelper::AddExtensionInstallHintToErrorMsg(context, error_message, required_extension);
+				throw MissingExtensionException(error_message);
+			}
+			// an extension is required to read this file, but it is not loaded - try to load it
+			ExtensionHelper::AutoLoadExtension(context, required_extension);
 			// success! glob again
 			// check the extension is loaded just in case to prevent an infinite loop here
 			if (!context.db->ExtensionIsLoaded(required_extension)) {

package/src/duckdb/src/common/filename_pattern.cpp

@@ -24,7 +24,7 @@ void FilenamePattern::SetFilenamePattern(const string &pattern) {
 	_pos = std::min(_pos, (idx_t)_base.length());
 }
 
-string FilenamePattern::CreateFilename(const FileSystem &fs, const string &path, const string &extension,
+string FilenamePattern::CreateFilename(FileSystem &fs, const string &path, const string &extension,
                                        idx_t offset) const {
 	string result(_base);
 	string replacement;

package/src/duckdb/src/common/gzip_file_system.cpp

@@ -145,6 +145,13 @@ void MiniZStreamWrapper::Initialize(CompressedFile &file, bool write) {
 bool MiniZStreamWrapper::Read(StreamData &sd) {
 	// Handling for the concatenated files
 	if (sd.refresh) {
+		auto available = (uint32_t)(sd.in_buff_end - sd.in_buff_start);
+		if (available <= GZIP_FOOTER_SIZE) {
+			// Only footer is available so we just close and return finished
+			Close();
+			return true;
+		}
+
 		sd.refresh = false;
 		auto body_ptr = sd.in_buff_start + GZIP_FOOTER_SIZE;
 		uint8_t gzip_hdr[GZIP_HEADER_MINSIZE];
@@ -200,18 +207,6 @@ bool MiniZStreamWrapper::Read(StreamData &sd) {
 
 	// if stream ended, deallocate inflator
 	if (ret == duckdb_miniz::MZ_STREAM_END) {
-		// Last read from file done and remaining bytes only for footer or less
-		if ((sd.in_buff_end < sd.in_buff.get() + sd.in_buf_size) && mz_stream_ptr->avail_in <= GZIP_FOOTER_SIZE) {
-			Close();
-			return true;
-		}
-		if (mz_stream_ptr->avail_in > GZIP_FOOTER_SIZE) {
-			// Definitely not concatenated gzip
-			if (*(sd.in_buff_start + GZIP_FOOTER_SIZE) != 0x1F) {
-				Close();
-				return true;
-			}
-		}
 		// Concatenated GZIP potentially coming up - refresh input buffer
 		sd.refresh = true;
 	}

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "re2/re2.h"
 
@@ -86,12 +87,15 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
 //		 currently, only expressions that cannot be evaluated during pushdown are removed.
 void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
                                               vector<unique_ptr<Expression>> &filters,
-                                              unordered_map<string, column_t> &column_map, idx_t table_index,
+                                              unordered_map<string, column_t> &column_map, LogicalGet &get,
                                               bool hive_enabled, bool filename_enabled) {
+
 	vector<string> pruned_files;
 	vector<bool> have_preserved_filter(filters.size(), false);
 	vector<unique_ptr<Expression>> pruned_filters;
+	unordered_set<idx_t> filters_applied_to_files;
 	duckdb_re2::RE2 regex(REGEX_STRING);
+	auto table_index = get.table_index;
 
 	if ((!filename_enabled && !hive_enabled) || filters.empty()) {
 		return;
@@ -121,11 +125,11 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
 			} else if (!result_value.GetValue<bool>()) {
 				// filter evaluates to false
 				should_prune_file = true;
-			}
-
-			// Use filter combiner to determine that this filter makes
-			if (!should_prune_file && combiner.AddFilter(std::move(filter_copy)) == FilterResult::UNSATISFIABLE) {
-				should_prune_file = true;
+				// convert the filter to a table filter.
+				if (filters_applied_to_files.find(j) == filters_applied_to_files.end()) {
+					get.extra_info.file_filters += filter->ToString();
+					filters_applied_to_files.insert(j);
+				}
 			}
 		}