duckdb 0.8.2-dev33.0 → 0.8.2-dev3300.0

package/src/duckdb/src/common/enums/date_part_specifier.cpp

@@ -65,6 +65,8 @@ bool TryGetDatePartSpecifier(const string &specifier_p, DatePartSpecifier &resul
 		result = DatePartSpecifier::TIMEZONE_HOUR;
 	} else if (specifier == "timezone_minute") {
 		result = DatePartSpecifier::TIMEZONE_MINUTE;
+	} else if (specifier == "julian" || specifier == "jd") {
+		result = DatePartSpecifier::JULIAN_DAY;
 	} else {
 		return false;
 	}

package/src/duckdb/src/common/enums/logical_operator_type.cpp

@@ -22,6 +22,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "ANY_JOIN";
 	case LogicalOperatorType::LOGICAL_ASOF_JOIN:
 		return "ASOF_JOIN";
+	case LogicalOperatorType::LOGICAL_DEPENDENT_JOIN:
+		return "DEPENDENT_JOIN";
 	case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
 		return "COMPARISON_JOIN";
 	case LogicalOperatorType::LOGICAL_DELIM_JOIN:
@@ -86,6 +88,8 @@ string LogicalOperatorToString(LogicalOperatorType type) {
 		return "VACUUM";
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE:
 		return "REC_CTE";
+	case LogicalOperatorType::LOGICAL_MATERIALIZED_CTE:
+		return "CTE";
 	case LogicalOperatorType::LOGICAL_CTE_REF:
 		return "CTE_SCAN";
 	case LogicalOperatorType::LOGICAL_SHOW:

package/src/duckdb/src/common/enums/optimizer_type.cpp

@@ -25,6 +25,8 @@ static DefaultOptimizerType internal_optimizer_types[] = {
     {"common_aggregate", OptimizerType::COMMON_AGGREGATE},
     {"column_lifetime", OptimizerType::COLUMN_LIFETIME},
     {"top_n", OptimizerType::TOP_N},
+    {"compressed_materialization", OptimizerType::COMPRESSED_MATERIALIZATION},
+    {"duplicate_groups", OptimizerType::DUPLICATE_GROUPS},
     {"reorder_filter", OptimizerType::REORDER_FILTER},
     {"extension", OptimizerType::EXTENSION},
     {nullptr, OptimizerType::INVALID}};

package/src/duckdb/src/common/enums/physical_operator_type.cpp

@@ -103,8 +103,12 @@ string PhysicalOperatorToString(PhysicalOperatorType type) {
 		return "VACUUM";
 	case PhysicalOperatorType::RECURSIVE_CTE:
 		return "REC_CTE";
+	case PhysicalOperatorType::CTE:
+		return "CTE";
 	case PhysicalOperatorType::RECURSIVE_CTE_SCAN:
 		return "REC_CTE_SCAN";
+	case PhysicalOperatorType::CTE_SCAN:
+		return "CTE_SCAN";
 	case PhysicalOperatorType::EXPRESSION_SCAN:
 		return "EXPRESSION_SCAN";
 	case PhysicalOperatorType::ALTER:

package/src/duckdb/src/common/exception.cpp

@@ -62,11 +62,11 @@ string Exception::ConstructMessageRecursive(const string &msg, std::vector<Excep
 #ifdef DEBUG
 	// Verify that we have the required amount of values for the message
 	idx_t parameter_count = 0;
-	for (idx_t i = 0; i < msg.size(); i++) {
+	for (idx_t i = 0; i + 1 < msg.size(); i++) {
 		if (msg[i] != '%') {
 			continue;
 		}
-		if (i < msg.size() && msg[i + 1] == '%') {
+		if (msg[i + 1] == '%') {
 			i++;
 			continue;
 		}

package/src/duckdb/src/common/extra_type_info.cpp

@@ -0,0 +1,483 @@
+#include "duckdb/common/extra_type_info.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "duckdb/common/enum_util.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/type_catalog_entry.hpp"
+#include "duckdb/common/string_map_set.hpp"
+
+namespace duckdb {
+
+//===--------------------------------------------------------------------===//
+// Extra Type Info
+//===--------------------------------------------------------------------===//
+ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type) : type(type) {
+}
+ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type, string alias) : type(type), alias(std::move(alias)) {
+}
+ExtraTypeInfo::~ExtraTypeInfo() {
+}
+
+bool ExtraTypeInfo::Equals(ExtraTypeInfo *other_p) const {
+	if (type == ExtraTypeInfoType::INVALID_TYPE_INFO || type == ExtraTypeInfoType::STRING_TYPE_INFO ||
+	    type == ExtraTypeInfoType::GENERIC_TYPE_INFO) {
+		if (!other_p) {
+			if (!alias.empty()) {
+				return false;
+			}
+			//! We only need to compare aliases when both types have them in this case
+			return true;
+		}
+		if (alias != other_p->alias) {
+			return false;
+		}
+		return true;
+	}
+	if (!other_p) {
+		return false;
+	}
+	if (type != other_p->type) {
+		return false;
+	}
+	return alias == other_p->alias && EqualsInternal(other_p);
+}
+
+void ExtraTypeInfo::Serialize(FieldWriter &writer) const {
+}
+
+void ExtraTypeInfo::Serialize(ExtraTypeInfo *info, FieldWriter &writer) {
+	if (!info) {
+		writer.WriteField<ExtraTypeInfoType>(ExtraTypeInfoType::INVALID_TYPE_INFO);
+		writer.WriteString(string());
+	} else {
+		writer.WriteField<ExtraTypeInfoType>(info->type);
+		info->Serialize(writer);
+		writer.WriteString(info->alias);
+	}
+}
+
+shared_ptr<ExtraTypeInfo> ExtraTypeInfo::Deserialize(FieldReader &reader) {
+	auto type = reader.ReadRequired<ExtraTypeInfoType>();
+	shared_ptr<ExtraTypeInfo> extra_info;
+	switch (type) {
+	case ExtraTypeInfoType::INVALID_TYPE_INFO: {
+		auto alias = reader.ReadField<string>(string());
+		if (!alias.empty()) {
+			return make_shared<ExtraTypeInfo>(type, alias);
+		}
+		return nullptr;
+	}
+	case ExtraTypeInfoType::GENERIC_TYPE_INFO: {
+		extra_info = make_shared<ExtraTypeInfo>(type);
+	} break;
+	case ExtraTypeInfoType::DECIMAL_TYPE_INFO:
+		extra_info = DecimalTypeInfo::Deserialize(reader);
+		break;
+	case ExtraTypeInfoType::STRING_TYPE_INFO:
+		extra_info = StringTypeInfo::Deserialize(reader);
+		break;
+	case ExtraTypeInfoType::LIST_TYPE_INFO:
+		extra_info = ListTypeInfo::Deserialize(reader);
+		break;
+	case ExtraTypeInfoType::STRUCT_TYPE_INFO:
+		extra_info = StructTypeInfo::Deserialize(reader);
+		break;
+	case ExtraTypeInfoType::USER_TYPE_INFO:
+		extra_info = UserTypeInfo::Deserialize(reader);
+		break;
+	case ExtraTypeInfoType::ENUM_TYPE_INFO:
+		extra_info = EnumTypeInfo::Deserialize(reader);
+		break;
+	case ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO:
+		extra_info = AggregateStateTypeInfo::Deserialize(reader);
+		break;
+	default:
+		throw InternalException("Unimplemented type info in ExtraTypeInfo::Deserialize");
+	}
+	auto alias = reader.ReadField<string>(string());
+	extra_info->alias = alias;
+	return extra_info;
+}
+
+bool ExtraTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	// Do nothing
+	return true;
+}
+
+//===--------------------------------------------------------------------===//
+// Decimal Type Info
+//===--------------------------------------------------------------------===//
+DecimalTypeInfo::DecimalTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::DECIMAL_TYPE_INFO) {
+}
+
+DecimalTypeInfo::DecimalTypeInfo(uint8_t width_p, uint8_t scale_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::DECIMAL_TYPE_INFO), width(width_p), scale(scale_p) {
+	D_ASSERT(width_p >= scale_p);
+}
+
+void DecimalTypeInfo::Serialize(FieldWriter &writer) const {
+	writer.WriteField<uint8_t>(width);
+	writer.WriteField<uint8_t>(scale);
+}
+
+shared_ptr<ExtraTypeInfo> DecimalTypeInfo::Deserialize(FieldReader &reader) {
+	auto width = reader.ReadRequired<uint8_t>();
+	auto scale = reader.ReadRequired<uint8_t>();
+	return make_shared<DecimalTypeInfo>(width, scale);
+}
+
+bool DecimalTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<DecimalTypeInfo>();
+	return width == other.width && scale == other.scale;
+}
+
+//===--------------------------------------------------------------------===//
+// String Type Info
+//===--------------------------------------------------------------------===//
+StringTypeInfo::StringTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::STRING_TYPE_INFO) {
+}
+
+StringTypeInfo::StringTypeInfo(string collation_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::STRING_TYPE_INFO), collation(std::move(collation_p)) {
+}
+
+void StringTypeInfo::Serialize(FieldWriter &writer) const {
+	writer.WriteString(collation);
+}
+
+shared_ptr<ExtraTypeInfo> StringTypeInfo::Deserialize(FieldReader &reader) {
+	auto collation = reader.ReadRequired<string>();
+	return make_shared<StringTypeInfo>(std::move(collation));
+}
+
+bool StringTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	// collation info has no impact on equality
+	return true;
+}
+
+//===--------------------------------------------------------------------===//
+// List Type Info
+//===--------------------------------------------------------------------===//
+ListTypeInfo::ListTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::LIST_TYPE_INFO) {
+}
+
+ListTypeInfo::ListTypeInfo(LogicalType child_type_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::LIST_TYPE_INFO), child_type(std::move(child_type_p)) {
+}
+
+void ListTypeInfo::Serialize(FieldWriter &writer) const {
+	writer.WriteSerializable(child_type);
+}
+
+shared_ptr<ExtraTypeInfo> ListTypeInfo::Deserialize(FieldReader &reader) {
+	auto child_type = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
+	return make_shared<ListTypeInfo>(std::move(child_type));
+}
+
+bool ListTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<ListTypeInfo>();
+	return child_type == other.child_type;
+}
+
+//===--------------------------------------------------------------------===//
+// Struct Type Info
+//===--------------------------------------------------------------------===//
+StructTypeInfo::StructTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::STRUCT_TYPE_INFO) {
+}
+
+StructTypeInfo::StructTypeInfo(child_list_t<LogicalType> child_types_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::STRUCT_TYPE_INFO), child_types(std::move(child_types_p)) {
+}
+
+void StructTypeInfo::Serialize(FieldWriter &writer) const {
+	writer.WriteField<uint32_t>(child_types.size());
+	auto &serializer = writer.GetSerializer();
+	for (idx_t i = 0; i < child_types.size(); i++) {
+		serializer.WriteString(child_types[i].first);
+		child_types[i].second.Serialize(serializer);
+	}
+}
+
+shared_ptr<ExtraTypeInfo> StructTypeInfo::Deserialize(FieldReader &reader) {
+	child_list_t<LogicalType> child_list;
+	auto child_types_size = reader.ReadRequired<uint32_t>();
+	auto &source = reader.GetSource();
+	for (uint32_t i = 0; i < child_types_size; i++) {
+		auto name = source.Read<string>();
+		auto type = LogicalType::Deserialize(source);
+		child_list.emplace_back(std::move(name), std::move(type));
+	}
+	return make_shared<StructTypeInfo>(std::move(child_list));
+}
+
+bool StructTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<StructTypeInfo>();
+	return child_types == other.child_types;
+}
+
+//===--------------------------------------------------------------------===//
+// Aggregate State Type Info
+//===--------------------------------------------------------------------===//
+AggregateStateTypeInfo::AggregateStateTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO) {
+}
+
+AggregateStateTypeInfo::AggregateStateTypeInfo(aggregate_state_t state_type_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::AGGREGATE_STATE_TYPE_INFO), state_type(std::move(state_type_p)) {
+}
+
+void AggregateStateTypeInfo::Serialize(FieldWriter &writer) const {
+	auto &serializer = writer.GetSerializer();
+	writer.WriteString(state_type.function_name);
+	state_type.return_type.Serialize(serializer);
+	writer.WriteField<uint32_t>(state_type.bound_argument_types.size());
+	for (idx_t i = 0; i < state_type.bound_argument_types.size(); i++) {
+		state_type.bound_argument_types[i].Serialize(serializer);
+	}
+}
+
+shared_ptr<ExtraTypeInfo> AggregateStateTypeInfo::Deserialize(FieldReader &reader) {
+	auto &source = reader.GetSource();
+
+	auto function_name = reader.ReadRequired<string>();
+	auto return_type = LogicalType::Deserialize(source);
+	auto bound_argument_types_size = reader.ReadRequired<uint32_t>();
+	vector<LogicalType> bound_argument_types;
+
+	for (uint32_t i = 0; i < bound_argument_types_size; i++) {
+		auto type = LogicalType::Deserialize(source);
+		bound_argument_types.push_back(std::move(type));
+	}
+	return make_shared<AggregateStateTypeInfo>(
+	    aggregate_state_t(std::move(function_name), std::move(return_type), std::move(bound_argument_types)));
+}
+
+bool AggregateStateTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<AggregateStateTypeInfo>();
+	return state_type.function_name == other.state_type.function_name &&
+	       state_type.return_type == other.state_type.return_type &&
+	       state_type.bound_argument_types == other.state_type.bound_argument_types;
+}
+
+//===--------------------------------------------------------------------===//
+// User Type Info
+//===--------------------------------------------------------------------===//
+UserTypeInfo::UserTypeInfo() : ExtraTypeInfo(ExtraTypeInfoType::USER_TYPE_INFO) {
+}
+
+UserTypeInfo::UserTypeInfo(string name_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::USER_TYPE_INFO), user_type_name(std::move(name_p)) {
+}
+
+void UserTypeInfo::Serialize(FieldWriter &writer) const {
+	writer.WriteString(user_type_name);
+}
+
+shared_ptr<ExtraTypeInfo> UserTypeInfo::Deserialize(FieldReader &reader) {
+	auto enum_name = reader.ReadRequired<string>();
+	return make_shared<UserTypeInfo>(std::move(enum_name));
+}
+
+bool UserTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<UserTypeInfo>();
+	return other.user_type_name == user_type_name;
+}
+
+//===--------------------------------------------------------------------===//
+// Enum Type Info
+//===--------------------------------------------------------------------===//
+PhysicalType EnumTypeInfo::DictType(idx_t size) {
+	if (size <= NumericLimits<uint8_t>::Maximum()) {
+		return PhysicalType::UINT8;
+	} else if (size <= NumericLimits<uint16_t>::Maximum()) {
+		return PhysicalType::UINT16;
+	} else if (size <= NumericLimits<uint32_t>::Maximum()) {
+		return PhysicalType::UINT32;
+	} else {
+		throw InternalException("Enum size must be lower than " + std::to_string(NumericLimits<uint32_t>::Maximum()));
+	}
+}
+
+template <class T>
+struct EnumTypeInfoTemplated : public EnumTypeInfo {
+	explicit EnumTypeInfoTemplated(Vector &values_insert_order_p, idx_t size_p)
+	    : EnumTypeInfo(values_insert_order_p, size_p) {
+		D_ASSERT(values_insert_order_p.GetType().InternalType() == PhysicalType::VARCHAR);
+
+		UnifiedVectorFormat vdata;
+		values_insert_order.ToUnifiedFormat(size_p, vdata);
+
+		auto data = UnifiedVectorFormat::GetData<string_t>(vdata);
+		for (idx_t i = 0; i < size_p; i++) {
+			auto idx = vdata.sel->get_index(i);
+			if (!vdata.validity.RowIsValid(idx)) {
+				throw InternalException("Attempted to create ENUM type with NULL value");
+			}
+			if (values.count(data[idx]) > 0) {
+				throw InvalidInputException("Attempted to create ENUM type with duplicate value %s",
+				                            data[idx].GetString());
+			}
+			values[data[idx]] = i;
+		}
+	}
+
+	static shared_ptr<EnumTypeInfoTemplated> Deserialize(FieldReader &reader, uint32_t size) {
+		Vector values_insert_order(LogicalType::VARCHAR, size);
+		values_insert_order.Deserialize(size, reader.GetSource());
+		return make_shared<EnumTypeInfoTemplated>(values_insert_order, size);
+	}
+
+	static shared_ptr<EnumTypeInfoTemplated> FormatDeserialize(FormatDeserializer &source, uint32_t size) {
+		Vector values_insert_order(LogicalType::VARCHAR, size);
+		values_insert_order.FormatDeserialize(source, size);
+		return make_shared<EnumTypeInfoTemplated>(values_insert_order, size);
+	}
+
+	const string_map_t<T> &GetValues() const {
+		return values;
+	}
+
+	EnumTypeInfoTemplated(const EnumTypeInfoTemplated &) = delete;
+	EnumTypeInfoTemplated &operator=(const EnumTypeInfoTemplated &) = delete;
+
+private:
+	string_map_t<T> values;
+};
+
+EnumTypeInfo::EnumTypeInfo(Vector &values_insert_order_p, idx_t dict_size_p)
+    : ExtraTypeInfo(ExtraTypeInfoType::ENUM_TYPE_INFO), values_insert_order(values_insert_order_p),
+      dict_type(EnumDictType::VECTOR_DICT), dict_size(dict_size_p) {
+}
+
+const EnumDictType &EnumTypeInfo::GetEnumDictType() const {
+	return dict_type;
+}
+
+const Vector &EnumTypeInfo::GetValuesInsertOrder() const {
+	return values_insert_order;
+}
+
+const idx_t &EnumTypeInfo::GetDictSize() const {
+	return dict_size;
+}
+
+LogicalType EnumTypeInfo::CreateType(Vector &ordered_data, idx_t size) {
+	// Generate EnumTypeInfo
+	shared_ptr<ExtraTypeInfo> info;
+	auto enum_internal_type = EnumTypeInfo::DictType(size);
+	switch (enum_internal_type) {
+	case PhysicalType::UINT8:
+		info = make_shared<EnumTypeInfoTemplated<uint8_t>>(ordered_data, size);
+		break;
+	case PhysicalType::UINT16:
+		info = make_shared<EnumTypeInfoTemplated<uint16_t>>(ordered_data, size);
+		break;
+	case PhysicalType::UINT32:
+		info = make_shared<EnumTypeInfoTemplated<uint32_t>>(ordered_data, size);
+		break;
+	default:
+		throw InternalException("Invalid Physical Type for ENUMs");
+	}
+	// Generate Actual Enum Type
+	return LogicalType(LogicalTypeId::ENUM, info);
+}
+
+template <class T>
+int64_t TemplatedGetPos(const string_map_t<T> &map, const string_t &key) {
+	auto it = map.find(key);
+	if (it == map.end()) {
+		return -1;
+	}
+	return it->second;
+}
+
+int64_t EnumType::GetPos(const LogicalType &type, const string_t &key) {
+	auto info = type.AuxInfo();
+	switch (type.InternalType()) {
+	case PhysicalType::UINT8:
+		return TemplatedGetPos(info->Cast<EnumTypeInfoTemplated<uint8_t>>().GetValues(), key);
+	case PhysicalType::UINT16:
+		return TemplatedGetPos(info->Cast<EnumTypeInfoTemplated<uint16_t>>().GetValues(), key);
+	case PhysicalType::UINT32:
+		return TemplatedGetPos(info->Cast<EnumTypeInfoTemplated<uint32_t>>().GetValues(), key);
+	default:
+		throw InternalException("ENUM can only have unsigned integers (except UINT64) as physical types");
+	}
+}
+
+string_t EnumType::GetString(const LogicalType &type, idx_t pos) {
+	D_ASSERT(pos < EnumType::GetSize(type));
+	return FlatVector::GetData<string_t>(EnumType::GetValuesInsertOrder(type))[pos];
+}
+
+shared_ptr<ExtraTypeInfo> EnumTypeInfo::Deserialize(FieldReader &reader) {
+	// deserialize the enum data
+	auto enum_size = reader.ReadRequired<uint32_t>();
+	auto enum_internal_type = EnumTypeInfo::DictType(enum_size);
+	switch (enum_internal_type) {
+	case PhysicalType::UINT8:
+		return EnumTypeInfoTemplated<uint8_t>::Deserialize(reader, enum_size);
+	case PhysicalType::UINT16:
+		return EnumTypeInfoTemplated<uint16_t>::Deserialize(reader, enum_size);
+	case PhysicalType::UINT32:
+		return EnumTypeInfoTemplated<uint32_t>::Deserialize(reader, enum_size);
+	default:
+		throw InternalException("Invalid Physical Type for ENUMs");
+	}
+}
+
+shared_ptr<ExtraTypeInfo> EnumTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto enum_size = deserializer.ReadProperty<idx_t>(200, "enum_size");
+	auto enum_internal_type = EnumTypeInfo::DictType(enum_size);
+	switch (enum_internal_type) {
+	case PhysicalType::UINT8:
+		return EnumTypeInfoTemplated<uint8_t>::FormatDeserialize(deserializer, enum_size);
+	case PhysicalType::UINT16:
+		return EnumTypeInfoTemplated<uint16_t>::FormatDeserialize(deserializer, enum_size);
+	case PhysicalType::UINT32:
+		return EnumTypeInfoTemplated<uint32_t>::FormatDeserialize(deserializer, enum_size);
+	default:
+		throw InternalException("Invalid Physical Type for ENUMs");
+	}
+}
+
+// Equalities are only used in enums with different catalog entries
+bool EnumTypeInfo::EqualsInternal(ExtraTypeInfo *other_p) const {
+	auto &other = other_p->Cast<EnumTypeInfo>();
+	if (dict_type != other.dict_type) {
+		return false;
+	}
+	D_ASSERT(dict_type == EnumDictType::VECTOR_DICT);
+	// We must check if both enums have the same size
+	if (other.dict_size != dict_size) {
+		return false;
+	}
+	auto other_vector_ptr = FlatVector::GetData<string_t>(other.values_insert_order);
+	auto this_vector_ptr = FlatVector::GetData<string_t>(values_insert_order);
+
+	// Now we must check if all strings are the same
+	for (idx_t i = 0; i < dict_size; i++) {
+		if (!Equals::Operation(other_vector_ptr[i], this_vector_ptr[i])) {
+			return false;
+		}
+	}
+	return true;
+}
+
+void EnumTypeInfo::Serialize(FieldWriter &writer) const {
+	if (dict_type != EnumDictType::VECTOR_DICT) {
+		throw InternalException("Cannot serialize non-vector dictionary ENUM types");
+	}
+	auto dict_size = GetDictSize();
+	// Store Dictionary Size
+	writer.WriteField<uint32_t>(dict_size);
+	// Store Vector Order By Insertion
+	((Vector &)GetValuesInsertOrder()).Serialize(dict_size, writer.GetSerializer()); // NOLINT - FIXME
+}
+
+void EnumTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
+	ExtraTypeInfo::FormatSerialize(serializer);
+	serializer.WriteProperty(200, "dict_size", dict_size);
+	((Vector &)values_insert_order).FormatSerialize(serializer, dict_size); // NOLINT - FIXME
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/field_writer.cpp

@@ -8,7 +8,6 @@ namespace duckdb {
 FieldWriter::FieldWriter(Serializer &serializer_p)
     : serializer(serializer_p), buffer(make_uniq<BufferedSerializer>()), field_count(0), finalized(false) {
 	buffer->SetVersion(serializer.GetVersion());
-	buffer->is_query_plan = serializer.is_query_plan;
 }
 
 FieldWriter::~FieldWriter() {
@@ -73,6 +72,7 @@ FieldReader::FieldReader(Deserializer &source_p) : source(source_p), field_count
 	max_field_count = source_p.Read<uint32_t>();
 	total_size = source_p.Read<uint64_t>();
 	D_ASSERT(max_field_count > 0);
+	D_ASSERT(total_size > 0);
 	source.SetRemainingData(total_size);
 }
 

package/src/duckdb/src/common/file_system.cpp

@@ -23,6 +23,14 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+
+#ifdef __MVS__
+#define _XOPEN_SOURCE_EXTENDED 1
+#include <sys/resource.h>
+// enjoy - https://reviews.llvm.org/D92110
+#define PATH_MAX _XOPEN_PATH_MAX
+#endif
+
 #else
 #include <string>
 #include <sysinfoapi.h>
@@ -63,11 +71,11 @@ string FileSystem::GetEnvVariable(const string &name) {
 }
 
 bool FileSystem::IsPathAbsolute(const string &path) {
-	auto path_separator = FileSystem::PathSeparator();
+	auto path_separator = PathSeparator(path);
 	return PathMatched(path, path_separator);
 }
 
-string FileSystem::PathSeparator() {
+string FileSystem::PathSeparator(const string &path) {
 	return "/";
 }
 
@@ -79,7 +87,14 @@ void FileSystem::SetWorkingDirectory(const string &path) {
 
 idx_t FileSystem::GetAvailableMemory() {
 	errno = 0;
+
+#ifdef __MVS__
+	struct rlimit limit;
+	int rlim_rc = getrlimit(RLIMIT_AS, &limit);
+	idx_t max_memory = MinValue<idx_t>(limit.rlim_max, UINTPTR_MAX);
+#else
 	idx_t max_memory = MinValue<idx_t>((idx_t)sysconf(_SC_PHYS_PAGES) * (idx_t)sysconf(_SC_PAGESIZE), UINTPTR_MAX);
+#endif
 	if (errno != 0) {
 		return DConstants::INVALID_INDEX;
 	}
@@ -152,7 +167,7 @@ string FileSystem::NormalizeAbsolutePath(const string &path) {
 	return result;
 }
 
-string FileSystem::PathSeparator() {
+string FileSystem::PathSeparator(const string &path) {
 	return "\\";
 }
 
@@ -195,11 +210,11 @@ string FileSystem::GetWorkingDirectory() {
 
 string FileSystem::JoinPath(const string &a, const string &b) {
 	// FIXME: sanitize paths
-	return a + PathSeparator() + b;
+	return a + PathSeparator(a) + b;
 }
 
 string FileSystem::ConvertSeparators(const string &path) {
-	auto separator_str = PathSeparator();
+	auto separator_str = PathSeparator(path);
 	char separator = separator_str[0];
 	if (separator == '/') {
 		// on unix-based systems we only accept / as a separator
@@ -214,7 +229,7 @@ string FileSystem::ExtractName(const string &path) {
 		return string();
 	}
 	auto normalized_path = ConvertSeparators(path);
-	auto sep = PathSeparator();
+	auto sep = PathSeparator(path);
 	auto splits = StringUtil::Split(normalized_path, sep);
 	D_ASSERT(!splits.empty());
 	return splits.back();
@@ -370,6 +385,10 @@ void FileSystem::UnregisterSubSystem(const string &name) {
 	throw NotImplementedException("%s: Can't unregister a sub system on a non-virtual file system", GetName());
 }
 
+void FileSystem::SetDisabledFileSystems(const vector<string> &names) {
+	throw NotImplementedException("%s: Can't disable file systems on a non-virtual file system", GetName());
+}
+
 vector<string> FileSystem::ListSubSystems() {
 	throw NotImplementedException("%s: Can't list sub systems on a non-virtual file system", GetName());
 }

package/src/duckdb/src/common/filename_pattern.cpp

@@ -24,7 +24,7 @@ void FilenamePattern::SetFilenamePattern(const string &pattern) {
 	_pos = std::min(_pos, (idx_t)_base.length());
 }
 
-string FilenamePattern::CreateFilename(const FileSystem &fs, const string &path, const string &extension,
+string FilenamePattern::CreateFilename(FileSystem &fs, const string &path, const string &extension,
                                        idx_t offset) const {
 	string result(_base);
 	string replacement;

package/src/duckdb/src/common/gzip_file_system.cpp

@@ -145,6 +145,13 @@ void MiniZStreamWrapper::Initialize(CompressedFile &file, bool write) {
 bool MiniZStreamWrapper::Read(StreamData &sd) {
 	// Handling for the concatenated files
 	if (sd.refresh) {
+		auto available = (uint32_t)(sd.in_buff_end - sd.in_buff_start);
+		if (available <= GZIP_FOOTER_SIZE) {
+			// Only footer is available so we just close and return finished
+			Close();
+			return true;
+		}
+
 		sd.refresh = false;
 		auto body_ptr = sd.in_buff_start + GZIP_FOOTER_SIZE;
 		uint8_t gzip_hdr[GZIP_HEADER_MINSIZE];
@@ -200,18 +207,6 @@ bool MiniZStreamWrapper::Read(StreamData &sd) {
 
 	// if stream ended, deallocate inflator
 	if (ret == duckdb_miniz::MZ_STREAM_END) {
-		// Last read from file done and remaining bytes only for footer or less
-		if ((sd.in_buff_end < sd.in_buff.get() + sd.in_buf_size) && mz_stream_ptr->avail_in <= GZIP_FOOTER_SIZE) {
-			Close();
-			return true;
-		}
-		if (mz_stream_ptr->avail_in > GZIP_FOOTER_SIZE) {
-			// Definitely not concatenated gzip
-			if (*(sd.in_buff_start + GZIP_FOOTER_SIZE) != 0x1F) {
-				Close();
-				return true;
-			}
-		}
 		// Concatenated GZIP potentially coming up - refresh input buffer
 		sd.refresh = true;
 	}