duckdb 0.8.2-dev4424.0 → 0.8.2-dev4514.0

package/binding.gyp

@@ -80,6 +80,7 @@
                 "src/duckdb/ub_src_function_aggregate.cpp", 
                 "src/duckdb/ub_src_function.cpp", 
                 "src/duckdb/ub_src_function_cast.cpp", 
+                "src/duckdb/ub_src_function_cast_union.cpp", 
                 "src/duckdb/ub_src_function_pragma.cpp", 
                 "src/duckdb/ub_src_function_scalar_compressed_materialization.cpp", 
                 "src/duckdb/ub_src_function_scalar.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev4424.0",
+  "version": "0.8.2-dev4514.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp

@@ -244,6 +244,9 @@ struct ExecuteSqlTableFunction {
 		auto result = make_uniq<BindData>();
 
 		result->con = make_uniq<Connection>(*context.db);
+		if (input.inputs[0].IsNull()) {
+			throw BinderException("json_execute_serialized_sql cannot execute NULL plan");
+		}
 		auto serialized = input.inputs[0].GetValueUnsafe<string>();
 		auto stmt = DeserializeSelectStatement(serialized, alc);
 		result->plan = result->con->RelationFromQuery(std::move(stmt));

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -1825,7 +1825,7 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		}
 	}
 
-	if (type.id() == LogicalTypeId::STRUCT) {
+	if (type.id() == LogicalTypeId::STRUCT || type.id() == LogicalTypeId::UNION) {
 		auto &child_types = StructType::GetChildTypes(type);
 		// set up the schema element for this struct
 		duckdb_parquet::format::SchemaElement schema_element;

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -15,6 +15,7 @@
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/serializer/buffered_file_writer.hpp"
 #include "duckdb/common/types/column/column_data_collection.hpp"
+#include "duckdb/function/copy_function.hpp"
 #endif
 
 #include "column_writer.hpp"
@@ -75,11 +76,11 @@ public:
 		return *writer;
 	}
 
-	static bool TypeIsSupported(const LogicalType &type);
+	static CopyTypeSupport TypeIsSupported(const LogicalType &type);
 
 private:
-	static bool DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type,
-	                                            duckdb_parquet::format::Type::type &type);
+	static CopyTypeSupport DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type,
+	                                                       duckdb_parquet::format::Type::type &type);
 	string file_name;
 	vector<LogicalType> sql_types;
 	vector<string> column_names;

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -77,7 +77,8 @@ private:
 	WriteStream &serializer;
 };
 
-bool ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type, Type::type &parquet_type) {
+CopyTypeSupport ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_type,
+                                                               Type::type &parquet_type) {
 	switch (duckdb_type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		parquet_type = Type::BOOLEAN;
@@ -95,9 +96,11 @@ bool ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_ty
 		parquet_type = Type::FLOAT;
 		break;
 	case LogicalTypeId::DOUBLE:
-	case LogicalTypeId::HUGEINT:
 		parquet_type = Type::DOUBLE;
 		break;
+	case LogicalTypeId::HUGEINT:
+		parquet_type = Type::DOUBLE;
+		return CopyTypeSupport::LOSSY;
 	case LogicalTypeId::ENUM:
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::VARCHAR:
@@ -141,47 +144,62 @@ bool ParquetWriter::DuckDBTypeToParquetTypeInternal(const LogicalType &duckdb_ty
 		}
 		break;
 	default:
-		// Anything that is not supported returns false
-		return false;
+		// Anything that is not supported
+		return CopyTypeSupport::UNSUPPORTED;
 	}
-	return true;
+	return CopyTypeSupport::SUPPORTED;
 }
 
 Type::type ParquetWriter::DuckDBTypeToParquetType(const LogicalType &duckdb_type) {
 	Type::type result;
-	if (!DuckDBTypeToParquetTypeInternal(duckdb_type, result)) {
+	auto type_supports = DuckDBTypeToParquetTypeInternal(duckdb_type, result);
+	if (type_supports == CopyTypeSupport::UNSUPPORTED) {
 		throw NotImplementedException("Unimplemented type for Parquet \"%s\"", duckdb_type.ToString());
 	}
 	return result;
 }
 
-bool ParquetWriter::TypeIsSupported(const LogicalType &type) {
+CopyTypeSupport ParquetWriter::TypeIsSupported(const LogicalType &type) {
 	Type::type unused;
 	auto id = type.id();
 	if (id == LogicalTypeId::LIST) {
 		auto &child_type = ListType::GetChildType(type);
 		return TypeIsSupported(child_type);
 	}
+	if (id == LogicalTypeId::UNION) {
+		auto count = UnionType::GetMemberCount(type);
+		for (idx_t i = 0; i < count; i++) {
+			auto &member_type = UnionType::GetMemberType(type, i);
+			auto type_support = TypeIsSupported(member_type);
+			if (type_support != CopyTypeSupport::SUPPORTED) {
+				return type_support;
+			}
+		}
+		return CopyTypeSupport::SUPPORTED;
+	}
 	if (id == LogicalTypeId::STRUCT) {
 		auto &children = StructType::GetChildTypes(type);
 		for (auto &child : children) {
 			auto &child_type = child.second;
-			if (!TypeIsSupported(child_type)) {
-				return false;
+			auto type_support = TypeIsSupported(child_type);
+			if (type_support != CopyTypeSupport::SUPPORTED) {
+				return type_support;
 			}
 		}
-		return true;
+		return CopyTypeSupport::SUPPORTED;
 	}
 	if (id == LogicalTypeId::MAP) {
 		auto &key_type = MapType::KeyType(type);
 		auto &value_type = MapType::ValueType(type);
-		if (!TypeIsSupported(key_type)) {
-			return false;
+		auto key_type_support = TypeIsSupported(key_type);
+		if (key_type_support != CopyTypeSupport::SUPPORTED) {
+			return key_type_support;
 		}
-		if (!TypeIsSupported(value_type)) {
-			return false;
+		auto value_type_support = TypeIsSupported(value_type);
+		if (value_type_support != CopyTypeSupport::SUPPORTED) {
+			return value_type_support;
 		}
-		return true;
+		return CopyTypeSupport::SUPPORTED;
 	}
 	return DuckDBTypeToParquetTypeInternal(type, unused);
 }

package/src/duckdb/src/common/enum_util.cpp

@@ -5974,6 +5974,8 @@ const char* EnumUtil::ToChars<UnionInvalidReason>(UnionInvalidReason value) {
 		return "NO_MEMBERS";
 	case UnionInvalidReason::VALIDITY_OVERLAP:
 		return "VALIDITY_OVERLAP";
+	case UnionInvalidReason::TAG_MISMATCH:
+		return "TAG_MISMATCH";
 	default:
 		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
 	}
@@ -5993,6 +5995,9 @@ UnionInvalidReason EnumUtil::FromString<UnionInvalidReason>(const char *value) {
 	if (StringUtil::Equals(value, "VALIDITY_OVERLAP")) {
 		return UnionInvalidReason::VALIDITY_OVERLAP;
 	}
+	if (StringUtil::Equals(value, "TAG_MISMATCH")) {
+		return UnionInvalidReason::TAG_MISMATCH;
+	}
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 

package/src/duckdb/src/common/types/vector.cpp

@@ -2007,6 +2007,9 @@ UnionInvalidReason UnionVector::CheckUnionValidity(Vector &vector, idx_t count,
 					return UnionInvalidReason::VALIDITY_OVERLAP;
 				}
 				found_valid = true;
+				if (tag != static_cast<union_tag_t>(member_idx)) {
+					return UnionInvalidReason::TAG_MISMATCH;
+				}
 			}
 		}
 	}

package/src/duckdb/src/common/types.cpp

@@ -398,7 +398,7 @@ string LogicalType::ToString() const {
 			if (i > 0) {
 				ret += ", ";
 			}
-			ret += "'" + KeywordHelper::WriteOptionallyQuoted(EnumType::GetString(*this, i).GetString(), '\'') + "'";
+			ret += KeywordHelper::WriteQuoted(EnumType::GetString(*this, i).GetString(), '\'');
 		}
 		ret += ")";
 		return ret;

package/src/duckdb/src/execution/column_binding_resolver.cpp

@@ -102,6 +102,7 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
 	default:
 		break;
 	}
+
 	// general case
 	// first visit the children of this operator
 	VisitOperatorChildren(op);

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp

@@ -55,6 +55,9 @@ struct SniffDialect {
 		if (machine.state == CSVState::INVALID) {
 			return;
 		}
+		if (machine.cur_rows < machine.options.sample_chunk_size && machine.state == CSVState::DELIMITER) {
+			sniffed_column_counts[machine.cur_rows] = ++machine.column_count;
+		}
 		if (machine.cur_rows < machine.options.sample_chunk_size && machine.state != CSVState::EMPTY_LINE) {
 			sniffed_column_counts[machine.cur_rows++] = machine.column_count;
 		}

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp

@@ -148,12 +148,18 @@ void CSVSniffer::DetectHeader() {
 			names.push_back(col_name);
 			name_collision_count[col_name] = 0;
 		}
+		if (best_header_row.size() < best_candidate->dialect_options.num_cols && options.null_padding) {
+			for (idx_t col = best_header_row.size(); col < best_candidate->dialect_options.num_cols; col++) {
+				names.push_back(GenerateColumnName(best_candidate->dialect_options.num_cols, col));
+			}
+		} else if (best_header_row.size() < best_candidate->dialect_options.num_cols) {
+			throw InternalException("Detected header has number of columns inferior to dialect detection");
+		}
 
 	} else {
 		best_candidate->dialect_options.header = false;
 		for (idx_t col = 0; col < best_candidate->dialect_options.num_cols; col++) {
-			string column_name = GenerateColumnName(best_candidate->dialect_options.num_cols, col);
-			names.push_back(column_name);
+			names.push_back(GenerateColumnName(best_candidate->dialect_options.num_cols, col));
 		}
 	}
 

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp

@@ -183,6 +183,10 @@ struct SniffValue {
 	}
 
 	inline static void Finalize(CSVStateMachine &machine, vector<TupleSniffing> &sniffed_values) {
+		if (machine.cur_rows < sniffed_values.size() && machine.state == CSVState::DELIMITER) {
+			// Started a new empty value
+			sniffed_values[machine.cur_rows].values.push_back(Value(machine.value));
+		}
 		if (machine.cur_rows < sniffed_values.size() && machine.state != CSVState::EMPTY_LINE) {
 			machine.VerifyUTF8();
 			sniffed_values[machine.cur_rows].line_number = machine.rows_read;
@@ -300,7 +304,7 @@ void CSVSniffer::DetectTypes() {
 
 		// Potentially Skip Notes (I also find this dirty, but it is what the original code does)
 		while (true_start < tuples.size()) {
-			if (tuples[true_start].values.size() < max_columns_found) {
+			if (tuples[true_start].values.size() < max_columns_found && !options.null_padding) {
 				true_start = tuples[true_start].line_number;
 				values_start++;
 			} else {

package/src/duckdb/src/function/cast/union/from_struct.cpp

@@ -0,0 +1,114 @@
+#include "duckdb/function/cast/bound_cast_data.hpp"
+
+namespace duckdb {
+
+bool StructToUnionCast::AllowImplicitCastFromStruct(const LogicalType &source, const LogicalType &target) {
+	if (source.id() != LogicalTypeId::STRUCT) {
+		return false;
+	}
+	auto target_fields = StructType::GetChildTypes(target);
+	auto fields = StructType::GetChildTypes(source);
+	if (target_fields.size() != fields.size()) {
+		// Struct should have the same amount of fields as the union
+		return false;
+	}
+	for (idx_t i = 0; i < target_fields.size(); i++) {
+		auto &target_field = target_fields[i].second;
+		auto &target_field_name = target_fields[i].first;
+		auto &field = fields[i].second;
+		auto &field_name = fields[i].first;
+		if (i == 0) {
+			// For the tag field we don't accept a type substitute as varchar
+			if (target_field != field) {
+				return false;
+			}
+			continue;
+		}
+		if (!StringUtil::CIEquals(target_field_name, field_name)) {
+			return false;
+		}
+		if (target_field != field && field != LogicalType::VARCHAR) {
+			// We allow the field to be VARCHAR, since unsupported types get cast to VARCHAR by EXPORT DATABASE (format
+			// PARQUET) i.e UNION(a BIT) becomes STRUCT(a VARCHAR)
+			return false;
+		}
+	}
+	return true;
+}
+
+// Physical Cast execution
+
+bool StructToUnionCast::Cast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
+	auto &cast_data = parameters.cast_data->Cast<StructBoundCastData>();
+	auto &lstate = parameters.local_state->Cast<StructCastLocalState>();
+
+	D_ASSERT(source.GetType().id() == LogicalTypeId::STRUCT);
+	D_ASSERT(result.GetType().id() == LogicalTypeId::UNION);
+	D_ASSERT(cast_data.target.id() == LogicalTypeId::UNION);
+
+	auto &source_children = StructVector::GetEntries(source);
+	auto &target_children = StructVector::GetEntries(result);
+
+	for (idx_t i = 0; i < source_children.size(); i++) {
+		auto &result_child_vector = *target_children[i];
+		auto &source_child_vector = *source_children[i];
+		CastParameters child_parameters(parameters, cast_data.child_cast_info[i].cast_data, lstate.local_states[i]);
+		auto converted =
+		    cast_data.child_cast_info[i].function(source_child_vector, result_child_vector, count, child_parameters);
+		(void)converted;
+		D_ASSERT(converted);
+	}
+
+	auto check_tags = UnionVector::CheckUnionValidity(result, count);
+	switch (check_tags) {
+	case UnionInvalidReason::TAG_OUT_OF_RANGE:
+		throw ConversionException("One or more of the tags do not point to a valid union member");
+	case UnionInvalidReason::VALIDITY_OVERLAP:
+		throw ConversionException("One or more rows in the produced UNION have validity set for more than 1 member");
+	case UnionInvalidReason::TAG_MISMATCH:
+		throw ConversionException(
+		    "One or more rows in the produced UNION have tags that don't point to the valid member");
+	case UnionInvalidReason::VALID:
+		break;
+	default:
+		throw InternalException("Struct to union cast failed for unknown reason");
+	}
+
+	if (source.GetVectorType() == VectorType::CONSTANT_VECTOR) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+		ConstantVector::SetNull(result, ConstantVector::IsNull(source));
+	} else {
+		source.Flatten(count);
+		FlatVector::Validity(result) = FlatVector::Validity(source);
+	}
+	result.Verify(count);
+	return true;
+}
+
+// Bind cast
+
+unique_ptr<BoundCastData> StructToUnionCast::BindData(BindCastInput &input, const LogicalType &source,
+                                                      const LogicalType &target) {
+	vector<BoundCastInfo> child_cast_info;
+	D_ASSERT(source.id() == LogicalTypeId::STRUCT);
+	D_ASSERT(target.id() == LogicalTypeId::UNION);
+
+	auto result_child_count = StructType::GetChildCount(target);
+	D_ASSERT(result_child_count == StructType::GetChildCount(source));
+
+	for (idx_t i = 0; i < result_child_count; i++) {
+		auto &source_child = StructType::GetChildType(source, i);
+		auto &target_child = StructType::GetChildType(target, i);
+
+		auto child_cast = input.GetCastFunction(source_child, target_child);
+		child_cast_info.push_back(std::move(child_cast));
+	}
+	return make_uniq<StructBoundCastData>(std::move(child_cast_info), target);
+}
+
+BoundCastInfo StructToUnionCast::Bind(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
+	auto cast_data = StructToUnionCast::BindData(input, source, target);
+	return BoundCastInfo(&StructToUnionCast::Cast, std::move(cast_data), StructBoundCastData::InitStructCastLocalState);
+}
+
+} // namespace duckdb

package/src/duckdb/src/function/cast/union_casts.cpp

@@ -11,33 +11,10 @@ namespace duckdb {
 //--------------------------------------------------------------------------------------------------
 // if the source can be implicitly cast to a member of the target union, the cast is valid
 
-struct ToUnionBoundCastData : public BoundCastData {
-	ToUnionBoundCastData(union_tag_t member_idx, string name, LogicalType type, int64_t cost,
-	                     BoundCastInfo member_cast_info)
-	    : tag(member_idx), name(std::move(name)), type(std::move(type)), cost(cost),
-	      member_cast_info(std::move(member_cast_info)) {
-	}
-
-	union_tag_t tag;
-	string name;
-	LogicalType type;
-	int64_t cost;
-	BoundCastInfo member_cast_info;
-
-public:
-	unique_ptr<BoundCastData> Copy() const override {
-		return make_uniq<ToUnionBoundCastData>(tag, name, type, cost, member_cast_info.Copy());
-	}
-
-	static bool SortByCostAscending(const ToUnionBoundCastData &left, const ToUnionBoundCastData &right) {
-		return left.cost < right.cost;
-	}
-};
-
 unique_ptr<BoundCastData> BindToUnionCast(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
 	D_ASSERT(target.id() == LogicalTypeId::UNION);
 
-	vector<ToUnionBoundCastData> candidates;
+	vector<UnionBoundCastData> candidates;
 
 	for (idx_t member_idx = 0; member_idx < UnionType::GetMemberCount(target); member_idx++) {
 		auto member_type = UnionType::GetMemberType(target, member_idx);
@@ -68,7 +45,7 @@ unique_ptr<BoundCastData> BindToUnionCast(BindCastInput &input, const LogicalTyp
 	}
 
 	// sort the candidate casts by cost
-	std::sort(candidates.begin(), candidates.end(), ToUnionBoundCastData::SortByCostAscending);
+	std::sort(candidates.begin(), candidates.end(), UnionBoundCastData::SortByCostAscending);
 
 	// select the lowest possible cost cast
 	auto &selected_cast = candidates[0];
@@ -95,11 +72,11 @@ unique_ptr<BoundCastData> BindToUnionCast(BindCastInput &input, const LogicalTyp
 	}
 
 	// otherwise, return the selected cast
-	return make_uniq<ToUnionBoundCastData>(std::move(selected_cast));
+	return make_uniq<UnionBoundCastData>(std::move(selected_cast));
 }
 
 unique_ptr<FunctionLocalState> InitToUnionLocalState(CastLocalStateParameters &parameters) {
-	auto &cast_data = parameters.cast_data->Cast<ToUnionBoundCastData>();
+	auto &cast_data = parameters.cast_data->Cast<UnionBoundCastData>();
 	if (!cast_data.member_cast_info.init_local_state) {
 		return nullptr;
 	}
@@ -109,7 +86,7 @@ unique_ptr<FunctionLocalState> InitToUnionLocalState(CastLocalStateParameters &p
 
 static bool ToUnionCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 	D_ASSERT(result.GetType().id() == LogicalTypeId::UNION);
-	auto &cast_data = parameters.cast_data->Cast<ToUnionBoundCastData>();
+	auto &cast_data = parameters.cast_data->Cast<UnionBoundCastData>();
 	auto &selected_member_vector = UnionVector::GetMember(result, cast_data.tag);
 
 	CastParameters child_parameters(parameters, cast_data.member_cast_info.cast_data, parameters.local_state);
@@ -127,7 +104,13 @@ static bool ToUnionCast(Vector &source, Vector &result, idx_t count, CastParamet
 
 BoundCastInfo DefaultCasts::ImplicitToUnionCast(BindCastInput &input, const LogicalType &source,
                                                 const LogicalType &target) {
-	return BoundCastInfo(&ToUnionCast, BindToUnionCast(input, source, target), InitToUnionLocalState);
+
+	D_ASSERT(target.id() == LogicalTypeId::UNION);
+	if (StructToUnionCast::AllowImplicitCastFromStruct(source, target)) {
+		return StructToUnionCast::Bind(input, source, target);
+	}
+	auto cast_data = BindToUnionCast(input, source, target);
+	return BoundCastInfo(&ToUnionCast, std::move(cast_data), InitToUnionLocalState);
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -143,7 +126,7 @@ BoundCastInfo DefaultCasts::ImplicitToUnionCast(BindCastInput &input, const Logi
 // INVALID:	UNION(A, B) 	->	UNION(A, C)		if B can't be implicitly cast to C
 // INVALID:	UNION(A, B, D) 	->	UNION(A, B, C)
 
-struct UnionToUnionBoundCastData : public BoundCastData {
+struct UnionUnionBoundCastData : public BoundCastData {
 
 	// mapping from source member index to target member index
 	// these are always the same size as the source member count
@@ -153,7 +136,7 @@ struct UnionToUnionBoundCastData : public BoundCastData {
 
 	LogicalType target_type;
 
-	UnionToUnionBoundCastData(vector<idx_t> tag_map, vector<BoundCastInfo> member_casts, LogicalType target_type)
+	UnionUnionBoundCastData(vector<idx_t> tag_map, vector<BoundCastInfo> member_casts, LogicalType target_type)
 	    : tag_map(std::move(tag_map)), member_casts(std::move(member_casts)), target_type(std::move(target_type)) {
 	}
 
@@ -163,7 +146,7 @@ public:
 		for (auto &member_cast : member_casts) {
 			member_casts_copy.push_back(member_cast.Copy());
 		}
-		return make_uniq<UnionToUnionBoundCastData>(tag_map, std::move(member_casts_copy), target_type);
+		return make_uniq<UnionUnionBoundCastData>(tag_map, std::move(member_casts_copy), target_type);
 	}
 };
 
@@ -203,11 +186,11 @@ unique_ptr<BoundCastData> BindUnionToUnionCast(BindCastInput &input, const Logic
 		}
 	}
 
-	return make_uniq<UnionToUnionBoundCastData>(tag_map, std::move(member_casts), target);
+	return make_uniq<UnionUnionBoundCastData>(tag_map, std::move(member_casts), target);
 }
 
 unique_ptr<FunctionLocalState> InitUnionToUnionLocalState(CastLocalStateParameters &parameters) {
-	auto &cast_data = parameters.cast_data->Cast<UnionToUnionBoundCastData>();
+	auto &cast_data = parameters.cast_data->Cast<UnionUnionBoundCastData>();
 	auto result = make_uniq<StructCastLocalState>();
 
 	for (auto &entry : cast_data.member_casts) {
@@ -222,7 +205,7 @@ unique_ptr<FunctionLocalState> InitUnionToUnionLocalState(CastLocalStateParamete
 }
 
 static bool UnionToUnionCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
-	auto &cast_data = parameters.cast_data->Cast<UnionToUnionBoundCastData>();
+	auto &cast_data = parameters.cast_data->Cast<UnionUnionBoundCastData>();
 	auto &lstate = parameters.local_state->Cast<StructCastLocalState>();
 
 	auto source_member_count = UnionType::GetMemberCount(source.GetType());
@@ -313,7 +296,7 @@ static bool UnionToUnionCast(Vector &source, Vector &result, idx_t count, CastPa
 static bool UnionToVarcharCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 	auto constant = source.GetVectorType() == VectorType::CONSTANT_VECTOR;
 	// first cast all union members to varchar
-	auto &cast_data = parameters.cast_data->Cast<UnionToUnionBoundCastData>();
+	auto &cast_data = parameters.cast_data->Cast<UnionUnionBoundCastData>();
 	Vector varchar_union(cast_data.target_type, count);
 
 	UnionToUnionCast(source, varchar_union, count, parameters);
@@ -356,6 +339,7 @@ static bool UnionToVarcharCast(Vector &source, Vector &result, idx_t count, Cast
 
 BoundCastInfo DefaultCasts::UnionCastSwitch(BindCastInput &input, const LogicalType &source,
                                             const LogicalType &target) {
+	D_ASSERT(source.id() == LogicalTypeId::UNION);
 	switch (target.id()) {
 	case LogicalTypeId::VARCHAR: {
 		// bind a cast in which we convert all members to VARCHAR first

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev4424"
+#define DUCKDB_VERSION "0.8.2-dev4514"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "b78b24ad26"
+#define DUCKDB_SOURCE_ID "38c6e8ccce"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -447,7 +447,7 @@ struct StructVector {
 	DUCKDB_API static vector<unique_ptr<Vector>> &GetEntries(Vector &vector);
 };
 
-enum class UnionInvalidReason : uint8_t { VALID, TAG_OUT_OF_RANGE, NO_MEMBERS, VALIDITY_OVERLAP };
+enum class UnionInvalidReason : uint8_t { VALID, TAG_OUT_OF_RANGE, NO_MEMBERS, VALIDITY_OVERLAP, TAG_MISMATCH };
 
 struct UnionVector {
 	// Unions are stored as structs, but the first child is always the "tag"

package/src/duckdb/src/include/duckdb/function/cast/bound_cast_data.hpp

@@ -81,4 +81,36 @@ public:
 	unique_ptr<FunctionLocalState> value_state;
 };
 
+struct UnionBoundCastData : public BoundCastData {
+	UnionBoundCastData(union_tag_t member_idx, string name, LogicalType type, int64_t cost,
+	                   BoundCastInfo member_cast_info)
+	    : tag(member_idx), name(std::move(name)), type(std::move(type)), cost(cost),
+	      member_cast_info(std::move(member_cast_info)) {
+	}
+
+	union_tag_t tag;
+	string name;
+	LogicalType type;
+	int64_t cost;
+	BoundCastInfo member_cast_info;
+
+public:
+	unique_ptr<BoundCastData> Copy() const override {
+		return make_uniq<UnionBoundCastData>(tag, name, type, cost, member_cast_info.Copy());
+	}
+
+	static bool SortByCostAscending(const UnionBoundCastData &left, const UnionBoundCastData &right) {
+		return left.cost < right.cost;
+	}
+};
+
+struct StructToUnionCast {
+public:
+	static bool AllowImplicitCastFromStruct(const LogicalType &source, const LogicalType &target);
+	static bool Cast(Vector &source, Vector &result, idx_t count, CastParameters &parameters);
+	static unique_ptr<BoundCastData> BindData(BindCastInput &input, const LogicalType &source,
+	                                          const LogicalType &target);
+	static BoundCastInfo Bind(BindCastInput &input, const LogicalType &source, const LogicalType &target);
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/copy_function.hpp

@@ -99,7 +99,9 @@ typedef void (*copy_flush_batch_t)(ClientContext &context, FunctionData &bind_da
                                    PreparedBatchData &batch);
 typedef idx_t (*copy_desired_batch_size_t)(ClientContext &context, FunctionData &bind_data);
 
-typedef bool (*copy_supports_type_t)(const LogicalType &type);
+enum class CopyTypeSupport { SUPPORTED, LOSSY, UNSUPPORTED };
+
+typedef CopyTypeSupport (*copy_supports_type_t)(const LogicalType &type);
 
 class CopyFunction : public Function {
 public:

package/src/duckdb/src/include/duckdb/optimizer/rule/empty_needle_removal.hpp

@@ -13,7 +13,7 @@
 namespace duckdb {
 
 // The Empty_needle_removal Optimization rule folds some foldable ConstantExpression
-//(e.g.: PREFIX('xyz', '') is TRUE, PREFIX(NULL, '') is NULL, so rewrite PREFIX(x, '') to (CASE WHEN x IS NOT NULL THEN)
+//(e.g.: PREFIX('xyz', '') is TRUE, PREFIX(NULL, '') is NULL, so rewrite PREFIX(x, '') to TRUE_OR_NULL(x)
 class EmptyNeedleRemovalRule : public Rule {
 public:
 	explicit EmptyNeedleRemovalRule(ExpressionRewriter &rewriter);

package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp

@@ -72,17 +72,21 @@ void ColumnLifetimeAnalyzer::VisitOperator(LogicalOperator &op) {
 		for (auto &cond : comp_join.conditions) {
 			if (cond.comparison == ExpressionType::COMPARE_EQUAL) {
 				has_equality = true;
+				break;
 			}
 		}
 		if (!has_equality) {
 			break;
 		}
-		// now, for each of the columns of the RHS, check which columns need to be projected
+		// visit current operator expressions so they are added to the referenced_columns
+		LogicalOperatorVisitor::VisitOperatorExpressions(op);
+
 		column_binding_set_t unused_bindings;
+		auto old_op_bindings = op.GetColumnBindings();
 		ExtractUnusedColumnBindings(op.children[1]->GetColumnBindings(), unused_bindings);
 
 		// now recurse into the filter and its children
-		StandardVisitOperator(op);
+		LogicalOperatorVisitor::VisitOperatorChildren(op);
 
 		// then generate the projection map
 		GenerateProjectionMap(op.children[1]->GetColumnBindings(), unused_bindings, comp_join.right_projection_map);
@@ -118,12 +122,14 @@ void ColumnLifetimeAnalyzer::VisitOperator(LogicalOperator &op) {
 		if (everything_referenced) {
 			break;
 		}
+		// first visit operator expressions to populate referenced columns
+		LogicalOperatorVisitor::VisitOperatorExpressions(op);
 		// filter, figure out which columns are not needed after the filter
 		column_binding_set_t unused_bindings;
 		ExtractUnusedColumnBindings(op.children[0]->GetColumnBindings(), unused_bindings);
 
 		// now recurse into the filter and its children
-		StandardVisitOperator(op);
+		LogicalOperatorVisitor::VisitOperatorChildren(op);
 
 		// then generate the projection map
 		GenerateProjectionMap(op.children[0]->GetColumnBindings(), unused_bindings, filter.projection_map);

package/src/duckdb/src/optimizer/statistics/operator/propagate_projection.cpp

@@ -11,7 +11,6 @@ unique_ptr<NodeStatistics> StatisticsPropagator::PropagateStatistics(LogicalProj
 		ReplaceWithEmptyResult(*node_ptr);
 		return std::move(node_stats);
 	}
-
 	// then propagate to each of the expressions
 	for (idx_t i = 0; i < proj.expressions.size(); i++) {
 		auto stats = PropagateExpression(proj.expressions[i]);

package/src/duckdb/src/parser/keyword_helper.cpp

@@ -36,7 +36,7 @@ string KeywordHelper::EscapeQuotes(const string &text, char quote) {
 string KeywordHelper::WriteQuoted(const string &text, char quote) {
 	// 1. Escapes all occurences of 'quote' by doubling them (escape in SQL)
 	// 2. Adds quotes around the string
-	return string(1, quote) + EscapeQuotes(text) + string(1, quote);
+	return string(1, quote) + EscapeQuotes(text, quote) + string(1, quote);
 }
 
 string KeywordHelper::WriteOptionallyQuoted(const string &text, char quote, bool allow_caps) {

package/src/duckdb/src/planner/binder/statement/bind_export.cpp

@@ -115,7 +115,95 @@ string CreateFileName(const string &id_suffix, TableCatalogEntry &table, const s
 	return StringUtil::Format("%s_%s%s.%s", schema, name, id_suffix, extension);
 }
 
-unique_ptr<QueryNode> CreateSelectStatement(CopyStatement &stmt, vector<unique_ptr<ParsedExpression>> select_list) {
+static bool IsSupported(CopyTypeSupport support_level) {
+	// For export purposes we don't want to lose information, so we only accept fully supported types
+	return support_level == CopyTypeSupport::SUPPORTED;
+}
+
+static LogicalType AlterLogicalType(const LogicalType &original, copy_supports_type_t type_check) {
+	D_ASSERT(type_check);
+	auto id = original.id();
+	switch (id) {
+	case LogicalTypeId::LIST: {
+		auto child = AlterLogicalType(ListType::GetChildType(original), type_check);
+		return LogicalType::LIST(child);
+	}
+	case LogicalTypeId::STRUCT: {
+		auto &original_children = StructType::GetChildTypes(original);
+		child_list_t<LogicalType> new_children;
+		for (auto &child : original_children) {
+			auto &child_name = child.first;
+			auto &child_type = child.second;
+
+			LogicalType new_type;
+			if (!IsSupported(type_check(child_type))) {
+				new_type = AlterLogicalType(child_type, type_check);
+			} else {
+				new_type = child_type;
+			}
+			new_children.push_back(std::make_pair(child_name, new_type));
+		}
+		return LogicalType::STRUCT(std::move(new_children));
+	}
+	case LogicalTypeId::UNION: {
+		auto member_count = UnionType::GetMemberCount(original);
+		child_list_t<LogicalType> new_children;
+		for (idx_t i = 0; i < member_count; i++) {
+			auto &child_name = UnionType::GetMemberName(original, i);
+			auto &child_type = UnionType::GetMemberType(original, i);
+
+			LogicalType new_type;
+			if (!IsSupported(type_check(child_type))) {
+				new_type = AlterLogicalType(child_type, type_check);
+			} else {
+				new_type = child_type;
+			}
+
+			new_children.push_back(std::make_pair(child_name, new_type));
+		}
+		return LogicalType::UNION(std::move(new_children));
+	}
+	case LogicalTypeId::MAP: {
+		auto &key_type = MapType::KeyType(original);
+		auto &value_type = MapType::ValueType(original);
+
+		LogicalType new_key_type;
+		LogicalType new_value_type;
+		if (!IsSupported(type_check(key_type))) {
+			new_key_type = AlterLogicalType(key_type, type_check);
+		} else {
+			new_key_type = key_type;
+		}
+
+		if (!IsSupported(type_check(value_type))) {
+			new_value_type = AlterLogicalType(value_type, type_check);
+		} else {
+			new_value_type = value_type;
+		}
+		return LogicalType::MAP(new_key_type, new_value_type);
+	}
+	default: {
+		D_ASSERT(!IsSupported(type_check(original)));
+		return LogicalType::VARCHAR;
+	}
+	}
+}
+
+static bool NeedsCast(LogicalType &type, copy_supports_type_t type_check) {
+	if (!type_check) {
+		return false;
+	}
+	if (IsSupported(type_check(type))) {
+		// The type is supported in it's entirety, no cast is required
+		return false;
+	}
+	// Change the type to something that is supported
+	type = AlterLogicalType(type, type_check);
+	return true;
+}
+
+static unique_ptr<QueryNode> CreateSelectStatement(CopyStatement &stmt, child_list_t<LogicalType> &select_list,
+                                                   copy_supports_type_t type_check) {
 	auto ref = make_uniq<BaseTableRef>();
 	ref->catalog_name = stmt.info->catalog;
 	ref->schema_name = stmt.info->schema;
@@ -123,7 +211,21 @@ unique_ptr<QueryNode> CreateSelectStatement(CopyStatement &stmt, vector<unique_p
 
 	auto statement = make_uniq<SelectNode>();
 	statement->from_table = std::move(ref);
-	statement->select_list = std::move(select_list);
+
+	vector<unique_ptr<ParsedExpression>> expressions;
+	for (auto &col : select_list) {
+		auto &name = col.first;
+		auto &type = col.second;
+
+		auto expression = make_uniq_base<ParsedExpression, ColumnRefExpression>(name);
+		if (NeedsCast(type, type_check)) {
+			// Add a cast to a type supported by the copy function
+			expression = make_uniq_base<ParsedExpression, CastExpression>(type, std::move(expression));
+		}
+		expressions.push_back(std::move(expression));
+	}
+
+	statement->select_list = std::move(expressions);
 	return std::move(statement);
 }
 
@@ -194,16 +296,10 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 		info->table = table.name;
 
 		// We can not export generated columns
-		vector<unique_ptr<ParsedExpression>> expressions;
+		child_list_t<LogicalType> select_list;
+
 		for (auto &col : table.GetColumns().Physical()) {
-			auto expression = make_uniq_base<ParsedExpression, ColumnRefExpression>(col.GetName());
-			auto is_supported = copy_function.function.supports_type;
-			if (is_supported && !is_supported(col.Type())) {
-				expression =
-				    make_uniq_base<ParsedExpression, CastExpression>(LogicalType::VARCHAR, std::move(expression));
-			}
-			expressions.push_back(std::move(expression));
-			info->select_list.push_back(col.GetName());
+			select_list.push_back(std::make_pair(col.Name(), col.Type()));
 		}
 
 		ExportedTableData exported_data;
@@ -220,7 +316,8 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {
 		// generate the copy statement and bind it
 		CopyStatement copy_stmt;
 		copy_stmt.info = std::move(info);
-		copy_stmt.select_statement = CreateSelectStatement(copy_stmt, std::move(expressions));
+		copy_stmt.select_statement =
+		    CreateSelectStatement(copy_stmt, select_list, copy_function.function.supports_type);
 
 		auto copy_binder = Binder::CreateBinder(context, this);
 		auto bound_statement = copy_binder->Bind(copy_stmt);

package/src/duckdb/src/storage/compression/rle.cpp

@@ -292,7 +292,11 @@ void RLESkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
 	scan_state.Skip(segment, skip_count);
 }
 
+template <bool ENTIRE_VECTOR>
 static bool CanEmitConstantVector(idx_t position, idx_t run_length, idx_t scan_count) {
+	if (!ENTIRE_VECTOR) {
+		return false;
+	}
 	if (scan_count != STANDARD_VECTOR_SIZE) {
 		// Only when we can fill an entire Vector can we emit a ConstantVector, because subsequent scans require the
 		// input Vector to be flat
@@ -330,9 +334,9 @@ static void RLEScanConstant(RLEScanState<T> &scan_state, rle_count_t *index_poin
 	return;
 }
 
-template <class T>
-void RLEScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
-                    idx_t result_offset) {
+template <class T, bool ENTIRE_VECTOR>
+void RLEScanPartialInternal(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
+                            idx_t result_offset) {
 	auto &scan_state = state.scan_state->Cast<RLEScanState<T>>();
 
 	auto data = scan_state.handle.Ptr() + segment.GetBlockOffset();
@@ -340,7 +344,8 @@ void RLEScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_c
 	auto index_pointer = reinterpret_cast<rle_count_t *>(data + scan_state.rle_count_offset);
 
 	// If we are scanning an entire Vector and it contains only a single run
-	if (CanEmitConstantVector(scan_state.position_in_entry, index_pointer[scan_state.entry_pos], scan_count)) {
+	if (CanEmitConstantVector<ENTIRE_VECTOR>(scan_state.position_in_entry, index_pointer[scan_state.entry_pos],
+	                                         scan_count)) {
 		RLEScanConstant<T>(scan_state, index_pointer, data_pointer, scan_count, result);
 		return;
 	}
@@ -357,9 +362,15 @@ void RLEScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_c
 	}
 }
 
+template <class T>
+void RLEScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
+                    idx_t result_offset) {
+	return RLEScanPartialInternal<T, false>(segment, state, scan_count, result, result_offset);
+}
+
 template <class T>
 void RLEScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
-	RLEScanPartial<T>(segment, state, scan_count, result, 0);
+	RLEScanPartialInternal<T, true>(segment, state, scan_count, result, 0);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/ub_src_function_cast_union.cpp

@@ -0,0 +1,2 @@
+#include "src/function/cast/union/from_struct.cpp"
+