duckdb 0.9.2-dev22.0 → 0.9.2-dev26.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.9.2-dev22.0",
+  "version": "0.9.2-dev26.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/icu/icu-timebucket.cpp

@@ -76,24 +76,21 @@ struct ICUTimeBucket : public ICUDateFunc {
 
 	static inline timestamp_t WidthConvertibleToDaysCommon(int32_t bucket_width_days, const timestamp_t ts,
 	                                                       const timestamp_t origin, icu::Calendar *calendar) {
-		const auto trunc_days = TruncationFactory(DatePartSpecifier::DAY);
 		const auto sub_days = SubtractFactory(DatePartSpecifier::DAY);
 
-		uint64_t tmp_micros = SetTime(calendar, ts);
-		trunc_days(calendar, tmp_micros);
-		timestamp_t truncated_ts = GetTimeUnsafe(calendar, tmp_micros);
-
-		int64_t ts_days = sub_days(calendar, origin, truncated_ts);
+		int64_t ts_days = sub_days(calendar, origin, ts);
 		int64_t result_days = (ts_days / bucket_width_days) * bucket_width_days;
 		if (result_days < NumericLimits<int32_t>::Minimum() || result_days > NumericLimits<int32_t>::Maximum()) {
 			throw OutOfRangeException("Timestamp out of range");
 		}
-		if (ts_days < 0 && ts_days % bucket_width_days != 0) {
-			result_days =
-			    SubtractOperatorOverflowCheck::Operation<int32_t, int32_t, int32_t>(result_days, bucket_width_days);
+		timestamp_t bucket = Add(calendar, origin, interval_t {0, static_cast<int32_t>(result_days), 0});
+		if (ts < bucket) {
+			D_ASSERT(ts < origin);
+			bucket = Add(calendar, bucket, interval_t {0, -bucket_width_days, 0});
+			D_ASSERT(ts > bucket);
 		}
 
-		return Add(calendar, origin, interval_t {0, static_cast<int32_t>(result_days), 0});
+		return bucket;
 	}
 
 	static inline timestamp_t WidthConvertibleToMonthsCommon(int32_t bucket_width_months, const timestamp_t ts,

package/src/duckdb/extension/icu/icu-timezone.cpp

@@ -81,6 +81,9 @@ static void ICUTimeZoneFunction(ClientContext &context, TableFunctionInput &data
 			break;
 		}
 
+		//	What PG reports is the total offset for today,
+		//	which is the ICU total offset (i.e., "raw") plus the DST offset.
+		raw_offset_ms += dst_offset_ms;
 		output.SetValue(2, index, Value::INTERVAL(Interval::FromMicro(raw_offset_ms * Interval::MICROS_PER_MSEC)));
 		output.SetValue(3, index, Value(dst_offset_ms != 0));
 		++index;

package/src/duckdb/extension/json/buffered_json_reader.cpp

@@ -23,7 +23,7 @@ bool JSONFileHandle::IsOpen() const {
 }
 
 void JSONFileHandle::Close() {
-	if (IsOpen() && file_handle->OnDiskFile()) {
+	if (IsOpen() && !file_handle->IsPipe()) {
 		file_handle->Close();
 		file_handle = nullptr;
 	}
@@ -72,30 +72,23 @@ void JSONFileHandle::ReadAtPosition(char *pointer, idx_t size, idx_t position, b
 	D_ASSERT(size != 0);
 	if (plain_file_source) {
 		file_handle->Read(pointer, size, position);
-		actual_reads++;
-
-		return;
-	}
-
-	if (sample_run) { // Cache the buffer
+	} else if (sample_run) { // Cache the buffer
 		file_handle->Read(pointer, size, position);
-		actual_reads++;
 
 		cached_buffers.emplace_back(allocator.Allocate(size));
 		memcpy(cached_buffers.back().get(), pointer, size);
 		cached_size += size;
+	} else {
+		if (!cached_buffers.empty() || position < cached_size) {
+			ReadFromCache(pointer, size, position);
+		}
 
-		return;
-	}
-
-	if (!cached_buffers.empty() || position < cached_size) {
-		ReadFromCache(pointer, size, position);
-		actual_reads++;
+		if (size != 0) {
+			file_handle->Read(pointer, size, position);
+		}
 	}
-
-	if (size != 0) {
-		file_handle->Read(pointer, size, position);
-		actual_reads++;
+	if (++actual_reads > requested_reads) {
+		throw InternalException("JSONFileHandle performed more actual reads than requested reads");
 	}
 }
 

package/src/duckdb/extension/json/json_scan.cpp

@@ -214,17 +214,22 @@ unique_ptr<GlobalTableFunctionState> JSONGlobalTableFunctionState::Init(ClientCo
 
 idx_t JSONGlobalTableFunctionState::MaxThreads() const {
 	auto &bind_data = state.bind_data;
-	if (bind_data.options.format == JSONFormat::NEWLINE_DELIMITED) {
-		return state.system_threads;
-	}
 
 	if (!state.json_readers.empty() && state.json_readers[0]->HasFileHandle()) {
+		// We opened and auto-detected a file, so we can get a better estimate
 		auto &reader = *state.json_readers[0];
-		if (reader.GetFormat() == JSONFormat::NEWLINE_DELIMITED) { // Auto-detected NDJSON
-			return state.system_threads;
+		if (bind_data.options.format == JSONFormat::NEWLINE_DELIMITED ||
+		    reader.GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
+			return MaxValue<idx_t>(state.json_readers[0]->GetFileHandle().FileSize() / bind_data.maximum_object_size,
+			                       1);
 		}
 	}
 
+	if (bind_data.options.format == JSONFormat::NEWLINE_DELIMITED) {
+		// We haven't opened any files, so this is our best bet
+		return state.system_threads;
+	}
+
 	// One reader per file
 	return bind_data.files.size();
 }

package/src/duckdb/extension/parquet/parquet_extension.cpp

@@ -740,8 +740,8 @@ static void GetFieldIDs(const Value &field_ids_value, ChildFieldIDs &field_ids,
 	}
 }
 
-unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyInfo &info, vector<string> &names,
-                                          vector<LogicalType> &sql_types) {
+unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, const CopyInfo &info, const vector<string> &names,
+                                          const vector<LogicalType> &sql_types) {
 	D_ASSERT(names.size() == sql_types.size());
 	bool row_group_size_bytes_set = false;
 	auto bind_data = make_uniq<ParquetWriteBindData>();

package/src/duckdb/src/catalog/catalog_entry/view_catalog_entry.cpp

@@ -32,6 +32,7 @@ unique_ptr<CreateInfo> ViewCatalogEntry::GetInfo() const {
 	result->query = unique_ptr_cast<SQLStatement, SelectStatement>(query->Copy());
 	result->aliases = aliases;
 	result->types = types;
+	result->temporary = temporary;
 	return std::move(result);
 }
 
@@ -58,23 +59,16 @@ string ViewCatalogEntry::ToSQL() const {
 		//! Return empty sql with view name so pragma view_tables don't complain
 		return sql;
 	}
-	return sql + "\n;";
+	auto info = GetInfo();
+	auto result = info->ToString();
+	return result + ";\n";
 }
 
 unique_ptr<CatalogEntry> ViewCatalogEntry::Copy(ClientContext &context) const {
 	D_ASSERT(!internal);
-	CreateViewInfo create_info(schema, name);
-	create_info.query = unique_ptr_cast<SQLStatement, SelectStatement>(query->Copy());
-	for (idx_t i = 0; i < aliases.size(); i++) {
-		create_info.aliases.push_back(aliases[i]);
-	}
-	for (idx_t i = 0; i < types.size(); i++) {
-		create_info.types.push_back(types[i]);
-	}
-	create_info.temporary = temporary;
-	create_info.sql = sql;
+	auto create_info = GetInfo();
 
-	return make_uniq<ViewCatalogEntry>(catalog, schema, create_info);
+	return make_uniq<ViewCatalogEntry>(catalog, schema, create_info->Cast<CreateViewInfo>());
 }
 
 } // namespace duckdb

package/src/duckdb/src/catalog/catalog_set.cpp

@@ -199,6 +199,8 @@ bool CatalogSet::AlterOwnership(CatalogTransaction transaction, ChangeOwnershipI
 bool CatalogSet::AlterEntry(CatalogTransaction transaction, const string &name, AlterInfo &alter_info) {
 	// lock the catalog for writing
 	lock_guard<mutex> write_lock(catalog.GetWriteLock());
+	// lock this catalog set to disallow reading
+	lock_guard<mutex> read_lock(catalog_lock);
 
 	// first check if the entry exists in the unordered set
 	EntryIndex entry_index;
@@ -210,9 +212,6 @@ bool CatalogSet::AlterEntry(CatalogTransaction transaction, const string &name,
 		throw CatalogException("Cannot alter entry \"%s\" because it is an internal system entry", entry->name);
 	}
 
-	// lock this catalog set to disallow reading
-	lock_guard<mutex> read_lock(catalog_lock);
-
 	// create a new entry and replace the currently stored one
 	// set the timestamp to the timestamp of the current transaction
 	// and point it to the updated table node
@@ -316,6 +315,7 @@ void CatalogSet::DropEntryInternal(CatalogTransaction transaction, EntryIndex en
 bool CatalogSet::DropEntry(CatalogTransaction transaction, const string &name, bool cascade, bool allow_drop_internal) {
 	// lock the catalog for writing
 	lock_guard<mutex> write_lock(catalog.GetWriteLock());
+	lock_guard<mutex> read_lock(catalog_lock);
 	// we can only delete an entry that exists
 	EntryIndex entry_index;
 	auto entry = GetEntryInternal(transaction, name, &entry_index);
@@ -326,7 +326,6 @@ bool CatalogSet::DropEntry(CatalogTransaction transaction, const string &name, b
 		throw CatalogException("Cannot drop entry \"%s\" because it is an internal system entry", entry->name);
 	}
 
-	lock_guard<mutex> read_lock(catalog_lock);
 	DropEntryInternal(transaction, std::move(entry_index), *entry, cascade);
 	return true;
 }

package/src/duckdb/src/common/arrow/appender/union_data.cpp

@@ -24,7 +24,7 @@ void ArrowUnionData::Append(ArrowAppendData &append_data, Vector &input, idx_t f
 
 	duckdb::vector<Vector> child_vectors;
 	for (const auto &child : UnionType::CopyMemberTypes(input.GetType())) {
-		child_vectors.emplace_back(child.second);
+		child_vectors.emplace_back(child.second, size);
 	}
 
 	for (idx_t input_idx = from; input_idx < to; input_idx++) {

package/src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -193,26 +193,26 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			InitializeAppenderForType<ArrowVarcharData<string_t>>(append_data);
 		} else {
-			InitializeAppenderForType<ArrowVarcharData<string_t, ArrowVarcharConverter, uint32_t>>(append_data);
+			InitializeAppenderForType<ArrowVarcharData<string_t, ArrowVarcharConverter, int32_t>>(append_data);
 		}
 		break;
 	case LogicalTypeId::UUID:
 		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			InitializeAppenderForType<ArrowVarcharData<hugeint_t, ArrowUUIDConverter>>(append_data);
 		} else {
-			InitializeAppenderForType<ArrowVarcharData<hugeint_t, ArrowUUIDConverter, uint32_t>>(append_data);
+			InitializeAppenderForType<ArrowVarcharData<hugeint_t, ArrowUUIDConverter, int32_t>>(append_data);
 		}
 		break;
 	case LogicalTypeId::ENUM:
 		switch (type.InternalType()) {
 		case PhysicalType::UINT8:
-			InitializeAppenderForType<ArrowEnumData<uint8_t>>(append_data);
+			InitializeAppenderForType<ArrowEnumData<int8_t>>(append_data);
 			break;
 		case PhysicalType::UINT16:
-			InitializeAppenderForType<ArrowEnumData<uint16_t>>(append_data);
+			InitializeAppenderForType<ArrowEnumData<int16_t>>(append_data);
 			break;
 		case PhysicalType::UINT32:
-			InitializeAppenderForType<ArrowEnumData<uint32_t>>(append_data);
+			InitializeAppenderForType<ArrowEnumData<int32_t>>(append_data);
 			break;
 		default:
 			throw InternalException("Unsupported internal enum type");
@@ -227,11 +227,20 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 	case LogicalTypeId::STRUCT:
 		InitializeAppenderForType<ArrowStructData>(append_data);
 		break;
-	case LogicalTypeId::LIST:
-		InitializeAppenderForType<ArrowListData>(append_data);
+	case LogicalTypeId::LIST: {
+		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
+			InitializeAppenderForType<ArrowListData<int64_t>>(append_data);
+		} else {
+			InitializeAppenderForType<ArrowListData<int32_t>>(append_data);
+		}
 		break;
+	}
 	case LogicalTypeId::MAP:
-		InitializeAppenderForType<ArrowMapData>(append_data);
+		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
+			InitializeAppenderForType<ArrowMapData<int64_t>>(append_data);
+		} else {
+			InitializeAppenderForType<ArrowMapData<int32_t>>(append_data);
+		}
 		break;
 	default:
 		throw NotImplementedException("Unsupported type in DuckDB -> Arrow Conversion: %s\n", type.ToString());

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -187,7 +187,11 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		break;
 	}
 	case LogicalTypeId::LIST: {
-		child.format = "+l";
+		if (options.arrow_offset_size == ArrowOffsetSize::LARGE) {
+			child.format = "+L";
+		} else {
+			child.format = "+l";
+		}
 		child.n_children = 1;
 		root_holder.nested_children.emplace_back();
 		root_holder.nested_children.back().resize(1);

package/src/duckdb/src/common/enum_util.cpp

@@ -64,6 +64,7 @@
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/types/vector.hpp"
 #include "duckdb/common/types/vector_buffer.hpp"
+#include "duckdb/core_functions/aggregate/quantile_enum.hpp"
 #include "duckdb/execution/index/art/art.hpp"
 #include "duckdb/execution/index/art/node.hpp"
 #include "duckdb/execution/operator/scan/csv/base_csv_reader.hpp"
@@ -4571,6 +4572,44 @@ ProfilerPrintFormat EnumUtil::FromString<ProfilerPrintFormat>(const char *value)
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 
+template<>
+const char* EnumUtil::ToChars<QuantileSerializationType>(QuantileSerializationType value) {
+	switch(value) {
+	case QuantileSerializationType::NON_DECIMAL:
+		return "NON_DECIMAL";
+	case QuantileSerializationType::DECIMAL_DISCRETE:
+		return "DECIMAL_DISCRETE";
+	case QuantileSerializationType::DECIMAL_DISCRETE_LIST:
+		return "DECIMAL_DISCRETE_LIST";
+	case QuantileSerializationType::DECIMAL_CONTINUOUS:
+		return "DECIMAL_CONTINUOUS";
+	case QuantileSerializationType::DECIMAL_CONTINUOUS_LIST:
+		return "DECIMAL_CONTINUOUS_LIST";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
+	}
+}
+
+template<>
+QuantileSerializationType EnumUtil::FromString<QuantileSerializationType>(const char *value) {
+	if (StringUtil::Equals(value, "NON_DECIMAL")) {
+		return QuantileSerializationType::NON_DECIMAL;
+	}
+	if (StringUtil::Equals(value, "DECIMAL_DISCRETE")) {
+		return QuantileSerializationType::DECIMAL_DISCRETE;
+	}
+	if (StringUtil::Equals(value, "DECIMAL_DISCRETE_LIST")) {
+		return QuantileSerializationType::DECIMAL_DISCRETE_LIST;
+	}
+	if (StringUtil::Equals(value, "DECIMAL_CONTINUOUS")) {
+		return QuantileSerializationType::DECIMAL_CONTINUOUS;
+	}
+	if (StringUtil::Equals(value, "DECIMAL_CONTINUOUS_LIST")) {
+		return QuantileSerializationType::DECIMAL_CONTINUOUS_LIST;
+	}
+	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
+}
+
 template<>
 const char* EnumUtil::ToChars<QueryNodeType>(QueryNodeType value) {
 	switch(value) {
@@ -5118,6 +5157,29 @@ SinkFinalizeType EnumUtil::FromString<SinkFinalizeType>(const char *value) {
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 
+template<>
+const char* EnumUtil::ToChars<SinkNextBatchType>(SinkNextBatchType value) {
+	switch(value) {
+	case SinkNextBatchType::READY:
+		return "READY";
+	case SinkNextBatchType::BLOCKED:
+		return "BLOCKED";
+	default:
+		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
+	}
+}
+
+template<>
+SinkNextBatchType EnumUtil::FromString<SinkNextBatchType>(const char *value) {
+	if (StringUtil::Equals(value, "READY")) {
+		return SinkNextBatchType::READY;
+	}
+	if (StringUtil::Equals(value, "BLOCKED")) {
+		return SinkNextBatchType::BLOCKED;
+	}
+	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
+}
+
 template<>
 const char* EnumUtil::ToChars<SinkResultType>(SinkResultType value) {
 	switch(value) {
@@ -6010,6 +6072,8 @@ const char* EnumUtil::ToChars<UnionInvalidReason>(UnionInvalidReason value) {
 		return "VALIDITY_OVERLAP";
 	case UnionInvalidReason::TAG_MISMATCH:
 		return "TAG_MISMATCH";
+	case UnionInvalidReason::NULL_TAG:
+		return "NULL_TAG";
 	default:
 		throw NotImplementedException(StringUtil::Format("Enum value: '%d' not implemented", value));
 	}
@@ -6032,6 +6096,9 @@ UnionInvalidReason EnumUtil::FromString<UnionInvalidReason>(const char *value) {
 	if (StringUtil::Equals(value, "TAG_MISMATCH")) {
 		return UnionInvalidReason::TAG_MISMATCH;
 	}
+	if (StringUtil::Equals(value, "NULL_TAG")) {
+		return UnionInvalidReason::NULL_TAG;
+	}
 	throw NotImplementedException(StringUtil::Format("Enum value: '%s' not implemented", value));
 }
 

package/src/duckdb/src/common/file_system.cpp

@@ -344,7 +344,7 @@ bool FileSystem::FileExists(const string &filename) {
 }
 
 bool FileSystem::IsPipe(const string &filename) {
-	throw NotImplementedException("%s: IsPipe is not implemented!", GetName());
+	return false;
 }
 
 void FileSystem::RemoveFile(const string &filename) {
@@ -500,6 +500,10 @@ bool FileHandle::CanSeek() {
 	return file_system.CanSeek();
 }
 
+bool FileHandle::IsPipe() {
+	return file_system.IsPipe(path);
+}
+
 string FileHandle::ReadLine() {
 	string result;
 	char buffer[1];

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -64,7 +64,10 @@ static void ConvertKnownColRefToConstants(unique_ptr<Expression> &expr,
 // 	- s3://bucket/var1=value1/bla/bla/var2=value2
 //  - http(s)://domain(:port)/lala/kasdl/var1=value1/?not-a-var=not-a-value
 //  - folder/folder/folder/../var1=value1/etc/.//var2=value2
-const string HivePartitioning::REGEX_STRING = "[\\/\\\\]([^\\/\\?\\\\]+)=([^\\/\\n\\?\\\\]+)";
+const string &HivePartitioning::RegexString() {
+	static string REGEX = "[\\/\\\\]([^\\/\\?\\\\]+)=([^\\/\\n\\?\\\\]+)";
+	return REGEX;
+}
 
 std::map<string, string> HivePartitioning::Parse(const string &filename, duckdb_re2::RE2 &regex) {
 	std::map<string, string> result;
@@ -79,7 +82,7 @@ std::map<string, string> HivePartitioning::Parse(const string &filename, duckdb_
 }
 
 std::map<string, string> HivePartitioning::Parse(const string &filename) {
-	duckdb_re2::RE2 regex(REGEX_STRING);
+	duckdb_re2::RE2 regex(RegexString());
 	return Parse(filename, regex);
 }
 
@@ -94,7 +97,7 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
 	vector<bool> have_preserved_filter(filters.size(), false);
 	vector<unique_ptr<Expression>> pruned_filters;
 	unordered_set<idx_t> filters_applied_to_files;
-	duckdb_re2::RE2 regex(REGEX_STRING);
+	duckdb_re2::RE2 regex(RegexString());
 	auto table_index = get.table_index;
 
 	if ((!filename_enabled && !hive_enabled) || filters.empty()) {

package/src/duckdb/src/common/multi_file_reader.cpp

@@ -102,7 +102,9 @@ bool MultiFileReader::ComplexFilterPushdown(ClientContext &context, vector<strin
 
 	unordered_map<string, column_t> column_map;
 	for (idx_t i = 0; i < get.column_ids.size(); i++) {
-		column_map.insert({get.names[get.column_ids[i]], i});
+		if (!IsRowIdColumnId(get.column_ids[i])) {
+			column_map.insert({get.names[get.column_ids[i]], i});
+		}
 	}
 
 	auto start_files = files.size();
@@ -432,7 +434,7 @@ void MultiFileReaderOptions::AutoDetectHiveTypesInternal(const string &file, Cli
 		}
 		Value value(part.second);
 		for (auto &candidate : candidates) {
-			const bool success = value.TryCastAs(context, candidate);
+			const bool success = value.TryCastAs(context, candidate, true);
 			if (success) {
 				hive_types_schema[name] = candidate;
 				break;

package/src/duckdb/src/common/types/list_segment.cpp

@@ -462,6 +462,10 @@ void SegmentPrimitiveFunction(ListSegmentFunctions &functions) {
 
 void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type) {
 
+	if (type.id() == LogicalTypeId::UNKNOWN) {
+		throw ParameterNotResolvedException();
+	}
+
 	auto physical_type = type.InternalType();
 	switch (physical_type) {
 	case PhysicalType::BIT:

package/src/duckdb/src/common/types/vector.cpp

@@ -1131,9 +1131,12 @@ void Vector::VerifyMap(Vector &vector_p, const SelectionVector &sel_p, idx_t cou
 
 void Vector::VerifyUnion(Vector &vector_p, const SelectionVector &sel_p, idx_t count) {
 #ifdef DEBUG
+
 	D_ASSERT(vector_p.GetType().id() == LogicalTypeId::UNION);
 	auto valid_check = UnionVector::CheckUnionValidity(vector_p, count, sel_p);
-	D_ASSERT(valid_check == UnionInvalidReason::VALID);
+	if (valid_check != UnionInvalidReason::VALID) {
+		throw InternalException("Union not valid, reason: %s", EnumUtil::ToString(valid_check));
+	}
 #endif // DEBUG
 }
 
@@ -1250,7 +1253,8 @@ void Vector::Verify(Vector &vector_p, const SelectionVector &sel_p, idx_t count)
 		}
 
 		if (vector->GetType().id() == LogicalTypeId::UNION) {
-			VerifyUnion(*vector, *sel, count);
+			// Pass in raw vector
+			VerifyUnion(vector_p, sel_p, count);
 		}
 	}
 
@@ -1911,7 +1915,13 @@ void UnionVector::SetToMember(Vector &union_vector, union_tag_t tag, Vector &mem
 		// if the member vector is constant, we can set the union to constant as well
 		union_vector.SetVectorType(VectorType::CONSTANT_VECTOR);
 		ConstantVector::GetData<union_tag_t>(tag_vector)[0] = tag;
-		ConstantVector::SetNull(union_vector, ConstantVector::IsNull(member_vector));
+		if (keep_tags_for_null) {
+			ConstantVector::SetNull(union_vector, false);
+			ConstantVector::SetNull(tag_vector, false);
+		} else {
+			ConstantVector::SetNull(union_vector, ConstantVector::IsNull(member_vector));
+			ConstantVector::SetNull(tag_vector, ConstantVector::IsNull(member_vector));
+		}
 
 	} else {
 		// otherwise flatten and set to flatvector
@@ -1962,53 +1972,75 @@ union_tag_t UnionVector::GetTag(const Vector &vector, idx_t index) {
 	return FlatVector::GetData<union_tag_t>(tag_vector)[index];
 }
 
-UnionInvalidReason UnionVector::CheckUnionValidity(Vector &vector, idx_t count, const SelectionVector &sel) {
-	D_ASSERT(vector.GetType().id() == LogicalTypeId::UNION);
-	auto member_count = UnionType::GetMemberCount(vector.GetType());
+//! Raw selection vector passed in (not merged with any other selection vectors)
+UnionInvalidReason UnionVector::CheckUnionValidity(Vector &vector_p, idx_t count, const SelectionVector &sel_p) {
+	D_ASSERT(vector_p.GetType().id() == LogicalTypeId::UNION);
+
+	// Will contain the (possibly) merged selection vector
+	const SelectionVector *sel = &sel_p;
+	SelectionVector owned_sel;
+	Vector *vector = &vector_p;
+	if (vector->GetVectorType() == VectorType::DICTIONARY_VECTOR) {
+		// In the case of a dictionary vector, unwrap the Vector, and merge the selection vectors.
+		auto &child = DictionaryVector::Child(*vector);
+		D_ASSERT(child.GetVectorType() != VectorType::DICTIONARY_VECTOR);
+		auto &dict_sel = DictionaryVector::SelVector(*vector);
+		// merge the selection vectors and verify the child
+		auto new_buffer = dict_sel.Slice(*sel, count);
+		owned_sel.Initialize(new_buffer);
+		sel = &owned_sel;
+		vector = &child;
+	} else if (vector->GetVectorType() == VectorType::CONSTANT_VECTOR) {
+		sel = ConstantVector::ZeroSelectionVector(count, owned_sel);
+	}
+
+	auto member_count = UnionType::GetMemberCount(vector_p.GetType());
 	if (member_count == 0) {
 		return UnionInvalidReason::NO_MEMBERS;
 	}
 
-	UnifiedVectorFormat union_vdata;
-	vector.ToUnifiedFormat(count, union_vdata);
+	UnifiedVectorFormat vector_vdata;
+	vector_p.ToUnifiedFormat(count, vector_vdata);
 
-	UnifiedVectorFormat tags_vdata;
-	auto &tag_vector = UnionVector::GetTags(vector);
-	tag_vector.ToUnifiedFormat(count, tags_vdata);
+	auto &entries = StructVector::GetEntries(vector_p);
+	duckdb::vector<UnifiedVectorFormat> child_vdata(entries.size());
+	for (idx_t entry_idx = 0; entry_idx < entries.size(); entry_idx++) {
+		auto &child = *entries[entry_idx];
+		child.ToUnifiedFormat(count, child_vdata[entry_idx]);
+	}
+
+	auto &tag_vdata = child_vdata[0];
 
-	// check that only one member is valid at a time
 	for (idx_t row_idx = 0; row_idx < count; row_idx++) {
-		auto union_mapped_row_idx = sel.get_index(row_idx);
-		if (!union_vdata.validity.RowIsValid(union_mapped_row_idx)) {
-			continue;
-		}
+		auto mapped_idx = sel->get_index(row_idx);
 
-		auto tag_mapped_row_idx = tags_vdata.sel->get_index(row_idx);
-		if (!tags_vdata.validity.RowIsValid(tag_mapped_row_idx)) {
+		if (!vector_vdata.validity.RowIsValid(mapped_idx)) {
 			continue;
 		}
 
-		auto tag = (UnifiedVectorFormat::GetData<union_tag_t>(tags_vdata))[tag_mapped_row_idx];
+		auto tag_idx = tag_vdata.sel->get_index(sel_p.get_index(row_idx));
+		if (!tag_vdata.validity.RowIsValid(tag_idx)) {
+			// we can't have NULL tags!
+			return UnionInvalidReason::NULL_TAG;
+		}
+		auto tag = UnifiedVectorFormat::GetData<union_tag_t>(tag_vdata)[tag_idx];
 		if (tag >= member_count) {
 			return UnionInvalidReason::TAG_OUT_OF_RANGE;
 		}
 
 		bool found_valid = false;
-		for (idx_t member_idx = 0; member_idx < member_count; member_idx++) {
-
-			UnifiedVectorFormat member_vdata;
-			auto &member = UnionVector::GetMember(vector, member_idx);
-			member.ToUnifiedFormat(count, member_vdata);
-
-			auto mapped_row_idx = member_vdata.sel->get_index(row_idx);
-			if (member_vdata.validity.RowIsValid(mapped_row_idx)) {
-				if (found_valid) {
-					return UnionInvalidReason::VALIDITY_OVERLAP;
-				}
-				found_valid = true;
-				if (tag != static_cast<union_tag_t>(member_idx)) {
-					return UnionInvalidReason::TAG_MISMATCH;
-				}
+		for (idx_t i = 0; i < member_count; i++) {
+			auto &member_vdata = child_vdata[1 + i]; // skip the tag
+			idx_t member_idx = member_vdata.sel->get_index(sel_p.get_index(row_idx));
+			if (!member_vdata.validity.RowIsValid(member_idx)) {
+				continue;
+			}
+			if (found_valid) {
+				return UnionInvalidReason::VALIDITY_OVERLAP;
+			}
+			found_valid = true;
+			if (tag != static_cast<union_tag_t>(i)) {
+				return UnionInvalidReason::TAG_MISMATCH;
 			}
 		}
 	}

package/src/duckdb/src/common/types.cpp

@@ -373,7 +373,9 @@ string LogicalType::ToString() const {
 		string ret = "UNION(";
 		size_t count = UnionType::GetMemberCount(*this);
 		for (size_t i = 0; i < count; i++) {
-			ret += UnionType::GetMemberName(*this, i) + " " + UnionType::GetMemberType(*this, i).ToString();
+			auto member_name = UnionType::GetMemberName(*this, i);
+			auto member_type = UnionType::GetMemberType(*this, i).ToString();
+			ret += StringUtil::Format("%s %s", SQLIdentifier(member_name), member_type);
 			if (i < count - 1) {
 				ret += ", ";
 			}