duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp

@@ -14,17 +14,18 @@
 #include "duckdb/common/vector_size.hpp"
 
 namespace duckdb {
+struct SelectionVector;
 struct ValidityMask;
 
 template <typename V>
 struct TemplatedValidityData {
-	static constexpr const int BITS_PER_VALUE = sizeof(V) * 8;
+	static constexpr const idx_t BITS_PER_VALUE = sizeof(V) * 8;
 	static constexpr const V MAX_ENTRY = V(~V(0));
 
 public:
 	inline explicit TemplatedValidityData(idx_t count) {
 		auto entry_count = EntryCount(count);
-		owned_data = make_unsafe_uniq_array<V>(entry_count);
+		owned_data = make_unsafe_uniq_array_uninitialized<V>(entry_count);
 		for (idx_t entry_idx = 0; entry_idx < entry_count; entry_idx++) {
 			owned_data[entry_idx] = MAX_ENTRY;
 		}
@@ -32,7 +33,7 @@ public:
 	inline TemplatedValidityData(const V *validity_mask, idx_t count) {
 		D_ASSERT(validity_mask);
 		auto entry_count = EntryCount(count);
-		owned_data = make_unsafe_uniq_array<V>(entry_count);
+		owned_data = make_unsafe_uniq_array_uninitialized<V>(entry_count);
 		for (idx_t entry_idx = 0; entry_idx < entry_count; entry_idx++) {
 			owned_data[entry_idx] = validity_mask[entry_idx];
 		}
@@ -60,9 +61,9 @@ struct TemplatedValidityMask {
 	using ValidityBuffer = TemplatedValidityData<V>;
 
 public:
-	static constexpr const int BITS_PER_VALUE = ValidityBuffer::BITS_PER_VALUE;
-	static constexpr const int STANDARD_ENTRY_COUNT = (STANDARD_VECTOR_SIZE + (BITS_PER_VALUE - 1)) / BITS_PER_VALUE;
-	static constexpr const int STANDARD_MASK_SIZE = STANDARD_ENTRY_COUNT * sizeof(validity_t);
+	static constexpr const idx_t BITS_PER_VALUE = ValidityBuffer::BITS_PER_VALUE;
+	static constexpr const idx_t STANDARD_ENTRY_COUNT = (STANDARD_VECTOR_SIZE + (BITS_PER_VALUE - 1)) / BITS_PER_VALUE;
+	static constexpr const idx_t STANDARD_MASK_SIZE = STANDARD_ENTRY_COUNT * sizeof(validity_t);
 
 public:
 	inline TemplatedValidityMask() : validity_mask(nullptr), target_count(STANDARD_VECTOR_SIZE) {
@@ -261,8 +262,9 @@ public:
 		for (idx_t i = 0; i < last_entry_index; i++) {
 			validity_mask[i] = 0;
 		}
-		auto last_entry_bits = count % static_cast<idx_t>(BITS_PER_VALUE);
-		validity_mask[last_entry_index] = (last_entry_bits == 0) ? 0 : (ValidityBuffer::MAX_ENTRY << (last_entry_bits));
+		auto last_entry_bits = count % BITS_PER_VALUE;
+		validity_mask[last_entry_index] =
+		    (last_entry_bits == 0) ? 0 : static_cast<V>(ValidityBuffer::MAX_ENTRY << (last_entry_bits));
 	}
 
 	//! Marks exactly "count" bits in the validity mask as valid (not null)
@@ -275,9 +277,10 @@ public:
 		for (idx_t i = 0; i < last_entry_index; i++) {
 			validity_mask[i] = ValidityBuffer::MAX_ENTRY;
 		}
-		auto last_entry_bits = count % static_cast<idx_t>(BITS_PER_VALUE);
-		validity_mask[last_entry_index] |=
-		    (last_entry_bits == 0) ? ValidityBuffer::MAX_ENTRY : ~(ValidityBuffer::MAX_ENTRY << (last_entry_bits));
+		auto last_entry_bits = count % BITS_PER_VALUE;
+		validity_mask[last_entry_index] |= (last_entry_bits == 0)
+		                                       ? ValidityBuffer::MAX_ENTRY
+		                                       : ~static_cast<V>(ValidityBuffer::MAX_ENTRY << (last_entry_bits));
 	}
 
 	inline bool IsMaskSet() const {
@@ -336,9 +339,11 @@ public:
 
 public:
 	DUCKDB_API void Resize(idx_t old_size, idx_t new_size);
-	DUCKDB_API idx_t TargetCount();
+	DUCKDB_API idx_t TargetCount() const;
 	DUCKDB_API void SliceInPlace(const ValidityMask &other, idx_t target_offset, idx_t source_offset, idx_t count);
 	DUCKDB_API void Slice(const ValidityMask &other, idx_t source_offset, idx_t count);
+	DUCKDB_API void CopySel(const ValidityMask &other, const SelectionVector &sel, idx_t source_offset,
+	                        idx_t target_offset, idx_t count);
 	DUCKDB_API void Combine(const ValidityMask &other, idx_t count);
 	DUCKDB_API string ToString(idx_t count) const;
 
@@ -348,4 +353,90 @@ public:
 	void Read(ReadStream &reader, idx_t count);
 };
 
+//===--------------------------------------------------------------------===//
+// ValidityArray
+//===--------------------------------------------------------------------===//
+struct ValidityArray {
+	inline ValidityArray() {
+	}
+
+	inline bool AllValid() const {
+		return !validity_mask;
+	}
+
+	inline void Initialize(idx_t count, bool initial = true) {
+		target_count = count;
+		validity_data = make_unsafe_uniq_array<bool>(count);
+		validity_mask = validity_data.get();
+		memset(validity_mask, initial, sizeof(bool) * count);
+	}
+
+	//! RowIsValidUnsafe should only be used if AllValid() is false: it achieves the same as RowIsValid but skips a
+	//! not-null check
+	inline bool RowIsValidUnsafe(idx_t row_idx) const {
+		D_ASSERT(validity_mask);
+		return validity_mask[row_idx];
+	}
+
+	//! Returns true if a row is valid (i.e. not null), false otherwise
+	inline bool RowIsValid(idx_t row_idx) const {
+		if (!validity_mask) {
+			return true;
+		}
+		return RowIsValidUnsafe(row_idx);
+	}
+
+	//! Same as SetValid, but skips a null check on validity_mask
+	inline void SetValidUnsafe(idx_t row_idx) {
+		D_ASSERT(validity_mask);
+		validity_mask[row_idx] = true;
+	}
+
+	//! Marks the entry at the specified row index as valid (i.e. not-null)
+	inline void SetValid(idx_t row_idx) {
+		if (!validity_mask) {
+			// if AllValid() we don't need to do anything
+			// the row is already valid
+			return;
+		}
+
+		SetValidUnsafe(row_idx);
+	}
+
+	inline void Pack(ValidityMask &mask, const idx_t count) const {
+		if (AllValid()) {
+			mask.Reset();
+			return;
+		}
+		mask.Initialize(count);
+
+		const auto entire_entries = count / ValidityMask::BITS_PER_VALUE;
+		const auto ragged = count % ValidityMask::BITS_PER_VALUE;
+		auto bits = mask.GetData();
+		idx_t row_idx = 0;
+		for (idx_t i = 0; i < entire_entries; ++i) {
+			validity_t entry = 0;
+			for (idx_t j = 0; j < ValidityMask::BITS_PER_VALUE; ++j) {
+				if (RowIsValidUnsafe(row_idx++)) {
+					entry |= validity_t(1) << j;
+				}
+			}
+			*bits++ = entry;
+		}
+		if (ragged) {
+			validity_t entry = 0;
+			for (idx_t j = 0; j < ragged; ++j) {
+				if (RowIsValidUnsafe(row_idx++)) {
+					entry |= validity_t(1) << j;
+				}
+			}
+			*bits++ = entry;
+		}
+	}
+
+	bool *validity_mask = nullptr;
+	unsafe_unique_array<bool> validity_data;
+	idx_t target_count = 0;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/value.hpp

@@ -33,6 +33,7 @@ class Value {
 	friend struct ListValue;
 	friend struct UnionValue;
 	friend struct ArrayValue;
+	friend struct MapValue;
 
 public:
 	//! Create an empty NULL value of the specified type
@@ -90,8 +91,8 @@ public:
 	DUCKDB_API static Value Numeric(const LogicalType &type, hugeint_t value);
 	DUCKDB_API static Value Numeric(const LogicalType &type, uhugeint_t value);
 
-	//! Create a tinyint Value from a specified value
-	DUCKDB_API static Value BOOLEAN(int8_t value);
+	//! Create a boolean Value from a specified value
+	DUCKDB_API static Value BOOLEAN(bool value);
 	//! Create a tinyint Value from a specified value
 	DUCKDB_API static Value TINYINT(int8_t value);
 	//! Create a smallint Value from a specified value
@@ -193,6 +194,8 @@ public:
 	//! Creates a bitstring by casting a specified string to a bitstring
 	DUCKDB_API static Value BIT(const_data_ptr_t data, idx_t len);
 	DUCKDB_API static Value BIT(const string &data);
+	DUCKDB_API static Value VARINT(const_data_ptr_t data, idx_t len);
+	DUCKDB_API static Value VARINT(const string &data);
 
 	//! Creates an aggregate state
 	DUCKDB_API static Value AGGREGATE_STATE(const LogicalType &type, const_data_ptr_t data, idx_t len); // NOLINT
@@ -307,7 +310,7 @@ private:
 
 	//! The value of the object, if it is of a constant size Type
 	union Val {
-		int8_t boolean;
+		bool boolean;
 		int8_t tinyint;
 		int16_t smallint;
 		int32_t integer;
@@ -417,6 +420,10 @@ struct StructValue {
 	DUCKDB_API static const vector<Value> &GetChildren(const Value &value);
 };
 
+struct MapValue {
+	DUCKDB_API static const vector<Value> &GetChildren(const Value &value);
+};
+
 struct ListValue {
 	DUCKDB_API static const vector<Value> &GetChildren(const Value &value);
 };
@@ -523,6 +530,8 @@ DUCKDB_API date_t Value::GetValue() const;
 template <>
 DUCKDB_API dtime_t Value::GetValue() const;
 template <>
+DUCKDB_API dtime_tz_t Value::GetValue() const;
+template <>
 DUCKDB_API timestamp_t Value::GetValue() const;
 template <>
 DUCKDB_API interval_t Value::GetValue() const;

package/src/duckdb/src/include/duckdb/common/types/varint.hpp

@@ -0,0 +1,107 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/types/varint.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/limits.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types.hpp"
+#include "duckdb/common/winapi.hpp"
+#include "duckdb/function/cast/default_casts.hpp"
+
+namespace duckdb {
+//! The Varint class is a static class that holds helper functions for the Varint type.
+class Varint {
+public:
+	//! Header size of a Varint is always 3 bytes.
+	DUCKDB_API static constexpr uint8_t VARINT_HEADER_SIZE = 3;
+	//! Verifies if a Varint is valid. i.e., if it has 3 header bytes. The header correctly represents the number of
+	//! data bytes, and the data bytes has no leading zero bytes.
+	DUCKDB_API static void Verify(const string_t &input);
+
+	//! Sets the header of a varint (i.e., char* blob), depending on the number of bytes that varint needs and if it's a
+	//! negative number
+	DUCKDB_API static void SetHeader(char *blob, uint64_t number_of_bytes, bool is_negative);
+	//! Initializes and returns a blob with value 0, allocated in Vector& result
+	DUCKDB_API static string_t InitializeVarintZero(Vector &result);
+	DUCKDB_API static string InitializeVarintZero();
+
+	//! Switch Case of To Varint Convertion
+	DUCKDB_API static BoundCastInfo NumericToVarintCastSwitch(const LogicalType &source);
+
+	//! ----------------------------------- Varchar Cast ----------------------------------- //
+	//! Function to prepare a varchar for conversion. We trim zero's, check for negative values, and what-not
+	//! Returns false if this is an invalid varchar
+	DUCKDB_API static bool VarcharFormatting(const string_t &value, idx_t &start_pos, idx_t &end_pos, bool &is_negative,
+	                                         bool &is_zero);
+
+	//! Converts a char to a Digit
+	DUCKDB_API static int CharToDigit(char c);
+	//! Converts a Digit to a char
+	DUCKDB_API static char DigitToChar(int digit);
+	//! Function to convert a string_t into a vector of bytes
+	DUCKDB_API static void GetByteArray(vector<uint8_t> &byte_array, bool &is_negative, const string_t &blob);
+	//! Function to convert VARINT blob to a VARCHAR
+	DUCKDB_API static string VarIntToVarchar(const string_t &blob);
+	//! Function to convert Varchar to VARINT blob
+	DUCKDB_API static string VarcharToVarInt(const string_t &value);
+	//! ----------------------------------- Double Cast ----------------------------------- //
+	DUCKDB_API static bool VarintToDouble(const string_t &blob, double &result, bool &strict);
+};
+
+//! ----------------------------------- (u)Integral Cast ----------------------------------- //
+struct IntCastToVarInt {
+	template <class SRC>
+	static inline string_t Operation(SRC input, Vector &result) {
+		return IntToVarInt(result, input);
+	}
+};
+
+//! ----------------------------------- (u)HugeInt Cast ----------------------------------- //
+struct HugeintCastToVarInt {
+	template <class SRC>
+	static inline string_t Operation(SRC input, Vector &result) {
+		throw InternalException("Unsupported type for cast to VARINT");
+	}
+};
+
+struct TryCastToVarInt {
+	template <class SRC, class DST>
+	static inline bool Operation(SRC input, DST &result, Vector &result_vector, CastParameters &parameters) {
+		throw InternalException("Unsupported type for try cast to VARINT");
+	}
+};
+
+template <>
+DUCKDB_API bool TryCastToVarInt::Operation(double double_value, string_t &result_value, Vector &result,
+                                           CastParameters &parameters);
+
+template <>
+DUCKDB_API bool TryCastToVarInt::Operation(float float_value, string_t &result_value, Vector &result,
+                                           CastParameters &parameters);
+
+template <>
+DUCKDB_API bool TryCastToVarInt::Operation(string_t input_value, string_t &result_value, Vector &result,
+                                           CastParameters &parameters);
+
+struct VarIntCastToVarchar {
+	template <class SRC>
+	DUCKDB_API static inline string_t Operation(SRC input, Vector &result) {
+		return StringVector::AddStringOrBlob(result, Varint::VarIntToVarchar(input));
+	}
+};
+
+struct VarintToDoubleCast {
+	template <class SRC, class DST>
+	DUCKDB_API static inline bool Operation(SRC input, DST &result, bool strict = false) {
+		return Varint::VarintToDouble(input, result, strict);
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -204,6 +204,8 @@ public:
 	DUCKDB_API void Serialize(Serializer &serializer, idx_t count);
 	DUCKDB_API void Deserialize(Deserializer &deserializer, idx_t count);
 
+	idx_t GetAllocationSize(idx_t cardinality) const;
+
 	// Getters
 	inline VectorType GetVectorType() const {
 		return vector_type;
@@ -470,8 +472,10 @@ struct FSSTVector {
 
 	DUCKDB_API static string_t AddCompressedString(Vector &vector, string_t data);
 	DUCKDB_API static string_t AddCompressedString(Vector &vector, const char *data, idx_t len);
-	DUCKDB_API static void RegisterDecoder(Vector &vector, buffer_ptr<void> &duckdb_fsst_decoder);
+	DUCKDB_API static void RegisterDecoder(Vector &vector, buffer_ptr<void> &duckdb_fsst_decoder,
+	                                       const idx_t string_block_limit);
 	DUCKDB_API static void *GetDecoder(const Vector &vector);
+	DUCKDB_API static vector<unsigned char> &GetDecompressBuffer(const Vector &vector);
 	//! Setting the string count is required to be able to correctly flatten the vector
 	DUCKDB_API static void SetCount(Vector &vector, idx_t count);
 	DUCKDB_API static idx_t GetCount(Vector &vector);

package/src/duckdb/src/include/duckdb/common/types/vector_buffer.hpp

@@ -72,7 +72,7 @@ public:
 	}
 	explicit VectorBuffer(idx_t data_size) : buffer_type(VectorBufferType::STANDARD_BUFFER) {
 		if (data_size > 0) {
-			data = make_unsafe_uniq_array<data_t>(data_size);
+			data = make_unsafe_uniq_array_uninitialized<data_t>(data_size);
 		}
 	}
 	explicit VectorBuffer(unsafe_unique_array<data_t> data_p)
@@ -199,12 +199,16 @@ public:
 	VectorFSSTStringBuffer();
 
 public:
-	void AddDecoder(buffer_ptr<void> &duckdb_fsst_decoder_p) {
+	void AddDecoder(buffer_ptr<void> &duckdb_fsst_decoder_p, const idx_t string_block_limit) {
 		duckdb_fsst_decoder = duckdb_fsst_decoder_p;
+		decompress_buffer.resize(string_block_limit + 1);
 	}
 	void *GetDecoder() {
 		return duckdb_fsst_decoder.get();
 	}
+	vector<unsigned char> &GetDecompressBuffer() {
+		return decompress_buffer;
+	}
 	void SetCount(idx_t count) {
 		total_string_count = count;
 	}
@@ -215,6 +219,7 @@ public:
 private:
 	buffer_ptr<void> duckdb_fsst_decoder;
 	idx_t total_string_count = 0;
+	vector<unsigned char> decompress_buffer;
 };
 
 class VectorStructBuffer : public VectorBuffer {

package/src/duckdb/src/include/duckdb/common/types.hpp

@@ -215,7 +215,7 @@ enum class LogicalTypeId : uint8_t {
 	BIT = 36,
 	STRING_LITERAL = 37, /* string literals, used for constant strings - only exists while binding */
 	INTEGER_LITERAL = 38,/* integer literals, used for constant integers - only exists while binding */
-
+	VARINT = 39,
 	UHUGEINT = 49,
 	HUGEINT = 50,
 	POINTER = 51,
@@ -318,6 +318,7 @@ struct LogicalType {
 	DUCKDB_API string ToString() const;
 	DUCKDB_API bool IsIntegral() const;
 	DUCKDB_API bool IsNumeric() const;
+	DUCKDB_API bool IsTemporal() const;
 	DUCKDB_API hash_t Hash() const;
 	DUCKDB_API void SetAlias(string alias);
 	DUCKDB_API bool HasAlias() const;
@@ -344,9 +345,6 @@ struct LogicalType {
 
 	DUCKDB_API bool IsValid() const;
 
-	template<class F>
-	bool Contains(F &&predicate) const;
-	bool Contains(LogicalTypeId type_id) const;
 
 private:
 	LogicalTypeId id_; // NOLINT: allow this naming for legacy reasons
@@ -382,6 +380,8 @@ public:
 	static constexpr const LogicalTypeId ANY = LogicalTypeId::ANY;
 	static constexpr const LogicalTypeId BLOB = LogicalTypeId::BLOB;
 	static constexpr const LogicalTypeId BIT = LogicalTypeId::BIT;
+	static constexpr const LogicalTypeId VARINT = LogicalTypeId::VARINT;
+
 	static constexpr const LogicalTypeId INTERVAL = LogicalTypeId::INTERVAL;
 	static constexpr const LogicalTypeId HUGEINT = LogicalTypeId::HUGEINT;
 	static constexpr const LogicalTypeId UHUGEINT = LogicalTypeId::UHUGEINT;
@@ -402,9 +402,7 @@ public:
 	DUCKDB_API static LogicalType MAP(const LogicalType &child);                 // NOLINT
 	DUCKDB_API static LogicalType MAP(LogicalType key, LogicalType value);       // NOLINT
 	DUCKDB_API static LogicalType UNION(child_list_t<LogicalType> members);      // NOLINT
-	DUCKDB_API static LogicalType ARRAY(const LogicalType &child, idx_t size);   // NOLINT
-	// an array of unknown size (only used for binding)
-	DUCKDB_API static LogicalType ARRAY(const LogicalType &child);        // NOLINT
+	DUCKDB_API static LogicalType ARRAY(const LogicalType &child, optional_idx index);   // NOLINT
 	DUCKDB_API static LogicalType ENUM(Vector &ordered_data, idx_t size); // NOLINT
 	// ANY but with special rules (default is LogicalType::ANY, 5)
 	DUCKDB_API static LogicalType ANY_PARAMS(LogicalType target, idx_t cast_score = 5); // NOLINT
@@ -546,37 +544,6 @@ struct aggregate_state_t {
 	vector<LogicalType> bound_argument_types;
 };
 
-template<class F>
-bool LogicalType::Contains(F &&predicate) const {
-	if(predicate(*this)) {
-		return true;
-	}
-	switch(id()) {
-	case LogicalTypeId::STRUCT: {
-		for(const auto &child : StructType::GetChildTypes(*this)) {
-			if(child.second.Contains(predicate)) {
-				return true;
-			}
-		}
-		}
-		break;
-	case LogicalTypeId::LIST:
-		return ListType::GetChildType(*this).Contains(predicate);
-	case LogicalTypeId::MAP:
-		return MapType::KeyType(*this).Contains(predicate) || MapType::ValueType(*this).Contains(predicate);
-	case LogicalTypeId::UNION:
-		for(const auto &child : UnionType::CopyMemberTypes(*this)) {
-			if(child.second.Contains(predicate)) {
-				return true;
-			}
-		}
-		break;
-	case LogicalTypeId::ARRAY:
-		return ArrayType::GetChildType(*this).Contains(predicate);
-	default:
-		return false;
-	}
-	return false;
-}
+
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/union_by_name.hpp

@@ -12,9 +12,32 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/helper.hpp"
+#include "duckdb/parallel/task_executor.hpp"
 
 namespace duckdb {
 
+template <class READER_TYPE, class OPTION_TYPE>
+class UnionByReaderTask : public BaseExecutorTask {
+public:
+	UnionByReaderTask(TaskExecutor &executor, ClientContext &context, const string &file, idx_t file_idx,
+	                  vector<typename READER_TYPE::UNION_READER_DATA> &readers, OPTION_TYPE &options)
+	    : BaseExecutorTask(executor), context(context), file_name(file), file_idx(file_idx), readers(readers),
+	      options(options) {
+	}
+
+	void ExecuteTask() override {
+		auto reader = make_uniq<READER_TYPE>(context, file_name, options);
+		readers[file_idx] = READER_TYPE::StoreUnionReader(std::move(reader), file_idx);
+	}
+
+private:
+	ClientContext &context;
+	const string &file_name;
+	idx_t file_idx;
+	vector<typename READER_TYPE::UNION_READER_DATA> &readers;
+	OPTION_TYPE &options;
+};
+
 class UnionByName {
 public:
 	static void CombineUnionTypes(const vector<string> &new_names, const vector<LogicalType> &new_types,
@@ -23,19 +46,28 @@ public:
 
 	//! Union all files(readers) by their col names
 	template <class READER_TYPE, class OPTION_TYPE>
-	static vector<unique_ptr<READER_TYPE>> UnionCols(ClientContext &context, const vector<string> &files,
-	                                                 vector<LogicalType> &union_col_types,
-	                                                 vector<string> &union_col_names, OPTION_TYPE &options) {
-		vector<unique_ptr<READER_TYPE>> union_readers;
-		case_insensitive_map_t<idx_t> union_names_map;
+	static vector<typename READER_TYPE::UNION_READER_DATA>
+	UnionCols(ClientContext &context, const vector<string> &files, vector<LogicalType> &union_col_types,
+	          vector<string> &union_col_names, OPTION_TYPE &options) {
+		vector<typename READER_TYPE::UNION_READER_DATA> union_readers;
+		union_readers.resize(files.size());
+
+		TaskExecutor executor(context);
+		// schedule tasks for all files
 		for (idx_t file_idx = 0; file_idx < files.size(); ++file_idx) {
-			const auto &file_name = files[file_idx];
-			auto reader = make_uniq<READER_TYPE>(context, file_name, options);
+			auto task = make_uniq<UnionByReaderTask<READER_TYPE, OPTION_TYPE>>(executor, context, files[file_idx],
+			                                                                   file_idx, union_readers, options);
+			executor.ScheduleTask(std::move(task));
+		}
+		// complete all tasks
+		executor.WorkOnTasks();
 
-			auto &col_names = reader->GetNames();
-			auto &sql_types = reader->GetTypes();
+		// now combine the result schemas
+		case_insensitive_map_t<idx_t> union_names_map;
+		for (auto &reader : union_readers) {
+			auto &col_names = reader->names;
+			auto &sql_types = reader->types;
 			CombineUnionTypes(col_names, sql_types, union_col_types, union_col_names, union_names_map);
-			union_readers.push_back(std::move(reader));
 		}
 		return union_readers;
 	}

package/src/duckdb/src/include/duckdb/common/vector_operations/generic_executor.hpp

@@ -213,6 +213,35 @@ struct StructTypeQuaternary {
 	}
 };
 
+template <class CHILD_TYPE>
+struct GenericListType {
+	vector<CHILD_TYPE> values;
+
+	using STRUCT_STATE = PrimitiveTypeState;
+
+	static bool ConstructType(STRUCT_STATE &state, idx_t i, GenericListType<CHILD_TYPE> &result) {
+		throw InternalException("FIXME: implement ConstructType for lists");
+	}
+
+	static void AssignResult(Vector &result, idx_t i, GenericListType<CHILD_TYPE> value) {
+		auto &child = ListVector::GetEntry(result);
+		auto current_size = ListVector::GetListSize(result);
+
+		// reserve space in the child element
+		auto list_size = value.values.size();
+		ListVector::Reserve(result, current_size + list_size);
+
+		auto list_entries = FlatVector::GetData<list_entry_t>(result);
+		list_entries[i].offset = current_size;
+		list_entries[i].length = list_size;
+
+		for (idx_t child_idx = 0; child_idx < list_size; child_idx++) {
+			CHILD_TYPE::AssignResult(child, current_size + child_idx, value.values[child_idx]);
+		}
+		ListVector::SetListSize(result, current_size + list_size);
+	}
+};
+
 //! The GenericExecutor can handle struct types in addition to primitive types
 struct GenericExecutor {
 private:

package/src/duckdb/src/include/duckdb/common/vector_operations/unary_executor.hpp

@@ -72,7 +72,6 @@ private:
 #endif
 
 		if (!mask.AllValid()) {
-			result_mask.EnsureWritable();
 			for (idx_t i = 0; i < count; i++) {
 				auto idx = sel_vector->get_index(i);
 				if (mask.RowIsValidUnsafe(idx)) {
@@ -83,9 +82,6 @@ private:
 				}
 			}
 		} else {
-			if (adds_nulls) {
-				result_mask.EnsureWritable();
-			}
 			for (idx_t i = 0; i < count; i++) {
 				auto idx = sel_vector->get_index(i);
 				result_data[i] =
@@ -133,9 +129,6 @@ private:
 				}
 			}
 		} else {
-			if (adds_nulls) {
-				result_mask.EnsureWritable();
-			}
 			for (idx_t i = 0; i < count; i++) {
 				result_data[i] =
 				    OPWRAPPER::template Operation<OP, INPUT_TYPE, RESULT_TYPE>(ldata[i], result_mask, i, dataptr);

package/src/duckdb/src/include/duckdb/common/vector_operations/vector_operations.hpp

@@ -192,6 +192,8 @@ struct VectorOperations {
 	                 idx_t target_offset);
 	static void Copy(const Vector &source, Vector &target, const SelectionVector &sel, idx_t source_count,
 	                 idx_t source_offset, idx_t target_offset);
+	static void Copy(const Vector &source, Vector &target, const SelectionVector &sel, idx_t source_count,
+	                 idx_t source_offset, idx_t target_offset, idx_t copy_count);
 
 	// Copy the data of <source> to the target location, setting null values to
 	// NullValue<T>. Used to store data without separate NULL mask.

package/src/duckdb/src/include/duckdb/common/winapi.hpp

@@ -10,11 +10,15 @@
 
 #ifndef DUCKDB_API
 #if defined(_WIN32) && !defined(__MINGW32__)
+#ifdef DUCKDB_STATIC_BUILD
+#define DUCKDB_API
+#else
 #if defined(DUCKDB_BUILD_LIBRARY) && !defined(DUCKDB_BUILD_LOADABLE_EXTENSION)
 #define DUCKDB_API __declspec(dllexport)
 #else
 #define DUCKDB_API __declspec(dllimport)
 #endif
+#endif
 #else
 #define DUCKDB_API
 #endif
@@ -22,11 +26,15 @@
 
 #ifndef DUCKDB_EXTENSION_API
 #ifdef _WIN32
+#ifdef DUCKDB_STATIC_BUILD
+#define DUCKDB_EXTENSION_API
+#else
 #ifdef DUCKDB_BUILD_LOADABLE_EXTENSION
 #define DUCKDB_EXTENSION_API __declspec(dllexport)
 #else
 #define DUCKDB_EXTENSION_API
 #endif
+#endif
 #else
 #define DUCKDB_EXTENSION_API __attribute__((visibility("default")))
 #endif