duckdb 0.6.2-dev971.0 → 0.7.1-dev2.0

package/src/duckdb/src/include/duckdb/common/types/bit.hpp

@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/types/bit.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/types.hpp"
+
+namespace duckdb {
+
+//! The Bit class is a static class that holds helper functions for the BIT type.
+class Bit {
+public:
+	//! Returns the number of bits in the bit string
+	DUCKDB_API static idx_t BitLength(string_t bits);
+	//! Returns the number of set bits in the bit string
+	DUCKDB_API static idx_t BitCount(string_t bits);
+	//! Returns the number of bytes in the bit string
+	DUCKDB_API static idx_t OctetLength(string_t bits);
+	//! Extracts the nth bit from bit string; the first (leftmost) bit is indexed 0
+	DUCKDB_API static idx_t GetBit(string_t bit_string, idx_t n);
+	//! Sets the nth bit in bit string to newvalue; the first (leftmost) bit is indexed 0
+	DUCKDB_API static void SetBit(const string_t &bit_string, idx_t n, idx_t new_value, string_t &result);
+	//! Returns first starting index of the specified substring within bits, or zero if it's not present.
+	DUCKDB_API static idx_t BitPosition(string_t substring, string_t bits);
+	//! Converts bits to a string, writing the output to the designated output string.
+	//! The string needs to have space for at least GetStringSize(bits) bytes.
+	DUCKDB_API static void ToString(string_t bits, char *output);
+	DUCKDB_API static string ToString(string_t str);
+	//! Returns the bit size of a string -> bit conversion
+	DUCKDB_API static bool TryGetBitStringSize(string_t str, idx_t &result_size, string *error_message);
+	//! Convert a string to a bit. This function should ONLY be called after calling GetBitSize, since it does NOT
+	//! perform data validation.
+	DUCKDB_API static void ToBit(string_t str, data_ptr_t output);
+	DUCKDB_API static string ToBit(string_t str);
+	DUCKDB_API static void SetEmptyBitString(string_t &target, string_t &input);
+
+	DUCKDB_API static void RightShift(const string_t &bit_string, const idx_t &shif, string_t &result);
+	DUCKDB_API static void LeftShift(const string_t &bit_string, const idx_t &shift, string_t &result);
+	DUCKDB_API static void BitwiseAnd(const string_t &rhs, const string_t &lhs, string_t &result);
+	DUCKDB_API static void BitwiseOr(const string_t &rhs, const string_t &lhs, string_t &result);
+	DUCKDB_API static void BitwiseXor(const string_t &rhs, const string_t &lhs, string_t &result);
+	DUCKDB_API static void BitwiseNot(const string_t &rhs, string_t &result);
+
+private:
+	//! Returns the amount of padded zeroes to fill up to a full byte. This information is stored in the first byte of
+	//! the bitstring.
+	DUCKDB_API static idx_t GetPadding(const string_t &bit_string);
+	DUCKDB_API static idx_t GetBitSize(string_t str);
+	DUCKDB_API static void SetBit(string_t &bit_string, idx_t n, idx_t new_value);
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/chunk_collection.hpp

@@ -109,7 +109,7 @@ public:
 			return nullptr;
 		}
 
-		auto res = move(chunks[0]);
+		auto res = std::move(chunks[0]);
 		chunks.erase(chunks.begin() + 0);
 		return res;
 	}

package/src/duckdb/src/include/duckdb/common/types/conflict_manager.hpp

@@ -0,0 +1,71 @@
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/execution/index/art/art.hpp"
+#include "duckdb/common/types/selection_vector.hpp"
+#include "duckdb/common/vector.hpp"
+
+namespace duckdb {
+
+class Index;
+class ConflictInfo;
+
+enum class ConflictManagerMode : uint8_t {
+	SCAN, // gather conflicts without throwing
+	THROW // throw on the conflicts that were not found during the scan
+};
+
+enum class LookupResultType : uint8_t { LOOKUP_MISS, LOOKUP_HIT, LOOKUP_NULL };
+
+class ConflictManager {
+public:
+	ConflictManager(VerifyExistenceType lookup_type, idx_t input_size, ConflictInfo *conflict_info = nullptr);
+
+public:
+	void SetIndexCount(idx_t count);
+	// These methods return a boolean indicating whether we should throw or not
+	bool AddMiss(idx_t chunk_index);
+	bool AddHit(idx_t chunk_index, row_t row_id);
+	bool AddNull(idx_t chunk_index);
+	VerifyExistenceType LookupType() const;
+	// This should be called before using the conflicts selection vector
+	void Finalize();
+	idx_t ConflictCount() const;
+	const ManagedSelection &Conflicts() const;
+	Vector &RowIds();
+	const ConflictInfo &GetConflictInfo() const;
+	void FinishLookup();
+	void SetMode(ConflictManagerMode mode);
+
+private:
+	bool IsConflict(LookupResultType type);
+	const unordered_set<idx_t> &InternalConflictSet() const;
+	Vector &InternalRowIds();
+	Vector &InternalIntermediate();
+	ManagedSelection &InternalSelection();
+	bool SingleIndexTarget() const;
+	bool ShouldThrow(idx_t chunk_index) const;
+	bool ShouldIgnoreNulls() const;
+	void AddConflictInternal(idx_t chunk_index, row_t row_id);
+	void AddToConflictSet(idx_t chunk_index);
+
+private:
+	VerifyExistenceType lookup_type;
+	idx_t input_size;
+	ConflictInfo *conflict_info;
+	idx_t index_count;
+	bool finalized = false;
+	ManagedSelection conflicts;
+	unique_ptr<Vector> row_ids;
+	// Used to check if a given conflict is part of the conflict target or not
+	unique_ptr<unordered_set<idx_t>> conflict_set;
+	// Contains 'input_size' booleans, indicating if a given index in the input chunk has a conflict
+	unique_ptr<Vector> intermediate_vector;
+	// Mapping from chunk_index to row_id
+	vector<row_t> row_id_map;
+	// Whether we have already found the one conflict target we're interested in
+	bool single_index_finished = false;
+	ConflictManagerMode mode;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/constraint_conflict_info.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/types/selection_vector.hpp"
+#include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/unordered_set.hpp"
+
+namespace duckdb {
+
+class Index;
+
+class ConflictInfo {
+public:
+	ConflictInfo(const unordered_set<column_t> &column_ids, bool only_check_unique = true)
+	    : column_ids(column_ids), only_check_unique(only_check_unique) {
+	}
+	const unordered_set<column_t> &column_ids;
+
+public:
+	bool ConflictTargetMatches(Index &index) const;
+	void VerifyAllConflictsMeetCondition() const;
+
+public:
+	bool only_check_unique = true;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp

@@ -144,6 +144,8 @@ public:
 
 	//! Hashes the DataChunk to the target vector
 	DUCKDB_API void Hash(Vector &result);
+	//! Hashes specific vectors of the DataChunk to the target vector
+	DUCKDB_API void Hash(vector<idx_t> &column_ids, Vector &result);
 
 	//! Returns a list of types of the vectors of this data chunk
 	DUCKDB_API vector<LogicalType> GetTypes();

package/src/duckdb/src/include/duckdb/common/types/hugeint.hpp

@@ -26,9 +26,9 @@ public:
 
 	template <class T>
 	static T Cast(hugeint_t input) {
-		T value;
-		TryCast(input, value);
-		return value;
+		T result = 0;
+		TryCast(input, result);
+		return result;
 	}
 
 	template <class T>

package/src/duckdb/src/include/duckdb/common/types/partitioned_column_data.hpp

@@ -28,7 +28,7 @@ public:
 	vector<unique_ptr<ColumnDataAppendState>> partition_append_states;
 };
 
-enum class PartitionedColumnDataType : uint8_t { RADIX, INVALID };
+enum class PartitionedColumnDataType : uint8_t { RADIX, HIVE, INVALID };
 
 //! Shared allocators for parallel partitioning
 struct PartitionAllocators {

package/src/duckdb/src/include/duckdb/common/types/row_data_collection_scanner.hpp

@@ -80,6 +80,9 @@ public:
 	//! Scans the next data chunk from the sorted data
 	void Scan(DataChunk &chunk);
 
+	//! Resets to the start and updates the flush flag
+	void Reset(bool flush = true);
+
 private:
 	//! The row data being scanned
 	RowDataCollection &rows;
@@ -98,7 +101,7 @@ private:
 	//! Whether the blocks can be flushed to disk
 	const bool external;
 	//! Whether to flush the blocks after scanning
-	const bool flush;
+	bool flush;
 	//! Whether we are unswizzling the blocks
 	const bool unswizzling;
 

package/src/duckdb/src/include/duckdb/common/types/selection_vector.hpp

@@ -40,10 +40,31 @@ struct SelectionVector {
 		Initialize(sel_vector);
 	}
 	explicit SelectionVector(buffer_ptr<SelectionData> data) {
-		Initialize(move(data));
+		Initialize(std::move(data));
+	}
+	SelectionVector &operator=(SelectionVector &&other) {
+		sel_vector = other.sel_vector;
+		other.sel_vector = nullptr;
+		selection_data = std::move(other.selection_data);
+		return *this;
 	}
 
 public:
+	static idx_t Inverted(const SelectionVector &src, SelectionVector &dst, idx_t source_size, idx_t count) {
+		idx_t src_idx = 0;
+		idx_t dst_idx = 0;
+		for (idx_t i = 0; i < count; i++) {
+			if (src_idx < source_size && src.get_index(src_idx) == i) {
+				src_idx++;
+				// This index is selected by 'src', skip it in 'dst'
+				continue;
+			}
+			// This index does not exist in 'src', add it to the selection of 'dst'
+			dst.set_index(dst_idx++, i);
+		}
+		return dst_idx;
+	}
+
 	void Initialize(sel_t *sel) {
 		selection_data.reset();
 		sel_vector = sel;
@@ -53,7 +74,7 @@ public:
 		sel_vector = selection_data->owned_data.get();
 	}
 	void Initialize(buffer_ptr<SelectionData> data) {
-		selection_data = move(data);
+		selection_data = std::move(data);
 		sel_vector = selection_data->owned_data.get();
 	}
 	void Initialize(const SelectionVector &other) {
@@ -97,8 +118,11 @@ private:
 
 class OptionalSelection {
 public:
-	explicit inline OptionalSelection(SelectionVector *sel_p) : sel(sel_p) {
-
+	explicit inline OptionalSelection(SelectionVector *sel_p) {
+		Initialize(sel_p);
+	}
+	void Initialize(SelectionVector *sel_p) {
+		sel = sel_p;
 		if (sel) {
 			vec.Initialize(sel->data());
 			sel = &vec;
@@ -127,4 +151,60 @@ private:
 	SelectionVector vec;
 };
 
+// Contains a selection vector, combined with a count
+class ManagedSelection {
+public:
+	explicit inline ManagedSelection(idx_t size, bool initialize = true)
+	    : initialized(initialize), size(size), internal_opt_selvec(nullptr) {
+		count = 0;
+		if (!initialized) {
+			return;
+		}
+		sel_vec.Initialize(size);
+		internal_opt_selvec.Initialize(&sel_vec);
+	}
+
+public:
+	bool Initialized() const {
+		return initialized;
+	}
+	void Initialize(idx_t size) {
+		D_ASSERT(!initialized);
+		this->size = size;
+		sel_vec.Initialize(size);
+		internal_opt_selvec.Initialize(&sel_vec);
+		initialized = true;
+	}
+
+	inline idx_t operator[](idx_t index) const {
+		D_ASSERT(index < size);
+		return sel_vec.get_index(index);
+	}
+	inline bool IndexMapsToLocation(idx_t idx, idx_t location) const {
+		return idx < count && sel_vec.get_index(idx) == location;
+	}
+	inline void Append(const idx_t idx) {
+		internal_opt_selvec.Append(count, idx);
+	}
+	inline idx_t Count() const {
+		return count;
+	}
+	inline idx_t Size() const {
+		return size;
+	}
+	inline const SelectionVector &Selection() const {
+		return sel_vec;
+	}
+	inline SelectionVector &Selection() {
+		return sel_vec;
+	}
+
+private:
+	bool initialized = false;
+	idx_t count;
+	idx_t size;
+	SelectionVector sel_vec;
+	OptionalSelection internal_opt_selvec;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp

@@ -83,15 +83,7 @@ public:
 		return !validity_mask;
 	}
 	inline bool CheckAllValid(idx_t count) const {
-		if (AllValid()) {
-			return true;
-		}
-		idx_t entry_count = ValidityBuffer::EntryCount(count);
-		idx_t valid_count = 0;
-		for (idx_t i = 0; i < entry_count; i++) {
-			valid_count += validity_mask[i] == ValidityBuffer::MAX_ENTRY;
-		}
-		return valid_count == entry_count;
+		return CountValid(count) == count;
 	}
 
 	inline bool CheckAllValid(idx_t to, idx_t from) const {

package/src/duckdb/src/include/duckdb/common/types/value.hpp

@@ -63,7 +63,7 @@ public:
 	// move assignment
 	DUCKDB_API Value &operator=(Value &&other) noexcept;
 
-	inline LogicalType &type() {
+	inline LogicalType &GetTypeMutable() {
 		return type_;
 	}
 	inline const LogicalType &type() const {
@@ -131,10 +131,6 @@ public:
 	                                  int32_t micros);
 	DUCKDB_API static Value INTERVAL(int32_t months, int32_t days, int64_t micros);
 	DUCKDB_API static Value INTERVAL(interval_t interval);
-	//! Creates a JSON Value
-	DUCKDB_API static Value JSON(const char *val);
-	DUCKDB_API static Value JSON(string_t val);
-	DUCKDB_API static Value JSON(string val);
 
 	// Create a enum Value from a specified uint value
 	DUCKDB_API static Value ENUM(uint64_t value, const LogicalType &original_type);
@@ -169,6 +165,9 @@ public:
 	}
 	//! Creates a blob by casting a specified string to a blob (i.e. interpreting \x characters)
 	DUCKDB_API static Value BLOB(const string &data);
+	//! Creates a bitstring by casting a specified string to a bitstring
+	DUCKDB_API static Value BIT(const_data_ptr_t data, idx_t len);
+	DUCKDB_API static Value BIT(const string &data);
 
 	template <class T>
 	T GetValue() const {
@@ -222,6 +221,8 @@ public:
 	DUCKDB_API bool TryCastAs(ClientContext &context, const LogicalType &target_type, bool strict = false);
 	DUCKDB_API bool DefaultTryCastAs(const LogicalType &target_type, bool strict = false);
 
+	DUCKDB_API void Reinterpret(LogicalType new_type);
+
 	//! Serializes a Value to a stand-alone binary blob
 	DUCKDB_API void Serialize(Serializer &serializer) const;
 	//! Deserializes a Value from a blob

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -32,6 +32,14 @@ class VectorListBuffer;
 
 struct SelCache;
 
+struct ConsecutiveChildListInfo {
+	ConsecutiveChildListInfo() : is_constant(true), needs_slicing(false), child_list_info(list_entry_t(0, 0)) {
+	}
+	bool is_constant;
+	bool needs_slicing;
+	list_entry_t child_list_info;
+};
+
 //!  Vector of values of a specified PhysicalType.
 class Vector {
 	friend struct ConstantVector;
@@ -108,10 +116,10 @@ public:
 
 	//! Converts this Vector to a printable string representation
 	DUCKDB_API string ToString(idx_t count) const;
-	DUCKDB_API void Print(idx_t count);
+	DUCKDB_API void Print(idx_t count) const;
 
 	DUCKDB_API string ToString() const;
-	DUCKDB_API void Print();
+	DUCKDB_API void Print() const;
 
 	//! Flatten the vector, removing any compression and turning it into a FLAT_VECTOR
 	DUCKDB_API void Flatten(idx_t count);
@@ -203,7 +211,7 @@ protected:
 class VectorChildBuffer : public VectorBuffer {
 public:
 	explicit VectorChildBuffer(Vector vector)
-	    : VectorBuffer(VectorBufferType::VECTOR_CHILD_BUFFER), data(move(vector)) {
+	    : VectorBuffer(VectorBufferType::VECTOR_CHILD_BUFFER), data(std::move(vector)) {
 	}
 
 public:
@@ -331,6 +339,12 @@ struct ListVector {
 	DUCKDB_API static void Append(Vector &target, const Vector &source, const SelectionVector &sel, idx_t source_size,
 	                              idx_t source_offset = 0);
 	DUCKDB_API static void PushBack(Vector &target, const Value &insert);
+	//! Returns the child_vector of list starting at offset until offset + count, and its length
+	DUCKDB_API static idx_t GetConsecutiveChildList(Vector &list, Vector &result, idx_t offset, idx_t count);
+	//! Returns information to only copy a section of a list child vector
+	DUCKDB_API static ConsecutiveChildListInfo GetConsecutiveChildListInfo(Vector &list, idx_t offset, idx_t count);
+	//! Slice and flatten a child vector to only contain a consecutive subsection of the child entries
+	DUCKDB_API static void GetConsecutiveChildSelVector(Vector &list, SelectionVector &sel, idx_t offset, idx_t count);
 	//! Share the entry of the other list vector
 	DUCKDB_API static void ReferenceEntry(Vector &vector, Vector &other);
 };

package/src/duckdb/src/include/duckdb/common/types/vector_buffer.hpp

@@ -58,7 +58,7 @@ public:
 		}
 	}
 	explicit VectorBuffer(unique_ptr<data_t[]> data_p)
-	    : buffer_type(VectorBufferType::STANDARD_BUFFER), data(move(data_p)) {
+	    : buffer_type(VectorBufferType::STANDARD_BUFFER), data(std::move(data_p)) {
 	}
 	virtual ~VectorBuffer() {
 	}
@@ -71,7 +71,7 @@ public:
 	}
 
 	void SetData(unique_ptr<data_t[]> new_data) {
-		data = move(new_data);
+		data = std::move(new_data);
 	}
 
 	VectorAuxiliaryData *GetAuxiliaryData() {
@@ -79,7 +79,7 @@ public:
 	}
 
 	void SetAuxiliaryData(unique_ptr<VectorAuxiliaryData> aux_data_p) {
-		aux_data = move(aux_data_p);
+		aux_data = std::move(aux_data_p);
 	}
 
 	static buffer_ptr<VectorBuffer> CreateStandardVector(PhysicalType type, idx_t capacity = STANDARD_VECTOR_SIZE);
@@ -109,7 +109,7 @@ public:
 	    : VectorBuffer(VectorBufferType::DICTIONARY_BUFFER), sel_vector(sel) {
 	}
 	explicit DictionaryBuffer(buffer_ptr<SelectionData> data)
-	    : VectorBuffer(VectorBufferType::DICTIONARY_BUFFER), sel_vector(move(data)) {
+	    : VectorBuffer(VectorBufferType::DICTIONARY_BUFFER), sel_vector(std::move(data)) {
 	}
 	explicit DictionaryBuffer(idx_t count = STANDARD_VECTOR_SIZE)
 	    : VectorBuffer(VectorBufferType::DICTIONARY_BUFFER), sel_vector(count) {
@@ -150,7 +150,7 @@ public:
 	}
 
 	void AddHeapReference(buffer_ptr<VectorBuffer> heap) {
-		references.push_back(move(heap));
+		references.push_back(std::move(heap));
 	}
 
 private:

package/src/duckdb/src/include/duckdb/common/types.hpp

@@ -257,7 +257,7 @@ enum class LogicalTypeId : uint8_t {
 	UBIGINT = 31,
 	TIMESTAMP_TZ = 32,
 	TIME_TZ = 34,
-	JSON = 35,
+	BIT = 36,
 
 	HUGEINT = 50,
 	POINTER = 51,
@@ -314,7 +314,7 @@ struct LogicalType {
 	inline LogicalType& operator=(LogicalType&& other) noexcept {
 		id_ = other.id_;
 		physical_type_ = other.physical_type_;
-		type_info_ = move(other.type_info_);
+		type_info_ = std::move(other.type_info_);
 		return *this;
 	}
 
@@ -390,7 +390,8 @@ public:
 	static constexpr const LogicalTypeId VARCHAR = LogicalTypeId::VARCHAR;
 	static constexpr const LogicalTypeId ANY = LogicalTypeId::ANY;
 	static constexpr const LogicalTypeId BLOB = LogicalTypeId::BLOB;
-	static constexpr const LogicalTypeId INTERVAL = LogicalTypeId::INTERVAL;
+    static constexpr const LogicalTypeId BIT = LogicalTypeId::BIT;
+    static constexpr const LogicalTypeId INTERVAL = LogicalTypeId::INTERVAL;
 	static constexpr const LogicalTypeId HUGEINT = LogicalTypeId::HUGEINT;
 	static constexpr const LogicalTypeId UUID = LogicalTypeId::UUID;
 	static constexpr const LogicalTypeId HASH = LogicalTypeId::UBIGINT;
@@ -398,7 +399,6 @@ public:
 	static constexpr const LogicalTypeId TABLE = LogicalTypeId::TABLE;
 	static constexpr const LogicalTypeId LAMBDA = LogicalTypeId::LAMBDA;
 	static constexpr const LogicalTypeId INVALID = LogicalTypeId::INVALID;
-	static constexpr const LogicalTypeId JSON = LogicalTypeId::JSON;
 	static constexpr const LogicalTypeId ROW_TYPE = LogicalTypeId::BIGINT;
 
 	// explicitly allowing these functions to be capitalized to be in-line with the remaining functions
@@ -481,6 +481,8 @@ DUCKDB_API LogicalTypeId TransformStringToLogicalTypeId(const string &str);
 
 DUCKDB_API LogicalType TransformStringToLogicalType(const string &str);
 
+DUCKDB_API LogicalType TransformStringToLogicalType(const string &str, ClientContext &context);
+
 //! The PhysicalType used by the row identifiers column
 extern const PhysicalType ROW_TYPE;
 
@@ -495,7 +497,7 @@ bool ApproxEqual(float l, float r);
 bool ApproxEqual(double l, double r);
 
 struct aggregate_state_t {
-	aggregate_state_t(string function_name_p, LogicalType return_type_p, vector<LogicalType> bound_argument_types_p) : function_name(move(function_name_p)), return_type(move(return_type_p)), bound_argument_types(move(bound_argument_types_p)) {
+	aggregate_state_t(string function_name_p, LogicalType return_type_p, vector<LogicalType> bound_argument_types_p) : function_name(std::move(function_name_p)), return_type(std::move(return_type_p)), bound_argument_types(std::move(bound_argument_types_p)) {
 	}
 
 	string function_name;

package/src/duckdb/src/include/duckdb/common/unicode_bar.hpp

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/unicode_bar.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+namespace duckdb {
+struct UnicodeBar {
+private:
+	static constexpr idx_t PARTIAL_BLOCKS_COUNT = 8;
+
+public:
+	static constexpr idx_t PartialBlocksCount() {
+		return PARTIAL_BLOCKS_COUNT;
+	}
+
+	static const char *const *PartialBlocks() {
+		static const char *PARTIAL_BLOCKS[PARTIAL_BLOCKS_COUNT] = {" ",
+		                                                           "\xE2\x96\x8F",
+		                                                           "\xE2\x96\x8E",
+		                                                           "\xE2\x96\x8D",
+		                                                           "\xE2\x96\x8C",
+		                                                           "\xE2\x96\x8B",
+		                                                           "\xE2\x96\x8A",
+		                                                           "\xE2\x96\x89"};
+		return PARTIAL_BLOCKS;
+	}
+
+	static const char *FullBlock() {
+		return "\xE2\x96\x88";
+	}
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/union_by_name.hpp

@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/union_by_name.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+#include <vector>
+#include <string>
+#include "duckdb/common/types.hpp"
+
+using std::string;
+using std::vector;
+
+namespace duckdb {
+
+template <class READER_TYPE, class OPTION_TYPE>
+class UnionByName {
+
+public:
+	//! Union all files(readers) by their col names
+	static vector<unique_ptr<READER_TYPE>>
+	UnionCols(ClientContext &context, const vector<string> &files, vector<LogicalType> &union_col_types,
+	          vector<string> &union_col_names, case_insensitive_map_t<idx_t> &union_names_map, OPTION_TYPE options) {
+		idx_t union_names_index = 0;
+		vector<unique_ptr<READER_TYPE>> union_readers;
+
+		for (idx_t file_idx = 0; file_idx < files.size(); ++file_idx) {
+			const auto file_name = files[file_idx];
+			auto reader = make_unique<READER_TYPE>(context, file_name, options);
+
+			auto &col_names = reader->names;
+			auto &sql_types = reader->return_types;
+			D_ASSERT(col_names.size() == sql_types.size());
+
+			for (idx_t col = 0; col < col_names.size(); ++col) {
+				auto union_find = union_names_map.find(col_names[col]);
+
+				if (union_find != union_names_map.end()) {
+					// given same name , union_col's type must compatible with col's type
+					LogicalType compatible_type;
+					compatible_type = LogicalType::MaxLogicalType(union_col_types[union_find->second], sql_types[col]);
+					union_col_types[union_find->second] = compatible_type;
+				} else {
+					union_names_map[col_names[col]] = union_names_index;
+					union_names_index++;
+
+					union_col_names.emplace_back(col_names[col]);
+					union_col_types.emplace_back(sql_types[col]);
+				}
+			}
+			union_readers.push_back(std::move(reader));
+		}
+		return union_readers;
+	}
+
+	//! Create information for reader's col mapping to union cols
+	static vector<unique_ptr<READER_TYPE>> CreateUnionMap(vector<unique_ptr<READER_TYPE>> union_readers,
+	                                                      vector<LogicalType> &union_col_types,
+	                                                      vector<string> &union_col_names,
+	                                                      case_insensitive_map_t<idx_t> &union_names_map) {
+		for (auto &reader : union_readers) {
+			auto &col_names = reader->names;
+			vector<bool> union_null_cols(union_col_names.size(), true);
+			vector<idx_t> union_idx_map(col_names.size(), 0);
+
+			for (idx_t col = 0; col < col_names.size(); ++col) {
+				idx_t union_idx = union_names_map[col_names[col]];
+				union_idx_map[col] = union_idx;
+				union_null_cols[union_idx] = false;
+			}
+
+			reader->union_col_types = union_col_types;
+			reader->union_idx_map = std::move(union_idx_map);
+			reader->union_null_cols = std::move(union_null_cols);
+		}
+		return union_readers;
+	}
+
+	//! Set nulls into the cols that mismtach union names
+	static void SetNullUnionCols(DataChunk &result, const vector<bool> &union_null_cols) {
+		for (idx_t col = 0; col < union_null_cols.size(); ++col) {
+			if (union_null_cols[col]) {
+				result.data[col].SetVectorType(VectorType::CONSTANT_VECTOR);
+				ConstantVector::SetNull(result.data[col], true);
+			}
+		}
+	}
+};
+
+} // namespace duckdb