duckdb 0.7.2-dev2706.0 → 0.7.2-dev2820.0

package/src/duckdb/extension/json/json_functions/read_json_objects.cpp

@@ -46,22 +46,22 @@ TableFunction GetReadJSONObjectsTableFunction(bool list_parameter, shared_ptr<JS
 	return table_function;
 }
 
-CreateTableFunctionInfo JSONFunctions::GetReadJSONObjectsFunction() {
+TableFunctionSet JSONFunctions::GetReadJSONObjectsFunction() {
 	TableFunctionSet function_set("read_json_objects");
 	auto function_info =
 	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::UNSTRUCTURED, JSONRecordType::JSON);
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
-	return CreateTableFunctionInfo(function_set);
+	return function_set;
 }
 
-CreateTableFunctionInfo JSONFunctions::GetReadNDJSONObjectsFunction() {
+TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() {
 	TableFunctionSet function_set("read_ndjson_objects");
 	auto function_info =
 	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::JSON);
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
-	return CreateTableFunctionInfo(function_set);
+	return function_set;
 }
 
 } // namespace duckdb

package/src/duckdb/extension/json/json_functions.cpp

@@ -121,8 +121,8 @@ JSONFunctionLocalState &JSONFunctionLocalState::ResetAndGet(ExpressionState &sta
 	return lstate;
 }
 
-vector<CreateScalarFunctionInfo> JSONFunctions::GetScalarFunctions() {
-	vector<CreateScalarFunctionInfo> functions;
+vector<ScalarFunctionSet> JSONFunctions::GetScalarFunctions() {
+	vector<ScalarFunctionSet> functions;
 
 	// Extract functions
 	AddAliases({"json_extract", "json_extract_path"}, GetExtractFunction(), functions);
@@ -153,14 +153,14 @@ vector<CreateScalarFunctionInfo> JSONFunctions::GetScalarFunctions() {
 	return functions;
 }
 
-vector<CreatePragmaFunctionInfo> JSONFunctions::GetPragmaFunctions() {
-	vector<CreatePragmaFunctionInfo> functions;
+vector<PragmaFunctionSet> JSONFunctions::GetPragmaFunctions() {
+	vector<PragmaFunctionSet> functions;
 	functions.push_back(GetExecuteJsonSerializedSqlPragmaFunction());
 	return functions;
 }
 
-vector<CreateTableFunctionInfo> JSONFunctions::GetTableFunctions() {
-	vector<CreateTableFunctionInfo> functions;
+vector<TableFunctionSet> JSONFunctions::GetTableFunctions() {
+	vector<TableFunctionSet> functions;
 
 	// Reads JSON as string
 	functions.push_back(GetReadJSONObjectsFunction());

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -34,6 +34,7 @@
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/storage/table/row_group.hpp"
+#include "duckdb/main/extension_util.hpp"
 #endif
 
 namespace duckdb {
@@ -687,20 +688,23 @@ unique_ptr<TableRef> ParquetScanReplacement(ClientContext &context, const string
 }
 
 void ParquetExtension::Load(DuckDB &db) {
+	auto &db_instance = *db.instance;
 	auto &fs = db.GetFileSystem();
 	fs.RegisterSubSystem(FileCompressionType::ZSTD, make_uniq<ZStdFileSystem>());
 
 	auto scan_fun = ParquetScanFunction::GetFunctionSet();
-	CreateTableFunctionInfo cinfo(scan_fun);
-	cinfo.name = "read_parquet";
-	CreateTableFunctionInfo pq_scan = cinfo;
-	pq_scan.name = "parquet_scan";
+	scan_fun.name = "read_parquet";
+	ExtensionUtil::RegisterFunction(db_instance, scan_fun);
+	scan_fun.name = "parquet_scan";
+	ExtensionUtil::RegisterFunction(db_instance, scan_fun);
 
+	// parquet_metadata
 	ParquetMetaDataFunction meta_fun;
-	CreateTableFunctionInfo meta_cinfo(MultiFileReader::CreateFunctionSet(meta_fun));
+	ExtensionUtil::RegisterFunction(db_instance, MultiFileReader::CreateFunctionSet(meta_fun));
 
+	// parquet_schema
 	ParquetSchemaFunction schema_fun;
-	CreateTableFunctionInfo schema_cinfo(MultiFileReader::CreateFunctionSet(schema_fun));
+	ExtensionUtil::RegisterFunction(db_instance, MultiFileReader::CreateFunctionSet(schema_fun));
 
 	CopyFunction function("parquet");
 	function.copy_to_bind = ParquetWriteBind;
@@ -714,24 +718,7 @@ void ParquetExtension::Load(DuckDB &db) {
 	function.copy_from_function = scan_fun.functions[0];
 
 	function.extension = "parquet";
-	CreateCopyFunctionInfo info(function);
-
-	Connection con(db);
-	con.BeginTransaction();
-	auto &context = *con.context;
-	auto &catalog = Catalog::GetSystemCatalog(context);
-
-	if (catalog.GetEntry<TableFunctionCatalogEntry>(context, DEFAULT_SCHEMA, "parquet_scan",
-	                                                OnEntryNotFound::RETURN_NULL)) {
-		throw InvalidInputException("Parquet extension is either already loaded or built-in");
-	}
-
-	catalog.CreateCopyFunction(context, info);
-	catalog.CreateTableFunction(context, cinfo);
-	catalog.CreateTableFunction(context, pq_scan);
-	catalog.CreateTableFunction(context, meta_cinfo);
-	catalog.CreateTableFunction(context, schema_cinfo);
-	con.Commit();
+	ExtensionUtil::RegisterFunction(db_instance, function);
 
 	auto &config = DBConfig::GetConfig(*db.instance);
 	config.replacement_scans.emplace_back(ParquetScanReplacement);

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -55,6 +55,30 @@ string PrintParquetElementToString(T &&entry) {
 	return ss.str();
 }
 
+template <class T>
+Value ParquetElementString(T &&value, bool is_set) {
+	if (!is_set) {
+		return Value();
+	}
+	return Value(ConvertParquetElementToString(value));
+}
+
+template <class T>
+Value ParquetElementInteger(T &&value, bool is_iset) {
+	if (!is_iset) {
+		return Value();
+	}
+	return Value::INTEGER(value);
+}
+
+template <class T>
+Value ParquetElementBigint(T &&value, bool is_iset) {
+	if (!is_iset) {
+		return Value();
+	}
+	return Value::BIGINT(value);
+}
+
 void ParquetMetaDataOperatorData::BindMetaData(vector<LogicalType> &return_types, vector<string> &names) {
 	names.emplace_back("file_name");
 	return_types.emplace_back(LogicalType::VARCHAR);
@@ -186,7 +210,7 @@ void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const
 			current_chunk.SetValue(5, count, Value::BIGINT(col_idx));
 
 			// file_offset, LogicalType::BIGINT
-			current_chunk.SetValue(6, count, Value::BIGINT(column.file_offset));
+			current_chunk.SetValue(6, count, ParquetElementBigint(column.file_offset, row_group.__isset.file_offset));
 
 			// num_values, LogicalType::BIGINT
 			current_chunk.SetValue(7, count, Value::BIGINT(col_meta.num_values));
@@ -206,13 +230,10 @@ void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const
 			                       ConvertParquetStats(column_type, schema_element, stats.__isset.max, stats.max));
 
 			// stats_null_count, LogicalType::BIGINT
-			current_chunk.SetValue(
-			    12, count, stats.__isset.null_count ? Value::BIGINT(stats.null_count) : Value(LogicalType::BIGINT));
+			current_chunk.SetValue(12, count, ParquetElementBigint(stats.null_count, stats.__isset.null_count));
 
 			// stats_distinct_count, LogicalType::BIGINT
-			current_chunk.SetValue(13, count,
-			                       stats.__isset.distinct_count ? Value::BIGINT(stats.distinct_count)
-			                                                    : Value(LogicalType::BIGINT));
+			current_chunk.SetValue(13, count, ParquetElementBigint(stats.distinct_count, stats.__isset.distinct_count));
 
 			// stats_min_value, LogicalType::VARCHAR
 			current_chunk.SetValue(
@@ -234,10 +255,13 @@ void ParquetMetaDataOperatorData::LoadFileMetaData(ClientContext &context, const
 			current_chunk.SetValue(17, count, Value(StringUtil::Join(encoding_string, ", ")));
 
 			// index_page_offset, LogicalType::BIGINT
-			current_chunk.SetValue(18, count, Value::BIGINT(col_meta.index_page_offset));
+			current_chunk.SetValue(
+			    18, count, ParquetElementBigint(col_meta.index_page_offset, col_meta.__isset.index_page_offset));
 
 			// dictionary_page_offset, LogicalType::BIGINT
-			current_chunk.SetValue(19, count, Value::BIGINT(col_meta.dictionary_page_offset));
+			current_chunk.SetValue(
+			    19, count,
+			    ParquetElementBigint(col_meta.dictionary_page_offset, col_meta.__isset.dictionary_page_offset));
 
 			// data_page_offset, LogicalType::BIGINT
 			current_chunk.SetValue(20, count, Value::BIGINT(col_meta.data_page_offset));
@@ -299,8 +323,10 @@ void ParquetMetaDataOperatorData::BindSchema(vector<LogicalType> &return_types,
 	return_types.emplace_back(LogicalType::VARCHAR);
 }
 
-Value ParquetLogicalTypeToString(const duckdb_parquet::format::LogicalType &type) {
-
+Value ParquetLogicalTypeToString(const duckdb_parquet::format::LogicalType &type, bool is_set) {
+	if (!is_set) {
+		return Value();
+	}
 	if (type.__isset.STRING) {
 		return Value(PrintParquetElementToString(type.STRING));
 	}
@@ -362,31 +388,31 @@ void ParquetMetaDataOperatorData::LoadSchemaData(ClientContext &context, const v
 		current_chunk.SetValue(1, count, column.name);
 
 		// type, LogicalType::VARCHAR
-		current_chunk.SetValue(2, count, ConvertParquetElementToString(column.type));
+		current_chunk.SetValue(2, count, ParquetElementString(column.type, column.__isset.type));
 
-		// type_length, LogicalType::VARCHAR
-		current_chunk.SetValue(3, count, Value::INTEGER(column.type_length));
+		// type_length, LogicalType::INTEGER
+		current_chunk.SetValue(3, count, ParquetElementInteger(column.type_length, column.__isset.type_length));
 
 		// repetition_type, LogicalType::VARCHAR
-		current_chunk.SetValue(4, count, ConvertParquetElementToString(column.repetition_type));
+		current_chunk.SetValue(4, count, ParquetElementString(column.repetition_type, column.__isset.repetition_type));
 
 		// num_children, LogicalType::BIGINT
-		current_chunk.SetValue(5, count, Value::BIGINT(column.num_children));
+		current_chunk.SetValue(5, count, ParquetElementBigint(column.num_children, column.__isset.num_children));
 
 		// converted_type, LogicalType::VARCHAR
-		current_chunk.SetValue(6, count, ConvertParquetElementToString(column.converted_type));
+		current_chunk.SetValue(6, count, ParquetElementString(column.converted_type, column.__isset.converted_type));
 
 		// scale, LogicalType::BIGINT
-		current_chunk.SetValue(7, count, Value::BIGINT(column.scale));
+		current_chunk.SetValue(7, count, ParquetElementBigint(column.scale, column.__isset.scale));
 
 		// precision, LogicalType::BIGINT
-		current_chunk.SetValue(8, count, Value::BIGINT(column.precision));
+		current_chunk.SetValue(8, count, ParquetElementBigint(column.precision, column.__isset.precision));
 
 		// field_id, LogicalType::BIGINT
-		current_chunk.SetValue(9, count, Value::BIGINT(column.field_id));
+		current_chunk.SetValue(9, count, ParquetElementBigint(column.field_id, column.__isset.field_id));
 
 		// logical_type, LogicalType::VARCHAR
-		current_chunk.SetValue(10, count, ParquetLogicalTypeToString(column.logicalType));
+		current_chunk.SetValue(10, count, ParquetLogicalTypeToString(column.logicalType, column.__isset.logicalType));
 
 		count++;
 		if (count >= STANDARD_VECTOR_SIZE) {

package/src/duckdb/src/catalog/catalog_entry/pragma_function_catalog_entry.cpp

@@ -5,8 +5,7 @@ namespace duckdb {
 
 PragmaFunctionCatalogEntry::PragmaFunctionCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema,
                                                        CreatePragmaFunctionInfo &info)
-    : StandardEntry(CatalogType::PRAGMA_FUNCTION_ENTRY, schema, catalog, info.name),
-      functions(std::move(info.functions)) {
+    : FunctionEntry(CatalogType::PRAGMA_FUNCTION_ENTRY, catalog, schema, info), functions(std::move(info.functions)) {
 }
 
 } // namespace duckdb

package/src/duckdb/src/catalog/catalog_entry/scalar_function_catalog_entry.cpp

@@ -5,7 +5,7 @@ namespace duckdb {
 
 ScalarFunctionCatalogEntry::ScalarFunctionCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema,
                                                        CreateScalarFunctionInfo &info)
-    : StandardEntry(CatalogType::SCALAR_FUNCTION_ENTRY, schema, catalog, info.name), functions(info.functions) {
+    : FunctionEntry(CatalogType::SCALAR_FUNCTION_ENTRY, catalog, schema, info), functions(info.functions) {
 }
 
 unique_ptr<CatalogEntry> ScalarFunctionCatalogEntry::AlterEntry(ClientContext &context, AlterInfo &info) {

package/src/duckdb/src/catalog/catalog_entry/scalar_macro_catalog_entry.cpp

@@ -7,9 +7,9 @@
 namespace duckdb {
 
 MacroCatalogEntry::MacroCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateMacroInfo &info)
-    : StandardEntry(
+    : FunctionEntry(
           (info.function->type == MacroType::SCALAR_MACRO ? CatalogType::MACRO_ENTRY : CatalogType::TABLE_MACRO_ENTRY),
-          schema, catalog, info.name),
+          catalog, schema, info),
       function(std::move(info.function)) {
 	this->temporary = info.temporary;
 	this->internal = info.internal;

package/src/duckdb/src/catalog/catalog_entry/table_function_catalog_entry.cpp

@@ -5,8 +5,7 @@ namespace duckdb {
 
 TableFunctionCatalogEntry::TableFunctionCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema,
                                                      CreateTableFunctionInfo &info)
-    : StandardEntry(CatalogType::TABLE_FUNCTION_ENTRY, schema, catalog, info.name),
-      functions(std::move(info.functions)) {
+    : FunctionEntry(CatalogType::TABLE_FUNCTION_ENTRY, catalog, schema, info), functions(std::move(info.functions)) {
 	D_ASSERT(this->functions.Size() > 0);
 }
 

package/src/duckdb/src/catalog/catalog_transaction.cpp

@@ -31,4 +31,8 @@ ClientContext &CatalogTransaction::GetContext() {
 	return *context;
 }
 
+CatalogTransaction CatalogTransaction::GetSystemTransaction(DatabaseInstance &db) {
+	return CatalogTransaction(db, 1, 1);
+}
+
 } // namespace duckdb

package/src/duckdb/src/catalog/duck_catalog.cpp

@@ -7,6 +7,9 @@
 #include "duckdb/catalog/default/default_schemas.hpp"
 #include "duckdb/function/built_in_functions.hpp"
 #include "duckdb/main/attached_database.hpp"
+#ifndef DISABLE_CORE_FUNCTIONS_EXTENSION
+#include "duckdb/core_functions/core_functions.hpp"
+#endif
 
 namespace duckdb {
 
@@ -22,7 +25,7 @@ void DuckCatalog::Initialize(bool load_builtin) {
 	// first initialize the base system catalogs
 	// these are never written to the WAL
 	// we start these at 1 because deleted entries default to 0
-	CatalogTransaction data(GetDatabase(), 1, 1);
+	auto data = CatalogTransaction::GetSystemTransaction(GetDatabase());
 
 	// create the default schema
 	CreateSchemaInfo info;
@@ -34,6 +37,10 @@ void DuckCatalog::Initialize(bool load_builtin) {
 		// initialize default functions
 		BuiltinFunctions builtin(data, *this);
 		builtin.Initialize();
+
+#ifndef DISABLE_CORE_FUNCTIONS_EXTENSION
+		CoreFunctions::RegisterFunctions(*this, data);
+#endif
 	}
 
 	Verify();

package/src/duckdb/src/common/enums/date_part_specifier.cpp

@@ -0,0 +1,82 @@
+#include "duckdb/common/enums/date_part_specifier.hpp"
+#include "duckdb/common/string_util.hpp"
+
+namespace duckdb {
+
+bool TryGetDatePartSpecifier(const string &specifier_p, DatePartSpecifier &result) {
+	auto specifier = StringUtil::Lower(specifier_p);
+	if (specifier == "year" || specifier == "yr" || specifier == "y" || specifier == "years" || specifier == "yrs") {
+		result = DatePartSpecifier::YEAR;
+	} else if (specifier == "month" || specifier == "mon" || specifier == "months" || specifier == "mons") {
+		result = DatePartSpecifier::MONTH;
+	} else if (specifier == "day" || specifier == "days" || specifier == "d" || specifier == "dayofmonth") {
+		result = DatePartSpecifier::DAY;
+	} else if (specifier == "decade" || specifier == "dec" || specifier == "decades" || specifier == "decs") {
+		result = DatePartSpecifier::DECADE;
+	} else if (specifier == "century" || specifier == "cent" || specifier == "centuries" || specifier == "c") {
+		result = DatePartSpecifier::CENTURY;
+	} else if (specifier == "millennium" || specifier == "mil" || specifier == "millenniums" ||
+	           specifier == "millennia" || specifier == "mils" || specifier == "millenium") {
+		result = DatePartSpecifier::MILLENNIUM;
+	} else if (specifier == "microseconds" || specifier == "microsecond" || specifier == "us" || specifier == "usec" ||
+	           specifier == "usecs" || specifier == "usecond" || specifier == "useconds") {
+		result = DatePartSpecifier::MICROSECONDS;
+	} else if (specifier == "milliseconds" || specifier == "millisecond" || specifier == "ms" || specifier == "msec" ||
+	           specifier == "msecs" || specifier == "msecond" || specifier == "mseconds") {
+		result = DatePartSpecifier::MILLISECONDS;
+	} else if (specifier == "second" || specifier == "sec" || specifier == "seconds" || specifier == "secs" ||
+	           specifier == "s") {
+		result = DatePartSpecifier::SECOND;
+	} else if (specifier == "minute" || specifier == "min" || specifier == "minutes" || specifier == "mins" ||
+	           specifier == "m") {
+		result = DatePartSpecifier::MINUTE;
+	} else if (specifier == "hour" || specifier == "hr" || specifier == "hours" || specifier == "hrs" ||
+	           specifier == "h") {
+		result = DatePartSpecifier::HOUR;
+	} else if (specifier == "epoch") {
+		// seconds since 1970-01-01
+		result = DatePartSpecifier::EPOCH;
+	} else if (specifier == "dow" || specifier == "dayofweek" || specifier == "weekday") {
+		// day of the week (Sunday = 0, Saturday = 6)
+		result = DatePartSpecifier::DOW;
+	} else if (specifier == "isodow") {
+		// isodow (Monday = 1, Sunday = 7)
+		result = DatePartSpecifier::ISODOW;
+	} else if (specifier == "week" || specifier == "weeks" || specifier == "w" || specifier == "weekofyear") {
+		// ISO week number
+		result = DatePartSpecifier::WEEK;
+	} else if (specifier == "doy" || specifier == "dayofyear") {
+		// day of the year (1-365/366)
+		result = DatePartSpecifier::DOY;
+	} else if (specifier == "quarter" || specifier == "quarters") {
+		// quarter of the year (1-4)
+		result = DatePartSpecifier::QUARTER;
+	} else if (specifier == "yearweek") {
+		// Combined isoyear and isoweek YYYYWW
+		result = DatePartSpecifier::YEARWEEK;
+	} else if (specifier == "isoyear") {
+		// ISO year (first week of the year may be in previous year)
+		result = DatePartSpecifier::ISOYEAR;
+	} else if (specifier == "era") {
+		result = DatePartSpecifier::ERA;
+	} else if (specifier == "timezone") {
+		result = DatePartSpecifier::TIMEZONE;
+	} else if (specifier == "timezone_hour") {
+		result = DatePartSpecifier::TIMEZONE_HOUR;
+	} else if (specifier == "timezone_minute") {
+		result = DatePartSpecifier::TIMEZONE_MINUTE;
+	} else {
+		return false;
+	}
+	return true;
+}
+
+DatePartSpecifier GetDatePartSpecifier(const string &specifier) {
+	DatePartSpecifier result;
+	if (!TryGetDatePartSpecifier(specifier, result)) {
+		throw ConversionException("extract specifier \"%s\" not recognized", specifier);
+	}
+	return result;
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/types/vector.cpp

@@ -18,6 +18,7 @@
 #include "duckdb/common/fsst.hpp"
 #include "fsst.h"
 #include "duckdb/common/types/bit.hpp"
+#include "duckdb/common/types/value_map.hpp"
 
 #include "duckdb/common/serializer/format_serializer.hpp"
 #include "duckdb/common/serializer/format_deserializer.hpp"
@@ -1256,7 +1257,7 @@ void Vector::UTFVerify(idx_t count) {
 void Vector::VerifyMap(Vector &vector_p, const SelectionVector &sel_p, idx_t count) {
 #ifdef DEBUG
 	D_ASSERT(vector_p.GetType().id() == LogicalTypeId::MAP);
-	auto valid_check = CheckMapValidity(vector_p, count, sel_p);
+	auto valid_check = MapVector::CheckMapValidity(vector_p, count, sel_p);
 	D_ASSERT(valid_check == MapInvalidReason::VALID);
 #endif // DEBUG
 }
@@ -1264,7 +1265,7 @@ void Vector::VerifyMap(Vector &vector_p, const SelectionVector &sel_p, idx_t cou
 void Vector::VerifyUnion(Vector &vector_p, const SelectionVector &sel_p, idx_t count) {
 #ifdef DEBUG
 	D_ASSERT(vector_p.GetType().id() == LogicalTypeId::UNION);
-	auto valid_check = CheckUnionValidity(vector_p, count, sel_p);
+	auto valid_check = UnionVector::CheckUnionValidity(vector_p, count, sel_p);
 	D_ASSERT(valid_check == UnionInvalidReason::VALID);
 #endif // DEBUG
 }
@@ -1439,6 +1440,9 @@ void Vector::Verify(idx_t count) {
 	Verify(*this, *flat_sel, count);
 }
 
+//===--------------------------------------------------------------------===//
+// FlatVector
+//===--------------------------------------------------------------------===//
 void FlatVector::SetNull(Vector &vector, idx_t idx, bool is_null) {
 	D_ASSERT(vector.GetVectorType() == VectorType::FLAT_VECTOR);
 	vector.validity.Set(idx, !is_null);
@@ -1451,6 +1455,9 @@ void FlatVector::SetNull(Vector &vector, idx_t idx, bool is_null) {
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// ConstantVector
+//===--------------------------------------------------------------------===//
 void ConstantVector::SetNull(Vector &vector, bool is_null) {
 	D_ASSERT(vector.GetVectorType() == VectorType::CONSTANT_VECTOR);
 	vector.validity.Set(0, !is_null);
@@ -1539,6 +1546,9 @@ void ConstantVector::Reference(Vector &vector, Vector &source, idx_t position, i
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// StringVector
+//===--------------------------------------------------------------------===//
 string_t StringVector::AddString(Vector &vector, const char *data, idx_t len) {
 	return StringVector::AddString(vector, string_t(data, len));
 }
@@ -1629,6 +1639,9 @@ void StringVector::AddHeapReference(Vector &vector, Vector &other) {
 	StringVector::AddBuffer(vector, other.auxiliary);
 }
 
+//===--------------------------------------------------------------------===//
+// FSSTVector
+//===--------------------------------------------------------------------===//
 string_t FSSTVector::AddCompressedString(Vector &vector, const char *data, idx_t len) {
 	return FSSTVector::AddCompressedString(vector, string_t(data, len));
 }
@@ -1714,6 +1727,9 @@ void FSSTVector::DecompressVector(const Vector &src, Vector &dst, idx_t src_offs
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// MapVector
+//===--------------------------------------------------------------------===//
 Vector &MapVector::GetKeys(Vector &vector) {
 	auto &entries = StructVector::GetEntries(ListVector::GetEntry(vector));
 	D_ASSERT(entries.size() == 2);
@@ -1732,6 +1748,67 @@ const Vector &MapVector::GetValues(const Vector &vector) {
 	return GetValues((Vector &)vector);
 }
 
+MapInvalidReason MapVector::CheckMapValidity(Vector &map, idx_t count, const SelectionVector &sel) {
+	D_ASSERT(map.GetType().id() == LogicalTypeId::MAP);
+	UnifiedVectorFormat map_vdata;
+
+	map.ToUnifiedFormat(count, map_vdata);
+	auto &map_validity = map_vdata.validity;
+
+	auto list_data = ListVector::GetData(map);
+	auto &keys = MapVector::GetKeys(map);
+	UnifiedVectorFormat key_vdata;
+	keys.ToUnifiedFormat(count, key_vdata);
+	auto &key_validity = key_vdata.validity;
+
+	for (idx_t row = 0; row < count; row++) {
+		auto mapped_row = sel.get_index(row);
+		auto row_idx = map_vdata.sel->get_index(mapped_row);
+		// map is allowed to be NULL
+		if (!map_validity.RowIsValid(row_idx)) {
+			continue;
+		}
+		row_idx = key_vdata.sel->get_index(row);
+		value_set_t unique_keys;
+		for (idx_t i = 0; i < list_data[row_idx].length; i++) {
+			auto index = list_data[row_idx].offset + i;
+			index = key_vdata.sel->get_index(index);
+			if (!key_validity.RowIsValid(index)) {
+				return MapInvalidReason::NULL_KEY;
+			}
+			auto value = keys.GetValue(index);
+			auto result = unique_keys.insert(value);
+			if (!result.second) {
+				return MapInvalidReason::DUPLICATE_KEY;
+			}
+		}
+	}
+	return MapInvalidReason::VALID;
+}
+
+void MapVector::MapConversionVerify(Vector &vector, idx_t count) {
+	auto valid_check = MapVector::CheckMapValidity(vector, count);
+	switch (valid_check) {
+	case MapInvalidReason::VALID:
+		break;
+	case MapInvalidReason::DUPLICATE_KEY: {
+		throw InvalidInputException("Map keys have to be unique");
+	}
+	case MapInvalidReason::NULL_KEY: {
+		throw InvalidInputException("Map keys can not be NULL");
+	}
+	case MapInvalidReason::NULL_KEY_LIST: {
+		throw InvalidInputException("The list of map keys is not allowed to be NULL");
+	}
+	default: {
+		throw InternalException("MapInvalidReason not implemented");
+	}
+	}
+}
+
+//===--------------------------------------------------------------------===//
+// StructVector
+//===--------------------------------------------------------------------===//
 vector<unique_ptr<Vector>> &StructVector::GetEntries(Vector &vector) {
 	D_ASSERT(vector.GetType().id() == LogicalTypeId::STRUCT || vector.GetType().id() == LogicalTypeId::UNION);
 
@@ -1750,6 +1827,9 @@ const vector<unique_ptr<Vector>> &StructVector::GetEntries(const Vector &vector)
 	return GetEntries((Vector &)vector);
 }
 
+//===--------------------------------------------------------------------===//
+// ListVector
+//===--------------------------------------------------------------------===//
 const Vector &ListVector::GetEntry(const Vector &vector) {
 	D_ASSERT(vector.GetType().id() == LogicalTypeId::LIST || vector.GetType().id() == LogicalTypeId::MAP);
 	if (vector.GetVectorType() == VectorType::DICTIONARY_VECTOR) {
@@ -1928,7 +2008,9 @@ void ListVector::GetConsecutiveChildSelVector(Vector &list, SelectionVector &sel
 	//	info.second.offset = 0;
 }
 
-// Union vector
+//===--------------------------------------------------------------------===//
+// UnionVector
+//===--------------------------------------------------------------------===//
 const Vector &UnionVector::GetMember(const Vector &vector, idx_t member_index) {
 	D_ASSERT(member_index < UnionType::GetMemberCount(vector.GetType()));
 	auto &entries = StructVector::GetEntries(vector);
@@ -2015,4 +2097,55 @@ union_tag_t UnionVector::GetTag(const Vector &vector, idx_t index) {
 	return FlatVector::GetData<union_tag_t>(tag_vector)[index];
 }
 
+UnionInvalidReason UnionVector::CheckUnionValidity(Vector &vector, idx_t count, const SelectionVector &sel) {
+	D_ASSERT(vector.GetType().id() == LogicalTypeId::UNION);
+	auto member_count = UnionType::GetMemberCount(vector.GetType());
+	if (member_count == 0) {
+		return UnionInvalidReason::NO_MEMBERS;
+	}
+
+	UnifiedVectorFormat union_vdata;
+	vector.ToUnifiedFormat(count, union_vdata);
+
+	UnifiedVectorFormat tags_vdata;
+	auto &tag_vector = UnionVector::GetTags(vector);
+	tag_vector.ToUnifiedFormat(count, tags_vdata);
+
+	// check that only one member is valid at a time
+	for (idx_t row_idx = 0; row_idx < count; row_idx++) {
+		auto union_mapped_row_idx = sel.get_index(row_idx);
+		if (!union_vdata.validity.RowIsValid(union_mapped_row_idx)) {
+			continue;
+		}
+
+		auto tag_mapped_row_idx = tags_vdata.sel->get_index(row_idx);
+		if (!tags_vdata.validity.RowIsValid(tag_mapped_row_idx)) {
+			continue;
+		}
+
+		auto tag = ((union_tag_t *)tags_vdata.data)[tag_mapped_row_idx];
+		if (tag >= member_count) {
+			return UnionInvalidReason::TAG_OUT_OF_RANGE;
+		}
+
+		bool found_valid = false;
+		for (idx_t member_idx = 0; member_idx < member_count; member_idx++) {
+
+			UnifiedVectorFormat member_vdata;
+			auto &member = UnionVector::GetMember(vector, member_idx);
+			member.ToUnifiedFormat(count, member_vdata);
+
+			auto mapped_row_idx = member_vdata.sel->get_index(row_idx);
+			if (member_vdata.validity.RowIsValid(mapped_row_idx)) {
+				if (found_valid) {
+					return UnionInvalidReason::VALIDITY_OVERLAP;
+				}
+				found_valid = true;
+			}
+		}
+	}
+
+	return UnionInvalidReason::VALID;
+}
+
 } // namespace duckdb