duckdb 0.3.5-dev1364.0 → 0.3.5-dev1402.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.5-dev1364.0",
+  "version": "0.3.5-dev1402.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -45605,6 +45605,126 @@ bool Value::ValuesAreEqual(const Value &result_value, const Value &value) {
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/scalar/nested_functions.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+
+
+namespace duckdb {
+
+enum class MapInvalidReason : uint8_t { VALID, NULL_KEY_LIST, NULL_KEY, DUPLICATE_KEY };
+
+struct VariableReturnBindData : public FunctionData {
+	LogicalType stype;
+
+	explicit VariableReturnBindData(const LogicalType &stype_p) : stype(stype_p) {
+	}
+
+	unique_ptr<FunctionData> Copy() const override {
+		return make_unique<VariableReturnBindData>(stype);
+	}
+	bool Equals(const FunctionData &other_p) const override {
+		auto &other = (const VariableReturnBindData &)other_p;
+		return stype == other.stype;
+	}
+};
+
+template <class T, class MAP_TYPE = map<T, idx_t>>
+struct HistogramAggState {
+	MAP_TYPE *hist;
+};
+
+struct ArraySliceFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct StructPackFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListValueFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListRangeFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct MapFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct MapExtractFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListExtractFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListConcatFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListContainsFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListFlattenFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListPositionFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListAggregateFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListDistinctFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListUniqueFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct ListSortFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct CardinalityFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct StructExtractFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+MapInvalidReason CheckMapValidity(Vector &map, idx_t count,
+                                  const SelectionVector &sel = *FlatVector::IncrementalSelectionVector());
+
+} // namespace duckdb
+
 
 #include <cstring> // strlen() on Solaris
 
@@ -46585,6 +46705,14 @@ void Vector::UTFVerify(idx_t count) {
 	UTFVerify(*flat_sel, count);
 }
 
+void Vector::VerifyMap(Vector &vector_p, const SelectionVector &sel_p, idx_t count) {
+#ifdef DEBUG
+	D_ASSERT(vector_p.GetType().id() == LogicalTypeId::MAP);
+	auto valid_check = CheckMapValidity(vector_p, count, sel_p);
+	D_ASSERT(valid_check == MapInvalidReason::VALID);
+#endif // DEBUG
+}
+
 void Vector::Verify(Vector &vector_p, const SelectionVector &sel_p, idx_t count) {
 #ifdef DEBUG
 	if (count == 0) {
@@ -46676,6 +46804,9 @@ void Vector::Verify(Vector &vector_p, const SelectionVector &sel_p, idx_t count)
 				}
 			}
 		}
+		if (vector->GetType().id() == LogicalTypeId::MAP) {
+			VerifyMap(*vector, *sel, count);
+		}
 	}
 
 	if (type.InternalType() == PhysicalType::LIST) {
@@ -86125,120 +86256,6 @@ void BuiltinFunctions::RegisterHolisticAggregates() {
 	Register<ReservoirQuantileFun>();
 }
 
-} // namespace duckdb
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/function/scalar/nested_functions.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-
-
-
-
-
-namespace duckdb {
-
-struct VariableReturnBindData : public FunctionData {
-	LogicalType stype;
-
-	explicit VariableReturnBindData(const LogicalType &stype_p) : stype(stype_p) {
-	}
-
-	unique_ptr<FunctionData> Copy() const override {
-		return make_unique<VariableReturnBindData>(stype);
-	}
-	bool Equals(const FunctionData &other_p) const override {
-		auto &other = (const VariableReturnBindData &)other_p;
-		return stype == other.stype;
-	}
-};
-
-template <class T, class MAP_TYPE = map<T, idx_t>>
-struct HistogramAggState {
-	MAP_TYPE *hist;
-};
-
-struct ArraySliceFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct StructPackFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListValueFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListRangeFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct MapFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct MapExtractFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListExtractFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListConcatFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListContainsFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListFlattenFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListPositionFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListAggregateFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListDistinctFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListUniqueFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct ListSortFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct CardinalityFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
-struct StructExtractFun {
-	static ScalarFunction GetFunction();
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
 } // namespace duckdb
 
 //===----------------------------------------------------------------------===//
@@ -96802,8 +96819,110 @@ void CardinalityFun::RegisterFunction(BuiltinFunctions &set) {
 
 
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/parser/expression_map.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+
 namespace duckdb {
 
+struct ValueHashFunction {
+	uint64_t operator()(const Value &value) const {
+		return (uint64_t)value.Hash();
+	}
+};
+
+struct ValueEquality {
+	bool operator()(const Value &a, const Value &b) const {
+		return Value::NotDistinctFrom(a, b);
+	}
+};
+
+template <typename T>
+using value_map_t = unordered_map<Value, T, ValueHashFunction, ValueEquality>;
+
+using value_set_t = unordered_set<Value, ValueHashFunction, ValueEquality>;
+
+} // namespace duckdb
+
+
+namespace duckdb {
+
+// TODO: this doesn't recursively verify maps if maps are nested
+MapInvalidReason CheckMapValidity(Vector &map, idx_t count, const SelectionVector &sel) {
+	D_ASSERT(map.GetType().id() == LogicalTypeId::MAP);
+	VectorData map_vdata;
+	map.Orrify(count, map_vdata);
+	auto &map_validity = map_vdata.validity;
+
+	auto &key_vector = *(StructVector::GetEntries(map)[0]);
+	VectorData key_vdata;
+	key_vector.Orrify(count, key_vdata);
+	auto key_data = (list_entry_t *)key_vdata.data;
+	auto &key_validity = key_vdata.validity;
+
+	auto &key_entries = ListVector::GetEntry(key_vector);
+	VectorData key_entry_vdata;
+	key_entries.Orrify(count, key_entry_vdata);
+	auto &entry_validity = key_entry_vdata.validity;
+
+	for (idx_t row = 0; row < count; row++) {
+		auto mapped_row = sel.get_index(row);
+		auto row_idx = map_vdata.sel->get_index(mapped_row);
+		// map is allowed to be NULL
+		if (!map_validity.RowIsValid(row_idx)) {
+			continue;
+		}
+		row_idx = key_vdata.sel->get_index(row);
+		if (!key_validity.RowIsValid(row_idx)) {
+			return MapInvalidReason::NULL_KEY_LIST;
+		}
+		value_set_t unique_keys;
+		for (idx_t i = 0; i < key_data[row_idx].length; i++) {
+			auto index = key_data[row_idx].offset + i;
+			index = key_entry_vdata.sel->get_index(index);
+			if (!entry_validity.RowIsValid(index)) {
+				return MapInvalidReason::NULL_KEY;
+			}
+			auto value = key_entries.GetValue(index);
+			auto result = unique_keys.insert(value);
+			if (!result.second) {
+				return MapInvalidReason::DUPLICATE_KEY;
+			}
+		}
+	}
+	return MapInvalidReason::VALID;
+}
+
+static void MapConversionVerify(Vector &vector, idx_t count) {
+	auto valid_check = CheckMapValidity(vector, count);
+	switch (valid_check) {
+	case MapInvalidReason::VALID:
+		break;
+	case MapInvalidReason::DUPLICATE_KEY: {
+		throw InvalidInputException("Map keys have to be unique");
+	}
+	case MapInvalidReason::NULL_KEY: {
+		throw InvalidInputException("Map keys can not be NULL");
+	}
+	case MapInvalidReason::NULL_KEY_LIST: {
+		throw InvalidInputException("The list of map keys is not allowed to be NULL");
+	}
+	default: {
+		throw InternalException("MapInvalidReason not implemented");
+	}
+	}
+}
+
 static void MapFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	D_ASSERT(result.GetType().id() == LogicalTypeId::MAP);
 
@@ -96817,19 +96936,19 @@ static void MapFunction(DataChunk &args, ExpressionState &state, Vector &result)
 
 	auto &child_entries = StructVector::GetEntries(result);
 	D_ASSERT(child_entries.size() == 2);
-	auto &key_vector = child_entries[0];
-	auto &value_vector = child_entries[1];
+	auto &key_vector = *child_entries[0];
+	auto &value_vector = *child_entries[1];
 	if (args.data.empty()) {
 		// no arguments: construct an empty map
-		ListVector::SetListSize(*key_vector, 0);
-		key_vector->SetVectorType(VectorType::CONSTANT_VECTOR);
-		auto list_data = ConstantVector::GetData<list_entry_t>(*key_vector);
+		ListVector::SetListSize(key_vector, 0);
+		key_vector.SetVectorType(VectorType::CONSTANT_VECTOR);
+		auto list_data = ConstantVector::GetData<list_entry_t>(key_vector);
 		list_data->offset = 0;
 		list_data->length = 0;
 
-		ListVector::SetListSize(*value_vector, 0);
-		value_vector->SetVectorType(VectorType::CONSTANT_VECTOR);
-		list_data = ConstantVector::GetData<list_entry_t>(*value_vector);
+		ListVector::SetListSize(value_vector, 0);
+		value_vector.SetVectorType(VectorType::CONSTANT_VECTOR);
+		list_data = ConstantVector::GetData<list_entry_t>(value_vector);
 		list_data->offset = 0;
 		list_data->length = 0;
 
@@ -96840,8 +96959,10 @@ static void MapFunction(DataChunk &args, ExpressionState &state, Vector &result)
 	if (ListVector::GetListSize(args.data[0]) != ListVector::GetListSize(args.data[1])) {
 		throw Exception("Key list has a different size from Value list");
 	}
-	key_vector->Reference(args.data[0]);
-	value_vector->Reference(args.data[1]);
+
+	key_vector.Reference(args.data[0]);
+	value_vector.Reference(args.data[1]);
+	MapConversionVerify(result, args.size());
 
 	result.Verify(args.size());
 }
@@ -105367,6 +105488,7 @@ void BuiltinFunctions::RegisterArrowFunctions() {
 
 
 
+
 namespace duckdb {
 
 void ShiftRight(unsigned char *ar, int size, int shift) {
@@ -105569,6 +105691,26 @@ void ArrowToDuckDBBlob(Vector &vector, ArrowArray &array, ArrowScanLocalState &s
 	}
 }
 
+void ArrowToDuckDBMapVerify(Vector &vector, idx_t count) {
+	auto valid_check = CheckMapValidity(vector, count);
+	switch (valid_check) {
+	case MapInvalidReason::VALID:
+		break;
+	case MapInvalidReason::DUPLICATE_KEY: {
+		throw InvalidInputException("Arrow map contains duplicate key, which isn't supported by DuckDB map type");
+	}
+	case MapInvalidReason::NULL_KEY: {
+		throw InvalidInputException("Arrow map contains NULL as map key, which isn't supported by DuckDB map type");
+	}
+	case MapInvalidReason::NULL_KEY_LIST: {
+		throw InvalidInputException("Arrow map contains NULL as key list, which isn't supported by DuckDB map type");
+	}
+	default: {
+		throw InternalException("MapInvalidReason not implemented");
+	}
+	}
+}
+
 void ArrowToDuckDBMapList(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
                           unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data, idx_t col_idx,
                           pair<idx_t, idx_t> &arrow_convert_idx, uint32_t *offsets, ValidityMask *parent_mask) {
@@ -105975,6 +106117,7 @@ void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLocalState
 			ArrowToDuckDBMapList(*child_entries[type_idx], *struct_arrow.children[type_idx], scan_state, size,
 			                     arrow_convert_data, col_idx, arrow_convert_idx, offsets, &struct_validity_mask);
 		}
+		ArrowToDuckDBMapVerify(vector, size);
 		break;
 	}
 	case LogicalTypeId::STRUCT: {

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "aa105267d"
-#define DUCKDB_VERSION "v0.3.5-dev1364"
+#define DUCKDB_SOURCE_ID "192e3cf8a"
+#define DUCKDB_VERSION "v0.3.5-dev1402"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -3718,6 +3718,8 @@ public:
 	//! Verify that the Vector is in a consistent, not corrupt state. DEBUG
 	//! FUNCTION ONLY!
 	DUCKDB_API void Verify(idx_t count);
+	//! Asserts that the CheckMapValidity returns MapInvalidReason::VALID
+	DUCKDB_API static void VerifyMap(Vector &map, const SelectionVector &sel, idx_t count);
 	DUCKDB_API static void Verify(Vector &vector, const SelectionVector &sel, idx_t count);
 	DUCKDB_API void UTFVerify(idx_t count);
 	DUCKDB_API void UTFVerify(const SelectionVector &sel, idx_t count);