duckdb 0.6.2-dev1255.0 → 0.6.2-dev1264.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1255.0",
+  "version": "0.6.2-dev1264.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/function/cast/map_cast.cpp

@@ -3,6 +3,18 @@
 
 namespace duckdb {
 
+unique_ptr<BoundCastData> MapBoundCastData::BindMapToMapCast(BindCastInput &input, const LogicalType &source,
+                                                             const LogicalType &target) {
+	vector<BoundCastInfo> child_cast_info;
+	auto source_key = MapType::KeyType(source);
+	auto target_key = MapType::KeyType(target);
+	auto source_val = MapType::ValueType(source);
+	auto target_val = MapType::ValueType(target);
+	auto key_cast = input.GetCastFunction(source_key, target_key);
+	auto value_cast = input.GetCastFunction(source_val, target_val);
+	return make_unique<MapBoundCastData>(move(key_cast), move(value_cast));
+}
+
 static bool MapToVarcharCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 	auto constant = source.GetVectorType() == VectorType::CONSTANT_VECTOR;
 	auto varchar_type = LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR);

package/src/duckdb/src/function/cast/string_cast.cpp

@@ -2,6 +2,7 @@
 #include "duckdb/function/cast/vector_cast_helpers.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/vector.hpp"
+#include "duckdb/function/scalar/nested_functions.hpp"
 
 namespace duckdb {
 
@@ -114,10 +115,10 @@ static BoundCastInfo VectorStringCastNumericSwitch(BindCastInput &input, const L
 	}
 }
 
+// string -> list casting
 bool VectorStringToList::StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
                                                     ValidityMask &result_mask, idx_t count, CastParameters &parameters,
                                                     const SelectionVector *sel) {
-
 	idx_t total_list_size = 0;
 	for (idx_t i = 0; i < count; i++) {
 		idx_t idx = i;
@@ -127,7 +128,7 @@ bool VectorStringToList::StringToNestedTypeCastLoop(string_t *source_data, Valid
 		if (!source_mask.RowIsValid(idx)) {
 			continue;
 		}
-		total_list_size += VectorStringToList::CountParts(source_data[idx]);
+		total_list_size += VectorStringToList::CountPartsList(source_data[idx]);
 	}
 
 	Vector varchar_vector(LogicalType::VARCHAR, total_list_size);
@@ -151,7 +152,7 @@ bool VectorStringToList::StringToNestedTypeCastLoop(string_t *source_data, Valid
 		}
 
 		list_data[i].offset = total;
-		if (!VectorStringToList::SplitStringifiedList(source_data[idx], child_data, total, varchar_vector)) {
+		if (!VectorStringToList::SplitStringList(source_data[idx], child_data, total, varchar_vector)) {
 			string text = "Type VARCHAR with value '" + source_data[idx].GetString() +
 			              "' can't be cast to the destination type LIST";
 			HandleVectorCastError::Operation<string_t>(text, result_mask, idx, parameters.error_message, all_converted);
@@ -176,6 +177,7 @@ static LogicalType InitVarcharStructType(const LogicalType &target) {
 	return LogicalType::STRUCT(child_types);
 }
 
+// string -> struct casting
 bool VectorStringToStruct::StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
                                                       ValidityMask &result_mask, idx_t count,
                                                       CastParameters &parameters, const SelectionVector *sel) {
@@ -220,8 +222,6 @@ bool VectorStringToStruct::StringToNestedTypeCastLoop(string_t *source_data, Val
 		auto &varchar_vector = *child_vectors[child_idx];
 		auto &result_child_vector = *result_children[child_idx];
 		auto &child_cast_info = cast_data.child_cast_info[child_idx];
-		// get the correct casting function (VARCHAR -> result_child_type) from cast_data
-		// casting functions are determined by BindStructtoStructCast
 		CastParameters child_parameters(parameters, child_cast_info.cast_data.get());
 		if (!child_cast_info.function(varchar_vector, result_child_vector, count, child_parameters)) {
 			all_converted = false;
@@ -230,6 +230,87 @@ bool VectorStringToStruct::StringToNestedTypeCastLoop(string_t *source_data, Val
 	return all_converted;
 }
 
+// string -> map casting
+bool VectorStringToMap::StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
+                                                   ValidityMask &result_mask, idx_t count, CastParameters &parameters,
+                                                   const SelectionVector *sel) {
+	idx_t total_elements = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			continue;
+		}
+		total_elements += (VectorStringToMap::CountPartsMap(source_data[idx]) + 1) / 2;
+	}
+
+	Vector varchar_key_vector(LogicalType::VARCHAR, total_elements);
+	Vector varchar_val_vector(LogicalType::VARCHAR, total_elements);
+	auto child_key_data = FlatVector::GetData<string_t>(varchar_key_vector);
+	auto child_val_data = FlatVector::GetData<string_t>(varchar_val_vector);
+
+	ListVector::Reserve(result, total_elements);
+	ListVector::SetListSize(result, total_elements);
+	auto list_data = ListVector::GetData(result);
+
+	bool all_converted = true;
+	idx_t total = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			result_mask.SetInvalid(idx);
+			continue;
+		}
+
+		list_data[i].offset = total;
+		if (!VectorStringToMap::SplitStringMap(source_data[idx], child_key_data, child_val_data, total,
+		                                       varchar_key_vector, varchar_val_vector)) {
+			string text = "Type VARCHAR with value '" + source_data[idx].GetString() +
+			              "' can't be cast to the destination type MAP";
+			FlatVector::SetNull(result, idx, true);
+			HandleVectorCastError::Operation<string_t>(text, result_mask, idx, parameters.error_message, all_converted);
+		}
+		list_data[i].length = total - list_data[i].offset;
+	}
+	D_ASSERT(total_elements == total);
+
+	auto &result_key_child = MapVector::GetKeys(result);
+	auto &result_val_child = MapVector::GetValues(result);
+	auto &cast_data = (MapBoundCastData &)*parameters.cast_data;
+
+	CastParameters key_params(parameters, cast_data.key_cast.cast_data.get());
+	if (!cast_data.key_cast.function(varchar_key_vector, result_key_child, total_elements, key_params)) {
+		all_converted = false;
+	}
+	CastParameters val_params(parameters, cast_data.value_cast.cast_data.get());
+	if (!cast_data.value_cast.function(varchar_val_vector, result_val_child, total_elements, val_params)) {
+		all_converted = false;
+	}
+
+	auto &key_validity = FlatVector::Validity(result_key_child);
+	if (!all_converted) {
+		for (idx_t row_idx = 0; row_idx < count; row_idx++) {
+			if (!result_mask.RowIsValid(row_idx)) {
+				continue;
+			}
+			auto list = list_data[row_idx];
+			for (idx_t list_idx = 0; list_idx < list.length; list_idx++) {
+				auto idx = list.offset + list_idx;
+				if (!key_validity.RowIsValid(idx)) {
+					result_mask.SetInvalid(row_idx);
+				}
+			}
+		}
+	}
+	MapConversionVerify(result, count);
+	return all_converted;
+}
+
 template <class T>
 bool StringToNestedTypeCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 	D_ASSERT(source.GetType().id() == LogicalTypeId::VARCHAR);
@@ -239,7 +320,6 @@ bool StringToNestedTypeCast(Vector &source, Vector &result, idx_t count, CastPar
 		auto source_data = ConstantVector::GetData<string_t>(source);
 		auto &source_mask = ConstantVector::Validity(source);
 		auto &result_mask = FlatVector::Validity(result);
-
 		auto ret = T::StringToNestedTypeCastLoop(source_data, source_mask, result, result_mask, 1, parameters, nullptr);
 		result.SetVectorType(VectorType::CONSTANT_VECTOR);
 		return ret;
@@ -296,6 +376,10 @@ BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const Logical
 	case LogicalTypeId::STRUCT:
 		return BoundCastInfo(&StringToNestedTypeCast<VectorStringToStruct>,
 		                     StructBoundCastData::BindStructToStructCast(input, InitVarcharStructType(target), target));
+	case LogicalTypeId::MAP:
+		return BoundCastInfo(&StringToNestedTypeCast<VectorStringToMap>,
+		                     MapBoundCastData::BindMapToMapCast(
+		                         input, LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR), target));
 	default:
 		return VectorStringCastNumericSwitch(input, source, target);
 	}

package/src/duckdb/src/function/cast/vector_cast_helpers.cpp

@@ -2,6 +2,7 @@
 
 namespace duckdb {
 
+// ------- Helper functions for splitting string nested types  -------
 static bool IsNull(const char *buf, idx_t start_pos, Vector &child, idx_t row_idx) {
 	if (buf[start_pos] == 'N' && buf[start_pos + 1] == 'U' && buf[start_pos + 2] == 'L' && buf[start_pos + 3] == 'L') {
 		FlatVector::SetNull(child, row_idx, true);
@@ -16,46 +17,6 @@ inline static void SkipWhitespace(const char *buf, idx_t &pos, idx_t len) {
 	}
 }
 
-static idx_t StringTrim(const char *buf, idx_t &start_pos, idx_t pos) {
-	idx_t trailing_whitespace = 0;
-	while (StringUtil::CharacterIsSpace(buf[pos - trailing_whitespace - 1])) {
-		trailing_whitespace++;
-	}
-	if ((buf[start_pos] == '"' && buf[pos - trailing_whitespace - 1] == '"') ||
-	    (buf[start_pos] == '\'' && buf[pos - trailing_whitespace - 1] == '\'')) {
-		start_pos++;
-		trailing_whitespace++;
-	}
-	return (pos - trailing_whitespace);
-}
-
-struct CountPartOperation {
-	idx_t count = 0;
-
-	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
-		count++;
-	}
-};
-
-struct SplitStringOperation {
-	SplitStringOperation(string_t *child_data, idx_t &child_start, Vector &child)
-	    : child_data(child_data), child_start(child_start), child(child) {
-	}
-
-	string_t *child_data;
-	idx_t &child_start;
-	Vector &child;
-
-	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
-		if ((pos - start_pos) == 4 && IsNull(buf, start_pos, child, child_start)) {
-			child_start++;
-			return;
-		}
-		child_data[child_start] = StringVector::AddString(child, buf + start_pos, pos - start_pos);
-		child_start++;
-	}
-};
-
 static bool SkipToCloseQuotes(idx_t &pos, const char *buf, idx_t &len) {
 	char quote = buf[pos];
 	pos++;
@@ -97,8 +58,53 @@ static bool SkipToClose(idx_t &idx, const char *buf, idx_t &len, idx_t &lvl, cha
 	return false;
 }
 
+static idx_t StringTrim(const char *buf, idx_t &start_pos, idx_t pos) {
+	idx_t trailing_whitespace = 0;
+	while (StringUtil::CharacterIsSpace(buf[pos - trailing_whitespace - 1])) {
+		trailing_whitespace++;
+	}
+	if ((buf[start_pos] == '"' && buf[pos - trailing_whitespace - 1] == '"') ||
+	    (buf[start_pos] == '\'' && buf[pos - trailing_whitespace - 1] == '\'')) {
+		start_pos++;
+		trailing_whitespace++;
+	}
+	return (pos - trailing_whitespace);
+}
+
+struct CountPartOperation {
+	idx_t count = 0;
+
+	bool HandleKey(const char *buf, idx_t start_pos, idx_t pos) {
+		count++;
+		return true;
+	}
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+		count++;
+	}
+};
+
+// ------- LIST SPLIT -------
+struct SplitStringListOperation {
+	SplitStringListOperation(string_t *child_data, idx_t &child_start, Vector &child)
+	    : child_data(child_data), child_start(child_start), child(child) {
+	}
+
+	string_t *child_data;
+	idx_t &child_start;
+	Vector &child;
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+		if ((pos - start_pos) == 4 && IsNull(buf, start_pos, child, child_start)) {
+			child_start++;
+			return;
+		}
+		child_data[child_start] = StringVector::AddString(child, buf + start_pos, pos - start_pos);
+		child_start++;
+	}
+};
+
 template <class OP>
-static bool SplitStringifiedListInternal(const string_t &input, OP &state) {
+static bool SplitStringListInternal(const string_t &input, OP &state) {
 	const char *buf = input.GetDataUnsafe();
 	idx_t len = input.GetSize();
 	idx_t lvl = 1;
@@ -143,19 +149,121 @@ static bool SplitStringifiedListInternal(const string_t &input, OP &state) {
 	return (pos == len && lvl == 0);
 }
 
-bool VectorStringToList::SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start,
-                                              Vector &child) {
-	SplitStringOperation state(child_data, child_start, child);
-	return SplitStringifiedListInternal<SplitStringOperation>(input, state);
+bool VectorStringToList::SplitStringList(const string_t &input, string_t *child_data, idx_t &child_start,
+                                         Vector &child) {
+	SplitStringListOperation state(child_data, child_start, child);
+	return SplitStringListInternal<SplitStringListOperation>(input, state);
+}
+
+idx_t VectorStringToList::CountPartsList(const string_t &input) {
+	CountPartOperation state;
+	SplitStringListInternal<CountPartOperation>(input, state);
+	return state.count;
+}
+
+// ------- MAP SPLIT -------
+struct SplitStringMapOperation {
+	SplitStringMapOperation(string_t *child_key_data, string_t *child_val_data, idx_t &child_start, Vector &varchar_key,
+	                        Vector &varchar_val)
+	    : child_key_data(child_key_data), child_val_data(child_val_data), child_start(child_start),
+	      varchar_key(varchar_key), varchar_val(varchar_val) {
+	}
+
+	string_t *child_key_data;
+	string_t *child_val_data;
+	idx_t &child_start;
+	Vector &varchar_key;
+	Vector &varchar_val;
+
+	bool HandleKey(const char *buf, idx_t start_pos, idx_t pos) {
+		if ((pos - start_pos) == 4 && IsNull(buf, start_pos, varchar_key, child_start)) {
+			FlatVector::SetNull(varchar_val, child_start, true);
+			child_start++;
+			return false;
+		}
+		child_key_data[child_start] = StringVector::AddString(varchar_key, buf + start_pos, pos - start_pos);
+		return true;
+	}
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+		if ((pos - start_pos) == 4 && IsNull(buf, start_pos, varchar_val, child_start)) {
+			child_start++;
+			return;
+		}
+		child_val_data[child_start] = StringVector::AddString(varchar_val, buf + start_pos, pos - start_pos);
+		child_start++;
+	}
+};
+
+template <class OP>
+static bool FindKeyOrValueMap(const char *buf, idx_t len, idx_t &pos, OP &state, bool key) {
+	auto start_pos = pos;
+	idx_t lvl = 0;
+	while (pos < len) {
+		if (buf[pos] == '"' || buf[pos] == '\'') {
+			SkipToCloseQuotes(pos, buf, len);
+		} else if (buf[pos] == '{') {
+			SkipToClose(pos, buf, len, lvl, '}');
+		} else if (buf[pos] == '[') {
+			SkipToClose(pos, buf, len, lvl, ']');
+		} else if (key && buf[pos] == '=') {
+			idx_t end_pos = StringTrim(buf, start_pos, pos);
+			return state.HandleKey(buf, start_pos, end_pos); // put string in KEY_child_vector
+		} else if (!key && (buf[pos] == ',' || buf[pos] == '}')) {
+			idx_t end_pos = StringTrim(buf, start_pos, pos);
+			state.HandleValue(buf, start_pos, end_pos); // put string in VALUE_child_vector
+			return true;
+		}
+		pos++;
+	}
+	return false;
+}
+
+template <class OP>
+static bool SplitStringMapInternal(const string_t &input, OP &state) {
+	const char *buf = input.GetDataUnsafe();
+	idx_t len = input.GetSize();
+	idx_t pos = 0;
+
+	SkipWhitespace(buf, pos, len);
+	if (pos == len || buf[pos] != '{') {
+		return false;
+	}
+	SkipWhitespace(buf, ++pos, len);
+	if (pos == len) {
+		return false;
+	}
+	if (buf[pos] == '}') {
+		SkipWhitespace(buf, ++pos, len);
+		return (pos == len);
+	}
+	while (pos < len) {
+		if (!FindKeyOrValueMap(buf, len, pos, state, true)) {
+			return false;
+		}
+		SkipWhitespace(buf, ++pos, len);
+		if (!FindKeyOrValueMap(buf, len, pos, state, false)) {
+			return false;
+		}
+		SkipWhitespace(buf, ++pos, len);
+	}
+	return true;
+}
+
+bool VectorStringToMap::SplitStringMap(const string_t &input, string_t *child_key_data, string_t *child_val_data,
+                                       idx_t &child_start, Vector &varchar_key, Vector &varchar_val) {
+	SplitStringMapOperation state(child_key_data, child_val_data, child_start, varchar_key, varchar_val);
+	return SplitStringMapInternal<SplitStringMapOperation>(input, state);
 }
 
-idx_t VectorStringToList::CountParts(const string_t &input) {
+idx_t VectorStringToMap::CountPartsMap(const string_t &input) {
 	CountPartOperation state;
-	SplitStringifiedListInternal<CountPartOperation>(input, state);
+	SplitStringMapInternal<CountPartOperation>(input, state);
 	return state.count;
 }
 
-static bool FindKey(const char *buf, idx_t len, idx_t &pos) {
+// ------- STRUCT SPLIT -------
+static bool FindKeyStruct(const char *buf, idx_t len, idx_t &pos) {
 	while (pos < len) {
 		if (buf[pos] == ':') {
 			return true;
@@ -165,8 +273,8 @@ static bool FindKey(const char *buf, idx_t len, idx_t &pos) {
 	return false;
 }
 
-static bool FindValue(const char *buf, idx_t len, idx_t &pos, Vector &varchar_child, idx_t &row_idx,
-                      ValidityMask *child_mask) {
+static bool FindValueStruct(const char *buf, idx_t len, idx_t &pos, Vector &varchar_child, idx_t &row_idx,
+                            ValidityMask *child_mask) {
 	auto start_pos = pos;
 	idx_t lvl = 0;
 	while (pos < len) {
@@ -209,7 +317,7 @@ bool VectorStringToStruct::SplitStruct(string_t &input, std::vector<std::unique_
 	} else {
 		while (pos < len) {
 			auto key_start = pos;
-			if (!FindKey(buf, len, pos)) {
+			if (!FindKeyStruct(buf, len, pos)) {
 				return false;
 			}
 			auto key_end = StringTrim(buf, key_start, pos);
@@ -221,7 +329,7 @@ bool VectorStringToStruct::SplitStruct(string_t &input, std::vector<std::unique_
 			}
 			child_idx = it->second;
 			SkipWhitespace(buf, ++pos, len);
-			if (!FindValue(buf, len, pos, *varchar_vectors[child_idx], row_idx, child_masks[child_idx])) {
+			if (!FindValueStruct(buf, len, pos, *varchar_vectors[child_idx], row_idx, child_masks[child_idx])) {
 				return false;
 			}
 			SkipWhitespace(buf, ++pos, len);

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev1255"
+#define DUCKDB_VERSION "0.6.2-dev1264"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "5aa5644949"
+#define DUCKDB_SOURCE_ID "19098105dc"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/function/cast/default_casts.hpp

@@ -108,6 +108,23 @@ public:
 	}
 };
 
+struct MapBoundCastData : public BoundCastData {
+	MapBoundCastData(BoundCastInfo key_cast, BoundCastInfo value_cast)
+	    : key_cast(move(key_cast)), value_cast(move(value_cast)) {
+	}
+
+	BoundCastInfo key_cast;
+	BoundCastInfo value_cast;
+
+	static unique_ptr<BoundCastData> BindMapToMapCast(BindCastInput &input, const LogicalType &source,
+	                                                  const LogicalType &target);
+
+public:
+	unique_ptr<BoundCastData> Copy() const override {
+		return make_unique<MapBoundCastData>(key_cast.Copy(), value_cast.Copy());
+	}
+};
+
 struct DefaultCasts {
 	static BoundCastInfo GetDefaultCastFunction(BindCastInput &input, const LogicalType &source,
 	                                            const LogicalType &target);

package/src/duckdb/src/include/duckdb/function/cast/vector_cast_helpers.hpp

@@ -199,8 +199,8 @@ struct VectorCastHelpers {
 };
 
 struct VectorStringToList {
-	static idx_t CountParts(const string_t &input);
-	static bool SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start, Vector &child);
+	static idx_t CountPartsList(const string_t &input);
+	static bool SplitStringList(const string_t &input, string_t *child_data, idx_t &child_start, Vector &child);
 	static bool StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
 	                                       ValidityMask &result_mask, idx_t count, CastParameters &parameters,
 	                                       const SelectionVector *sel);
@@ -214,4 +214,13 @@ struct VectorStringToStruct {
 	                                       const SelectionVector *sel);
 };
 
+struct VectorStringToMap {
+	static idx_t CountPartsMap(const string_t &input);
+	static bool SplitStringMap(const string_t &input, string_t *child_key_data, string_t *child_val_data,
+	                           idx_t &child_start, Vector &varchar_key, Vector &varchar_val);
+	static bool StringToNestedTypeCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result,
+	                                       ValidityMask &result_mask, idx_t count, CastParameters &parameters,
+	                                       const SelectionVector *sel);
+};
+
 } // namespace duckdb