duckdb 0.5.2-dev815.0 → 0.5.2-dev833.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev815.0",
+  "version": "0.5.2-dev833.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -29106,7 +29106,7 @@ struct RowOperations {
 namespace duckdb {
 
 template <class OP, class RETURN_TYPE, typename... ARGS>
-RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
+RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&...args) {
 	D_ASSERT(radix_bits <= sizeof(hash_t) * 8);
 	switch (radix_bits) {
 	case 1:
@@ -29135,7 +29135,7 @@ RETURN_TYPE RadixBitsSwitch(idx_t radix_bits, ARGS &&... args) {
 }
 
 template <class OP, class RETURN_TYPE, idx_t radix_bits_1, typename... ARGS>
-RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&... args) {
+RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&...args) {
 	D_ASSERT(radix_bits_2 <= sizeof(hash_t) * 8);
 	switch (radix_bits_2) {
 	case 1:
@@ -29164,7 +29164,7 @@ RETURN_TYPE DoubleRadixBitsSwitch2(idx_t radix_bits_2, ARGS &&... args) {
 }
 
 template <class OP, class RETURN_TYPE, typename... ARGS>
-RETURN_TYPE DoubleRadixBitsSwitch1(idx_t radix_bits_1, idx_t radix_bits_2, ARGS &&... args) {
+RETURN_TYPE DoubleRadixBitsSwitch1(idx_t radix_bits_1, idx_t radix_bits_2, ARGS &&...args) {
 	D_ASSERT(radix_bits_1 <= sizeof(hash_t) * 8);
 	switch (radix_bits_1) {
 	case 1:
@@ -49455,6 +49455,9 @@ void Vector::Initialize(bool zero_data, idx_t capacity) {
 			memset(data, 0, capacity * type_size);
 		}
 	}
+	if (capacity > STANDARD_VECTOR_SIZE) {
+		validity.Resize(STANDARD_VECTOR_SIZE, capacity);
+	}
 }
 
 struct DataArrays {
@@ -93661,6 +93664,13 @@ unique_ptr<FunctionData> HistogramBindFunction(ClientContext &context, Aggregate
                                                vector<unique_ptr<Expression>> &arguments) {
 
 	D_ASSERT(arguments.size() == 1);
+
+	if (arguments[0]->return_type.id() == LogicalTypeId::LIST ||
+	    arguments[0]->return_type.id() == LogicalTypeId::STRUCT ||
+	    arguments[0]->return_type.id() == LogicalTypeId::MAP) {
+		throw NotImplementedException("Unimplemented type for histogram %s", arguments[0]->return_type.ToString());
+	}
+
 	child_list_t<LogicalType> struct_children;
 	struct_children.push_back({"key", LogicalType::LIST(arguments[0]->return_type)});
 	struct_children.push_back({"value", LogicalType::LIST(LogicalType::UBIGINT)});
@@ -95714,6 +95724,11 @@ struct VectorCastHelpers {
 	}
 };
 
+struct VectorStringifiedListParser {
+	static idx_t CountParts(const string_t &input);
+	static bool SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start, Vector &child);
+};
+
 } // namespace duckdb
 
 
@@ -96484,20 +96499,8 @@ BoundCastInfo DefaultCasts::EnumCastSwitch(BindCastInput &input, const LogicalTy
 
 namespace duckdb {
 
-struct ListBoundCastData : public BoundCastData {
-	explicit ListBoundCastData(BoundCastInfo child_cast) : child_cast_info(move(child_cast)) {
-	}
-
-	BoundCastInfo child_cast_info;
-
-public:
-	unique_ptr<BoundCastData> Copy() const override {
-		return make_unique<ListBoundCastData>(child_cast_info.Copy());
-	}
-};
-
-unique_ptr<BoundCastData> BindListToListCast(BindCastInput &input, const LogicalType &source,
-                                             const LogicalType &target) {
+unique_ptr<BoundCastData> ListBoundCastData::BindListToListCast(BindCastInput &input, const LogicalType &source,
+                                                                const LogicalType &target) {
 	vector<BoundCastInfo> child_cast_info;
 	auto &source_child_type = ListType::GetChildType(source);
 	auto &result_child_type = ListType::GetChildType(target);
@@ -96608,11 +96611,11 @@ static bool ListToVarcharCast(Vector &source, Vector &result, idx_t count, CastP
 BoundCastInfo DefaultCasts::ListCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
 	switch (target.id()) {
 	case LogicalTypeId::LIST:
-		return BoundCastInfo(ListToListCast, BindListToListCast(input, source, target));
+		return BoundCastInfo(ListToListCast, ListBoundCastData::BindListToListCast(input, source, target));
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::JSON:
-		return BoundCastInfo(ListToVarcharCast,
-		                     BindListToListCast(input, source, LogicalType::LIST(LogicalType::VARCHAR)));
+		return BoundCastInfo(ListToVarcharCast, ListBoundCastData::BindListToListCast(
+		                                            input, source, LogicalType::LIST(LogicalType::VARCHAR)));
 	default:
 		return DefaultCasts::TryVectorNullCast;
 	}
@@ -96950,9 +96953,97 @@ static BoundCastInfo VectorStringCastNumericSwitch(BindCastInput &input, const L
 	}
 }
 
+bool StringListCastLoop(string_t *source_data, ValidityMask &source_mask, Vector &result, ValidityMask &result_mask,
+                        idx_t count, CastParameters &parameters, const SelectionVector *sel) {
+
+	idx_t total_list_size = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			continue;
+		}
+		total_list_size += VectorStringifiedListParser::CountParts(source_data[idx]);
+	}
+
+	Vector varchar_vector(LogicalType::VARCHAR, total_list_size);
+
+	ListVector::Reserve(result, total_list_size);
+	ListVector::SetListSize(result, total_list_size);
+
+	auto list_data = ListVector::GetData(result);
+	auto child_data = FlatVector::GetData<string_t>(varchar_vector);
+
+	bool all_converted = true;
+	idx_t total = 0;
+	for (idx_t i = 0; i < count; i++) {
+		idx_t idx = i;
+		if (sel) {
+			idx = sel->get_index(i);
+		}
+		if (!source_mask.RowIsValid(idx)) {
+			result_mask.SetInvalid(i);
+			continue;
+		}
+
+		list_data[i].offset = total;
+		auto valid =
+		    VectorStringifiedListParser::SplitStringifiedList(source_data[idx], child_data, total, varchar_vector);
+		if (!valid) {
+			string text = "Type VARCHAR with value '" + source_data[idx].GetString() +
+			              "' can't be cast to the destination type LIST";
+			HandleVectorCastError::Operation<string_t>(text, result_mask, idx, parameters.error_message, all_converted);
+		}
+		list_data[i].length = total - list_data[i].offset; // length is the amount of parts coming from this string
+	}
+	D_ASSERT(total_list_size == total);
+
+	auto &result_child = ListVector::GetEntry(result);
+	auto &cast_data = (ListBoundCastData &)*parameters.cast_data;
+	CastParameters child_parameters(parameters, cast_data.child_cast_info.cast_data.get());
+	return cast_data.child_cast_info.function(varchar_vector, result_child, total_list_size, child_parameters) &&
+	       all_converted;
+}
+
+bool StringListCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
+	D_ASSERT(source.GetType().id() == LogicalTypeId::VARCHAR);
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+
+	switch (source.GetVectorType()) {
+	case VectorType::CONSTANT_VECTOR: {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+
+		auto source_data = ConstantVector::GetData<string_t>(source);
+		auto &source_mask = ConstantVector::Validity(source);
+		auto &result_mask = ConstantVector::Validity(result);
+
+		return StringListCastLoop(source_data, source_mask, result, result_mask, 1, parameters, nullptr);
+	}
+	default: {
+		UnifiedVectorFormat unified_source;
+		result.SetVectorType(VectorType::FLAT_VECTOR);
+
+		source.ToUnifiedFormat(count, unified_source);
+		auto source_sel = unified_source.sel;
+		auto source_data = (string_t *)unified_source.data;
+		auto &source_mask = unified_source.validity;
+		auto &result_mask = FlatVector::Validity(result);
+
+		return StringListCastLoop(source_data, source_mask, result, result_mask, count, parameters, source_sel);
+	}
+	}
+}
+
+BoundCastInfo StringToListCast(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
+	// second argument allows for a secondary casting function to be passed in the CastParameters
+	return BoundCastInfo(&StringListCast,
+	                     ListBoundCastData::BindListToListCast(input, LogicalType::LIST(LogicalType::VARCHAR), target));
+}
+
 BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const LogicalType &source,
                                              const LogicalType &target) {
-	// now switch on the target type
 	switch (target.id()) {
 	case LogicalTypeId::DATE:
 		return BoundCastInfo(&VectorCastHelpers::TryCastErrorLoop<string_t, date_t, duckdb::TryCastErrorMessage>);
@@ -96980,6 +97071,8 @@ BoundCastInfo DefaultCasts::StringCastSwitch(BindCastInput &input, const Logical
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::JSON:
 		return &DefaultCasts::ReinterpretCast;
+	case LogicalTypeId::LIST:
+		return StringToListCast(input, source, target);
 	default:
 		return VectorStringCastNumericSwitch(input, source, target);
 	}
@@ -97341,6 +97434,144 @@ BoundCastInfo DefaultCasts::UUIDCastSwitch(BindCastInput &input, const LogicalTy
 } // namespace duckdb
 
 
+namespace duckdb {
+
+struct CountPartOperation {
+	idx_t count = 0;
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+		count++;
+	}
+};
+
+struct SplitStringOperation {
+	SplitStringOperation(string_t *child_data, idx_t &child_start, Vector &child)
+	    : child_data(child_data), child_start(child_start), child(child) {
+	}
+
+	string_t *child_data;
+	idx_t &child_start;
+	Vector &child;
+
+	void HandleValue(const char *buf, idx_t start_pos, idx_t pos) {
+
+		if ((pos - start_pos) >= 4 && buf[start_pos] == 'N' && buf[start_pos + 1] == 'U' && buf[start_pos + 2] == 'L' &&
+		    buf[start_pos + 3] == 'L') {
+			FlatVector::SetNull(child, child_start, true);
+			child_start++;
+			return;
+		}
+		child_data[child_start] = StringVector::AddString(child, buf + start_pos, pos - start_pos);
+		child_start++;
+	}
+};
+
+static bool SkipToCloseQuotes(idx_t &pos, const char *buf, idx_t &len) {
+	char quote = buf[pos];
+	pos++;
+
+	while (pos < len) {
+		if (buf[pos] == quote) {
+			return true;
+		}
+		pos++;
+	}
+	return false;
+}
+
+static bool SkipToClose(idx_t &idx, const char *buf, idx_t &len, idx_t &lvl) {
+	while (idx < len) {
+		if (buf[idx] == '[') {
+			if (!SkipToClose(++idx, buf, len, lvl)) {
+				return false;
+			}
+			lvl++;
+			idx++;
+		}
+		if (buf[idx] == '"' || buf[idx] == '\'') {
+			SkipToCloseQuotes(idx, buf, len);
+		}
+		if (buf[idx] == ']') {
+			lvl--;
+			return true;
+		}
+		idx++;
+	}
+	return false;
+}
+
+template <class OP>
+static bool SplitStringifiedListInternal(const string_t &input, OP &state) {
+	const char *buf = input.GetDataUnsafe();
+	idx_t len = input.GetSize();
+	idx_t lvl = 1;
+	idx_t pos = 0;
+
+	while (pos < len && StringUtil::CharacterIsSpace(buf[pos])) {
+		pos++;
+	}
+	if (pos == len || buf[pos] != '[') {
+		return false;
+	}
+	pos++;
+	while (pos < len && StringUtil::CharacterIsSpace(buf[pos])) {
+		pos++;
+	}
+
+	idx_t start_pos = pos;
+	while (pos < len) {
+		if (buf[pos] == '[') {
+			if (!SkipToClose(++pos, buf, len, ++lvl)) {
+				return false;
+			}
+		} else if (buf[pos] == '"' || buf[pos] == '\'') {
+			SkipToCloseQuotes(pos, buf, len);
+		} else if (buf[pos] == ',' || buf[pos] == ']') {
+			idx_t trailing_whitespace = 0;
+			while (StringUtil::CharacterIsSpace(buf[pos - trailing_whitespace - 1])) {
+				trailing_whitespace++;
+			}
+			if (!(buf[pos] == ']' && start_pos == (pos))) {
+				state.HandleValue(buf, start_pos, pos - trailing_whitespace);
+			} // else the list is empty
+			if (buf[pos] == ']') {
+				lvl--;
+				break;
+			}
+			while (pos + 1 < len && StringUtil::CharacterIsSpace(buf[pos + 1])) {
+				pos++;
+			}
+			start_pos = pos + 1;
+		}
+		pos++;
+	}
+	pos++;
+	while (pos < len) {
+		if (!StringUtil::CharacterIsSpace(buf[pos])) {
+			return false;
+		}
+		pos++;
+	}
+	if (lvl != 0) {
+		return false;
+	}
+	return true;
+}
+
+bool VectorStringifiedListParser::SplitStringifiedList(const string_t &input, string_t *child_data, idx_t &child_start,
+                                                       Vector &child) {
+	SplitStringOperation state(child_data, child_start, child);
+	return SplitStringifiedListInternal<SplitStringOperation>(input, state);
+}
+
+idx_t VectorStringifiedListParser::CountParts(const string_t &input) {
+	CountPartOperation state;
+	SplitStringifiedListInternal<CountPartOperation>(input, state);
+	return state.count;
+}
+} // namespace duckdb
+
+
 namespace duckdb {
 
 //! The target type determines the preferred implicit casts