duckdb 0.5.2-dev863.0 → 0.5.2-dev870.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev863.0",
+  "version": "0.5.2-dev870.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -16820,14 +16820,25 @@ struct ConcatFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
-struct ConcatWSFun {
-	static void RegisterFunction(BuiltinFunctions &set);
-};
-
 struct LengthFun {
 	static void RegisterFunction(BuiltinFunctions &set);
+	static inline bool IsCharacter(char c) {
+		return (c & 0xc0) != 0x80;
+	}
+
 	template <class TA, class TR>
 	static inline TR Length(TA input) {
+		auto input_data = input.GetDataUnsafe();
+		auto input_length = input.GetSize();
+		TR length = 0;
+		for (idx_t i = 0; i < input_length; i++) {
+			length += IsCharacter(input_data[i]);
+		}
+		return length;
+	}
+
+	template <class TA, class TR>
+	static inline TR GraphemeCount(TA input) {
 		auto input_data = input.GetDataUnsafe();
 		auto input_length = input.GetSize();
 		for (idx_t i = 0; i < input_length; i++) {
@@ -16881,7 +16892,8 @@ struct RegexpFun {
 
 struct SubstringFun {
 	static void RegisterFunction(BuiltinFunctions &set);
-	static string_t SubstringScalarFunction(Vector &result, string_t input, int64_t offset, int64_t length);
+	static string_t SubstringUnicode(Vector &result, string_t input, int64_t offset, int64_t length);
+	static string_t SubstringGrapheme(Vector &result, string_t input, int64_t offset, int64_t length);
 };
 
 struct PrintfFun {
@@ -104993,7 +105005,7 @@ list_entry_t SliceValue(Vector &result, list_entry_t input, int64_t begin, int64
 template <>
 string_t SliceValue(Vector &result, string_t input, int32_t begin, int32_t end) {
 	// one-based - zero has strange semantics
-	return SubstringFun::SubstringScalarFunction(result, input, begin + 1, end - begin);
+	return SubstringFun::SubstringUnicode(result, input, begin + 1, end - begin);
 }
 
 template <typename INPUT_TYPE, typename INDEX_TYPE>
@@ -106341,7 +106353,7 @@ static void ExecuteListExtract(Vector &result, Vector &list, Vector &offsets, co
 static void ExecuteStringExtract(Vector &result, Vector &input_vector, Vector &subscript_vector, const idx_t count) {
 	BinaryExecutor::Execute<string_t, int64_t, string_t>(
 	    input_vector, subscript_vector, result, count, [&](string_t input_string, int64_t subscript) {
-		    return SubstringFun::SubstringScalarFunction(result, input_string, subscript, 1);
+		    return SubstringFun::SubstringUnicode(result, input_string, subscript, 1);
 	    });
 }
 
@@ -113771,54 +113783,84 @@ void JaroWinklerFun::RegisterFunction(BuiltinFunctions &set) {
 
 namespace duckdb {
 
+struct LeftRightUnicode {
+	template <class TA, class TR>
+	static inline TR Operation(TA input) {
+		return LengthFun::Length<TA, TR>(input);
+	}
+
+	static string_t Substring(Vector &result, string_t input, int64_t offset, int64_t length) {
+		return SubstringFun::SubstringUnicode(result, input, offset, length);
+	}
+};
+
+struct LeftRightGrapheme {
+	template <class TA, class TR>
+	static inline TR Operation(TA input) {
+		return LengthFun::GraphemeCount<TA, TR>(input);
+	}
+
+	static string_t Substring(Vector &result, string_t input, int64_t offset, int64_t length) {
+		return SubstringFun::SubstringGrapheme(result, input, offset, length);
+	}
+};
+
+template <class OP>
 static string_t LeftScalarFunction(Vector &result, const string_t str, int64_t pos) {
 	if (pos >= 0) {
-		return SubstringFun::SubstringScalarFunction(result, str, 1, pos);
+		return OP::Substring(result, str, 1, pos);
 	}
 
-	int64_t num_characters = LengthFun::Length<string_t, int64_t>(str);
+	int64_t num_characters = OP::template Operation<string_t, int64_t>(str);
 	pos = MaxValue<int64_t>(0, num_characters + pos);
-	return SubstringFun::SubstringScalarFunction(result, str, 1, pos);
+	return OP::Substring(result, str, 1, pos);
 }
 
+template <class OP>
 static void LeftFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	auto &str_vec = args.data[0];
 	auto &pos_vec = args.data[1];
 
 	BinaryExecutor::Execute<string_t, int64_t, string_t>(
 	    str_vec, pos_vec, result, args.size(),
-	    [&](string_t str, int64_t pos) { return LeftScalarFunction(result, str, pos); });
+	    [&](string_t str, int64_t pos) { return LeftScalarFunction<OP>(result, str, pos); });
 }
 
 void LeftFun::RegisterFunction(BuiltinFunctions &set) {
-	set.AddFunction(
-	    ScalarFunction("left", {LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR, LeftFunction));
+	set.AddFunction(ScalarFunction("left", {LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR,
+	                               LeftFunction<LeftRightUnicode>));
+	set.AddFunction(ScalarFunction("left_grapheme", {LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR,
+	                               LeftFunction<LeftRightGrapheme>));
 }
 
+template <class OP>
 static string_t RightScalarFunction(Vector &result, const string_t str, int64_t pos) {
-	int64_t num_characters = LengthFun::Length<string_t, int64_t>(str);
+	int64_t num_characters = OP::template Operation<string_t, int64_t>(str);
 	if (pos >= 0) {
 		int64_t len = MinValue<int64_t>(num_characters, pos);
 		int64_t start = num_characters - len + 1;
-		return SubstringFun::SubstringScalarFunction(result, str, start, len);
+		return OP::Substring(result, str, start, len);
 	}
 
 	int64_t len = num_characters - MinValue<int64_t>(num_characters, -pos);
 	int64_t start = num_characters - len + 1;
-	return SubstringFun::SubstringScalarFunction(result, str, start, len);
+	return OP::Substring(result, str, start, len);
 }
 
+template <class OP>
 static void RightFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	auto &str_vec = args.data[0];
 	auto &pos_vec = args.data[1];
 	BinaryExecutor::Execute<string_t, int64_t, string_t>(
 	    str_vec, pos_vec, result, args.size(),
-	    [&](string_t str, int64_t pos) { return RightScalarFunction(result, str, pos); });
+	    [&](string_t str, int64_t pos) { return RightScalarFunction<OP>(result, str, pos); });
 }
 
 void RightFun::RegisterFunction(BuiltinFunctions &set) {
-	set.AddFunction(
-	    ScalarFunction("right", {LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR, RightFunction));
+	set.AddFunction(ScalarFunction("right", {LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR,
+	                               RightFunction<LeftRightUnicode>));
+	set.AddFunction(ScalarFunction("right_grapheme", {LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR,
+	                               RightFunction<LeftRightGrapheme>));
 }
 
 } // namespace duckdb
@@ -113833,7 +113875,7 @@ void RightFun::RegisterFunction(BuiltinFunctions &set) {
 
 namespace duckdb {
 
-// length returns the size in characters
+// length returns the number of unicode codepoints
 struct StringLengthOperator {
 	template <class TA, class TR>
 	static inline TR Operation(TA input) {
@@ -113841,6 +113883,13 @@ struct StringLengthOperator {
 	}
 };
 
+struct GraphemeCountOperator {
+	template <class TA, class TR>
+	static inline TR Operation(TA input) {
+		return LengthFun::GraphemeCount<TA, TR>(input);
+	}
+};
+
 struct ArrayLengthOperator {
 	template <class TA, class TR>
 	static inline TR Operation(TA input) {
@@ -113911,6 +113960,12 @@ void LengthFun::RegisterFunction(BuiltinFunctions &set) {
 	length.name = "len";
 	set.AddFunction(length);
 
+	ScalarFunctionSet length_grapheme("length_grapheme");
+	length_grapheme.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::BIGINT,
+	                                           ScalarFunction::UnaryFunction<string_t, int64_t, GraphemeCountOperator>,
+	                                           nullptr, nullptr, LengthPropagateStats));
+	set.AddFunction(length_grapheme);
+
 	ScalarFunctionSet array_length("array_length");
 	array_length.AddFunction(array_length_unary);
 	array_length.AddFunction(ScalarFunction(
@@ -116186,7 +116241,92 @@ string_t SubstringASCII(Vector &result, string_t input, int64_t offset, int64_t
 	return SubstringSlice(result, input_data, start, end - start);
 }
 
-string_t SubstringFun::SubstringScalarFunction(Vector &result, string_t input, int64_t offset, int64_t length) {
+string_t SubstringFun::SubstringUnicode(Vector &result, string_t input, int64_t offset, int64_t length) {
+	auto input_data = input.GetDataUnsafe();
+	auto input_size = input.GetSize();
+
+	if (length == 0) {
+		return SubstringEmptyString(result);
+	}
+	// first figure out which direction we need to scan
+	idx_t start_pos;
+	idx_t end_pos;
+	if (offset < 0) {
+		start_pos = 0;
+		end_pos = DConstants::INVALID_INDEX;
+
+		// negative offset: scan backwards
+		int64_t start, end;
+
+		// we express start and end as unicode codepoints from the back
+		offset--;
+		if (length < 0) {
+			// negative length
+			start = -offset - length;
+			end = -offset;
+		} else {
+			// positive length
+			start = -offset;
+			end = -offset - length;
+		}
+		if (end <= 0) {
+			end_pos = input_size;
+		}
+		int64_t current_character = 0;
+		for (idx_t i = input_size; i > 0; i--) {
+			if (LengthFun::IsCharacter(input_data[i - 1])) {
+				current_character++;
+				if (current_character == start) {
+					start_pos = i;
+					break;
+				} else if (current_character == end) {
+					end_pos = i;
+				}
+			}
+		}
+		if (end_pos == DConstants::INVALID_INDEX) {
+			return SubstringEmptyString(result);
+		}
+	} else {
+		start_pos = DConstants::INVALID_INDEX;
+		end_pos = input_size;
+
+		// positive offset: scan forwards
+		int64_t start, end;
+
+		// we express start and end as unicode codepoints from the front
+		if (length < 0) {
+			// negative length
+			start = MaxValue<int64_t>(0, offset + length - 1);
+			end = offset - 1;
+		} else {
+			// positive length
+			start = MaxValue<int64_t>(0, offset - 1);
+			end = offset + length - 1;
+		}
+
+		int64_t current_character = 0;
+		for (idx_t i = 0; i < input_size; i++) {
+			if (LengthFun::IsCharacter(input_data[i])) {
+				if (current_character == start) {
+					start_pos = i;
+				} else if (current_character == end) {
+					end_pos = i;
+					break;
+				}
+				current_character++;
+			}
+		}
+		if (start_pos == DConstants::INVALID_INDEX || end == 0 || end <= start) {
+			return SubstringEmptyString(result);
+		}
+	}
+	D_ASSERT(end_pos >= start_pos);
+	// after we have found these, we can slice the substring
+	return SubstringSlice(result, input_data, start_pos, end_pos - start_pos);
+}
+
+string_t SubstringFun::SubstringGrapheme(Vector &result, string_t input, int64_t offset, int64_t length) {
 	auto input_data = input.GetDataUnsafe();
 	auto input_size = input.GetSize();
 
@@ -116247,6 +116387,19 @@ string_t SubstringFun::SubstringScalarFunction(Vector &result, string_t input, i
 	return SubstringSlice(result, input_data, start_pos, end_pos - start_pos);
 }
 
+struct SubstringUnicodeOp {
+	static string_t Substring(Vector &result, string_t input, int64_t offset, int64_t length) {
+		return SubstringFun::SubstringUnicode(result, input, offset, length);
+	}
+};
+
+struct SubstringGraphemeOp {
+	static string_t Substring(Vector &result, string_t input, int64_t offset, int64_t length) {
+		return SubstringFun::SubstringGrapheme(result, input, offset, length);
+	}
+};
+
+template <class OP>
 static void SubstringFunction(DataChunk &args, ExpressionState &state, Vector &result) {
 	auto &input_vector = args.data[0];
 	auto &offset_vector = args.data[1];
@@ -116256,13 +116409,12 @@ static void SubstringFunction(DataChunk &args, ExpressionState &state, Vector &r
 		TernaryExecutor::Execute<string_t, int64_t, int64_t, string_t>(
 		    input_vector, offset_vector, length_vector, result, args.size(),
 		    [&](string_t input_string, int64_t offset, int64_t length) {
-			    return SubstringFun::SubstringScalarFunction(result, input_string, offset, length);
+			    return OP::Substring(result, input_string, offset, length);
 		    });
 	} else {
 		BinaryExecutor::Execute<string_t, int64_t, string_t>(
 		    input_vector, offset_vector, result, args.size(), [&](string_t input_string, int64_t offset) {
-			    return SubstringFun::SubstringScalarFunction(result, input_string, offset,
-			                                                 NumericLimits<int64_t>::Maximum() - offset);
+			    return OP::Substring(result, input_string, offset, NumericLimits<int64_t>::Maximum() - offset);
 		    });
 	}
 }
@@ -116304,13 +116456,23 @@ static unique_ptr<BaseStatistics> SubstringPropagateStats(ClientContext &context
 void SubstringFun::RegisterFunction(BuiltinFunctions &set) {
 	ScalarFunctionSet substr("substring");
 	substr.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT},
-	                                  LogicalType::VARCHAR, SubstringFunction, nullptr, nullptr,
+	                                  LogicalType::VARCHAR, SubstringFunction<SubstringUnicodeOp>, nullptr, nullptr,
 	                                  SubstringPropagateStats));
 	substr.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR,
-	                                  SubstringFunction, nullptr, nullptr, SubstringPropagateStats));
+	                                  SubstringFunction<SubstringUnicodeOp>, nullptr, nullptr,
+	                                  SubstringPropagateStats));
 	set.AddFunction(substr);
 	substr.name = "substr";
 	set.AddFunction(substr);
+
+	ScalarFunctionSet substr_grapheme("substring_grapheme");
+	substr_grapheme.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT, LogicalType::BIGINT},
+	                                           LogicalType::VARCHAR, SubstringFunction<SubstringGraphemeOp>, nullptr,
+	                                           nullptr, SubstringPropagateStats));
+	substr_grapheme.AddFunction(ScalarFunction({LogicalType::VARCHAR, LogicalType::BIGINT}, LogicalType::VARCHAR,
+	                                           SubstringFunction<SubstringGraphemeOp>, nullptr, nullptr,
+	                                           SubstringPropagateStats));
+	set.AddFunction(substr_grapheme);
 }
 
 } // namespace duckdb