duckdb 0.7.2-dev0.0 → 0.7.2-dev1034.0

package/src/duckdb/src/function/scalar/string/hex.cpp

@@ -0,0 +1,201 @@
+#include "duckdb/common/exception.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types/blob.hpp"
+#include "duckdb/common/vector_operations/unary_executor.hpp"
+#include "duckdb/common/vector_operations/vector_operations.hpp"
+#include "duckdb/function/scalar/string_functions.hpp"
+
+namespace duckdb {
+
+struct HexStrOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		auto data = input.GetDataUnsafe();
+		auto size = input.GetSize();
+
+		// Allocate empty space
+		auto target = StringVector::EmptyString(result, size * 2);
+		auto output = target.GetDataWriteable();
+
+		for (idx_t i = 0; i < size; ++i) {
+			*output = Blob::HEX_TABLE[(data[i] >> 4) & 0x0F];
+			output++;
+			*output = Blob::HEX_TABLE[data[i] & 0x0F];
+			output++;
+		}
+
+		target.Finalize();
+		return target;
+	}
+};
+
+struct FromHexOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		auto data = input.GetDataUnsafe();
+		auto size = input.GetSize();
+
+		if (size > NumericLimits<uint32_t>::Maximum()) {
+			throw InvalidInputException("Hexadecimal input length larger than 2^32 are not supported");
+		}
+
+		D_ASSERT(size <= NumericLimits<uint32_t>::Maximum());
+		auto buffer_size = (size + 1) / 2;
+
+		// Allocate empty space
+		auto target = StringVector::EmptyString(result, buffer_size);
+		auto output = target.GetDataWriteable();
+
+		// Treated as a single byte
+		idx_t i = 0;
+		if (size % 2 != 0) {
+			*output = StringUtil::GetHexValue(data[i]);
+			i++;
+			output++;
+		}
+
+		for (; i < size; i += 2) {
+			uint8_t major = StringUtil::GetHexValue(data[i]);
+			uint8_t minor = StringUtil::GetHexValue(data[i + 1]);
+			*output = (major << 4) | minor;
+			output++;
+		}
+
+		target.Finalize();
+		return target;
+	}
+};
+
+struct HexIntegralOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		// Sufficient space for maximum length
+		char buffer[sizeof(INPUT_TYPE) * 2];
+		char *ptr = buffer;
+		idx_t buffer_size = 0;
+
+		bool seen_non_zero = false;
+		for (idx_t offset = sizeof(INPUT_TYPE) * 8; offset >= 4; offset -= 4) {
+			uint8_t byte = (input >> (offset - 4)) & 0x0F;
+			if (byte == 0 && !seen_non_zero && offset > 4) {
+				continue;
+			}
+			seen_non_zero = true;
+			*ptr = Blob::HEX_TABLE[byte];
+			ptr++;
+			buffer_size++;
+		}
+
+		// Allocate empty space
+		auto target = StringVector::EmptyString(result, buffer_size);
+		auto output = target.GetDataWriteable();
+		memcpy(output, buffer, buffer_size);
+
+		target.Finalize();
+		return target;
+	}
+};
+
+struct HexHugeIntOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		char buffer[sizeof(INPUT_TYPE) * 2];
+		char *ptr = buffer;
+		idx_t buffer_size = 0;
+
+		uint64_t lower = input.lower;
+		int64_t upper = input.upper;
+
+		bool seen_non_zero = false;
+		for (idx_t offset = 64; offset >= 4; offset -= 4) {
+			uint8_t byte = (upper >> (offset - 4)) & 0x0F;
+
+			if (byte == 0 && !seen_non_zero) {
+				continue;
+			}
+			seen_non_zero = true;
+			*ptr = Blob::HEX_TABLE[byte];
+			ptr++;
+			buffer_size++;
+		}
+
+		for (idx_t offset = 64; offset >= 4; offset -= 4) {
+			uint8_t byte = (lower >> (offset - 4)) & 0x0F;
+
+			// at least one byte space
+			if (byte == 0 && !seen_non_zero && offset > 4) {
+				continue;
+			}
+			seen_non_zero = true;
+			*ptr = Blob::HEX_TABLE[byte];
+			ptr++;
+			buffer_size++;
+		}
+
+		// Allocate empty space
+		auto target = StringVector::EmptyString(result, buffer_size);
+		auto output = target.GetDataWriteable();
+		memcpy(output, buffer, buffer_size);
+
+		target.Finalize();
+		return target;
+	}
+};
+
+static void ToHexFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 1);
+	auto &input = args.data[0];
+	idx_t count = args.size();
+
+	switch (input.GetType().InternalType()) {
+	case PhysicalType::VARCHAR:
+		UnaryExecutor::ExecuteString<string_t, string_t, HexStrOperator>(input, result, count);
+		break;
+	case PhysicalType::INT64:
+		UnaryExecutor::ExecuteString<int64_t, string_t, HexIntegralOperator>(input, result, count);
+		break;
+	case PhysicalType::INT128:
+		UnaryExecutor::ExecuteString<hugeint_t, string_t, HexHugeIntOperator>(input, result, count);
+		break;
+	case PhysicalType::UINT64:
+		UnaryExecutor::ExecuteString<uint64_t, string_t, HexIntegralOperator>(input, result, count);
+		break;
+	default:
+		throw NotImplementedException("Specifier type not implemented");
+	}
+}
+
+static void FromHexFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	D_ASSERT(args.ColumnCount() == 1);
+	D_ASSERT(args.data[0].GetType().InternalType() == PhysicalType::VARCHAR);
+	auto &input = args.data[0];
+	idx_t count = args.size();
+
+	UnaryExecutor::ExecuteString<string_t, string_t, FromHexOperator>(input, result, count);
+}
+
+void HexFun::RegisterFunction(BuiltinFunctions &set) {
+	ScalarFunctionSet to_hex("to_hex");
+	ScalarFunctionSet from_hex("from_hex");
+
+	to_hex.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, ToHexFunction));
+
+	to_hex.AddFunction(ScalarFunction({LogicalType::BIGINT}, LogicalType::VARCHAR, ToHexFunction));
+
+	to_hex.AddFunction(ScalarFunction({LogicalType::UBIGINT}, LogicalType::VARCHAR, ToHexFunction));
+
+	to_hex.AddFunction(ScalarFunction({LogicalType::HUGEINT}, LogicalType::VARCHAR, ToHexFunction));
+
+	from_hex.AddFunction(ScalarFunction({LogicalType::VARCHAR}, LogicalType::BLOB, FromHexFunction));
+
+	set.AddFunction(to_hex);
+	set.AddFunction(from_hex);
+
+	// mysql
+	to_hex.name = "hex";
+	from_hex.name = "unhex";
+	set.AddFunction(to_hex);
+	set.AddFunction(from_hex);
+}
+
+} // namespace duckdb

package/src/duckdb/src/function/scalar/string/instr.cpp

@@ -3,7 +3,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
-#include "duckdb/storage/statistics/string_statistics.hpp"
+
 #include "utf8proc.hpp"
 
 namespace duckdb {
@@ -42,12 +42,8 @@ static unique_ptr<BaseStatistics> InStrPropagateStats(ClientContext &context, Fu
 	auto &expr = input.expr;
 	D_ASSERT(child_stats.size() == 2);
 	// can only propagate stats if the children have stats
-	if (!child_stats[0]) {
-		return nullptr;
-	}
 	// for strpos, we only care if the FIRST string has unicode or not
-	auto &sstats = (StringStatistics &)*child_stats[0];
-	if (!sstats.has_unicode) {
+	if (!StringStats::CanContainUnicode(child_stats[0])) {
 		expr.function.function = ScalarFunction::BinaryFunction<string_t, string_t, int64_t, InstrAsciiOperator>;
 	}
 	return nullptr;

package/src/duckdb/src/function/scalar/string/length.cpp

@@ -4,7 +4,7 @@
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
-#include "duckdb/storage/statistics/string_statistics.hpp"
+
 #include "duckdb/planner/expression/bound_parameter_expression.hpp"
 #include "utf8proc.hpp"
 
@@ -78,11 +78,7 @@ static unique_ptr<BaseStatistics> LengthPropagateStats(ClientContext &context, F
 	auto &expr = input.expr;
 	D_ASSERT(child_stats.size() == 1);
 	// can only propagate stats if the children have stats
-	if (!child_stats[0]) {
-		return nullptr;
-	}
-	auto &sstats = (StringStatistics &)*child_stats[0];
-	if (!sstats.has_unicode) {
+	if (!StringStats::CanContainUnicode(child_stats[0])) {
 		expr.function.function = ScalarFunction::UnaryFunction<string_t, int64_t, StrLenOperator>;
 	}
 	return nullptr;

package/src/duckdb/src/function/scalar/string/like.cpp

@@ -2,7 +2,7 @@
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/scalar/string_functions.hpp"
 #include "duckdb/planner/expression/bound_function_expression.hpp"
-#include "duckdb/storage/statistics/string_statistics.hpp"
+
 #include "duckdb/execution/expression_executor.hpp"
 
 namespace duckdb {
@@ -482,11 +482,7 @@ static unique_ptr<BaseStatistics> ILikePropagateStats(ClientContext &context, Fu
 	auto &expr = input.expr;
 	D_ASSERT(child_stats.size() >= 1);
 	// can only propagate stats if the children have stats
-	if (!child_stats[0]) {
-		return nullptr;
-	}
-	auto &sstats = (StringStatistics &)*child_stats[0];
-	if (!sstats.has_unicode) {
+	if (!StringStats::CanContainUnicode(child_stats[0])) {
 		expr.function.function = ScalarFunction::BinaryFunction<string_t, string_t, bool, ASCII_OP>;
 	}
 	return nullptr;

package/src/duckdb/src/function/scalar/string/regexp/regexp_extract_all.cpp

@@ -0,0 +1,243 @@
+#include "duckdb/function/scalar/regexp.hpp"
+#include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/planner/expression/bound_function_expression.hpp"
+#include "duckdb/function/scalar/string_functions.hpp"
+#include "re2/re2.h"
+
+namespace duckdb {
+
+using regexp_util::CreateStringPiece;
+using regexp_util::Extract;
+using regexp_util::ParseRegexOptions;
+using regexp_util::TryParseConstantPattern;
+
+unique_ptr<FunctionLocalState>
+RegexpExtractAll::InitLocalState(ExpressionState &state, const BoundFunctionExpression &expr, FunctionData *bind_data) {
+	auto &info = (RegexpBaseBindData &)*bind_data;
+	if (info.constant_pattern) {
+		return make_unique<RegexLocalState>(info, true);
+	}
+	return nullptr;
+}
+
+// Forwards startpos automatically
+bool ExtractAll(duckdb_re2::StringPiece &input, duckdb_re2::RE2 &pattern, idx_t *startpos,
+                duckdb_re2::StringPiece *groups, int ngroups) {
+
+	D_ASSERT(pattern.ok());
+	D_ASSERT(pattern.NumberOfCapturingGroups() == ngroups);
+
+	if (!pattern.Match(input, *startpos, input.size(), pattern.Anchored(), groups, ngroups + 1)) {
+		return false;
+	}
+	idx_t consumed = static_cast<size_t>(groups[0].end() - (input.begin() + *startpos));
+	if (!consumed) {
+		// Empty match found, have to manually forward the input
+		// to avoid an infinite loop
+		// FIXME: support unicode characters
+		consumed++;
+		while (*startpos + consumed < input.length() && !LengthFun::IsCharacter(input[*startpos + consumed])) {
+			consumed++;
+		}
+	}
+	*startpos += consumed;
+	return true;
+}
+
+void ExtractSingleTuple(const string_t &string, duckdb_re2::RE2 &pattern, int32_t group, RegexStringPieceArgs &args,
+                        Vector &result, idx_t row) {
+	auto input = CreateStringPiece(string);
+
+	auto &child_vector = ListVector::GetEntry(result);
+	auto list_content = FlatVector::GetData<string_t>(child_vector);
+	auto &child_validity = FlatVector::Validity(child_vector);
+
+	auto current_list_size = ListVector::GetListSize(result);
+	auto current_list_capacity = ListVector::GetListCapacity(result);
+
+	auto result_data = FlatVector::GetData<list_entry_t>(result);
+	auto &list_entry = result_data[row];
+	list_entry.offset = current_list_size;
+
+	if (group < 0) {
+		list_entry.length = 0;
+		return;
+	}
+	// If the requested group index is out of bounds
+	// we want to throw only if there is a match
+	bool throw_on_group_found = (idx_t)group > args.size;
+
+	idx_t startpos = 0;
+	for (idx_t iteration = 0; ExtractAll(input, pattern, &startpos, args.group_buffer, args.size); iteration++) {
+		if (!iteration && throw_on_group_found) {
+			throw InvalidInputException("Pattern has %d groups. Cannot access group %d", args.size, group);
+		}
+
+		// Make sure we have enough room for the new entries
+		if (current_list_size + 1 >= current_list_capacity) {
+			ListVector::Reserve(result, current_list_capacity * 2);
+			current_list_capacity = ListVector::GetListCapacity(result);
+			list_content = FlatVector::GetData<string_t>(child_vector);
+		}
+
+		// Write the captured groups into the list-child vector
+		auto &match_group = args.group_buffer[group];
+
+		idx_t child_idx = current_list_size;
+		if (match_group.empty()) {
+			// This group was not matched
+			list_content[child_idx] = string_t(string.GetDataUnsafe(), 0);
+			if (match_group.begin() == nullptr) {
+				// This group is optional
+				child_validity.SetInvalid(child_idx);
+			}
+		} else {
+			// Every group is a substring of the original, we can find out the offset using the pointer
+			// the 'match_group' address is guaranteed to be bigger than that of the source
+			D_ASSERT((const char *)match_group.begin() >= string.GetDataUnsafe());
+			idx_t offset = match_group.begin() - string.GetDataUnsafe();
+			list_content[child_idx] = string_t(string.GetDataUnsafe() + offset, match_group.size());
+		}
+		current_list_size++;
+		if (startpos > input.size()) {
+			// Empty match found at the end of the string
+			break;
+		}
+	}
+	list_entry.length = current_list_size - list_entry.offset;
+	ListVector::SetListSize(result, current_list_size);
+}
+
+int32_t GetGroupIndex(DataChunk &args, idx_t row, int32_t &result) {
+	if (args.ColumnCount() < 3) {
+		result = 0;
+		return true;
+	}
+	UnifiedVectorFormat format;
+	args.data[2].ToUnifiedFormat(args.size(), format);
+	idx_t index = format.sel->get_index(row);
+	if (!format.validity.RowIsValid(index)) {
+		return false;
+	}
+	result = ((int32_t *)format.data)[index];
+	return true;
+}
+
+duckdb_re2::RE2 &GetPattern(const RegexpBaseBindData &info, ExpressionState &state,
+                            unique_ptr<duckdb_re2::RE2> &pattern_p) {
+	if (info.constant_pattern) {
+		auto &lstate = (RegexLocalState &)*ExecuteFunctionState::GetFunctionState(state);
+		return lstate.constant_pattern;
+	}
+	D_ASSERT(pattern_p);
+	return *pattern_p;
+}
+
+RegexStringPieceArgs &GetGroupsBuffer(const RegexpBaseBindData &info, ExpressionState &state,
+                                      unique_ptr<RegexStringPieceArgs> &groups_p) {
+	if (info.constant_pattern) {
+		auto &lstate = (RegexLocalState &)*ExecuteFunctionState::GetFunctionState(state);
+		return lstate.group_buffer;
+	}
+	D_ASSERT(groups_p);
+	return *groups_p;
+}
+
+void RegexpExtractAll::Execute(DataChunk &args, ExpressionState &state, Vector &result) {
+	auto &func_expr = (BoundFunctionExpression &)state.expr;
+	const auto &info = (RegexpBaseBindData &)*func_expr.bind_info;
+
+	auto &strings = args.data[0];
+	auto &patterns = args.data[1];
+	D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
+	auto &output_child = ListVector::GetEntry(result);
+
+	UnifiedVectorFormat strings_data;
+	strings.ToUnifiedFormat(args.size(), strings_data);
+
+	UnifiedVectorFormat pattern_data;
+	patterns.ToUnifiedFormat(args.size(), pattern_data);
+
+	ListVector::Reserve(result, STANDARD_VECTOR_SIZE);
+	// Reference the 'strings' StringBuffer, because we won't need to allocate new data
+	// for the result, all returned strings are substrings of the originals
+	output_child.SetAuxiliary(strings.GetAuxiliary());
+
+	// Avoid doing extra work if all the inputs are constant
+	idx_t tuple_count = args.AllConstant() ? 1 : args.size();
+
+	unique_ptr<RegexStringPieceArgs> non_const_args;
+	unique_ptr<duckdb_re2::RE2> stored_re;
+	if (!info.constant_pattern) {
+		non_const_args = make_unique<RegexStringPieceArgs>();
+	} else {
+		// Verify that the constant pattern is valid
+		auto &re = GetPattern(info, state, stored_re);
+		auto group_count_p = re.NumberOfCapturingGroups();
+		if (group_count_p == -1) {
+			throw InvalidInputException("Pattern failed to parse, error: '%s'", re.error());
+		}
+	}
+
+	for (idx_t row = 0; row < tuple_count; row++) {
+		bool pattern_valid = true;
+		if (!info.constant_pattern) {
+			// Check if the pattern is NULL or not,
+			// and compile the pattern if it's not constant
+			auto pattern_idx = pattern_data.sel->get_index(row);
+			if (!pattern_data.validity.RowIsValid(pattern_idx)) {
+				pattern_valid = false;
+			} else {
+				auto &pattern_p = ((string_t *)pattern_data.data)[pattern_idx];
+				auto pattern_strpiece = CreateStringPiece(pattern_p);
+				stored_re = make_unique<duckdb_re2::RE2>(pattern_strpiece, info.options);
+
+				// Increase the size of the args buffer if needed
+				auto group_count_p = stored_re->NumberOfCapturingGroups();
+				if (group_count_p == -1) {
+					throw InvalidInputException("Pattern failed to parse, error: '%s'", stored_re->error());
+				}
+				non_const_args->SetSize(group_count_p);
+			}
+		}
+
+		auto string_idx = strings_data.sel->get_index(row);
+		int32_t group_index;
+		if (!pattern_valid || !strings_data.validity.RowIsValid(string_idx) || !GetGroupIndex(args, row, group_index)) {
+			// If something is NULL, the result is NULL
+			// FIXME: do we even need 'SPECIAL_HANDLING'?
+			auto result_data = FlatVector::GetData<list_entry_t>(result);
+			auto &result_validity = FlatVector::Validity(result);
+			result_data[row].length = 0;
+			result_data[row].offset = ListVector::GetListSize(result);
+			result_validity.SetInvalid(row);
+			continue;
+		}
+
+		auto &re = GetPattern(info, state, stored_re);
+		auto &groups = GetGroupsBuffer(info, state, non_const_args);
+		auto &string = ((string_t *)strings_data.data)[string_idx];
+		ExtractSingleTuple(string, re, group_index, groups, result, row);
+	}
+
+	if (args.AllConstant()) {
+		result.SetVectorType(VectorType::CONSTANT_VECTOR);
+	}
+}
+
+unique_ptr<FunctionData> RegexpExtractAll::Bind(ClientContext &context, ScalarFunction &bound_function,
+                                                vector<unique_ptr<Expression>> &arguments) {
+	D_ASSERT(arguments.size() >= 2);
+
+	duckdb_re2::RE2::Options options;
+
+	string constant_string;
+	bool constant_pattern = TryParseConstantPattern(context, *arguments[1], constant_string);
+
+	if (arguments.size() >= 4) {
+		ParseRegexOptions(context, *arguments[3], options);
+	}
+	return make_unique<RegexpExtractBindData>(options, std::move(constant_string), constant_pattern, "");
+}
+
+} // namespace duckdb

package/src/duckdb/src/function/scalar/string/regexp/regexp_util.cpp

@@ -0,0 +1,79 @@
+#include "duckdb/function/scalar/regexp.hpp"
+#include "duckdb/execution/expression_executor.hpp"
+
+namespace duckdb {
+
+namespace regexp_util {
+
+bool TryParseConstantPattern(ClientContext &context, Expression &expr, string &constant_string) {
+	if (!expr.IsFoldable()) {
+		return false;
+	}
+	Value pattern_str = ExpressionExecutor::EvaluateScalar(context, expr);
+	if (!pattern_str.IsNull() && pattern_str.type().id() == LogicalTypeId::VARCHAR) {
+		constant_string = StringValue::Get(pattern_str);
+		return true;
+	}
+	return false;
+}
+
+void ParseRegexOptions(const string &options, duckdb_re2::RE2::Options &result, bool *global_replace) {
+	for (idx_t i = 0; i < options.size(); i++) {
+		switch (options[i]) {
+		case 'c':
+			// case-sensitive matching
+			result.set_case_sensitive(true);
+			break;
+		case 'i':
+			// case-insensitive matching
+			result.set_case_sensitive(false);
+			break;
+		case 'l':
+			// literal matching
+			result.set_literal(true);
+			break;
+		case 'm':
+		case 'n':
+		case 'p':
+			// newline-sensitive matching
+			result.set_dot_nl(false);
+			break;
+		case 's':
+			// non-newline-sensitive matching
+			result.set_dot_nl(true);
+			break;
+		case 'g':
+			// global replace, only available for regexp_replace
+			if (global_replace) {
+				*global_replace = true;
+			} else {
+				throw InvalidInputException("Option 'g' (global replace) is only valid for regexp_replace");
+			}
+			break;
+		case ' ':
+		case '\t':
+		case '\n':
+			// ignore whitespace
+			break;
+		default:
+			throw InvalidInputException("Unrecognized Regex option %c", options[i]);
+		}
+	}
+}
+
+void ParseRegexOptions(ClientContext &context, Expression &expr, RE2::Options &target, bool *global_replace) {
+	if (expr.HasParameter()) {
+		throw ParameterNotResolvedException();
+	}
+	if (!expr.IsFoldable()) {
+		throw InvalidInputException("Regex options field must be a constant");
+	}
+	Value options_str = ExpressionExecutor::EvaluateScalar(context, expr);
+	if (!options_str.IsNull() && options_str.type().id() == LogicalTypeId::VARCHAR) {
+		ParseRegexOptions(StringValue::Get(options_str), target, global_replace);
+	}
+}
+
+} // namespace regexp_util
+
+} // namespace duckdb