duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/core_functions/scalar/string/url_encode.cpp

@@ -0,0 +1,49 @@
+#include "duckdb/common/vector_operations/unary_executor.hpp"
+#include "duckdb/core_functions/scalar/string_functions.hpp"
+#include "duckdb/common/string_util.hpp"
+
+namespace duckdb {
+
+struct URLEncodeOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		auto input_str = input.GetData();
+		auto input_size = input.GetSize();
+		idx_t result_length = StringUtil::URLEncodeSize(input_str, input_size);
+		auto result_str = StringVector::EmptyString(result, result_length);
+		StringUtil::URLEncodeBuffer(input_str, input_size, result_str.GetDataWriteable());
+		result_str.Finalize();
+		return result_str;
+	}
+};
+
+static void URLEncodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	UnaryExecutor::ExecuteString<string_t, string_t, URLEncodeOperator>(args.data[0], result, args.size());
+}
+
+ScalarFunction UrlEncodeFun::GetFunction() {
+	return ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, URLEncodeFunction);
+}
+
+struct URLDecodeOperator {
+	template <class INPUT_TYPE, class RESULT_TYPE>
+	static RESULT_TYPE Operation(INPUT_TYPE input, Vector &result) {
+		auto input_str = input.GetData();
+		auto input_size = input.GetSize();
+		idx_t result_length = StringUtil::URLDecodeSize(input_str, input_size);
+		auto result_str = StringVector::EmptyString(result, result_length);
+		StringUtil::URLDecodeBuffer(input_str, input_size, result_str.GetDataWriteable());
+		result_str.Finalize();
+		return result_str;
+	}
+};
+
+static void URLDecodeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
+	UnaryExecutor::ExecuteString<string_t, string_t, URLDecodeOperator>(args.data[0], result, args.size());
+}
+
+ScalarFunction UrlDecodeFun::GetFunction() {
+	return ScalarFunction({LogicalType::VARCHAR}, LogicalType::VARCHAR, URLDecodeFunction);
+}
+
+} // namespace duckdb

package/src/duckdb/src/core_functions/scalar/struct/struct_pack.cpp

@@ -18,7 +18,7 @@ static void StructPackFunction(DataChunk &args, ExpressionState &state, Vector &
 #endif
 	bool all_const = true;
 	auto &child_entries = StructVector::GetEntries(result);
-	for (size_t i = 0; i < args.ColumnCount(); i++) {
+	for (idx_t i = 0; i < args.ColumnCount(); i++) {
 		if (args.data[i].GetVectorType() != VectorType::CONSTANT_VECTOR) {
 			all_const = false;
 		}
@@ -26,7 +26,6 @@ static void StructPackFunction(DataChunk &args, ExpressionState &state, Vector &
 		child_entries[i]->Reference(args.data[i]);
 	}
 	result.SetVectorType(all_const ? VectorType::CONSTANT_VECTOR : VectorType::FLAT_VECTOR);
-
 	result.Verify(args.size());
 }
 

package/src/duckdb/src/core_functions/scalar/union/union_extract.cpp

@@ -45,7 +45,9 @@ static unique_ptr<FunctionData> UnionExtractBind(ClientContext &context, ScalarF
 	if (arguments[0]->return_type.id() == LogicalTypeId::UNKNOWN) {
 		throw ParameterNotResolvedException();
 	}
-	D_ASSERT(LogicalTypeId::UNION == arguments[0]->return_type.id());
+	if (arguments[0]->return_type.id() != LogicalTypeId::UNION) {
+		throw BinderException("union_extract can only take a union parameter");
+	}
 	idx_t union_member_count = UnionType::GetMemberCount(arguments[0]->return_type);
 	if (union_member_count == 0) {
 		throw InternalException("Can't extract something from an empty union");
@@ -88,7 +90,7 @@ static unique_ptr<FunctionData> UnionExtractBind(ClientContext &context, ScalarF
 		for (idx_t i = 0; i < union_member_count; i++) {
 			candidates.push_back(UnionType::GetMemberName(arguments[0]->return_type, i));
 		}
-		auto closest_settings = StringUtil::TopNLevenshtein(candidates, key);
+		auto closest_settings = StringUtil::TopNJaroWinkler(candidates, key);
 		auto message = StringUtil::CandidatesMessage(closest_settings, "Candidate Entries");
 		throw BinderException("Could not find key \"%s\" in union\n%s", key, message);
 	}

package/src/duckdb/src/execution/adaptive_filter.cpp

@@ -1,12 +1,12 @@
 #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 #include "duckdb/execution/adaptive_filter.hpp"
 #include "duckdb/planner/table_filter.hpp"
+#include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/common/vector.hpp"
 
 namespace duckdb {
 
-AdaptiveFilter::AdaptiveFilter(const Expression &expr)
-    : iteration_count(0), observe_interval(10), execute_interval(20), warmup(true) {
+AdaptiveFilter::AdaptiveFilter(const Expression &expr) : observe_interval(10), execute_interval(20), warmup(true) {
 	auto &conj_expr = expr.Cast<BoundConjunctionExpression>();
 	D_ASSERT(conj_expr.children.size() > 1);
 	for (idx_t idx = 0; idx < conj_expr.children.size(); idx++) {
@@ -18,15 +18,34 @@ AdaptiveFilter::AdaptiveFilter(const Expression &expr)
 	right_random_border = 100 * (conj_expr.children.size() - 1);
 }
 
-AdaptiveFilter::AdaptiveFilter(TableFilterSet *table_filters)
-    : iteration_count(0), observe_interval(10), execute_interval(20), warmup(true) {
-	for (auto &table_filter : table_filters->filters) {
-		permutation.push_back(table_filter.first);
+AdaptiveFilter::AdaptiveFilter(const TableFilterSet &table_filters)
+    : observe_interval(10), execute_interval(20), warmup(true) {
+	for (idx_t idx = 0; idx < table_filters.filters.size(); idx++) {
+		permutation.push_back(idx);
 		swap_likeliness.push_back(100);
 	}
 	swap_likeliness.pop_back();
-	right_random_border = 100 * (table_filters->filters.size() - 1);
+	right_random_border = 100 * (table_filters.filters.size() - 1);
+}
+
+AdaptiveFilterState AdaptiveFilter::BeginFilter() const {
+	if (permutation.size() <= 1) {
+		return AdaptiveFilterState();
+	}
+	AdaptiveFilterState state;
+	state.start_time = high_resolution_clock::now();
+	return state;
 }
+
+void AdaptiveFilter::EndFilter(AdaptiveFilterState state) {
+	if (permutation.size() <= 1) {
+		// nothing to permute
+		return;
+	}
+	auto end_time = high_resolution_clock::now();
+	AdaptRuntimeStatistics(duration_cast<duration<double>>(end_time - state.start_time).count());
+}
+
 void AdaptiveFilter::AdaptRuntimeStatistics(double duration) {
 	iteration_count++;
 	runtime_sum += duration;
@@ -35,7 +54,7 @@ void AdaptiveFilter::AdaptRuntimeStatistics(double duration) {
 		// the last swap was observed
 		if (observe && iteration_count == observe_interval) {
 			// keep swap if runtime decreased, else reverse swap
-			if (prev_mean - (runtime_sum / iteration_count) <= 0) {
+			if (prev_mean - (runtime_sum / static_cast<double>(iteration_count)) <= 0) {
 				// reverse swap because runtime didn't decrease
 				std::swap(permutation[swap_idx], permutation[swap_idx + 1]);
 
@@ -54,11 +73,11 @@ void AdaptiveFilter::AdaptRuntimeStatistics(double duration) {
 			runtime_sum = 0.0;
 		} else if (!observe && iteration_count == execute_interval) {
 			// save old mean to evaluate swap
-			prev_mean = runtime_sum / iteration_count;
+			prev_mean = runtime_sum / static_cast<double>(iteration_count);
 
 			// get swap index and swap likeliness
-			std::uniform_int_distribution<int> distribution(1, NumericCast<int>(right_random_border)); // a <= i <= b
-			auto random_number = UnsafeNumericCast<idx_t>(distribution(generator) - 1);
+			// a <= i <= b
+			auto random_number = generator.NextRandomInteger(1, NumericCast<uint32_t>(right_random_border));
 
 			swap_idx = random_number / 100;                    // index to be swapped
 			idx_t likeliness = random_number - 100 * swap_idx; // random number between [0, 100)

package/src/duckdb/src/execution/aggregate_hashtable.cpp

@@ -9,6 +9,7 @@
 #include "duckdb/common/types/row/tuple_data_iterator.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/execution/ht_entry.hpp"
 #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
 
 namespace duckdb {
@@ -122,7 +123,7 @@ idx_t GroupedAggregateHashTable::InitialCapacity() {
 
 idx_t GroupedAggregateHashTable::GetCapacityForCount(idx_t count) {
 	count = MaxValue<idx_t>(InitialCapacity(), count);
-	return NextPowerOfTwo(NumericCast<uint64_t>(static_cast<double>(count) * LOAD_FACTOR));
+	return NextPowerOfTwo(LossyNumericCast<uint64_t>(static_cast<double>(count) * LOAD_FACTOR));
 }
 
 idx_t GroupedAggregateHashTable::Capacity() const {
@@ -130,7 +131,7 @@ idx_t GroupedAggregateHashTable::Capacity() const {
 }
 
 idx_t GroupedAggregateHashTable::ResizeThreshold() const {
-	return NumericCast<idx_t>(static_cast<double>(Capacity()) / LOAD_FACTOR);
+	return LossyNumericCast<idx_t>(static_cast<double>(Capacity()) / LOAD_FACTOR);
 }
 
 idx_t GroupedAggregateHashTable::ApplyBitMask(hash_t hash) const {
@@ -146,7 +147,7 @@ void GroupedAggregateHashTable::Verify() {
 			continue;
 		}
 		auto hash = Load<hash_t>(entry.GetPointer() + hash_offset);
-		D_ASSERT(entry.GetSalt() == aggr_ht_entry_t::ExtractSalt(hash));
+		D_ASSERT(entry.GetSalt() == ht_entry_t::ExtractSalt(hash));
 		total_count++;
 	}
 	D_ASSERT(total_count == Count());
@@ -154,7 +155,7 @@ void GroupedAggregateHashTable::Verify() {
 }
 
 void GroupedAggregateHashTable::ClearPointerTable() {
-	std::fill_n(entries, capacity, aggr_ht_entry_t(0));
+	std::fill_n(entries, capacity, ht_entry_t::GetEmptyEntry());
 }
 
 void GroupedAggregateHashTable::ResetCount() {
@@ -173,8 +174,8 @@ void GroupedAggregateHashTable::Resize(idx_t size) {
 	}
 
 	capacity = size;
-	hash_map = buffer_manager.GetBufferAllocator().Allocate(capacity * sizeof(aggr_ht_entry_t));
-	entries = reinterpret_cast<aggr_ht_entry_t *>(hash_map.get());
+	hash_map = buffer_manager.GetBufferAllocator().Allocate(capacity * sizeof(ht_entry_t));
+	entries = reinterpret_cast<ht_entry_t *>(hash_map.get());
 	ClearPointerTable();
 	bitmask = capacity - 1;
 
@@ -201,7 +202,7 @@ void GroupedAggregateHashTable::Resize(idx_t size) {
 					}
 					auto &entry = entries[entry_idx];
 					D_ASSERT(!entry.IsOccupied());
-					entry.SetSalt(aggr_ht_entry_t::ExtractSalt(hash));
+					entry.SetSalt(ht_entry_t::ExtractSalt(hash));
 					entry.SetPointer(row_location);
 					D_ASSERT(entry.IsOccupied());
 				}
@@ -333,7 +334,7 @@ idx_t GroupedAggregateHashTable::FindOrCreateGroupsInternal(DataChunk &groups, V
 		const auto &hash = hashes[r];
 		ht_offsets[r] = ApplyBitMask(hash);
 		D_ASSERT(ht_offsets[r] == hash % capacity);
-		hash_salts[r] = aggr_ht_entry_t::ExtractSalt(hash);
+		hash_salts[r] = ht_entry_t::ExtractSalt(hash);
 	}
 
 	// we start out with all entries [0, 1, 2, ..., groups.size()]
@@ -354,7 +355,7 @@ idx_t GroupedAggregateHashTable::FindOrCreateGroupsInternal(DataChunk &groups, V
 	auto &chunk_state = state.append_state.chunk_state;
 	TupleDataCollection::ToUnifiedFormat(chunk_state, state.group_chunk);
 	if (!state.group_data) {
-		state.group_data = make_unsafe_uniq_array<UnifiedVectorFormat>(state.group_chunk.ColumnCount());
+		state.group_data = make_unsafe_uniq_array_uninitialized<UnifiedVectorFormat>(state.group_chunk.ColumnCount());
 	}
 	TupleDataCollection::GetVectorData(chunk_state, state.group_data.get());
 
@@ -380,13 +381,9 @@ idx_t GroupedAggregateHashTable::FindOrCreateGroupsInternal(DataChunk &groups, V
 						// Same salt, compare group keys
 						state.group_compare_vector.set_index(need_compare_count++, index);
 						break;
-					} else {
-						// Different salts, move to next entry (linear probing)
-						if (++ht_offset >= capacity) {
-							ht_offset = 0;
-						}
-						continue;
 					}
+					// Different salts, move to next entry (linear probing)
+					IncrementAndWrap(ht_offset, bitmask);
 				} else { // Cell is unoccupied, let's claim it
 					// Set salt (also marks as occupied)
 					entry.SetSalt(salt);
@@ -439,9 +436,7 @@ idx_t GroupedAggregateHashTable::FindOrCreateGroupsInternal(DataChunk &groups, V
 		for (idx_t i = 0; i < no_match_count; i++) {
 			const auto index = state.no_match_vector.get_index(i);
 			auto &ht_offset = ht_offsets[index];
-			if (++ht_offset >= capacity) {
-				ht_offset = 0;
-			}
+			IncrementAndWrap(ht_offset, bitmask);
 		}
 		sel_vector = &state.no_match_vector;
 		remaining_entries = no_match_count;

package/src/duckdb/src/execution/expression_executor/execute_conjunction.cpp

@@ -2,7 +2,6 @@
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/planner/expression/bound_conjunction_expression.hpp"
 #include "duckdb/execution/adaptive_filter.hpp"
-#include "duckdb/common/chrono.hpp"
 
 #include <random>
 
@@ -60,8 +59,7 @@ idx_t ExpressionExecutor::Select(const BoundConjunctionExpression &expr, Express
 
 	if (expr.type == ExpressionType::CONJUNCTION_AND) {
 		// get runtime statistics
-		auto start_time = high_resolution_clock::now();
-
+		auto filter_state = state.adaptive_filter->BeginFilter();
 		const SelectionVector *current_sel = sel;
 		idx_t current_count = count;
 		idx_t false_count = 0;
@@ -96,14 +94,12 @@ idx_t ExpressionExecutor::Select(const BoundConjunctionExpression &expr, Express
 				current_sel = true_sel;
 			}
 		}
-
 		// adapt runtime statistics
-		auto end_time = high_resolution_clock::now();
-		state.adaptive_filter->AdaptRuntimeStatistics(duration_cast<duration<double>>(end_time - start_time).count());
+		state.adaptive_filter->EndFilter(filter_state);
 		return current_count;
 	} else {
 		// get runtime statistics
-		auto start_time = high_resolution_clock::now();
+		auto filter_state = state.adaptive_filter->BeginFilter();
 
 		const SelectionVector *current_sel = sel;
 		idx_t current_count = count;
@@ -135,8 +131,7 @@ idx_t ExpressionExecutor::Select(const BoundConjunctionExpression &expr, Express
 		}
 
 		// adapt runtime statistics
-		auto end_time = high_resolution_clock::now();
-		state.adaptive_filter->AdaptRuntimeStatistics(duration_cast<duration<double>>(end_time - start_time).count());
+		state.adaptive_filter->EndFilter(filter_state);
 		return result_count;
 	}
 }

package/src/duckdb/src/execution/expression_executor.cpp

@@ -249,7 +249,7 @@ static inline idx_t DefaultSelectLoop(const SelectionVector *bsel, const uint8_t
 	for (idx_t i = 0; i < count; i++) {
 		auto bidx = bsel->get_index(i);
 		auto result_idx = sel->get_index(i);
-		if (bdata[bidx] > 0 && (NO_NULL || mask.RowIsValid(bidx))) {
+		if ((NO_NULL || mask.RowIsValid(bidx)) && bdata[bidx] > 0) {
 			if (HAS_TRUE_SEL) {
 				true_sel->set_index(true_count++, result_idx);
 			}