duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/common/progress_bar/terminal_progress_bar_display.cpp

@@ -14,7 +14,61 @@ int32_t TerminalProgressBarDisplay::NormalizePercentage(double percentage) {
 	return int32_t(percentage);
 }
 
-void TerminalProgressBarDisplay::PrintProgressInternal(int32_t percentage) {
+static string FormatETA(double seconds, bool elapsed = false) {
+	// for terminal rendering purposes, we need to make sure the length is always the same
+	// we pad the end with spaces if that is not the case
+	// the maximum length here is "(~10.35 minutes remaining)" (26 bytes)
+	// always pad to this amount
+	static constexpr idx_t RENDER_SIZE = 26;
+
+	if (seconds < 0 || seconds == 2147483647) {
+		// Invalid or unknown ETA, skip rendering estimate
+		return string(RENDER_SIZE, ' ');
+	}
+
+	if (!elapsed && seconds > 3600 * 99) {
+		// estimate larger than 99 hours remaining, treat this as invalid/unknown ETA
+		return string(RENDER_SIZE, ' ');
+	}
+
+	// Round to nearest centisecond as integer
+	auto total_centiseconds = static_cast<uint64_t>(std::llround(seconds * 100.0));
+
+	// Split into seconds and centiseconds
+	uint64_t total_seconds = total_centiseconds / 100;
+	uint32_t centiseconds = static_cast<uint32_t>(total_centiseconds % 100);
+
+	// Break down total_seconds
+	uint64_t hours = total_seconds / 3600;
+	uint32_t minutes = static_cast<uint32_t>((total_seconds % 3600) / 60);
+	uint32_t secs = static_cast<uint32_t>(total_seconds % 60);
+	string result;
+	result = "(";
+	if (!elapsed) {
+		if (hours == 0 && minutes == 0 && secs == 0) {
+			result += StringUtil::Format("<1 second");
+		} else if (hours == 0 && minutes == 0) {
+			result += StringUtil::Format("~%u second%s", secs, secs > 1 ? "s" : "");
+		} else if (hours == 0) {
+			auto minute_fraction = static_cast<uint32_t>(static_cast<double>(secs) / 60.0 * 10);
+			result += StringUtil::Format("~%u.%u minutes", minutes, minute_fraction);
+		} else {
+			auto hour_fraction = static_cast<uint32_t>(static_cast<double>(minutes) / 60.0 * 10);
+			result += StringUtil::Format("~%llu.%u hours", hours, hour_fraction);
+		}
+		result += " remaining";
+	} else {
+		result += StringUtil::Format("%02llu:%02u:%02u.%02llu", hours, minutes, secs, centiseconds);
+		result += " elapsed";
+	}
+	result += ")";
+	if (result.size() < RENDER_SIZE) {
+		result += string(RENDER_SIZE - result.size(), ' ');
+	}
+	return result;
+}
+
+void TerminalProgressBarDisplay::PrintProgressInternal(int32_t percentage, double seconds, bool finished) {
 	string result;
 	// we divide the number of blocks by the percentage
 	// 0%   = 0
@@ -52,22 +106,42 @@ void TerminalProgressBarDisplay::PrintProgressInternal(int32_t percentage) {
 	}
 	result += PROGRESS_END;
 	result += " ";
+	result += FormatETA(seconds, finished);
 
 	Printer::RawPrint(OutputStream::STREAM_STDOUT, result);
 }
 
 void TerminalProgressBarDisplay::Update(double percentage) {
+	const double current_time = GetElapsedDuration();
+
+	// Filters go from 0 to 1, percentage is from 0-100
+	const double filter_percentage = percentage / 100.0;
+
+	ukf.Update(filter_percentage, current_time);
+
+	double estimated_seconds_remaining;
+	//  If the query is mostly completed, there can be oscillation of estimated
+	//  time to completion since the progress updates seem sparse near the very
+	//  end of the query, so clamp time remaining to not oscillate with estimates
+	//  that are unlikely to be correct.
+	if (filter_percentage > 0.99) {
+		estimated_seconds_remaining = 0.5;
+	} else {
+		estimated_seconds_remaining = std::min(ukf.GetEstimatedRemainingSeconds(), 2147483647.0);
+	}
 	auto percentage_int = NormalizePercentage(percentage);
-	if (percentage_int == rendered_percentage) {
-		return;
+
+	TerminalProgressBarDisplayedProgressInfo updated_progress_info = {(idx_t)percentage_int,
+	                                                                  (idx_t)estimated_seconds_remaining};
+	if (displayed_progress_info != updated_progress_info) {
+		PrintProgressInternal(percentage_int, estimated_seconds_remaining);
+		Printer::Flush(OutputStream::STREAM_STDOUT);
+		displayed_progress_info = updated_progress_info;
 	}
-	PrintProgressInternal(percentage_int);
-	Printer::Flush(OutputStream::STREAM_STDOUT);
-	rendered_percentage = percentage_int;
 }
 
 void TerminalProgressBarDisplay::Finish() {
-	PrintProgressInternal(100);
+	PrintProgressInternal(100, GetElapsedDuration(), true);
 	Printer::RawPrint(OutputStream::STREAM_STDOUT, "\n");
 	Printer::Flush(OutputStream::STREAM_STDOUT);
 }

package/src/duckdb/src/common/progress_bar/unscented_kalman_filter.cpp

@@ -0,0 +1,288 @@
+#include "duckdb/common/progress_bar/unscented_kalman_filter.hpp"
+#include "duckdb.hpp"
+namespace duckdb {
+
+UnscentedKalmanFilter::UnscentedKalmanFilter()
+    : x(STATE_DIM, 0.0), P(STATE_DIM, vector<double>(STATE_DIM, 0.0)), Q(STATE_DIM, vector<double>(STATE_DIM, 0.0)),
+      R(OBS_DIM, vector<double>(OBS_DIM, 0.0)), last_time(0.0), initialized(false), last_progress(-1.0),
+      scale_factor(1.0) {
+
+	// Calculate UKF parameters
+	lambda = ALPHA * ALPHA * (STATE_DIM + KAPPA) - STATE_DIM;
+
+	// Initialize weights
+	wm.resize(SIGMA_POINTS);
+	wc.resize(SIGMA_POINTS);
+
+	wm[0] = lambda / (STATE_DIM + lambda);
+	wc[0] = lambda / (STATE_DIM + lambda) + (1 - ALPHA * ALPHA + BETA);
+
+	for (size_t i = 1; i < SIGMA_POINTS; i++) {
+		wm[i] = wc[i] = 0.5 / (STATE_DIM + lambda);
+	}
+
+	// Initialize covariance matrices
+	P[0][0] = 0.165;  // progress variance
+	P[1][1] = 0.0098; // velocity variance
+
+	Q[0][0] = 0.01; // process noise for progress
+	Q[1][1] = 0.0;  // process noise for velocity
+
+	R[0][0] = 0.000077; // measurement noise for progress
+}
+
+void UnscentedKalmanFilter::Update(double progress, double time) {
+	progress *= scale_factor;
+	if (!initialized) {
+		Initialize(progress, time);
+		return;
+	}
+	Predict(time);
+	if (last_progress != progress) {
+		UpdateInternal(progress);
+		last_progress = progress;
+	}
+}
+
+void UnscentedKalmanFilter::Initialize(double initial_progress, double current_time) {
+	// If the initial progress is zero we can't yet initalize, since the filter
+	// wont' have a reasonable guess about query velocity, just wait until some
+	// progress has been made.
+	if (initial_progress == 0.0 || current_time == 0.0) {
+		return;
+	}
+	// If the initial progress value is very small, it needs to be scaled up so
+	// its the same relative magnitude of updates that was used to determine
+	// the P, Q, and R parameters of the Kalman filter.
+	//
+	// The settings for the Kalman filter make assumptions around the magnitude of
+	// measurement noise.  If the progress updates are very small, the updates
+	// could be considered noise by the Kalman filter.
+	scale_factor = std::max(1.0, 0.1 / initial_progress);
+	initial_progress *= scale_factor;
+	x[0] = initial_progress;
+	x[1] = initial_progress / current_time; // initial velocity guess
+	last_time = current_time;
+	last_progress = initial_progress;
+	initialized = true;
+}
+
+// Matrix operations
+vector<vector<double>> UnscentedKalmanFilter::MatrixSqrt(const vector<vector<double>> &mat) {
+	size_t n = mat.size();
+	vector<vector<double>> L(n, vector<double>(n, 0.0));
+
+	// Cholesky decomposition
+	for (size_t i = 0; i < n; i++) {
+		for (size_t j = 0; j <= i; j++) {
+			if (i == j) {
+				double sum = 0;
+				for (size_t k = 0; k < j; k++) {
+					sum += L[j][k] * L[j][k];
+				}
+				L[j][j] = std::sqrt(mat[j][j] - sum);
+			} else {
+				double sum = 0;
+				for (size_t k = 0; k < j; k++) {
+					sum += L[i][k] * L[j][k];
+				}
+				L[i][j] = (mat[i][j] - sum) / L[j][j];
+			}
+		}
+	}
+	return L;
+}
+
+// Generate sigma points
+vector<vector<double>> UnscentedKalmanFilter::GenerateSigmaPoints() {
+	vector<vector<double>> sigma_points(SIGMA_POINTS, vector<double>(STATE_DIM));
+
+	// Calculate sqrt((n + lambda) * P)
+	auto scaled_P = P;
+	double scale = STATE_DIM + lambda;
+	for (size_t i = 0; i < STATE_DIM; i++) {
+		for (size_t j = 0; j < STATE_DIM; j++) {
+			scaled_P[i][j] *= scale;
+		}
+	}
+
+	auto sqrt_P = MatrixSqrt(scaled_P);
+
+	// First sigma point is the mean
+	sigma_points[0] = x;
+
+	// Remaining sigma points
+	for (size_t i = 0; i < STATE_DIM; i++) {
+		for (size_t j = 0; j < STATE_DIM; j++) {
+			sigma_points[i + 1][j] = x[j] + sqrt_P[j][i];
+			sigma_points[i + 1 + STATE_DIM][j] = x[j] - sqrt_P[j][i];
+		}
+	}
+
+	return sigma_points;
+}
+
+// State transition function
+vector<double> UnscentedKalmanFilter::StateTransition(const vector<double> &state, double dt) {
+	vector<double> new_state(STATE_DIM);
+	new_state[0] = state[0] + state[1] * dt; // progress += velocity * dt
+	new_state[1] = state[1];                 // velocity remains constant
+
+	// Clamp progress to [0, 1]
+	new_state[0] = std::max(0.0, std::min(1.0, new_state[0]));
+
+	return new_state;
+}
+
+// Measurement function (identity for progress)
+vector<double> UnscentedKalmanFilter::MeasurementFunction(const vector<double> &state) {
+	return {state[0]};
+}
+
+void UnscentedKalmanFilter::Predict(double current_time) {
+	D_ASSERT(initialized);
+
+	double dt = current_time - last_time;
+	last_time = current_time;
+
+	if (dt <= 0) {
+		return;
+	}
+
+	// Generate sigma points
+	auto sigma_points = GenerateSigmaPoints();
+
+	// Propagate sigma points through state transition
+	vector<vector<double>> sigma_points_pred(SIGMA_POINTS, vector<double>(STATE_DIM));
+	for (size_t i = 0; i < SIGMA_POINTS; i++) {
+		sigma_points_pred[i] = StateTransition(sigma_points[i], dt);
+	}
+
+	// Predict state mean
+	std::fill(x.begin(), x.end(), 0.0);
+	for (size_t i = 0; i < SIGMA_POINTS; i++) {
+		for (size_t j = 0; j < STATE_DIM; j++) {
+			x[j] += wm[i] * sigma_points_pred[i][j];
+		}
+	}
+
+	// Predict covariance
+	for (size_t i = 0; i < STATE_DIM; i++) {
+		std::fill(P[i].begin(), P[i].end(), 0.0);
+	}
+
+	for (size_t i = 0; i < SIGMA_POINTS; i++) {
+		for (size_t j = 0; j < STATE_DIM; j++) {
+			for (size_t k = 0; k < STATE_DIM; k++) {
+				P[j][k] += wc[i] * (sigma_points_pred[i][j] - x[j]) * (sigma_points_pred[i][k] - x[k]);
+			}
+		}
+	}
+
+	// Add process noise
+	for (size_t i = 0; i < STATE_DIM; i++) {
+		for (size_t j = 0; j < STATE_DIM; j++) {
+			P[i][j] += Q[i][j];
+		}
+	}
+}
+
+void UnscentedKalmanFilter::UpdateInternal(double measured_progress) {
+	D_ASSERT(initialized);
+	// Generate sigma points
+	auto sigma_points = GenerateSigmaPoints();
+
+	// Transform sigma points through measurement function
+	vector<vector<double>> Z_sigma(SIGMA_POINTS, vector<double>(OBS_DIM));
+	for (size_t i = 0; i < SIGMA_POINTS; i++) {
+		Z_sigma[i] = MeasurementFunction(sigma_points[i]);
+	}
+
+	// Predict measurement mean
+	vector<double> z_pred(OBS_DIM, 0.0);
+	for (size_t i = 0; i < SIGMA_POINTS; i++) {
+		for (size_t j = 0; j < OBS_DIM; j++) {
+			z_pred[j] += wm[i] * Z_sigma[i][j];
+		}
+	}
+
+	// Innovation covariance
+	vector<vector<double>> S(OBS_DIM, vector<double>(OBS_DIM, 0.0));
+	for (size_t i = 0; i < SIGMA_POINTS; i++) {
+		for (size_t j = 0; j < OBS_DIM; j++) {
+			for (size_t k = 0; k < OBS_DIM; k++) {
+				S[j][k] += wc[i] * (Z_sigma[i][j] - z_pred[j]) * (Z_sigma[i][k] - z_pred[k]);
+			}
+		}
+	}
+
+	// Add measurement noise
+	for (size_t i = 0; i < OBS_DIM; i++) {
+		for (size_t j = 0; j < OBS_DIM; j++) {
+			S[i][j] += R[i][j];
+		}
+	}
+
+	// Cross correlation
+	vector<vector<double>> T(STATE_DIM, vector<double>(OBS_DIM, 0.0));
+	for (size_t i = 0; i < SIGMA_POINTS; i++) {
+		for (size_t j = 0; j < STATE_DIM; j++) {
+			for (size_t k = 0; k < OBS_DIM; k++) {
+				T[j][k] += wc[i] * (sigma_points[i][j] - x[j]) * (Z_sigma[i][k] - z_pred[k]);
+			}
+		}
+	}
+
+	// Kalman gain (simplified for 1D measurement)
+	vector<vector<double>> K(STATE_DIM, vector<double>(OBS_DIM));
+	double S_inv = 1.0 / S[0][0];
+	for (size_t i = 0; i < STATE_DIM; i++) {
+		K[i][0] = T[i][0] * S_inv;
+	}
+
+	// Update state
+	vector<double> innovation = {measured_progress - z_pred[0]};
+	for (size_t i = 0; i < STATE_DIM; i++) {
+		x[i] += K[i][0] * innovation[0];
+	}
+
+	// Update covariance
+	for (size_t i = 0; i < STATE_DIM; i++) {
+		for (size_t j = 0; j < STATE_DIM; j++) {
+			P[i][j] -= K[i][0] * S[0][0] * K[j][0];
+		}
+	}
+
+	// Ensure progress stays in bounds
+	x[0] = std::max(0.0, std::min(1.0, x[0]));
+}
+
+double UnscentedKalmanFilter::GetProgress() const {
+	return x[0];
+}
+
+double UnscentedKalmanFilter::GetVelocity() const {
+	return x[1];
+}
+
+double UnscentedKalmanFilter::GetEstimatedRemainingSeconds() const {
+	if (!initialized) {
+		return 2147483647.0;
+	}
+	if (x[1] <= 0) {
+		// velocity is negative or zero - we estimate this will never finish
+		return NumericLimits<double>::Maximum();
+	}
+	double remaining_progress = (1.0 * scale_factor) - x[0];
+	return std::max(remaining_progress / x[1], 0.0);
+}
+
+double UnscentedKalmanFilter::GetProgressVariance() const {
+	return P[0][0];
+}
+
+double UnscentedKalmanFilter::GetVelocityVariance() const {
+	return P[1][1];
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/radix_partitioning.cpp

@@ -51,9 +51,9 @@ RETURN_TYPE RadixBitsSwitch(const idx_t radix_bits, ARGS &&... args) {
 	case 10:
 		return OP::template Operation<10>(std::forward<ARGS>(args)...);
 	case 11:
-		return OP::template Operation<10>(std::forward<ARGS>(args)...);
+		return OP::template Operation<11>(std::forward<ARGS>(args)...);
 	case 12:
-		return OP::template Operation<10>(std::forward<ARGS>(args)...);
+		return OP::template Operation<12>(std::forward<ARGS>(args)...);
 	default:
 		throw InternalException(
 		    "radix_bits higher than RadixPartitioning::MAX_RADIX_BITS encountered in RadixBitsSwitch");

package/src/duckdb/src/common/row_operations/row_aggregate.cpp

@@ -1,10 +1,3 @@
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/common/types/row_operations/row_aggregate.cpp
-//
-//
-//===----------------------------------------------------------------------===//
 #include "duckdb/catalog/catalog_entry/aggregate_function_catalog_entry.hpp"
 #include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/common/types/row/tuple_data_layout.hpp"
@@ -22,10 +15,14 @@ void RowOperations::InitializeStates(TupleDataLayout &layout, Vector &addresses,
 	auto aggr_idx = layout.ColumnCount();
 
 	for (const auto &aggr : layout.GetAggregates()) {
-		for (idx_t i = 0; i < count; ++i) {
-			auto row_idx = sel.get_index(i);
-			auto row = pointers[row_idx];
-			aggr.function.initialize(aggr.function, row + offsets[aggr_idx]);
+		if (sel.IsSet()) {
+			for (idx_t i = 0; i < count; ++i) {
+				aggr.function.initialize(aggr.function, pointers[sel.get_index_unsafe(i)] + offsets[aggr_idx]);
+			}
+		} else {
+			for (idx_t i = 0; i < count; ++i) {
+				aggr.function.initialize(aggr.function, pointers[i] + offsets[aggr_idx]);
+			}
 		}
 		++aggr_idx;
 	}

package/src/duckdb/src/common/row_operations/row_external.cpp

@@ -1,10 +1,3 @@
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/common/types/row_operations/row_external.cpp
-//
-//
-//===----------------------------------------------------------------------===//
 #include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/common/types/row/row_layout.hpp"
 

package/src/duckdb/src/common/row_operations/row_gather.cpp

@@ -1,8 +1,3 @@
-//===--------------------------------------------------------------------===//
-// row_gather.cpp
-// Description: This file contains the implementation of the gather operators
-//===--------------------------------------------------------------------===//
-
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/operator/constant_operators.hpp"
 #include "duckdb/common/row_operations/row_operations.hpp"

package/src/duckdb/src/common/row_operations/row_matcher.cpp

@@ -8,7 +8,11 @@ namespace duckdb {
 
 using ValidityBytes = TupleDataLayout::ValidityBytes;
 
-template <bool NO_MATCH_SEL, class T, class OP, bool LHS_ALL_VALID>
+#ifdef DUCKDB_SMALLER_BINARY
+template <bool NO_MATCH_SEL, class T, class OP>
+#else
+template <bool NO_MATCH_SEL, class T, class OP, bool LHS_ALL_VALID, bool RHS_ALL_VALID>
+#endif
 static idx_t TemplatedMatchLoop(const TupleDataVectorFormat &lhs_format, SelectionVector &sel, const idx_t count,
                                 const TupleDataLayout &rhs_layout, Vector &rhs_row_locations, const idx_t col_idx,
                                 SelectionVector *no_match_sel, idx_t &no_match_count) {
@@ -19,26 +23,33 @@ static idx_t TemplatedMatchLoop(const TupleDataVectorFormat &lhs_format, Selecti
 	const auto lhs_data = UnifiedVectorFormat::GetData<T>(lhs_format.unified);
 	const auto &lhs_validity = lhs_format.unified.validity;
 
+#ifdef DUCKDB_SMALLER_BINARY
+	const auto LHS_ALL_VALID = lhs_validity.AllValid();
+	const auto RHS_ALL_VALID = rhs_layout.AllValid();
+#endif
+
 	// RHS
 	const auto rhs_locations = FlatVector::GetData<data_ptr_t>(rhs_row_locations);
 	const auto rhs_offset_in_row = rhs_layout.GetOffsets()[col_idx];
 	idx_t entry_idx;
 	idx_t idx_in_entry;
 	ValidityBytes::GetEntryIndex(col_idx, entry_idx, idx_in_entry);
+	const auto rhs_column_count = rhs_layout.ColumnCount();
 
 	idx_t match_count = 0;
 	for (idx_t i = 0; i < count; i++) {
 		const auto idx = sel.get_index(i);
 
 		const auto lhs_idx = lhs_sel.get_index(idx);
-		const auto lhs_null = LHS_ALL_VALID ? false : !lhs_validity.RowIsValid(lhs_idx);
-
 		const auto &rhs_location = rhs_locations[idx];
-		const ValidityBytes rhs_mask(rhs_location, rhs_layout.ColumnCount());
-		const auto rhs_null = !rhs_mask.RowIsValid(rhs_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry);
 
-		if (COMPARISON_OP::template Operation<T>(lhs_data[lhs_idx], Load<T>(rhs_location + rhs_offset_in_row), lhs_null,
-		                                         rhs_null)) {
+		if (COMPARISON_OP::template Operation<T>(
+		        lhs_data[lhs_idx], Load<T>(rhs_location + rhs_offset_in_row),
+		        LHS_ALL_VALID ? false : !lhs_validity.RowIsValidUnsafe(lhs_idx),
+		        RHS_ALL_VALID ? false
+		                      : !ValidityBytes::RowIsValid(
+		                            ValidityBytes(rhs_location, rhs_column_count).GetValidityEntryUnsafe(entry_idx),
+		                            idx_in_entry))) {
 			sel.set_index(match_count++, idx);
 		} else if (NO_MATCH_SEL) {
 			no_match_sel->set_index(no_match_count++, idx);
@@ -51,13 +62,28 @@ template <bool NO_MATCH_SEL, class T, class OP>
 static idx_t TemplatedMatch(Vector &, const TupleDataVectorFormat &lhs_format, SelectionVector &sel, const idx_t count,
                             const TupleDataLayout &rhs_layout, Vector &rhs_row_locations, const idx_t col_idx,
                             const vector<MatchFunction> &, SelectionVector *no_match_sel, idx_t &no_match_count) {
+#ifdef DUCKDB_SMALLER_BINARY
+	return TemplatedMatchLoop<NO_MATCH_SEL, T, OP>(lhs_format, sel, count, rhs_layout, rhs_row_locations, col_idx,
+	                                               no_match_sel, no_match_count);
+#else
 	if (lhs_format.unified.validity.AllValid()) {
-		return TemplatedMatchLoop<NO_MATCH_SEL, T, OP, true>(lhs_format, sel, count, rhs_layout, rhs_row_locations,
-		                                                     col_idx, no_match_sel, no_match_count);
+		if (rhs_layout.AllValid()) {
+			return TemplatedMatchLoop<NO_MATCH_SEL, T, OP, true, true>(
+			    lhs_format, sel, count, rhs_layout, rhs_row_locations, col_idx, no_match_sel, no_match_count);
+		} else {
+			return TemplatedMatchLoop<NO_MATCH_SEL, T, OP, true, false>(
+			    lhs_format, sel, count, rhs_layout, rhs_row_locations, col_idx, no_match_sel, no_match_count);
+		}
 	} else {
-		return TemplatedMatchLoop<NO_MATCH_SEL, T, OP, false>(lhs_format, sel, count, rhs_layout, rhs_row_locations,
-		                                                      col_idx, no_match_sel, no_match_count);
+		if (rhs_layout.AllValid()) {
+			return TemplatedMatchLoop<NO_MATCH_SEL, T, OP, false, true>(
+			    lhs_format, sel, count, rhs_layout, rhs_row_locations, col_idx, no_match_sel, no_match_count);
+		} else {
+			return TemplatedMatchLoop<NO_MATCH_SEL, T, OP, false, false>(
+			    lhs_format, sel, count, rhs_layout, rhs_row_locations, col_idx, no_match_sel, no_match_count);
+		}
 	}
+#endif
 }
 
 template <bool NO_MATCH_SEL, class OP>
@@ -85,8 +111,10 @@ static idx_t StructMatchEquality(Vector &lhs_vector, const TupleDataVectorFormat
 		const auto lhs_null = lhs_validity.AllValid() ? false : !lhs_validity.RowIsValid(lhs_idx);
 
 		const auto &rhs_location = rhs_locations[idx];
-		const ValidityBytes rhs_mask(rhs_location, rhs_layout.ColumnCount());
-		const auto rhs_null = !rhs_mask.RowIsValid(rhs_mask.GetValidityEntryUnsafe(entry_idx), idx_in_entry);
+		const auto rhs_null =
+		    !rhs_layout.AllValid() &&
+		    !ValidityBytes::RowIsValid(
+		        ValidityBytes(rhs_location, rhs_layout.ColumnCount()).GetValidityEntryUnsafe(entry_idx), idx_in_entry);
 
 		// For structs there is no value to compare, here we match NULLs and let recursion do the rest
 		// So we use the comparison only if rhs or LHS is NULL and COMPARE_NULL is true
@@ -204,15 +232,18 @@ static idx_t GenericNestedMatch(Vector &lhs_vector, const TupleDataVectorFormat
 	return SelectComparison<OP>(sliced, key, sel, count, &sel, nullptr);
 }
 
-void RowMatcher::Initialize(const bool no_match_sel, const TupleDataLayout &layout, const Predicates &predicates) {
-	match_functions.reserve(predicates.size());
-	for (idx_t col_idx = 0; col_idx < predicates.size(); col_idx++) {
-		match_functions.push_back(GetMatchFunction(no_match_sel, layout.GetTypes()[col_idx], predicates[col_idx]));
+void RowMatcher::Initialize(const bool no_match_sel, const TupleDataLayout &layout, const Predicates &predicates,
+                            vector<column_t> columns_p) {
+	if (columns_p.empty()) {
+		// Assume all columns
+		columns_p.reserve(predicates.size());
+		for (column_t col_idx = 0; col_idx < predicates.size(); col_idx++) {
+			columns_p.emplace_back(col_idx);
+		}
 	}
-}
 
-void RowMatcher::Initialize(const bool no_match_sel, const TupleDataLayout &layout, const Predicates &predicates,
-                            vector<column_t> &columns) {
+	rhs_layout = &layout;
+	columns = columns_p;
 
 	// The columns must have the same size as the predicates vector
 	D_ASSERT(columns.size() == predicates.size());
@@ -222,27 +253,14 @@ void RowMatcher::Initialize(const bool no_match_sel, const TupleDataLayout &layo
 
 	match_functions.reserve(predicates.size());
 	for (idx_t idx = 0; idx < predicates.size(); idx++) {
-		column_t col_idx = columns[idx];
+		const column_t col_idx = columns[idx];
 		match_functions.push_back(GetMatchFunction(no_match_sel, layout.GetTypes()[col_idx], predicates[idx]));
+		rhs_types.push_back(layout.GetTypes()[col_idx]);
 	}
 }
 
 idx_t RowMatcher::Match(DataChunk &lhs, const vector<TupleDataVectorFormat> &lhs_formats, SelectionVector &sel,
-                        idx_t count, const TupleDataLayout &rhs_layout, Vector &rhs_row_locations,
-                        SelectionVector *no_match_sel, idx_t &no_match_count) {
-	D_ASSERT(!match_functions.empty());
-	for (idx_t col_idx = 0; col_idx < match_functions.size(); col_idx++) {
-		const auto &match_function = match_functions[col_idx];
-		count =
-		    match_function.function(lhs.data[col_idx], lhs_formats[col_idx], sel, count, rhs_layout, rhs_row_locations,
-		                            col_idx, match_function.child_functions, no_match_sel, no_match_count);
-	}
-	return count;
-}
-
-idx_t RowMatcher::Match(DataChunk &lhs, const vector<TupleDataVectorFormat> &lhs_formats, SelectionVector &sel,
-                        idx_t count, const TupleDataLayout &rhs_layout, Vector &rhs_row_locations,
-                        SelectionVector *no_match_sel, idx_t &no_match_count, const vector<column_t> &columns) {
+                        idx_t count, Vector &rhs_row_locations, SelectionVector *no_match_sel, idx_t &no_match_count) {
 	D_ASSERT(!match_functions.empty());
 
 	// The column_ids must have the same size as the match_functions vector
@@ -258,7 +276,7 @@ idx_t RowMatcher::Match(DataChunk &lhs, const vector<TupleDataVectorFormat> &lhs
 
 		const auto &match_function = match_functions[fun_idx];
 		count =
-		    match_function.function(lhs.data[col_idx], lhs_formats[col_idx], sel, count, rhs_layout, rhs_row_locations,
+		    match_function.function(lhs.data[col_idx], lhs_formats[col_idx], sel, count, *rhs_layout, rhs_row_locations,
 		                            col_idx, match_function.child_functions, no_match_sel, no_match_count);
 	}
 	return count;

package/src/duckdb/src/common/row_operations/row_scatter.cpp

@@ -1,9 +1,3 @@
-//===--------------------------------------------------------------------===//
-// row_scatter.cpp
-// Description: This file contains the implementation of the row scattering
-//              operators
-//===--------------------------------------------------------------------===//
-
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/row_operations/row_operations.hpp"

package/src/duckdb/src/common/serializer/buffered_file_reader.cpp

@@ -23,6 +23,10 @@ BufferedFileReader::BufferedFileReader(FileSystem &fs, unique_ptr<FileHandle> ha
 }
 
 void BufferedFileReader::ReadData(data_ptr_t target_buffer, uint64_t read_size) {
+	ReadData(QueryContext(), target_buffer, read_size);
+}
+
+void BufferedFileReader::ReadData(QueryContext context, data_ptr_t target_buffer, uint64_t read_size) {
 	// first copy anything we can from the buffer
 	data_ptr_t end_ptr = target_buffer + read_size;
 	while (true) {

package/src/duckdb/src/common/serializer/buffered_file_writer.cpp

@@ -90,6 +90,10 @@ void BufferedFileWriter::Truncate(idx_t size) {
 		handle->Truncate(NumericCast<int64_t>(size));
 		// reset anything written in the buffer
 		offset = 0;
+		// Reset the seek position if applicable
+		if (handle->CanSeek() && handle->SeekPosition() > size) {
+			handle->Seek(size);
+		}
 	}
 }
 

package/src/duckdb/src/common/serializer/memory_stream.cpp

@@ -75,6 +75,10 @@ void MemoryStream::WriteData(const_data_ptr_t source, idx_t write_size) {
 }
 
 void MemoryStream::ReadData(data_ptr_t destination, idx_t read_size) {
+	ReadData(QueryContext(), destination, read_size);
+}
+
+void MemoryStream::ReadData(QueryContext context, data_ptr_t destination, idx_t read_size) {
 	if (position + read_size > capacity) {
 		throw SerializationException("Failed to deserialize: not enough data in buffer to fulfill read request");
 	}