duckdb 1.3.4 → 1.4.0

package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp

@@ -152,4 +152,61 @@ TO ExactNumericCast(float val) {
 	return res;
 }
 
+template <class T>
+struct NextUnsigned {};
+
+template <>
+struct NextUnsigned<uint8_t> {
+	using type = uint16_t;
+};
+
+template <>
+struct NextUnsigned<uint16_t> {
+	using type = uint32_t;
+};
+
+template <>
+struct NextUnsigned<uint32_t> {
+	using type = uint64_t;
+};
+
+template <>
+struct NextUnsigned<uint64_t> {
+#if ((__GNUC__ >= 5) || defined(__clang__)) && defined(__SIZEOF_INT128__)
+	using type = __uint128_t;
+#else
+	using type = uhugeint_t;
+#endif
+};
+
+template <class TYPE>
+class FastMod {
+	using NEXT_TYPE = typename NextUnsigned<TYPE>::type;
+	static_assert(sizeof(NEXT_TYPE) != 0, "NextUnsigned not available for this type");
+
+public:
+	explicit FastMod(TYPE divisor_p) : divisor(divisor_p), multiplier((static_cast<TYPE>(-1) / divisor) + 1) {
+	}
+
+	TYPE Div(const TYPE &val) const {
+		return static_cast<TYPE>((static_cast<NEXT_TYPE>(val) * multiplier) >> (sizeof(TYPE) * 8)); // NOLINT
+	}
+
+	TYPE Mod(const TYPE &val, const TYPE &quotient) const {
+		return val - quotient * divisor;
+	}
+
+	TYPE Mod(const TYPE &val) const {
+		return Mod(val, Div(val));
+	}
+
+	const TYPE &GetDivisor() const {
+		return divisor;
+	}
+
+private:
+	const TYPE divisor;
+	const TYPE multiplier;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp

@@ -24,10 +24,10 @@ public:
 
 	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
 		GetFileSystem().Read(handle, buffer, nr_bytes, location);
-	};
+	}
 
 	void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
-		GetFileSystem().Write(handle, buffer, nr_bytes, location);
+		throw InternalException("writing on the OpenerFileSystem is undefined");
 	}
 
 	int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
@@ -41,7 +41,7 @@ public:
 	int64_t GetFileSize(FileHandle &handle) override {
 		return GetFileSystem().GetFileSize(handle);
 	}
-	time_t GetLastModifiedTime(FileHandle &handle) override {
+	timestamp_t GetLastModifiedTime(FileHandle &handle) override {
 		return GetFileSystem().GetLastModifiedTime(handle);
 	}
 	string GetVersionTag(FileHandle &handle) override {
@@ -143,6 +143,10 @@ public:
 		GetFileSystem().SetDisabledFileSystems(names);
 	}
 
+	bool SubSystemIsDisabled(const string &name) override {
+		return GetFileSystem().SubSystemIsDisabled(name);
+	}
+
 	vector<string> ListSubSystems() override {
 		return GetFileSystem().ListSubSystems();
 	}

package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp

@@ -591,6 +591,16 @@ DUCKDB_API bool TryCast::Operation(dtime_t input, dtime_t &result, bool strict);
 template <>
 DUCKDB_API bool TryCast::Operation(dtime_t input, dtime_tz_t &result, bool strict);
 
+//===--------------------------------------------------------------------===//
+// Time (ns) Casts
+//===--------------------------------------------------------------------===//
+template <>
+DUCKDB_API bool TryCast::Operation(dtime_t input, dtime_ns_t &result, bool strict);
+template <>
+DUCKDB_API bool TryCast::Operation(dtime_ns_t input, dtime_ns_t &result, bool strict);
+template <>
+DUCKDB_API bool TryCast::Operation(dtime_ns_t input, dtime_t &result, bool strict);
+
 //===--------------------------------------------------------------------===//
 // Time With Time Zone Casts (Offset)
 //===--------------------------------------------------------------------===//
@@ -652,6 +662,15 @@ DUCKDB_API bool TryCast::Operation(string_t input, dtime_t &result, bool strict)
 template <>
 dtime_t Cast::Operation(string_t input);
 //===--------------------------------------------------------------------===//
+// String -> Time_NS Casts
+//===--------------------------------------------------------------------===//
+template <>
+DUCKDB_API bool TryCastErrorMessage::Operation(string_t input, dtime_ns_t &result, CastParameters &parameters);
+template <>
+DUCKDB_API bool TryCast::Operation(string_t input, dtime_ns_t &result, bool strict);
+template <>
+dtime_ns_t Cast::Operation(string_t input);
+//===--------------------------------------------------------------------===//
 // String -> TimeTZ Casts
 //===--------------------------------------------------------------------===//
 template <>
@@ -666,12 +685,18 @@ dtime_tz_t Cast::Operation(string_t input);
 template <>
 DUCKDB_API bool TryCastErrorMessage::Operation(string_t input, timestamp_t &result, CastParameters &parameters);
 template <>
+DUCKDB_API bool TryCastErrorMessage::Operation(string_t input, timestamp_tz_t &result, CastParameters &parameters);
+template <>
 DUCKDB_API bool TryCast::Operation(string_t input, timestamp_t &result, bool strict);
 template <>
+DUCKDB_API bool TryCast::Operation(string_t input, timestamp_tz_t &result, bool strict);
+template <>
 DUCKDB_API bool TryCast::Operation(string_t input, timestamp_ns_t &result, bool strict);
 template <>
 timestamp_t Cast::Operation(string_t input);
 template <>
+timestamp_tz_t Cast::Operation(string_t input);
+template <>
 timestamp_ns_t Cast::Operation(string_t input);
 //===--------------------------------------------------------------------===//
 // String -> Interval Casts
@@ -717,6 +742,33 @@ DUCKDB_API bool TryCastToTimestampMS::Operation(date_t input, timestamp_t &resul
 template <>
 DUCKDB_API bool TryCastToTimestampSec::Operation(date_t input, timestamp_t &result, bool strict);
 
+//===--------------------------------------------------------------------===//
+// string -> Non-Standard Time types
+//===--------------------------------------------------------------------===//
+struct TryCastToTimeNS {
+	template <class SRC, class DST>
+	static inline bool Operation(SRC input, DST &result, bool strict = false) {
+		throw InternalException("Unsupported type for try cast to time (ns)");
+	}
+};
+
+template <>
+DUCKDB_API bool TryCastToTimeNS::Operation(string_t input, dtime_ns_t &result, bool strict);
+
+//===--------------------------------------------------------------------===//
+// Non-Standard Time -> string/time types
+//===--------------------------------------------------------------------===//
+
+struct CastFromTimeNS {
+	template <class SRC>
+	static inline string_t Operation(SRC input, Vector &result) {
+		throw duckdb::NotImplementedException("Cast to string could not be performed!");
+	}
+};
+
+template <>
+duckdb::string_t CastFromTimeNS::Operation(duckdb::dtime_ns_t input, Vector &result);
+
 //===--------------------------------------------------------------------===//
 // Non-Standard Timestamps -> string/timestamp types
 //===--------------------------------------------------------------------===//
@@ -803,6 +855,12 @@ struct CastTimestampNsToTime {
 		throw duckdb::NotImplementedException("Cast to TIME could not be performed!");
 	}
 };
+struct CastTimestampNsToTimeNs {
+	template <class SRC, class DST>
+	static inline DST Operation(SRC input) {
+		throw duckdb::NotImplementedException("Cast to TIME_NS could not be performed!");
+	}
+};
 struct CastTimestampNsToUs {
 	template <class SRC, class DST>
 	static inline DST Operation(SRC input) {
@@ -862,6 +920,8 @@ duckdb::date_t CastTimestampNsToDate::Operation(duckdb::timestamp_t input);
 template <>
 duckdb::dtime_t CastTimestampNsToTime::Operation(duckdb::timestamp_t input);
 template <>
+duckdb::dtime_ns_t CastTimestampNsToTimeNs::Operation(duckdb::timestamp_ns_t input);
+template <>
 duckdb::timestamp_t CastTimestampNsToUs::Operation(duckdb::timestamp_t input);
 template <>
 duckdb::date_t CastTimestampSecToDate::Operation(duckdb::timestamp_t input);
@@ -986,6 +1046,30 @@ template <>
 DUCKDB_API bool TryCastToUUID::Operation(string_t input, hugeint_t &result, Vector &result_vector,
                                          CastParameters &parameters);
 
+struct CastFromUUIDToBlob {
+	template <class SRC>
+	static inline string_t Operation(SRC input, Vector &result) {
+		throw duckdb::NotImplementedException("Cast from uuid to blob could not be performed!");
+	}
+};
+template <>
+duckdb::string_t CastFromUUIDToBlob::Operation(duckdb::hugeint_t input, Vector &vector);
+
+struct TryCastBlobToUUID {
+	template <class SRC, class DST>
+	static inline bool Operation(SRC input, DST &result, Vector &result_vector, CastParameters &parameters) {
+		throw InternalException("Unsupported type for try cast blob to uuid");
+	}
+	template <class SRC, class DST>
+	static inline bool Operation(SRC input, DST &result, bool strict) {
+		throw InternalException("Unsupported type for try cast blob to uuid");
+	}
+};
+template <>
+bool TryCastBlobToUUID::Operation(string_t input, hugeint_t &result, Vector &result_vector, CastParameters &parameters);
+template <>
+bool TryCastBlobToUUID::Operation(string_t input, hugeint_t &result, bool strict);
+
 //===--------------------------------------------------------------------===//
 // Pointers
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/operator/interpolate.hpp

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/operator/interpolate.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/numeric_utils.hpp"
+#include "duckdb/common/types/interval.hpp"
+#include "duckdb/common/types/timestamp.hpp"
+
+namespace duckdb {
+
+//	Linear interpolation between two values
+struct InterpolateOperator {
+	template <typename TARGET_TYPE>
+	static inline TARGET_TYPE Operation(const TARGET_TYPE &lo, const double d, const TARGET_TYPE &hi) {
+		const auto delta = static_cast<double>(hi - lo);
+		return LossyNumericCast<TARGET_TYPE>(lo + static_cast<TARGET_TYPE>(delta * d));
+	}
+};
+
+template <>
+double InterpolateOperator::Operation(const double &lo, const double d, const double &hi);
+template <>
+dtime_t InterpolateOperator::Operation(const dtime_t &lo, const double d, const dtime_t &hi);
+template <>
+timestamp_t InterpolateOperator::Operation(const timestamp_t &lo, const double d, const timestamp_t &hi);
+template <>
+hugeint_t InterpolateOperator::Operation(const hugeint_t &lo, const double d, const hugeint_t &hi);
+template <>
+uhugeint_t InterpolateOperator::Operation(const uhugeint_t &lo, const double d, const uhugeint_t &hi);
+template <>
+interval_t InterpolateOperator::Operation(const interval_t &lo, const double d, const interval_t &hi);
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/operator/string_cast.hpp

@@ -17,6 +17,10 @@
 
 namespace duckdb {
 
+struct date_t;
+struct timestamp_t;
+struct timestamp_ns_t;
+
 //! StringCast
 class Vector;
 
@@ -62,6 +66,8 @@ DUCKDB_API duckdb::string_t StringCast::Operation(date_t input, Vector &result);
 template <>
 DUCKDB_API duckdb::string_t StringCast::Operation(dtime_t input, Vector &result);
 template <>
+DUCKDB_API duckdb::string_t StringCast::Operation(dtime_ns_t input, Vector &result);
+template <>
 DUCKDB_API duckdb::string_t StringCast::Operation(timestamp_t input, Vector &result);
 template <>
 DUCKDB_API duckdb::string_t StringCast::Operation(timestamp_ns_t input, Vector &result);

package/src/duckdb/src/include/duckdb/common/optionally_owned_ptr.hpp

@@ -26,8 +26,12 @@ public:
 	optionally_owned_ptr(unique_ptr<T> &&owned_p) // NOLINT: allow implicit creation from moved unique_ptr
 	    : owned(std::move(owned_p)), ptr(owned) {
 	}
+	optionally_owned_ptr(shared_ptr<T> owned_p) // NOLINT: allow implicit creation from shared_ptr
+	    : owned_shared(std::move(owned_p)), ptr(*owned_shared) {
+	}
 	// Move constructor
-	optionally_owned_ptr(optionally_owned_ptr &&other) noexcept : owned(std::move(other.owned)), ptr(other.ptr) {
+	optionally_owned_ptr(optionally_owned_ptr &&other) noexcept
+	    : owned(std::move(other.owned)), owned_shared(std::move(other.owned_shared)), ptr(other.ptr) {
 		other.ptr = nullptr;
 	}
 	// Copy constructor
@@ -55,7 +59,13 @@ public:
 		return ptr.get();
 	}
 	bool is_owned() const { // NOLINT: mimic std casing
-		return owned != nullptr;
+		return owned != nullptr || owned_shared != nullptr;
+	}
+	const unique_ptr<T> &get_owned_unique() const {
+		return owned;
+	}
+	const shared_ptr<T> &get_owned_shared() const {
+		return owned_shared;
 	}
 	// this looks dirty - but this is the default behavior of raw pointers
 	T *get_mutable() const { // NOLINT: mimic std casing
@@ -64,11 +74,13 @@ public:
 
 	optionally_owned_ptr<T> &operator=(T &ref) {
 		owned = nullptr;
+		owned_shared = nullptr;
 		ptr = optional_ptr<T>(ref);
 		return *this;
 	}
 	optionally_owned_ptr<T> &operator=(T *ref) {
 		owned = nullptr;
+		owned_shared = nullptr;
 		ptr = optional_ptr<T>(ref);
 		return *this;
 	}
@@ -77,6 +89,9 @@ public:
 		if (owned != rhs.owned) {
 			return false;
 		}
+		if (owned_shared != rhs.owned_shared) {
+			return false;
+		}
 		return ptr == rhs.ptr;
 	}
 
@@ -86,6 +101,7 @@ public:
 
 private:
 	unique_ptr<T> owned;
+	shared_ptr<T> owned_shared;
 	optional_ptr<T> ptr;
 };
 

package/src/duckdb/src/include/duckdb/common/owning_string_map.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/string_map_set.hpp"
 #include "duckdb/common/allocator.hpp"
 #include "duckdb/common/pair.hpp"
+#include "duckdb/common/map.hpp"
 #include "duckdb/storage/arena_allocator.hpp"
 #include <cstring>
 
@@ -46,6 +47,13 @@ public:
 			return map.insert(make_pair(CopyString(entry.first), std::move(entry.second)));
 		}
 	}
+	std::pair<iterator, bool> emplace(value_type &&entry) { // NOLINT: match std style
+		auto it = find(entry.first);
+		if (it != end()) {
+			return std::make_pair(it, false);
+		}
+		return insert(std::move(entry));
+	}
 	size_type erase(const key_type &k) { // NOLINT: match std style
 		auto entry = map.find(k);
 		if (entry == map.end()) {

package/src/duckdb/src/include/duckdb/common/pipe_file_system.hpp

@@ -14,7 +14,7 @@ namespace duckdb {
 
 class PipeFileSystem : public FileSystem {
 public:
-	static unique_ptr<FileHandle> OpenPipe(unique_ptr<FileHandle> handle);
+	static unique_ptr<FileHandle> OpenPipe(QueryContext context, unique_ptr<FileHandle> handle);
 
 	int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
 	int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override;

package/src/duckdb/src/include/duckdb/common/progress_bar/display/terminal_progress_bar_display.hpp

@@ -11,20 +11,59 @@
 #include "duckdb/common/constants.hpp"
 #include "duckdb/common/progress_bar/progress_bar_display.hpp"
 #include "duckdb/common/unicode_bar.hpp"
+#include "duckdb/common/progress_bar/unscented_kalman_filter.hpp"
+#include <chrono>
+#include <thread>
 
 namespace duckdb {
 
+struct TerminalProgressBarDisplayedProgressInfo {
+	optional_idx percentage;
+	optional_idx estimated_seconds_remaining;
+
+	bool operator==(const TerminalProgressBarDisplayedProgressInfo &other) const {
+		return percentage == other.percentage && estimated_seconds_remaining == other.estimated_seconds_remaining;
+	}
+
+	bool operator!=(const TerminalProgressBarDisplayedProgressInfo &other) const {
+		return !(*this == other);
+	}
+};
+
 class TerminalProgressBarDisplay : public ProgressBarDisplay {
+private:
+	UnscentedKalmanFilter ukf;
+	std::chrono::steady_clock::time_point start_time;
+
+	// track the progress info that has been previously
+	// displayed to prevent redundant updates
+	struct TerminalProgressBarDisplayedProgressInfo displayed_progress_info;
+
+	double GetElapsedDuration() {
+		auto now = std::chrono::steady_clock::now();
+		return std::chrono::duration<double>(now - start_time).count();
+	}
+	void StopPeriodicUpdates();
+
 public:
-	TerminalProgressBarDisplay() = default;
-	~TerminalProgressBarDisplay() override = default;
+	TerminalProgressBarDisplay() {
+		start_time = std::chrono::steady_clock::now();
+		displayed_progress_info = {optional_idx(), optional_idx()};
+	}
+
+	~TerminalProgressBarDisplay() override {
+	}
 
 public:
 	void Update(double percentage) override;
 	void Finish() override;
 
 private:
-	int32_t rendered_percentage = -1;
+	std::mutex mtx;
+	std::thread periodic_update_thread;
+	std::condition_variable cv;
+	void PeriodicUpdate();
+
 	static constexpr const idx_t PARTIAL_BLOCK_COUNT = UnicodeBar::PartialBlocksCount();
 #ifndef DUCKDB_ASCII_TREE_RENDERER
 	const char *PROGRESS_EMPTY = " ";                                  // NOLINT
@@ -39,11 +78,11 @@ private:
 	const char *PROGRESS_START = "[";
 	const char *PROGRESS_END = "]";
 #endif
-	static constexpr const idx_t PROGRESS_BAR_WIDTH = 60;
+	static constexpr const idx_t PROGRESS_BAR_WIDTH = 38;
 
 private:
 	static int32_t NormalizePercentage(double percentage);
-	void PrintProgressInternal(int32_t percentage);
+	void PrintProgressInternal(int32_t percentage, double estimated_remaining_seconds, bool is_finished = false);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/progress_bar/progress_bar.hpp

@@ -52,7 +52,7 @@ public:
 	//! Updates the progress bar and prints it to the screen
 	void Update(bool final);
 	QueryProgress GetDetailedQueryProgress();
-	void PrintProgress(int percentage);
+	void PrintProgress(double percentage);
 	void FinishProgressBarPrint();
 	bool ShouldPrint(bool final) const;
 	bool PrintEnabled() const;

package/src/duckdb/src/include/duckdb/common/progress_bar/unscented_kalman_filter.hpp

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/progress_bar/unscented_kalman_filter.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb.hpp"
+#include <chrono>
+
+namespace duckdb {
+
+class UnscentedKalmanFilter {
+private:
+	static constexpr size_t STATE_DIM = 2; // [progress, velocity]
+	static constexpr size_t OBS_DIM = 1;   // [progress]
+	static constexpr size_t SIGMA_POINTS = 2 * STATE_DIM + 1;
+
+	// UKF parameters
+	static constexpr double ALPHA = 0.044;
+	static constexpr double BETA = 1.0;
+	static constexpr double KAPPA = 0.0;
+
+	double lambda;
+	vector<double> wm, wc; // weights for mean and covariance
+
+	// State: [progress (0-1), velocity (progress/second)]
+	vector<double> x;         // state estimate
+	vector<vector<double>> P; // covariance matrix
+	vector<vector<double>> Q; // process noise
+	vector<vector<double>> R; // measurement noise
+
+	double last_time;
+	bool initialized;
+	double last_progress;
+	double scale_factor;
+
+	// Helper functions
+	vector<vector<double>> MatrixSqrt(const vector<vector<double>> &mat);
+	vector<vector<double>> GenerateSigmaPoints();
+	vector<double> StateTransition(const vector<double> &state, double dt);
+	vector<double> MeasurementFunction(const vector<double> &state);
+
+public:
+	UnscentedKalmanFilter();
+
+	void Update(double progress, double time);
+
+	double GetEstimatedRemainingSeconds() const;
+
+private:
+	void Initialize(double initial_progress, double current_time);
+	void Predict(double current_time);
+	void UpdateInternal(double measured_progress);
+
+	double GetProgress() const;
+	double GetVelocity() const;
+	double GetProgressVariance() const;
+	double GetVelocityVariance() const;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/row_operations/row_matcher.hpp

@@ -34,27 +34,18 @@ struct RowMatcher {
 public:
 	using Predicates = vector<ExpressionType>;
 
-	//! Initializes the RowMatcher, filling match_functions using layout and predicates
-	void Initialize(const bool no_match_sel, const TupleDataLayout &layout, const Predicates &predicates);
-
-	//! Initializes the RowMatcher, filling match_functions using layout and equality_predicates but only for the given
-	//! columns
+	//! Initializes the RowMatcher, filling match_functions using layout and equality_predicates
+	//! If columns is empty, we assume column indices 0, 1, 2, etc.
 	void Initialize(const bool no_match_sel, const TupleDataLayout &layout, const Predicates &predicates,
-	                vector<column_t> &columns);
+	                vector<column_t> columns = {});
 
+public:
 	//! Given a DataChunk on the LHS, on which we've called TupleDataCollection::ToUnifiedFormat,
 	//! we match it with rows on the RHS, according to the given layout and locations.
 	//! Initially, 'sel' has 'count' entries which point to what needs to be compared.
 	//! After matching is done, this returns how many matching entries there are, which 'sel' is modified to point to
 	idx_t Match(DataChunk &lhs, const vector<TupleDataVectorFormat> &lhs_formats, SelectionVector &sel, idx_t count,
-	            const TupleDataLayout &rhs_layout, Vector &rhs_row_locations, SelectionVector *no_match_sel,
-	            idx_t &no_match_count);
-
-	//! Same as Match above, but only compares the column indexes in columns. Needs to be initialized with the same
-	//! columns.
-	idx_t Match(DataChunk &lhs, const vector<TupleDataVectorFormat> &lhs_formats, SelectionVector &sel, idx_t count,
-	            const TupleDataLayout &rhs_layout, Vector &rhs_row_locations, SelectionVector *no_match_sel,
-	            idx_t &no_match_count, const vector<column_t> &columns);
+	            Vector &rhs_row_locations, SelectionVector *no_match_sel, idx_t &no_match_count);
 
 private:
 	//! Gets the templated match function for a given column
@@ -69,7 +60,10 @@ private:
 	MatchFunction GetListMatchFunction(const ExpressionType predicate);
 
 private:
+	optional_ptr<const TupleDataLayout> rhs_layout;
+	vector<column_t> columns;
 	vector<MatchFunction> match_functions;
+	vector<LogicalType> rhs_types;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp

@@ -84,18 +84,6 @@ struct RowOperations {
 	                   const idx_t count, const RowLayout &layout, const idx_t col_no, const idx_t build_size = 0,
 	                   data_ptr_t heap_ptr = nullptr);
 
-	//===--------------------------------------------------------------------===//
-	// Comparison Operators
-	//===--------------------------------------------------------------------===//
-	//! Compare a block of key data against the row values to produce an updated selection that matches
-	//! and a second (optional) selection of non-matching values.
-	//! Returns the number of matches remaining in the selection.
-	using Predicates = vector<ExpressionType>;
-
-	static idx_t Match(DataChunk &columns, UnifiedVectorFormat col_data[], const TupleDataLayout &layout, Vector &rows,
-	                   const Predicates &predicates, SelectionVector &sel, idx_t count, SelectionVector *no_match,
-	                   idx_t &no_match_count);
-
 	//===--------------------------------------------------------------------===//
 	// Heap Operators
 	//===--------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp

@@ -27,6 +27,8 @@ public:
 
 public:
 	void ReadData(data_ptr_t buffer, uint64_t read_size) override;
+	void ReadData(QueryContext context, data_ptr_t buffer, uint64_t read_size) override;
+
 	//! Returns true if the reader has finished reading the entire file
 	bool Finished();
 

package/src/duckdb/src/include/duckdb/common/serializer/deserializer.hpp

@@ -179,17 +179,32 @@ public:
 	}
 
 	template <class FUNC>
-	void ReadList(const field_id_t field_id, const char *tag, FUNC func) {
-		OnPropertyBegin(field_id, tag);
+	void ReadListInternal(FUNC func) {
 		auto size = OnListBegin();
 		List list {*this};
 		for (idx_t i = 0; i < size; i++) {
 			func(list, i);
 		}
 		OnListEnd();
+	}
+
+	template <class FUNC>
+	void ReadList(const field_id_t field_id, const char *tag, FUNC func) {
+		OnPropertyBegin(field_id, tag);
+		ReadListInternal(func);
 		OnPropertyEnd();
 	}
 
+	template <class FUNC>
+	void ReadOptionalList(const field_id_t field_id, const char *tag, FUNC func) {
+		if (!OnOptionalPropertyBegin(field_id, tag)) {
+			OnOptionalPropertyEnd(false);
+			return;
+		}
+		ReadListInternal(func);
+		OnOptionalPropertyEnd(true);
+	}
+
 	template <class FUNC>
 	void ReadObject(const field_id_t field_id, const char *tag, FUNC func) {
 		OnPropertyBegin(field_id, tag);

package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp

@@ -54,6 +54,7 @@ public:
 	//! Read data from the stream.
 	//! Throws if the read would exceed the capacity of the stream
 	void ReadData(data_ptr_t buffer, idx_t read_size) override;
+	void ReadData(QueryContext context, data_ptr_t buffer, idx_t read_size) override;
 
 	//! Rewind the stream to the start, keeping the capacity and the backing buffer intact
 	void Rewind();