duckdb 1.3.4 → 1.4.1-dev2.0

package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp

@@ -96,6 +96,8 @@ public:
 
 	bool HasExtension() const;
 
+	ArrowArrayPhysicalType GetPhysicalType() const;
+
 	//! The Arrow Type Extension data, if any
 	shared_ptr<ArrowTypeExtensionData> extension_data;
 
@@ -114,13 +116,17 @@ protected:
 
 using arrow_column_map_t = unordered_map<idx_t, shared_ptr<ArrowType>>;
 
-struct ArrowTableType {
+struct ArrowTableSchema {
 public:
-	void AddColumn(idx_t index, shared_ptr<ArrowType> type);
+	void AddColumn(idx_t index, shared_ptr<ArrowType> type, const string &name);
 	const arrow_column_map_t &GetColumns() const;
+	vector<LogicalType> &GetTypes();
+	vector<string> &GetNames();
 
 private:
 	arrow_column_map_t arrow_convert_data;
+	vector<LogicalType> types;
+	vector<string> column_names;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_type_info.hpp

@@ -21,6 +21,8 @@ namespace duckdb {
 
 class ArrowType;
 
+enum class ArrowArrayPhysicalType : uint8_t { DICTIONARY_ENCODED, RUN_END_ENCODED, DEFAULT };
+
 struct ArrowTypeInfo {
 public:
 	explicit ArrowTypeInfo() : type() {

package/src/duckdb/src/include/duckdb/function/table/arrow.hpp

@@ -65,7 +65,7 @@ public:
 	//! The (optional) dependency of this function (used in Python for example)
 	shared_ptr<DependencyItem> dependency;
 	//! Arrow table data
-	ArrowTableType arrow_table;
+	ArrowTableSchema arrow_table;
 	//! Whether projection pushdown is enabled on the scan
 	bool projection_pushdown_enabled = true;
 };
@@ -89,10 +89,9 @@ public:
 struct ArrowScanLocalState;
 struct ArrowArrayScanState {
 public:
-	explicit ArrowArrayScanState(ArrowScanLocalState &state, ClientContext &context);
+	explicit ArrowArrayScanState(ClientContext &context);
 
 public:
-	ArrowScanLocalState &state;
 	// Hold ownership over the Arrow Arrays owned by DuckDB to allow for zero-copy
 	shared_ptr<ArrowArrayWrapper> owned_data;
 	unordered_map<idx_t, unique_ptr<ArrowArrayScanState>> children;
@@ -153,7 +152,7 @@ public:
 	ArrowArrayScanState &GetState(idx_t child_idx) {
 		auto it = array_states.find(child_idx);
 		if (it == array_states.end()) {
-			auto child_p = make_uniq<ArrowArrayScanState>(*this, context);
+			auto child_p = make_uniq<ArrowArrayScanState>(context);
 			auto &child = *child_p;
 			array_states.emplace(child_idx, std::move(child_p));
 			return child;
@@ -181,6 +180,26 @@ struct ArrowScanGlobalState : public GlobalTableFunctionState {
 	}
 };
 
+struct ArrowToDuckDBConversion {
+	static void SetValidityMask(Vector &vector, ArrowArray &array, idx_t chunk_offset, idx_t size,
+	                            int64_t parent_offset, int64_t nested_offset, bool add_null = false);
+
+	static void ColumnArrowToDuckDBRunEndEncoded(Vector &vector, const ArrowArray &array, idx_t chunk_offset,
+	                                             ArrowArrayScanState &array_state, idx_t size,
+	                                             const ArrowType &arrow_type, int64_t nested_offset = -1,
+	                                             ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0);
+
+	static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, idx_t chunk_offset,
+	                                ArrowArrayScanState &array_state, idx_t size, const ArrowType &arrow_type,
+	                                int64_t nested_offset = -1, ValidityMask *parent_mask = nullptr,
+	                                uint64_t parent_offset = 0, bool ignore_extensions = false);
+
+	static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, idx_t chunk_offset,
+	                                          ArrowArrayScanState &array_state, idx_t size, const ArrowType &arrow_type,
+	                                          int64_t nested_offset = -1, const ValidityMask *parent_mask = nullptr,
+	                                          uint64_t parent_offset = 0);
+};
+
 struct ArrowTableFunction {
 public:
 	static void RegisterFunction(BuiltinFunctions &set);
@@ -214,9 +233,8 @@ public:
 
 	//! Scan Function
 	static void ArrowScanFunction(ClientContext &context, TableFunctionInput &data, DataChunk &output);
-	static void PopulateArrowTableType(DBConfig &config, ArrowTableType &arrow_table,
-	                                   const ArrowSchemaWrapper &schema_p, vector<string> &names,
-	                                   vector<LogicalType> &return_types);
+	static void PopulateArrowTableSchema(DBConfig &config, ArrowTableSchema &arrow_table,
+	                                     const ArrowSchema &arrow_schema);
 
 protected:
 	//! Defines Maximum Number of Threads
@@ -226,13 +244,10 @@ protected:
 	static OperatorPartitionData ArrowGetPartitionData(ClientContext &context, TableFunctionGetPartitionInput &input);
 
 	//! Specify if a given type can be pushed-down by the arrow engine
-	static bool ArrowPushdownType(const LogicalType &type);
+	static bool ArrowPushdownType(const FunctionData &bind_data, idx_t col_idx);
 	//! -----Utility Functions:-----
 	//! Gets Arrow Table's Cardinality
 	static unique_ptr<NodeStatistics> ArrowScanCardinality(ClientContext &context, const FunctionData *bind_data);
-	//! Gets the progress on the table scan, used for Progress Bars
-	static double ArrowProgress(ClientContext &context, const FunctionData *bind_data,
-	                            const GlobalTableFunctionState *global_state);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/direct_file_reader.hpp

@@ -0,0 +1,39 @@
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// direct_file_reader.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/multi_file/base_file_reader.hpp"
+
+namespace duckdb {
+
+class DirectFileReader : public BaseFileReader {
+public:
+	explicit DirectFileReader(OpenFileInfo file_p, const LogicalType &type);
+	~DirectFileReader() override;
+
+public:
+	unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, const string &name) override;
+
+	bool TryInitializeScan(ClientContext &context, GlobalTableFunctionState &gstate,
+	                       LocalTableFunctionState &lstate) override;
+	void Scan(ClientContext &context, GlobalTableFunctionState &global_state, LocalTableFunctionState &local_state,
+	          DataChunk &chunk) override;
+	void FinishFile(ClientContext &context, GlobalTableFunctionState &gstate) override;
+
+	string GetReaderType() const override {
+		return "File";
+	};
+
+private:
+	bool done;
+	LogicalType type;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp

@@ -18,6 +18,7 @@
 #include "duckdb/function/scalar/strftime_format.hpp"
 #include "duckdb/function/table_function.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp"
+#include "duckdb/common/csv_writer.hpp"
 
 namespace duckdb {
 class BaseScanner;
@@ -40,25 +41,14 @@ struct BaseCSVData : public TableFunctionData {
 };
 
 struct WriteCSVData : public BaseCSVData {
-	WriteCSVData(string file_path, vector<LogicalType> sql_types, vector<string> names)
-	    : sql_types(std::move(sql_types)) {
-		files.push_back(std::move(file_path));
+	explicit WriteCSVData(vector<string> names) {
 		options.name_list = std::move(names);
 		if (options.dialect_options.state_machine_options.escape == '\0') {
 			options.dialect_options.state_machine_options.escape = options.dialect_options.state_machine_options.quote;
 		}
 	}
-
-	//! The file path of the CSV file to read or write
-	vector<string> files;
-	//! The SQL types to write
-	vector<LogicalType> sql_types;
-	//! The newline string to write
-	string newline = "\n";
 	//! The size of the CSV file (in bytes) that we buffer before we flush it to disk
 	idx_t flush_size = 4096ULL * 8ULL;
-	//! For each byte whether the CSV file requires quotes when containing the byte
-	unsafe_unique_array<bool> requires_quotes;
 	//! Expressions used to convert the input into strings
 	vector<unique_ptr<Expression>> cast_expressions;
 };

package/src/duckdb/src/include/duckdb/function/table/read_file.hpp

@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/table/read_file.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/multi_file/multi_file_function.hpp"
+#include "utf8proc_wrapper.hpp"
+
+namespace duckdb {
+
+struct ReadFileBindData : public TableFunctionData {
+	unique_ptr<BaseFileReaderOptions> options;
+
+	static constexpr const idx_t FILE_NAME_COLUMN = 0;
+	static constexpr const idx_t FILE_CONTENT_COLUMN = 1;
+	static constexpr const idx_t FILE_SIZE_COLUMN = 2;
+	static constexpr const idx_t FILE_LAST_MODIFIED_COLUMN = 3;
+};
+
+struct ReadFileGlobalState : public GlobalTableFunctionState {
+	ReadFileGlobalState() {
+	}
+
+	shared_ptr<MultiFileList> file_list;
+	vector<idx_t> column_ids;
+	bool requires_file_open = false;
+};
+
+struct ReadBlobOperation {
+	static constexpr const char *NAME = "read_blob";
+	static constexpr const char *FILE_TYPE = "blob";
+
+	static inline LogicalType TYPE() {
+		return LogicalType::BLOB;
+	}
+};
+
+struct ReadTextOperation {
+	static constexpr const char *NAME = "read_text";
+	static constexpr const char *FILE_TYPE = "text";
+
+	static inline LogicalType TYPE() {
+		return LogicalType::VARCHAR;
+	}
+};
+
+template <class OP>
+struct DirectMultiFileInfo : MultiFileReaderInterface {
+	static unique_ptr<MultiFileReaderInterface> CreateInterface(ClientContext &context);
+	unique_ptr<BaseFileReaderOptions> InitializeOptions(ClientContext &context,
+	                                                    optional_ptr<TableFunctionInfo> info) override;
+	bool ParseCopyOption(ClientContext &context, const string &key, const vector<Value> &values,
+	                     BaseFileReaderOptions &options, vector<string> &expected_names,
+	                     vector<LogicalType> &expected_types) override;
+	bool ParseOption(ClientContext &context, const string &key, const Value &val, MultiFileOptions &file_options,
+	                 BaseFileReaderOptions &options) override;
+	unique_ptr<TableFunctionData> InitializeBindData(MultiFileBindData &multi_file_data,
+	                                                 unique_ptr<BaseFileReaderOptions> options) override;
+	void BindReader(ClientContext &context, vector<LogicalType> &return_types, vector<string> &names,
+	                MultiFileBindData &bind_data) override;
+	optional_idx MaxThreads(const MultiFileBindData &bind_data_p, const MultiFileGlobalState &global_state,
+	                        FileExpandResult expand_result) override;
+	unique_ptr<GlobalTableFunctionState> InitializeGlobalState(ClientContext &context, MultiFileBindData &bind_data,
+	                                                           MultiFileGlobalState &global_state) override;
+	unique_ptr<LocalTableFunctionState> InitializeLocalState(ExecutionContext &, GlobalTableFunctionState &) override;
+	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, GlobalTableFunctionState &gstate,
+	                                        BaseUnionData &union_data, const MultiFileBindData &bind_data_p) override;
+	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, GlobalTableFunctionState &gstate,
+	                                        const OpenFileInfo &file, idx_t file_idx,
+	                                        const MultiFileBindData &bind_data) override;
+	shared_ptr<BaseFileReader> CreateReader(ClientContext &context, const OpenFileInfo &file,
+	                                        BaseFileReaderOptions &options,
+	                                        const MultiFileOptions &file_options) override;
+	unique_ptr<NodeStatistics> GetCardinality(const MultiFileBindData &bind_data, idx_t file_count) override;
+	FileGlobInput GetGlobInput() override;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp

@@ -47,6 +47,14 @@ struct DuckDBSchemasFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct DuckDBConnectionCountFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct DuckDBApproxDatabaseCountFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct DuckDBColumnsFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
@@ -147,6 +155,10 @@ struct DuckDBViewsFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct EnableLoggingFun {
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct TestType {
 	TestType(LogicalType type_p, string name_p)
 	    : type(std::move(type_p)), name(std::move(name_p)), min_value(Value::MinimumValue(type)),
@@ -164,7 +176,7 @@ struct TestType {
 
 struct TestAllTypesFun {
 	static void RegisterFunction(BuiltinFunctions &set);
-	static vector<TestType> GetTestTypes(bool large_enum = false);
+	static vector<TestType> GetTestTypes(bool large_enum = false, bool large_bignum = false);
 };
 
 struct TestVectorTypesFun {

package/src/duckdb/src/include/duckdb/function/table_function.hpp

@@ -111,9 +111,10 @@ struct TableFunctionBindInput {
 struct TableFunctionInitInput {
 	TableFunctionInitInput(optional_ptr<const FunctionData> bind_data_p, vector<column_t> column_ids_p,
 	                       const vector<idx_t> &projection_ids_p, optional_ptr<TableFilterSet> filters_p,
-	                       optional_ptr<SampleOptions> sample_options_p = nullptr)
+	                       optional_ptr<SampleOptions> sample_options_p = nullptr,
+	                       optional_ptr<const PhysicalOperator> op_p = nullptr)
 	    : bind_data(bind_data_p), column_ids(std::move(column_ids_p)), projection_ids(projection_ids_p),
-	      filters(filters_p), sample_options(sample_options_p) {
+	      filters(filters_p), sample_options(sample_options_p), op(op_p) {
 		for (auto &col_id : column_ids) {
 			column_indexes.emplace_back(col_id);
 		}
@@ -121,9 +122,10 @@ struct TableFunctionInitInput {
 
 	TableFunctionInitInput(optional_ptr<const FunctionData> bind_data_p, vector<ColumnIndex> column_indexes_p,
 	                       const vector<idx_t> &projection_ids_p, optional_ptr<TableFilterSet> filters_p,
-	                       optional_ptr<SampleOptions> sample_options_p = nullptr)
+	                       optional_ptr<SampleOptions> sample_options_p = nullptr,
+	                       optional_ptr<const PhysicalOperator> op_p = nullptr)
 	    : bind_data(bind_data_p), column_indexes(std::move(column_indexes_p)), projection_ids(projection_ids_p),
-	      filters(filters_p), sample_options(sample_options_p) {
+	      filters(filters_p), sample_options(sample_options_p), op(op_p) {
 		for (auto &col_id : column_indexes) {
 			column_ids.emplace_back(col_id.GetPrimaryIndex());
 		}
@@ -135,6 +137,7 @@ struct TableFunctionInitInput {
 	const vector<idx_t> projection_ids;
 	optional_ptr<TableFilterSet> filters;
 	optional_ptr<SampleOptions> sample_options;
+	optional_ptr<const PhysicalOperator> op;
 
 	bool CanRemoveFilterColumns() const {
 		if (projection_ids.empty()) {
@@ -288,7 +291,7 @@ typedef BindInfo (*table_function_get_bind_info_t)(const optional_ptr<FunctionDa
 
 typedef unique_ptr<MultiFileReader> (*table_function_get_multi_file_reader_t)(const TableFunction &);
 
-typedef bool (*table_function_supports_pushdown_type_t)(const LogicalType &type);
+typedef bool (*table_function_supports_pushdown_type_t)(const FunctionData &bind_data, idx_t col_idx);
 
 typedef double (*table_function_progress_t)(ClientContext &context, const FunctionData *bind_data,
                                             const GlobalTableFunctionState *global_state);

package/src/duckdb/src/include/duckdb/function/window/window_aggregate_function.hpp

@@ -16,17 +16,17 @@ namespace duckdb {
 
 class WindowAggregateExecutor : public WindowExecutor {
 public:
-	WindowAggregateExecutor(BoundWindowExpression &wexpr, ClientContext &context, WindowSharedExpressions &shared,
+	WindowAggregateExecutor(BoundWindowExpression &wexpr, ClientContext &client, WindowSharedExpressions &shared,
 	                        WindowAggregationMode mode);
 
-	void Sink(DataChunk &sink_chunk, DataChunk &coll_chunk, const idx_t input_idx, WindowExecutorGlobalState &gstate,
-	          WindowExecutorLocalState &lstate) const override;
-	void Finalize(WindowExecutorGlobalState &gstate, WindowExecutorLocalState &lstate,
-	              CollectionPtr collection) const override;
+	void Sink(ExecutionContext &context, DataChunk &sink_chunk, DataChunk &coll_chunk, const idx_t input_idx,
+	          OperatorSinkInput &sink) const override;
+	void Finalize(ExecutionContext &context, CollectionPtr collection, OperatorSinkInput &sink) const override;
 
-	unique_ptr<WindowExecutorGlobalState> GetGlobalState(const idx_t payload_count, const ValidityMask &partition_mask,
-	                                                     const ValidityMask &order_mask) const override;
-	unique_ptr<WindowExecutorLocalState> GetLocalState(const WindowExecutorGlobalState &gstate) const override;
+	unique_ptr<GlobalSinkState> GetGlobalState(ClientContext &client, const idx_t payload_count,
+	                                           const ValidityMask &partition_mask,
+	                                           const ValidityMask &order_mask) const override;
+	unique_ptr<LocalSinkState> GetLocalState(ExecutionContext &context, const GlobalSinkState &gstate) const override;
 
 	const WindowAggregationMode mode;
 
@@ -37,8 +37,8 @@ public:
 	unique_ptr<Expression> filter_ref;
 
 protected:
-	void EvaluateInternal(WindowExecutorGlobalState &gstate, WindowExecutorLocalState &lstate, DataChunk &eval_chunk,
-	                      Vector &result, idx_t count, idx_t row_idx) const override;
+	void EvaluateInternal(ExecutionContext &context, DataChunk &eval_chunk, Vector &result, idx_t count, idx_t row_idx,
+	                      OperatorSinkInput &sink) const override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/window/window_aggregator.hpp

@@ -17,27 +17,6 @@ class WindowCollection;
 class WindowCursor;
 struct WindowSharedExpressions;
 
-class WindowAggregatorState {
-public:
-	WindowAggregatorState();
-	virtual ~WindowAggregatorState() {
-	}
-
-	template <class TARGET>
-	TARGET &Cast() {
-		DynamicCastCheck<TARGET>(this);
-		return reinterpret_cast<TARGET &>(*this);
-	}
-	template <class TARGET>
-	const TARGET &Cast() const {
-		DynamicCastCheck<TARGET>(this);
-		return reinterpret_cast<const TARGET &>(*this);
-	}
-
-	//! Allocator for aggregates
-	ArenaAllocator allocator;
-};
-
 class WindowAggregator {
 public:
 	using CollectionPtr = optional_ptr<WindowCollection>;
@@ -110,19 +89,20 @@ public:
 	virtual ~WindowAggregator();
 
 	//	Threading states
-	virtual unique_ptr<WindowAggregatorState> GetGlobalState(ClientContext &context, idx_t group_count,
-	                                                         const ValidityMask &partition_mask) const;
-	virtual unique_ptr<WindowAggregatorState> GetLocalState(const WindowAggregatorState &gstate) const = 0;
+	virtual unique_ptr<GlobalSinkState> GetGlobalState(ClientContext &client, idx_t group_count,
+	                                                   const ValidityMask &partition_mask) const;
+	virtual unique_ptr<LocalSinkState> GetLocalState(ExecutionContext &context,
+	                                                 const GlobalSinkState &gstate) const = 0;
 
 	//	Build
-	virtual void Sink(WindowAggregatorState &gstate, WindowAggregatorState &lstate, DataChunk &sink_chunk,
-	                  DataChunk &coll_chunk, idx_t input_idx, optional_ptr<SelectionVector> filter_sel, idx_t filtered);
-	virtual void Finalize(WindowAggregatorState &gstate, WindowAggregatorState &lstate, CollectionPtr collection,
-	                      const FrameStats &stats);
+	virtual void Sink(ExecutionContext &context, DataChunk &sink_chunk, DataChunk &coll_chunk, idx_t input_idx,
+	                  optional_ptr<SelectionVector> filter_sel, idx_t filtered, OperatorSinkInput &sink);
+	virtual void Finalize(ExecutionContext &context, CollectionPtr collection, const FrameStats &stats,
+	                      OperatorSinkInput &sink);
 
 	//	Probe
-	virtual void Evaluate(const WindowAggregatorState &gsink, WindowAggregatorState &lstate, const DataChunk &bounds,
-	                      Vector &result, idx_t count, idx_t row_idx) const = 0;
+	virtual void Evaluate(ExecutionContext &context, const DataChunk &bounds, Vector &result, idx_t count,
+	                      idx_t row_idx, OperatorSinkInput &sink) const = 0;
 
 	//! The window function
 	const BoundWindowExpression &wexpr;
@@ -140,21 +120,15 @@ public:
 	vector<column_t> child_idx;
 };
 
-class WindowAggregatorGlobalState : public WindowAggregatorState {
+class WindowAggregatorGlobalState : public GlobalSinkState {
 public:
-	WindowAggregatorGlobalState(ClientContext &context, const WindowAggregator &aggregator_p, idx_t group_count)
-	    : context(context), aggregator(aggregator_p), aggr(aggregator.wexpr), locals(0), finalized(0) {
-
-		if (aggr.filter) {
-			// 	Start with all invalid and set the ones that pass
-			filter_mask.Initialize(group_count, false);
-		} else {
-			filter_mask.InitializeEmpty(group_count);
-		}
-	}
+	WindowAggregatorGlobalState(ClientContext &client, const WindowAggregator &aggregator_p, idx_t group_count);
+
+	//! The client we are in
+	ClientContext &client;
 
-	//! The context we are in
-	ClientContext &context;
+	//! Global allocator
+	ArenaAllocator allocator;
 
 	//! The aggregator data
 	const WindowAggregator &aggregator;
@@ -175,17 +149,20 @@ public:
 	std::atomic<idx_t> finalized;
 };
 
-class WindowAggregatorLocalState : public WindowAggregatorState {
+class WindowAggregatorLocalState : public LocalSinkState {
 public:
 	using CollectionPtr = optional_ptr<WindowCollection>;
 
 	static void InitSubFrames(SubFrames &frames, const WindowExcludeMode exclude_mode);
 
-	WindowAggregatorLocalState() {
-	}
+	explicit WindowAggregatorLocalState(ExecutionContext &context);
 
-	void Sink(WindowAggregatorGlobalState &gastate, DataChunk &sink_chunk, DataChunk &coll_chunk, idx_t row_idx);
-	virtual void Finalize(WindowAggregatorGlobalState &gastate, CollectionPtr collection);
+	void Sink(ExecutionContext &context, WindowAggregatorGlobalState &gastate, DataChunk &sink_chunk,
+	          DataChunk &coll_chunk, idx_t row_idx);
+	virtual void Finalize(ExecutionContext &context, WindowAggregatorGlobalState &gastate, CollectionPtr collection);
+
+	//! Global allocator
+	ArenaAllocator allocator;
 
 	//! The state used for reading the collection
 	unique_ptr<WindowCursor> cursor;

package/src/duckdb/src/include/duckdb/function/window/window_collection.hpp

@@ -143,4 +143,146 @@ public:
 	DataChunk chunk;
 };
 
+class WindowCollectionChunkScanner {
+public:
+	WindowCollectionChunkScanner(ColumnDataCollection &collection, const vector<column_t> &scan_ids,
+	                             const idx_t begin_idx)
+	    : collection(collection), curr_idx(0) {
+		collection.InitializeScan(state, scan_ids);
+		collection.InitializeScanChunk(state, chunk);
+
+		Seek(begin_idx);
+	}
+
+	void Seek(idx_t begin_idx) {
+		idx_t chunk_idx;
+		idx_t seg_idx;
+		idx_t row_idx;
+		for (; curr_idx > begin_idx; --curr_idx) {
+			collection.PrevScanIndex(state, chunk_idx, seg_idx, row_idx);
+		}
+		for (; curr_idx < begin_idx; ++curr_idx) {
+			collection.NextScanIndex(state, chunk_idx, seg_idx, row_idx);
+		}
+	}
+
+	bool Scan() {
+		const auto result = collection.Scan(state, chunk);
+		++curr_idx;
+		return result;
+	}
+
+	idx_t Scanned() const {
+		return state.next_row_index;
+	}
+
+	//! Return a struct type for comparing keys
+	LogicalType PrefixStructType(column_t end, column_t begin = 0);
+	//! Reference the chunk into a struct vector matching the keys
+	static void ReferenceStructColumns(DataChunk &chunk, Vector &vec, column_t end, column_t begin = 0);
+
+	ColumnDataCollection &collection;
+	ColumnDataScanState state;
+	DataChunk chunk;
+	idx_t curr_idx;
+};
+
+template <typename OP>
+static void WindowDeltaScanner(ColumnDataCollection &collection, idx_t block_begin, idx_t block_end,
+                               const vector<column_t> &scan_cols, const idx_t key_count, OP operation) {
+
+	//	Stop if there is no work to do
+	if (!collection.Count()) {
+		return;
+	}
+
+	//	Start back one to get the overlap
+	idx_t block_curr = block_begin ? block_begin - 1 : 0;
+
+	//	Scan the sort columns
+	WindowCollectionChunkScanner scanner(collection, scan_cols, block_curr);
+	auto &scanned = scanner.chunk;
+
+	//	Shifted buffer for the next values
+	DataChunk next;
+	collection.InitializeScanChunk(scanner.state, next);
+
+	//	Delay buffer for the previous row
+	DataChunk delayed;
+	collection.InitializeScanChunk(scanner.state, delayed);
+
+	//	Only compare the key arguments.
+	const auto key_type = scanner.PrefixStructType(key_count);
+	Vector compare_curr(key_type);
+	Vector compare_prev(key_type);
+
+	bool boundary_compare = (block_begin > 0);
+	idx_t row_idx = 1;
+	if (!scanner.Scan()) {
+		return;
+	}
+
+	//	Process chunks offset by 1
+	SelectionVector next_sel(1, STANDARD_VECTOR_SIZE);
+	SelectionVector distinct(STANDARD_VECTOR_SIZE);
+	SelectionVector matching(STANDARD_VECTOR_SIZE);
+
+	// In order to reuse the verbose `distinct from` logic for both the main vector comparisons
+	// and single element boundary comparisons, we alternate between single element compares
+	// and count-1 compares.
+	while (block_curr < block_end) {
+		//	Compare the current to the previous;
+		DataChunk *curr = nullptr;
+		DataChunk *prev = nullptr;
+
+		idx_t count = 0;
+		if (boundary_compare) {
+			//	Save the last row of the scanned chunk
+			count = 1;
+			sel_t last = UnsafeNumericCast<sel_t>(scanned.size() - 1);
+			SelectionVector sel(&last);
+			delayed.Reset();
+			scanned.Copy(delayed, sel, count);
+			prev = &delayed;
+
+			// Try to read the next chunk
+			++block_curr;
+			row_idx = scanner.Scanned();
+			if (block_curr >= block_end || !scanner.Scan()) {
+				break;
+			}
+			curr = &scanned;
+		} else {
+			//	Compare the [1..size) values with the [0..size-1) values
+			count = scanned.size() - 1;
+			if (!count) {
+				//	1 row scanned, so just skip the rest of the loop.
+				boundary_compare = true;
+				continue;
+			}
+			prev = &scanned;
+
+			// Slice the current back one into the previous
+			next.Slice(scanned, next_sel, count);
+			curr = &next;
+		}
+
+		//	Reference the comparison prefix as a struct to simplify the compares.
+		scanner.ReferenceStructColumns(*prev, compare_prev, key_count);
+		scanner.ReferenceStructColumns(*curr, compare_curr, key_count);
+
+		const auto ndistinct =
+		    VectorOperations::DistinctFrom(compare_curr, compare_prev, nullptr, count, &distinct, &matching);
+
+		//	If n is 0, neither SV has been filled in?
+		auto match_sel = ndistinct ? &matching : FlatVector::IncrementalSelectionVector();
+
+		operation(row_idx, *prev, *curr, ndistinct, distinct, *match_sel);
+
+		//	Transition between comparison ranges.
+		boundary_compare = !boundary_compare;
+		row_idx += count;
+	}
+}
+
 } // namespace duckdb