duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/execution/merge_sort_tree.hpp

@@ -9,13 +9,16 @@
 #pragma once
 
 #include "duckdb/common/array.hpp"
+#include "duckdb/common/atomic.hpp"
 #include "duckdb/common/helper.hpp"
+#include "duckdb/common/mutex.hpp"
 #include "duckdb/common/pair.hpp"
 #include "duckdb/common/printer.hpp"
 #include "duckdb/common/typedefs.hpp"
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/vector_operations/aggregate_executor.hpp"
 #include <iomanip>
+#include <thread>
 
 namespace duckdb {
 
@@ -102,7 +105,14 @@ struct MergeSortTree {
 
 	explicit MergeSortTree(const CMP &cmp = CMP()) : cmp(cmp) {
 	}
-	explicit MergeSortTree(Elements &&lowest_level, const CMP &cmp = CMP());
+
+	inline Elements &LowestLevel() {
+		return tree[0].first;
+	}
+
+	Elements &Allocate(idx_t count);
+
+	void Build();
 
 	idx_t SelectNth(const SubFrames &frames, idx_t n) const;
 
@@ -120,6 +130,47 @@ struct MergeSortTree {
 	static constexpr auto CASCADING = C;
 
 protected:
+	//! Parallel build machinery
+	mutex build_lock;
+	atomic<idx_t> build_level;
+	atomic<idx_t> build_complete;
+	idx_t build_run;
+	idx_t build_run_length;
+	idx_t build_num_runs;
+
+	bool TryNextRun(idx_t &level_idx, idx_t &run_idx) {
+		const auto fanout = F;
+
+		lock_guard<mutex> stage_guard(build_lock);
+
+		// Finished with this level?
+		if (build_complete >= build_num_runs) {
+			++build_level;
+			if (build_level >= tree.size()) {
+				return false;
+			}
+
+			const auto count = LowestLevel().size();
+			build_run_length *= fanout;
+			build_num_runs = (count + build_run_length - 1) / build_run_length;
+			build_run = 0;
+			build_complete = 0;
+		}
+
+		// If all runs are in flight,
+		// yield until the next level is ready
+		if (build_run >= build_num_runs) {
+			return false;
+		}
+
+		level_idx = build_level;
+		run_idx = build_run++;
+
+		return true;
+	}
+
+	void BuildRun(idx_t level_idx, idx_t run_idx);
+
 	RunElement StartGames(Games &losers, const RunElements &elements, const RunElement &sentinel) {
 		const auto elem_nodes = elements.size();
 		const auto game_nodes = losers.size();
@@ -224,7 +275,7 @@ protected:
 				out << 'd';
 				for (size_t i = 0; i < level.first.size(); ++i) {
 					out << ((i && i % level_width == 0) ? group_separator : separator);
-					out << std::setw(number_width) << level.first[i];
+					out << std::setw(NumericCast<int32_t>(number_width)) << level.first[i];
 				}
 				out << '\n';
 			}
@@ -237,7 +288,7 @@ protected:
 					for (idx_t idx = 0; idx < cascading_idcs_cnt; idx += FANOUT) {
 						out << ((idx && ((idx / FANOUT) % (level_width / CASCADING + 2) == 0)) ? group_separator
 						                                                                       : separator);
-						out << std::setw(number_width) << level.second[idx + child_nr];
+						out << std::setw(NumericCast<int32_t>(number_width)) << level.second[idx + child_nr];
 					}
 					out << '\n';
 				}
@@ -250,93 +301,133 @@ protected:
 };
 
 template <typename E, typename O, typename CMP, uint64_t F, uint64_t C>
-MergeSortTree<E, O, CMP, F, C>::MergeSortTree(Elements &&lowest_level, const CMP &cmp) : cmp(cmp) {
+vector<E> &MergeSortTree<E, O, CMP, F, C>::Allocate(idx_t count) {
 	const auto fanout = F;
 	const auto cascading = C;
-	const auto count = lowest_level.size();
+	Elements lowest_level(count);
 	tree.emplace_back(Level(std::move(lowest_level), Offsets()));
 
-	const RunElement SENTINEL(MergeSortTraits<ElementType>::SENTINEL(), MergeSortTraits<idx_t>::SENTINEL());
-
-	//	Fan in parent levels until we are at the top
-	//	Note that we don't build the top layer as that would just be all the data.
 	for (idx_t child_run_length = 1; child_run_length < count;) {
 		const auto run_length = child_run_length * fanout;
 		const auto num_runs = (count + run_length - 1) / run_length;
 
 		Elements elements;
-		elements.reserve(count);
+		elements.resize(count);
 
 		//	Allocate cascading pointers only if there is room
 		Offsets cascades;
 		if (cascading > 0 && run_length > cascading) {
 			const auto num_cascades = fanout * num_runs * (run_length / cascading + 2);
-			cascades.reserve(num_cascades);
+			cascades.resize(num_cascades);
 		}
 
-		//	Create each parent run by merging the child runs using a tournament tree
-		// 	https://en.wikipedia.org/wiki/K-way_merge_algorithm
-		//	TODO: Because the runs are independent, they can be parallelised with parallel_for
-		const auto &child_level = tree.back();
-		for (idx_t run_idx = 0; run_idx < num_runs; ++run_idx) {
-			//	Position markers for scanning the children.
-			using Bounds = pair<idx_t, idx_t>;
-			array<Bounds, fanout> bounds;
-			//	Start with first element of each (sorted) child run
-			RunElements players;
-			const auto child_base = run_idx * run_length;
-			for (idx_t child_run = 0; child_run < fanout; ++child_run) {
-				const auto child_idx = child_base + child_run * child_run_length;
-				bounds[child_run] = {MinValue<idx_t>(child_idx, count),
-				                     MinValue<idx_t>(child_idx + child_run_length, count)};
-				if (bounds[child_run].first != bounds[child_run].second) {
-					players[child_run] = {child_level.first[child_idx], child_run};
-				} else {
-					//	Empty child
-					players[child_run] = SENTINEL;
-				}
-			}
+		//	Insert completed level and move up to the next one
+		tree.emplace_back(std::move(elements), std::move(cascades));
+		child_run_length = run_length;
+	}
 
-			//	Play the first round and extract the winner
-			Games games;
-			auto winner = StartGames(games, players, SENTINEL);
-			while (winner != SENTINEL) {
-				// Add fractional cascading pointers
-				// if we are on a fraction boundary
-				if (cascading > 0 && run_length > cascading && elements.size() % cascading == 0) {
-					for (idx_t i = 0; i < fanout; ++i) {
-						cascades.emplace_back(bounds[i].first);
-					}
-				}
+	//	Set up for parallel build
+	build_level = 1;
+	build_complete = 0;
+	build_run = 0;
+	build_run_length = fanout;
+	build_num_runs = (count + build_run_length - 1) / build_run_length;
 
-				//	Insert new winner element into the current run
-				elements.emplace_back(winner.first);
-				const auto child_run = winner.second;
-				auto &child_idx = bounds[child_run].first;
-				++child_idx;
-
-				//	Move to the next entry in the child run (if any)
-				if (child_idx < bounds[child_run].second) {
-					winner = ReplayGames(games, child_run, {child_level.first[child_idx], child_run});
-				} else {
-					winner = ReplayGames(games, child_run, SENTINEL);
-				}
-			}
+	return LowestLevel();
+}
 
-			// Add terminal cascade pointers to the end
-			if (cascading > 0 && run_length > cascading) {
-				for (idx_t j = 0; j < 2; ++j) {
-					for (idx_t i = 0; i < fanout; ++i) {
-						cascades.emplace_back(bounds[i].first);
-					}
-				}
-			}
+template <typename E, typename O, typename CMP, uint64_t F, uint64_t C>
+void MergeSortTree<E, O, CMP, F, C>::Build() {
+	//	Fan in parent levels until we are at the top
+	//	Note that we don't build the top layer as that would just be all the data.
+	while (build_level.load() < tree.size()) {
+		idx_t level_idx;
+		idx_t run_idx;
+		if (TryNextRun(level_idx, run_idx)) {
+			BuildRun(level_idx, run_idx);
+		} else {
+			std::this_thread::yield();
 		}
+	}
+}
 
-		//	Insert completed level and move up to the next one
-		tree.emplace_back(std::move(elements), std::move(cascades));
+template <typename E, typename O, typename CMP, uint64_t F, uint64_t C>
+void MergeSortTree<E, O, CMP, F, C>::BuildRun(idx_t level_idx, idx_t run_idx) {
+	//	Create each parent run by merging the child runs using a tournament tree
+	// 	https://en.wikipedia.org/wiki/K-way_merge_algorithm
+	const auto fanout = F;
+	const auto cascading = C;
+
+	auto &elements = tree[level_idx].first;
+	auto &cascades = tree[level_idx].second;
+	const auto &child_level = tree[level_idx - 1];
+	const auto count = elements.size();
+
+	idx_t child_run_length = 1;
+	auto run_length = child_run_length * fanout;
+	for (idx_t l = 1; l < level_idx; ++l) {
 		child_run_length = run_length;
+		run_length *= fanout;
+	}
+
+	const RunElement SENTINEL(MergeSortTraits<ElementType>::SENTINEL(), MergeSortTraits<idx_t>::SENTINEL());
+
+	//	Position markers for scanning the children.
+	using Bounds = pair<OffsetType, OffsetType>;
+	array<Bounds, fanout> bounds;
+	//	Start with first element of each (sorted) child run
+	RunElements players;
+	const auto child_base = run_idx * run_length;
+	for (idx_t child_run = 0; child_run < fanout; ++child_run) {
+		const auto child_idx = child_base + child_run * child_run_length;
+		bounds[child_run] = {OffsetType(MinValue<idx_t>(child_idx, count)),
+		                     OffsetType(MinValue<idx_t>(child_idx + child_run_length, count))};
+		if (bounds[child_run].first != bounds[child_run].second) {
+			players[child_run] = {child_level.first[child_idx], child_run};
+		} else {
+			//	Empty child
+			players[child_run] = SENTINEL;
+		}
 	}
+
+	//	Play the first round and extract the winner
+	Games games;
+	auto element_idx = child_base;
+	auto cascade_idx = fanout * run_idx * (run_length / cascading + 2);
+	auto winner = StartGames(games, players, SENTINEL);
+	while (winner != SENTINEL) {
+		// Add fractional cascading pointers
+		// if we are on a fraction boundary
+		if (!cascades.empty() && element_idx % cascading == 0) {
+			for (idx_t i = 0; i < fanout; ++i) {
+				cascades[cascade_idx++] = bounds[i].first;
+			}
+		}
+
+		//	Insert new winner element into the current run
+		elements[element_idx++] = winner.first;
+		const auto child_run = winner.second;
+		auto &child_idx = bounds[child_run].first;
+		++child_idx;
+
+		//	Move to the next entry in the child run (if any)
+		if (child_idx < bounds[child_run].second) {
+			winner = ReplayGames(games, child_run, {child_level.first[child_idx], child_run});
+		} else {
+			winner = ReplayGames(games, child_run, SENTINEL);
+		}
+	}
+
+	// Add terminal cascade pointers to the end
+	if (!cascades.empty()) {
+		for (idx_t j = 0; j < 2; ++j) {
+			for (idx_t i = 0; i < fanout; ++i) {
+				cascades[cascade_idx++] = bounds[i].first;
+			}
+		}
+	}
+
+	++build_complete;
 }
 
 template <typename E, typename O, typename CMP, uint64_t F, uint64_t C>

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/aggregate_object.hpp

@@ -27,7 +27,7 @@ struct AggregateObject { // NOLINT: work-around bug in clang-tidy
 	AggregateObject(AggregateFunction function, FunctionData *bind_data, idx_t child_count, idx_t payload_size,
 	                AggregateType aggr_type, PhysicalType return_type, Expression *filter = nullptr);
 	explicit AggregateObject(BoundAggregateExpression *aggr);
-	explicit AggregateObject(BoundWindowExpression &window);
+	explicit AggregateObject(const BoundWindowExpression &window);
 
 	FunctionData *GetFunctionData() const {
 		return bind_data_wrapper ? bind_data_wrapper->function_data.get() : nullptr;

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp

@@ -131,7 +131,7 @@ public:
 	}
 
 public:
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
 	//! Toggle multi-scan capability on a hash table, which prevents the scan of the aggregate from being destructive
 	//! If this is not toggled the GetData method will destroy the hash table as it is scanning it
 	static void SetMultiScan(GlobalSinkState &state);

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp

@@ -51,7 +51,7 @@ public:
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
 
 	//! Create a perfect aggregate hash table for this node
 	unique_ptr<PerfectAggregateHashTable> CreateHT(Allocator &allocator, ClientContext &context) const;

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_streaming_window.hpp

@@ -18,7 +18,7 @@ class PhysicalStreamingWindow : public PhysicalOperator {
 public:
 	static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::STREAMING_WINDOW;
 
-	static bool IsStreamingFunction(unique_ptr<Expression> &expr);
+	static bool IsStreamingFunction(ClientContext &context, unique_ptr<Expression> &expr);
 
 public:
 	PhysicalStreamingWindow(vector<LogicalType> types, vector<unique_ptr<Expression>> select_list,
@@ -35,11 +35,28 @@ public:
 	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
 	                           GlobalOperatorState &gstate, OperatorState &state) const override;
 
+	OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, GlobalOperatorState &gstate,
+	                                        OperatorState &state) const final;
+
+	bool RequiresFinalExecute() const final {
+		return true;
+	}
+
 	OrderPreservationType OperatorOrder() const override {
 		return OrderPreservationType::FIXED_ORDER;
 	}
 
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
+
+private:
+	void ExecuteFunctions(ExecutionContext &context, DataChunk &chunk, DataChunk &delayed,
+	                      GlobalOperatorState &gstate_p, OperatorState &state_p) const;
+	void ExecuteInput(ExecutionContext &context, DataChunk &delayed, DataChunk &input, DataChunk &chunk,
+	                  GlobalOperatorState &gstate, OperatorState &state) const;
+	void ExecuteDelayed(ExecutionContext &context, DataChunk &delayed, DataChunk &input, DataChunk &chunk,
+	                    GlobalOperatorState &gstate, OperatorState &state) const;
+	void ExecuteShifted(ExecutionContext &context, DataChunk &delayed, DataChunk &input, DataChunk &chunk,
+	                    GlobalOperatorState &gstate, OperatorState &state) const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp

@@ -51,7 +51,7 @@ public:
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
 
 	bool IsSink() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp

@@ -77,7 +77,7 @@ public:
 public:
 	idx_t MaxThreads(ClientContext &context);
 
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/ungrouped_aggregate_state.hpp

@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/aggregate/ungrouped_aggregate_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/expression_executor.hpp"
+#include "duckdb/function/aggregate_function.hpp"
+#include "duckdb/planner/expression/bound_aggregate_expression.hpp"
+
+namespace duckdb {
+struct DistinctAggregateData;
+struct LocalUngroupedAggregateState;
+
+struct UngroupedAggregateState {
+	explicit UngroupedAggregateState(const vector<unique_ptr<Expression>> &aggregate_expressions);
+	~UngroupedAggregateState();
+
+	void Move(UngroupedAggregateState &other);
+
+public:
+	//! Aggregates
+	const vector<unique_ptr<Expression>> &aggregate_expressions;
+	//! The aggregate values
+	vector<unsafe_unique_array<data_t>> aggregate_data;
+	//! The bind data
+	vector<optional_ptr<FunctionData>> bind_data;
+	//! The destructors
+	vector<aggregate_destructor_t> destructors;
+	//! Counts (used for verification)
+	unique_array<atomic<idx_t>> counts;
+};
+
+struct GlobalUngroupedAggregateState {
+public:
+	GlobalUngroupedAggregateState(Allocator &client_allocator, const vector<unique_ptr<Expression>> &aggregates)
+	    : client_allocator(client_allocator), allocator(client_allocator), state(aggregates) {
+	}
+
+	mutable mutex lock;
+	//! Client allocator
+	Allocator &client_allocator;
+	//! Global arena allocator
+	ArenaAllocator allocator;
+	//! Allocator pool
+	mutable vector<unique_ptr<ArenaAllocator>> stored_allocators;
+	//! The global aggregate state
+	UngroupedAggregateState state;
+
+public:
+	//! Create an ArenaAllocator with cross-thread lifetime
+	ArenaAllocator &CreateAllocator() const;
+	void Combine(LocalUngroupedAggregateState &other);
+	void CombineDistinct(LocalUngroupedAggregateState &other, DistinctAggregateData &distinct_data);
+	void Finalize(DataChunk &result);
+};
+
+struct LocalUngroupedAggregateState {
+public:
+	explicit LocalUngroupedAggregateState(GlobalUngroupedAggregateState &gstate);
+
+	//! Local arena allocator
+	ArenaAllocator &allocator;
+	//! The local aggregate state
+	UngroupedAggregateState state;
+
+public:
+	void Sink(DataChunk &payload_chunk, idx_t payload_idx, idx_t aggr_idx);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp

@@ -19,7 +19,7 @@ namespace duckdb {
 class CSVFileScan;
 class ScannerResult {
 public:
-	ScannerResult(CSVStates &states, CSVStateMachine &state_machine);
+	ScannerResult(CSVStates &states, CSVStateMachine &state_machine, idx_t result_size);
 
 	//! Adds a Value to the result
 	static inline void SetQuoted(ScannerResult &result, idx_t quoted_position) {
@@ -28,17 +28,36 @@ public:
 		}
 		result.quoted = true;
 	}
-	//! Adds a Row to the result
 	static inline void SetEscaped(ScannerResult &result) {
 		result.escaped = true;
 	}
-	// Variable to keep information regarding quoted and escaped values
+	static inline void SetComment(ScannerResult &result, idx_t buffer_pos) {
+		result.comment = true;
+	}
+	static inline bool UnsetComment(ScannerResult &result, idx_t buffer_pos) {
+		result.comment = false;
+		return false;
+	}
+	static inline bool IsCommentSet(const ScannerResult &result) {
+		return result.comment == true;
+	}
+
+	//! Variable to keep information regarding quoted and escaped values
 	bool quoted = false;
 	bool escaped = false;
+	//! Variable to keep track if we are in a comment row. Hence won't add it
+	bool comment = false;
 	idx_t quoted_position = 0;
 
+	//! Size of the result
+	const idx_t result_size;
+
 	CSVStateMachine &state_machine;
 
+	void Print() const {
+		state_machine.Print();
+	}
+
 protected:
 	CSVStates &states;
 };
@@ -66,16 +85,16 @@ public:
 
 	void SetIterator(const CSVIterator &it);
 
-	idx_t GetBoundaryIndex() {
+	idx_t GetBoundaryIndex() const {
 		return iterator.GetBoundaryIdx();
 	}
 
-	idx_t GetLinesRead() {
+	idx_t GetLinesRead() const {
 		return lines_read;
 	}
 
-	idx_t GetIteratorPosition() {
-		return iterator.pos.buffer_pos;
+	CSVPosition GetIteratorPosition() const {
+		return iterator.pos;
 	}
 
 	CSVStateMachine &GetStateMachine();
@@ -95,9 +114,13 @@ public:
 
 	bool ever_quoted = false;
 
+	//! Shared pointer to the buffer_manager, this is shared across multiple scanners
+	shared_ptr<CSVBufferManager> buffer_manager;
+
 	//! Skips Notes and/or parts of the data, starting from the top.
 	//! notes are dirty lines on top of the file, before the actual data
-	void SkipCSVRows(idx_t rows_to_skip);
+	static CSVIterator SkipCSVRows(shared_ptr<CSVBufferManager> buffer_manager,
+	                               const shared_ptr<CSVStateMachine> &state_machine, idx_t rows_to_skip);
 
 protected:
 	//! Boundaries of this scanner
@@ -110,9 +133,6 @@ protected:
 	//! Hold the current buffer ptr
 	char *buffer_handle_ptr = nullptr;
 
-	//! Shared pointer to the buffer_manager, this is shared across multiple scanners
-	shared_ptr<CSVBufferManager> buffer_manager;
-
 	//! If this scanner has been initialized
 	bool initialized = false;
 	//! How many lines were read by this scanner
@@ -122,7 +142,7 @@ protected:
 	//! Initializes the scanner
 	virtual void Initialize();
 
-	inline bool ContainsZeroByte(uint64_t v) {
+	inline static bool ContainsZeroByte(uint64_t v) {
 		return (v - UINT64_C(0x0101010101010101)) & ~(v)&UINT64_C(0x8080808080808080);
 	}
 
@@ -158,11 +178,20 @@ protected:
 					lines_read++;
 
 				} else if (states.states[0] != CSVState::CARRIAGE_RETURN) {
-					if (T::AddRow(result, iterator.pos.buffer_pos)) {
-						iterator.pos.buffer_pos++;
-						bytes_read = iterator.pos.buffer_pos - start_pos;
-						lines_read++;
-						return;
+					if (T::IsCommentSet(result)) {
+						if (T::UnsetComment(result, iterator.pos.buffer_pos)) {
+							iterator.pos.buffer_pos++;
+							bytes_read = iterator.pos.buffer_pos - start_pos;
+							lines_read++;
+							return;
+						}
+					} else {
+						if (T::AddRow(result, iterator.pos.buffer_pos)) {
+							iterator.pos.buffer_pos++;
+							bytes_read = iterator.pos.buffer_pos - start_pos;
+							lines_read++;
+							return;
+						}
 					}
 					lines_read++;
 				}
@@ -177,11 +206,20 @@ protected:
 						return;
 					}
 				} else if (states.states[0] != CSVState::CARRIAGE_RETURN) {
-					if (T::AddRow(result, iterator.pos.buffer_pos)) {
-						iterator.pos.buffer_pos++;
-						bytes_read = iterator.pos.buffer_pos - start_pos;
-						lines_read++;
-						return;
+					if (T::IsCommentSet(result)) {
+						if (T::UnsetComment(result, iterator.pos.buffer_pos)) {
+							iterator.pos.buffer_pos++;
+							bytes_read = iterator.pos.buffer_pos - start_pos;
+							lines_read++;
+							return;
+						}
+					} else {
+						if (T::AddRow(result, iterator.pos.buffer_pos)) {
+							iterator.pos.buffer_pos++;
+							bytes_read = iterator.pos.buffer_pos - start_pos;
+							lines_read++;
+							return;
+						}
 					}
 				}
 				iterator.pos.buffer_pos++;
@@ -225,7 +263,8 @@ protected:
 					    Load<uint64_t>(reinterpret_cast<const_data_ptr_t>(&buffer_handle_ptr[iterator.pos.buffer_pos]));
 					if (ContainsZeroByte((value ^ state_machine->transition_array.delimiter) &
 					                     (value ^ state_machine->transition_array.new_line) &
-					                     (value ^ state_machine->transition_array.carriage_return))) {
+					                     (value ^ state_machine->transition_array.carriage_return) &
+					                     (value ^ state_machine->transition_array.comment))) {
 						break;
 					}
 					iterator.pos.buffer_pos += 8;
@@ -241,6 +280,25 @@ protected:
 				T::QuotedNewLine(result);
 				iterator.pos.buffer_pos++;
 				break;
+			case CSVState::COMMENT: {
+				T::SetComment(result, iterator.pos.buffer_pos);
+				iterator.pos.buffer_pos++;
+				while (iterator.pos.buffer_pos + 8 < to_pos) {
+					uint64_t value =
+					    Load<uint64_t>(reinterpret_cast<const_data_ptr_t>(&buffer_handle_ptr[iterator.pos.buffer_pos]));
+					if (ContainsZeroByte((value ^ state_machine->transition_array.new_line) &
+					                     (value ^ state_machine->transition_array.carriage_return))) {
+						break;
+					}
+					iterator.pos.buffer_pos += 8;
+				}
+				while (state_machine->transition_array
+				           .skip_comment[static_cast<uint8_t>(buffer_handle_ptr[iterator.pos.buffer_pos])] &&
+				       iterator.pos.buffer_pos < to_pos - 1) {
+					iterator.pos.buffer_pos++;
+				}
+				break;
+			}
 			default:
 				iterator.pos.buffer_pos++;
 				break;