duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp

@@ -12,12 +12,15 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/execution/operator/csv_scanner/quote_rules.hpp"
 #include "duckdb/execution/operator/csv_scanner/column_count_scanner.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_schema.hpp"
+#include "duckdb/execution/operator/csv_scanner/header_value.hpp"
 
 namespace duckdb {
 struct DateTimestampSniffing {
 	bool initialized = false;
 	bool had_match = false;
 	vector<string> format;
+	idx_t initial_size;
 };
 //! Struct to store the result of the Sniffer
 struct SnifferResult {
@@ -30,6 +33,36 @@ struct SnifferResult {
 	vector<string> names;
 };
 
+//! All the options that will be used to sniff the dialect of the CSV file
+struct DialectCandidates {
+	//! The constructor populates all of our the options that will be used in our sniffer search space
+	explicit DialectCandidates(const CSVStateMachineOptions &options);
+
+	//! Static functions to get defaults of the search space
+	static vector<char> GetDefaultDelimiter();
+
+	static vector<vector<char>> GetDefaultQuote();
+
+	static vector<QuoteRule> GetDefaultQuoteRule();
+
+	static vector<vector<char>> GetDefaultEscape();
+
+	static vector<char> GetDefaultComment();
+
+	string Print();
+
+	//! Candidates for the delimiter
+	vector<char> delim_candidates;
+	//! Candidates for the comment
+	vector<char> comment_candidates;
+	//! Quote-Rule Candidates
+	vector<QuoteRule> quoterule_candidates;
+	//! Candidates for the quote option
+	unordered_map<uint8_t, vector<char>> quote_candidates_map;
+	//! Candidates for the escape option
+	unordered_map<uint8_t, vector<char>> escape_candidates_map;
+};
+
 //! This represents the data related to columns that have been set by the user
 //! e.g., from a copy command
 struct SetColumns {
@@ -46,12 +79,12 @@ struct SetColumns {
 	//! Column Names that were detected
 	const vector<string> *names = nullptr;
 	//! If columns are set
-	bool IsSet();
+	bool IsSet() const;
 	//! How many columns
-	idx_t Size();
+	idx_t Size() const;
 	//! Helper function that checks if candidate is acceptable based on the number of columns it produces
-	inline bool IsCandidateUnacceptable(idx_t num_cols, bool null_padding, bool ignore_errors,
-	                                    bool last_value_always_empty) {
+	inline bool IsCandidateUnacceptable(const idx_t num_cols, bool null_padding, bool ignore_errors,
+	                                    bool last_value_always_empty) const {
 		if (!IsSet() || ignore_errors) {
 			// We can't say its unacceptable if it's not set or if we ignore errors
 			return false;
@@ -75,24 +108,17 @@ struct SetColumns {
 	}
 };
 
-struct HeaderValue {
-	HeaderValue() : is_null(true) {
-	}
-	explicit HeaderValue(string_t value_p) {
-		value = value_p;
-	}
-	bool IsNull() {
-		return is_null;
-	}
-	bool is_null = false;
-	string_t value {};
+//! Struct used to know if we have a date or timestamp type already identified in this CSV File
+struct HasType {
+	bool date = false;
+	bool timestamp = false;
 };
 
 //! Sniffer that detects Header, Dialect and Types of CSV Files
 class CSVSniffer {
 public:
 	explicit CSVSniffer(CSVReaderOptions &options_p, shared_ptr<CSVBufferManager> buffer_manager_p,
-	                    CSVStateMachineCache &state_machine_cache, SetColumns set_columns = {});
+	                    CSVStateMachineCache &state_machine_cache, bool default_null_to_varchar = true);
 
 	//! Main method that sniffs the CSV file, returns the types, names and options as a result
 	//! CSV Sniffing consists of five steps:
@@ -103,8 +129,22 @@ public:
 	//! 5. Type Replacement: Replaces the types of the columns if the user specified them
 	SnifferResult SniffCSV(bool force_match = false);
 
+	//! I call it adaptive, since that's a sexier term.
+	//! In practice this Function that only sniffs the first two rows, to verify if a header exists and what are the
+	//! data types It does this considering a priorly set CSV schema. If there is a mismatch of the schema it runs the
+	//! full on blazing all guns sniffer, if that still fails it tells the user to union_by_name.
+	//! It returns the projection order.
+	SnifferResult AdaptiveSniff(CSVSchema &file_schema);
+
+	//! Function that only sniffs the first two rows, to verify if a header exists and what are the data types
+	SnifferResult MinimalSniff();
+
 	static NewLineIdentifier DetectNewLineDelimiter(CSVBufferManager &buffer_manager);
 
+	//! If a string_t value can be cast to a type
+	static bool CanYouCastIt(ClientContext &context, const string_t value, const LogicalType &type,
+	                         const DialectOptions &dialect_options, const bool is_null, const char decimal_separator);
+
 private:
 	//! CSV State Machine Cache
 	CSVStateMachineCache &state_machine_cache;
@@ -120,6 +160,7 @@ private:
 	SetColumns set_columns;
 	shared_ptr<CSVErrorHandler> error_handler;
 	shared_ptr<CSVErrorHandler> detection_error_handler;
+
 	//! Sets the result options
 	void SetResultOptions();
 
@@ -129,24 +170,14 @@ private:
 	//! First phase of auto detection: detect CSV dialect (i.e. delimiter, quote rules, etc)
 	void DetectDialect();
 	//! Functions called in the main DetectDialect(); function
-	//! 1. Generates the search space candidates for the dialect
-	void GenerateCandidateDetectionSearchSpace(vector<char> &delim_candidates, vector<QuoteRule> &quoterule_candidates,
-	                                           unordered_map<uint8_t, vector<char>> &quote_candidates_map,
-	                                           unordered_map<uint8_t, vector<char>> &escape_candidates_map);
-	//! 2. Generates the search space candidates for the state machines
+	//! 1. Generates the search space candidates for the state machines
 	void GenerateStateMachineSearchSpace(vector<unique_ptr<ColumnCountScanner>> &column_count_scanners,
-	                                     const vector<char> &delimiter_candidates,
-	                                     const vector<QuoteRule> &quoterule_candidates,
-	                                     const unordered_map<uint8_t, vector<char>> &quote_candidates_map,
-	                                     const unordered_map<uint8_t, vector<char>> &escape_candidates_map);
-	//! 2.1 If the user set a number of lines to skip, make sure we skip them
-	//! We skip from the first scanner and then apply information on the remaining scanners.
-	void SkipLines(vector<unique_ptr<ColumnCountScanner>> &csv_state_machines);
-
-	//! 3. Analyzes if dialect candidate is a good candidate to be considered, if so, it adds it to the candidates
+	                                     const DialectCandidates &dialect_candidates);
+
+	//! 2. Analyzes if dialect candidate is a good candidate to be considered, if so, it adds it to the candidates
 	void AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner>, idx_t &rows_read, idx_t &best_consistent_rows,
-	                             idx_t &prev_padding_count);
-	//! 4. Refine Candidates over remaining chunks
+	                             idx_t &prev_padding_count, idx_t &min_ignored_rows);
+	//! 3. Refine Candidates over remaining chunks
 	void RefineCandidates();
 
 	//! Checks if candidate still produces good values for the next chunk
@@ -155,22 +186,25 @@ private:
 	//! ------------------------------------------------------//
 	//! ------------------- Type Detection ------------------ //
 	//! ------------------------------------------------------//
-	//! Second phase of auto detection: detect types, format template candidates
+	//! Second phase of auto-detection: detect types, format template candidates
 	//! ordered by descending specificity (~ from high to low)
 	void DetectTypes();
 	//! Change the date format for the type to the string
 	//! Try to cast a string value to the specified sql type
-	void SetDateFormat(CSVStateMachine &candidate, const string &format_specifier, const LogicalTypeId &sql_type);
+	static void SetDateFormat(CSVStateMachine &candidate, const string &format_specifier,
+	                          const LogicalTypeId &sql_type);
 
 	//! Function that initialized the necessary variables used for date and timestamp detection
 	void InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, const string &separator,
 	                                         const LogicalType &sql_type);
+	//! Sets user defined date and time formats (if any)
+	void SetUserDefinedDateTimeFormat(CSVStateMachine &candidate);
 	//! Functions that performs detection for date and timestamp formats
 	void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator,
-	                                   string_t &dummy_val);
-	//! If a string_t value can be cast to a type
-	bool CanYouCastIt(const string_t value, const LogicalType &type, const DialectOptions &dialect_options,
-	                  const bool is_null, const char decimal_separator);
+	                                   const string_t &dummy_val);
+	//! Sniffs the types from a data chunk
+	void SniffTypes(DataChunk &data_chunk, CSVStateMachine &state_machine,
+	                unordered_map<idx_t, vector<LogicalType>> &info_sql_types_candidates, idx_t start_idx_detection);
 
 	//! Variables for Type Detection
 	//! Format Candidates for Date and Timestamp Types
@@ -194,11 +228,19 @@ private:
 	void RefineTypes();
 	bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type);
 	vector<LogicalType> detected_types;
+	//! If when finding a SQLNULL type in type detection we default it to varchar
+	const bool default_null_to_varchar;
 	//! ------------------------------------------------------//
 	//! ------------------ Header Detection ----------------- //
 	//! ------------------------------------------------------//
 	void DetectHeader();
-	bool DetectHeaderWithSetColumn();
+	static bool DetectHeaderWithSetColumn(ClientContext &context, vector<HeaderValue> &best_header_row,
+	                                      SetColumns &set_columns, CSVReaderOptions &options);
+	static vector<string>
+	DetectHeaderInternal(ClientContext &context, vector<HeaderValue> &best_header_row, CSVStateMachine &state_machine,
+	                     SetColumns &set_columns,
+	                     unordered_map<idx_t, vector<LogicalType>> &best_sql_types_candidates_per_column_idx,
+	                     CSVReaderOptions &options, CSVErrorHandler &error_handler);
 	vector<string> names;
 
 	//! ------------------------------------------------------//

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state.hpp

@@ -24,7 +24,8 @@ enum class CSVState : uint8_t {
 	INVALID = 7,          //! Got to an Invalid State, this should error.
 	NOT_SET = 8,          //! If the state is not set, usually the first state before getting the first character
 	QUOTED_NEW_LINE = 9,  //! If we have a quoted newline
-	EMPTY_SPACE = 10      //! If we have empty spaces in the beginning and end of value
+	EMPTY_SPACE = 10,     //! If we have empty spaces in the beginning and end of value
+	COMMENT = 11          //! If we are in a comment state, and hence have to skip the whole line
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine.hpp

@@ -31,10 +31,15 @@ struct CSVStates {
 		       (states[1] == CSVState::RECORD_SEPARATOR || states[1] == CSVState::CARRIAGE_RETURN);
 	}
 
+	inline bool WasStandard() {
+		return states[0] == CSVState::STANDARD;
+	}
+
 	inline bool EmptyLastValue() {
 		// It is a new row, if the previous state is not a record separator, and the current one is
 		return states[0] == CSVState::DELIMITER &&
-		       (states[1] == CSVState::RECORD_SEPARATOR || states[1] == CSVState::CARRIAGE_RETURN);
+		       (states[1] == CSVState::RECORD_SEPARATOR || states[1] == CSVState::CARRIAGE_RETURN ||
+		        states[1] == CSVState::DELIMITER);
 	}
 
 	inline bool EmptyLine() {
@@ -46,6 +51,10 @@ struct CSVStates {
 		return states[1] == CSVState::NOT_SET;
 	}
 
+	inline bool IsComment() {
+		return states[1] == CSVState::COMMENT;
+	}
+
 	inline bool IsCurrentNewRow() {
 		return states[1] == CSVState::RECORD_SEPARATOR || states[1] == CSVState::CARRIAGE_RETURN;
 	}
@@ -89,6 +98,14 @@ public:
 		states.states[1] = transition_array[static_cast<uint8_t>(current_char)][static_cast<uint8_t>(states.states[1])];
 	}
 
+	void Print() {
+		std::cout << "State Machine Options" << '\n';
+		std::cout << "Delim: " << state_machine_options.delimiter.GetValue() << '\n';
+		std::cout << "Quote: " << state_machine_options.quote.GetValue() << '\n';
+		std::cout << "Escape: " << state_machine_options.escape.GetValue() << '\n';
+		std::cout << "Comment: " << state_machine_options.comment.GetValue() << '\n';
+		std::cout << "---------------------" << '\n';
+	}
 	//! The Transition Array is a Finite State Machine
 	//! It holds the transitions of all states, on all 256 possible different characters
 	const StateMachine &transition_array;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp

@@ -19,7 +19,7 @@ namespace duckdb {
 //! Class to wrap the state machine matrix
 class StateMachine {
 public:
-	static constexpr uint32_t NUM_STATES = 11;
+	static constexpr uint32_t NUM_STATES = 13;
 	static constexpr uint32_t NUM_TRANSITIONS = 256;
 	CSVState state_machine[NUM_TRANSITIONS][NUM_STATES];
 	//! Transitions where we might skip processing
@@ -27,12 +27,16 @@ public:
 	bool skip_standard[256];
 	//! For the Quoted State
 	bool skip_quoted[256];
+	//! For the Comment State
+	bool skip_comment[256];
 
 	uint64_t delimiter = 0;
 	uint64_t new_line = 0;
 	uint64_t carriage_return = 0;
 	uint64_t quote = 0;
 	uint64_t escape = 0;
+	uint64_t comment = 0;
+
 	const CSVState *operator[](idx_t i) const {
 		return state_machine[i];
 	}
@@ -48,8 +52,9 @@ struct HashCSVStateMachineConfig {
 		auto h_delimiter = Hash(config.delimiter.GetValue());
 		auto h_quote = Hash(config.quote.GetValue());
 		auto h_escape = Hash(config.escape.GetValue());
-		auto h_newline = Hash((uint8_t)config.new_line.GetValue());
-		return CombineHash(h_delimiter, CombineHash(h_quote, CombineHash(h_escape, h_newline)));
+		auto h_newline = Hash(static_cast<uint8_t>(config.new_line.GetValue()));
+		auto h_comment = Hash(static_cast<uint8_t>(config.comment.GetValue()));
+		return CombineHash(h_delimiter, CombineHash(h_quote, CombineHash(h_escape, CombineHash(h_newline, h_comment))));
 	}
 };
 
@@ -81,10 +86,7 @@ private:
 	//! Cache on delimiter|quote|escape|newline
 	unordered_map<CSVStateMachineOptions, StateMachine, HashCSVStateMachineConfig> state_machine_cache;
 	//! Default value for options used to intialize CSV State Machine Cache
-	const vector<char> default_delimiter = {',', '|', ';', '\t'};
-	const vector<vector<char>> default_quote = {{'\"'}, {'\"', '\''}, {'\0'}};
-	const vector<QuoteRule> default_quote_rule = {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
-	const vector<vector<char>> default_escape = {{'\0', '\"', '\''}, {'\\'}, {'\0'}};
+
 	//! Because the state machine cache can be accessed in Parallel we need a mutex.
 	mutex main_mutex;
 };

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp

@@ -20,7 +20,6 @@ namespace duckdb {
 
 //! CSV Global State is used in the CSV Reader Table Function, it controls what each thread
 struct CSVGlobalState : public GlobalTableFunctionState {
-public:
 	CSVGlobalState(ClientContext &context, const shared_ptr<CSVBufferManager> &buffer_manager_p,
 	               const CSVReaderOptions &options, idx_t system_threads_p, const vector<string> &files,
 	               vector<column_t> column_ids_p, const ReadCSVData &bind_data);
@@ -41,8 +40,8 @@ public:
 
 	//! Calculates the Max Threads that will be used by this CSV Reader
 	idx_t MaxThreads() const override;
-	//! We hold information on the current scanner boundary
-	CSVIterator current_boundary;
+
+	bool IsDone() const;
 
 private:
 	//! Reference to the client context that created this scan
@@ -67,7 +66,7 @@ private:
 
 	const ReadCSVData &bind_data;
 
-	vector<LogicalType> file_schema;
+	CSVSchema file_schema;
 
 	bool single_threaded = false;
 
@@ -77,6 +76,8 @@ private:
 	shared_ptr<CSVBufferUsage> current_buffer_in_use;
 
 	unordered_map<idx_t, idx_t> threads_per_file;
+	//! We hold information on the current scanner boundary
+	CSVIterator current_boundary;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/header_value.hpp

@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/header_value.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types/string_type.hpp"
+
+namespace duckdb {
+struct HeaderValue {
+	HeaderValue() : is_null(true) {
+	}
+	explicit HeaderValue(const string_t value_p) {
+		value = value_p.GetString();
+	}
+	bool IsNull() {
+		return is_null;
+	}
+	bool is_null = false;
+	string value;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner_boundary.hpp

@@ -25,11 +25,9 @@ namespace duckdb {
 
 //! Information stored in the buffer
 struct CSVBoundary {
-	CSVBoundary(idx_t file_idx, idx_t buffer_idx, idx_t buffer_pos, idx_t boundary_idx, idx_t end_pos);
+	CSVBoundary(idx_t buffer_idx, idx_t buffer_pos, idx_t boundary_idx, idx_t end_pos);
 	CSVBoundary();
 	void Print();
-	//! File index where we start scanning [0-idx], a scanner can never go over one file.
-	idx_t file_idx = 0;
 	//! Start Buffer index of the file where we start scanning
 	idx_t buffer_idx = 0;
 	//! Start Buffer position of the buffer of the file where we start scanning
@@ -43,10 +41,8 @@ struct CSVBoundary {
 };
 
 struct CSVPosition {
-	CSVPosition(idx_t file_idx, idx_t buffer_idx, idx_t buffer_pos);
+	CSVPosition(idx_t buffer_idx, idx_t buffer_pos);
 	CSVPosition();
-	//! File index where we start scanning [0-idx], a scanner can never go over one file.
-	idx_t file_idx = 0;
 	//! Start Buffer index of the file where we start scanning
 	idx_t buffer_idx = 0;
 	//! Start Buffer position of the buffer of the file where we start scanning
@@ -55,8 +51,6 @@ struct CSVPosition {
 };
 struct CSVIterator {
 public:
-	CSVIterator(idx_t file_idx, idx_t buffer_idx, idx_t buffer_pos, idx_t boundary_idx, idx_t buffer_size);
-
 	CSVIterator();
 
 	void Print();
@@ -68,12 +62,13 @@ public:
 
 	//! Getters
 	idx_t GetEndPos() const;
-	idx_t GetFileIdx() const;
 	idx_t GetBufferIdx() const;
 	idx_t GetBoundaryIdx() const;
 
 	void SetCurrentPositionToBoundary();
 
+	void SetCurrentBoundaryToPosition(bool single_threaded);
+
 	void SetStart(idx_t pos);
 
 	//! 8 MB TODO: Should benchmarks other values
@@ -83,6 +78,8 @@ public:
 
 	bool done = false;
 
+	bool first_one = true;
+
 private:
 	//! The original setting
 	CSVBoundary boundary;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/skip_scanner.hpp

@@ -33,6 +33,9 @@ public:
 	static inline bool EmptyLine(SkipResult &result, const idx_t buffer_pos);
 	//! Handles QuotedNewline State
 	static inline void QuotedNewLine(SkipResult &result);
+	//! Code to unset comment
+	static inline bool UnsetComment(SkipResult &result, idx_t buffer_pos);
+	//! Internal code to add a row
 	inline void InternalAddRow();
 };
 

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp

@@ -15,8 +15,8 @@ namespace duckdb {
 //! Basically which char, quote and escape were used to generate it.
 struct CSVStateMachineOptions {
 	CSVStateMachineOptions() {};
-	CSVStateMachineOptions(char delimiter_p, char quote_p, char escape_p, NewLineIdentifier new_line_p)
-	    : delimiter(delimiter_p), quote(quote_p), escape(escape_p), new_line(new_line_p) {};
+	CSVStateMachineOptions(char delimiter_p, char quote_p, char escape_p, char comment_p, NewLineIdentifier new_line_p)
+	    : delimiter(delimiter_p), quote(quote_p), escape(escape_p), comment(comment_p), new_line(new_line_p) {};
 
 	//! Delimiter to separate columns within each line
 	CSVOption<char> delimiter = ',';
@@ -24,12 +24,14 @@ struct CSVStateMachineOptions {
 	CSVOption<char> quote = '\"';
 	//! Escape character to escape quote character
 	CSVOption<char> escape = '\0';
+	//! Comment character to skip a line
+	CSVOption<char> comment = '\0';
 	//! New Line separator
 	CSVOption<NewLineIdentifier> new_line = NewLineIdentifier::NOT_SET;
 
 	bool operator==(const CSVStateMachineOptions &other) const {
 		return delimiter == other.delimiter && quote == other.quote && escape == other.escape &&
-		       new_line == other.new_line;
+		       new_line == other.new_line && comment == other.comment;
 	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp

@@ -36,7 +36,7 @@ public:
 	    : buffer_pos(buffer_pos_p), buffer_size(buffer_size_p), buffer_idx(buffer_idx_p) {
 	}
 
-	idx_t operator-(const LinePosition &other) {
+	idx_t operator-(const LinePosition &other) const {
 		if (other.buffer_idx == buffer_idx) {
 			return buffer_pos - other.buffer_pos;
 		}
@@ -47,7 +47,7 @@ public:
 		return buffer_pos == other.buffer_pos && buffer_idx == other.buffer_idx && buffer_size == other.buffer_size;
 	}
 
-	idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl = false) {
+	idx_t GetGlobalPosition(idx_t requested_buffer_size, bool first_char_nl = false) const {
 		return requested_buffer_size * buffer_idx + buffer_pos + first_char_nl;
 	}
 	idx_t buffer_pos = 0;
@@ -64,14 +64,15 @@ public:
 
 	//! Reconstructs the current line to be used in error messages
 	string ReconstructCurrentLine(bool &first_char_nl,
-	                              unordered_map<idx_t, shared_ptr<CSVBufferHandle>> &buffer_handles);
+	                              unordered_map<idx_t, shared_ptr<CSVBufferHandle>> &buffer_handles,
+	                              bool reconstruct_line) const;
 };
 
 class StringValueResult;
 
 class CurrentError {
 public:
-	CurrentError(CSVErrorType type, idx_t col_idx_p, idx_t chunk_idx_p, LinePosition error_position_p,
+	CurrentError(CSVErrorType type, idx_t col_idx_p, idx_t chunk_idx_p, const LinePosition &error_position_p,
 	             idx_t current_line_size_p)
 	    : type(type), col_idx(col_idx_p), chunk_idx(chunk_idx_p), current_line_size(current_line_size_p),
 	      error_position(error_position_p) {};
@@ -120,7 +121,7 @@ public:
 		current_errors.back().error_message = std::move(error_message);
 	}
 
-	bool HasErrorType(CSVErrorType type) {
+	bool HasErrorType(CSVErrorType type) const {
 		for (auto &error : current_errors) {
 			if (type == error.type) {
 				return true;
@@ -139,7 +140,7 @@ private:
 
 struct ParseTypeInfo {
 	ParseTypeInfo() {};
-	ParseTypeInfo(LogicalType &type, bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
+	ParseTypeInfo(const LogicalType &type, bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
 		type_id = type.id();
 		internal_type = type.InternalType();
 		if (type.id() == LogicalTypeId::DECIMAL) {
@@ -158,9 +159,9 @@ class StringValueResult : public ScannerResult {
 public:
 	StringValueResult(CSVStates &states, CSVStateMachine &state_machine,
 	                  const shared_ptr<CSVBufferHandle> &buffer_handle, Allocator &buffer_allocator,
-	                  bool figure_out_new_line, idx_t buffer_position, CSVErrorHandler &error_handler,
-	                  CSVIterator &iterator, bool store_line_size, shared_ptr<CSVFileScan> csv_file_scan,
-	                  idx_t &lines_read, bool sniffing);
+	                  idx_t result_size_p, idx_t buffer_position, CSVErrorHandler &error_handler, CSVIterator &iterator,
+	                  bool store_line_size, shared_ptr<CSVFileScan> csv_file_scan, idx_t &lines_read, bool sniffing,
+	                  string path);
 
 	~StringValueResult();
 
@@ -172,6 +173,7 @@ public:
 	LinePosition last_position;
 	char *buffer_ptr;
 	idx_t buffer_size;
+	idx_t position_before_comment;
 
 	//! CSV Options that impact the parsing
 	const uint32_t number_of_columns;
@@ -186,8 +188,7 @@ public:
 	DataChunk parse_chunk;
 	idx_t number_of_rows = 0;
 	idx_t cur_col_id = 0;
-	bool figure_out_new_line;
-	idx_t result_size;
+	bool figure_out_new_line = false;
 	//! Information to properly handle errors
 	CSVErrorHandler &error_handler;
 	CSVIterator &iterator;
@@ -225,6 +226,12 @@ public:
 	//! We store borked rows so we can generate multiple errors during flushing
 	unordered_set<idx_t> borked_rows;
 
+	const string path;
+
+	//! Variable used when trying to figure out where a new segment starts, we must always start from a Valid
+	//! (i.e., non-comment) line.
+	bool first_line_is_comment = false;
+
 	//! Specialized code for quoted values, makes sure to remove quotes and escapes
 	static inline void AddQuotedValue(StringValueResult &result, const idx_t buffer_pos);
 	//! Adds a Value to the result
@@ -235,7 +242,7 @@ public:
 	static inline void InvalidState(StringValueResult &result);
 	//! Handles QuotedNewline State
 	static inline void QuotedNewLine(StringValueResult &result);
-	void NullPaddingQuotedNewlineCheck();
+	void NullPaddingQuotedNewlineCheck() const;
 	//! Handles EmptyLine states
 	static inline bool EmptyLine(StringValueResult &result, const idx_t buffer_pos);
 	inline bool AddRowInternal();
@@ -243,10 +250,21 @@ public:
 	void HandleUnicodeError(idx_t col_idx, LinePosition &error_position);
 	bool HandleTooManyColumnsError(const char *value_ptr, const idx_t size);
 	inline void AddValueToVector(const char *value_ptr, const idx_t size, bool allocate = false);
+	static inline void SetComment(StringValueResult &result, idx_t buffer_pos);
+	static inline bool UnsetComment(StringValueResult &result, idx_t buffer_pos);
 
 	DataChunk &ToChunk();
 	//! Resets the state of the result
 	void Reset();
+
+	//! BOM skipping (https://en.wikipedia.org/wiki/Byte_order_mark)
+	void SkipBOM() const;
+	//! If we should Print Error Lines
+	//! We only really care about error lines if we are going to error or store them in a rejects table
+	bool PrintErrorLine() const;
+	//! Removes last added line, usually because we figured out later on that it's an ill-formed line
+	//! or that it does not fit our schema
+	void RemoveLastLine();
 };
 
 //! Our dialect scanner basically goes over the CSV and actually parses the values to a DuckDB vector of string_t
@@ -255,11 +273,13 @@ public:
 	StringValueScanner(idx_t scanner_idx, const shared_ptr<CSVBufferManager> &buffer_manager,
 	                   const shared_ptr<CSVStateMachine> &state_machine,
 	                   const shared_ptr<CSVErrorHandler> &error_handler, const shared_ptr<CSVFileScan> &csv_file_scan,
-	                   bool sniffing = false, CSVIterator boundary = {}, bool figure_out_nl = false);
+	                   bool sniffing = false, const CSVIterator &boundary = {},
+	                   idx_t result_size = STANDARD_VECTOR_SIZE);
 
 	StringValueScanner(const shared_ptr<CSVBufferManager> &buffer_manager,
 	                   const shared_ptr<CSVStateMachine> &state_machine,
-	                   const shared_ptr<CSVErrorHandler> &error_handler);
+	                   const shared_ptr<CSVErrorHandler> &error_handler, idx_t result_size = STANDARD_VECTOR_SIZE,
+	                   const CSVIterator &boundary = {});
 
 	StringValueResult &ParseChunk() override;
 
@@ -269,13 +289,13 @@ public:
 	//! Function that creates and returns a non-boundary CSV Scanner, can be used for internal csv reading.
 	static unique_ptr<StringValueScanner> GetCSVScanner(ClientContext &context, CSVReaderOptions &options);
 
-	bool FinishedIterator();
+	bool FinishedIterator() const;
 
 	//! Creates a new string with all escaped values removed
 	static string_t RemoveEscape(const char *str_ptr, idx_t end, char escape, Vector &vector);
 
 	//! If we can directly cast the type when consuming the CSV file, or we have to do it later
-	static bool CanDirectlyCast(const LogicalType &type);
+	static bool CanDirectlyCast(const LogicalType &type, bool icu_loaded);
 
 	const idx_t scanner_idx;
 
@@ -294,9 +314,6 @@ private:
 	//! Function used to move from one buffer to the other, if necessary
 	bool MoveToNextBuffer();
 
-	//! BOM skipping (https://en.wikipedia.org/wiki/Byte_order_mark)
-	void SkipBOM();
-
 	void SkipUntilNewLine();
 
 	void SetStart();

package/src/duckdb/src/include/duckdb/execution/operator/filter/physical_filter.hpp

@@ -33,7 +33,7 @@ public:
 		return true;
 	}
 
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
 
 protected:
 	OperatorResultType ExecuteInternal(ExecutionContext &context, DataChunk &input, DataChunk &chunk,