duckdb 0.10.1-dev9.0 → 0.10.2-dev0.0

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp

@@ -0,0 +1,166 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_reader_options.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_buffer.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_option.hpp"
+#include "duckdb/execution/operator/csv_scanner/state_machine_options.hpp"
+#include "duckdb/common/map.hpp"
+#include "duckdb/function/scalar/strftime_format.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/types.hpp"
+#include "duckdb/common/multi_file_reader_options.hpp"
+
+namespace duckdb {
+
+struct DialectOptions {
+	CSVStateMachineOptions state_machine_options;
+	//! Expected number of columns
+	idx_t num_cols = 0;
+	//! Whether or not the file has a header line
+	CSVOption<bool> header = false;
+	//! The date format to use (if any is specified)
+	map<LogicalTypeId, CSVOption<StrpTimeFormat>> date_format = {{LogicalTypeId::DATE, {}},
+	                                                             {LogicalTypeId::TIMESTAMP, {}}};
+	//! How many leading rows to skip
+	CSVOption<idx_t> skip_rows = 0;
+};
+
+struct CSVReaderOptions {
+	//===--------------------------------------------------------------------===//
+	// CommonCSVOptions
+	//===--------------------------------------------------------------------===//
+	//! See struct above.
+	DialectOptions dialect_options;
+	//! Whether or not we should ignore InvalidInput errors
+	bool ignore_errors = false;
+	//! Rejects table name
+	string rejects_table_name;
+	//! Rejects table entry limit (0 = no limit)
+	idx_t rejects_limit = 0;
+	//! Columns to use as recovery key for rejected rows when reading with ignore_errors = true
+	vector<string> rejects_recovery_columns;
+	//! Index of the recovery columns
+	vector<idx_t> rejects_recovery_column_ids;
+	//! Number of samples to buffer
+	idx_t buffer_sample_size = (idx_t)STANDARD_VECTOR_SIZE * 50;
+	//! Specifies the string that represents a null value
+	string null_str;
+	//! Whether file is compressed or not, and if so which compression type
+	//! AUTO_DETECT (default; infer from file extension)
+	FileCompressionType compression = FileCompressionType::AUTO_DETECT;
+	//! Option to convert quoted values to NULL values
+	bool allow_quoted_nulls = true;
+
+	//===--------------------------------------------------------------------===//
+	// CSVAutoOptions
+	//===--------------------------------------------------------------------===//
+	//! SQL Type list mapping of name to SQL type index in sql_type_list
+	case_insensitive_map_t<idx_t> sql_types_per_column;
+	//! User-defined SQL type list
+	vector<LogicalType> sql_type_list;
+	//! User-defined name list
+	vector<string> name_list;
+	//! Types considered as candidates for auto detection ordered by descending specificity (~ from high to low)
+	vector<LogicalType> auto_type_candidates = {LogicalType::VARCHAR, LogicalType::TIMESTAMP, LogicalType::DATE,
+	                                            LogicalType::TIME,    LogicalType::DOUBLE,    LogicalType::BIGINT,
+	                                            LogicalType::BOOLEAN, LogicalType::SQLNULL};
+	//! In case the sniffer found a mismatch error from user defined types or dialect
+	string sniffer_user_mismatch_error;
+	//! In case the sniffer found a mismatch error from user defined types or dialect
+	vector<bool> was_type_manually_set;
+	//===--------------------------------------------------------------------===//
+	// ReadCSVOptions
+	//===--------------------------------------------------------------------===//
+	//! Maximum CSV line size: specified because if we reach this amount, we likely have wrong delimiters (default: 2MB)
+	//! note that this is the guaranteed line length that will succeed, longer lines may be accepted if slightly above
+	idx_t maximum_line_size = 2097152;
+	//! Whether or not header names shall be normalized
+	bool normalize_names = false;
+	//! True, if column with that index must skip null check
+	vector<bool> force_not_null;
+	//! Number of sample chunks used in auto-detection
+	idx_t sample_size_chunks = 20480 / STANDARD_VECTOR_SIZE;
+	//! Consider all columns to be of type varchar
+	bool all_varchar = false;
+	//! Whether or not to automatically detect dialect and datatypes
+	bool auto_detect = true;
+	//! The file path of the CSV file to read
+	string file_path;
+	//! Multi-file reader options
+	MultiFileReaderOptions file_options;
+	//! Buffer Size (Parallel Scan)
+	idx_t buffer_size = CSVBuffer::CSV_BUFFER_SIZE;
+	//! Decimal separator when reading as numeric
+	string decimal_separator = ".";
+	//! Whether or not to pad rows that do not have enough columns with NULL values
+	bool null_padding = false;
+	//! If we should attempt to run parallel scanning over one file
+	bool parallel = true;
+
+	//! User defined parameters for the csv function concatenated on a string
+	string user_defined_parameters;
+	//===--------------------------------------------------------------------===//
+	// WriteCSVOptions
+	//===--------------------------------------------------------------------===//
+	//! True, if column with that index must be quoted
+	vector<bool> force_quote;
+	//! Prefix/suffix/custom newline the entire file once (enables writing of files as JSON arrays)
+	string prefix;
+	string suffix;
+	string write_newline;
+
+	//! The date format to use (if any is specified)
+	map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
+	//! The date format to use for writing (if any is specified)
+	map<LogicalTypeId, StrfTimeFormat> write_date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
+	//! Whether or not a type format is specified
+	map<LogicalTypeId, bool> has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}};
+
+	void Serialize(Serializer &serializer) const;
+	static CSVReaderOptions Deserialize(Deserializer &deserializer);
+
+	void SetCompression(const string &compression);
+
+	bool GetHeader() const;
+	void SetHeader(bool has_header);
+
+	string GetEscape() const;
+	void SetEscape(const string &escape);
+
+	int64_t GetSkipRows() const;
+	void SetSkipRows(int64_t rows);
+
+	string GetQuote() const;
+	void SetQuote(const string &quote);
+	void SetDelimiter(const string &delimiter);
+	string GetDelimiter() const;
+
+	NewLineIdentifier GetNewline() const;
+	void SetNewline(const string &input);
+	//! Set an option that is supported by both reading and writing functions, called by
+	//! the SetReadOption and SetWriteOption methods
+	bool SetBaseOption(const string &loption, const Value &value);
+
+	//! loption - lowercase string
+	//! set - argument(s) to the option
+	//! expected_names - names expected if the option is "columns"
+	void SetReadOption(const string &loption, const Value &value, vector<string> &expected_names);
+	void SetWriteOption(const string &loption, const Value &value);
+	void SetDateFormat(LogicalTypeId type, const string &format, bool read_format);
+	void ToNamedParameters(named_parameter_map_t &out);
+	void FromNamedParameters(named_parameter_map_t &in, ClientContext &context, vector<LogicalType> &return_types,
+	                         vector<string> &names);
+
+	string ToString() const;
+	//! If the type for column with idx i was manually set
+	bool WasTypeManuallySet(idx_t i) const;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp

@@ -0,0 +1,191 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_sniffer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
+#include "duckdb/common/vector.hpp"
+#include "duckdb/execution/operator/csv_scanner/quote_rules.hpp"
+#include "duckdb/execution/operator/csv_scanner/column_count_scanner.hpp"
+
+namespace duckdb {
+struct DateTimestampSniffing {
+	bool initialized = false;
+	vector<string> format;
+};
+//! Struct to store the result of the Sniffer
+struct SnifferResult {
+	SnifferResult(vector<LogicalType> return_types_p, vector<string> names_p)
+	    : return_types(std::move(return_types_p)), names(std::move(names_p)) {
+	}
+	//! Return Types that were detected
+	vector<LogicalType> return_types;
+	//! Column Names that were detected
+	vector<string> names;
+};
+
+//! This represents the data related to columns that have been set by the user
+//! e.g., from a copy command
+struct SetColumns {
+	SetColumns(const vector<LogicalType> *types_p, const vector<string> *names_p) : types(types_p), names(names_p) {
+		if (!types) {
+			D_ASSERT(!types && !names);
+		} else {
+			D_ASSERT(types->size() == names->size());
+		}
+	}
+	SetColumns() {};
+	//! Return Types that were detected
+	const vector<LogicalType> *types = nullptr;
+	//! Column Names that were detected
+	const vector<string> *names = nullptr;
+	//! If columns are set
+	bool IsSet();
+	//! How many columns
+	idx_t Size();
+	//! Helper function that checks if candidate is acceptable based on the number of columns it produces
+	inline bool IsCandidateUnacceptable(idx_t num_cols, bool null_padding, bool ignore_errors,
+	                                    bool last_value_always_empty) {
+		if (!IsSet() || ignore_errors) {
+			// We can't say its unacceptable if it's not set or if we ignore errors
+			return false;
+		}
+		idx_t size = Size();
+		// If the columns are set and there is a mismatch with the expected number of columns, with null_padding and
+		// ignore_errors not set, we don't have a suitable candidate.
+		// Note that we compare with max_columns_found + 1, because some broken files have the behaviour where two
+		// columns are represented as: | col 1 | col_2 |
+		if (num_cols == size || num_cols == size + last_value_always_empty) {
+			// Good Candidate
+			return false;
+		}
+		// if we detected more columns than we have set, it's all good because we can null-pad them
+		if (null_padding && num_cols > size) {
+			return false;
+		}
+
+		// Unacceptable
+		return true;
+	}
+};
+
+//! Sniffer that detects Header, Dialect and Types of CSV Files
+class CSVSniffer {
+public:
+	explicit CSVSniffer(CSVReaderOptions &options_p, shared_ptr<CSVBufferManager> buffer_manager_p,
+	                    CSVStateMachineCache &state_machine_cache, SetColumns set_columns = {});
+
+	//! Main method that sniffs the CSV file, returns the types, names and options as a result
+	//! CSV Sniffing consists of five steps:
+	//! 1. Dialect Detection: Generate the CSV Options (delimiter, quote, escape, etc.)
+	//! 2. Type Detection: Figures out the types of the columns (For one chunk)
+	//! 3. Type Refinement: Refines the types of the columns for the remaining chunks
+	//! 4. Header Detection: Figures out if  the CSV file has a header and produces the names of the columns
+	//! 5. Type Replacement: Replaces the types of the columns if the user specified them
+	SnifferResult SniffCSV(bool force_match = false);
+
+	static NewLineIdentifier DetectNewLineDelimiter(CSVBufferManager &buffer_manager);
+
+private:
+	//! CSV State Machine Cache
+	CSVStateMachineCache &state_machine_cache;
+	//! Highest number of columns found
+	idx_t max_columns_found = 0;
+	//! Current Candidates being considered
+	vector<unique_ptr<ColumnCountScanner>> candidates;
+	//! Reference to original CSV Options, it will be modified as a result of the sniffer.
+	CSVReaderOptions &options;
+	//! Buffer being used on sniffer
+	shared_ptr<CSVBufferManager> buffer_manager;
+	//! Information regarding columns that were set by user/query
+	SetColumns set_columns;
+	shared_ptr<CSVErrorHandler> error_handler;
+	shared_ptr<CSVErrorHandler> detection_error_handler;
+	//! Sets the result options
+	void SetResultOptions();
+
+	//! ------------------------------------------------------//
+	//! ----------------- Dialect Detection ----------------- //
+	//! ------------------------------------------------------//
+	//! First phase of auto detection: detect CSV dialect (i.e. delimiter, quote rules, etc)
+	void DetectDialect();
+	//! Functions called in the main DetectDialect(); function
+	//! 1. Generates the search space candidates for the dialect
+	void GenerateCandidateDetectionSearchSpace(vector<char> &delim_candidates, vector<QuoteRule> &quoterule_candidates,
+	                                           unordered_map<uint8_t, vector<char>> &quote_candidates_map,
+	                                           unordered_map<uint8_t, vector<char>> &escape_candidates_map);
+	//! 2. Generates the search space candidates for the state machines
+	void GenerateStateMachineSearchSpace(vector<unique_ptr<ColumnCountScanner>> &column_count_scanners,
+	                                     const vector<char> &delimiter_candidates,
+	                                     const vector<QuoteRule> &quoterule_candidates,
+	                                     const unordered_map<uint8_t, vector<char>> &quote_candidates_map,
+	                                     const unordered_map<uint8_t, vector<char>> &escape_candidates_map);
+	//! 3. Analyzes if dialect candidate is a good candidate to be considered, if so, it adds it to the candidates
+	void AnalyzeDialectCandidate(unique_ptr<ColumnCountScanner>, idx_t &rows_read, idx_t &best_consistent_rows,
+	                             idx_t &prev_padding_count);
+	//! 4. Refine Candidates over remaining chunks
+	void RefineCandidates();
+
+	//! Checks if candidate still produces good values for the next chunk
+	bool RefineCandidateNextChunk(ColumnCountScanner &candidate);
+
+	//! ------------------------------------------------------//
+	//! ------------------- Type Detection ------------------ //
+	//! ------------------------------------------------------//
+	//! Second phase of auto detection: detect types, format template candidates
+	//! ordered by descending specificity (~ from high to low)
+	void DetectTypes();
+	//! Change the date format for the type to the string
+	//! Try to cast a string value to the specified sql type
+	bool TryCastValue(const DialectOptions &dialect_options, const string &decimal_separator, const Value &value,
+	                  const LogicalType &sql_type);
+	void SetDateFormat(CSVStateMachine &candidate, const string &format_specifier, const LogicalTypeId &sql_type);
+
+	//! Function that initialized the necessary variables used for date and timestamp detection
+	void InitializeDateAndTimeStampDetection(CSVStateMachine &candidate, const string &separator,
+	                                         const LogicalType &sql_type);
+	//! Functions that performs detection for date and timestamp formats
+	void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, const LogicalType &sql_type, const string &separator,
+	                                   Value &dummy_val);
+
+	//! Variables for Type Detection
+	//! Format Candidates for Date and Timestamp Types
+	const map<LogicalTypeId, vector<const char *>> format_template_candidates = {
+	    {LogicalTypeId::DATE, {"%m-%d-%Y", "%m-%d-%y", "%d-%m-%Y", "%d-%m-%y", "%Y-%m-%d", "%y-%m-%d"}},
+	    {LogicalTypeId::TIMESTAMP,
+	     {"%Y-%m-%d %H:%M:%S.%f", "%m-%d-%Y %I:%M:%S %p", "%m-%d-%y %I:%M:%S %p", "%d-%m-%Y %H:%M:%S",
+	      "%d-%m-%y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%y-%m-%d %H:%M:%S"}},
+	};
+	unordered_map<idx_t, vector<LogicalType>> best_sql_types_candidates_per_column_idx;
+	map<LogicalTypeId, vector<string>> best_format_candidates;
+	unique_ptr<StringValueScanner> best_candidate;
+	vector<Value> best_header_row;
+	//! Variable used for sniffing date and timestamp
+	map<LogicalTypeId, DateTimestampSniffing> format_candidates;
+
+	//! ------------------------------------------------------//
+	//! ------------------ Type Refinement ------------------ //
+	//! ------------------------------------------------------//
+	void RefineTypes();
+	bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type);
+	vector<LogicalType> detected_types;
+	//! ------------------------------------------------------//
+	//! ------------------ Header Detection ----------------- //
+	//! ------------------------------------------------------//
+	void DetectHeader();
+	bool DetectHeaderWithSetColumn();
+	vector<string> names;
+
+	//! ------------------------------------------------------//
+	//! ------------------ Type Replacement ----------------- //
+	//! ------------------------------------------------------//
+	void ReplaceTypes();
+	vector<bool> manually_set;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state.hpp

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cstdint>
+
+namespace duckdb {
+
+//! All States of CSV Parsing
+enum class CSVState : uint8_t {
+	STANDARD = 0,         //! Regular unquoted field state
+	DELIMITER = 1,        //! State after encountering a field separator (e.g., ;)
+	RECORD_SEPARATOR = 2, //! State after encountering a record separator (i.e., \n)
+	CARRIAGE_RETURN = 3,  //! State after encountering a carriage return(i.e., \r)
+	QUOTED = 4,           //! State when inside a quoted field
+	UNQUOTED = 5,         //! State when leaving a quoted field
+	ESCAPE = 6,           //! State when encountering an escape character (e.g., \)
+	INVALID = 7,          //! Got to an Invalid State, this should error.
+	NOT_SET = 8,          //! If the state is not set, usually the first state before getting the first character
+	QUOTED_NEW_LINE = 9,  //! If we have a quoted newline
+	EMPTY_SPACE = 10      //! If we have empty spaces in the beginning and end of value
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine.hpp

@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_state_machine.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp"
+
+namespace duckdb {
+
+//! State of necessary CSV States to parse file
+//! Current, previous, and state before the previous
+struct CSVStates {
+	void Initialize() {
+		states[0] = CSVState::NOT_SET;
+		states[1] = CSVState::NOT_SET;
+	}
+	inline bool NewValue() {
+		return states[1] == CSVState::DELIMITER;
+	}
+
+	inline bool NewRow() {
+		// It is a new row, if the previous state is not a record separator, and the current one is
+		return states[0] != CSVState::RECORD_SEPARATOR && states[0] != CSVState::CARRIAGE_RETURN &&
+		       (states[1] == CSVState::RECORD_SEPARATOR || states[1] == CSVState::CARRIAGE_RETURN);
+	}
+
+	inline bool EmptyLastValue() {
+		// It is a new row, if the previous state is not a record separator, and the current one is
+		return states[0] == CSVState::DELIMITER &&
+		       (states[1] == CSVState::RECORD_SEPARATOR || states[1] == CSVState::CARRIAGE_RETURN);
+	}
+
+	inline bool EmptyLine() {
+		return (states[1] == CSVState::CARRIAGE_RETURN || states[1] == CSVState::RECORD_SEPARATOR) &&
+		       (states[0] == CSVState::RECORD_SEPARATOR || states[0] == CSVState::NOT_SET);
+	}
+
+	inline bool IsNotSet() {
+		return states[1] == CSVState::NOT_SET;
+	}
+
+	inline bool IsCurrentNewRow() {
+		return states[1] == CSVState::RECORD_SEPARATOR || states[1] == CSVState::CARRIAGE_RETURN;
+	}
+
+	inline bool IsCarriageReturn() {
+		return states[1] == CSVState::CARRIAGE_RETURN;
+	}
+
+	inline bool IsQuoted() {
+		return states[0] == CSVState::QUOTED;
+	}
+	inline bool IsEscaped() {
+		return states[1] == CSVState::ESCAPE || (states[0] == CSVState::UNQUOTED && states[1] == CSVState::QUOTED);
+	}
+	inline bool IsQuotedCurrent() {
+		return states[1] == CSVState::QUOTED || states[1] == CSVState::QUOTED_NEW_LINE;
+	}
+	CSVState states[2];
+};
+
+//! The CSV State Machine comprises a state transition array (STA).
+//! The STA indicates the current state of parsing based on both the current and preceding characters.
+//! This reveals whether we are dealing with a Field, a New Line, a Delimiter, and so forth.
+//! The STA's creation depends on the provided quote, character, and delimiter options for that state machine.
+//! The motivation behind implementing an STA is to remove branching in regular CSV Parsing by predicting and detecting
+//! the states. Note: The State Machine is currently utilized solely in the CSV Sniffer.
+class CSVStateMachine {
+public:
+	explicit CSVStateMachine(CSVReaderOptions &options_p, const CSVStateMachineOptions &state_machine_options,
+	                         CSVStateMachineCache &csv_state_machine_cache_p);
+
+	explicit CSVStateMachine(const StateMachine &transition_array, const CSVReaderOptions &options);
+
+	//! Transition all states to next state, that depends on the current char
+	inline void Transition(CSVStates &states, char current_char) const {
+		states.states[0] = states.states[1];
+		states.states[1] = transition_array[static_cast<uint8_t>(current_char)][static_cast<uint8_t>(states.states[1])];
+	}
+
+	//! The Transition Array is a Finite State Machine
+	//! It holds the transitions of all states, on all 256 possible different characters
+	const StateMachine &transition_array;
+	//! Options of this state machine
+	const CSVStateMachineOptions state_machine_options;
+	//! CSV Reader Options
+	const CSVReaderOptions &options;
+	//! Dialect options resulting from sniffing
+	DialectOptions dialect_options;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp

@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/storage/object_cache.hpp"
+#include "duckdb/common/types/hash.hpp"
+#include "duckdb/execution/operator/csv_scanner/state_machine_options.hpp"
+#include "duckdb/execution/operator/csv_scanner/quote_rules.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state.hpp"
+
+namespace duckdb {
+
+//! Class to wrap the state machine matrix
+class StateMachine {
+public:
+	static constexpr uint32_t NUM_STATES = 11;
+	static constexpr uint32_t NUM_TRANSITIONS = 256;
+	CSVState state_machine[NUM_TRANSITIONS][NUM_STATES];
+	//! Transitions where we might skip processing
+	//! For the Standard State
+	bool skip_standard[256];
+	//! For the Quoted State
+	bool skip_quoted[256];
+
+	uint64_t delimiter = 0;
+	uint64_t new_line = 0;
+	uint64_t carriage_return = 0;
+	uint64_t quote = 0;
+	uint64_t escape = 0;
+	const CSVState *operator[](idx_t i) const {
+		return state_machine[i];
+	}
+
+	CSVState *operator[](idx_t i) {
+		return state_machine[i];
+	}
+};
+
+//! Hash function used in out state machine cache, it hashes and combines all options used to generate a state machine
+struct HashCSVStateMachineConfig {
+	size_t operator()(CSVStateMachineOptions const &config) const noexcept {
+		auto h_delimiter = Hash(config.delimiter.GetValue());
+		auto h_quote = Hash(config.quote.GetValue());
+		auto h_escape = Hash(config.escape.GetValue());
+		auto h_newline = Hash((uint8_t)config.new_line.GetValue());
+		return CombineHash(h_delimiter, CombineHash(h_quote, CombineHash(h_escape, h_newline)));
+	}
+};
+
+//! The CSVStateMachineCache caches state machines, although small ~2kb, the actual creation of multiple State Machines
+//! can become a bottleneck on sniffing, when reading very small csv files.
+//! Hence the cache stores State Machines based on their different delimiter|quote|escape options.
+class CSVStateMachineCache : public ObjectCacheEntry {
+public:
+	CSVStateMachineCache();
+	~CSVStateMachineCache() override = default;
+	//! Gets a state machine from the cache, if it's not from one the default options
+	//! It first caches it, then returns it.
+	static CSVStateMachineCache &Get(ClientContext &context);
+
+	//! Gets a state machine from the cache, if it's not from one the default options
+	//! It first caches it, then returns it.
+	const StateMachine &Get(const CSVStateMachineOptions &state_machine_options);
+
+	static string ObjectType() {
+		return "CSV_STATE_MACHINE_CACHE";
+	}
+
+	string GetObjectType() override {
+		return ObjectType();
+	}
+
+private:
+	void Insert(const CSVStateMachineOptions &state_machine_options);
+	//! Cache on delimiter|quote|escape|newline
+	unordered_map<CSVStateMachineOptions, StateMachine, HashCSVStateMachineConfig> state_machine_cache;
+	//! Default value for options used to intialize CSV State Machine Cache
+	const vector<char> default_delimiter = {',', '|', ';', '\t'};
+	const vector<vector<char>> default_quote = {{'\"'}, {'\"', '\''}, {'\0'}};
+	const vector<QuoteRule> default_quote_rule = {QuoteRule::QUOTES_RFC, QuoteRule::QUOTES_OTHER, QuoteRule::NO_QUOTES};
+	const vector<vector<char>> default_escape = {{'\0', '\"', '\''}, {'\\'}, {'\0'}};
+	//! Because the state machine cache can be accessed in Parallel we need a mutex.
+	mutex main_mutex;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp

@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/global_csv_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_error.hpp"
+#include "duckdb/function/table/read_csv.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp"
+#include "duckdb/execution/operator/csv_scanner/string_value_scanner.hpp"
+
+namespace duckdb {
+
+//! CSV Global State is used in the CSV Reader Table Function, it controls what each thread
+struct CSVGlobalState : public GlobalTableFunctionState {
+public:
+	CSVGlobalState(ClientContext &context, const shared_ptr<CSVBufferManager> &buffer_manager_p,
+	               const CSVReaderOptions &options, idx_t system_threads_p, const vector<string> &files,
+	               vector<column_t> column_ids_p, const ReadCSVData &bind_data);
+
+	~CSVGlobalState() override {
+	}
+
+	//! Generates a CSV Scanner, with information regarding the piece of buffer it should be read.
+	//! In case it returns a nullptr it means we are done reading these files.
+	unique_ptr<StringValueScanner> Next();
+
+	void FillRejectsTable();
+
+	void DecrementThread();
+
+	//! Returns Current Progress of this CSV Read
+	double GetProgress(const ReadCSVData &bind_data) const;
+
+	//! Calculates the Max Threads that will be used by this CSV Reader
+	idx_t MaxThreads() const override;
+	//! We hold information on the current scanner boundary
+	CSVIterator current_boundary;
+
+private:
+	//! Reference to the client context that created this scan
+	ClientContext &context;
+
+	vector<shared_ptr<CSVFileScan>> file_scans;
+
+	//! Mutex to lock when getting next batch of bytes (Parallel Only)
+	mutable mutex main_mutex;
+
+	//! Basically max number of threads in DuckDB
+	idx_t system_threads;
+
+	//! Number of threads being used in this scanner
+	idx_t running_threads = 1;
+	//! The column ids to read
+	vector<column_t> column_ids;
+
+	string sniffer_mismatch_error;
+
+	bool finished = false;
+
+	const ReadCSVData &bind_data;
+
+	vector<LogicalType> file_schema;
+
+	bool single_threaded = false;
+
+	atomic<idx_t> scanner_idx;
+
+	atomic<idx_t> last_file_idx;
+	shared_ptr<CSVBufferUsage> current_buffer_in_use;
+};
+
+} // namespace duckdb