duckdb 0.8.2-dev87.0 → 0.9.1-dev0.0

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer.hpp

@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_buffer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_file_handle.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/storage/buffer/block_handle.hpp"
+
+namespace duckdb {
+
+class CSVBufferHandle {
+public:
+	CSVBufferHandle(BufferHandle handle_p, idx_t actual_size_p, const bool is_first_buffer_p,
+	                const bool is_final_buffer_p, idx_t csv_global_state_p, idx_t start_position_p, idx_t file_idx_p)
+	    : handle(std::move(handle_p)), actual_size(actual_size_p), is_first_buffer(is_first_buffer_p),
+	      is_last_buffer(is_final_buffer_p), csv_global_start(csv_global_state_p), start_position(start_position_p),
+	      file_idx(file_idx_p) {};
+	CSVBufferHandle()
+	    : actual_size(0), is_first_buffer(false), is_last_buffer(false), csv_global_start(0), start_position(0),
+	      file_idx(0) {};
+	//! Handle created during allocation
+	BufferHandle handle;
+	const idx_t actual_size;
+	const bool is_first_buffer;
+	const bool is_last_buffer;
+	const idx_t csv_global_start;
+	const idx_t start_position;
+	const idx_t file_idx;
+	inline char *Ptr() {
+		return char_ptr_cast(handle.Ptr());
+	}
+};
+
+//! CSV Buffers are parts of a decompressed CSV File.
+//! For a decompressed file of 100Mb. With our Buffer size set to 32Mb, we would generate 4 buffers.
+//! One for the first 32Mb, second and third for the other 32Mb, and the last one with 4 Mb
+//! These buffers are actually used for sniffing and parsing!
+class CSVBuffer {
+public:
+	//! Constructor for Initial Buffer
+	CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
+	          idx_t &global_csv_current_position, idx_t file_number);
+
+	//! Constructor for `Next()` Buffers
+	CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position,
+	          idx_t file_number_p);
+
+	//! Creates a new buffer with the next part of the CSV File
+	shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number);
+
+	//! Gets the buffer actual size
+	idx_t GetBufferSize();
+
+	//! Gets the start position of the buffer, only relevant for the first time it's scanned
+	idx_t GetStart();
+
+	//! If this buffer is the last buffer of the CSV File
+	bool IsCSVFileLastBuffer();
+
+	//! Allocates internal buffer, sets 'block' and 'handle' variables.
+	void AllocateBuffer(idx_t buffer_size);
+
+	void Reload(CSVFileHandle &file_handle);
+	//! Wrapper for the Pin Function, if it can seek, it means that the buffer might have been destroyed, hence we must
+	//! Scan it from the disk file again.
+	unique_ptr<CSVBufferHandle> Pin(CSVFileHandle &file_handle);
+	//! Wrapper for the unpin
+	void Unpin();
+	char *Ptr() {
+		return char_ptr_cast(handle.Ptr());
+	}
+
+	static constexpr idx_t CSV_BUFFER_SIZE = 32000000; // 32MB
+	//! In case the file has a size < 32MB, we will use this size instead
+	//! This is to avoid mallocing a lot of memory for a small file
+	//! And if it's a compressed file we can't use the actual size of the file
+	static constexpr idx_t CSV_MINIMUM_BUFFER_SIZE = 10000000; // 10MB
+	//! If this is the last buffer of the CSV File
+	bool last_buffer = false;
+
+private:
+	ClientContext &context;
+	//! Actual size can be smaller than the buffer size in case we allocate it too optimistically.
+	idx_t actual_buffer_size;
+	//! We need to check for Byte Order Mark, to define the start position of this buffer
+	//! https://en.wikipedia.org/wiki/Byte_order_mark#UTF-8
+	idx_t start_position = 0;
+	//! If this is the first buffer of the CSV File
+	bool first_buffer = false;
+	//! Global position from the CSV File where this buffer starts
+	idx_t global_csv_start = 0;
+	//! Number of the file that is in this buffer
+	idx_t file_number = 0;
+	//! If we can seek in the file or not.
+	//! If we can't seek, this means we can't destroy the buffers
+	bool can_seek;
+	//! -------- Allocated Block ---------//
+	//! Block created in allocation
+	shared_ptr<BlockHandle> block;
+	BufferHandle handle;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp

@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_file_handle.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+
+namespace duckdb {
+class CSVBuffer;
+class CSVStateMachine;
+
+//! This class is used to manage the CSV buffers.  Buffers are cached when used for auto detection.
+//! When parsing, buffer are not cached and just returned.
+class CSVBufferManager {
+public:
+	CSVBufferManager(ClientContext &context, unique_ptr<CSVFileHandle> file_handle, const CSVReaderOptions &options,
+	                 idx_t file_idx = 0);
+	//! Returns a buffer from a buffer id (starting from 0). If it's in the auto-detection then we cache new buffers
+	//! Otherwise we remove them from the cache if they are already there, or just return them bypassing the cache.
+	unique_ptr<CSVBufferHandle> GetBuffer(const idx_t pos);
+	//! Returns the starting position of the first buffer
+	idx_t GetStartPos();
+	//! unique_ptr to the file handle, gets stolen after sniffing
+	unique_ptr<CSVFileHandle> file_handle;
+	//! Initializes the buffer manager, during it's construction/reset
+	void Initialize();
+
+	void UnpinBuffer(idx_t cache_idx);
+
+	ClientContext &context;
+	idx_t skip_rows = 0;
+	idx_t file_idx;
+	bool done = false;
+
+private:
+	//! Reads next buffer in reference to cached_buffers.front()
+	bool ReadNextAndCacheIt();
+	vector<shared_ptr<CSVBuffer>> cached_buffers;
+	shared_ptr<CSVBuffer> last_buffer;
+	idx_t global_csv_pos = 0;
+	//! The size of the buffer, if the csv file has a smaller size than this, we will use that instead to malloc less
+	idx_t buffer_size;
+	//! Starting position of first buffer
+	idx_t start_pos = 0;
+};
+
+class CSVBufferIterator {
+public:
+	explicit CSVBufferIterator(shared_ptr<CSVBufferManager> buffer_manager_p)
+	    : buffer_manager(std::move(buffer_manager_p)) {
+		cur_pos = buffer_manager->GetStartPos();
+	};
+
+	//! This functions templates an operation over the CSV File
+	template <class OP, class T>
+	inline bool Process(CSVStateMachine &machine, T &result) {
+
+		OP::Initialize(machine);
+		//! If current buffer is not set we try to get a new one
+		if (!cur_buffer_handle) {
+			cur_pos = 0;
+			if (cur_buffer_idx == 0) {
+				cur_pos = buffer_manager->GetStartPos();
+			}
+			cur_buffer_handle = buffer_manager->GetBuffer(cur_buffer_idx++);
+			D_ASSERT(cur_buffer_handle);
+		}
+		while (cur_buffer_handle) {
+			char *buffer_handle_ptr = cur_buffer_handle->Ptr();
+			while (cur_pos < cur_buffer_handle->actual_size) {
+				if (OP::Process(machine, result, buffer_handle_ptr[cur_pos], cur_pos)) {
+					//! Not-Done Processing the File, but the Operator is happy!
+					OP::Finalize(machine, result);
+					return false;
+				}
+				cur_pos++;
+			}
+			cur_buffer_handle = buffer_manager->GetBuffer(cur_buffer_idx++);
+			cur_pos = 0;
+		}
+		//! Done Processing the File
+		OP::Finalize(machine, result);
+		return true;
+	}
+	//! Returns true if the iterator is finished
+	bool Finished();
+	//! Resets the iterator
+	void Reset();
+
+private:
+	idx_t cur_pos = 0;
+	idx_t cur_buffer_idx = 0;
+	shared_ptr<CSVBufferManager> buffer_manager;
+	unique_ptr<CSVBufferHandle> cur_buffer_handle;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_file_handle.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/csv_file_handle.hpp
+// duckdb/execution/operator/scan/csv/csv_file_handle.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -20,15 +20,13 @@ class FileSystem;
 struct CSVFileHandle {
 public:
 	CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<FileHandle> file_handle_p, const string &path_p,
-	              FileCompressionType compression, bool enable_reset = true);
+	              FileCompressionType compression);
 
 	mutex main_mutex;
 
 public:
 	bool CanSeek();
 	void Seek(idx_t position);
-	idx_t SeekPosition();
-	void Reset();
 	bool OnDiskFile();
 
 	idx_t FileSize();
@@ -38,29 +36,24 @@ public:
 	idx_t Read(void *buffer, idx_t nr_bytes);
 
 	string ReadLine();
-	void DisableReset();
+
+	string GetFilePath();
 
 	static unique_ptr<FileHandle> OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
 	                                             FileCompressionType compression);
 	static unique_ptr<CSVFileHandle> OpenFile(FileSystem &fs, Allocator &allocator, const string &path,
-	                                          FileCompressionType compression, bool enable_reset);
+	                                          FileCompressionType compression);
 
 private:
-	FileSystem &fs;
-	Allocator &allocator;
 	unique_ptr<FileHandle> file_handle;
 	string path;
-	FileCompressionType compression;
-	bool reset_enabled = true;
 	bool can_seek = false;
 	bool on_disk_file = false;
 	idx_t file_size = 0;
-	// reset support
-	AllocatedData cached_buffer;
-	idx_t read_position = 0;
-	idx_t buffer_size = 0;
-	idx_t buffer_capacity = 0;
+
 	idx_t requested_bytes = 0;
+	//! If we finished reading the file
+	bool finished = false;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_line_info.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/csv_line_info.hpp
+// duckdb/execution/operator/scan/csv/csv_line_info.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -18,11 +18,16 @@ public:
 	bool CanItGetLine(idx_t file_idx, idx_t batch_idx);
 
 	//! Return the 1-indexed line number
-	idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true);
+	idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true,
+	              bool stop_at_first = true);
+	//! In case an error happened we have to increment the lines read of that batch
+	void Increment(idx_t file_idx, idx_t batch_idx);
 	//! Verify if the CSV File was read correctly from [0,batch_idx] batches.
 	void Verify(idx_t file_idx, idx_t batch_idx, idx_t cur_first_pos);
-	//! Lines read per batch, <batch_index,count>
-	unordered_map<idx_t, idx_t> lines_read;
+	//! Lines read per batch, <file_index, <batch_index, count>>
+	vector<unordered_map<idx_t, idx_t>> lines_read;
+	//! Lines read per batch, <file_index, <batch_index, count>>
+	vector<unordered_map<idx_t, idx_t>> lines_errored;
 	//! Set of batches that have been initialized but are not yet finished.
 	vector<set<idx_t>> current_batches;
 	//! Pointer to CSV Reader Mutex

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_reader_options.hpp

@@ -1,18 +1,17 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/csv_reader_options.hpp
+// duckdb/execution/operator/scan/csv/csv_reader_options.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/execution/operator/persistent/csv_buffer.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_buffer.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
 #include "duckdb/common/types/value.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/multi_file_reader_options.hpp"
@@ -28,35 +27,69 @@ enum class NewLineIdentifier : uint8_t {
 
 enum class ParallelMode { AUTOMATIC = 0, PARALLEL = 1, SINGLE_THREADED = 2 };
 
-struct BufferedCSVReaderOptions {
+//! Struct that holds the configuration of a CSV State Machine
+//! Basically which char, quote and escape were used to generate it.
+struct CSVStateMachineOptions {
+	CSVStateMachineOptions() {};
+	CSVStateMachineOptions(char delimiter_p, char quote_p, char escape_p)
+	    : delimiter(delimiter_p), quote(quote_p), escape(escape_p) {};
+
+	//! Delimiter to separate columns within each line
+	char delimiter = ',';
+	//! Quote used for columns that contain reserved characters, e.g '
+	char quote = '\"';
+	//! Escape character to escape quote character
+	char escape = '\0';
+
+	bool operator==(const CSVStateMachineOptions &other) const {
+		return delimiter == other.delimiter && quote == other.quote && escape == other.escape;
+	}
+};
+
+struct DialectOptions {
+	CSVStateMachineOptions state_machine_options;
+	//! New Line separator
+	NewLineIdentifier new_line = NewLineIdentifier::NOT_SET;
+	//! Expected number of columns
+	idx_t num_cols = 0;
+	//! Whether or not the file has a header line
+	bool header = false;
+	//! The date format to use (if any is specified)
+	map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
+	//! Whether or not a type format is specified
+	map<LogicalTypeId, bool> has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}};
+	//! How many leading rows to skip
+	idx_t skip_rows = 0;
+	//! True start of the first CSV Buffer (After skipping empty lines, headers, notes and so on)
+	idx_t true_start = 0;
+};
+
+struct CSVReaderOptions {
 	//===--------------------------------------------------------------------===//
 	// CommonCSVOptions
 	//===--------------------------------------------------------------------===//
-
+	//! See struct above.
+	DialectOptions dialect_options;
 	//! Whether or not a delimiter was defined by the user
 	bool has_delimiter = false;
-	//! Delimiter to separate columns within each line
-	string delimiter = ",";
 	//! Whether or not a new_line was defined by the user
 	bool has_newline = false;
-	//! New Line separator
-	NewLineIdentifier new_line = NewLineIdentifier::NOT_SET;
 	//! Whether or not a quote was defined by the user
 	bool has_quote = false;
-	//! Quote used for columns that contain reserved characters, e.g., delimiter
-	string quote = "\"";
 	//! Whether or not an escape character was defined by the user
 	bool has_escape = false;
-	//! Escape character to escape quote character
-	string escape;
 	//! Whether or not a header information was given by the user
 	bool has_header = false;
-	//! Whether or not the file has a header line
-	bool header = false;
 	//! Whether or not we should ignore InvalidInput errors
 	bool ignore_errors = false;
-	//! Expected number of columns
-	idx_t num_cols = 0;
+	//! Rejects table name
+	string rejects_table_name;
+	//! Rejects table entry limit (0 = no limit)
+	idx_t rejects_limit = 0;
+	//! Columns to use as recovery key for rejected rows when reading with ignore_errors = true
+	vector<string> rejects_recovery_columns;
+	//! Index of the recovery columns
+	vector<idx_t> rejects_recovery_column_ids;
 	//! Number of samples to buffer
 	idx_t buffer_sample_size = STANDARD_VECTOR_SIZE * 50;
 	//! Specifies the string that represents a null value
@@ -84,9 +117,6 @@ struct BufferedCSVReaderOptions {
 	//===--------------------------------------------------------------------===//
 	// ReadCSVOptions
 	//===--------------------------------------------------------------------===//
-
-	//! How many leading rows to skip
-	idx_t skip_rows = 0;
 	//! Whether or not the skip_rows is set by the user
 	bool skip_rows_set = false;
 	//! Maximum CSV line size: specified because if we reach this amount, we likely have wrong delimiters (default: 2MB)
@@ -96,12 +126,10 @@ struct BufferedCSVReaderOptions {
 	bool normalize_names = false;
 	//! True, if column with that index must skip null check
 	vector<bool> force_not_null;
+	//! Number of sample chunks used in auto-detection
+	idx_t sample_size_chunks = 20480 / STANDARD_VECTOR_SIZE;
 	//! Consider all columns to be of type varchar
 	bool all_varchar = false;
-	//! Size of sample chunk used for dialect and type detection
-	idx_t sample_chunk_size = STANDARD_VECTOR_SIZE;
-	//! Number of sample chunks used for type detection
-	idx_t sample_chunks = 10;
 	//! Whether or not to automatically detect dialect and datatypes
 	bool auto_detect = false;
 	//! The file path of the CSV file to read
@@ -109,7 +137,7 @@ struct BufferedCSVReaderOptions {
 	//! Multi-file reader options
 	MultiFileReaderOptions file_options;
 	//! Buffer Size (Parallel Scan)
-	idx_t buffer_size = CSVBuffer::INITIAL_BUFFER_SIZE_COLOSSAL;
+	idx_t buffer_size = CSVBuffer::CSV_BUFFER_SIZE;
 	//! Decimal separator when reading as numeric
 	string decimal_separator = ".";
 	//! Whether or not to pad rows that do not have enough columns with NULL values
@@ -130,22 +158,32 @@ struct BufferedCSVReaderOptions {
 	string write_newline;
 
 	//! The date format to use (if any is specified)
-	std::map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
+	map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
 	//! The date format to use for writing (if any is specified)
-	std::map<LogicalTypeId, StrfTimeFormat> write_date_format = {{LogicalTypeId::DATE, {}},
-	                                                             {LogicalTypeId::TIMESTAMP, {}}};
+	map<LogicalTypeId, StrfTimeFormat> write_date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
 	//! Whether or not a type format is specified
-	std::map<LogicalTypeId, bool> has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}};
+	map<LogicalTypeId, bool> has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}};
 
-	void Serialize(FieldWriter &writer) const;
-	void Deserialize(FieldReader &reader);
+	void Serialize(Serializer &serializer) const;
+	static CSVReaderOptions Deserialize(Deserializer &deserializer);
 
 	void SetCompression(const string &compression);
+
+	bool GetHeader() const;
 	void SetHeader(bool has_header);
+
+	string GetEscape() const;
 	void SetEscape(const string &escape);
+
+	int64_t GetSkipRows() const;
+	void SetSkipRows(int64_t rows);
+
+	string GetQuote() const;
 	void SetQuote(const string &quote);
 	void SetDelimiter(const string &delimiter);
+	string GetDelimiter() const;
 
+	NewLineIdentifier GetNewline() const;
 	void SetNewline(const string &input);
 	//! Set an option that is supported by both reading and writing functions, called by
 	//! the SetReadOption and SetWriteOption methods
@@ -155,10 +193,18 @@ struct BufferedCSVReaderOptions {
 	//! set - argument(s) to the option
 	//! expected_names - names expected if the option is "columns"
 	void SetReadOption(const string &loption, const Value &value, vector<string> &expected_names);
-
 	void SetWriteOption(const string &loption, const Value &value);
 	void SetDateFormat(LogicalTypeId type, const string &format, bool read_format);
+	void ToNamedParameters(named_parameter_map_t &out);
+	void FromNamedParameters(named_parameter_map_t &in, ClientContext &context, vector<LogicalType> &return_types,
+	                         vector<string> &names);
+
+	string ToString() const;
+
+	named_parameter_map_t OutputReadSettings();
 
-	std::string ToString() const;
+public:
+	//! Whether columns were explicitly provided through named parameters
+	bool explicitly_set_columns = false;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_sniffer.hpp

@@ -0,0 +1,129 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_sniffer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/scan/csv/csv_state_machine.hpp"
+#include "duckdb/common/vector.hpp"
+#include "duckdb/execution/operator/scan/csv/quote_rules.hpp"
+
+namespace duckdb {
+//! Struct to store the result of the Sniffer
+struct SnifferResult {
+	SnifferResult(vector<LogicalType> return_types_p, vector<string> names_p)
+	    : return_types(std::move(return_types_p)), names(std::move(names_p)) {
+	}
+	//! Return Types that were detected
+	vector<LogicalType> return_types;
+	//! Column Names that were detected
+	vector<string> names;
+};
+
+//! Sniffer that detects Header, Dialect and Types of CSV Files
+class CSVSniffer {
+public:
+	explicit CSVSniffer(CSVReaderOptions &options_p, shared_ptr<CSVBufferManager> buffer_manager_p,
+	                    CSVStateMachineCache &state_machine_cache, bool explicit_set_columns = false);
+
+	//! Main method that sniffs the CSV file, returns the types, names and options as a result
+	//! CSV Sniffing consists of five steps:
+	//! 1. Dialect Detection: Generate the CSV Options (delimiter, quote, escape, etc.)
+	//! 2. Type Detection: Figures out the types of the columns (For one chunk)
+	//! 3. Header Detection: Figures out if  the CSV file has a header and produces the names of the columns
+	//! 4. Type Replacement: Replaces the types of the columns if the user specified them
+	//! 5. Type Refinement: Refines the types of the columns for the remaining chunks
+	SnifferResult SniffCSV();
+
+private:
+	//! CSV State Machine Cache
+	CSVStateMachineCache &state_machine_cache;
+	//! Highest number of columns found
+	idx_t max_columns_found = 0;
+	//! Current Candidates being considered
+	vector<unique_ptr<CSVStateMachine>> candidates;
+	//! Reference to original CSV Options, it will be modified as a result of the sniffer.
+	CSVReaderOptions &options;
+	//! Buffer being used on sniffer
+	shared_ptr<CSVBufferManager> buffer_manager;
+
+	//! ------------------------------------------------------//
+	//! ----------------- Dialect Detection ----------------- //
+	//! ------------------------------------------------------//
+	//! First phase of auto detection: detect CSV dialect (i.e. delimiter, quote rules, etc)
+	void DetectDialect();
+	//! Functions called in the main DetectDialect(); function
+	//! 1. Generates the search space candidates for the dialect
+	void GenerateCandidateDetectionSearchSpace(vector<char> &delim_candidates, vector<QuoteRule> &quoterule_candidates,
+	                                           unordered_map<uint8_t, vector<char>> &quote_candidates_map,
+	                                           unordered_map<uint8_t, vector<char>> &escape_candidates_map);
+	//! 2. Generates the search space candidates for the state machines
+	void GenerateStateMachineSearchSpace(vector<unique_ptr<CSVStateMachine>> &csv_state_machines,
+	                                     const vector<char> &delimiter_candidates,
+	                                     const vector<QuoteRule> &quoterule_candidates,
+	                                     const unordered_map<uint8_t, vector<char>> &quote_candidates_map,
+	                                     const unordered_map<uint8_t, vector<char>> &escape_candidates_map);
+	//! 3. Analyzes if dialect candidate is a good candidate to be considered, if so, it adds it to the candidates
+	void AnalyzeDialectCandidate(unique_ptr<CSVStateMachine>, idx_t &rows_read, idx_t &best_consistent_rows,
+	                             idx_t &prev_padding_count);
+	//! 4. Refine Candidates over remaining chunks
+	void RefineCandidates();
+	//! Checks if candidate still produces good values for the next chunk
+	bool RefineCandidateNextChunk(CSVStateMachine &candidate);
+
+	//! ------------------------------------------------------//
+	//! ------------------- Type Detection ------------------ //
+	//! ------------------------------------------------------//
+	//! Second phase of auto detection: detect types, format template candidates
+	//! ordered by descending specificity (~ from high to low)
+	void DetectTypes();
+	//! Change the date format for the type to the string
+	//! Try to cast a string value to the specified sql type
+	bool TryCastValue(CSVStateMachine &candidate, const Value &value, const LogicalType &sql_type);
+	void SetDateFormat(CSVStateMachine &candidate, const string &format_specifier, const LogicalTypeId &sql_type);
+	//! Functions that performs detection for date and timestamp formats
+	void DetectDateAndTimeStampFormats(CSVStateMachine &candidate, map<LogicalTypeId, bool> &has_format_candidates,
+	                                   map<LogicalTypeId, vector<string>> &format_candidates,
+	                                   const LogicalType &sql_type, const string &separator, Value &dummy_val);
+
+	//! Variables for Type Detection
+	//! Format Candidates for Date and Timestamp Types
+	const map<LogicalTypeId, vector<const char *>> format_template_candidates = {
+	    {LogicalTypeId::DATE, {"%m-%d-%Y", "%m-%d-%y", "%d-%m-%Y", "%d-%m-%y", "%Y-%m-%d", "%y-%m-%d"}},
+	    {LogicalTypeId::TIMESTAMP,
+	     {"%Y-%m-%d %H:%M:%S.%f", "%m-%d-%Y %I:%M:%S %p", "%m-%d-%y %I:%M:%S %p", "%d-%m-%Y %H:%M:%S",
+	      "%d-%m-%y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%y-%m-%d %H:%M:%S"}},
+	};
+	unordered_map<idx_t, vector<LogicalType>> best_sql_types_candidates_per_column_idx;
+	map<LogicalTypeId, vector<string>> best_format_candidates;
+	unique_ptr<CSVStateMachine> best_candidate;
+	idx_t best_start_with_header = 0;
+	idx_t best_start_without_header = 0;
+	vector<Value> best_header_row;
+
+	//! ------------------------------------------------------//
+	//! ------------------ Header Detection ----------------- //
+	//! ------------------------------------------------------//
+	void DetectHeader();
+	vector<string> names;
+	//! If Column Names and Types have been explicitly set
+	const bool explicit_set_columns;
+
+	//! ------------------------------------------------------//
+	//! ------------------ Type Replacement ----------------- //
+	//! ------------------------------------------------------//
+	void ReplaceTypes();
+
+	//! ------------------------------------------------------//
+	//! ------------------ Type Refinement ------------------ //
+	//! ------------------------------------------------------//
+	void RefineTypes();
+	bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type);
+	vector<LogicalType> detected_types;
+};
+
+} // namespace duckdb