duckdb 0.8.2-dev37.0 → 0.8.2-dev3989.0

package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp

@@ -47,7 +47,7 @@ public:
 	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
 	unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
 	SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
-	void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
+	SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
 
 	bool IsSink() const override {
 		return true;

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/base_csv_reader.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/base_csv_reader.hpp
+// duckdb/execution/operator/scan/csv/base_csv_reader.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -15,9 +15,9 @@
 #include "duckdb/common/enums/file_compression_type.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/common/queue.hpp"
-#include "duckdb/execution/operator/persistent/csv_reader_options.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
-#include "duckdb/execution/operator/persistent/csv_line_info.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_line_info.hpp"
 
 #include <sstream>
 
@@ -30,19 +30,19 @@ struct StrpTimeFormat;
 class FileOpener;
 class FileSystem;
 
-enum class ParserMode : uint8_t { PARSING = 0, SNIFFING_DIALECT = 1, SNIFFING_DATATYPES = 2, PARSING_HEADER = 3 };
+enum class ParserMode : uint8_t { PARSING = 0, SNIFFING_DATATYPES = 1, PARSING_HEADER = 2 };
 
 //! Buffered CSV reader is a class that reads values from a stream and parses them as a CSV file
 class BaseCSVReader {
 public:
-	BaseCSVReader(ClientContext &context, BufferedCSVReaderOptions options,
+	BaseCSVReader(ClientContext &context, CSVReaderOptions options,
 	              const vector<LogicalType> &requested_types = vector<LogicalType>());
-	~BaseCSVReader();
+	virtual ~BaseCSVReader();
 
 	ClientContext &context;
 	FileSystem &fs;
 	Allocator &allocator;
-	BufferedCSVReaderOptions options;
+	CSVReaderOptions options;
 	vector<LogicalType> return_types;
 	vector<string> names;
 	MultiFileReaderData reader_data;
@@ -73,25 +73,25 @@ public:
 	const vector<LogicalType> &GetTypes() {
 		return return_types;
 	}
-
 	//! Get the 1-indexed global line number for the given local error line
-	virtual idx_t GetLineError(idx_t line_error, idx_t buffer_idx) {
+	virtual idx_t GetLineError(idx_t line_error, idx_t buffer_idx, bool stop_at_first = true) {
 		return line_error + 1;
 	};
 
 	//! Initialize projection indices to select all columns
 	void InitializeProjection();
 
+	static unique_ptr<CSVFileHandle> OpenCSV(ClientContext &context, const CSVReaderOptions &options);
+
+	static bool TryCastDateVector(map<LogicalTypeId, StrpTimeFormat> &options, Vector &input_vector,
+	                              Vector &result_vector, idx_t count, string &error_message, idx_t &line_error);
+
+	static bool TryCastTimestampVector(map<LogicalTypeId, StrpTimeFormat> &options, Vector &input_vector,
+	                                   Vector &result_vector, idx_t count, string &error_message);
+
 protected:
 	//! Initializes the parse_chunk with varchar columns and aligns info with new number of cols
 	void InitParseChunk(idx_t num_cols);
-	//! Change the date format for the type to the string
-	void SetDateFormat(const string &format_specifier, const LogicalTypeId &sql_type);
-	//! Try to cast a string value to the specified sql type
-	bool TryCastValue(const Value &value, const LogicalType &sql_type);
-	//! Try to cast a vector of values to the specified sql type
-	bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type);
-
 	//! Adds a value to the current row
 	void AddValue(string_t str_val, idx_t &column, vector<idx_t> &escape_positions, bool has_quotes,
 	              idx_t buffer_idx = 0);
@@ -100,8 +100,6 @@ protected:
 	//! Finalizes a chunk, parsing all values that have been added so far and adding them to the insert_chunk
 	bool Flush(DataChunk &insert_chunk, idx_t buffer_idx = 0, bool try_add_line = false);
 
-	unique_ptr<CSVFileHandle> OpenCSV(const BufferedCSVReaderOptions &options);
-
 	void VerifyUTF8(idx_t col_idx);
 	void VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, int64_t offset = 0);
 	string GetLineNumberStr(idx_t linenr, bool linenr_estimated, idx_t buffer_idx = 0);
@@ -109,11 +107,13 @@ protected:
 	//! Sets the newline delimiter
 	void SetNewLineDelimiter(bool carry = false, bool carry_followed_by_nl = false);
 
+	//! Verifies that the line length did not go over a pre-defined limit.
+	void VerifyLineLength(idx_t line_size, idx_t buffer_idx = 0);
+
 protected:
 	//! Whether or not the current row's columns have overflown return_types.size()
 	bool error_column_overflow = false;
 	//! Number of sniffed columns - only used when auto-detecting
-	vector<idx_t> sniffed_column_counts;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/buffered_csv_reader.hpp

@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/buffered_csv_reader.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/scan/csv/base_csv_reader.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp"
+
+namespace duckdb {
+struct CopyInfo;
+struct CSVFileHandle;
+struct FileHandle;
+struct StrpTimeFormat;
+
+class FileOpener;
+class FileSystem;
+
+//! Buffered CSV reader is a class that reads values from a stream and parses them as a CSV file
+class BufferedCSVReader : public BaseCSVReader {
+	//! Initial buffer read size; can be extended for long lines
+	static constexpr idx_t INITIAL_BUFFER_SIZE = 16384;
+	//! Larger buffer size for non disk files
+	static constexpr idx_t INITIAL_BUFFER_SIZE_LARGE = 10000000; // 10MB
+
+public:
+	BufferedCSVReader(ClientContext &context, CSVReaderOptions options,
+	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
+	BufferedCSVReader(ClientContext &context, string filename, CSVReaderOptions options,
+	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
+	virtual ~BufferedCSVReader() {
+	}
+
+	unsafe_unique_array<char> buffer;
+	idx_t buffer_size;
+	idx_t position;
+	idx_t start = 0;
+
+	vector<unsafe_unique_array<char>> cached_buffers;
+
+	unique_ptr<CSVFileHandle> file_handle;
+	//! CSV State Machine Cache
+	CSVStateMachineCache state_machine_cache;
+
+public:
+	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
+	void ParseCSV(DataChunk &insert_chunk);
+	static string ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column, const vector<string> &names);
+
+private:
+	//! Initialize Parser
+	void Initialize(const vector<LogicalType> &requested_types);
+	//! Skips skip_rows, reads header row from input stream
+	void SkipRowsAndReadHeader(idx_t skip_rows, bool skip_header);
+	//! Resets the buffer
+	void ResetBuffer();
+	//! Reads a new buffer from the CSV file if the current one has been exhausted
+	bool ReadBuffer(idx_t &start, idx_t &line_start);
+	//! Try to parse a single datachunk from the file. Throws an exception if anything goes wrong.
+	void ParseCSV(ParserMode mode);
+	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
+	bool TryParseCSV(ParserMode mode, DataChunk &insert_chunk, string &error_message);
+	//! Skip Empty lines for tables with over one column
+	void SkipEmptyLines();
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer.hpp

@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_buffer.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/constants.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_file_handle.hpp"
+#include "duckdb/storage/buffer_manager.hpp"
+#include "duckdb/storage/block_manager.hpp"
+#include "duckdb/storage/buffer/block_handle.hpp"
+
+namespace duckdb {
+
+class CSVBufferHandle {
+public:
+	CSVBufferHandle(BufferHandle handle_p, idx_t actual_size_p, const bool is_first_buffer_p,
+	                const bool is_final_buffer_p, idx_t csv_global_state_p, idx_t start_position_p, idx_t file_idx_p)
+	    : handle(std::move(handle_p)), actual_size(actual_size_p), is_first_buffer(is_first_buffer_p),
+	      is_last_buffer(is_final_buffer_p), csv_global_start(csv_global_state_p), start_position(start_position_p),
+	      file_idx(file_idx_p) {};
+	CSVBufferHandle()
+	    : actual_size(0), is_first_buffer(false), is_last_buffer(false), csv_global_start(0), start_position(0),
+	      file_idx(0) {};
+	//! Handle created during allocation
+	BufferHandle handle;
+	const idx_t actual_size;
+	const bool is_first_buffer;
+	const bool is_last_buffer;
+	const idx_t csv_global_start;
+	const idx_t start_position;
+	const idx_t file_idx;
+	inline char *Ptr() {
+		return char_ptr_cast(handle.Ptr());
+	}
+};
+
+//! CSV Buffers are parts of a decompressed CSV File.
+//! For a decompressed file of 100Mb. With our Buffer size set to 32Mb, we would generate 4 buffers.
+//! One for the first 32Mb, second and third for the other 32Mb, and the last one with 4 Mb
+//! These buffers are actually used for sniffing and parsing!
+class CSVBuffer {
+public:
+	//! Constructor for Initial Buffer
+	CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
+	          idx_t &global_csv_current_position, idx_t file_number);
+
+	//! Constructor for `Next()` Buffers
+	CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position,
+	          idx_t file_number_p);
+
+	//! Creates a new buffer with the next part of the CSV File
+	shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number);
+
+	//! Gets the buffer actual size
+	idx_t GetBufferSize();
+
+	//! Gets the start position of the buffer, only relevant for the first time it's scanned
+	idx_t GetStart();
+
+	//! If this buffer is the last buffer of the CSV File
+	bool IsCSVFileLastBuffer();
+
+	//! Allocates internal buffer, sets 'block' and 'handle' variables.
+	void AllocateBuffer(idx_t buffer_size);
+
+	void Reload(CSVFileHandle &file_handle);
+	//! Wrapper for the Pin Function, if it can seek, it means that the buffer might have been destroyed, hence we must
+	//! Scan it from the disk file again.
+	unique_ptr<CSVBufferHandle> Pin(CSVFileHandle &file_handle);
+	//! Wrapper for the unpin
+	void Unpin();
+	char *Ptr() {
+		return char_ptr_cast(handle.Ptr());
+	}
+
+	static constexpr idx_t CSV_BUFFER_SIZE = 32000000; // 32MB
+	//! In case the file has a size < 32MB, we will use this size instead
+	//! This is to avoid mallocing a lot of memory for a small file
+	//! And if it's a compressed file we can't use the actual size of the file
+	static constexpr idx_t CSV_MINIMUM_BUFFER_SIZE = 10000000; // 10MB
+	//! If this is the last buffer of the CSV File
+	bool last_buffer = false;
+
+private:
+	ClientContext &context;
+	//! Actual size can be smaller than the buffer size in case we allocate it too optimistically.
+	idx_t file_size;
+	//! We need to check for Byte Order Mark, to define the start position of this buffer
+	//! https://en.wikipedia.org/wiki/Byte_order_mark#UTF-8
+	idx_t start_position = 0;
+	//! If this is the first buffer of the CSV File
+	bool first_buffer = false;
+	//! Global position from the CSV File where this buffer starts
+	idx_t global_csv_start = 0;
+	//! Number of the file that is in this buffer
+	idx_t file_number = 0;
+	//! If we can seek in the file or not.
+	//! If we can't seek, this means we can't destroy the buffers
+	bool can_seek;
+	//! -------- Allocated Block ---------//
+	//! Block created in allocation
+	shared_ptr<BlockHandle> block;
+	BufferHandle handle;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp

@@ -0,0 +1,103 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/main/client_context.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_file_handle.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+
+namespace duckdb {
+class CSVBuffer;
+class CSVStateMachine;
+
+//! This class is used to manage the CSV buffers.  Buffers are cached when used for auto detection.
+//! When parsing, buffer are not cached and just returned.
+class CSVBufferManager {
+public:
+	CSVBufferManager(ClientContext &context, unique_ptr<CSVFileHandle> file_handle, const CSVReaderOptions &options,
+	                 idx_t file_idx = 0);
+	//! Returns a buffer from a buffer id (starting from 0). If it's in the auto-detection then we cache new buffers
+	//! Otherwise we remove them from the cache if they are already there, or just return them bypassing the cache.
+	unique_ptr<CSVBufferHandle> GetBuffer(const idx_t pos);
+	//! Returns the starting position of the first buffer
+	idx_t GetStartPos();
+	//! unique_ptr to the file handle, gets stolen after sniffing
+	unique_ptr<CSVFileHandle> file_handle;
+	//! Initializes the buffer manager, during it's construction/reset
+	void Initialize();
+
+	void UnpinBuffer(idx_t cache_idx);
+
+	ClientContext &context;
+	idx_t skip_rows = 0;
+	idx_t file_idx;
+	bool done = false;
+
+private:
+	//! Reads next buffer in reference to cached_buffers.front()
+	bool ReadNextAndCacheIt();
+	vector<shared_ptr<CSVBuffer>> cached_buffers;
+	shared_ptr<CSVBuffer> last_buffer;
+	idx_t global_csv_pos = 0;
+	//! The size of the buffer, if the csv file has a smaller size than this, we will use that instead to malloc less
+	idx_t buffer_size;
+	//! Starting position of first buffer
+	idx_t start_pos = 0;
+};
+
+class CSVBufferIterator {
+public:
+	explicit CSVBufferIterator(shared_ptr<CSVBufferManager> buffer_manager_p)
+	    : buffer_manager(std::move(buffer_manager_p)) {
+		cur_pos = buffer_manager->GetStartPos();
+	};
+
+	//! This functions templates an operation over the CSV File
+	template <class OP, class T>
+	inline bool Process(CSVStateMachine &machine, T &result) {
+
+		OP::Initialize(machine);
+		//! If current buffer is not set we try to get a new one
+		if (!cur_buffer_handle) {
+			cur_pos = 0;
+			if (cur_buffer_idx == 0) {
+				cur_pos = buffer_manager->GetStartPos();
+			}
+			cur_buffer_handle = buffer_manager->GetBuffer(cur_buffer_idx++);
+			D_ASSERT(cur_buffer_handle);
+		}
+		while (cur_buffer_handle) {
+			char *buffer_handle_ptr = cur_buffer_handle->Ptr();
+			while (cur_pos < cur_buffer_handle->actual_size) {
+				if (OP::Process(machine, result, buffer_handle_ptr[cur_pos], cur_pos)) {
+					//! Not-Done Processing the File, but the Operator is happy!
+					OP::Finalize(machine, result);
+					return false;
+				}
+				cur_pos++;
+			}
+			cur_buffer_handle = buffer_manager->GetBuffer(cur_buffer_idx++);
+			cur_pos = 0;
+		}
+		//! Done Processing the File
+		OP::Finalize(machine, result);
+		return true;
+	}
+	//! Returns true if the iterator is finished
+	bool Finished();
+	//! Resets the iterator
+	void Reset();
+
+private:
+	idx_t cur_pos = 0;
+	idx_t cur_buffer_idx = 0;
+	shared_ptr<CSVBufferManager> buffer_manager;
+	unique_ptr<CSVBufferHandle> cur_buffer_handle;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_file_handle.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/csv_file_handle.hpp
+// duckdb/execution/operator/scan/csv/csv_file_handle.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -20,15 +20,13 @@ class FileSystem;
 struct CSVFileHandle {
 public:
 	CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<FileHandle> file_handle_p, const string &path_p,
-	              FileCompressionType compression, bool enable_reset = true);
+	              FileCompressionType compression);
 
 	mutex main_mutex;
 
 public:
 	bool CanSeek();
 	void Seek(idx_t position);
-	idx_t SeekPosition();
-	void Reset();
 	bool OnDiskFile();
 
 	idx_t FileSize();
@@ -38,29 +36,24 @@ public:
 	idx_t Read(void *buffer, idx_t nr_bytes);
 
 	string ReadLine();
-	void DisableReset();
+
+	string GetFilePath();
 
 	static unique_ptr<FileHandle> OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
 	                                             FileCompressionType compression);
 	static unique_ptr<CSVFileHandle> OpenFile(FileSystem &fs, Allocator &allocator, const string &path,
-	                                          FileCompressionType compression, bool enable_reset);
+	                                          FileCompressionType compression);
 
 private:
-	FileSystem &fs;
-	Allocator &allocator;
 	unique_ptr<FileHandle> file_handle;
 	string path;
-	FileCompressionType compression;
-	bool reset_enabled = true;
 	bool can_seek = false;
 	bool on_disk_file = false;
 	idx_t file_size = 0;
-	// reset support
-	AllocatedData cached_buffer;
-	idx_t read_position = 0;
-	idx_t buffer_size = 0;
-	idx_t buffer_capacity = 0;
+
 	idx_t requested_bytes = 0;
+	//! If we finished reading the file
+	bool finished = false;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_line_info.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/csv_line_info.hpp
+// duckdb/execution/operator/scan/csv/csv_line_info.hpp
 //
 //
 //===----------------------------------------------------------------------===//
@@ -18,11 +18,12 @@ public:
 	bool CanItGetLine(idx_t file_idx, idx_t batch_idx);
 
 	//! Return the 1-indexed line number
-	idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true);
+	idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true,
+	              bool stop_at_first = true);
 	//! Verify if the CSV File was read correctly from [0,batch_idx] batches.
 	void Verify(idx_t file_idx, idx_t batch_idx, idx_t cur_first_pos);
-	//! Lines read per batch, <batch_index,count>
-	unordered_map<idx_t, idx_t> lines_read;
+	//! Lines read per batch, <file_index, <batch_index, count>>
+	vector<unordered_map<idx_t, idx_t>> lines_read;
 	//! Set of batches that have been initialized but are not yet finished.
 	vector<set<idx_t>> current_batches;
 	//! Pointer to CSV Reader Mutex

package/src/duckdb/src/include/duckdb/execution/operator/{persistent → scan/csv}/csv_reader_options.hpp

@@ -1,14 +1,14 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/execution/operator/persistent/csv_reader_options.hpp
+// duckdb/execution/operator/scan/csv/csv_reader_options.hpp
 //
 //
 //===----------------------------------------------------------------------===//
 
 #pragma once
 
-#include "duckdb/execution/operator/persistent/csv_buffer.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_buffer.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/function/scalar/strftime_format.hpp"
 #include "duckdb/common/types/value.hpp"
@@ -28,35 +28,75 @@ enum class NewLineIdentifier : uint8_t {
 
 enum class ParallelMode { AUTOMATIC = 0, PARALLEL = 1, SINGLE_THREADED = 2 };
 
-struct BufferedCSVReaderOptions {
+//! Struct that holds the configuration of a CSV State Machine
+//! Basically which char, quote and escape were used to generate it.
+struct CSVStateMachineOptions {
+	CSVStateMachineOptions() {};
+	CSVStateMachineOptions(char delimiter_p, char quote_p, char escape_p)
+	    : delimiter(delimiter_p), quote(quote_p), escape(escape_p) {};
+
+	//! Delimiter to separate columns within each line
+	char delimiter = ',';
+	//! Quote used for columns that contain reserved characters, e.g '
+	char quote = '\"';
+	//! Escape character to escape quote character
+	char escape = '\0';
+
+	bool operator==(const CSVStateMachineOptions &other) const {
+		return delimiter == other.delimiter && quote == other.quote && escape == other.escape;
+	}
+
+	void Serialize(FieldWriter &writer) const;
+	void Deserialize(FieldReader &reader);
+};
+
+struct DialectOptions {
+	CSVStateMachineOptions state_machine_options;
+	//! New Line separator
+	NewLineIdentifier new_line = NewLineIdentifier::NOT_SET;
+	//! Expected number of columns
+	idx_t num_cols = 0;
+	//! Whether or not the file has a header line
+	bool header = false;
+	//! The date format to use (if any is specified)
+	map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
+	//! Whether or not a type format is specified
+	map<LogicalTypeId, bool> has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}};
+	//! How many leading rows to skip
+	idx_t skip_rows = 0;
+	//! True start of the first CSV Buffer (After skipping empty lines, headers, notes and so on)
+	idx_t true_start = 0;
+
+	void Serialize(FieldWriter &writer) const;
+	void Deserialize(FieldReader &reader);
+};
+
+struct CSVReaderOptions {
 	//===--------------------------------------------------------------------===//
 	// CommonCSVOptions
 	//===--------------------------------------------------------------------===//
-
+	//! See struct above.
+	DialectOptions dialect_options;
 	//! Whether or not a delimiter was defined by the user
 	bool has_delimiter = false;
-	//! Delimiter to separate columns within each line
-	string delimiter = ",";
 	//! Whether or not a new_line was defined by the user
 	bool has_newline = false;
-	//! New Line separator
-	NewLineIdentifier new_line = NewLineIdentifier::NOT_SET;
 	//! Whether or not a quote was defined by the user
 	bool has_quote = false;
-	//! Quote used for columns that contain reserved characters, e.g., delimiter
-	string quote = "\"";
 	//! Whether or not an escape character was defined by the user
 	bool has_escape = false;
-	//! Escape character to escape quote character
-	string escape;
 	//! Whether or not a header information was given by the user
 	bool has_header = false;
-	//! Whether or not the file has a header line
-	bool header = false;
 	//! Whether or not we should ignore InvalidInput errors
 	bool ignore_errors = false;
-	//! Expected number of columns
-	idx_t num_cols = 0;
+	//! Rejects table name
+	string rejects_table_name;
+	//! Rejects table entry limit (0 = no limit)
+	idx_t rejects_limit = 0;
+	//! Columns to use as recovery key for rejected rows when reading with ignore_errors = true
+	vector<string> rejects_recovery_columns;
+	//! Index of the recovery columns
+	vector<idx_t> rejects_recovery_column_ids;
 	//! Number of samples to buffer
 	idx_t buffer_sample_size = STANDARD_VECTOR_SIZE * 50;
 	//! Specifies the string that represents a null value
@@ -84,9 +124,6 @@ struct BufferedCSVReaderOptions {
 	//===--------------------------------------------------------------------===//
 	// ReadCSVOptions
 	//===--------------------------------------------------------------------===//
-
-	//! How many leading rows to skip
-	idx_t skip_rows = 0;
 	//! Whether or not the skip_rows is set by the user
 	bool skip_rows_set = false;
 	//! Maximum CSV line size: specified because if we reach this amount, we likely have wrong delimiters (default: 2MB)
@@ -109,7 +146,7 @@ struct BufferedCSVReaderOptions {
 	//! Multi-file reader options
 	MultiFileReaderOptions file_options;
 	//! Buffer Size (Parallel Scan)
-	idx_t buffer_size = CSVBuffer::INITIAL_BUFFER_SIZE_COLOSSAL;
+	idx_t buffer_size = CSVBuffer::CSV_BUFFER_SIZE;
 	//! Decimal separator when reading as numeric
 	string decimal_separator = ".";
 	//! Whether or not to pad rows that do not have enough columns with NULL values
@@ -129,16 +166,13 @@ struct BufferedCSVReaderOptions {
 	string suffix;
 	string write_newline;
 
-	//! The date format to use (if any is specified)
-	std::map<LogicalTypeId, StrpTimeFormat> date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
 	//! The date format to use for writing (if any is specified)
-	std::map<LogicalTypeId, StrfTimeFormat> write_date_format = {{LogicalTypeId::DATE, {}},
-	                                                             {LogicalTypeId::TIMESTAMP, {}}};
-	//! Whether or not a type format is specified
-	std::map<LogicalTypeId, bool> has_format = {{LogicalTypeId::DATE, false}, {LogicalTypeId::TIMESTAMP, false}};
+	map<LogicalTypeId, StrfTimeFormat> write_date_format = {{LogicalTypeId::DATE, {}}, {LogicalTypeId::TIMESTAMP, {}}};
 
 	void Serialize(FieldWriter &writer) const;
 	void Deserialize(FieldReader &reader);
+	void FormatSerialize(FormatSerializer &serializer) const;
+	static CSVReaderOptions FormatDeserialize(FormatDeserializer &deserializer);
 
 	void SetCompression(const string &compression);
 	void SetHeader(bool has_header);
@@ -155,10 +189,9 @@ struct BufferedCSVReaderOptions {
 	//! set - argument(s) to the option
 	//! expected_names - names expected if the option is "columns"
 	void SetReadOption(const string &loption, const Value &value, vector<string> &expected_names);
-
 	void SetWriteOption(const string &loption, const Value &value);
 	void SetDateFormat(LogicalTypeId type, const string &format, bool read_format);
 
-	std::string ToString() const;
+	string ToString() const;
 };
 } // namespace duckdb