duckdb 0.6.2-dev971.0 → 0.7.1-dev100.0

package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp

@@ -14,11 +14,15 @@ namespace duckdb {
 
 class Node256 : public Node {
 public:
+	//! Empty Node256
 	explicit Node256();
+	//! ART pointers to the child nodes
 	ARTPointer children[256];
 
 public:
 	static Node256 *New();
+	//! Returns the memory size of the Node256
+	idx_t MemorySize(ART &art, const bool &recurse) override;
 	//! Get position of a specific byte, returns DConstants::INVALID_INDEX if not exists
 	idx_t GetChildPos(uint8_t k) override;
 	//! Get the position of the first child that is greater or equal to the specific byte, or DConstants::INVALID_INDEX
@@ -28,18 +32,23 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node256 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
 	void ReplaceChildPointer(idx_t pos, Node *node) override;
+	//! Returns whether the child at pos is in memory
+	bool ChildIsInMemory(idx_t pos) override;
 
 	//! Insert a new child node at key_byte into the Node256
-	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
+	static void InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) shrink to Node48
-	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node256 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
+	static void EraseChild(ART &art, Node *&node, idx_t pos);
+
 	//! Returns the size (maximum capacity) of the Node256
-	static idx_t GetSize();
+	static constexpr idx_t GetSize() {
+		return 256;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp

@@ -14,14 +14,17 @@ namespace duckdb {
 
 class Node4 : public Node {
 public:
-	Node4();
-
+	//! Empty Node4
+	explicit Node4();
+	//! Array containing all partial key bytes
 	uint8_t key[4];
-	// Pointers to the child nodes
+	//! ART pointers to the child nodes
 	ARTPointer children[4];
 
 public:
 	static Node4 *New();
+	//! Returns the memory size of the Node4
+	idx_t MemorySize(ART &art, const bool &recurse) override;
 	//! Get position of a byte, returns DConstants::INVALID_INDEX if not exists
 	idx_t GetChildPos(uint8_t k) override;
 	//! Get the position of the first child that is greater or equal to the specific byte, or DConstants::INVALID_INDEX
@@ -31,18 +34,23 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node4 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
 	void ReplaceChildPointer(idx_t pos, Node *node) override;
+	//! Returns whether the child at pos is in memory
+	bool ChildIsInMemory(idx_t pos) override;
 
 	//! Insert a new child node at key_byte into the Node4
-	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
+	static void InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) merge with last child
-	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node4 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
+	static void EraseChild(ART &art, Node *&node, idx_t pos);
+
 	//! Returns the size (maximum capacity) of the Node4
-	static idx_t GetSize();
+	static constexpr idx_t GetSize() {
+		return 4;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp

@@ -14,12 +14,17 @@ namespace duckdb {
 
 class Node48 : public Node {
 public:
+	//! Empty Node48
 	explicit Node48();
+	//! Array containing all possible partial key bytes, those not set have an EMPTY_MARKER
 	uint8_t child_index[256];
+	//! ART pointers to the child nodes
 	ARTPointer children[48];
 
 public:
 	static Node48 *New();
+	//! Returns the memory size of the Node48
+	idx_t MemorySize(ART &art, const bool &recurse) override;
 	//! Get position of a specific byte, returns DConstants::INVALID_INDEX if not exists
 	idx_t GetChildPos(uint8_t k) override;
 	//! Get the position of the first child that is greater or equal to the specific byte, or DConstants::INVALID_INDEX
@@ -29,18 +34,22 @@ public:
 	idx_t GetMin() override;
 	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
 	idx_t GetNextPos(idx_t pos) override;
+	//! Get the next position in the node, or DConstants::INVALID_INDEX if there is no next position
+	idx_t GetNextPosAndByte(idx_t pos, uint8_t &byte) override;
 	//! Get Node48 child
 	Node *GetChild(ART &art, idx_t pos) override;
 	//! Replace child pointer
 	void ReplaceChildPointer(idx_t pos, Node *node) override;
+	//! Returns whether the child at pos is in memory
+	bool ChildIsInMemory(idx_t pos) override;
 
 	//! Insert a new child node at key_byte into the Node48
-	static void InsertChild(Node *&node, uint8_t key_byte, Node *new_child);
+	static void InsertChild(ART &art, Node *&node, uint8_t key_byte, Node *new_child);
 	//! Erase the child at pos and (if necessary) shrink to Node16
-	static void EraseChild(Node *&node, int pos, ART &art);
-	//! Merge Node48 into l_node
-	static bool Merge(MergeInfo &info, idx_t depth, Node *&l_parent, idx_t l_pos);
+	static void EraseChild(ART &art, Node *&node, idx_t pos);
 	//! Returns the size (maximum capacity) of the Node48
-	static idx_t GetSize();
+	static constexpr idx_t GetSize() {
+		return 48;
+	}
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp

@@ -12,45 +12,51 @@
 #include "duckdb/storage/meta_block_writer.hpp"
 
 namespace duckdb {
+class ART;
+
 class Prefix {
 	static constexpr idx_t PREFIX_INLINE_BYTES = 8;
 
 public:
+	//! Empty prefix
 	Prefix();
-	// Prefix created from key starting on `depth`
+	//! Construct prefix from key starting at depth
 	Prefix(Key &key, uint32_t depth, uint32_t size);
-	// Prefix created from other prefix up to size
+	//! Construct prefix from other prefix up to size
 	Prefix(Prefix &other_prefix, uint32_t size);
 	~Prefix();
 
-	// Returns the Prefix's size
-	uint32_t Size() const;
-	//! Return a pointer to the prefix data
+	//! Returns the prefix size
+	inline uint32_t Size() const {
+		return size;
+	}
+	//! Returns the memory size of the prefix
+	idx_t MemorySize();
+	//! Returns a pointer to the prefix data
 	uint8_t *GetPrefixData();
+	//! Returns a const pointer to the prefix data
 	const uint8_t *GetPrefixData() const;
 
-	// Subscript operator
+	//! Subscript operator
 	uint8_t &operator[](idx_t idx);
-
-	// Assign operator
+	//! Assign operator
 	Prefix &operator=(const Prefix &src);
-
-	// Move operator
+	//! Move operator
 	Prefix &operator=(Prefix &&other) noexcept;
 
-	// Concatenate Prefix with a key and another prefix
-	// Used when deleting a Node.
-	// other.prefix + key + this->Prefix
-	void Concatenate(uint8_t key, Prefix &other);
-	// Reduces the prefix in n elements, and returns what would be the first one as a key
-	uint8_t Reduce(uint32_t n);
-	// Serializes Prefix
+	//! Concatenate prefix with a partial key byte and another prefix: other.prefix + byte + this->prefix
+	//! Used when deleting a node
+	void Concatenate(ART &art, uint8_t key, Prefix &other);
+	//! Reduces the prefix in n bytes, and returns the new first byte
+	uint8_t Reduce(ART &art, uint32_t n);
+
+	//! Serializes the prefix
 	void Serialize(duckdb::MetaBlockWriter &writer);
-	// Deserializes Prefix
+	//! Deserializes the prefix
 	void Deserialize(duckdb::MetaBlockReader &reader);
 
-	// Compare the key with the prefix of the node, return the position where it mismatches
-	uint32_t KeyMismatchPosition(Key &key, uint64_t depth);
+	//! Compare the key with the prefix of the node, return the position where they mismatch
+	uint32_t KeyMismatchPosition(Key &key, uint32_t depth);
 	//! Compare this prefix to another prefix, return the position where they mismatch, or size otherwise
 	uint32_t MismatchPosition(Prefix &other);
 

package/src/duckdb/src/include/duckdb/execution/index/art/swizzleable_pointer.hpp

@@ -27,6 +27,8 @@ public:
 	//! Transforms from Node* to uint64_t
 	SwizzleablePointer &operator=(const Node *ptr);
 
+	//! Checks if pointer is swizzled
+	bool IsSwizzled();
 	//! Unswizzle the pointer (if possible)
 	Node *Unswizzle(ART &art);
 
@@ -34,7 +36,7 @@ public:
 		return pointer;
 	}
 
-	//! Deletes the underlying object (if necessary) and set the pointer to null_ptr
+	//! Deletes the underlying object (if necessary) and set the pointer to nullptr
 	void Reset();
 
 private:
@@ -42,10 +44,8 @@ private:
 
 	friend bool operator!=(const SwizzleablePointer &s_ptr, const uint64_t &ptr);
 
-	//! Extracts block info from swizzled pointer
+	//! Extracts the block info from swizzled pointer
 	BlockPointer GetSwizzledBlockInfo();
-	//! Checks if pointer is swizzled
-	bool IsSwizzled();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp

@@ -16,7 +16,7 @@ namespace duckdb {
 class PhysicalExplainAnalyze : public PhysicalOperator {
 public:
 	PhysicalExplainAnalyze(vector<LogicalType> types)
-	    : PhysicalOperator(PhysicalOperatorType::EXPLAIN_ANALYZE, move(types), 1) {
+	    : PhysicalOperator(PhysicalOperatorType::EXPLAIN_ANALYZE, std::move(types), 1) {
 	}
 
 public:

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit_percent.hpp

@@ -19,9 +19,9 @@ public:
 	PhysicalLimitPercent(vector<LogicalType> types, double limit_percent, idx_t offset,
 	                     unique_ptr<Expression> limit_expression, unique_ptr<Expression> offset_expression,
 	                     idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::LIMIT_PERCENT, move(types), estimated_cardinality),
-	      limit_percent(limit_percent), offset_value(offset), limit_expression(move(limit_expression)),
-	      offset_expression(move(offset_expression)) {
+	    : PhysicalOperator(PhysicalOperatorType::LIMIT_PERCENT, std::move(types), estimated_cardinality),
+	      limit_percent(limit_percent), offset_value(offset), limit_expression(std::move(limit_expression)),
+	      offset_expression(std::move(offset_expression)) {
 	}
 
 	double limit_percent;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_load.hpp

@@ -18,7 +18,7 @@ class PhysicalLoad : public PhysicalOperator {
 public:
 	explicit PhysicalLoad(unique_ptr<LoadInfo> info, idx_t estimated_cardinality)
 	    : PhysicalOperator(PhysicalOperatorType::LOAD, {LogicalType::BOOLEAN}, estimated_cardinality),
-	      info(move(info)) {
+	      info(std::move(info)) {
 	}
 
 	unique_ptr<LoadInfo> info;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_pragma.hpp

@@ -19,7 +19,7 @@ class PhysicalPragma : public PhysicalOperator {
 public:
 	PhysicalPragma(PragmaFunction function_p, PragmaInfo info_p, idx_t estimated_cardinality)
 	    : PhysicalOperator(PhysicalOperatorType::PRAGMA, {LogicalType::BOOLEAN}, estimated_cardinality),
-	      function(move(function_p)), info(move(info_p)) {
+	      function(std::move(function_p)), info(std::move(info_p)) {
 	}
 
 	//! The pragma function to call

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_prepare.hpp

@@ -18,7 +18,7 @@ class PhysicalPrepare : public PhysicalOperator {
 public:
 	PhysicalPrepare(string name, shared_ptr<PreparedStatementData> prepared, idx_t estimated_cardinality)
 	    : PhysicalOperator(PhysicalOperatorType::PREPARE, {LogicalType::BOOLEAN}, estimated_cardinality), name(name),
-	      prepared(move(prepared)) {
+	      prepared(std::move(prepared)) {
 	}
 
 	string name;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_reservoir_sample.hpp

@@ -17,8 +17,8 @@ namespace duckdb {
 class PhysicalReservoirSample : public PhysicalOperator {
 public:
 	PhysicalReservoirSample(vector<LogicalType> types, unique_ptr<SampleOptions> options, idx_t estimated_cardinality)
-	    : PhysicalOperator(PhysicalOperatorType::RESERVOIR_SAMPLE, move(types), estimated_cardinality),
-	      options(move(options)) {
+	    : PhysicalOperator(PhysicalOperatorType::RESERVOIR_SAMPLE, std::move(types), estimated_cardinality),
+	      options(std::move(options)) {
 	}
 
 	unique_ptr<SampleOptions> options;

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_set.hpp

@@ -30,13 +30,13 @@ public:
 	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
 	             LocalSourceState &lstate) const override;
 
+	static void SetExtensionVariable(ClientContext &context, ExtensionOption &extension_option, const string &name,
+	                                 SetScope scope, const Value &value);
+
 public:
 	const std::string name;
 	const Value value;
 	const SetScope scope;
-
-private:
-	void SetExtensionVariable(ExecutionContext &context, DBConfig &config, ExtensionOption &extension_option) const;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_transaction.hpp

@@ -18,7 +18,7 @@ class PhysicalTransaction : public PhysicalOperator {
 public:
 	explicit PhysicalTransaction(unique_ptr<TransactionInfo> info, idx_t estimated_cardinality)
 	    : PhysicalOperator(PhysicalOperatorType::TRANSACTION, {LogicalType::BOOLEAN}, estimated_cardinality),
-	      info(move(info)) {
+	      info(std::move(info)) {
 	}
 
 	unique_ptr<TransactionInfo> info;

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_cross_product.hpp

@@ -56,7 +56,7 @@ public:
 
 class CrossProductExecutor {
 public:
-	CrossProductExecutor(ColumnDataCollection &rhs);
+	explicit CrossProductExecutor(ColumnDataCollection &rhs);
 
 	OperatorResultType Execute(DataChunk &input, DataChunk &output);
 

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_positional_join.hpp

@@ -0,0 +1,50 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/join/physical_positional_join.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/physical_operator.hpp"
+#include "duckdb/common/types/column_data_collection.hpp"
+
+namespace duckdb {
+
+//! PhysicalPositionalJoin represents a cross product between two tables
+class PhysicalPositionalJoin : public PhysicalOperator {
+public:
+	PhysicalPositionalJoin(vector<LogicalType> types, unique_ptr<PhysicalOperator> left,
+	                       unique_ptr<PhysicalOperator> right, idx_t estimated_cardinality);
+
+public:
+	// Operator Interface
+	OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
+	                           GlobalOperatorState &gstate, OperatorState &state) const override;
+
+public:
+	// Source interface
+	void GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	             LocalSourceState &lstate) const override;
+
+	bool IsSource() const override {
+		return true;
+	}
+
+public:
+	// Sink Interface
+	unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
+	SinkResultType Sink(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate,
+	                    DataChunk &input) const override;
+
+	bool IsSink() const override {
+		return true;
+	}
+
+public:
+	void BuildPipelines(Pipeline &current, MetaPipeline &meta_pipeline) override;
+	vector<const PhysicalOperator *> GetSources() const override;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp

@@ -35,22 +35,22 @@ class BaseCSVReader {
 public:
 	BaseCSVReader(ClientContext &context, BufferedCSVReaderOptions options,
 	              const vector<LogicalType> &requested_types = vector<LogicalType>());
-
-	BaseCSVReader(FileSystem &fs, Allocator &allocator, FileOpener *opener, BufferedCSVReaderOptions options,
-	              const vector<LogicalType> &requested_types = vector<LogicalType>());
 	~BaseCSVReader();
 
+	ClientContext &context;
 	FileSystem &fs;
 	Allocator &allocator;
 	FileOpener *opener;
 	BufferedCSVReaderOptions options;
-	vector<LogicalType> sql_types;
-	vector<string> col_names;
+	vector<LogicalType> return_types;
+	vector<string> names;
 
 	//! remap parse_chunk col to insert_chunk col, because when
 	//! union_by_name option on insert_chunk may have more cols
 	vector<idx_t> insert_cols_idx;
-	vector<idx_t> insert_nulls_idx;
+	vector<idx_t> union_idx_map;
+	vector<bool> union_null_cols;
+	vector<LogicalType> union_col_types;
 
 	idx_t linenr = 0;
 	bool linenr_estimated = false;
@@ -70,10 +70,6 @@ public:
 
 	ParserMode mode;
 
-public:
-	//! Fill nulls into the cols that mismtach union names
-	void SetNullUnionCols(DataChunk &insert_chunk);
-
 protected:
 	//! Initializes the parse_chunk with varchar columns and aligns info with new number of cols
 	void InitParseChunk(idx_t num_cols);
@@ -99,8 +95,11 @@ protected:
 	void VerifyUTF8(idx_t col_idx, idx_t row_idx, DataChunk &chunk, int64_t offset = 0);
 	static string GetLineNumberStr(idx_t linenr, bool linenr_estimated);
 
+	//! Sets the newline delimiter
+	void SetNewLineDelimiter(bool carry = false, bool carry_followed_by_nl = false);
+
 protected:
-	//! Whether or not the current row's columns have overflown sql_types.size()
+	//! Whether or not the current row's columns have overflown return_types.size()
 	bool error_column_overflow = false;
 	//! Number of sniffed columns - only used when auto-detecting
 	vector<idx_t> sniffed_column_counts;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/buffered_csv_reader.hpp

@@ -55,7 +55,7 @@ class BufferedCSVReader : public BaseCSVReader {
 public:
 	BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options,
 	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
-	BufferedCSVReader(FileSystem &fs, Allocator &allocator, FileOpener *opener, BufferedCSVReaderOptions options,
+	BufferedCSVReader(ClientContext &context, string filename, BufferedCSVReaderOptions options,
 	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
 	~BufferedCSVReader();
 
@@ -73,6 +73,7 @@ public:
 public:
 	//! Extract a single DataChunk from the CSV file and stores it in insert_chunk
 	void ParseCSV(DataChunk &insert_chunk);
+	static string ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column, const vector<string> &names);
 
 private:
 	//! Initialize Parser

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_buffer.hpp

@@ -20,13 +20,15 @@ public:
 	static constexpr idx_t INITIAL_BUFFER_SIZE_COLOSSAL = 32000000; // 32MB
 
 	//! Constructor for Initial Buffer
-	CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle);
+	CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
+	          idx_t &global_csv_current_position);
 
 	//! Constructor for `Next()` Buffers
-	CSVBuffer(ClientContext &context, BufferHandle handle, idx_t buffer_size_p, idx_t actual_size_p, bool final_buffer);
+	CSVBuffer(ClientContext &context, BufferHandle handle, idx_t buffer_size_p, idx_t actual_size_p, bool final_buffer,
+	          idx_t global_csv_current_position);
 
 	//! Creates a new buffer with the next part of the CSV File
-	unique_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t set_buffer_size);
+	unique_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t &global_csv_current_position);
 
 	//! Gets the buffer actual size
 	idx_t GetBufferSize();
@@ -40,6 +42,8 @@ public:
 	//! If this buffer is the first buffer of the CSV File
 	bool IsCSVFileFirstBuffer();
 
+	idx_t GetCSVGlobalStart();
+
 	BufferHandle AllocateBuffer(idx_t buffer_size);
 
 	char *Ptr() {
@@ -59,5 +63,7 @@ private:
 	bool last_buffer = false;
 	//! If this is the first buffer of the CSV File
 	bool first_buffer = false;
+	//! Global position from the CSV File where this buffer starts
+	idx_t global_csv_start = 0;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_file_handle.hpp

@@ -15,7 +15,7 @@ namespace duckdb {
 
 struct CSVFileHandle {
 public:
-	explicit CSVFileHandle(unique_ptr<FileHandle> file_handle_p) : file_handle(move(file_handle_p)) {
+	explicit CSVFileHandle(unique_ptr<FileHandle> file_handle_p) : file_handle(std::move(file_handle_p)) {
 		can_seek = file_handle->CanSeek();
 		plain_file_source = file_handle->OnDiskFile() && can_seek;
 		file_size = file_handle->GetFileSize();
@@ -98,7 +98,7 @@ public:
 					if (buffer_size > 0) {
 						memcpy(new_buffer.get(), cached_buffer.get(), buffer_size);
 					}
-					cached_buffer = move(new_buffer);
+					cached_buffer = std::move(new_buffer);
 				}
 				memcpy(cached_buffer.get() + buffer_size, (char *)buffer + result_offset, bytes_read);
 				buffer_size += bytes_read;

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp

@@ -13,9 +13,17 @@
 #include "duckdb/function/scalar/strftime.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/field_writer.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
 
 namespace duckdb {
 
+enum NewLineIdentifier {
+	SINGLE = 1,   // Either \r or \n
+	CARRY_ON = 2, // \r\n
+	MIX = 3,      // Hippie-Land, can't run it multithreaded
+	NOT_SET = 4
+};
+
 struct BufferedCSVReaderOptions {
 	//===--------------------------------------------------------------------===//
 	// CommonCSVOptions
@@ -25,7 +33,16 @@ struct BufferedCSVReaderOptions {
 	bool has_delimiter = false;
 	//! Delimiter to separate columns within each line
 	string delimiter = ",";
-	//! Whether or not a quote sign was defined by the user
+	//! Whether or not a new_line was defined by the user
+	bool has_newline = false;
+	//! New Line separator
+	NewLineIdentifier new_line = NewLineIdentifier::NOT_SET;
+
+	//! Whether or not an option was provided for parallel
+	bool has_parallel = false;
+	//! Whether or not the read will use the ParallelCSVReader
+	bool use_parallel = false;
+	//! Whether or not a quote was defined by the user
 	bool has_quote = false;
 	//! Quote used for columns that contain reserved characters, e.g., delimiter
 	string quote = "\"";
@@ -54,8 +71,10 @@ struct BufferedCSVReaderOptions {
 	//===--------------------------------------------------------------------===//
 	// CSVAutoOptions
 	//===--------------------------------------------------------------------===//
-	//! SQL Types defined per specific column
-	unordered_map<string, LogicalType> sql_types_per_column;
+	//! SQL Type list mapping of name to SQL type index in sql_type_list
+	case_insensitive_map_t<idx_t> sql_types_per_column;
+	//! User-defined SQL type list
+	vector<LogicalType> sql_type_list;
 	//===--------------------------------------------------------------------===//
 	// ReadCSVOptions
 	//===--------------------------------------------------------------------===//
@@ -87,6 +106,8 @@ struct BufferedCSVReaderOptions {
 	bool union_by_name = false;
 	//! Buffer Size (Parallel Scan)
 	idx_t buffer_size = CSVBuffer::INITIAL_BUFFER_SIZE_COLOSSAL;
+	//! Decimal separator when reading as numeric
+	string decimal_separator = ".";
 
 	//===--------------------------------------------------------------------===//
 	// WriteCSVOptions
@@ -106,7 +127,14 @@ struct BufferedCSVReaderOptions {
 	void Serialize(FieldWriter &writer) const;
 	void Deserialize(FieldReader &reader);
 
+	void SetCompression(const string &compression);
+	void SetHeader(bool has_header);
+	void SetEscape(const string &escape);
+	void SetQuote(const string &quote);
 	void SetDelimiter(const string &delimiter);
+	void SetParallel(bool use_parallel);
+
+	void SetNewline(const string &input);
 	//! Set an option that is supported by both reading and writing functions, called by
 	//! the SetReadOption and SetWriteOption methods
 	bool SetBaseOption(const string &loption, const Value &value);

package/src/duckdb/src/include/duckdb/execution/operator/persistent/parallel_csv_reader.hpp

@@ -21,7 +21,7 @@ namespace duckdb {
 struct CSVBufferRead {
 	CSVBufferRead(shared_ptr<CSVBuffer> buffer_p, idx_t buffer_start_p, idx_t buffer_end_p, idx_t batch_index,
 	              idx_t estimated_linenr)
-	    : buffer(move(buffer_p)), buffer_start(buffer_start_p), buffer_end(buffer_end_p), batch_index(batch_index),
+	    : buffer(std::move(buffer_p)), buffer_start(buffer_start_p), buffer_end(buffer_end_p), batch_index(batch_index),
 	      estimated_linenr(estimated_linenr) {
 		if (buffer) {
 			if (buffer_end > buffer->GetBufferSize()) {
@@ -76,7 +76,7 @@ struct CSVBufferRead {
 			for (; cur_pos < length; cur_pos++) {
 				intersection[cur_pos] = next_buffer_ptr[nxt_buffer_pos++];
 			}
-			intersections.emplace_back(move(intersection));
+			intersections.emplace_back(std::move(intersection));
 			return string_t(intersections.back().get(), length);
 		}
 	}
@@ -91,6 +91,10 @@ struct CSVBufferRead {
 	idx_t estimated_linenr;
 };
 
+struct VerificationPositions {
+	idx_t beginning_of_first_line = 0;
+	idx_t end_of_last_line = 0;
+};
 //! Buffered CSV reader is a class that reads values from a stream and parses them as a CSV file
 class ParallelCSVReader : public BaseCSVReader {
 public:
@@ -111,7 +115,10 @@ public:
 	//! If this flag is set, it means we are about to try to read our last row.
 	bool reached_remainder_state = false;
 
+	bool finished = false;
+
 	unique_ptr<CSVBufferRead> buffer;
+	VerificationPositions GetVerificationPositions();
 
 public:
 	void SetBufferRead(unique_ptr<CSVBufferRead> buffer);
@@ -134,8 +141,13 @@ private:
 	//! when changing the buffer end the first time.
 	//! It returns FALSE if the parser should jump to the final state of parsing or not
 	bool BufferRemainder();
+
+	bool NewLineDelimiter(bool carry, bool carry_followed_by_nl, bool first_char);
+
 	//! Parses a CSV file with a one-byte delimiter, escape and quote character
 	bool TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message, bool try_add_line = false);
+	//! Position of the first read line and last read line for verification purposes
+	VerificationPositions verification_positions;
 };
 
 } // namespace duckdb