duckdb 0.10.2-dev0.0 → 0.10.2-dev3.0

package/src/duckdb/src/include/duckdb/common/vector_operations/unary_executor.hpp

@@ -50,7 +50,7 @@ template <class OP>
 struct UnaryStringOperator {
 	template <class INPUT_TYPE, class RESULT_TYPE>
 	static RESULT_TYPE Operation(INPUT_TYPE input, ValidityMask &mask, idx_t idx, void *dataptr) {
-		auto vector = (Vector *)dataptr;
+		auto vector = reinterpret_cast<Vector *>(dataptr);
 		return OP::template Operation<INPUT_TYPE, RESULT_TYPE>(input, *vector);
 	}
 };
@@ -192,7 +192,8 @@ public:
 
 	template <class INPUT_TYPE, class RESULT_TYPE, class FUNC = std::function<RESULT_TYPE(INPUT_TYPE)>>
 	static void Execute(Vector &input, Vector &result, idx_t count, FUNC fun) {
-		ExecuteStandard<INPUT_TYPE, RESULT_TYPE, UnaryLambdaWrapper, FUNC>(input, result, count, (void *)&fun, false);
+		ExecuteStandard<INPUT_TYPE, RESULT_TYPE, UnaryLambdaWrapper, FUNC>(input, result, count,
+		                                                                   reinterpret_cast<void *>(&fun), false);
 	}
 
 	template <class INPUT_TYPE, class RESULT_TYPE, class OP>

package/src/duckdb/src/include/duckdb/common/vector_operations/vector_operations.hpp

@@ -86,54 +86,72 @@ struct VectorOperations {
 	//===--------------------------------------------------------------------===//
 	// Select Comparisons
 	//===--------------------------------------------------------------------===//
-	static idx_t Equals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count, SelectionVector *true_sel,
-	                    SelectionVector *false_sel);
-	static idx_t NotEquals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                       SelectionVector *true_sel, SelectionVector *false_sel);
-	static idx_t GreaterThan(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                         SelectionVector *true_sel, SelectionVector *false_sel);
-	static idx_t GreaterThanEquals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                               SelectionVector *true_sel, SelectionVector *false_sel);
-	static idx_t LessThan(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                      SelectionVector *true_sel, SelectionVector *false_sel);
-	static idx_t LessThanEquals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                            SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t Equals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                    optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                    optional_ptr<ValidityMask> null_mask = nullptr);
+	static idx_t NotEquals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                       optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                       optional_ptr<ValidityMask> null_mask = nullptr);
+	static idx_t GreaterThan(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                         optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                         optional_ptr<ValidityMask> null_mask = nullptr);
+	static idx_t GreaterThanEquals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                               optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                               optional_ptr<ValidityMask> null_mask = nullptr);
+	static idx_t LessThan(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                      optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                      optional_ptr<ValidityMask> null_mask = nullptr);
+	static idx_t LessThanEquals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                            optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                            optional_ptr<ValidityMask> null_mask = nullptr);
 
 	// true := A != B with nulls being equal
-	static idx_t DistinctFrom(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                          SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t DistinctFrom(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                          optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel);
 	// true := A == B with nulls being equal
-	static idx_t NotDistinctFrom(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                             SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t NotDistinctFrom(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                             optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel);
 	// true := A > B with nulls being maximal
-	static idx_t DistinctGreaterThan(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                                 SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t DistinctGreaterThan(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                                 optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                                 optional_ptr<ValidityMask> null_mask = nullptr);
 	// true := A >= B with nulls being maximal
-	static idx_t DistinctGreaterThanEquals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                                       SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t DistinctGreaterThanEquals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel,
+	                                       idx_t count, optional_ptr<SelectionVector> true_sel,
+	                                       optional_ptr<SelectionVector> false_sel,
+	                                       optional_ptr<ValidityMask> null_mask = nullptr);
 	// true := A < B with nulls being maximal
-	static idx_t DistinctLessThan(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                              SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t DistinctLessThan(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                              optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                              optional_ptr<ValidityMask> null_mask = nullptr);
 	// true := A <= B with nulls being maximal
-	static idx_t DistinctLessThanEquals(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                                    SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t DistinctLessThanEquals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel,
+	                                    idx_t count, optional_ptr<SelectionVector> true_sel,
+	                                    optional_ptr<SelectionVector> false_sel,
+	                                    optional_ptr<ValidityMask> null_mask = nullptr);
 
 	// true := A > B with nulls being minimal
-	static idx_t DistinctGreaterThanNullsFirst(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                                           SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t DistinctGreaterThanNullsFirst(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel,
+	                                           idx_t count, optional_ptr<SelectionVector> true_sel,
+	                                           optional_ptr<SelectionVector> false_sel,
+	                                           optional_ptr<ValidityMask> null_mask = nullptr);
 	// true := A < B with nulls being minimal
-	static idx_t DistinctLessThanNullsFirst(Vector &left, Vector &right, const SelectionVector *sel, idx_t count,
-	                                        SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t DistinctLessThanNullsFirst(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel,
+	                                        idx_t count, optional_ptr<SelectionVector> true_sel,
+	                                        optional_ptr<SelectionVector> false_sel,
+	                                        optional_ptr<ValidityMask> null_mask = nullptr);
 
 	//===--------------------------------------------------------------------===//
 	// Nested Comparisons
 	//===--------------------------------------------------------------------===//
 	// true := A != B with nulls being equal
-	static idx_t NestedNotEquals(Vector &left, Vector &right, const SelectionVector &sel, idx_t count,
-	                             SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t NestedNotEquals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                             optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                             optional_ptr<ValidityMask> null_mask = nullptr);
 	// true := A == B with nulls being equal
-	static idx_t NestedEquals(Vector &left, Vector &right, const SelectionVector &sel, idx_t count,
-	                          SelectionVector *true_sel, SelectionVector *false_sel);
+	static idx_t NestedEquals(Vector &left, Vector &right, optional_ptr<const SelectionVector> sel, idx_t count,
+	                          optional_ptr<SelectionVector> true_sel, optional_ptr<SelectionVector> false_sel,
+	                          optional_ptr<ValidityMask> null_mask = nullptr);
 
 	//===--------------------------------------------------------------------===//
 	// Hash functions

package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp

@@ -19,9 +19,8 @@ class VirtualFileSystem : public FileSystem {
 public:
 	VirtualFileSystem();
 
-	unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = FileLockType::NO_LOCK,
-	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
-	                                FileOpener *opener = nullptr) override;
+	unique_ptr<FileHandle> OpenFile(const string &path, FileOpenFlags flags,
+	                                optional_ptr<FileOpener> opener = nullptr) override;
 
 	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
 	void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
@@ -39,22 +38,22 @@ public:
 	void FileSync(FileHandle &handle) override;
 
 	// need to look up correct fs for this
-	bool DirectoryExists(const string &directory) override;
-	void CreateDirectory(const string &directory) override;
+	bool DirectoryExists(const string &directory, optional_ptr<FileOpener> opener) override;
+	void CreateDirectory(const string &directory, optional_ptr<FileOpener> opener) override;
 
-	void RemoveDirectory(const string &directory) override;
+	void RemoveDirectory(const string &directory, optional_ptr<FileOpener> opener) override;
 
 	bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
 	               FileOpener *opener = nullptr) override;
 
-	void MoveFile(const string &source, const string &target) override;
+	void MoveFile(const string &source, const string &target, optional_ptr<FileOpener> opener) override;
 
-	bool FileExists(const string &filename) override;
+	bool FileExists(const string &filename, optional_ptr<FileOpener> opener) override;
 
-	bool IsPipe(const string &filename) override;
-	virtual void RemoveFile(const string &filename) override;
+	bool IsPipe(const string &filename, optional_ptr<FileOpener> opener) override;
+	void RemoveFile(const string &filename, optional_ptr<FileOpener> opener) override;
 
-	virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override;
+	vector<string> Glob(const string &path, FileOpener *opener = nullptr) override;
 
 	void RegisterSubSystem(unique_ptr<FileSystem> fs) override;
 

package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp

@@ -255,7 +255,7 @@ struct SumFun {
 struct SumNoOverflowFun {
 	static constexpr const char *Name = "sum_no_overflow";
 	static constexpr const char *Parameters = "arg";
-	static constexpr const char *Description = "Calculates the sum value for all tuples in arg without overflow checks.";
+	static constexpr const char *Description = "Internal only. Calculates the sum value for all tuples in arg without overflow checks.";
 	static constexpr const char *Example = "sum_no_overflow(A)";
 
 	static AggregateFunctionSet GetFunctions();

package/src/duckdb/src/include/duckdb/core_functions/aggregate/sum_helpers.hpp

@@ -65,6 +65,18 @@ struct RegularAdd {
 	}
 };
 
+struct HugeintAdd {
+	template <class STATE, class T>
+	static void AddNumber(STATE &state, T input) {
+		state.value = Hugeint::Add(state.value, input);
+	}
+
+	template <class STATE, class T>
+	static void AddConstant(STATE &state, T input, idx_t count) {
+		AddNumber(state, Hugeint::Multiply(input, count));
+	}
+};
+
 struct KahanAdd {
 	template <class STATE, class T>
 	static void AddNumber(STATE &state, T input) {
@@ -77,7 +89,7 @@ struct KahanAdd {
 	}
 };
 
-struct HugeintAdd {
+struct AddToHugeint {
 	static void AddValue(hugeint_t &result, uint64_t value, int positive) {
 		// integer summation taken from Tim Gubner et al. - Efficient Query Processing
 		// with Optimistically Compressed Hash Tables & Strings in the USSR

package/src/duckdb/src/include/duckdb/core_functions/function_list.hpp

@@ -27,7 +27,7 @@ struct StaticFunctionDefinition {
 	get_aggregate_function_t get_aggregate_function;
 	get_aggregate_function_set_t get_aggregate_function_set;
 
-	static StaticFunctionDefinition *GetFunctionList();
+	static const StaticFunctionDefinition *GetFunctionList();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/core_functions/lambda_functions.hpp

@@ -88,7 +88,8 @@ public:
 			result_validity = &FlatVector::Validity(result);
 
 			if (list_column.GetType().id() == LogicalTypeId::SQLNULL) {
-				result_validity->SetInvalid(0);
+				result.SetVectorType(VectorType::CONSTANT_VECTOR);
+				ConstantVector::SetNull(result, true);
 				result_is_null = true;
 				return;
 			}

package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp

@@ -29,7 +29,7 @@ struct FlushMoveState;
    stores them in the HT. It uses linear probing for collision resolution.
 */
 
-struct aggr_ht_entry_t {
+struct aggr_ht_entry_t { // NOLINT
 public:
 	explicit aggr_ht_entry_t(hash_t value_p) : value(value_p) {
 	}

package/src/duckdb/src/include/duckdb/execution/column_binding_resolver.hpp

@@ -19,7 +19,7 @@ namespace duckdb {
 //! are used within the execution engine
 class ColumnBindingResolver : public LogicalOperatorVisitor {
 public:
-	ColumnBindingResolver(bool verify_only = false);
+	explicit ColumnBindingResolver(bool verify_only = false);
 
 	void VisitOperator(LogicalOperator &op) override;
 	static void Verify(LogicalOperator &op);

package/src/duckdb/src/include/duckdb/execution/expression_executor.hpp

@@ -158,6 +158,6 @@ private:
 private:
 	// it is possible to create an expression executor without a ClientContext - but it should be avoided
 	DUCKDB_API ExpressionExecutor();
-	DUCKDB_API ExpressionExecutor(const vector<unique_ptr<Expression>> &exprs);
+	DUCKDB_API explicit ExpressionExecutor(const vector<unique_ptr<Expression>> &exprs);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp

@@ -10,7 +10,6 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
-#include "duckdb/common/cycle_counter.hpp"
 #include "duckdb/function/function.hpp"
 
 namespace duckdb {
@@ -29,7 +28,6 @@ struct ExpressionState {
 	vector<unique_ptr<ExpressionState>> child_states;
 	vector<LogicalType> types;
 	DataChunk intermediate_chunk;
-	CycleCounter profiler;
 
 public:
 	void AddChild(Expression *expr);
@@ -48,14 +46,14 @@ public:
 	}
 	template <class TARGET>
 	const TARGET &Cast() const {
-		D_ASSERT(dynamic_cast<const TARGET *>(this));
+		DynamicCastCheck<TARGET>(this);
 		return reinterpret_cast<const TARGET &>(*this);
 	}
 };
 
 struct ExecuteFunctionState : public ExpressionState {
 	ExecuteFunctionState(const Expression &expr, ExpressionExecutorState &root);
-	~ExecuteFunctionState();
+	~ExecuteFunctionState() override;
 
 	unique_ptr<FunctionLocalState> local_state;
 
@@ -70,7 +68,6 @@ struct ExpressionExecutorState {
 
 	unique_ptr<ExpressionState> root_state;
 	ExpressionExecutor *executor = nullptr;
-	CycleCounter profiler;
 
 	void Verify();
 };

package/src/duckdb/src/include/duckdb/execution/merge_sort_tree.hpp

@@ -226,7 +226,7 @@ protected:
 					out << ((i && i % level_width == 0) ? group_separator : separator);
 					out << std::setw(number_width) << level.first[i];
 				}
-				out << std::endl;
+				out << '\n';
 			}
 			// Print the pointers
 			if (!level.second.empty()) {
@@ -239,7 +239,7 @@ protected:
 						                                                                       : separator);
 						out << std::setw(number_width) << level.second[idx + child_nr];
 					}
-					out << std::endl;
+					out << '\n';
 				}
 			}
 			level_width *= FANOUT;
@@ -254,7 +254,7 @@ MergeSortTree<E, O, CMP, F, C>::MergeSortTree(Elements &&lowest_level, const CMP
 	const auto fanout = F;
 	const auto cascading = C;
 	const auto count = lowest_level.size();
-	tree.emplace_back(Level(lowest_level, Offsets()));
+	tree.emplace_back(Level(std::move(lowest_level), Offsets()));
 
 	const RunElement SENTINEL(MergeSortTraits<ElementType>::SENTINEL(), MergeSortTraits<idx_t>::SENTINEL());
 

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/aggregate_object.hpp

@@ -17,13 +17,13 @@ class BoundAggregateExpression;
 class BoundWindowExpression;
 
 struct FunctionDataWrapper {
-	FunctionDataWrapper(unique_ptr<FunctionData> function_data_p) : function_data(std::move(function_data_p)) {
+	explicit FunctionDataWrapper(unique_ptr<FunctionData> function_data_p) : function_data(std::move(function_data_p)) {
 	}
 
 	unique_ptr<FunctionData> function_data;
 };
 
-struct AggregateObject {
+struct AggregateObject { // NOLINT: work-around bug in clang-tidy
 	AggregateObject(AggregateFunction function, FunctionData *bind_data, idx_t child_count, idx_t payload_size,
 	                AggregateType aggr_type, PhysicalType return_type, Expression *filter = nullptr);
 	explicit AggregateObject(BoundAggregateExpression *aggr);

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/distinct_aggregate_data.hpp

@@ -44,7 +44,7 @@ private:
 
 struct DistinctAggregateData {
 public:
-	DistinctAggregateData(const DistinctAggregateCollectionInfo &info);
+	explicit DistinctAggregateData(const DistinctAggregateCollectionInfo &info);
 	DistinctAggregateData(const DistinctAggregateCollectionInfo &info, const GroupingSet &groups,
 	                      const vector<unique_ptr<Expression>> *group_expressions);
 	//! The data used by the hashtables

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp

@@ -252,7 +252,7 @@ protected:
 			Initialize();
 			initialized = true;
 		}
-		if (!iterator.done) {
+		if (!iterator.done && cur_buffer_handle) {
 			Process(result);
 		}
 		FinalizeChunkProcess();

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp

@@ -48,9 +48,6 @@ public:
 	ColumnCountScanner(shared_ptr<CSVBufferManager> buffer_manager, const shared_ptr<CSVStateMachine> &state_machine,
 	                   shared_ptr<CSVErrorHandler> error_handler);
 
-	~ColumnCountScanner() {
-	}
-
 	ColumnCountResult &ParseChunk() override;
 
 	ColumnCountResult &GetResult() override;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp

@@ -18,16 +18,17 @@ namespace duckdb {
 
 class CSVBufferHandle {
 public:
-	CSVBufferHandle(BufferHandle handle_p, idx_t actual_size_p, const bool is_final_buffer_p, idx_t file_idx_p,
-	                idx_t buffer_index_p)
-	    : handle(std::move(handle_p)), actual_size(actual_size_p), is_last_buffer(is_final_buffer_p),
-	      file_idx(file_idx_p), buffer_idx(buffer_index_p) {};
-	CSVBufferHandle() : actual_size(0), is_last_buffer(false), file_idx(0), buffer_idx(0) {};
+	CSVBufferHandle(BufferHandle handle_p, idx_t actual_size_p, idx_t requested_size_p, const bool is_final_buffer_p,
+	                idx_t file_idx_p, idx_t buffer_index_p)
+	    : handle(std::move(handle_p)), actual_size(actual_size_p), requested_size(requested_size_p),
+	      is_last_buffer(is_final_buffer_p), file_idx(file_idx_p), buffer_idx(buffer_index_p) {};
+	CSVBufferHandle() : actual_size(0), requested_size(0), is_last_buffer(false), file_idx(0), buffer_idx(0) {};
 	~CSVBufferHandle() {
 	}
 	//! Handle created during allocation
 	BufferHandle handle;
 	const idx_t actual_size;
+	const idx_t requested_size;
 	const bool is_last_buffer;
 	const idx_t file_idx;
 	const idx_t buffer_idx;
@@ -86,13 +87,15 @@ private:
 	ClientContext &context;
 	//! Actual size can be smaller than the buffer size in case we allocate it too optimistically.
 	idx_t actual_buffer_size;
+	idx_t requested_size;
 	//! Global position from the CSV File where this buffer starts
 	idx_t global_csv_start = 0;
 	//! Number of the file that is in this buffer
 	idx_t file_number = 0;
 	//! If we can seek in the file or not.
-	//! If we can't seek, this means we can't destroy the buffers
 	bool can_seek;
+	//! If this file is being fed by a pipe.
+	bool is_pipe;
 	//! Buffer Index, used as a batch index for insertion-order preservation
 	idx_t buffer_idx = 0;
 	//! -------- Allocated Block ---------//

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp

@@ -42,10 +42,12 @@ public:
 	//! once.
 	bool Done();
 
+	void ResetBufferManager();
 	string GetFilePath();
 
 	ClientContext &context;
 	idx_t skip_rows = 0;
+	bool sniffing = false;
 
 private:
 	//! Reads next buffer in reference to cached_buffers.front()
@@ -69,6 +71,7 @@ private:
 	//! If the file_handle used seek
 	bool has_seeked = false;
 	unordered_set<idx_t> reset_when_possible;
+	bool is_pipe;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp

@@ -37,61 +37,79 @@ public:
 };
 
 enum CSVErrorType : uint8_t {
-	CAST_ERROR = 0,                // If when casting a value from string to the column type fails
-	COLUMN_NAME_TYPE_MISMATCH = 1, // If there is a mismatch between Column Names and Types
-	INCORRECT_COLUMN_AMOUNT = 2,   // If the CSV is missing a column
-	UNTERMINATED_QUOTES = 3,       // If a quote is not terminated
-	SNIFFING = 4,          // If something went wrong during sniffing and was not possible to find suitable candidates
-	MAXIMUM_LINE_SIZE = 5, // Maximum line size was exceeded by a line in the CSV File
-	NULLPADDED_QUOTED_NEW_VALUE = 6, // If the null_padding option is set and we have quoted new values in parallel
-	INVALID_UNICODE = 7
-
+	CAST_ERROR = 0,                //! If when casting a value from string to the column type fails
+	COLUMN_NAME_TYPE_MISMATCH = 1, //! If there is a mismatch between Column Names and Types
+	TOO_FEW_COLUMNS = 2,           //! If the CSV has too few columns
+	TOO_MANY_COLUMNS = 3,          //! If the CSV has too many  column
+	UNTERMINATED_QUOTES = 4,       //! If a quote is not terminated
+	SNIFFING = 5,          //! If something went wrong during sniffing and was not possible to find suitable candidates
+	MAXIMUM_LINE_SIZE = 6, //! Maximum line size was exceeded by a line in the CSV File
+	NULLPADDED_QUOTED_NEW_VALUE = 7, //! If the null_padding option is set, and we have quoted new values in parallel
+	INVALID_UNICODE = 8              //! If we have invalid unicode values
 };
 
 class CSVError {
 public:
 	CSVError() {};
-	CSVError(string error_message, CSVErrorType type, idx_t column_idx, vector<Value> row, LinesPerBoundary error_info);
+	CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info,
+	         idx_t row_byte_position, int64_t byte_position, const CSVReaderOptions &reader_options,
+	         const string &fixes);
 	CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info);
 	//! Produces error messages for column name -> type mismatch.
 	static CSVError ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column, const vector<string> &names);
 	//! Produces error messages for casting errors
 	static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error,
-	                          idx_t column_idx, vector<Value> &row, LinesPerBoundary error_info, LogicalTypeId type);
+	                          idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position,
+	                          int64_t byte_position, LogicalTypeId type);
 	//! Produces error for when the line size exceeds the maximum line size option
-	static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info);
+	static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info,
+	                              string &csv_row, idx_t byte_position);
 	//! Produces error for when the sniffer couldn't find viable options
 	static CSVError SniffingError(string &file_path);
 	//! Produces error messages for unterminated quoted values
-	static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, string_t *vector_ptr,
-	                                        idx_t vector_line_start, idx_t current_column, LinesPerBoundary error_info);
+	static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column,
+	                                        LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position,
+	                                        int64_t byte_position);
 	//! Produces error messages for null_padding option is set and we have quoted new values in parallel
 	static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info);
 	//! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line
-	static CSVError IncorrectColumnAmountError(const CSVReaderOptions &options, string_t *vector_ptr,
-	                                           idx_t vector_line_start, idx_t actual_columns,
-	                                           LinesPerBoundary error_info);
-	//! Produces error message when we detect an invalid utf-8 value
-	static CSVError InvalidUTF8(const CSVReaderOptions &options, LinesPerBoundary error_info);
+	static CSVError IncorrectColumnAmountError(const CSVReaderOptions &state_machine, idx_t actual_columns,
+	                                           LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position,
+	                                           int64_t byte_position);
+	static CSVError InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info,
+	                            string &csv_row, idx_t row_byte_position, int64_t byte_position);
+
 	idx_t GetBoundaryIndex() {
 		return error_info.boundary_idx;
 	}
 
+	//! We might want to remove newline in errors if we are doing them for the rejects tables
+	void RemoveNewLine(string &error);
+
 	//! Actual error message
 	string error_message;
+	//! Full error message used in throws
+	//! 1. The Actual error
+	//! 2. How to fix it
+	//! 3. Options that generated the error
+	string full_error_message;
 	//! Error Type
 	CSVErrorType type;
 	//! Column Index where error happened
 	idx_t column_idx;
-	//! Values from the row where error happened
-	vector<Value> row;
+	//! Original CSV row where error happened
+	string csv_row;
 	//! Line information regarding this error
 	LinesPerBoundary error_info;
+	//! Byte position of where the row starts
+	idx_t row_byte_position;
+	//! Byte Position where error occurred.
+	int64_t byte_position;
 };
 
 class CSVErrorHandler {
 public:
-	CSVErrorHandler(bool ignore_errors = false);
+	explicit CSVErrorHandler(bool ignore_errors = false);
 	//! Throws the error
 	void Error(CSVError csv_error, bool force_error = false);
 	//! If we have a cached error, and we can now error, we error.

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp

@@ -28,6 +28,9 @@ public:
 	bool CanSeek();
 	void Seek(idx_t position);
 	bool OnDiskFile();
+	bool IsPipe();
+
+	void Reset();
 
 	idx_t FileSize();
 
@@ -50,6 +53,8 @@ private:
 	string path;
 	bool can_seek = false;
 	bool on_disk_file = false;
+	bool is_pipe = false;
+
 	idx_t file_size = 0;
 
 	idx_t requested_bytes = 0;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp

@@ -35,6 +35,7 @@ public:
 	const vector<string> &GetNames();
 	const vector<LogicalType> &GetTypes();
 	void InitializeProjection();
+	void Finish();
 
 	//! Initialize the actual names and types to be scanned from the file
 	void InitializeFileNamesTypes();

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp

@@ -24,10 +24,12 @@ class Deserializer;
 //! Wrapper for CSV Options that can be manually set by the user
 //! It is important to make this difference for options that can be automatically sniffed AND manually set.
 template <typename T>
-struct CSVOption {
+struct CSVOption { // NOLINT: work-around bug in clang-tidy
 public:
-	CSVOption(T value_p) : value(value_p) {};
-	CSVOption(T value_p, bool set_by_user_p) : value(value_p), set_by_user(set_by_user_p) {};
+	CSVOption(T value_p) : value(value_p) { // NOLINT: allow implicit conversion from value
+	}
+	CSVOption(T value_p, bool set_by_user_p) : value(value_p), set_by_user(set_by_user_p) {
+	}
 	CSVOption() {};
 
 	//! Sets value.
@@ -73,7 +75,7 @@ public:
 		return value != other;
 	}
 	//! Returns CSV Option value
-	const T GetValue() const {
+	inline const T GetValue() const {
 		return value;
 	}
 	bool IsSetByUser() const {