duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/column_count_scanner.hpp

@@ -17,29 +17,46 @@
 
 namespace duckdb {
 
+//! Result of a sniffed tuples using the column count scanner
+struct ColumnCount {
+	//! Number of columns found in a row
+	idx_t number_of_columns = 0;
+	//! If all values from this row onwards are null
+	bool last_value_always_empty = true;
+	//! If this row is potentially a comment
+	bool is_comment = false;
+	//! If this row is potentially a mid-line comment
+	bool is_mid_comment = false;
+};
+
 class ColumnCountResult : public ScannerResult {
 public:
-	ColumnCountResult(CSVStates &states, CSVStateMachine &state_machine);
-	inline idx_t &operator[](size_t index) {
+	ColumnCountResult(CSVStates &states, CSVStateMachine &state_machine, idx_t result_size);
+	inline ColumnCount &operator[](size_t index) {
 		return column_counts[index];
 	}
 
-	idx_t column_counts[STANDARD_VECTOR_SIZE];
+	vector<ColumnCount> column_counts;
 	idx_t current_column_count = 0;
 	bool error = false;
-	bool last_value_always_empty = true;
 	idx_t result_position = 0;
+	bool cur_line_starts_as_comment = false;
 
 	//! Adds a Value to the result
-	static inline void AddValue(ColumnCountResult &result, const idx_t buffer_pos);
+	static inline void AddValue(ColumnCountResult &result, idx_t buffer_pos);
 	//! Adds a Row to the result
-	static inline bool AddRow(ColumnCountResult &result, const idx_t buffer_pos);
+	static inline bool AddRow(ColumnCountResult &result, idx_t buffer_pos);
 	//! Behavior when hitting an invalid state
 	static inline void InvalidState(ColumnCountResult &result);
 	//! Handles QuotedNewline State
 	static inline void QuotedNewLine(ColumnCountResult &result);
 	//! Handles EmptyLine states
-	static inline bool EmptyLine(ColumnCountResult &result, const idx_t buffer_pos);
+	static inline bool EmptyLine(ColumnCountResult &result, idx_t buffer_pos);
+	//! Handles unset comment
+	static inline bool UnsetComment(ColumnCountResult &result, idx_t buffer_pos);
+
+	static inline void SetComment(ColumnCountResult &result, idx_t buffer_pos);
+
 	inline void InternalAddRow();
 };
 
@@ -47,7 +64,8 @@ public:
 class ColumnCountScanner : public BaseScanner {
 public:
 	ColumnCountScanner(shared_ptr<CSVBufferManager> buffer_manager, const shared_ptr<CSVStateMachine> &state_machine,
-	                   shared_ptr<CSVErrorHandler> error_handler);
+	                   shared_ptr<CSVErrorHandler> error_handler, idx_t result_size = STANDARD_VECTOR_SIZE,
+	                   CSVIterator iterator = {});
 
 	ColumnCountResult &ParseChunk() override;
 
@@ -63,6 +81,7 @@ private:
 	ColumnCountResult result;
 
 	idx_t column_count;
+	idx_t result_size;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp

@@ -22,7 +22,7 @@ class CSVStateMachine;
 class CSVBufferManager {
 public:
 	CSVBufferManager(ClientContext &context, const CSVReaderOptions &options, const string &file_path,
-	                 const idx_t file_idx);
+	                 const idx_t file_idx, bool per_file_single_threaded = false);
 	//! Returns a buffer from a buffer id (starting from 0). If it's in the auto-detection then we cache new buffers
 	//! Otherwise we remove them from the cache if they are already there, or just return them bypassing the cache.
 	shared_ptr<CSVBufferHandle> GetBuffer(const idx_t buffer_idx);
@@ -48,6 +48,7 @@ public:
 	ClientContext &context;
 	idx_t skip_rows = 0;
 	bool sniffing = false;
+	const bool per_file_single_threaded;
 
 private:
 	//! Reads next buffer in reference to cached_buffers.front()

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp

@@ -9,12 +9,12 @@
 #pragma once
 
 #include "duckdb/common/typedefs.hpp"
-#include "duckdb/common/types/hash.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "duckdb/common/unordered_map.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+#include "duckdb/execution/operator/csv_scanner/header_value.hpp"
 
 namespace duckdb {
 
@@ -53,38 +53,43 @@ public:
 	CSVError() {};
 	CSVError(string error_message, CSVErrorType type, idx_t column_idx, string csv_row, LinesPerBoundary error_info,
 	         idx_t row_byte_position, optional_idx byte_position, const CSVReaderOptions &reader_options,
-	         const string &fixes);
+	         const string &fixes, const string &current_path);
 	CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info);
 	//! Produces error messages for column name -> type mismatch.
 	static CSVError ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column, const vector<string> &names);
 	//! Produces error messages for casting errors
 	static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error,
 	                          idx_t column_idx, string &csv_row, LinesPerBoundary error_info, idx_t row_byte_position,
-	                          optional_idx byte_position, LogicalTypeId type);
+	                          optional_idx byte_position, LogicalTypeId type, const string &current_path);
 	//! Produces error for when the line size exceeds the maximum line size option
 	static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info,
-	                              string &csv_row, idx_t byte_position);
-	//! Produces error for when the sniffer couldn't find viable options
-	static CSVError SniffingError(string &file_path);
+	                              string &csv_row, idx_t byte_position, const string &current_path);
+	//! Produces an error message for a dialect sniffing error.
+	static CSVError SniffingError(const CSVReaderOptions &options, const string &search_space);
+	//! Produces an error message for a header sniffing error.
+	static CSVError HeaderSniffingError(const CSVReaderOptions &options, const vector<HeaderValue> &best_header_row,
+	                                    idx_t column_count, char delimiter);
 	//! Produces error messages for unterminated quoted values
 	static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, idx_t current_column,
 	                                        LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position,
-	                                        optional_idx byte_position);
+	                                        optional_idx byte_position, const string &current_path);
 	//! Produces error messages for null_padding option is set and we have quoted new values in parallel
-	static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info);
+	static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info,
+	                                const string &current_path);
 	//! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line
 	static CSVError IncorrectColumnAmountError(const CSVReaderOptions &state_machine, idx_t actual_columns,
 	                                           LinesPerBoundary error_info, string &csv_row, idx_t row_byte_position,
-	                                           optional_idx byte_position);
+	                                           optional_idx byte_position, const string &current_path);
 	static CSVError InvalidUTF8(const CSVReaderOptions &options, idx_t current_column, LinesPerBoundary error_info,
-	                            string &csv_row, idx_t row_byte_position, optional_idx byte_position);
+	                            string &csv_row, idx_t row_byte_position, optional_idx byte_position,
+	                            const string &current_path);
 
-	idx_t GetBoundaryIndex() {
+	idx_t GetBoundaryIndex() const {
 		return error_info.boundary_idx;
 	}
 
 	//! We might want to remove newline in errors if we are doing them for the rejects tables
-	void RemoveNewLine(string &error);
+	static void RemoveNewLine(string &error);
 
 	//! Actual error message
 	string error_message;
@@ -94,15 +99,15 @@ public:
 	//! 3. Options that generated the error
 	string full_error_message;
 	//! Error Type
-	CSVErrorType type;
+	CSVErrorType type {};
 	//! Column Index where error happened
-	idx_t column_idx;
+	idx_t column_idx {};
 	//! Original CSV row where error happened
 	string csv_row;
 	//! Line information regarding this error
 	LinesPerBoundary error_info;
 	//! Byte position of where the row starts
-	idx_t row_byte_position;
+	idx_t row_byte_position {};
 	//! Byte Position where error occurred.
 	optional_idx byte_position;
 };
@@ -116,17 +121,14 @@ public:
 	void ErrorIfNeeded();
 	//! Inserts a finished error info
 	void Insert(idx_t boundary_idx, idx_t rows);
-	//! Method that actually throws the error
-	void ThrowError(CSVError csv_error);
-	//! If we processed all boundaries before the one that error-ed
-	bool CanGetLine(idx_t boundary_index);
-	//! Return the 1-indexed line number
 	idx_t GetLine(const LinesPerBoundary &error_info);
 	void NewMaxLineSize(idx_t scan_line_size);
+	//! Returns true if there are any errors
+	bool AnyErrors();
 	//! Set of errors
 	map<LinesPerBoundary, vector<CSVError>> errors;
 
-	idx_t GetMaxLineLength() {
+	idx_t GetMaxLineLength() const {
 		return max_line_length;
 	}
 	void DontPrintErrorLine() {
@@ -134,8 +136,15 @@ public:
 	}
 
 private:
+	//! Private methods should always be locked by parent method.
 	//! If we should print the line of an error
-	bool PrintLineNumber(CSVError &error);
+	bool PrintLineNumber(const CSVError &error) const;
+	//! Method that actually throws the error
+	void ThrowError(const CSVError &csv_error);
+	//! If we processed all boundaries before the one that error-ed
+	bool CanGetLine(idx_t boundary_index);
+	//! Return the 1-indexed line number
+	idx_t GetLineInternal(const LinesPerBoundary &error_info);
 	//! CSV Error Handler Mutex
 	mutex main_mutex;
 	//! Map of <boundary indexes> -> lines read

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp

@@ -24,7 +24,6 @@ public:
 
 	mutex main_mutex;
 
-public:
 	bool CanSeek();
 	void Seek(idx_t position);
 	bool OnDiskFile();
@@ -46,7 +45,9 @@ public:
 	                                             FileCompressionType compression);
 	static unique_ptr<CSVFileHandle> OpenFile(FileSystem &fs, Allocator &allocator, const string &path,
 	                                          FileCompressionType compression);
-	bool uncompressed = false;
+	FileCompressionType compression_type;
+
+	double GetProgress();
 
 private:
 	unique_ptr<FileHandle> file_handle;
@@ -54,6 +55,7 @@ private:
 	bool can_seek = false;
 	bool on_disk_file = false;
 	bool is_pipe = false;
+	idx_t uncompressed_bytes_read = 0;
 
 	idx_t file_size = 0;
 

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp

@@ -12,33 +12,74 @@
 #include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_error.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_schema.hpp"
 
 namespace duckdb {
 struct ReadCSVData;
+class CSVFileScan;
+
+struct CSVUnionData {
+	~CSVUnionData();
+
+	string file_name;
+	vector<string> names;
+	vector<LogicalType> types;
+	CSVReaderOptions options;
+	unique_ptr<CSVFileScan> reader;
+
+	const string &GetFileName() {
+		return file_name;
+	}
+};
+
 //! Struct holding information over a CSV File we will scan
 class CSVFileScan {
+public:
+	using UNION_READER_DATA = unique_ptr<CSVUnionData>;
+
 public:
 	//! Constructor for when a CSV File Scan is being constructed over information acquired during sniffing
 	//! This means the options are alreadu set, and the buffer manager is already up and runinng.
 	CSVFileScan(ClientContext &context, shared_ptr<CSVBufferManager> buffer_manager,
 	            shared_ptr<CSVStateMachine> state_machine, const CSVReaderOptions &options,
-	            const ReadCSVData &bind_data, const vector<column_t> &column_ids, vector<LogicalType> &file_schema);
+	            const ReadCSVData &bind_data, const vector<column_t> &column_ids, CSVSchema &file_schema);
 	//! Constructor for new CSV Files, we must initialize the buffer manager and the state machine
 	//! Path to this file
 	CSVFileScan(ClientContext &context, const string &file_path, const CSVReaderOptions &options, const idx_t file_idx,
-	            const ReadCSVData &bind_data, const vector<column_t> &column_ids,
-	            const vector<LogicalType> &file_schema);
+	            const ReadCSVData &bind_data, const vector<column_t> &column_ids, CSVSchema &file_schema,
+	            bool per_file_single_threaded);
 
-	CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options);
+	CSVFileScan(ClientContext &context, const string &file_name, const CSVReaderOptions &options);
 
-	const string &GetFileName();
+	void SetStart();
+	const string &GetFileName() const;
 	const vector<string> &GetNames();
 	const vector<LogicalType> &GetTypes();
 	void InitializeProjection();
 	void Finish();
 
+	static unique_ptr<CSVUnionData> StoreUnionReader(unique_ptr<CSVFileScan> scan_p, idx_t file_idx) {
+		auto data = make_uniq<CSVUnionData>();
+		if (file_idx == 0) {
+			data->file_name = scan_p->file_path;
+			data->options = scan_p->options;
+			data->names = scan_p->names;
+			data->types = scan_p->types;
+			data->reader = std::move(scan_p);
+		} else {
+			data->file_name = scan_p->file_path;
+			data->options = std::move(scan_p->options);
+			data->names = std::move(scan_p->names);
+			data->types = std::move(scan_p->types);
+		}
+		data->options.auto_detect = false;
+		return data;
+	}
+
 	//! Initialize the actual names and types to be scanned from the file
 	void InitializeFileNamesTypes();
+
+public:
 	const string file_path;
 	//! File Index
 	idx_t file_idx;
@@ -67,5 +108,7 @@ public:
 
 	//! Options for this CSV Reader
 	CSVReaderOptions options;
+
+	CSVIterator start_iterator;
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp

@@ -13,9 +13,11 @@
 namespace duckdb {
 
 enum class NewLineIdentifier : uint8_t {
-	SINGLE = 1,   // Either \r or \n
+	SINGLE_N = 1, // \n
 	CARRY_ON = 2, // \r\n
-	NOT_SET = 3
+	NOT_SET = 3,
+	SINGLE_R = 4 // \r
+
 };
 
 class Serializer;
@@ -131,8 +133,10 @@ private:
 
 	std::string FormatValueInternal(const NewLineIdentifier &val) const {
 		switch (val) {
-		case NewLineIdentifier::SINGLE:
+		case NewLineIdentifier::SINGLE_N:
 			return "\\n";
+		case NewLineIdentifier::SINGLE_R:
+			return "\\r";
 		case NewLineIdentifier::CARRY_ON:
 			return "\\r\\n";
 		case NewLineIdentifier::NOT_SET:

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp

@@ -31,6 +31,7 @@ struct DialectOptions {
 	                                                             {LogicalTypeId::TIMESTAMP, {}}};
 	//! How many leading rows to skip
 	CSVOption<idx_t> skip_rows = 0;
+	idx_t rows_until_header = 0;
 };
 
 struct CSVReaderOptions {
@@ -39,18 +40,18 @@ struct CSVReaderOptions {
 	//===--------------------------------------------------------------------===//
 	//! See struct above.
 	DialectOptions dialect_options;
-	//! Whether or not we should ignore InvalidInput errors
+	//! Whether we should ignore InvalidInput errors
 	CSVOption<bool> ignore_errors = false;
 	//! Whether we store CSV Errors in the rejects table or not
 	CSVOption<bool> store_rejects = false;
 	//! Rejects table name (Name of the table the store rejects errors)
 	CSVOption<string> rejects_table_name = {"reject_errors"};
-	//! Rejects Scan name name  (Name of the table the store rejects scans)
+	//! Rejects Scan name  (Name of the table the store rejects scans)
 	CSVOption<string> rejects_scan_name = {"reject_scans"};
 	//! Rejects table entry limit (0 = no limit)
 	idx_t rejects_limit = 0;
 	//! Number of samples to buffer
-	idx_t buffer_sample_size = (idx_t)STANDARD_VECTOR_SIZE * 50;
+	idx_t buffer_sample_size = static_cast<idx_t>(STANDARD_VECTOR_SIZE * 50);
 	//! Specifies the strings that represents a null value
 	vector<string> null_str = {""};
 	//! Whether file is compressed or not, and if so which compression type
@@ -58,6 +59,7 @@ struct CSVReaderOptions {
 	FileCompressionType compression = FileCompressionType::AUTO_DETECT;
 	//! Option to convert quoted values to NULL values
 	bool allow_quoted_nulls = true;
+	char comment;
 
 	//===--------------------------------------------------------------------===//
 	// CSVAutoOptions
@@ -68,6 +70,8 @@ struct CSVReaderOptions {
 	vector<LogicalType> sql_type_list;
 	//! User-defined name list
 	vector<string> name_list;
+	//! If the names and types were set by the columns parameter
+	bool columns_set = false;
 	//! Types considered as candidates for auto detection ordered by descending specificity (~ from high to low)
 	vector<LogicalType> auto_type_candidates = {LogicalType::VARCHAR,   LogicalType::DOUBLE, LogicalType::BIGINT,
 	                                            LogicalType::TIMESTAMP, LogicalType::DATE,   LogicalType::TIME,
@@ -88,8 +92,10 @@ struct CSVReaderOptions {
 	unordered_set<string> force_not_null_names;
 	//! True, if column with that index must skip null check
 	vector<bool> force_not_null;
+	//! Result size of sniffing phases
+	static constexpr idx_t sniff_size = 2048;
 	//! Number of sample chunks used in auto-detection
-	idx_t sample_size_chunks = 20480 / STANDARD_VECTOR_SIZE;
+	idx_t sample_size_chunks = 20480 / sniff_size;
 	//! Consider all columns to be of type varchar
 	bool all_varchar = false;
 	//! Whether or not to automatically detect dialect and datatypes
@@ -136,18 +142,21 @@ struct CSVReaderOptions {
 	string GetEscape() const;
 	void SetEscape(const string &escape);
 
-	int64_t GetSkipRows() const;
+	idx_t GetSkipRows() const;
+
 	void SetSkipRows(int64_t rows);
 
-	string GetQuote() const;
 	void SetQuote(const string &quote);
+	string GetQuote() const;
+	void SetComment(const string &comment);
+	string GetComment() const;
 	void SetDelimiter(const string &delimiter);
 	string GetDelimiter() const;
 
 	//! If we can safely ignore errors (i.e., they are being ignored and not being stored in a rejects table)
 	bool IgnoreErrors() const;
 
-	NewLineIdentifier GetNewline() const;
+	string GetNewline() const;
 	void SetNewline(const string &input);
 	//! Set an option that is supported by both reading and writing functions, called by
 	//! the SetReadOption and SetWriteOption methods
@@ -160,17 +169,18 @@ struct CSVReaderOptions {
 	void SetWriteOption(const string &loption, const Value &value);
 	void SetDateFormat(LogicalTypeId type, const string &format, bool read_format);
 	void ToNamedParameters(named_parameter_map_t &out);
-	void FromNamedParameters(named_parameter_map_t &in, ClientContext &context, vector<LogicalType> &return_types,
-	                         vector<string> &names);
+	void FromNamedParameters(named_parameter_map_t &in, ClientContext &context);
 
-	string ToString() const;
+	string ToString(const string &current_file_path) const;
 	//! If the type for column with idx i was manually set
 	bool WasTypeManuallySet(idx_t i) const;
 
-	string NewLineIdentifierToString() {
+	string NewLineIdentifierToString() const {
 		switch (dialect_options.state_machine_options.new_line.GetValue()) {
-		case NewLineIdentifier::SINGLE:
+		case NewLineIdentifier::SINGLE_N:
 			return "\\n";
+		case NewLineIdentifier::SINGLE_R:
+			return "\\r";
 		case NewLineIdentifier::CARRY_ON:
 			return "\\r\\n";
 		default:

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_schema.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types.hpp"
+
+namespace duckdb {
+//! Basic CSV Column Info
+struct CSVColumnInfo {
+	CSVColumnInfo(string &name_p, LogicalType &type_p) : name(name_p), type(type_p) {
+	}
+	string name;
+	LogicalType type;
+};
+
+//! Basic CSV Schema
+struct CSVSchema {
+	void Initialize(vector<string> &names, vector<LogicalType> &types, const string &file_path);
+	bool Empty() const;
+	bool SchemasMatch(string &error_message, vector<string> &names, vector<LogicalType> &types,
+	                  const string &cur_file_path);
+
+private:
+	static bool CanWeCastIt(LogicalTypeId source, LogicalTypeId destination);
+	vector<CSVColumnInfo> columns;
+	unordered_map<string, idx_t> name_idx_map;
+	string file_path;
+};
+} // namespace duckdb