duckdb 0.10.1-dev9.0 → 0.10.2-dev0.0

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_casting.hpp

@@ -0,0 +1,154 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_casting.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+#include "duckdb/common/vector_operations/unary_executor.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
+#include "duckdb/common/operator/decimal_cast_operators.hpp"
+
+namespace duckdb {
+class CSVCast {
+	template <class OP, class T>
+	static bool TemplatedTryCastFloatingVector(const CSVReaderOptions &options, Vector &input_vector,
+	                                           Vector &result_vector, idx_t count, CastParameters &parameters,
+	                                           idx_t &line_error) {
+		D_ASSERT(input_vector.GetType().id() == LogicalTypeId::VARCHAR);
+		bool all_converted = true;
+		idx_t row = 0;
+		UnaryExecutor::Execute<string_t, T>(input_vector, result_vector, count, [&](string_t input) {
+			T result;
+			if (!OP::Operation(input, result, parameters)) {
+				line_error = row;
+				all_converted = false;
+			} else {
+				row++;
+			}
+			return result;
+		});
+		return all_converted;
+	}
+
+	template <class OP, class T>
+	static bool TemplatedTryCastDecimalVector(const CSVReaderOptions &options, Vector &input_vector,
+	                                          Vector &result_vector, idx_t count, CastParameters &parameters,
+	                                          uint8_t width, uint8_t scale, idx_t &line_error) {
+		D_ASSERT(input_vector.GetType().id() == LogicalTypeId::VARCHAR);
+		bool all_converted = true;
+		auto &validity_mask = FlatVector::Validity(result_vector);
+		idx_t cur_line = 0;
+		UnaryExecutor::Execute<string_t, T>(input_vector, result_vector, count, [&](string_t input) {
+			T result;
+			if (!OP::Operation(input, result, parameters, width, scale)) {
+				if (all_converted) {
+					line_error = cur_line;
+				}
+				validity_mask.SetInvalid(cur_line);
+				all_converted = false;
+				cur_line++;
+			} else {
+				cur_line++;
+			}
+			return result;
+		});
+		return all_converted;
+	}
+
+	struct TryCastDateOperator {
+		static bool Operation(const map<LogicalTypeId, CSVOption<StrpTimeFormat>> &options, string_t input,
+		                      date_t &result, string &error_message) {
+			return options.at(LogicalTypeId::DATE).GetValue().TryParseDate(input, result, error_message);
+		}
+	};
+
+	struct TryCastTimestampOperator {
+		static bool Operation(const map<LogicalTypeId, CSVOption<StrpTimeFormat>> &options, string_t input,
+		                      timestamp_t &result, string &error_message) {
+			return options.at(LogicalTypeId::TIMESTAMP).GetValue().TryParseTimestamp(input, result, error_message);
+		}
+	};
+
+	template <class OP, class T>
+	static bool TemplatedTryCastDateVector(const map<LogicalTypeId, CSVOption<StrpTimeFormat>> &options,
+	                                       Vector &input_vector, Vector &result_vector, idx_t count,
+	                                       CastParameters &parameters, idx_t &line_error, bool nullify_error) {
+		D_ASSERT(input_vector.GetType().id() == LogicalTypeId::VARCHAR);
+		bool all_converted = true;
+		idx_t cur_line = 0;
+		auto &validity_mask = FlatVector::Validity(result_vector);
+		UnaryExecutor::Execute<string_t, T>(input_vector, result_vector, count, [&](string_t input) {
+			T result;
+			if (!OP::Operation(options, input, result, *parameters.error_message)) {
+				if (all_converted) {
+					line_error = cur_line;
+				}
+				if (nullify_error) {
+					validity_mask.SetInvalid(cur_line);
+				}
+				all_converted = false;
+			}
+			cur_line++;
+			return result;
+		});
+		return all_converted;
+	}
+
+public:
+	static bool TryCastDateVector(const map<LogicalTypeId, CSVOption<StrpTimeFormat>> &options, Vector &input_vector,
+	                              Vector &result_vector, idx_t count, CastParameters &parameters, idx_t &line_error,
+	                              bool nullify_error = false) {
+		return TemplatedTryCastDateVector<TryCastDateOperator, date_t>(options, input_vector, result_vector, count,
+		                                                               parameters, line_error, nullify_error);
+	}
+	static bool TryCastTimestampVector(const map<LogicalTypeId, CSVOption<StrpTimeFormat>> &options,
+	                                   Vector &input_vector, Vector &result_vector, idx_t count,
+	                                   CastParameters &parameters, bool nullify_error = false) {
+		idx_t line_error;
+		return TemplatedTryCastDateVector<TryCastTimestampOperator, timestamp_t>(
+		    options, input_vector, result_vector, count, parameters, line_error, nullify_error);
+	}
+	static bool TryCastFloatingVectorCommaSeparated(const CSVReaderOptions &options, Vector &input_vector,
+	                                                Vector &result_vector, idx_t count, CastParameters &parameters,
+	                                                const LogicalType &result_type, idx_t &line_error) {
+		switch (result_type.InternalType()) {
+		case PhysicalType::DOUBLE:
+			return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, double>(
+			    options, input_vector, result_vector, count, parameters, line_error);
+		case PhysicalType::FLOAT:
+			return TemplatedTryCastFloatingVector<TryCastErrorMessageCommaSeparated, float>(
+			    options, input_vector, result_vector, count, parameters, line_error);
+		default:
+			throw InternalException("Unimplemented physical type for floating");
+		}
+	}
+
+	static bool TryCastDecimalVectorCommaSeparated(const CSVReaderOptions &options, Vector &input_vector,
+	                                               Vector &result_vector, idx_t count, CastParameters &parameters,
+	                                               const LogicalType &result_type, idx_t &line_error) {
+		auto width = DecimalType::GetWidth(result_type);
+		auto scale = DecimalType::GetScale(result_type);
+		switch (result_type.InternalType()) {
+		case PhysicalType::INT16:
+			return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int16_t>(
+			    options, input_vector, result_vector, count, parameters, width, scale, line_error);
+		case PhysicalType::INT32:
+			return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int32_t>(
+			    options, input_vector, result_vector, count, parameters, width, scale, line_error);
+		case PhysicalType::INT64:
+			return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, int64_t>(
+			    options, input_vector, result_vector, count, parameters, width, scale, line_error);
+		case PhysicalType::INT128:
+			return TemplatedTryCastDecimalVector<TryCastToDecimalCommaSeparated, hugeint_t>(
+			    options, input_vector, result_vector, count, parameters, width, scale, line_error);
+		default:
+			throw InternalException("Unimplemented physical type for decimal");
+		}
+	}
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp

@@ -0,0 +1,130 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_error.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/typedefs.hpp"
+#include "duckdb/common/types/hash.hpp"
+#include "duckdb/common/mutex.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/types/string_type.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
+
+namespace duckdb {
+
+//! Object that holds information on how many lines each csv batch read.
+
+class LinesPerBoundary {
+public:
+	LinesPerBoundary();
+	LinesPerBoundary(idx_t boundary_idx, idx_t lines_in_batch);
+
+	idx_t boundary_idx = 0;
+	idx_t lines_in_batch = 0;
+
+	bool operator<(const LinesPerBoundary &other) const {
+		if (boundary_idx < other.boundary_idx) {
+			return true;
+		}
+		return lines_in_batch < other.lines_in_batch;
+	}
+};
+
+enum CSVErrorType : uint8_t {
+	CAST_ERROR = 0,                // If when casting a value from string to the column type fails
+	COLUMN_NAME_TYPE_MISMATCH = 1, // If there is a mismatch between Column Names and Types
+	INCORRECT_COLUMN_AMOUNT = 2,   // If the CSV is missing a column
+	UNTERMINATED_QUOTES = 3,       // If a quote is not terminated
+	SNIFFING = 4,          // If something went wrong during sniffing and was not possible to find suitable candidates
+	MAXIMUM_LINE_SIZE = 5, // Maximum line size was exceeded by a line in the CSV File
+	NULLPADDED_QUOTED_NEW_VALUE = 6, // If the null_padding option is set and we have quoted new values in parallel
+	INVALID_UNICODE = 7
+
+};
+
+class CSVError {
+public:
+	CSVError() {};
+	CSVError(string error_message, CSVErrorType type, idx_t column_idx, vector<Value> row, LinesPerBoundary error_info);
+	CSVError(string error_message, CSVErrorType type, LinesPerBoundary error_info);
+	//! Produces error messages for column name -> type mismatch.
+	static CSVError ColumnTypesError(case_insensitive_map_t<idx_t> sql_types_per_column, const vector<string> &names);
+	//! Produces error messages for casting errors
+	static CSVError CastError(const CSVReaderOptions &options, string &column_name, string &cast_error,
+	                          idx_t column_idx, vector<Value> &row, LinesPerBoundary error_info, LogicalTypeId type);
+	//! Produces error for when the line size exceeds the maximum line size option
+	static CSVError LineSizeError(const CSVReaderOptions &options, idx_t actual_size, LinesPerBoundary error_info);
+	//! Produces error for when the sniffer couldn't find viable options
+	static CSVError SniffingError(string &file_path);
+	//! Produces error messages for unterminated quoted values
+	static CSVError UnterminatedQuotesError(const CSVReaderOptions &options, string_t *vector_ptr,
+	                                        idx_t vector_line_start, idx_t current_column, LinesPerBoundary error_info);
+	//! Produces error messages for null_padding option is set and we have quoted new values in parallel
+	static CSVError NullPaddingFail(const CSVReaderOptions &options, LinesPerBoundary error_info);
+	//! Produces error for incorrect (e.g., smaller and lower than the predefined) number of columns in a CSV Line
+	static CSVError IncorrectColumnAmountError(const CSVReaderOptions &options, string_t *vector_ptr,
+	                                           idx_t vector_line_start, idx_t actual_columns,
+	                                           LinesPerBoundary error_info);
+	//! Produces error message when we detect an invalid utf-8 value
+	static CSVError InvalidUTF8(const CSVReaderOptions &options, LinesPerBoundary error_info);
+	idx_t GetBoundaryIndex() {
+		return error_info.boundary_idx;
+	}
+
+	//! Actual error message
+	string error_message;
+	//! Error Type
+	CSVErrorType type;
+	//! Column Index where error happened
+	idx_t column_idx;
+	//! Values from the row where error happened
+	vector<Value> row;
+	//! Line information regarding this error
+	LinesPerBoundary error_info;
+};
+
+class CSVErrorHandler {
+public:
+	CSVErrorHandler(bool ignore_errors = false);
+	//! Throws the error
+	void Error(CSVError csv_error, bool force_error = false);
+	//! If we have a cached error, and we can now error, we error.
+	void ErrorIfNeeded();
+	//! Inserts a finished error info
+	void Insert(idx_t boundary_idx, idx_t rows);
+	//! Method that actually throws the error
+	void ThrowError(CSVError csv_error);
+	//! If we processed all boundaries before the one that error-ed
+	bool CanGetLine(idx_t boundary_index);
+	//! Return the 1-indexed line number
+	idx_t GetLine(const LinesPerBoundary &error_info);
+	void NewMaxLineSize(idx_t scan_line_size);
+	//! Set of errors
+	map<LinesPerBoundary, vector<CSVError>> errors;
+
+	idx_t GetMaxLineLength() {
+		return max_line_length;
+	}
+	void DontPrintErrorLine() {
+		print_line = false;
+	}
+
+private:
+	//! If we should print the line of an error
+	bool PrintLineNumber(CSVError &error);
+	//! CSV Error Handler Mutex
+	mutex main_mutex;
+	//! Map of <boundary indexes> -> lines read
+	unordered_map<idx_t, LinesPerBoundary> lines_per_batch_map;
+	idx_t max_line_length = 0;
+	bool ignore_errors = false;
+	bool print_line = true;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp

@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_file_handle.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/file_system.hpp"
+#include "duckdb/common/mutex.hpp"
+#include "duckdb/common/helper.hpp"
+#include "duckdb/common/allocator.hpp"
+
+namespace duckdb {
+class Allocator;
+class FileSystem;
+
+struct CSVFileHandle {
+public:
+	CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<FileHandle> file_handle_p, const string &path_p,
+	              FileCompressionType compression);
+
+	mutex main_mutex;
+
+public:
+	bool CanSeek();
+	void Seek(idx_t position);
+	bool OnDiskFile();
+
+	idx_t FileSize();
+
+	bool FinishedReading();
+
+	idx_t Read(void *buffer, idx_t nr_bytes);
+
+	string ReadLine();
+
+	string GetFilePath();
+
+	static unique_ptr<FileHandle> OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
+	                                             FileCompressionType compression);
+	static unique_ptr<CSVFileHandle> OpenFile(FileSystem &fs, Allocator &allocator, const string &path,
+	                                          FileCompressionType compression);
+	bool uncompressed = false;
+
+private:
+	unique_ptr<FileHandle> file_handle;
+	string path;
+	bool can_seek = false;
+	bool on_disk_file = false;
+	idx_t file_size = 0;
+
+	idx_t requested_bytes = 0;
+	//! If we finished reading the file
+	bool finished = false;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp

@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_error.hpp"
+
+namespace duckdb {
+struct ReadCSVData;
+//! Struct holding information over a CSV File we will scan
+class CSVFileScan {
+public:
+	//! Constructor for when a CSV File Scan is being constructed over information acquired during sniffing
+	//! This means the options are alreadu set, and the buffer manager is already up and runinng.
+	CSVFileScan(ClientContext &context, shared_ptr<CSVBufferManager> buffer_manager,
+	            shared_ptr<CSVStateMachine> state_machine, const CSVReaderOptions &options,
+	            const ReadCSVData &bind_data, const vector<column_t> &column_ids, vector<LogicalType> &file_schema);
+	//! Constructor for new CSV Files, we must initialize the buffer manager and the state machine
+	//! Path to this file
+	CSVFileScan(ClientContext &context, const string &file_path, const CSVReaderOptions &options, const idx_t file_idx,
+	            const ReadCSVData &bind_data, const vector<column_t> &column_ids,
+	            const vector<LogicalType> &file_schema);
+
+	CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options);
+
+	const string &GetFileName();
+	const vector<string> &GetNames();
+	const vector<LogicalType> &GetTypes();
+	void InitializeProjection();
+
+	//! Initialize the actual names and types to be scanned from the file
+	void InitializeFileNamesTypes();
+	const string file_path;
+	//! File Index
+	idx_t file_idx;
+	//! Buffer Manager for the CSV File
+	shared_ptr<CSVBufferManager> buffer_manager;
+	//! State Machine for this file
+	shared_ptr<CSVStateMachine> state_machine;
+	//! How many bytes were read up to this point
+	atomic<idx_t> bytes_read {0};
+	//! Size of this file
+	idx_t file_size;
+	//! Line Info used in error messages
+	shared_ptr<CSVErrorHandler> error_handler;
+	//! Whether or not this is an on-disk file
+	bool on_disk_file = true;
+
+	vector<string> names;
+	vector<LogicalType> types;
+	MultiFileReaderData reader_data;
+
+	vector<LogicalType> file_types;
+
+	//! Variables to handle projection pushdown
+	set<idx_t> projected_columns;
+	std::vector<std::pair<idx_t, idx_t>> projection_ids;
+
+	//! Options for this CSV Reader
+	CSVReaderOptions options;
+};
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp

@@ -0,0 +1,155 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/csv_scanner/csv_option.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/function/scalar/strftime_format.hpp"
+
+namespace duckdb {
+
+enum class NewLineIdentifier : uint8_t {
+	SINGLE = 1,   // Either \r or \n
+	CARRY_ON = 2, // \r\n
+	NOT_SET = 3
+};
+
+class Serializer;
+class Deserializer;
+
+//! Wrapper for CSV Options that can be manually set by the user
+//! It is important to make this difference for options that can be automatically sniffed AND manually set.
+template <typename T>
+struct CSVOption {
+public:
+	CSVOption(T value_p) : value(value_p) {};
+	CSVOption(T value_p, bool set_by_user_p) : value(value_p), set_by_user(set_by_user_p) {};
+	CSVOption() {};
+
+	//! Sets value.
+	//! If by user it also toggles the set_by user flag
+	void Set(T value_p, bool by_user = true) {
+		D_ASSERT(!(by_user && set_by_user));
+		if (!set_by_user) {
+			// If it's not set by user we can change the value
+			value = value_p;
+			set_by_user = by_user;
+		}
+	}
+
+	//! Sets value.
+	//! If by user it also toggles the set_by user flag
+	void Set(CSVOption value_p, bool by_user = true) {
+		D_ASSERT(!(by_user && set_by_user));
+		if (!set_by_user) {
+			// If it's not set by user we can change the value
+			value = value_p;
+			set_by_user = by_user;
+		}
+	}
+
+	// this is due to a hacky implementation in the read csv relation
+	void ChangeSetByUserTrue() {
+		set_by_user = true;
+	}
+
+	bool operator==(const CSVOption &other) const {
+		return value == other.value;
+	}
+
+	bool operator!=(const CSVOption &other) const {
+		return value != other.value;
+	}
+
+	bool operator==(const T &other) const {
+		return value == other;
+	}
+
+	bool operator!=(const T &other) const {
+		return value != other;
+	}
+	//! Returns CSV Option value
+	const T GetValue() const {
+		return value;
+	}
+	bool IsSetByUser() const {
+		return set_by_user;
+	}
+
+	//! Returns a formatted string with information regarding how this option was set
+	string FormatSet() const {
+		if (set_by_user) {
+			return "(Set By User)";
+		}
+		return "(Auto-Detected)";
+	}
+
+	//! Returns a formatted string with the actual value of this option
+	string FormatValue() const {
+		return FormatValueInternal(value);
+	}
+
+	//! Serializes CSV Option
+	DUCKDB_API void Serialize(Serializer &serializer) const;
+
+	//! Deserializes CSV Option
+	DUCKDB_API static CSVOption<T> Deserialize(Deserializer &deserializer);
+
+private:
+	//! If this option was manually set by the user
+	bool set_by_user = false;
+	T value;
+
+	//! --------------------------------------------------- //
+	//! Functions used to convert a value to a string
+	//! --------------------------------------------------- //
+
+	template <typename U>
+	std::string FormatValueInternal(const U &val) const {
+		throw InternalException("Type not accepted as CSV Option.");
+	}
+
+	std::string FormatValueInternal(const std::string &val) const {
+		return val;
+	}
+
+	std::string FormatValueInternal(const idx_t &val) const {
+		return to_string(val);
+	}
+
+	std::string FormatValueInternal(const char &val) const {
+		string char_val;
+		char_val += val;
+		return char_val;
+	}
+
+	std::string FormatValueInternal(const NewLineIdentifier &val) const {
+		switch (val) {
+		case NewLineIdentifier::SINGLE:
+			return "\\n";
+		case NewLineIdentifier::CARRY_ON:
+			return "\\r\\n";
+		case NewLineIdentifier::NOT_SET:
+			return "Single-Line File";
+		default:
+			throw InternalException("Invalid Newline Detected.");
+		}
+	}
+
+	std::string FormatValueInternal(const StrpTimeFormat &val) const {
+		return val.format_specifier;
+	}
+
+	std::string FormatValueInternal(const bool &val) const {
+		if (val) {
+			return "true";
+		}
+		return "false";
+	}
+};
+
+} // namespace duckdb