duckdb 1.0.1-dev22.0 → 1.0.1-dev27.0

package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp

@@ -1,5 +1,6 @@
 #include "duckdb/execution/operator/csv_scanner/csv_state_machine.hpp"
 #include "duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
 
 namespace duckdb {
 
@@ -8,6 +9,13 @@ void InitializeTransitionArray(StateMachine &transition_array, const CSVState cu
 		transition_array[i][static_cast<uint8_t>(cur_state)] = state;
 	}
 }
+
+// Shift and OR to replicate across all bytes
+void ShiftAndReplicateBits(uint64_t &value) {
+	value |= value << 8;
+	value |= value << 16;
+	value |= value << 32;
+}
 void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_options) {
 	D_ASSERT(state_machine_cache.find(state_machine_options) == state_machine_cache.end());
 	// Initialize transition array with default values to the Standard option
@@ -24,6 +32,9 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		case CSVState::ESCAPE:
 			InitializeTransitionArray(transition_array, cur_state, CSVState::INVALID);
 			break;
+		case CSVState::COMMENT:
+			InitializeTransitionArray(transition_array, cur_state, CSVState::COMMENT);
+			break;
 		default:
 			InitializeTransitionArray(transition_array, cur_state, CSVState::STANDARD);
 			break;
@@ -33,6 +44,7 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	uint8_t delimiter = static_cast<uint8_t>(state_machine_options.delimiter.GetValue());
 	uint8_t quote = static_cast<uint8_t>(state_machine_options.quote.GetValue());
 	uint8_t escape = static_cast<uint8_t>(state_machine_options.escape.GetValue());
+	uint8_t comment = static_cast<uint8_t>(state_machine_options.comment.GetValue());
 
 	auto new_line_id = state_machine_options.new_line.GetValue();
 
@@ -47,6 +59,9 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		} else {
 			transition_array[static_cast<uint8_t>('\r')][state] = CSVState::RECORD_SEPARATOR;
 		}
+		if (comment != '\0') {
+			transition_array[comment][state] = CSVState::COMMENT;
+		}
 	}
 	// 2) Field Separator State
 	transition_array[delimiter][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::DELIMITER;
@@ -63,6 +78,9 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	if (delimiter != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::EMPTY_SPACE;
 	}
+	if (comment != '\0') {
+		transition_array[comment][static_cast<uint8_t>(CSVState::DELIMITER)] = CSVState::COMMENT;
+	}
 
 	// 3) Record Separator State
 	transition_array[delimiter][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::DELIMITER;
@@ -79,6 +97,9 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	if (delimiter != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::EMPTY_SPACE;
 	}
+	if (comment != '\0') {
+		transition_array[comment][static_cast<uint8_t>(CSVState::RECORD_SEPARATOR)] = CSVState::COMMENT;
+	}
 
 	// 4) Carriage Return State
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] =
@@ -89,6 +110,9 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	if (delimiter != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] = CSVState::EMPTY_SPACE;
 	}
+	if (comment != '\0') {
+		transition_array[comment][static_cast<uint8_t>(CSVState::CARRIAGE_RETURN)] = CSVState::COMMENT;
+	}
 
 	// 5) Quoted State
 	transition_array[quote][static_cast<uint8_t>(CSVState::QUOTED)] = CSVState::UNQUOTED;
@@ -111,12 +135,15 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	if (state_machine_options.quote == state_machine_options.escape) {
 		transition_array[escape][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::QUOTED;
 	}
+	if (comment != '\0') {
+		transition_array[comment][static_cast<uint8_t>(CSVState::UNQUOTED)] = CSVState::COMMENT;
+	}
 	// 7) Escaped State
 	transition_array[quote][static_cast<uint8_t>(CSVState::ESCAPE)] = CSVState::QUOTED;
 	transition_array[escape][static_cast<uint8_t>(CSVState::ESCAPE)] = CSVState::QUOTED;
 
 	// 8) Not Set
-	transition_array[delimiter][static_cast<uint8_t>(static_cast<uint8_t>(CSVState::NOT_SET))] = CSVState::DELIMITER;
+	transition_array[delimiter][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::DELIMITER;
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::RECORD_SEPARATOR;
 	if (new_line_id == NewLineIdentifier::CARRY_ON) {
 		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::NOT_SET)] =
@@ -125,19 +152,21 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::NOT_SET)] =
 		    CSVState::RECORD_SEPARATOR;
 	}
-	transition_array[static_cast<uint8_t>(quote)][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::QUOTED;
+	transition_array[quote][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::QUOTED;
 	if (delimiter != ' ') {
 		transition_array[' '][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::EMPTY_SPACE;
 	}
+	if (comment != '\0') {
+		transition_array[comment][static_cast<uint8_t>(CSVState::NOT_SET)] = CSVState::COMMENT;
+	}
 	// 9) Quoted NewLine
 	transition_array[quote][static_cast<uint8_t>(CSVState::QUOTED_NEW_LINE)] = CSVState::UNQUOTED;
 	if (state_machine_options.quote != state_machine_options.escape) {
 		transition_array[escape][static_cast<uint8_t>(CSVState::QUOTED_NEW_LINE)] = CSVState::ESCAPE;
 	}
 
-	// 10) Empty Value State
-	transition_array[delimiter][static_cast<uint8_t>(static_cast<uint8_t>(CSVState::EMPTY_SPACE))] =
-	    CSVState::DELIMITER;
+	// 10) Empty Value State (Not first value)
+	transition_array[delimiter][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::DELIMITER;
 	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] =
 	    CSVState::RECORD_SEPARATOR;
 	if (new_line_id == NewLineIdentifier::CARRY_ON) {
@@ -148,15 +177,31 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 		    CSVState::RECORD_SEPARATOR;
 	}
 	transition_array[quote][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::QUOTED;
+	if (comment != '\0') {
+		transition_array[comment][static_cast<uint8_t>(CSVState::EMPTY_SPACE)] = CSVState::COMMENT;
+	}
+
+	// 11) Comment State
+	transition_array[static_cast<uint8_t>('\n')][static_cast<uint8_t>(CSVState::COMMENT)] = CSVState::RECORD_SEPARATOR;
+	if (new_line_id == NewLineIdentifier::CARRY_ON) {
+		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::COMMENT)] =
+		    CSVState::CARRIAGE_RETURN;
+	} else {
+		transition_array[static_cast<uint8_t>('\r')][static_cast<uint8_t>(CSVState::COMMENT)] =
+		    CSVState::RECORD_SEPARATOR;
+	}
+
 	// Initialize characters we can skip during processing, for Standard and Quoted states
 	for (idx_t i = 0; i < StateMachine::NUM_TRANSITIONS; i++) {
 		transition_array.skip_standard[i] = true;
 		transition_array.skip_quoted[i] = true;
+		transition_array.skip_comment[i] = true;
 	}
 	// For standard states we only care for delimiters \r and \n
 	transition_array.skip_standard[delimiter] = false;
 	transition_array.skip_standard[static_cast<uint8_t>('\n')] = false;
 	transition_array.skip_standard[static_cast<uint8_t>('\r')] = false;
+	transition_array.skip_standard[comment] = false;
 
 	// For quoted we only care about quote, escape and for delimiters \r and \n
 	transition_array.skip_quoted[quote] = false;
@@ -164,6 +209,9 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	transition_array.skip_quoted[static_cast<uint8_t>('\n')] = false;
 	transition_array.skip_quoted[static_cast<uint8_t>('\r')] = false;
 
+	transition_array.skip_comment[static_cast<uint8_t>('\r')] = false;
+	transition_array.skip_comment[static_cast<uint8_t>('\n')] = false;
+
 	transition_array.delimiter = delimiter;
 	transition_array.new_line = static_cast<uint8_t>('\n');
 	transition_array.carriage_return = static_cast<uint8_t>('\r');
@@ -171,36 +219,32 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	transition_array.escape = escape;
 
 	// Shift and OR to replicate across all bytes
-	transition_array.delimiter |= transition_array.delimiter << 8;
-	transition_array.delimiter |= transition_array.delimiter << 16;
-	transition_array.delimiter |= transition_array.delimiter << 32;
-
-	transition_array.new_line |= transition_array.new_line << 8;
-	transition_array.new_line |= transition_array.new_line << 16;
-	transition_array.new_line |= transition_array.new_line << 32;
-
-	transition_array.carriage_return |= transition_array.carriage_return << 8;
-	transition_array.carriage_return |= transition_array.carriage_return << 16;
-	transition_array.carriage_return |= transition_array.carriage_return << 32;
-
-	transition_array.quote |= transition_array.quote << 8;
-	transition_array.quote |= transition_array.quote << 16;
-	transition_array.quote |= transition_array.quote << 32;
-
-	transition_array.escape |= transition_array.escape << 8;
-	transition_array.escape |= transition_array.escape << 16;
-	transition_array.escape |= transition_array.escape << 32;
+	ShiftAndReplicateBits(transition_array.delimiter);
+	ShiftAndReplicateBits(transition_array.new_line);
+	ShiftAndReplicateBits(transition_array.carriage_return);
+	ShiftAndReplicateBits(transition_array.quote);
+	ShiftAndReplicateBits(transition_array.escape);
+	ShiftAndReplicateBits(transition_array.comment);
 }
 
 CSVStateMachineCache::CSVStateMachineCache() {
+	auto default_quote = DialectCandidates::GetDefaultQuote();
+	auto default_escape = DialectCandidates::GetDefaultEscape();
+	auto default_quote_rule = DialectCandidates::GetDefaultQuoteRule();
+	auto default_delimiter = DialectCandidates::GetDefaultDelimiter();
+	auto default_comment = DialectCandidates::GetDefaultComment();
+
 	for (auto quoterule : default_quote_rule) {
 		const auto &quote_candidates = default_quote[static_cast<uint8_t>(quoterule)];
 		for (const auto &quote : quote_candidates) {
 			for (const auto &delimiter : default_delimiter) {
 				const auto &escape_candidates = default_escape[static_cast<uint8_t>(quoterule)];
 				for (const auto &escape : escape_candidates) {
-					Insert({delimiter, quote, escape, NewLineIdentifier::SINGLE});
-					Insert({delimiter, quote, escape, NewLineIdentifier::CARRY_ON});
+					for (const auto &comment : default_comment) {
+						Insert({delimiter, quote, escape, comment, NewLineIdentifier::SINGLE_N});
+						Insert({delimiter, quote, escape, comment, NewLineIdentifier::SINGLE_R});
+						Insert({delimiter, quote, escape, comment, NewLineIdentifier::CARRY_ON});
+					}
 				}
 			}
 		}

package/src/duckdb/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp

@@ -1,13 +1,16 @@
 #include "duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp"
-#include "duckdb/function/table/read_csv.hpp"
+
 #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
+#include "duckdb/execution/operator/csv_scanner/skip_scanner.hpp"
+#include "duckdb/function/table/read_csv.hpp"
 
 namespace duckdb {
+CSVUnionData::~CSVUnionData() {
+}
 
 CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr<CSVBufferManager> buffer_manager_p,
                          shared_ptr<CSVStateMachine> state_machine_p, const CSVReaderOptions &options_p,
-                         const ReadCSVData &bind_data, const vector<column_t> &column_ids,
-                         vector<LogicalType> &file_schema)
+                         const ReadCSVData &bind_data, const vector<column_t> &column_ids, CSVSchema &file_schema)
     : file_path(options_p.file_path), file_idx(0), buffer_manager(std::move(buffer_manager_p)),
       state_machine(std::move(state_machine_p)), file_size(buffer_manager->file_handle->FileSize()),
       error_handler(make_shared_ptr<CSVErrorHandler>(options_p.ignore_errors.GetValue())),
@@ -23,7 +26,8 @@ CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr<CSVBufferManager> bu
 		                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
 		InitializeFileNamesTypes();
 		return;
-	} else if (!bind_data.column_info.empty()) {
+	}
+	if (!bind_data.column_info.empty()) {
 		// Serialized Union By name
 		names = bind_data.column_info[0].names;
 		types = bind_data.column_info[0].types;
@@ -32,51 +36,56 @@ CSVFileScan::CSVFileScan(ClientContext &context, shared_ptr<CSVBufferManager> bu
 		InitializeFileNamesTypes();
 		return;
 	}
-	names = bind_data.return_names;
-	types = bind_data.return_types;
-	file_schema = bind_data.return_types;
+	names = bind_data.csv_names;
+	types = bind_data.csv_types;
+	file_schema.Initialize(names, types, file_path);
 	multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
 	                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
 
 	InitializeFileNamesTypes();
+	SetStart();
+}
+
+void CSVFileScan::SetStart() {
+	idx_t rows_to_skip = options.GetSkipRows() + state_machine->dialect_options.header.GetValue();
+	rows_to_skip = std::max(rows_to_skip, state_machine->dialect_options.rows_until_header +
+	                                          state_machine->dialect_options.header.GetValue());
+	if (rows_to_skip == 0) {
+		start_iterator.first_one = true;
+		return;
+	}
+	SkipScanner skip_scanner(buffer_manager, state_machine, error_handler, rows_to_skip);
+	skip_scanner.ParseChunk();
+	start_iterator = skip_scanner.GetIterator();
 }
 
 CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, const CSVReaderOptions &options_p,
                          const idx_t file_idx_p, const ReadCSVData &bind_data, const vector<column_t> &column_ids,
-                         const vector<LogicalType> &file_schema)
+                         CSVSchema &file_schema, bool per_file_single_threaded)
     : file_path(file_path_p), file_idx(file_idx_p),
       error_handler(make_shared_ptr<CSVErrorHandler>(options_p.ignore_errors.GetValue())), options(options_p) {
 	auto multi_file_reader = MultiFileReader::CreateDefault("CSV Scan");
-	if (file_idx < bind_data.union_readers.size()) {
-		// we are doing UNION BY NAME - fetch the options from the union reader for this file
-		optional_ptr<CSVFileScan> union_reader_ptr;
-		if (file_idx == 0) {
-			union_reader_ptr = bind_data.initial_reader.get();
-		} else {
-			union_reader_ptr = bind_data.union_readers[file_idx].get();
-		}
-		if (union_reader_ptr) {
-			auto &union_reader = *union_reader_ptr;
-			// Initialize Buffer Manager
-			buffer_manager = union_reader.buffer_manager;
-			// Initialize On Disk and Size of file
-			on_disk_file = union_reader.on_disk_file;
-			file_size = union_reader.file_size;
-			names = union_reader.GetNames();
-			options = union_reader.options;
-			types = union_reader.GetTypes();
-			state_machine = union_reader.state_machine;
-			multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind,
-			                                    bind_data.return_types, bind_data.return_names, column_ids, nullptr,
-			                                    file_path, context, nullptr);
-
-			InitializeFileNamesTypes();
-			return;
-		}
+	if (file_idx == 0 && bind_data.initial_reader) {
+		auto &union_reader = *bind_data.initial_reader;
+		// Initialize Buffer Manager
+		buffer_manager = union_reader.buffer_manager;
+		// Initialize On Disk and Size of file
+		on_disk_file = union_reader.on_disk_file;
+		file_size = union_reader.file_size;
+		names = union_reader.GetNames();
+		options = union_reader.options;
+		types = union_reader.GetTypes();
+		state_machine = union_reader.state_machine;
+		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
+		                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
+
+		InitializeFileNamesTypes();
+		SetStart();
+		return;
 	}
 
 	// Initialize Buffer Manager
-	buffer_manager = make_shared_ptr<CSVBufferManager>(context, options, file_path, file_idx);
+	buffer_manager = make_shared_ptr<CSVBufferManager>(context, options, file_path, file_idx, per_file_single_threaded);
 	// Initialize On Disk and Size of file
 	on_disk_file = buffer_manager->file_handle->OnDiskFile();
 	file_size = buffer_manager->file_handle->FileSize();
@@ -84,13 +93,21 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons
 	auto &state_machine_cache = CSVStateMachineCache::Get(context);
 
 	if (file_idx < bind_data.column_info.size()) {
-		// Serialized Union By name
+		// (Serialized) Union By name
 		names = bind_data.column_info[file_idx].names;
 		types = bind_data.column_info[file_idx].types;
-		options.dialect_options.num_cols = names.size();
-		if (options.auto_detect) {
-			CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
-			sniffer.SniffCSV();
+		if (file_idx < bind_data.union_readers.size()) {
+			// union readers - use cached options
+			D_ASSERT(names == bind_data.union_readers[file_idx]->names);
+			D_ASSERT(types == bind_data.union_readers[file_idx]->types);
+			options = bind_data.union_readers[file_idx]->options;
+		} else {
+			// Serialized union by name - sniff again
+			options.dialect_options.num_cols = names.size();
+			if (options.auto_detect) {
+				CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
+				sniffer.SniffCSV();
+			}
 		}
 		state_machine = make_shared_ptr<CSVStateMachine>(
 		    state_machine_cache.Get(options.dialect_options.state_machine_options), options);
@@ -98,40 +115,41 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_path_p, cons
 		multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
 		                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
 		InitializeFileNamesTypes();
+		SetStart();
 		return;
 	}
-	// Sniff it (We only really care about dialect detection, if types or number of columns are different this will
-	// error out during scanning)
-	if (options.auto_detect && file_idx > 0) {
-		CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
-		auto result = sniffer.SniffCSV();
-		if (!file_schema.empty()) {
-			if (!options.file_options.filename && !options.file_options.hive_partitioning &&
-			    file_schema.size() != result.return_types.size()) {
-				throw InvalidInputException("Mismatch between the schema of different files");
-			}
+	// Sniff it!
+	names = bind_data.csv_names;
+	types = bind_data.csv_types;
+	if (options.auto_detect && bind_data.files.size() > 1) {
+		if (file_schema.Empty()) {
+			CSVSniffer sniffer(options, buffer_manager, state_machine_cache);
+			auto result = sniffer.SniffCSV();
+			file_schema.Initialize(result.names, result.return_types, options.file_path);
+		} else if (file_idx > 0 && buffer_manager->file_handle->FileSize() > 0) {
+			options.file_path = file_path;
+			CSVSniffer sniffer(options, buffer_manager, state_machine_cache, false);
+			auto result = sniffer.AdaptiveSniff(file_schema);
+			names = result.names;
+			types = result.return_types;
 		}
 	}
 	if (options.dialect_options.num_cols == 0) {
 		// We need to define the number of columns, if the sniffer is not running this must be in the sql_type_list
 		options.dialect_options.num_cols = options.sql_type_list.size();
 	}
-
 	if (options.dialect_options.state_machine_options.new_line == NewLineIdentifier::NOT_SET) {
 		options.dialect_options.state_machine_options.new_line = CSVSniffer::DetectNewLineDelimiter(*buffer_manager);
 	}
-
-	names = bind_data.csv_names;
-	types = bind_data.csv_types;
 	state_machine = make_shared_ptr<CSVStateMachine>(
 	    state_machine_cache.Get(options.dialect_options.state_machine_options), options);
-
 	multi_file_reader->InitializeReader(*this, options.file_options, bind_data.reader_bind, bind_data.return_types,
 	                                    bind_data.return_names, column_ids, nullptr, file_path, context, nullptr);
 	InitializeFileNamesTypes();
+	SetStart();
 }
 
-CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVReaderOptions &options_p)
+CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, const CSVReaderOptions &options_p)
     : file_path(file_name), file_idx(0),
       error_handler(make_shared_ptr<CSVErrorHandler>(options_p.ignore_errors.GetValue())), options(options_p) {
 	buffer_manager = make_shared_ptr<CSVBufferManager>(context, options, file_path, file_idx);
@@ -156,6 +174,7 @@ CSVFileScan::CSVFileScan(ClientContext &context, const string &file_name, CSVRea
 	// Initialize State Machine
 	state_machine = make_shared_ptr<CSVStateMachine>(
 	    state_machine_cache.Get(options.dialect_options.state_machine_options), options);
+	SetStart();
 }
 
 void CSVFileScan::InitializeFileNamesTypes() {
@@ -180,9 +199,11 @@ void CSVFileScan::InitializeFileNamesTypes() {
 	}
 
 	// We need to be sure that our types are also following the cast_map
-	for (idx_t i = 0; i < reader_data.column_ids.size(); i++) {
-		if (reader_data.cast_map.find(reader_data.column_ids[i]) != reader_data.cast_map.end()) {
-			file_types[i] = reader_data.cast_map[reader_data.column_ids[i]];
+	if (!reader_data.cast_map.empty()) {
+		for (idx_t i = 0; i < reader_data.column_ids.size(); i++) {
+			if (reader_data.cast_map.find(reader_data.column_ids[i]) != reader_data.cast_map.end()) {
+				file_types[i] = reader_data.cast_map[reader_data.column_ids[i]];
+			}
 		}
 	}
 
@@ -195,7 +216,7 @@ void CSVFileScan::InitializeFileNamesTypes() {
 	file_types = sorted_types;
 }
 
-const string &CSVFileScan::GetFileName() {
+const string &CSVFileScan::GetFileName() const {
 	return file_path;
 }
 const vector<string> &CSVFileScan::GetNames() {

package/src/duckdb/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp

@@ -1,9 +1,11 @@
 #include "duckdb/execution/operator/csv_scanner/global_csv_state.hpp"
-#include "duckdb/main/client_data.hpp"
-#include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
+
 #include "duckdb/execution/operator/csv_scanner/csv_sniffer.hpp"
+#include "duckdb/execution/operator/csv_scanner/scanner_boundary.hpp"
+#include "duckdb/execution/operator/csv_scanner/skip_scanner.hpp"
 #include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
 #include "duckdb/main/appender.hpp"
+#include "duckdb/main/client_data.hpp"
 
 namespace duckdb {
 
@@ -22,7 +24,7 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr<CSVBuf
 	} else {
 		// If not we need to construct it for the first file
 		file_scans.emplace_back(
-		    make_uniq<CSVFileScan>(context, files[0], options, 0U, bind_data, column_ids, file_schema));
+		    make_uniq<CSVFileScan>(context, files[0], options, 0U, bind_data, column_ids, file_schema, false));
 	};
 	// There are situations where we only support single threaded scanning
 	bool many_csv_files = files.size() > 1 && files.size() > system_threads * 2;
@@ -30,13 +32,18 @@ CSVGlobalState::CSVGlobalState(ClientContext &context_p, const shared_ptr<CSVBuf
 	last_file_idx = 0;
 	scanner_idx = 0;
 	running_threads = MaxThreads();
-	if (single_threaded) {
-		current_boundary = CSVIterator();
-	} else {
-		auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size;
-		current_boundary = CSVIterator(0, 0, 0, 0, buffer_size);
+	current_boundary = file_scans.back()->start_iterator;
+	current_boundary.SetCurrentBoundaryToPosition(single_threaded);
+	if (current_boundary.done && context.client_data->debug_set_max_line_length) {
+		context.client_data->debug_max_line_length = current_boundary.pos.buffer_pos;
 	}
-	current_buffer_in_use = make_shared_ptr<CSVBufferUsage>(*file_scans.back()->buffer_manager, 0);
+	current_buffer_in_use =
+	    make_shared_ptr<CSVBufferUsage>(*file_scans.back()->buffer_manager, current_boundary.GetBufferIdx());
+}
+
+bool CSVGlobalState::IsDone() const {
+	lock_guard<mutex> parallel_lock(main_mutex);
+	return current_boundary.done;
 }
 
 double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const {
@@ -44,13 +51,24 @@ double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const {
 	idx_t total_files = bind_data.files.size();
 	// get the progress WITHIN the current file
 	double percentage = 0;
-	if (file_scans.back()->file_size == 0) {
+	if (file_scans.front()->file_size == 0) {
 		percentage = 1.0;
 	} else {
 		// for compressed files, readed bytes may greater than files size.
 		for (auto &file : file_scans) {
-			percentage +=
-			    (double(1) / double(total_files)) * std::min(1.0, double(file->bytes_read) / double(file->file_size));
+			double file_progress;
+			if (!file->buffer_manager) {
+				// We are done with this file, so it's 100%
+				file_progress = 1.0;
+			} else if (file->buffer_manager->file_handle->compression_type == FileCompressionType::GZIP ||
+			           file->buffer_manager->file_handle->compression_type == FileCompressionType::ZSTD) {
+				// This file is not done, and is a compressed file
+				file_progress = file->buffer_manager->file_handle->GetProgress();
+			} else {
+				file_progress = static_cast<double>(file->bytes_read);
+			}
+			// This file is an uncompressed file, so we use the more price bytes_read from the scanner
+			percentage += (double(1) / double(total_files)) * std::min(1.0, file_progress / double(file->file_size));
 		}
 	}
 	return percentage * 100;
@@ -58,29 +76,45 @@ double CSVGlobalState::GetProgress(const ReadCSVData &bind_data_p) const {
 
 unique_ptr<StringValueScanner> CSVGlobalState::Next(optional_ptr<StringValueScanner> previous_scanner) {
 	if (single_threaded) {
-		idx_t cur_idx = last_file_idx++;
-		if (cur_idx >= bind_data.files.size()) {
-			return nullptr;
-		}
-		shared_ptr<CSVFileScan> current_file;
-		if (cur_idx == 0) {
-			current_file = file_scans.back();
-		} else {
-			lock_guard<mutex> parallel_lock(main_mutex);
-			file_scans.emplace_back(make_shared_ptr<CSVFileScan>(context, bind_data.files[cur_idx], bind_data.options,
-			                                                     cur_idx, bind_data, column_ids, file_schema));
-			current_file = file_scans.back();
-		}
-		if (previous_scanner) {
-			lock_guard<mutex> parallel_lock(main_mutex);
-			previous_scanner->buffer_tracker.reset();
-			current_buffer_in_use.reset();
-			previous_scanner->csv_file_scan->Finish();
-		}
-		auto csv_scanner =
-		    make_uniq<StringValueScanner>(scanner_idx++, current_file->buffer_manager, current_file->state_machine,
-		                                  current_file->error_handler, current_file, false, current_boundary);
-		return csv_scanner;
+		idx_t cur_idx;
+		bool empty_file = false;
+		do {
+			{
+				lock_guard<mutex> parallel_lock(main_mutex);
+				cur_idx = last_file_idx++;
+				if (cur_idx >= bind_data.files.size()) {
+					// No more files to scan
+					return nullptr;
+				}
+				if (cur_idx == 0) {
+					D_ASSERT(!previous_scanner);
+					auto current_file = file_scans.front();
+					return make_uniq<StringValueScanner>(scanner_idx++, current_file->buffer_manager,
+					                                     current_file->state_machine, current_file->error_handler,
+					                                     current_file, false, current_boundary);
+				}
+			}
+			auto file_scan = make_shared_ptr<CSVFileScan>(context, bind_data.files[cur_idx], bind_data.options, cur_idx,
+			                                              bind_data, column_ids, file_schema, true);
+			empty_file = file_scan->file_size == 0;
+			if (!empty_file) {
+				lock_guard<mutex> parallel_lock(main_mutex);
+				file_scans.emplace_back(std::move(file_scan));
+				auto current_file = file_scans.back();
+				current_boundary = current_file->start_iterator;
+				current_boundary.SetCurrentBoundaryToPosition(single_threaded);
+				current_buffer_in_use = make_shared_ptr<CSVBufferUsage>(*file_scans.back()->buffer_manager,
+				                                                        current_boundary.GetBufferIdx());
+				if (previous_scanner) {
+					previous_scanner->buffer_tracker.reset();
+					current_buffer_in_use.reset();
+					previous_scanner->csv_file_scan->Finish();
+				}
+				return make_uniq<StringValueScanner>(scanner_idx++, current_file->buffer_manager,
+				                                     current_file->state_machine, current_file->error_handler,
+				                                     current_file, false, current_boundary);
+			}
+		} while (empty_file);
 	}
 	lock_guard<mutex> parallel_lock(main_mutex);
 	if (finished) {
@@ -108,20 +142,24 @@ unique_ptr<StringValueScanner> CSVGlobalState::Next(optional_ptr<StringValueScan
 	// We then produce the next boundary
 	if (!current_boundary.Next(*current_file.buffer_manager)) {
 		// This means we are done scanning the current file
-		auto current_file_idx = current_file.file_idx + 1;
-		if (current_file_idx < bind_data.files.size()) {
-			// If we have a next file we have to construct the file scan for that
-			file_scans.emplace_back(make_shared_ptr<CSVFileScan>(context, bind_data.files[current_file_idx],
-			                                                     bind_data.options, current_file_idx, bind_data,
-			                                                     column_ids, file_schema));
-			// And re-start the boundary-iterator
-			auto buffer_size = file_scans.back()->buffer_manager->GetBuffer(0)->actual_size;
-			current_boundary = CSVIterator(current_file_idx, 0, 0, 0, buffer_size);
-			current_buffer_in_use = make_shared_ptr<CSVBufferUsage>(*file_scans.back()->buffer_manager, 0);
-		} else {
-			// If not we are done with this CSV Scanning
-			finished = true;
-		}
+		do {
+			auto current_file_idx = file_scans.back()->file_idx + 1;
+			if (current_file_idx < bind_data.files.size()) {
+				// If we have a next file we have to construct the file scan for that
+				file_scans.emplace_back(make_shared_ptr<CSVFileScan>(context, bind_data.files[current_file_idx],
+				                                                     bind_data.options, current_file_idx, bind_data,
+				                                                     column_ids, file_schema, false));
+				// And re-start the boundary-iterator
+				current_boundary = file_scans.back()->start_iterator;
+				current_boundary.SetCurrentBoundaryToPosition(single_threaded);
+				current_buffer_in_use = make_shared_ptr<CSVBufferUsage>(*file_scans.back()->buffer_manager,
+				                                                        current_boundary.GetBufferIdx());
+			} else {
+				// If not we are done with this CSV Scanning
+				finished = true;
+				break;
+			}
+		} while (current_boundary.done);
 	}
 	// We initialize the scan
 	return csv_scanner;
@@ -132,7 +170,7 @@ idx_t CSVGlobalState::MaxThreads() const {
 	if (single_threaded) {
 		return system_threads;
 	}
-	idx_t total_threads = file_scans.back()->file_size / CSVIterator::BYTES_PER_THREAD + 1;
+	idx_t total_threads = file_scans.front()->file_size / CSVIterator::BYTES_PER_THREAD + 1;
 
 	if (total_threads < system_threads) {
 		return total_threads;