duckdb 1.0.1-dev21.0 → 1.0.1-dev27.0

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp

@@ -53,7 +53,7 @@ shared_ptr<CSVBuffer> CSVBuffer::Next(CSVFileHandle &file_handle, idx_t buffer_s
 void CSVBuffer::AllocateBuffer(idx_t buffer_size) {
 	auto &buffer_manager = BufferManager::GetBufferManager(context);
 	bool can_destroy = !is_pipe;
-	handle = buffer_manager.Allocate(MemoryTag::CSV_READER, MaxValue<idx_t>(Storage::BLOCK_SIZE, buffer_size),
+	handle = buffer_manager.Allocate(MemoryTag::CSV_READER, MaxValue<idx_t>(buffer_manager.GetBlockSize(), buffer_size),
 	                                 can_destroy, &block);
 }
 

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp

@@ -4,8 +4,9 @@
 namespace duckdb {
 
 CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOptions &options, const string &file_path_p,
-                                   const idx_t file_idx_p)
-    : context(context_p), file_idx(file_idx_p), file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) {
+                                   const idx_t file_idx_p, bool per_file_single_threaded_p)
+    : context(context_p), per_file_single_threaded(per_file_single_threaded_p), file_idx(file_idx_p),
+      file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) {
 	D_ASSERT(!file_path.empty());
 	file_handle = ReadCSV::OpenCSV(file_path, options.compression, context);
 	is_pipe = file_handle->IsPipe();
@@ -71,7 +72,7 @@ shared_ptr<CSVBufferHandle> CSVBufferManager::GetBuffer(const idx_t pos) {
 			done = true;
 		}
 	}
-	if (pos != 0 && (sniffing || file_handle->CanSeek())) {
+	if (pos != 0 && (sniffing || file_handle->CanSeek() || per_file_single_threaded)) {
 		// We don't need to unpin the buffers here if we are not sniffing since we
 		// control it per-thread on the scan
 		if (cached_buffers[pos - 1]) {

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp

@@ -1,17 +1,19 @@
 #include "duckdb/execution/operator/csv_scanner/csv_file_handle.hpp"
 #include "duckdb/common/exception/binder_exception.hpp"
 #include "duckdb/common/numeric_utils.hpp"
+#include "duckdb/common/compressed_file_system.hpp"
+#include "duckdb/common/string_util.hpp"
 
 namespace duckdb {
 
 CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<FileHandle> file_handle_p,
                              const string &path_p, FileCompressionType compression)
-    : file_handle(std::move(file_handle_p)), path(path_p) {
+    : compression_type(compression), file_handle(std::move(file_handle_p)), path(path_p) {
 	can_seek = file_handle->CanSeek();
 	on_disk_file = file_handle->OnDiskFile();
 	file_size = file_handle->GetFileSize();
 	is_pipe = file_handle->IsPipe();
-	uncompressed = compression == FileCompressionType::UNCOMPRESSED;
+	compression_type = file_handle->GetFileCompressionType();
 }
 
 unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
@@ -29,6 +31,10 @@ unique_ptr<CSVFileHandle> CSVFileHandle::OpenFile(FileSystem &fs, Allocator &all
 	return make_uniq<CSVFileHandle>(fs, allocator, std::move(file_handle), path, compression);
 }
 
+double CSVFileHandle::GetProgress() {
+	return static_cast<double>(file_handle->GetProgress());
+}
+
 bool CSVFileHandle::CanSeek() {
 	return can_seek;
 }
@@ -72,6 +78,7 @@ idx_t CSVFileHandle::Read(void *buffer, idx_t nr_bytes) {
 	if (!finished) {
 		finished = bytes_read == 0;
 	}
+	uncompressed_bytes_read += static_cast<idx_t>(bytes_read);
 	return UnsafeNumericCast<idx_t>(bytes_read);
 }
 

package/src/duckdb/src/execution/operator/csv_scanner/scanner/base_scanner.cpp

@@ -5,15 +5,15 @@
 
 namespace duckdb {
 
-ScannerResult::ScannerResult(CSVStates &states_p, CSVStateMachine &state_machine_p)
-    : state_machine(state_machine_p), states(states_p) {
+ScannerResult::ScannerResult(CSVStates &states_p, CSVStateMachine &state_machine_p, idx_t result_size_p)
+    : result_size(result_size_p), state_machine(state_machine_p), states(states_p) {
 }
 
 BaseScanner::BaseScanner(shared_ptr<CSVBufferManager> buffer_manager_p, shared_ptr<CSVStateMachine> state_machine_p,
                          shared_ptr<CSVErrorHandler> error_handler_p, bool sniffing_p,
                          shared_ptr<CSVFileScan> csv_file_scan_p, CSVIterator iterator_p)
     : csv_file_scan(std::move(csv_file_scan_p)), sniffing(sniffing_p), error_handler(std::move(error_handler_p)),
-      state_machine(std::move(state_machine_p)), iterator(iterator_p), buffer_manager(std::move(buffer_manager_p)) {
+      state_machine(std::move(state_machine_p)), buffer_manager(std::move(buffer_manager_p)), iterator(iterator_p) {
 	D_ASSERT(buffer_manager);
 	D_ASSERT(state_machine);
 	// Initialize current buffer handle
@@ -41,19 +41,15 @@ bool BaseScanner::FinishedFile() {
 	return iterator.pos.buffer_pos + 1 == cur_buffer_handle->actual_size;
 }
 
-void BaseScanner::SkipCSVRows(idx_t rows_to_skip) {
+CSVIterator BaseScanner::SkipCSVRows(shared_ptr<CSVBufferManager> buffer_manager,
+                                     const shared_ptr<CSVStateMachine> &state_machine, idx_t rows_to_skip) {
 	if (rows_to_skip == 0) {
-		return;
+		return {};
 	}
-	SkipScanner row_skipper(buffer_manager, state_machine, error_handler, rows_to_skip);
+	auto error_handler = make_shared_ptr<CSVErrorHandler>();
+	SkipScanner row_skipper(std::move(buffer_manager), state_machine, error_handler, rows_to_skip);
 	row_skipper.ParseChunk();
-	iterator.pos.buffer_pos = row_skipper.GetIteratorPosition();
-	if (row_skipper.state_machine->options.dialect_options.state_machine_options.new_line ==
-	        NewLineIdentifier::CARRY_ON &&
-	    row_skipper.states.states[1] == CSVState::CARRIAGE_RETURN) {
-		iterator.pos.buffer_pos++;
-	}
-	lines_read += row_skipper.GetLinesRead();
+	return row_skipper.GetIterator();
 }
 
 CSVIterator &BaseScanner::GetIterator() {
@@ -65,19 +61,19 @@ void BaseScanner::SetIterator(const CSVIterator &it) {
 }
 
 ScannerResult &BaseScanner::ParseChunk() {
-	throw InternalException("ParseChunk() from CSV Base Scanner is mot implemented");
+	throw InternalException("ParseChunk() from CSV Base Scanner is not implemented");
 }
 
 ScannerResult &BaseScanner::GetResult() {
-	throw InternalException("GetResult() from CSV Base Scanner is mot implemented");
+	throw InternalException("GetResult() from CSV Base Scanner is not implemented");
 }
 
 void BaseScanner::Initialize() {
-	throw InternalException("Initialize() from CSV Base Scanner is mot implemented");
+	throw InternalException("Initialize() from CSV Base Scanner is not implemented");
 }
 
 void BaseScanner::FinalizeChunkProcess() {
-	throw InternalException("FinalizeChunkProcess() from CSV Base Scanner is mot implemented");
+	throw InternalException("FinalizeChunkProcess() from CSV Base Scanner is not implemented");
 }
 
 CSVStateMachine &BaseScanner::GetStateMachine() {

package/src/duckdb/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp

@@ -2,37 +2,66 @@
 
 namespace duckdb {
 
-ColumnCountResult::ColumnCountResult(CSVStates &states, CSVStateMachine &state_machine)
-    : ScannerResult(states, state_machine) {
+ColumnCountResult::ColumnCountResult(CSVStates &states, CSVStateMachine &state_machine, idx_t result_size)
+    : ScannerResult(states, state_machine, result_size) {
+	column_counts.resize(result_size);
 }
 
-void ColumnCountResult::AddValue(ColumnCountResult &result, const idx_t buffer_pos) {
+void ColumnCountResult::AddValue(ColumnCountResult &result, idx_t buffer_pos) {
 	result.current_column_count++;
 }
 
 inline void ColumnCountResult::InternalAddRow() {
-	column_counts[result_position++] = current_column_count + 1;
+	column_counts[result_position].number_of_columns = current_column_count + 1;
 	current_column_count = 0;
 }
 
-bool ColumnCountResult::AddRow(ColumnCountResult &result, const idx_t buffer_pos) {
+bool ColumnCountResult::AddRow(ColumnCountResult &result, idx_t buffer_pos) {
 	result.InternalAddRow();
 	if (!result.states.EmptyLastValue()) {
-		result.last_value_always_empty = false;
+		idx_t col_count_idx = result.result_position;
+		for (idx_t i = 0; i < result.result_position + 1; i++) {
+			if (!result.column_counts[col_count_idx].last_value_always_empty) {
+				break;
+			}
+			result.column_counts[col_count_idx--].last_value_always_empty = false;
+		}
 	}
-	if (result.result_position >= STANDARD_VECTOR_SIZE) {
+	result.result_position++;
+	if (result.result_position >= result.result_size) {
 		// We sniffed enough rows
 		return true;
 	}
 	return false;
 }
 
+void ColumnCountResult::SetComment(ColumnCountResult &result, idx_t buffer_pos) {
+	if (!result.states.WasStandard()) {
+		result.cur_line_starts_as_comment = true;
+	}
+	result.comment = true;
+}
+
+bool ColumnCountResult::UnsetComment(ColumnCountResult &result, idx_t buffer_pos) {
+	// If we are unsetting a comment, it means this row started with a comment char.
+	// We add the row but tag it as a comment
+	bool done = result.AddRow(result, buffer_pos);
+	if (result.cur_line_starts_as_comment) {
+		result.column_counts[result.result_position - 1].is_comment = true;
+	} else {
+		result.column_counts[result.result_position - 1].is_mid_comment = true;
+	}
+	result.comment = false;
+	result.cur_line_starts_as_comment = false;
+	return done;
+}
+
 void ColumnCountResult::InvalidState(ColumnCountResult &result) {
 	result.result_position = 0;
 	result.error = true;
 }
 
-bool ColumnCountResult::EmptyLine(ColumnCountResult &result, const idx_t buffer_pos) {
+bool ColumnCountResult::EmptyLine(ColumnCountResult &result, idx_t buffer_pos) {
 	// nop
 	return false;
 }
@@ -43,15 +72,24 @@ void ColumnCountResult::QuotedNewLine(ColumnCountResult &result) {
 
 ColumnCountScanner::ColumnCountScanner(shared_ptr<CSVBufferManager> buffer_manager,
                                        const shared_ptr<CSVStateMachine> &state_machine,
-                                       shared_ptr<CSVErrorHandler> error_handler)
-    : BaseScanner(std::move(buffer_manager), state_machine, std::move(error_handler)), result(states, *state_machine),
-      column_count(1) {
+                                       shared_ptr<CSVErrorHandler> error_handler, idx_t result_size_p,
+                                       CSVIterator iterator)
+    : BaseScanner(std::move(buffer_manager), state_machine, std::move(error_handler), true, nullptr, iterator),
+      result(states, *state_machine, result_size_p), column_count(1), result_size(result_size_p) {
 	sniffing = true;
 }
 
 unique_ptr<StringValueScanner> ColumnCountScanner::UpgradeToStringValueScanner() {
-	auto scanner = make_uniq<StringValueScanner>(0U, buffer_manager, state_machine, error_handler, nullptr, true);
-	return scanner;
+	idx_t rows_to_skip =
+	    std::max(state_machine->dialect_options.skip_rows.GetValue(), state_machine->dialect_options.rows_until_header);
+	auto iterator = SkipCSVRows(buffer_manager, state_machine, rows_to_skip);
+	if (iterator.done) {
+		CSVIterator it {};
+		return make_uniq<StringValueScanner>(0U, buffer_manager, state_machine, error_handler, nullptr, true, it,
+		                                     result_size);
+	}
+	return make_uniq<StringValueScanner>(0U, buffer_manager, state_machine, error_handler, nullptr, true, iterator,
+	                                     result_size);
 }
 
 ColumnCountResult &ColumnCountScanner::ParseChunk() {
@@ -70,12 +108,12 @@ void ColumnCountScanner::Initialize() {
 }
 
 void ColumnCountScanner::FinalizeChunkProcess() {
-	if (result.result_position == STANDARD_VECTOR_SIZE || result.error) {
+	if (result.result_position == result.result_size || result.error) {
 		// We are done
 		return;
 	}
 	// We run until we have a full chunk, or we are done scanning
-	while (!FinishedFile() && result.result_position < STANDARD_VECTOR_SIZE && !result.error) {
+	while (!FinishedFile() && result.result_position < result.result_size && !result.error) {
 		if (iterator.pos.buffer_pos == cur_buffer_handle->actual_size) {
 			// Move to next buffer
 			cur_buffer_handle = buffer_manager->GetBuffer(++iterator.pos.buffer_idx);
@@ -85,7 +123,13 @@ void ColumnCountScanner::FinalizeChunkProcess() {
 					return;
 				}
 				// This means we reached the end of the file, we must add a last line if there is any to be added
-				result.InternalAddRow();
+				if (result.comment) {
+					// If it's a comment we add the last line via unsetcomment
+					result.UnsetComment(result, NumericLimits<idx_t>::Maximum());
+				} else {
+					// OW, we do a regular AddRow
+					result.AddRow(result, NumericLimits<idx_t>::Maximum());
+				}
 				return;
 			}
 			iterator.pos.buffer_pos = 0;

package/src/duckdb/src/execution/operator/csv_scanner/scanner/csv_schema.cpp

@@ -0,0 +1,105 @@
+#include "duckdb/execution/operator/csv_scanner/csv_schema.hpp"
+
+namespace duckdb {
+
+struct TypeIdxPair {
+	TypeIdxPair(LogicalType type_p, idx_t idx_p) : type(std::move(type_p)), idx(idx_p) {
+	}
+	TypeIdxPair() {
+	}
+	LogicalType type;
+	idx_t idx {};
+};
+
+// We only really care about types that can be set in the sniffer_auto, or are sniffed by default
+// If the user manually sets them, we should never get a cast issue from the sniffer!
+bool CSVSchema::CanWeCastIt(LogicalTypeId source, LogicalTypeId destination) {
+	if (destination == LogicalTypeId::VARCHAR || source == destination) {
+		// We can always cast to varchar
+		// And obviously don't have to do anything if they are equal.
+		return true;
+	}
+	switch (source) {
+	case LogicalTypeId::SQLNULL:
+		return true;
+	case LogicalTypeId::TINYINT:
+		return destination == LogicalTypeId::SMALLINT || destination == LogicalTypeId::INTEGER ||
+		       destination == LogicalTypeId::BIGINT || destination == LogicalTypeId::DECIMAL ||
+		       destination == LogicalTypeId::FLOAT || destination == LogicalTypeId::DOUBLE;
+	case LogicalTypeId::SMALLINT:
+		return destination == LogicalTypeId::INTEGER || destination == LogicalTypeId::BIGINT ||
+		       destination == LogicalTypeId::DECIMAL || destination == LogicalTypeId::FLOAT ||
+		       destination == LogicalTypeId::DOUBLE;
+	case LogicalTypeId::INTEGER:
+		return destination == LogicalTypeId::BIGINT || destination == LogicalTypeId::DECIMAL ||
+		       destination == LogicalTypeId::FLOAT || destination == LogicalTypeId::DOUBLE;
+	case LogicalTypeId::BIGINT:
+		return destination == LogicalTypeId::DECIMAL || destination == LogicalTypeId::FLOAT ||
+		       destination == LogicalTypeId::DOUBLE;
+	case LogicalTypeId::FLOAT:
+		return destination == LogicalTypeId::DOUBLE;
+	default:
+		return false;
+	}
+}
+
+void CSVSchema::Initialize(vector<string> &names, vector<LogicalType> &types, const string &file_path_p) {
+	if (!columns.empty()) {
+		throw InternalException("CSV Schema is already populated, this should not happen.");
+	}
+	file_path = file_path_p;
+	D_ASSERT(names.size() == types.size() && !names.empty());
+	for (idx_t i = 0; i < names.size(); i++) {
+		// Populate our little schema
+		columns.push_back({names[i], types[i]});
+		name_idx_map[names[i]] = i;
+	}
+}
+
+bool CSVSchema::Empty() const {
+	return columns.empty();
+}
+
+bool CSVSchema::SchemasMatch(string &error_message, vector<string> &names, vector<LogicalType> &types,
+                             const string &cur_file_path) {
+	D_ASSERT(names.size() == types.size());
+	bool match = true;
+	unordered_map<string, TypeIdxPair> current_schema;
+	for (idx_t i = 0; i < names.size(); i++) {
+		// Populate our little schema
+		current_schema[names[i]] = {types[i], i};
+	}
+	// Here we check if the schema of a given file matched our original schema
+	// We consider it's not a match if:
+	// 1. The file misses columns that were defined in the original schema.
+	// 2. They have a column match, but the types do not match.
+	std::ostringstream error;
+	error << "Schema mismatch between globbed files."
+	      << "\n";
+	error << "Main file schema: " << file_path << "\n";
+	error << "Current file: " << cur_file_path << "\n";
+
+	for (auto &column : columns) {
+		if (current_schema.find(column.name) == current_schema.end()) {
+			error << "Column with name: \"" << column.name << "\" is missing"
+			      << "\n";
+			match = false;
+		} else {
+			if (!CanWeCastIt(current_schema[column.name].type.id(), column.type.id())) {
+				error << "Column with name: \"" << column.name
+				      << "\" is expected to have type: " << column.type.ToString();
+				error << " But has type: " << current_schema[column.name].type.ToString() << "\n";
+				match = false;
+			}
+		}
+	}
+
+	// Lets suggest some potential fixes
+	error << "Potential Fix: Since your schema has a mismatch, consider setting union_by_name=true.";
+	if (!match) {
+		error_message = error.str();
+	}
+	return match;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/csv_scanner/scanner/scanner_boundary.cpp

@@ -2,28 +2,15 @@
 
 namespace duckdb {
 
-CSVPosition::CSVPosition(idx_t file_idx_p, idx_t buffer_idx_p, idx_t buffer_pos_p)
-    : file_idx(file_idx_p), buffer_idx(buffer_idx_p), buffer_pos(buffer_pos_p) {
+CSVPosition::CSVPosition(idx_t buffer_idx_p, idx_t buffer_pos_p) : buffer_idx(buffer_idx_p), buffer_pos(buffer_pos_p) {
 }
 CSVPosition::CSVPosition() {
 }
 
-CSVBoundary::CSVBoundary(idx_t file_idx_p, idx_t buffer_idx_p, idx_t buffer_pos_p, idx_t boundary_idx_p,
-                         idx_t end_pos_p)
-    : file_idx(file_idx_p), buffer_idx(buffer_idx_p), buffer_pos(buffer_pos_p), boundary_idx(boundary_idx_p),
-      end_pos(end_pos_p) {
+CSVBoundary::CSVBoundary(idx_t buffer_idx_p, idx_t buffer_pos_p, idx_t boundary_idx_p, idx_t end_pos_p)
+    : buffer_idx(buffer_idx_p), buffer_pos(buffer_pos_p), boundary_idx(boundary_idx_p), end_pos(end_pos_p) {
 }
-CSVBoundary::CSVBoundary()
-    : file_idx(0), buffer_idx(0), buffer_pos(0), boundary_idx(0), end_pos(NumericLimits<idx_t>::Maximum()) {
-}
-CSVIterator::CSVIterator(idx_t file_idx, idx_t buffer_idx, idx_t buffer_pos, idx_t boundary_idx, idx_t buffer_size)
-    : pos(file_idx, buffer_idx, buffer_pos), is_set(true) {
-	// The end of our boundary will be the buffer size itself it that's smaller than where we want to go
-	if (buffer_size < buffer_pos + BYTES_PER_THREAD) {
-		boundary = {file_idx, buffer_idx, buffer_pos, boundary_idx, buffer_size};
-	} else {
-		boundary = {file_idx, buffer_idx, buffer_pos, boundary_idx, buffer_pos + BYTES_PER_THREAD};
-	}
+CSVBoundary::CSVBoundary() : buffer_idx(0), buffer_pos(0), boundary_idx(0), end_pos(NumericLimits<idx_t>::Maximum()) {
 }
 
 CSVIterator::CSVIterator() : is_set(false) {
@@ -32,7 +19,6 @@ CSVIterator::CSVIterator() : is_set(false) {
 void CSVBoundary::Print() {
 #ifndef DUCKDB_DISABLE_PRINT
 	std::cout << "---Boundary: " << boundary_idx << " ---" << '\n';
-	std::cout << "File Index:: " << file_idx << '\n';
 	std::cout << "Buffer Index: " << buffer_idx << '\n';
 	std::cout << "Buffer Pos: " << buffer_pos << '\n';
 	std::cout << "End Pos: " << end_pos << '\n';
@@ -51,6 +37,8 @@ bool CSVIterator::Next(CSVBufferManager &buffer_manager) {
 	if (!is_set) {
 		return false;
 	}
+	// If we are calling next this is not the first one anymore
+	first_one = false;
 	boundary.boundary_idx++;
 	// This is our start buffer
 	auto buffer = buffer_manager.GetBuffer(boundary.buffer_idx);
@@ -84,12 +72,8 @@ idx_t CSVIterator::GetEndPos() const {
 	return boundary.end_pos;
 }
 
-idx_t CSVIterator::GetFileIdx() const {
-	return pos.file_idx;
-}
-
 idx_t CSVIterator::GetBufferIdx() const {
-	return boundary.buffer_idx;
+	return pos.buffer_idx;
 }
 
 idx_t CSVIterator::GetBoundaryIdx() const {
@@ -97,11 +81,27 @@ idx_t CSVIterator::GetBoundaryIdx() const {
 }
 
 void CSVIterator::SetCurrentPositionToBoundary() {
-	pos.file_idx = boundary.file_idx;
 	pos.buffer_idx = boundary.buffer_idx;
 	pos.buffer_pos = boundary.buffer_pos;
 }
 
+void CSVIterator::SetCurrentBoundaryToPosition(bool single_threaded) {
+	if (single_threaded) {
+		is_set = false;
+		return;
+	}
+	boundary.buffer_idx = pos.buffer_idx;
+	if (pos.buffer_pos == 0) {
+		boundary.end_pos = CSVIterator::BYTES_PER_THREAD;
+	} else {
+		boundary.end_pos = ((pos.buffer_pos + CSVIterator::BYTES_PER_THREAD - 1) / CSVIterator::BYTES_PER_THREAD) *
+		                   CSVIterator::BYTES_PER_THREAD;
+	}
+
+	boundary.buffer_pos = boundary.end_pos - CSVIterator::BYTES_PER_THREAD;
+	is_set = true;
+}
+
 void CSVIterator::SetStart(idx_t start) {
 	boundary.buffer_pos = start;
 }

package/src/duckdb/src/execution/operator/csv_scanner/scanner/skip_scanner.cpp

@@ -4,7 +4,7 @@
 namespace duckdb {
 
 SkipResult::SkipResult(CSVStates &states, CSVStateMachine &state_machine, idx_t rows_to_skip_p)
-    : ScannerResult(states, state_machine), rows_to_skip(rows_to_skip_p) {
+    : ScannerResult(states, state_machine, STANDARD_VECTOR_SIZE), rows_to_skip(rows_to_skip_p) {
 }
 
 void SkipResult::AddValue(SkipResult &result, const idx_t buffer_pos) {
@@ -19,6 +19,14 @@ void SkipResult::QuotedNewLine(SkipResult &result) {
 	// nop
 }
 
+bool SkipResult::UnsetComment(SkipResult &result, idx_t buffer_pos) {
+	// If we are unsetting a comment, it means this row started with a comment char.
+	// We add the row but tag it as a comment
+	bool done = result.AddRow(result, buffer_pos);
+	result.comment = false;
+	return done;
+}
+
 bool SkipResult::AddRow(SkipResult &result, const idx_t buffer_pos) {
 	result.InternalAddRow();
 	if (result.row_count >= result.rows_to_skip) {
@@ -38,6 +46,7 @@ bool SkipResult::EmptyLine(SkipResult &result, const idx_t buffer_pos) {
 	}
 	return false;
 }
+
 SkipScanner::SkipScanner(shared_ptr<CSVBufferManager> buffer_manager, const shared_ptr<CSVStateMachine> &state_machine,
                          shared_ptr<CSVErrorHandler> error_handler, idx_t rows_to_skip)
     : BaseScanner(std::move(buffer_manager), state_machine, std::move(error_handler)),
@@ -58,6 +67,20 @@ void SkipScanner::Initialize() {
 }
 
 void SkipScanner::FinalizeChunkProcess() {
-	// nop
+	// We continue skipping until we skipped enough rows, or we have nothing else to read.
+	while (!FinishedFile() && result.row_count < result.rows_to_skip) {
+		cur_buffer_handle = buffer_manager->GetBuffer(++iterator.pos.buffer_idx);
+		if (cur_buffer_handle) {
+			iterator.pos.buffer_pos = 0;
+			buffer_handle_ptr = cur_buffer_handle->Ptr();
+			Process(result);
+		}
+	}
+	// Skip Carriage Return
+	if (state_machine->options.dialect_options.state_machine_options.new_line == NewLineIdentifier::CARRY_ON &&
+	    states.states[1] == CSVState::CARRIAGE_RETURN) {
+		iterator.pos.buffer_pos++;
+	}
+	iterator.done = FinishedFile();
 }
 } // namespace duckdb