duckdb 1.1.4-dev2.0 → 1.1.4-dev9.0

package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp

@@ -1,30 +1,14 @@
 #include "duckdb/execution/operator/aggregate/physical_window.hpp"
 
-#include "duckdb/common/operator/add.hpp"
-#include "duckdb/common/operator/cast_operators.hpp"
-#include "duckdb/common/operator/comparison_operators.hpp"
-#include "duckdb/common/operator/subtract.hpp"
-#include "duckdb/common/optional_ptr.hpp"
-#include "duckdb/common/radix_partitioning.hpp"
-#include "duckdb/common/row_operations/row_operations.hpp"
 #include "duckdb/common/sort/partition_state.hpp"
-
-#include "duckdb/common/types/column/column_data_consumer.hpp"
-#include "duckdb/common/types/row/row_data_collection_scanner.hpp"
-#include "duckdb/common/uhugeint.hpp"
-#include "duckdb/common/vector_operations/vector_operations.hpp"
-#include "duckdb/common/windows_undefs.hpp"
-#include "duckdb/execution/expression_executor.hpp"
-#include "duckdb/execution/window_executor.hpp"
-#include "duckdb/execution/window_segment_tree.hpp"
-#include "duckdb/main/client_config.hpp"
-#include "duckdb/main/config.hpp"
-#include "duckdb/parallel/base_pipeline_event.hpp"
-#include "duckdb/planner/expression/bound_reference_expression.hpp"
+#include "duckdb/function/window/window_aggregate_function.hpp"
+#include "duckdb/function/window/window_executor.hpp"
+#include "duckdb/function/window/window_rank_function.hpp"
+#include "duckdb/function/window/window_rownumber_function.hpp"
+#include "duckdb/function/window/window_shared_expressions.hpp"
+#include "duckdb/function/window/window_value_function.hpp"
 #include "duckdb/planner/expression/bound_window_expression.hpp"
-
-#include <algorithm>
-#include <cmath>
+//
 #include <numeric>
 
 namespace duckdb {
@@ -102,6 +86,8 @@ public:
 	ValidityMask partition_mask;
 	//! The order boundary mask
 	OrderMasks order_masks;
+	//! The fully materialised data collection
+	unique_ptr<WindowCollection> collection;
 	//! External paging
 	bool external;
 	// The processing stage for this group
@@ -145,6 +131,8 @@ public:
 	unique_ptr<WindowPartitionGlobalSinkState> global_partition;
 	//! The execution functions
 	Executors executors;
+	//! The shared expressions library
+	WindowSharedExpressions shared;
 };
 
 class WindowPartitionGlobalSinkState : public PartitionGlobalSinkState {
@@ -201,7 +189,7 @@ PhysicalWindow::PhysicalWindow(vector<LogicalType> types, vector<unique_ptr<Expr
 	idx_t max_orders = 0;
 	for (idx_t i = 0; i < select_list.size(); ++i) {
 		auto &expr = select_list[i];
-		D_ASSERT(expr->expression_class == ExpressionClass::BOUND_WINDOW);
+		D_ASSERT(expr->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
 		auto &bound_window = expr->Cast<BoundWindowExpression>();
 		if (bound_window.partitions.empty() && bound_window.orders.empty()) {
 			is_order_dependent = true;
@@ -215,34 +203,34 @@ PhysicalWindow::PhysicalWindow(vector<LogicalType> types, vector<unique_ptr<Expr
 }
 
 static unique_ptr<WindowExecutor> WindowExecutorFactory(BoundWindowExpression &wexpr, ClientContext &context,
-                                                        WindowAggregationMode mode) {
-	switch (wexpr.type) {
+                                                        WindowSharedExpressions &shared, WindowAggregationMode mode) {
+	switch (wexpr.GetExpressionType()) {
 	case ExpressionType::WINDOW_AGGREGATE:
-		return make_uniq<WindowAggregateExecutor>(wexpr, context, mode);
+		return make_uniq<WindowAggregateExecutor>(wexpr, context, shared, mode);
 	case ExpressionType::WINDOW_ROW_NUMBER:
-		return make_uniq<WindowRowNumberExecutor>(wexpr, context);
+		return make_uniq<WindowRowNumberExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_RANK_DENSE:
-		return make_uniq<WindowDenseRankExecutor>(wexpr, context);
+		return make_uniq<WindowDenseRankExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_RANK:
-		return make_uniq<WindowRankExecutor>(wexpr, context);
+		return make_uniq<WindowRankExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_PERCENT_RANK:
-		return make_uniq<WindowPercentRankExecutor>(wexpr, context);
+		return make_uniq<WindowPercentRankExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_CUME_DIST:
-		return make_uniq<WindowCumeDistExecutor>(wexpr, context);
+		return make_uniq<WindowCumeDistExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_NTILE:
-		return make_uniq<WindowNtileExecutor>(wexpr, context);
+		return make_uniq<WindowNtileExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_LEAD:
 	case ExpressionType::WINDOW_LAG:
-		return make_uniq<WindowLeadLagExecutor>(wexpr, context);
+		return make_uniq<WindowLeadLagExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_FIRST_VALUE:
-		return make_uniq<WindowFirstValueExecutor>(wexpr, context);
+		return make_uniq<WindowFirstValueExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_LAST_VALUE:
-		return make_uniq<WindowLastValueExecutor>(wexpr, context);
+		return make_uniq<WindowLastValueExecutor>(wexpr, context, shared);
 	case ExpressionType::WINDOW_NTH_VALUE:
-		return make_uniq<WindowNthValueExecutor>(wexpr, context);
+		return make_uniq<WindowNthValueExecutor>(wexpr, context, shared);
 		break;
 	default:
-		throw InternalException("Window aggregate type %s", ExpressionTypeToString(wexpr.type));
+		throw InternalException("Window aggregate type %s", ExpressionTypeToString(wexpr.GetExpressionType()));
 	}
 }
 
@@ -256,7 +244,7 @@ WindowGlobalSinkState::WindowGlobalSinkState(const PhysicalWindow &op, ClientCon
 	for (idx_t expr_idx = 0; expr_idx < op.select_list.size(); ++expr_idx) {
 		D_ASSERT(op.select_list[expr_idx]->GetExpressionClass() == ExpressionClass::BOUND_WINDOW);
 		auto &wexpr = op.select_list[expr_idx]->Cast<BoundWindowExpression>();
-		auto wexec = WindowExecutorFactory(wexpr, context, mode);
+		auto wexec = WindowExecutorFactory(wexpr, context, shared, mode);
 		executors.emplace_back(std::move(wexec));
 	}
 
@@ -568,6 +556,15 @@ WindowHashGroup::WindowHashGroup(WindowGlobalSinkState &gstate, const idx_t hash
 	if (rows) {
 		blocks = rows->blocks.size();
 	}
+
+	// Set up the collection for any fully materialised data
+	const auto &shared = WindowSharedExpressions::GetSortedExpressions(gstate.shared.coll_shared);
+	vector<LogicalType> types;
+	for (auto &expr : shared) {
+		types.emplace_back(expr->return_type);
+	}
+	auto &buffer_manager = BufferManager::GetBufferManager(gstate.context);
+	collection = make_uniq<WindowCollection>(buffer_manager, count, types);
 }
 
 // Per-thread scan state
@@ -606,6 +603,19 @@ protected:
 	void Sink();
 	void Finalize();
 	void GetData(DataChunk &chunk);
+
+	//! Storage and evaluation for the fully materialised data
+	unique_ptr<WindowBuilder> builder;
+	ExpressionExecutor coll_exec;
+	DataChunk coll_chunk;
+
+	//! Storage and evaluation for chunks used in the sink/build phase
+	ExpressionExecutor sink_exec;
+	DataChunk sink_chunk;
+
+	//! Storage and evaluation for chunks used in the evaluate phase
+	ExpressionExecutor eval_exec;
+	DataChunk eval_chunk;
 };
 
 WindowHashGroup::ExecutorGlobalStates &WindowHashGroup::Initialize(WindowGlobalSinkState &gsink) {
@@ -660,8 +670,30 @@ void WindowLocalSourceState::Sink() {
 				break;
 			}
 
+			//	Compute fully materialised expressions
+			if (coll_chunk.data.empty()) {
+				coll_chunk.SetCardinality(input_chunk);
+			} else {
+				coll_chunk.Reset();
+				coll_exec.Execute(input_chunk, coll_chunk);
+				auto collection = window_hash_group->collection.get();
+				if (!builder || &builder->collection != collection) {
+					builder = make_uniq<WindowBuilder>(*collection);
+				}
+
+				builder->Sink(coll_chunk, input_idx);
+			}
+
+			// Compute sink expressions
+			if (sink_chunk.data.empty()) {
+				sink_chunk.SetCardinality(input_chunk);
+			} else {
+				sink_chunk.Reset();
+				sink_exec.Execute(input_chunk, sink_chunk);
+			}
+
 			for (idx_t w = 0; w < executors.size(); ++w) {
-				executors[w]->Sink(input_chunk, input_idx, window_hash_group->count, *gestates[w], *local_states[w]);
+				executors[w]->Sink(sink_chunk, coll_chunk, input_idx, *gestates[w], *local_states[w]);
 			}
 
 			window_hash_group->sunk += input_chunk.size();
@@ -677,15 +709,20 @@ void WindowLocalSourceState::Finalize() {
 	D_ASSERT(task);
 	D_ASSERT(task->stage == WindowGroupStage::FINALIZE);
 
+	// First finalize the collection (so the executors can use it)
+	auto &gsink = gsource.gsink;
+	if (window_hash_group->collection) {
+		window_hash_group->collection->Combine(gsink.shared.coll_validity);
+	}
+
 	// Finalize all the executors.
 	// Parallel finalisation is handled internally by the executor,
 	// and should not return until all threads have completed work.
-	auto &gsink = gsource.gsink;
 	const auto &executors = gsink.executors;
 	auto &gestates = window_hash_group->gestates;
 	auto &local_states = window_hash_group->thread_states.at(task->thread_idx);
 	for (idx_t w = 0; w < executors.size(); ++w) {
-		executors[w]->Finalize(*gestates[w], *local_states[w]);
+		executors[w]->Finalize(*gestates[w], *local_states[w], window_hash_group->collection);
 	}
 
 	//	Mark this range as done
@@ -693,7 +730,9 @@ void WindowLocalSourceState::Finalize() {
 	task->begin_idx = task->end_idx;
 }
 
-WindowLocalSourceState::WindowLocalSourceState(WindowGlobalSourceState &gsource) : gsource(gsource), batch_index(0) {
+WindowLocalSourceState::WindowLocalSourceState(WindowGlobalSourceState &gsource)
+    : gsource(gsource), batch_index(0), coll_exec(gsource.context), sink_exec(gsource.context),
+      eval_exec(gsource.context) {
 	auto &gsink = gsource.gsink;
 	auto &global_partition = *gsink.global_partition;
 
@@ -706,6 +745,11 @@ WindowLocalSourceState::WindowLocalSourceState(WindowGlobalSourceState &gsource)
 	}
 	output_chunk.Initialize(global_partition.allocator, output_types);
 
+	auto &shared = gsink.shared;
+	shared.PrepareCollection(coll_exec, coll_chunk);
+	shared.PrepareSink(sink_exec, sink_chunk);
+	shared.PrepareEvaluate(eval_exec, eval_chunk);
+
 	++gsource.locals;
 }
 
@@ -819,7 +863,13 @@ void WindowLocalSourceState::GetData(DataChunk &result) {
 		auto &gstate = *gestates[expr_idx];
 		auto &lstate = *local_states[expr_idx];
 		auto &result = output_chunk.data[expr_idx];
-		executor.Evaluate(position, input_chunk, result, lstate, gstate);
+		if (eval_chunk.data.empty()) {
+			eval_chunk.SetCardinality(input_chunk);
+		} else {
+			eval_chunk.Reset();
+			eval_exec.Execute(input_chunk, eval_chunk);
+		}
+		executor.Evaluate(position, eval_chunk, result, lstate, gstate);
 	}
 	output_chunk.SetCardinality(input_chunk);
 	output_chunk.Verify();
@@ -856,7 +906,10 @@ unique_ptr<GlobalSourceState> PhysicalWindow::GetGlobalSourceState(ClientContext
 	return make_uniq<WindowGlobalSourceState>(context, gsink);
 }
 
-bool PhysicalWindow::SupportsBatchIndex() const {
+bool PhysicalWindow::SupportsPartitioning(const OperatorPartitionInfo &partition_info) const {
+	if (partition_info.RequiresPartitionColumns()) {
+		return false;
+	}
 	//	We can only preserve order for single partitioning
 	//	or work stealing causes out of order batch numbers
 	auto &wexpr = select_list[order_idx]->Cast<BoundWindowExpression>();
@@ -878,19 +931,30 @@ OrderPreservationType PhysicalWindow::SourceOrder() const {
 	return OrderPreservationType::FIXED_ORDER;
 }
 
-double PhysicalWindow::GetProgress(ClientContext &context, GlobalSourceState &gsource_p) const {
+ProgressData PhysicalWindow::GetProgress(ClientContext &context, GlobalSourceState &gsource_p) const {
 	auto &gsource = gsource_p.Cast<WindowGlobalSourceState>();
 	const auto returned = gsource.returned.load();
 
 	auto &gsink = gsource.gsink;
 	const auto count = gsink.global_partition->count.load();
-	return count ? (double(returned) / double(count)) : -1;
+	ProgressData res;
+	if (count) {
+		res.done = double(returned);
+		res.total = double(count);
+	} else {
+		res.SetInvalid();
+	}
+	return res;
 }
 
-idx_t PhysicalWindow::GetBatchIndex(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate_p,
-                                    LocalSourceState &lstate_p) const {
+OperatorPartitionData PhysicalWindow::GetPartitionData(ExecutionContext &context, DataChunk &chunk,
+                                                       GlobalSourceState &gstate_p, LocalSourceState &lstate_p,
+                                                       const OperatorPartitionInfo &partition_info) const {
+	if (partition_info.RequiresPartitionColumns()) {
+		throw InternalException("PhysicalWindow::GetPartitionData: partition columns not supported");
+	}
 	auto &lstate = lstate_p.Cast<WindowLocalSourceState>();
-	return lstate.batch_index;
+	return OperatorPartitionData(lstate.batch_index);
 }
 
 SourceResultType PhysicalWindow::GetData(ExecutionContext &context, DataChunk &chunk,

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp

@@ -58,7 +58,7 @@ void CSVBuffer::AllocateBuffer(idx_t buffer_size) {
 	block = handle.GetBlockHandle();
 }
 
-idx_t CSVBuffer::GetBufferSize() {
+idx_t CSVBuffer::GetBufferSize() const {
 	return actual_buffer_size;
 }
 
@@ -87,7 +87,7 @@ void CSVBuffer::Unpin() {
 	}
 }
 
-bool CSVBuffer::IsCSVFileLastBuffer() {
+bool CSVBuffer::IsCSVFileLastBuffer() const {
 	return last_buffer;
 }
 

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp

@@ -4,20 +4,18 @@
 namespace duckdb {
 
 CSVBufferManager::CSVBufferManager(ClientContext &context_p, const CSVReaderOptions &options, const string &file_path_p,
-                                   const idx_t file_idx_p, bool per_file_single_threaded_p)
+                                   const idx_t file_idx_p, bool per_file_single_threaded_p,
+                                   unique_ptr<CSVFileHandle> file_handle_p)
     : context(context_p), per_file_single_threaded(per_file_single_threaded_p), file_idx(file_idx_p),
-      file_path(file_path_p), buffer_size(CSVBuffer::CSV_BUFFER_SIZE) {
+      file_path(file_path_p), buffer_size(options.buffer_size_option.GetValue()) {
 	D_ASSERT(!file_path.empty());
-	file_handle = ReadCSV::OpenCSV(file_path, options.compression, context);
+	if (file_handle_p) {
+		file_handle = std::move(file_handle_p);
+	} else {
+		file_handle = ReadCSV::OpenCSV(file_path, options, context);
+	}
 	is_pipe = file_handle->IsPipe();
 	skip_rows = options.dialect_options.skip_rows.GetValue();
-	auto file_size = file_handle->FileSize();
-	if (file_size > 0 && file_size < buffer_size) {
-		buffer_size = CSVBuffer::CSV_MINIMUM_BUFFER_SIZE;
-	}
-	if (options.buffer_size < buffer_size) {
-		buffer_size = options.buffer_size;
-	}
 	Initialize();
 }
 
@@ -148,4 +146,15 @@ string CSVBufferManager::GetFilePath() const {
 	return file_path;
 }
 
+bool CSVBufferManager::IsBlockUnloaded(idx_t block_idx) {
+	if (block_idx < cached_buffers.size()) {
+		return cached_buffers[block_idx]->IsUnloaded();
+	}
+	return false;
+}
+
+idx_t CSVBufferManager::GetBytesRead() const {
+	return bytes_read;
+}
+
 } // namespace duckdb

package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp

@@ -3,12 +3,14 @@
 #include "duckdb/common/numeric_utils.hpp"
 #include "duckdb/common/compressed_file_system.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/execution/operator/csv_scanner/csv_reader_options.hpp"
 
 namespace duckdb {
 
-CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<FileHandle> file_handle_p,
-                             const string &path_p, FileCompressionType compression)
-    : compression_type(compression), file_handle(std::move(file_handle_p)), path(path_p) {
+CSVFileHandle::CSVFileHandle(DBConfig &config, unique_ptr<FileHandle> file_handle_p, const string &path_p,
+                             const CSVReaderOptions &options)
+    : compression_type(options.compression), file_handle(std::move(file_handle_p)),
+      encoder(config, options.encoding, options.buffer_size_option.GetValue()), path(path_p) {
 	can_seek = file_handle->CanSeek();
 	on_disk_file = file_handle->OnDiskFile();
 	file_size = file_handle->GetFileSize();
@@ -25,21 +27,21 @@ unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &
 	return file_handle;
 }
 
-unique_ptr<CSVFileHandle> CSVFileHandle::OpenFile(FileSystem &fs, Allocator &allocator, const string &path,
-                                                  FileCompressionType compression) {
-	auto file_handle = CSVFileHandle::OpenFileHandle(fs, allocator, path, compression);
-	return make_uniq<CSVFileHandle>(fs, allocator, std::move(file_handle), path, compression);
+unique_ptr<CSVFileHandle> CSVFileHandle::OpenFile(DBConfig &config, FileSystem &fs, Allocator &allocator,
+                                                  const string &path, const CSVReaderOptions &options) {
+	auto file_handle = OpenFileHandle(fs, allocator, path, options.compression);
+	return make_uniq<CSVFileHandle>(config, std::move(file_handle), path, options);
 }
 
-double CSVFileHandle::GetProgress() {
+double CSVFileHandle::GetProgress() const {
 	return static_cast<double>(file_handle->GetProgress());
 }
 
-bool CSVFileHandle::CanSeek() {
+bool CSVFileHandle::CanSeek() const {
 	return can_seek;
 }
 
-void CSVFileHandle::Seek(idx_t position) {
+void CSVFileHandle::Seek(const idx_t position) const {
 	if (!can_seek) {
 		if (is_pipe) {
 			throw InternalException("Trying to seek a piped CSV File.");
@@ -49,7 +51,7 @@ void CSVFileHandle::Seek(idx_t position) {
 	file_handle->Seek(position);
 }
 
-bool CSVFileHandle::OnDiskFile() {
+bool CSVFileHandle::OnDiskFile() const {
 	return on_disk_file;
 }
 
@@ -59,22 +61,27 @@ void CSVFileHandle::Reset() {
 	requested_bytes = 0;
 }
 
-bool CSVFileHandle::IsPipe() {
+bool CSVFileHandle::IsPipe() const {
 	return is_pipe;
 }
 
-idx_t CSVFileHandle::FileSize() {
+idx_t CSVFileHandle::FileSize() const {
 	return file_size;
 }
 
-bool CSVFileHandle::FinishedReading() {
+bool CSVFileHandle::FinishedReading() const {
 	return finished;
 }
 
 idx_t CSVFileHandle::Read(void *buffer, idx_t nr_bytes) {
 	requested_bytes += nr_bytes;
 	// if this is a plain file source OR we can seek we are not caching anything
-	auto bytes_read = file_handle->Read(buffer, nr_bytes);
+	idx_t bytes_read = 0;
+	if (encoder.encoding_name == "utf-8") {
+		bytes_read = static_cast<idx_t>(file_handle->Read(buffer, nr_bytes));
+	} else {
+		bytes_read = encoder.Encode(*file_handle, static_cast<char *>(buffer), nr_bytes);
+	}
 	if (!finished) {
 		finished = bytes_read == 0;
 	}

package/src/duckdb/src/execution/operator/csv_scanner/encode/csv_encoder.cpp

@@ -0,0 +1,95 @@
+#include "duckdb/execution/operator/csv_scanner/encode/csv_encoder.hpp"
+#include "duckdb/common/exception.hpp"
+#include "duckdb/main/config.hpp"
+#include "duckdb/function/encoding_function.hpp"
+
+namespace duckdb {
+
+void CSVEncoderBuffer::Initialize(idx_t encoded_size) {
+	encoded_buffer_size = encoded_size;
+	encoded_buffer = std::unique_ptr<char[]>(new char[encoded_size]);
+}
+
+char *CSVEncoderBuffer::Ptr() const {
+	return encoded_buffer.get();
+}
+
+idx_t CSVEncoderBuffer::GetCapacity() const {
+	return encoded_buffer_size;
+}
+
+idx_t CSVEncoderBuffer::GetSize() const {
+	return actual_encoded_buffer_size;
+}
+
+void CSVEncoderBuffer::SetSize(const idx_t buffer_size) {
+	D_ASSERT(buffer_size <= encoded_buffer_size);
+	actual_encoded_buffer_size = buffer_size;
+}
+
+bool CSVEncoderBuffer::HasDataToRead() const {
+	return cur_pos < actual_encoded_buffer_size;
+}
+
+void CSVEncoderBuffer::Reset() {
+	cur_pos = 0;
+	actual_encoded_buffer_size = 0;
+}
+
+CSVEncoder::CSVEncoder(DBConfig &config, const string &encoding_name_to_find, idx_t buffer_size) {
+	encoding_name = StringUtil::Lower(encoding_name_to_find);
+	auto function = config.GetEncodeFunction(encoding_name_to_find);
+	if (!function) {
+		auto loaded_encodings = config.GetLoadedEncodedFunctions();
+		std::ostringstream error;
+		error << "The CSV Reader does not support the encoding: \"" << encoding_name_to_find << "\"\n";
+		error << "The currently supported encodings are: " << '\n';
+		for (auto &encoding_function : loaded_encodings) {
+			error << "*  " << encoding_function.get().GetType() << '\n';
+		}
+		throw InvalidInputException(error.str());
+	}
+	// We ensure that the encoded buffer size is an even number to make the two byte lookup on utf-16 work
+	idx_t encoded_buffer_size = buffer_size % 2 != 0 ? buffer_size - 1 : buffer_size;
+	D_ASSERT(encoded_buffer_size > 0);
+	encoded_buffer.Initialize(encoded_buffer_size);
+	remaining_bytes_buffer.Initialize(function->GetBytesPerIteration());
+	encoding_function = function;
+}
+
+idx_t CSVEncoder::Encode(FileHandle &file_handle_input, char *output_buffer, const idx_t decoded_buffer_size) {
+	idx_t output_buffer_pos = 0;
+	// Check if we have some left-overs. These can either be
+	// 1. missing decoded bytes
+	if (remaining_bytes_buffer.HasDataToRead()) {
+		D_ASSERT(remaining_bytes_buffer.cur_pos == 0);
+		const auto remaining_bytes_buffer_ptr = remaining_bytes_buffer.Ptr();
+		for (; remaining_bytes_buffer.cur_pos < remaining_bytes_buffer.GetSize(); remaining_bytes_buffer.cur_pos++) {
+			output_buffer[output_buffer_pos++] = remaining_bytes_buffer_ptr[remaining_bytes_buffer.cur_pos];
+		}
+		remaining_bytes_buffer.Reset();
+	}
+	// 2. remaining encoded buffer
+	if (encoded_buffer.HasDataToRead()) {
+		encoding_function->GetFunction()(
+		    encoded_buffer.Ptr(), encoded_buffer.cur_pos, encoded_buffer.GetSize(), output_buffer, output_buffer_pos,
+		    decoded_buffer_size, remaining_bytes_buffer.Ptr(), remaining_bytes_buffer.actual_encoded_buffer_size);
+	}
+	// 3. a new encoded buffer from the file
+	while (output_buffer_pos < decoded_buffer_size) {
+		idx_t current_decoded_buffer_start = output_buffer_pos;
+		encoded_buffer.Reset();
+		auto actual_encoded_bytes =
+		    static_cast<idx_t>(file_handle_input.Read(encoded_buffer.Ptr(), encoded_buffer.GetCapacity()));
+		encoded_buffer.SetSize(actual_encoded_bytes);
+		encoding_function->GetFunction()(
+		    encoded_buffer.Ptr(), encoded_buffer.cur_pos, encoded_buffer.GetSize(), output_buffer, output_buffer_pos,
+		    decoded_buffer_size, remaining_bytes_buffer.Ptr(), remaining_bytes_buffer.actual_encoded_buffer_size);
+		if (output_buffer_pos == current_decoded_buffer_start) {
+			return output_buffer_pos;
+		}
+	}
+	return output_buffer_pos;
+}
+
+} // namespace duckdb

package/src/duckdb/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp

@@ -141,12 +141,17 @@ void ColumnCountScanner::FinalizeChunkProcess() {
 			cur_buffer_handle = buffer_manager->GetBuffer(++iterator.pos.buffer_idx);
 			if (!cur_buffer_handle) {
 				buffer_handle_ptr = nullptr;
+				if (states.IsQuotedCurrent() && !states.IsUnquoted()) {
+					// We are finishing our file on a quoted value that is never unquoted, straight to jail.
+					result.error = true;
+					return;
+				}
 				if (states.EmptyLine() || states.NewRow() || states.IsCurrentNewRow() || states.IsNotSet()) {
 					return;
 				}
 				// This means we reached the end of the file, we must add a last line if there is any to be added
 				if (result.comment) {
-					// If it's a comment we add the last line via unsetcomment
+					// If it's a comment we add the last line via unset comment
 					result.UnsetComment(result, NumericLimits<idx_t>::Maximum());
 				} else {
 					// OW, we do a regular AddRow

package/src/duckdb/src/execution/operator/csv_scanner/scanner/csv_schema.cpp

@@ -43,7 +43,46 @@ bool CSVSchema::CanWeCastIt(LogicalTypeId source, LogicalTypeId destination) {
 	}
 }
 
-void CSVSchema::Initialize(vector<string> &names, vector<LogicalType> &types, const string &file_path_p) {
+void CSVSchema::MergeSchemas(CSVSchema &other, bool null_padding) {
+	// TODO: We could also merge names, maybe by giving preference to non-generated names?
+	const vector<LogicalType> candidates_by_specificity = {LogicalType::BOOLEAN, LogicalType::BIGINT,
+	                                                       LogicalType::DOUBLE, LogicalType::VARCHAR};
+	for (idx_t i = 0; i < columns.size() && i < other.columns.size(); i++) {
+		auto this_type = columns[i].type.id();
+		auto other_type = other.columns[i].type.id();
+		if (columns[i].type != other.columns[i].type) {
+			if (CanWeCastIt(this_type, other_type)) {
+				// If we can cast this to other, this becomes other
+				columns[i].type = other.columns[i].type;
+			} else if (!CanWeCastIt(other_type, this_type)) {
+				// If we can't cast this to other or other to this, we see which parent they can be both cast to
+				for (const auto &type : candidates_by_specificity) {
+					if (CanWeCastIt(this_type, type.id()) && CanWeCastIt(other_type, type.id())) {
+						columns[i].type = type;
+						break;
+					}
+				}
+			}
+		}
+	}
+
+	if (null_padding && other.columns.size() > columns.size()) {
+		for (idx_t i = columns.size(); i < other.columns.size(); i++) {
+			auto name = other.columns[i].name;
+			auto type = other.columns[i].type;
+			columns.push_back({name, type});
+			name_idx_map[name] = i;
+		}
+	}
+}
+
+CSVSchema::CSVSchema(vector<string> &names, vector<LogicalType> &types, const string &file_path, idx_t rows_read_p,
+                     const bool empty_p)
+    : rows_read(rows_read_p), empty(empty_p) {
+	Initialize(names, types, file_path);
+}
+
+void CSVSchema::Initialize(const vector<string> &names, const vector<LogicalType> &types, const string &file_path_p) {
 	if (!columns.empty()) {
 		throw InternalException("CSV Schema is already populated, this should not happen.");
 	}
@@ -51,15 +90,49 @@ void CSVSchema::Initialize(vector<string> &names, vector<LogicalType> &types, co
 	D_ASSERT(names.size() == types.size() && !names.empty());
 	for (idx_t i = 0; i < names.size(); i++) {
 		// Populate our little schema
-		columns.push_back({names[i], types[i]});
+		auto name = names.at(i);
+		auto type = types.at(i);
+		columns.push_back({name, type});
 		name_idx_map[names[i]] = i;
 	}
 }
 
+vector<string> CSVSchema::GetNames() const {
+	vector<string> names;
+	for (auto &column : columns) {
+		names.push_back(column.name);
+	}
+	return names;
+}
+
+vector<LogicalType> CSVSchema::GetTypes() const {
+	vector<LogicalType> types;
+	for (auto &column : columns) {
+		types.push_back(column.type);
+	}
+	return types;
+}
+
 bool CSVSchema::Empty() const {
 	return columns.empty();
 }
 
+bool CSVSchema::MatchColumns(const CSVSchema &other) const {
+	return other.columns.size() == columns.size() || empty || other.empty;
+}
+
+string CSVSchema::GetPath() const {
+	return file_path;
+}
+
+idx_t CSVSchema::GetColumnCount() const {
+	return columns.size();
+}
+
+idx_t CSVSchema::GetRowsRead() const {
+	return rows_read;
+}
+
 bool CSVSchema::SchemasMatch(string &error_message, SnifferResult &sniffer_result, const string &cur_file_path,
                              bool is_minimal_sniffer) const {
 	D_ASSERT(sniffer_result.names.size() == sniffer_result.return_types.size());