duckdb 1.3.3-dev0.0 → 1.3.3-dev4.0

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -1,4 +1,5 @@
 #include "duckdb/common/bind_helpers.hpp"
+#include "duckdb/common/csv_writer.hpp"
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/multi_file/multi_file_reader.hpp"
 #include "duckdb/common/serializer/memory_stream.hpp"
@@ -20,8 +21,6 @@
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/common/multi_file/multi_file_function.hpp"
 
-#include <limits>
-
 namespace duckdb {
 
 void AreOptionsEqual(char str_1, char str_2, const string &name_str_1, const string &name_str_2) {
@@ -54,11 +53,6 @@ void StringDetection(const string &str_1, const string &str_2, const string &nam
 //===--------------------------------------------------------------------===//
 // Bind
 //===--------------------------------------------------------------------===//
-void WriteQuoteOrEscape(WriteStream &writer, char quote_or_escape) {
-	if (quote_or_escape != '\0') {
-		writer.Write(quote_or_escape);
-	}
-}
 
 void BaseCSVData::Finalize() {
 	auto delimiter_string = options.dialect_options.state_machine_options.delimiter.GetValue();
@@ -120,12 +114,6 @@ void BaseCSVData::Finalize() {
 	}
 }
 
-string TransformNewLine(string new_line) {
-	new_line = StringUtil::Replace(new_line, "\\r", "\r");
-	return StringUtil::Replace(new_line, "\\n", "\n");
-	;
-}
-
 static vector<unique_ptr<Expression>> CreateCastExpressions(WriteCSVData &bind_data, ClientContext &context,
                                                             const vector<string> &names,
                                                             const vector<LogicalType> &sql_types) {
@@ -184,7 +172,7 @@ static vector<unique_ptr<Expression>> CreateCastExpressions(WriteCSVData &bind_d
 
 static unique_ptr<FunctionData> WriteCSVBind(ClientContext &context, CopyFunctionBindInput &input,
                                              const vector<string> &names, const vector<LogicalType> &sql_types) {
-	auto bind_data = make_uniq<WriteCSVData>(input.info.file_path, sql_types, names);
+	auto bind_data = make_uniq<WriteCSVData>(names);
 
 	// check all the options in the copy info
 	for (auto &option : input.info.options) {
@@ -217,110 +205,51 @@ static unique_ptr<FunctionData> WriteCSVBind(ClientContext &context, CopyFunctio
 	auto expressions = CreateCastExpressions(*bind_data, context, names, sql_types);
 	bind_data->cast_expressions = std::move(expressions);
 
-	bind_data->requires_quotes = make_unsafe_uniq_array<bool>(256);
-	memset(bind_data->requires_quotes.get(), 0, sizeof(bool) * 256);
-	bind_data->requires_quotes['\n'] = true;
-	bind_data->requires_quotes['\r'] = true;
-	bind_data->requires_quotes['#'] = true;
-	bind_data->requires_quotes[NumericCast<idx_t>(
-	    bind_data->options.dialect_options.state_machine_options.delimiter.GetValue()[0])] = true;
-	bind_data->requires_quotes[NumericCast<idx_t>(
-	    bind_data->options.dialect_options.state_machine_options.quote.GetValue())] = true;
-
-	if (!bind_data->options.write_newline.empty()) {
-		bind_data->newline = TransformNewLine(bind_data->options.write_newline);
-	}
 	return std::move(bind_data);
 }
 
-//===--------------------------------------------------------------------===//
-// Helper writing functions
-//===--------------------------------------------------------------------===//
-static string AddEscapes(char to_be_escaped, const char escape, const string &val) {
-	idx_t i = 0;
-	string new_val = "";
-	idx_t found = val.find(to_be_escaped);
-
-	while (found != string::npos) {
-		while (i < found) {
-			new_val += val[i];
-			i++;
-		}
-		if (escape != '\0') {
-			new_val += escape;
-			found = val.find(to_be_escaped, found + 1);
-		}
-	}
-	while (i < val.length()) {
-		new_val += val[i];
-		i++;
-	}
-	return new_val;
-}
-
-static bool RequiresQuotes(WriteCSVData &csv_data, const char *str, idx_t len) {
-	auto &options = csv_data.options;
-	// check if the string is equal to the null string
-	if (len == options.null_str[0].size() && memcmp(str, options.null_str[0].c_str(), len) == 0) {
-		return true;
-	}
-	auto str_data = reinterpret_cast<const_data_ptr_t>(str);
-	for (idx_t i = 0; i < len; i++) {
-		if (csv_data.requires_quotes[str_data[i]]) {
-			// this byte requires quotes - write a quoted string
-			return true;
-		}
-	}
-	// no newline, quote or delimiter in the string
-	// no quoting or escaping necessary
-	return false;
-}
-
-static void WriteQuotedString(WriteStream &writer, WriteCSVData &csv_data, const char *str, idx_t len,
-                              bool force_quote) {
-	auto &options = csv_data.options;
-	if (!force_quote) {
-		// force quote is disabled: check if we need to add quotes anyway
-		force_quote = RequiresQuotes(csv_data, str, len);
-	}
-	// If a quote is set to none (i.e., null-terminator) we skip the quotation
-	if (force_quote && options.dialect_options.state_machine_options.quote.GetValue() != '\0') {
-		// quoting is enabled: we might need to escape things in the string
-		bool requires_escape = false;
-		// simple CSV
-		// do a single loop to check for a quote or escape value
-		for (idx_t i = 0; i < len; i++) {
-			if (str[i] == options.dialect_options.state_machine_options.quote.GetValue() ||
-			    str[i] == options.dialect_options.state_machine_options.escape.GetValue()) {
-				requires_escape = true;
-				break;
-			}
-		}
-
-		if (!requires_escape) {
-			// fast path: no need to escape anything
-			WriteQuoteOrEscape(writer, options.dialect_options.state_machine_options.quote.GetValue());
-			writer.WriteData(const_data_ptr_cast(str), len);
-			WriteQuoteOrEscape(writer, options.dialect_options.state_machine_options.quote.GetValue());
-			return;
-		}
-
-		// slow path: need to add escapes
-		string new_val(str, len);
-		new_val = AddEscapes(options.dialect_options.state_machine_options.escape.GetValue(),
-		                     options.dialect_options.state_machine_options.escape.GetValue(), new_val);
-		if (options.dialect_options.state_machine_options.escape !=
-		    options.dialect_options.state_machine_options.quote) {
-			// need to escape quotes separately
-			new_val = AddEscapes(options.dialect_options.state_machine_options.quote.GetValue(),
-			                     options.dialect_options.state_machine_options.escape.GetValue(), new_val);
-		}
-		WriteQuoteOrEscape(writer, options.dialect_options.state_machine_options.quote.GetValue());
-		writer.WriteData(const_data_ptr_cast(new_val.c_str()), new_val.size());
-		WriteQuoteOrEscape(writer, options.dialect_options.state_machine_options.quote.GetValue());
-	} else {
-		writer.WriteData(const_data_ptr_cast(str), len);
-	}
+static void CSVListCopyOptions(ClientContext &context, CopyOptionsInput &input) {
+	auto &copy_options = input.options;
+	copy_options["auto_detect"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY);
+	copy_options["sample_size"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::READ_ONLY);
+	copy_options["skip"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::READ_ONLY);
+	copy_options["max_line_size"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::READ_ONLY);
+	copy_options["maximum_line_size"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::READ_ONLY);
+	copy_options["ignore_errors"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY);
+	copy_options["buffer_size"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::READ_ONLY);
+	copy_options["decimal_separator"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_ONLY);
+	copy_options["null_padding"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY);
+	copy_options["parallel"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY);
+	copy_options["allow_quoted_nulls"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY);
+	copy_options["store_rejects"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_ONLY);
+	copy_options["force_not_null"] = CopyOption(LogicalType::ANY, CopyOptionMode::READ_ONLY);
+	copy_options["rejects_table"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_ONLY);
+	copy_options["rejects_scan"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_ONLY);
+	copy_options["rejects_limit"] = CopyOption(LogicalType::BIGINT, CopyOptionMode::READ_ONLY);
+	copy_options["encoding"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_ONLY);
+	copy_options["thousands"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_ONLY);
+
+	copy_options["force_quote"] = CopyOption(LogicalType::ANY, CopyOptionMode::WRITE_ONLY);
+	copy_options["prefix"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::WRITE_ONLY);
+	copy_options["suffix"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::WRITE_ONLY);
+
+	copy_options["new_line"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["date_format"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["dateformat"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["timestamp_format"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["timestampformat"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["quote"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["comment"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["delim"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["delimiter"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["sep"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["separator"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["escape"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["header"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_WRITE);
+	copy_options["nullstr"] = CopyOption(LogicalType::ANY, CopyOptionMode::READ_WRITE);
+	copy_options["null"] = CopyOption(LogicalType::ANY, CopyOptionMode::READ_WRITE);
+	copy_options["compression"] = CopyOption(LogicalType::VARCHAR, CopyOptionMode::READ_WRITE);
+	copy_options["strict_mode"] = CopyOption(LogicalType::BOOLEAN, CopyOptionMode::READ_WRITE);
 }
 
 //===--------------------------------------------------------------------===//
@@ -328,66 +257,35 @@ static void WriteQuotedString(WriteStream &writer, WriteCSVData &csv_data, const
 //===--------------------------------------------------------------------===//
 struct LocalWriteCSVData : public LocalFunctionData {
 public:
-	LocalWriteCSVData(ClientContext &context, vector<unique_ptr<Expression>> &expressions)
-	    : executor(context, expressions), stream(Allocator::Get(context)) {
+	LocalWriteCSVData(ClientContext &context, vector<unique_ptr<Expression>> &expressions, const idx_t &flush_size)
+	    : executor(context, expressions), writer_local_state(context, flush_size) {
 	}
 
 public:
 	//! Used to execute the expressions that transform input -> string
 	ExpressionExecutor executor;
-	//! The thread-local buffer to write data into
-	MemoryStream stream;
 	//! A chunk with VARCHAR columns to cast intermediates into
 	DataChunk cast_chunk;
-	//! If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY
-	bool written_anything = false;
+	//! Local state for the CSV writer
+	CSVWriterState writer_local_state;
 };
 
 struct GlobalWriteCSVData : public GlobalFunctionData {
-	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileCompressionType compression)
-	    : fs(fs), written_anything(false) {
-		handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW |
-		                                    FileLockType::WRITE_LOCK | compression);
-	}
-
-	//! Write generic data, e.g., CSV header
-	void WriteData(const_data_ptr_t data, idx_t size) {
-		lock_guard<mutex> flock(lock);
-		handle->Write((void *)data, size);
-	}
-
-	void WriteData(const char *data, idx_t size) {
-		WriteData(const_data_ptr_cast(data), size);
-	}
-
-	//! Write rows
-	void WriteRows(const_data_ptr_t data, idx_t size, const string &newline) {
-		lock_guard<mutex> flock(lock);
-		if (written_anything) {
-			handle->Write((void *)newline.c_str(), newline.length());
-		} else {
-			written_anything = true;
-		}
-		handle->Write((void *)data, size);
+	GlobalWriteCSVData(CSVReaderOptions &options, FileSystem &fs, const string &file_path,
+	                   FileCompressionType compression)
+	    : writer(options, fs, file_path, compression) {
 	}
 
 	idx_t FileSize() {
-		lock_guard<mutex> flock(lock);
-		return handle->GetFileSize();
+		return writer.FileSize();
 	}
 
-	FileSystem &fs;
-	//! The mutex for writing to the physical file
-	mutex lock;
-	//! The file handle to write to
-	unique_ptr<FileHandle> handle;
-	//! If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY
-	bool written_anything;
+	CSVWriter writer;
 };
 
 static unique_ptr<LocalFunctionData> WriteCSVInitializeLocal(ExecutionContext &context, FunctionData &bind_data) {
 	auto &csv_data = bind_data.Cast<WriteCSVData>();
-	auto local_data = make_uniq<LocalWriteCSVData>(context.client, csv_data.cast_expressions);
+	auto local_data = make_uniq<LocalWriteCSVData>(context.client, csv_data.cast_expressions, csv_data.flush_size);
 
 	// create the chunk with VARCHAR types
 	vector<LogicalType> types;
@@ -402,36 +300,15 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &options = csv_data.options;
 	auto global_data =
-	    make_uniq<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), file_path, options.compression);
-
-	if (!options.prefix.empty()) {
-		global_data->WriteData(options.prefix.c_str(), options.prefix.size());
-	}
-
-	if (!(options.dialect_options.header.IsSetByUser() && !options.dialect_options.header.GetValue())) {
-		MemoryStream stream(Allocator::Get(context));
-		// write the header line to the file
-		for (idx_t i = 0; i < csv_data.options.name_list.size(); i++) {
-			if (i != 0) {
-				WriteQuoteOrEscape(stream, options.dialect_options.state_machine_options.delimiter.GetValue()[0]);
-			}
-			WriteQuotedString(stream, csv_data, csv_data.options.name_list[i].c_str(),
-			                  csv_data.options.name_list[i].size(), false);
-		}
-		stream.WriteData(const_data_ptr_cast(csv_data.newline.c_str()), csv_data.newline.size());
+	    make_uniq<GlobalWriteCSVData>(options, FileSystem::GetFileSystem(context), file_path, options.compression);
 
-		global_data->WriteData(stream.GetData(), stream.GetPosition());
-	}
+	global_data->writer.Initialize();
 
 	return std::move(global_data);
 }
 
-static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_data, DataChunk &cast_chunk,
-                                  MemoryStream &writer, DataChunk &input, bool &written_anything,
-                                  ExpressionExecutor &executor) {
-	auto &csv_data = bind_data.Cast<WriteCSVData>();
-	auto &options = csv_data.options;
-
+static void WriteCSVChunkInternal(CSVWriter &writer, CSVWriterState &writer_local_state, DataChunk &cast_chunk,
+                                  DataChunk &input, ExpressionExecutor &executor) {
 	// first cast the columns of the chunk to varchar
 	cast_chunk.Reset();
 	cast_chunk.SetCardinality(input);
@@ -439,53 +316,17 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat
 	executor.Execute(input, cast_chunk);
 
 	cast_chunk.Flatten();
-	// now loop over the vectors and output the values
-	for (idx_t row_idx = 0; row_idx < cast_chunk.size(); row_idx++) {
-		if (row_idx == 0 && !written_anything) {
-			written_anything = true;
-		} else {
-			writer.WriteData(const_data_ptr_cast(csv_data.newline.c_str()), csv_data.newline.size());
-		}
-		// write values
-		D_ASSERT(options.null_str.size() == 1);
-		for (idx_t col_idx = 0; col_idx < cast_chunk.ColumnCount(); col_idx++) {
-			if (col_idx != 0) {
-				WriteQuoteOrEscape(writer, options.dialect_options.state_machine_options.delimiter.GetValue()[0]);
-			}
-			if (FlatVector::IsNull(cast_chunk.data[col_idx], row_idx)) {
-				// write null value
-				writer.WriteData(const_data_ptr_cast(options.null_str[0].c_str()), options.null_str[0].size());
-				continue;
-			}
 
-			// non-null value, fetch the string value from the cast chunk
-			auto str_data = FlatVector::GetData<string_t>(cast_chunk.data[col_idx]);
-			// FIXME: we could gain some performance here by checking for certain types if they ever require quotes
-			// (e.g. integers only require quotes if the delimiter is a number, decimals only require quotes if the
-			// delimiter is a number or "." character)
-			WriteQuotedString(writer, csv_data, str_data[row_idx].GetData(), str_data[row_idx].GetSize(),
-			                  csv_data.options.force_quote[col_idx]);
-		}
-	}
+	writer.WriteChunk(cast_chunk, writer_local_state);
 }
 
 static void WriteCSVSink(ExecutionContext &context, FunctionData &bind_data, GlobalFunctionData &gstate,
                          LocalFunctionData &lstate, DataChunk &input) {
-	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &local_data = lstate.Cast<LocalWriteCSVData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
 
-	// write data into the local buffer
-	WriteCSVChunkInternal(context.client, bind_data, local_data.cast_chunk, local_data.stream, input,
-	                      local_data.written_anything, local_data.executor);
-
-	// check if we should flush what we have currently written
-	auto &writer = local_data.stream;
-	if (writer.GetPosition() >= csv_data.flush_size) {
-		global_state.WriteRows(writer.GetData(), writer.GetPosition(), csv_data.newline);
-		writer.Rewind();
-		local_data.written_anything = false;
-	}
+	WriteCSVChunkInternal(global_state.writer, local_data.writer_local_state, local_data.cast_chunk, input,
+	                      local_data.executor);
 }
 
 //===--------------------------------------------------------------------===//
@@ -495,13 +336,7 @@ static void WriteCSVCombine(ExecutionContext &context, FunctionData &bind_data,
                             LocalFunctionData &lstate) {
 	auto &local_data = lstate.Cast<LocalWriteCSVData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
-	auto &csv_data = bind_data.Cast<WriteCSVData>();
-	auto &writer = local_data.stream;
-	// flush the local writer
-	if (local_data.written_anything) {
-		global_state.WriteRows(writer.GetData(), writer.GetPosition(), csv_data.newline);
-		writer.Rewind();
-	}
+	global_state.writer.Flush(local_data.writer_local_state);
 }
 
 //===--------------------------------------------------------------------===//
@@ -512,16 +347,12 @@ void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFun
 	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &options = csv_data.options;
 
-	MemoryStream stream(Allocator::Get(context));
 	if (!options.suffix.empty()) {
-		stream.WriteData(const_data_ptr_cast(options.suffix.c_str()), options.suffix.size());
-	} else if (global_state.written_anything) {
-		stream.WriteData(const_data_ptr_cast(csv_data.newline.c_str()), csv_data.newline.size());
+		global_state.writer.WriteRawString(options.suffix);
+	} else if (global_state.writer.WrittenAnything()) {
+		global_state.writer.WriteRawString(global_state.writer.writer_options.newline);
 	}
-	global_state.WriteData(stream.GetData(), stream.GetPosition());
-
-	global_state.handle->Close();
-	global_state.handle.reset();
+	global_state.writer.Close();
 }
 
 //===--------------------------------------------------------------------===//
@@ -540,11 +371,13 @@ CopyFunctionExecutionMode WriteCSVExecutionMode(bool preserve_insertion_order, b
 // Prepare Batch
 //===--------------------------------------------------------------------===//
 struct WriteCSVBatchData : public PreparedBatchData {
-	explicit WriteCSVBatchData(Allocator &allocator) : stream(allocator) {
+	explicit WriteCSVBatchData(ClientContext &context, const idx_t flush_size)
+	    : writer_local_state(make_uniq<CSVWriterState>(context, flush_size)) {
+		writer_local_state->require_manual_flush = true;
 	}
 
 	//! The thread-local buffer to write data into
-	MemoryStream stream;
+	unique_ptr<CSVWriterState> writer_local_state;
 };
 
 unique_ptr<PreparedBatchData> WriteCSVPrepareBatch(ClientContext &context, FunctionData &bind_data,
@@ -561,12 +394,12 @@ unique_ptr<PreparedBatchData> WriteCSVPrepareBatch(ClientContext &context, Funct
 	auto &original_types = collection->Types();
 	auto expressions = CreateCastExpressions(csv_data, context, csv_data.options.name_list, original_types);
 	ExpressionExecutor executor(context, expressions);
+	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
 
 	// write CSV chunks to the batch data
-	bool written_anything = false;
-	auto batch = make_uniq<WriteCSVBatchData>(Allocator::Get(context));
+	auto batch = make_uniq<WriteCSVBatchData>(context, NextPowerOfTwo(collection->SizeInBytes()));
 	for (auto &chunk : collection->Chunks()) {
-		WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->stream, chunk, written_anything, executor);
+		WriteCSVChunkInternal(global_state.writer, *batch->writer_local_state, cast_chunk, chunk, executor);
 	}
 	return std::move(batch);
 }
@@ -578,10 +411,7 @@ void WriteCSVFlushBatch(ClientContext &context, FunctionData &bind_data, GlobalF
                         PreparedBatchData &batch) {
 	auto &csv_batch = batch.Cast<WriteCSVBatchData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
-	auto &csv_data = bind_data.Cast<WriteCSVData>();
-	auto &writer = csv_batch.stream;
-	global_state.WriteRows(writer.GetData(), writer.GetPosition(), csv_data.newline);
-	writer.Rewind();
+	global_state.writer.Flush(*csv_batch.writer_local_state);
 }
 
 //===--------------------------------------------------------------------===//
@@ -593,12 +423,13 @@ bool WriteCSVRotateFiles(FunctionData &, const optional_idx &file_size_bytes) {
 
 bool WriteCSVRotateNextFile(GlobalFunctionData &gstate, FunctionData &, const optional_idx &file_size_bytes) {
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
-	return global_state.FileSize() > file_size_bytes.GetIndex();
+	return file_size_bytes.IsValid() && global_state.FileSize() > file_size_bytes.GetIndex();
 }
 
 void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 	CopyFunction info("csv");
 	info.copy_to_bind = WriteCSVBind;
+	info.copy_options = CSVListCopyOptions;
 	info.copy_to_initialize_local = WriteCSVInitializeLocal;
 	info.copy_to_initialize_global = WriteCSVInitializeGlobal;
 	info.copy_to_sink = WriteCSVSink;

package/src/duckdb/src/function/table/direct_file_reader.cpp

@@ -0,0 +1,172 @@
+#include "duckdb/function/table/direct_file_reader.hpp"
+#include "duckdb/function/table/read_file.hpp"
+#include "duckdb/storage/caching_file_system.hpp"
+
+namespace duckdb {
+
+DirectFileReader::DirectFileReader(OpenFileInfo file_p, const LogicalType &type)
+    : BaseFileReader(std::move(file_p)), done(false), type(type) {
+	columns.push_back(MultiFileColumnDefinition("filename", LogicalType::VARCHAR));
+	columns.push_back(MultiFileColumnDefinition("content", type));
+	columns.push_back(MultiFileColumnDefinition("size", LogicalType::BIGINT));
+	columns.push_back(MultiFileColumnDefinition("last_modified", LogicalType::TIMESTAMP_TZ));
+}
+
+DirectFileReader::~DirectFileReader() {
+}
+
+unique_ptr<BaseStatistics> DirectFileReader::GetStatistics(ClientContext &context, const string &name) {
+	return nullptr;
+}
+
+bool DirectFileReader::TryInitializeScan(ClientContext &context, GlobalTableFunctionState &gstate,
+                                         LocalTableFunctionState &lstate) {
+	auto &state = gstate.Cast<ReadFileGlobalState>();
+	return file_list_idx.GetIndex() < state.file_list->GetTotalFileCount() && !done;
+};
+
+static void AssertMaxFileSize(const string &file_name, idx_t file_size) {
+	const auto max_file_size = NumericLimits<uint32_t>::Maximum();
+	if (file_size > max_file_size) {
+		auto max_byte_size_format = StringUtil::BytesToHumanReadableString(max_file_size);
+		auto file_byte_size_format = StringUtil::BytesToHumanReadableString(file_size);
+		auto error_msg = StringUtil::Format("File '%s' size (%s) exceeds maximum allowed file (%s)", file_name.c_str(),
+		                                    file_byte_size_format, max_byte_size_format);
+		throw InvalidInputException(error_msg);
+	}
+}
+
+static inline void VERIFY(const string &filename, const string_t &content) {
+	if (Utf8Proc::Analyze(content.GetData(), content.GetSize()) == UnicodeType::INVALID) {
+		throw InvalidInputException("read_text: could not read content of file '%s' as valid UTF-8 encoded text. You "
+		                            "may want to use read_blob instead.",
+		                            filename);
+	}
+}
+
+void DirectFileReader::Scan(ClientContext &context, GlobalTableFunctionState &global_state,
+                            LocalTableFunctionState &local_state, DataChunk &output) {
+	auto &state = global_state.Cast<ReadFileGlobalState>();
+	if (done || file_list_idx.GetIndex() >= state.file_list->GetTotalFileCount()) {
+		return;
+	}
+
+	auto files = state.file_list;
+	auto fs = CachingFileSystem::Get(context);
+	idx_t out_idx = 0;
+
+	// We utilize projection pushdown here to only read the file content if the 'data' column is requested
+	unique_ptr<CachingFileHandle> file_handle = nullptr;
+
+	// Given the columns requested, do we even need to open the file?
+	if (state.requires_file_open) {
+		auto flags = FileFlags::FILE_FLAGS_READ;
+		if (FileSystem::IsRemoteFile(file.path)) {
+			flags |= FileFlags::FILE_FLAGS_DIRECT_IO;
+		}
+		file_handle = fs.OpenFile(QueryContext(context), file, flags);
+	}
+
+	for (idx_t col_idx = 0; col_idx < state.column_ids.size(); col_idx++) {
+		// We utilize projection pushdown to avoid potentially expensive fs operations.
+		auto proj_idx = state.column_ids[col_idx];
+		if (proj_idx == COLUMN_IDENTIFIER_ROW_ID) {
+			continue;
+		}
+		try {
+			switch (proj_idx) {
+			case ReadFileBindData::FILE_NAME_COLUMN: {
+				auto &file_name_vector = output.data[col_idx];
+				auto file_name_string = StringVector::AddString(file_name_vector, file.path);
+				FlatVector::GetData<string_t>(file_name_vector)[out_idx] = file_name_string;
+			} break;
+			case ReadFileBindData::FILE_CONTENT_COLUMN: {
+				auto file_size_raw = file_handle->GetFileSize();
+				AssertMaxFileSize(file.path, file_size_raw);
+				auto file_size = UnsafeNumericCast<int64_t>(file_size_raw);
+				auto &file_content_vector = output.data[col_idx];
+				auto content_string = StringVector::EmptyString(file_content_vector, file_size_raw);
+
+				auto remaining_bytes = UnsafeNumericCast<int64_t>(file_size);
+
+				// Read in batches of 100mb
+				constexpr auto MAX_READ_SIZE = 100LL * 1024 * 1024;
+				while (remaining_bytes > 0) {
+					const auto bytes_to_read = MinValue<int64_t>(remaining_bytes, MAX_READ_SIZE);
+					const auto content_string_ptr = content_string.GetDataWriteable() + (file_size - remaining_bytes);
+
+					idx_t actually_read;
+					if (file_handle->IsRemoteFile()) {
+						// Remote file: caching read
+						data_ptr_t read_ptr;
+						actually_read = NumericCast<idx_t>(bytes_to_read);
+						auto buffer_handle = file_handle->Read(read_ptr, actually_read);
+						memcpy(content_string_ptr, read_ptr, actually_read);
+					} else {
+						// Local file: non-caching read
+						actually_read = NumericCast<idx_t>(file_handle->GetFileHandle().Read(
+						    content_string_ptr, UnsafeNumericCast<idx_t>(bytes_to_read)));
+					}
+
+					if (actually_read == 0) {
+						// Uh oh, random EOF?
+						throw IOException("Failed to read file '%s' at offset %lu, unexpected EOF", file.path,
+						                  file_size - remaining_bytes);
+					}
+					remaining_bytes -= NumericCast<int64_t>(actually_read);
+				}
+
+				content_string.Finalize();
+
+				if (type == LogicalType::VARCHAR) {
+					VERIFY(file.path, content_string);
+				}
+
+				FlatVector::GetData<string_t>(file_content_vector)[out_idx] = content_string;
+			} break;
+			case ReadFileBindData::FILE_SIZE_COLUMN: {
+				auto &file_size_vector = output.data[col_idx];
+				FlatVector::GetData<int64_t>(file_size_vector)[out_idx] =
+				    NumericCast<int64_t>(file_handle->GetFileSize());
+			} break;
+			case ReadFileBindData::FILE_LAST_MODIFIED_COLUMN: {
+				auto &last_modified_vector = output.data[col_idx];
+				// This can sometimes fail (e.g. httpfs file system cant always parse the last modified time
+				// correctly)
+				try {
+					auto timestamp_seconds = file_handle->GetLastModifiedTime();
+					FlatVector::GetData<timestamp_tz_t>(last_modified_vector)[out_idx] =
+					    timestamp_tz_t(timestamp_seconds);
+				} catch (std::exception &ex) {
+					ErrorData error(ex);
+					if (error.Type() == ExceptionType::CONVERSION) {
+						FlatVector::SetNull(last_modified_vector, out_idx, true);
+					} else {
+						throw;
+					}
+				}
+			} break;
+			default:
+				break;
+			}
+		}
+		// Filesystems are not required to support all operations, so we just set the column to NULL if not
+		// implemented
+		catch (std::exception &ex) {
+			ErrorData error(ex);
+			if (error.Type() == ExceptionType::NOT_IMPLEMENTED) {
+				FlatVector::SetNull(output.data[col_idx], out_idx, true);
+			} else {
+				throw;
+			}
+		}
+	}
+	output.SetCardinality(1);
+	done = true;
+};
+
+void DirectFileReader::FinishFile(ClientContext &context, GlobalTableFunctionState &gstate) {
+	return;
+};
+
+} // namespace duckdb