duckdb 0.7.2-dev3546.0 → 0.7.2-dev3710.0

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -724,6 +724,11 @@ unique_ptr<TableRef> ParquetScanReplacement(ClientContext &context, const string
 	vector<unique_ptr<ParsedExpression>> children;
 	children.push_back(make_uniq<ConstantExpression>(Value(table_name)));
 	table_function->function = make_uniq<FunctionExpression>("parquet_scan", std::move(children));
+
+	if (!FileSystem::HasGlob(table_name)) {
+		table_function->alias = FileSystem::ExtractBaseName(table_name);
+	}
+
 	return std::move(table_function);
 }
 

package/src/duckdb/src/common/exception.cpp

@@ -59,6 +59,23 @@ string Exception::GetStackTrace(int max_depth) {
 }
 
 string Exception::ConstructMessageRecursive(const string &msg, std::vector<ExceptionFormatValue> &values) {
+#ifdef DEBUG
+	// Verify that we have the required amount of values for the message
+	idx_t parameter_count = 0;
+	for (idx_t i = 0; i < msg.size(); i++) {
+		if (msg[i] != '%') {
+			continue;
+		}
+		if (i < msg.size() && msg[i + 1] == '%') {
+			i++;
+			continue;
+		}
+		parameter_count++;
+	}
+	if (parameter_count != values.size()) {
+		throw InternalException("Expected %d parameters, received %d", parameter_count, values.size());
+	}
+#endif
 	return ExceptionFormatValue::Format(msg, values);
 }
 

package/src/duckdb/src/common/exception_format_value.cpp

@@ -3,6 +3,7 @@
 #include "fmt/format.h"
 #include "fmt/printf.h"
 #include "duckdb/common/types/hugeint.hpp"
+#include "duckdb/parser/keyword_helper.hpp"
 
 namespace duckdb {
 
@@ -40,6 +41,19 @@ template <>
 ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(string value) {
 	return ExceptionFormatValue(std::move(value));
 }
+
+template <>
+ExceptionFormatValue
+ExceptionFormatValue::CreateFormatValue(SQLString value) { // NOLINT: templating requires us to copy value here
+	return KeywordHelper::WriteQuoted(value.raw_string, '\'');
+}
+
+template <>
+ExceptionFormatValue
+ExceptionFormatValue::CreateFormatValue(SQLIdentifier value) { // NOLINT: templating requires us to copy value here
+	return KeywordHelper::WriteOptionallyQuoted(value.raw_string, '"');
+}
+
 template <>
 ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const char *value) {
 	return ExceptionFormatValue(string(value));

package/src/duckdb/src/common/file_system.cpp

@@ -11,6 +11,7 @@
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/main/database.hpp"
 #include "duckdb/main/extension_helper.hpp"
+#include "duckdb/common/windows_util.hpp"
 
 #include <cstdint>
 #include <cstdio>
@@ -53,6 +54,14 @@ bool PathMatched(const string &path, const string &sub_path) {
 
 #ifndef _WIN32
 
+string FileSystem::GetEnvVariable(const string &name) {
+	const char *env = getenv(name.c_str());
+	if (!env) {
+		return string();
+	}
+	return env;
+}
+
 bool FileSystem::IsPathAbsolute(const string &path) {
 	auto path_separator = FileSystem::PathSeparator();
 	return PathMatched(path, path_separator);
@@ -85,36 +94,59 @@ string FileSystem::GetWorkingDirectory() {
 	}
 	return string(buffer.get());
 }
+
+string FileSystem::NormalizeAbsolutePath(const string &path) {
+	D_ASSERT(IsPathAbsolute(path));
+	return path;
+}
+
 #else
 
-bool FileSystem::IsPathAbsolute(const string &path) {
-	// 1) A single backslash
-	auto sub_path = FileSystem::PathSeparator();
-	if (PathMatched(path, sub_path)) {
-		return true;
+string FileSystem::GetEnvVariable(const string &env) {
+	// first convert the environment variable name to the correct encoding
+	auto env_w = WindowsUtil::UTF8ToUnicode(env.c_str());
+	// use _wgetenv to get the value
+	auto res_w = _wgetenv(env_w.c_str());
+	if (!res_w) {
+		// no environment variable of this name found
+		return string();
 	}
-	// 2) check if starts with a double-backslash (i.e., \\)
-	sub_path += FileSystem::PathSeparator();
-	if (PathMatched(path, sub_path)) {
+	return WindowsUtil::UnicodeToUTF8(res_w);
+}
+
+bool FileSystem::IsPathAbsolute(const string &path) {
+	// 1) A single backslash or forward-slash
+	if (PathMatched(path, "\\") || PathMatched(path, "/")) {
 		return true;
 	}
-	// 3) A disk designator with a backslash (e.g., C:\)
+	// 2) A disk designator with a backslash (e.g., C:\ or C:/)
 	auto path_aux = path;
 	path_aux.erase(0, 1);
-	sub_path = ":" + FileSystem::PathSeparator();
-	if (PathMatched(path_aux, sub_path)) {
+	if (PathMatched(path_aux, ":\\") || PathMatched(path_aux, ":/")) {
 		return true;
 	}
 	return false;
 }
 
+string FileSystem::NormalizeAbsolutePath(const string &path) {
+	D_ASSERT(IsPathAbsolute(path));
+	auto result = StringUtil::Lower(FileSystem::ConvertSeparators(path));
+	if (PathMatched(result, "\\")) {
+		// Path starts with a single backslash or forward slash
+		// prepend drive letter
+		return GetWorkingDirectory().substr(0, 2) + result;
+	}
+	return result;
+}
+
 string FileSystem::PathSeparator() {
 	return "\\";
 }
 
 void FileSystem::SetWorkingDirectory(const string &path) {
-	if (!SetCurrentDirectory(path.c_str())) {
-		throw IOException("Could not change working directory!");
+	auto unicode_path = WindowsUtil::UTF8ToUnicode(path.c_str());
+	if (!SetCurrentDirectoryW(unicode_path.c_str())) {
+		throw IOException("Could not change working directory to \"%s\"", path);
 	}
 }
 
@@ -134,16 +166,16 @@ idx_t FileSystem::GetAvailableMemory() {
 }
 
 string FileSystem::GetWorkingDirectory() {
-	idx_t count = GetCurrentDirectory(0, nullptr);
+	idx_t count = GetCurrentDirectoryW(0, nullptr);
 	if (count == 0) {
 		throw IOException("Could not get working directory!");
 	}
-	auto buffer = make_unsafe_array<char>(count);
-	idx_t ret = GetCurrentDirectory(count, buffer.get());
+	auto buffer = make_unsafe_array<wchar_t>(count);
+	idx_t ret = GetCurrentDirectoryW(count, buffer.get());
 	if (count != ret + 1) {
 		throw IOException("Could not get working directory!");
 	}
-	return string(buffer.get(), ret);
+	return WindowsUtil::UnicodeToUTF8(buffer.get());
 }
 
 #endif
@@ -161,13 +193,7 @@ string FileSystem::ConvertSeparators(const string &path) {
 		return path;
 	}
 	// on windows-based systems we accept both
-	string result = path;
-	for (idx_t i = 0; i < result.size(); i++) {
-		if (result[i] == '/') {
-			result[i] = separator;
-		}
-	}
-	return result;
+	return StringUtil::Replace(path, "/", separator_str);
 }
 
 string FileSystem::ExtractName(const string &path) {
@@ -202,14 +228,10 @@ string FileSystem::GetHomeDirectory(optional_ptr<FileOpener> opener) {
 	}
 	// fallback to the default home directories for the specified system
 #ifdef DUCKDB_WINDOWS
-	const char *homedir = getenv("USERPROFILE");
+	return FileSystem::GetEnvVariable("USERPROFILE");
 #else
-	const char *homedir = getenv("HOME");
+	return FileSystem::GetEnvVariable("HOME");
 #endif
-	if (homedir) {
-		return homedir;
-	}
-	return string();
 }
 
 string FileSystem::GetHomeDirectory() {
@@ -305,6 +327,20 @@ void FileSystem::FileSync(FileHandle &handle) {
 	throw NotImplementedException("%s: FileSync is not implemented!", GetName());
 }
 
+bool FileSystem::HasGlob(const string &str) {
+	for (idx_t i = 0; i < str.size(); i++) {
+		switch (str[i]) {
+		case '*':
+		case '?':
+		case '[':
+			return true;
+		default:
+			break;
+		}
+	}
+	return false;
+}
+
 vector<string> FileSystem::Glob(const string &path, FileOpener *opener) {
 	throw NotImplementedException("%s: Glob is not implemented!", GetName());
 }
@@ -333,12 +369,8 @@ vector<string> FileSystem::GlobFiles(const string &pattern, ClientContext &conte
 	auto result = Glob(pattern);
 	if (result.empty()) {
 		string required_extension;
-		const string prefixes[] = {"http://", "https://", "s3://"};
-		for (auto &prefix : prefixes) {
-			if (StringUtil::StartsWith(pattern, prefix)) {
-				required_extension = "httpfs";
-				break;
-			}
+		if (FileSystem::IsRemoteFile(pattern)) {
+			required_extension = "httpfs";
 		}
 		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
 			// an extension is required to read this file but it is not loaded - try to load it
@@ -455,4 +487,14 @@ FileType FileHandle::GetType() {
 	return file_system.GetFileType(*this);
 }
 
+bool FileSystem::IsRemoteFile(const string &path) {
+	const string prefixes[] = {"http://", "https://", "s3://"};
+	for (auto &prefix : prefixes) {
+		if (StringUtil::StartsWith(path, prefix)) {
+			return true;
+		}
+	}
+	return false;
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/local_file_system.cpp

@@ -162,8 +162,9 @@ static FileType GetFileTypeInternal(int fd) { // LCOV_EXCL_START
 	}
 } // LCOV_EXCL_STOP
 
-unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock_type,
+unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
                                                  FileCompressionType compression, FileOpener *opener) {
+	auto path = FileSystem::ExpandPath(path_p, opener);
 	if (compression != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
@@ -506,8 +507,9 @@ public:
 	};
 };
 
-unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock_type,
+unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t flags, FileLockType lock_type,
                                                  FileCompressionType compression, FileOpener *opener) {
+	auto path = FileSystem::ExpandPath(path_p, opener);
 	if (compression != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
@@ -775,7 +777,7 @@ void LocalFileSystem::MoveFile(const string &source, const string &target) {
 	auto source_unicode = WindowsUtil::UTF8ToUnicode(source.c_str());
 	auto target_unicode = WindowsUtil::UTF8ToUnicode(target.c_str());
 	if (!MoveFileW(source_unicode.c_str(), target_unicode.c_str())) {
-		throw IOException("Could not move file");
+		throw IOException("Could not move file: %s", GetLastErrorAsString());
 	}
 }
 
@@ -819,19 +821,6 @@ idx_t LocalFileSystem::SeekPosition(FileHandle &handle) {
 	return GetFilePointer(handle);
 }
 
-static bool HasGlob(const string &str) {
-	for (idx_t i = 0; i < str.size(); i++) {
-		switch (str[i]) {
-		case '*':
-		case '?':
-		case '[':
-			return true;
-		default:
-			break;
-		}
-	}
-	return false;
-}
 static bool IsCrawl(const string &glob) {
 	// glob must match exactly
 	return glob == "**";

package/src/duckdb/src/common/types.cpp

@@ -349,7 +349,7 @@ string LogicalType::ToString() const {
 		auto &child_types = StructType::GetChildTypes(*this);
 		string ret = "STRUCT(";
 		for (size_t i = 0; i < child_types.size(); i++) {
-			ret += KeywordHelper::WriteOptionallyQuoted(child_types[i].first) + " " + child_types[i].second.ToString();
+			ret += StringUtil::Format("%s %s", SQLIdentifier(child_types[i].first), child_types[i].second);
 			if (i < child_types.size() - 1) {
 				ret += ", ";
 			}

package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp

@@ -77,7 +77,7 @@ static void AppendFilteredToResult(Vector &lambda_vector, list_entry_t *result_e
 		}
 
 		// found a true value
-		if (lambda_validity.RowIsValid(entry) && lambda_values[entry] > 0) {
+		if (lambda_validity.RowIsValid(entry) && lambda_values[entry]) {
 			true_sel.set_index(true_count++, i);
 			curr_list_len++;
 		}

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -1172,6 +1172,16 @@ void BufferedCSVReader::SkipEmptyLines() {
 	}
 }
 
+void UpdateMaxLineLength(ClientContext &context, idx_t line_length) {
+	if (!context.client_data->debug_set_max_line_length) {
+		return;
+	}
+	if (line_length < context.client_data->debug_max_line_length) {
+		return;
+	}
+	context.client_data->debug_max_line_length = line_length;
+}
+
 bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message) {
 	// used for parsing algorithm
 	bool finished_chunk = false;
@@ -1239,9 +1249,7 @@ add_row : {
 		return false;
 	}
 	finished_chunk = AddRow(insert_chunk, column, error_message);
-	if (context.client_data->max_line_length < position - line_start) {
-		context.client_data->max_line_length = position - line_start;
-	}
+	UpdateMaxLineLength(context, position - line_start);
 	if (!error_message.empty()) {
 		return false;
 	}
@@ -1379,9 +1387,7 @@ final_state:
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
 		SkipEmptyLines();
-		if (context.client_data->max_line_length < position - line_start) {
-			context.client_data->max_line_length = position - line_start;
-		}
+		UpdateMaxLineLength(context, position - line_start);
 		if (!error_message.empty()) {
 			return false;
 		}

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -183,6 +183,12 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 }
 
 void BufferedCSVReaderOptions::SetWriteOption(const string &loption, const Value &value) {
+	if (loption == "new_line") {
+		// Steal this from SetBaseOption so we can write different newlines (e.g., format JSON ARRAY)
+		write_newline = ParseString(value, loption);
+		return;
+	}
+
 	if (SetBaseOption(loption, value)) {
 		return;
 	}
@@ -199,6 +205,10 @@ void BufferedCSVReaderOptions::SetWriteOption(const string &loption, const Value
 		}
 		SetDateFormat(LogicalTypeId::TIMESTAMP, format, false);
 		SetDateFormat(LogicalTypeId::TIMESTAMP_TZ, format, false);
+	} else if (loption == "prefix") {
+		prefix = ParseString(value, loption);
+	} else if (loption == "suffix") {
+		suffix = ParseString(value, loption);
 	} else {
 		throw BinderException("Unrecognized option CSV writer \"%s\"", loption);
 	}

package/src/duckdb/src/execution/operator/persistent/physical_export.cpp

@@ -51,8 +51,8 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, CopyInfo &info,
 		ss << KeywordHelper::WriteOptionallyQuoted(exported_table.schema_name) << ".";
 	}
 
-	ss << KeywordHelper::WriteOptionallyQuoted(exported_table.table_name) << " FROM '" << exported_table.file_path
-	   << "' (";
+	ss << StringUtil::Format("%s FROM %s (", SQLIdentifier(exported_table.table_name),
+	                         SQLString(exported_table.file_path));
 
 	// write the copy options
 	ss << "FORMAT '" << info.format << "'";

package/src/duckdb/src/function/pragma/pragma_queries.cpp

@@ -41,7 +41,7 @@ string PragmaShowTables(ClientContext &context, const FunctionParameters &parame
 	)
 	SELECT "name"
 	FROM db_objects
-	ORDER BY "name";)EOF", where_clause, where_clause, where_clause);
+	ORDER BY "name";)EOF", where_clause, where_clause);
 	// clang-format on
 
 	return pragma_query;
@@ -50,15 +50,17 @@ string PragmaShowTables(ClientContext &context, const FunctionParameters &parame
 string PragmaShowTablesExpanded(ClientContext &context, const FunctionParameters &parameters) {
 	return R"(
 			SELECT
+				t.database_name AS database,
+				t.schema_name AS schema,
 				t.table_name,
 				LIST(c.column_name order by c.column_index) AS column_names,
 				LIST(c.data_type order by c.column_index) AS column_types,
-				FIRST(t.temporary) AS temporary
+				FIRST(t.temporary) AS temporary,
 			FROM duckdb_tables t
 			JOIN duckdb_columns c
 			USING (table_oid)
-			GROUP BY t.table_name
-			ORDER BY t.table_name;
+			GROUP BY t.database_name, t.schema_name, t.table_name
+			ORDER BY t.database_name, t.schema_name, t.table_name;
 	)";
 }
 

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -1,15 +1,16 @@
 #include "duckdb/common/bind_helpers.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/function/scalar/string_functions.hpp"
 #include "duckdb/function/table/read_csv.hpp"
 #include "duckdb/parser/parsed_data/copy_info.hpp"
-#include "duckdb/common/multi_file_reader.hpp"
-#include "duckdb/common/types/column/column_data_collection.hpp"
+
 #include <limits>
 
 namespace duckdb {
@@ -57,6 +58,15 @@ void BaseCSVData::Finalize() {
 			SubstringDetection(options.escape, options.null_str, "ESCAPE", "NULL");
 		}
 	}
+
+	if (!options.prefix.empty() || !options.suffix.empty()) {
+		if (options.prefix.empty() || options.suffix.empty()) {
+			throw BinderException("COPY ... (FORMAT CSV) must have both PREFIX and SUFFIX, or none at all");
+		}
+		if (options.header) {
+			throw BinderException("COPY ... (FORMAT CSV)'s HEADER cannot be combined with PREFIX/SUFFIX");
+		}
+	}
 }
 
 static unique_ptr<FunctionData> WriteCSVBind(ClientContext &context, CopyInfo &info, vector<string> &names,
@@ -85,6 +95,9 @@ static unique_ptr<FunctionData> WriteCSVBind(ClientContext &context, CopyInfo &i
 		bind_data->requires_quotes[bind_data->options.delimiter[0]] = true;
 		bind_data->requires_quotes[bind_data->options.quote[0]] = true;
 	}
+	if (!bind_data->options.write_newline.empty()) {
+		bind_data->newline = bind_data->options.write_newline;
+	}
 	return std::move(bind_data);
 }
 
@@ -251,24 +264,41 @@ struct LocalWriteCSVData : public LocalFunctionData {
 	BufferedSerializer serializer;
 	//! A chunk with VARCHAR columns to cast intermediates into
 	DataChunk cast_chunk;
+	//! If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY
+	bool written_anything = false;
 };
 
 struct GlobalWriteCSVData : public GlobalFunctionData {
-	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileCompressionType compression) : fs(fs) {
+	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileCompressionType compression)
+	    : fs(fs), written_anything(false) {
 		handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
 		                     FileLockType::WRITE_LOCK, compression);
 	}
 
+	//! Write generic data, e.g., CSV header
 	void WriteData(const_data_ptr_t data, idx_t size) {
 		lock_guard<mutex> flock(lock);
 		handle->Write((void *)data, size);
 	}
 
+	//! Write rows
+	void WriteRows(const_data_ptr_t data, idx_t size, const string &newline) {
+		lock_guard<mutex> flock(lock);
+		if (written_anything) {
+			handle->Write((void *)newline.c_str(), newline.length());
+		} else {
+			written_anything = true;
+		}
+		handle->Write((void *)data, size);
+	}
+
 	FileSystem &fs;
 	//! The mutex for writing to the physical file
 	mutex lock;
 	//! The file handle to write to
 	unique_ptr<FileHandle> handle;
+	//! If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY
+	bool written_anything;
 };
 
 static unique_ptr<LocalFunctionData> WriteCSVInitializeLocal(ExecutionContext &context, FunctionData &bind_data) {
@@ -290,6 +320,10 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 	auto global_data =
 	    make_uniq<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), file_path, options.compression);
 
+	if (!options.prefix.empty()) {
+		global_data->WriteData((const_data_ptr_t)options.prefix.c_str(), options.prefix.size());
+	}
+
 	if (options.header) {
 		BufferedSerializer serializer;
 		// write the header line to the file
@@ -304,11 +338,12 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 
 		global_data->WriteData(serializer.blob.data.get(), serializer.blob.size);
 	}
+
 	return std::move(global_data);
 }
 
 static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_data, DataChunk &cast_chunk,
-                                  BufferedSerializer &writer, DataChunk &input) {
+                                  BufferedSerializer &writer, DataChunk &input, bool &written_anything) {
 	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &options = csv_data.options;
 
@@ -338,6 +373,11 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat
 	cast_chunk.Flatten();
 	// now loop over the vectors and output the values
 	for (idx_t row_idx = 0; row_idx < cast_chunk.size(); row_idx++) {
+		if (row_idx == 0 && !written_anything) {
+			written_anything = true;
+		} else {
+			writer.WriteBufferData(csv_data.newline);
+		}
 		// write values
 		for (idx_t col_idx = 0; col_idx < cast_chunk.ColumnCount(); col_idx++) {
 			if (col_idx != 0) {
@@ -357,7 +397,6 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat
 			WriteQuotedString(writer, csv_data, str_data[row_idx].GetData(), str_data[row_idx].GetSize(),
 			                  csv_data.options.force_quote[col_idx]);
 		}
-		writer.WriteBufferData(csv_data.newline);
 	}
 }
 
@@ -368,13 +407,15 @@ static void WriteCSVSink(ExecutionContext &context, FunctionData &bind_data, Glo
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
 
 	// write data into the local buffer
-	WriteCSVChunkInternal(context.client, bind_data, local_data.cast_chunk, local_data.serializer, input);
+	WriteCSVChunkInternal(context.client, bind_data, local_data.cast_chunk, local_data.serializer, input,
+	                      local_data.written_anything);
 
 	// check if we should flush what we have currently written
 	auto &writer = local_data.serializer;
 	if (writer.blob.size >= csv_data.flush_size) {
-		global_state.WriteData(writer.blob.data.get(), writer.blob.size);
+		global_state.WriteRows(writer.blob.data.get(), writer.blob.size, csv_data.newline);
 		writer.Reset();
+		local_data.written_anything = false;
 	}
 }
 
@@ -385,10 +426,11 @@ static void WriteCSVCombine(ExecutionContext &context, FunctionData &bind_data,
                             LocalFunctionData &lstate) {
 	auto &local_data = lstate.Cast<LocalWriteCSVData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
+	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &writer = local_data.serializer;
 	// flush the local writer
-	if (writer.blob.size > 0) {
-		global_state.WriteData(writer.blob.data.get(), writer.blob.size);
+	if (local_data.written_anything) {
+		global_state.WriteRows(writer.blob.data.get(), writer.blob.size, csv_data.newline);
 		writer.Reset();
 	}
 }
@@ -398,6 +440,16 @@ static void WriteCSVCombine(ExecutionContext &context, FunctionData &bind_data,
 //===--------------------------------------------------------------------===//
 void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate) {
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
+	auto &csv_data = bind_data.Cast<WriteCSVData>();
+	auto &options = csv_data.options;
+
+	BufferedSerializer serializer;
+	if (!options.suffix.empty()) {
+		serializer.WriteBufferData(options.suffix);
+	} else if (global_state.written_anything) {
+		serializer.WriteBufferData(csv_data.newline);
+	}
+	global_state.WriteData(serializer.blob.data.get(), serializer.blob.size);
 
 	global_state.handle->Close();
 	global_state.handle.reset();
@@ -434,10 +486,11 @@ unique_ptr<PreparedBatchData> WriteCSVPrepareBatch(ClientContext &context, Funct
 	DataChunk cast_chunk;
 	cast_chunk.Initialize(Allocator::Get(context), types);
 
-	auto batch = make_uniq<WriteCSVBatchData>();
 	// write CSV chunks to the batch data
+	bool written_anything = false;
+	auto batch = make_uniq<WriteCSVBatchData>();
 	for (auto &chunk : collection->Chunks()) {
-		WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->serializer, chunk);
+		WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->serializer, chunk, written_anything);
 	}
 	return std::move(batch);
 }
@@ -449,8 +502,9 @@ void WriteCSVFlushBatch(ClientContext &context, FunctionData &bind_data, GlobalF
                         PreparedBatchData &batch) {
 	auto &csv_batch = batch.Cast<WriteCSVBatchData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
+	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &writer = csv_batch.serializer;
-	global_state.WriteData(writer.blob.data.get(), writer.blob.size);
+	global_state.WriteRows(writer.blob.data.get(), writer.blob.size, csv_data.newline);
 	writer.Reset();
 }