duckdb 0.7.2-dev3515.0 → 0.7.2-dev3666.0

package/src/duckdb/extension/parquet/parquet-extension.cpp

@@ -724,6 +724,11 @@ unique_ptr<TableRef> ParquetScanReplacement(ClientContext &context, const string
 	vector<unique_ptr<ParsedExpression>> children;
 	children.push_back(make_uniq<ConstantExpression>(Value(table_name)));
 	table_function->function = make_uniq<FunctionExpression>("parquet_scan", std::move(children));
+
+	if (!FileSystem::HasGlob(table_name)) {
+		table_function->alias = FileSystem::ExtractBaseName(table_name);
+	}
+
 	return std::move(table_function);
 }
 

package/src/duckdb/src/catalog/catalog_entry/macro_catalog_entry.cpp

@@ -0,0 +1,42 @@
+#include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
+#include "duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp"
+#include "duckdb/common/field_writer.hpp"
+#include "duckdb/function/scalar_macro_function.hpp"
+
+namespace duckdb {
+
+MacroCatalogEntry::MacroCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateMacroInfo &info)
+    : FunctionEntry(
+          (info.function->type == MacroType::SCALAR_MACRO ? CatalogType::MACRO_ENTRY : CatalogType::TABLE_MACRO_ENTRY),
+          catalog, schema, info),
+      function(std::move(info.function)) {
+	this->temporary = info.temporary;
+	this->internal = info.internal;
+}
+
+ScalarMacroCatalogEntry::ScalarMacroCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateMacroInfo &info)
+    : MacroCatalogEntry(catalog, schema, info) {
+}
+
+TableMacroCatalogEntry::TableMacroCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateMacroInfo &info)
+    : MacroCatalogEntry(catalog, schema, info) {
+}
+
+unique_ptr<CreateMacroInfo> MacroCatalogEntry::GetInfoForSerialization() const {
+	auto info = make_uniq<CreateMacroInfo>(type);
+	info->catalog = catalog.GetName();
+	info->schema = schema.name;
+	info->name = name;
+	info->function = function->Copy();
+	return info;
+}
+void MacroCatalogEntry::Serialize(Serializer &serializer) const {
+	auto info = GetInfoForSerialization();
+	info->Serialize(serializer);
+}
+
+unique_ptr<CreateMacroInfo> MacroCatalogEntry::Deserialize(Deserializer &main_source, ClientContext &context) {
+	return unique_ptr_cast<CreateInfo, CreateMacroInfo>(CreateInfo::Deserialize(main_source));
+}
+
+} // namespace duckdb

package/src/duckdb/src/catalog/catalog_search_path.cpp

@@ -145,6 +145,11 @@ void CatalogSearchPath::Set(vector<CatalogSearchEntry> new_paths, bool is_set_sc
 			                       is_set_schema ? "schema" : "search_path", path.ToString());
 		}
 	}
+	if (is_set_schema) {
+		if (new_paths[0].catalog == TEMP_CATALOG || new_paths[0].catalog == SYSTEM_CATALOG) {
+			throw CatalogException("SET schema cannot be set to internal schema \"%s\"", new_paths[0].catalog);
+		}
+	}
 	this->set_paths = std::move(new_paths);
 	SetPaths(set_paths);
 }

package/src/duckdb/src/catalog/catalog_set.cpp

@@ -84,7 +84,7 @@ bool CatalogSet::CreateEntry(CatalogTransaction transaction, const string &name,
 			throw InternalException("Attempting to create temporary entry \"%s\" in non-temporary catalog", name);
 		}
 		if (!value->temporary && catalog.IsTemporaryCatalog() && name != DEFAULT_SCHEMA) {
-			throw InternalException("Attempting to create non-temporary entry \"%s\" in temporary catalog", name);
+			throw InvalidInputException("Cannot create non-temporary entry \"%s\" in temporary catalog", name);
 		}
 	}
 	// lock the catalog for writing

package/src/duckdb/src/common/constants.cpp

@@ -2,6 +2,7 @@
 
 #include "duckdb/common/limits.hpp"
 #include "duckdb/common/vector_size.hpp"
+#include "duckdb/common/serializer.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/common/file_system.cpp

@@ -305,6 +305,20 @@ void FileSystem::FileSync(FileHandle &handle) {
 	throw NotImplementedException("%s: FileSync is not implemented!", GetName());
 }
 
+bool FileSystem::HasGlob(const string &str) {
+	for (idx_t i = 0; i < str.size(); i++) {
+		switch (str[i]) {
+		case '*':
+		case '?':
+		case '[':
+			return true;
+		default:
+			break;
+		}
+	}
+	return false;
+}
+
 vector<string> FileSystem::Glob(const string &path, FileOpener *opener) {
 	throw NotImplementedException("%s: Glob is not implemented!", GetName());
 }
@@ -333,12 +347,8 @@ vector<string> FileSystem::GlobFiles(const string &pattern, ClientContext &conte
 	auto result = Glob(pattern);
 	if (result.empty()) {
 		string required_extension;
-		const string prefixes[] = {"http://", "https://", "s3://"};
-		for (auto &prefix : prefixes) {
-			if (StringUtil::StartsWith(pattern, prefix)) {
-				required_extension = "httpfs";
-				break;
-			}
+		if (FileSystem::IsRemoteFile(pattern)) {
+			required_extension = "httpfs";
 		}
 		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
 			// an extension is required to read this file but it is not loaded - try to load it
@@ -455,4 +465,14 @@ FileType FileHandle::GetType() {
 	return file_system.GetFileType(*this);
 }
 
+bool FileSystem::IsRemoteFile(const string &path) {
+	const string prefixes[] = {"http://", "https://", "s3://"};
+	for (auto &prefix : prefixes) {
+		if (StringUtil::StartsWith(path, prefix)) {
+			return true;
+		}
+	}
+	return false;
+}
+
 } // namespace duckdb

package/src/duckdb/src/common/local_file_system.cpp

@@ -819,19 +819,6 @@ idx_t LocalFileSystem::SeekPosition(FileHandle &handle) {
 	return GetFilePointer(handle);
 }
 
-static bool HasGlob(const string &str) {
-	for (idx_t i = 0; i < str.size(); i++) {
-		switch (str[i]) {
-		case '*':
-		case '?':
-		case '[':
-			return true;
-		default:
-			break;
-		}
-	}
-	return false;
-}
 static bool IsCrawl(const string &glob) {
 	// glob must match exactly
 	return glob == "**";

package/src/duckdb/src/common/types/vector.cpp

@@ -1868,7 +1868,7 @@ idx_t ListVector::GetListSize(const Vector &vec) {
 		return ListVector::GetListSize(child);
 	}
 	D_ASSERT(vec.auxiliary);
-	return ((VectorListBuffer &)*vec.auxiliary).size;
+	return ((VectorListBuffer &)*vec.auxiliary).GetSize();
 }
 
 idx_t ListVector::GetListCapacity(const Vector &vec) {
@@ -1877,7 +1877,7 @@ idx_t ListVector::GetListCapacity(const Vector &vec) {
 		return ListVector::GetListSize(child);
 	}
 	D_ASSERT(vec.auxiliary);
-	return ((VectorListBuffer &)*vec.auxiliary).capacity;
+	return ((VectorListBuffer &)*vec.auxiliary).GetCapacity();
 }
 
 void ListVector::ReferenceEntry(Vector &vector, Vector &other) {
@@ -1894,7 +1894,7 @@ void ListVector::SetListSize(Vector &vec, idx_t size) {
 		auto &child = DictionaryVector::Child(vec);
 		ListVector::SetListSize(child, size);
 	}
-	((VectorListBuffer &)*vec.auxiliary).size = size;
+	((VectorListBuffer &)*vec.auxiliary).SetSize(size);
 }
 
 void ListVector::Append(Vector &target, const Vector &source, idx_t source_size, idx_t source_offset) {

package/src/duckdb/src/common/types/vector_buffer.cpp

@@ -58,12 +58,12 @@ VectorStructBuffer::~VectorStructBuffer() {
 }
 
 VectorListBuffer::VectorListBuffer(unique_ptr<Vector> vector, idx_t initial_capacity)
-    : VectorBuffer(VectorBufferType::LIST_BUFFER), capacity(initial_capacity), child(std::move(vector)) {
+    : VectorBuffer(VectorBufferType::LIST_BUFFER), child(std::move(vector)), capacity(initial_capacity) {
 }
 
 VectorListBuffer::VectorListBuffer(const LogicalType &list_type, idx_t initial_capacity)
-    : VectorBuffer(VectorBufferType::LIST_BUFFER), capacity(initial_capacity),
-      child(make_uniq<Vector>(ListType::GetChildType(list_type), initial_capacity)) {
+    : VectorBuffer(VectorBufferType::LIST_BUFFER),
+      child(make_uniq<Vector>(ListType::GetChildType(list_type), initial_capacity)), capacity(initial_capacity) {
 }
 
 void VectorListBuffer::Reserve(idx_t to_reserve) {
@@ -96,6 +96,14 @@ void VectorListBuffer::PushBack(const Value &insert) {
 	child->SetValue(size++, insert);
 }
 
+void VectorListBuffer::SetCapacity(idx_t new_capacity) {
+	this->capacity = new_capacity;
+}
+
+void VectorListBuffer::SetSize(idx_t new_size) {
+	this->size = new_size;
+}
+
 VectorListBuffer::~VectorListBuffer() {
 }
 

package/src/duckdb/src/common/types/vector_cache.cpp

@@ -16,7 +16,7 @@ public:
 			owned_data = allocator.Allocate(capacity * GetTypeIdSize(internal_type));
 			// child data of the list
 			auto &child_type = ListType::GetChildType(type);
-			child_caches.push_back(make_buffer<VectorCacheBuffer>(allocator, child_type));
+			child_caches.push_back(make_buffer<VectorCacheBuffer>(allocator, child_type, capacity));
 			auto child_vector = make_uniq<Vector>(child_type, false, false);
 			auxiliary = make_shared<VectorListBuffer>(std::move(child_vector));
 			break;
@@ -24,7 +24,7 @@ public:
 		case PhysicalType::STRUCT: {
 			auto &child_types = StructType::GetChildTypes(type);
 			for (auto &child_type : child_types) {
-				child_caches.push_back(make_buffer<VectorCacheBuffer>(allocator, child_type.second));
+				child_caches.push_back(make_buffer<VectorCacheBuffer>(allocator, child_type.second, capacity));
 			}
 			auto struct_buffer = make_shared<VectorStructBuffer>(type);
 			auxiliary = std::move(struct_buffer);
@@ -48,13 +48,13 @@ public:
 			// reinitialize the VectorListBuffer
 			AssignSharedPointer(result.auxiliary, auxiliary);
 			// propagate through child
+			auto &child_cache = (VectorCacheBuffer &)*child_caches[0];
 			auto &list_buffer = (VectorListBuffer &)*result.auxiliary;
-			list_buffer.capacity = capacity;
-			list_buffer.size = 0;
+			list_buffer.SetCapacity(child_cache.capacity);
+			list_buffer.SetSize(0);
 			list_buffer.SetAuxiliaryData(nullptr);
 
 			auto &list_child = list_buffer.GetChild();
-			auto &child_cache = (VectorCacheBuffer &)*child_caches[0];
 			child_cache.ResetFromCache(list_child, child_caches[0]);
 			break;
 		}

package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp

@@ -1172,6 +1172,16 @@ void BufferedCSVReader::SkipEmptyLines() {
 	}
 }
 
+void UpdateMaxLineLength(ClientContext &context, idx_t line_length) {
+	if (!context.client_data->debug_set_max_line_length) {
+		return;
+	}
+	if (line_length < context.client_data->debug_max_line_length) {
+		return;
+	}
+	context.client_data->debug_max_line_length = line_length;
+}
+
 bool BufferedCSVReader::TryParseSimpleCSV(DataChunk &insert_chunk, string &error_message) {
 	// used for parsing algorithm
 	bool finished_chunk = false;
@@ -1239,9 +1249,7 @@ add_row : {
 		return false;
 	}
 	finished_chunk = AddRow(insert_chunk, column, error_message);
-	if (context.client_data->max_line_length < position - line_start) {
-		context.client_data->max_line_length = position - line_start;
-	}
+	UpdateMaxLineLength(context, position - line_start);
 	if (!error_message.empty()) {
 		return false;
 	}
@@ -1379,9 +1387,7 @@ final_state:
 		AddValue(string_t(buffer.get() + start, position - start - offset), column, escape_positions, has_quotes);
 		finished_chunk = AddRow(insert_chunk, column, error_message);
 		SkipEmptyLines();
-		if (context.client_data->max_line_length < position - line_start) {
-			context.client_data->max_line_length = position - line_start;
-		}
+		UpdateMaxLineLength(context, position - line_start);
 		if (!error_message.empty()) {
 			return false;
 		}

package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp

@@ -183,6 +183,12 @@ void BufferedCSVReaderOptions::SetReadOption(const string &loption, const Value
 }
 
 void BufferedCSVReaderOptions::SetWriteOption(const string &loption, const Value &value) {
+	if (loption == "new_line") {
+		// Steal this from SetBaseOption so we can write different newlines (e.g., format JSON ARRAY)
+		write_newline = ParseString(value, loption);
+		return;
+	}
+
 	if (SetBaseOption(loption, value)) {
 		return;
 	}
@@ -199,6 +205,10 @@ void BufferedCSVReaderOptions::SetWriteOption(const string &loption, const Value
 		}
 		SetDateFormat(LogicalTypeId::TIMESTAMP, format, false);
 		SetDateFormat(LogicalTypeId::TIMESTAMP_TZ, format, false);
+	} else if (loption == "prefix") {
+		prefix = ParseString(value, loption);
+	} else if (loption == "suffix") {
+		suffix = ParseString(value, loption);
 	} else {
 		throw BinderException("Unrecognized option CSV writer \"%s\"", loption);
 	}

package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp

@@ -7,9 +7,9 @@
 
 namespace duckdb {
 
-PhysicalCreateType::PhysicalCreateType(unique_ptr<CreateTypeInfo> info, idx_t estimated_cardinality)
+PhysicalCreateType::PhysicalCreateType(unique_ptr<CreateTypeInfo> info_p, idx_t estimated_cardinality)
     : PhysicalOperator(PhysicalOperatorType::CREATE_TYPE, {LogicalType::BIGINT}, estimated_cardinality),
-      info(std::move(info)) {
+      info(std::move(info_p)) {
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/function/macro_function.cpp

@@ -5,6 +5,7 @@
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/function/scalar_macro_function.hpp"
+#include "duckdb/function/table_macro_function.hpp"
 #include "duckdb/parser/expression/columnref_expression.hpp"
 #include "duckdb/parser/expression/comparison_expression.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
@@ -91,4 +92,46 @@ string MacroFunction::ToSQL(const string &schema, const string &name) const {
 	return StringUtil::Format("CREATE MACRO %s.%s(%s) AS ", schema, name, StringUtil::Join(param_strings, ", "));
 }
 
+void MacroFunction::Serialize(Serializer &main_serializer) const {
+	FieldWriter writer(main_serializer);
+	writer.WriteField(type);
+	writer.WriteSerializableList(parameters);
+	writer.WriteField<uint32_t>((uint32_t)default_parameters.size());
+	auto &serializer = writer.GetSerializer();
+	for (auto &kv : default_parameters) {
+		serializer.WriteString(kv.first);
+		kv.second->Serialize(serializer);
+	}
+	SerializeInternal(writer);
+	writer.Finalize();
+}
+
+unique_ptr<MacroFunction> MacroFunction::Deserialize(Deserializer &main_source) {
+	FieldReader reader(main_source);
+	auto type = reader.ReadRequired<MacroType>();
+	auto parameters = reader.ReadRequiredSerializableList<ParsedExpression>();
+	auto default_param_count = reader.ReadRequired<uint32_t>();
+	unordered_map<string, unique_ptr<ParsedExpression>> default_parameters;
+	auto &source = reader.GetSource();
+	for (idx_t i = 0; i < default_param_count; i++) {
+		auto name = source.Read<string>();
+		default_parameters[name] = ParsedExpression::Deserialize(source);
+	}
+	unique_ptr<MacroFunction> result;
+	switch (type) {
+	case MacroType::SCALAR_MACRO:
+		result = ScalarMacroFunction::Deserialize(reader);
+		break;
+	case MacroType::TABLE_MACRO:
+		result = TableMacroFunction::Deserialize(reader);
+		break;
+	default:
+		throw InternalException("Cannot deserialize macro type");
+	}
+	result->parameters = std::move(parameters);
+	result->default_parameters = std::move(default_parameters);
+	reader.Finalize();
+	return result;
+}
+
 } // namespace duckdb

package/src/duckdb/src/function/pragma/pragma_queries.cpp

@@ -50,15 +50,17 @@ string PragmaShowTables(ClientContext &context, const FunctionParameters &parame
 string PragmaShowTablesExpanded(ClientContext &context, const FunctionParameters &parameters) {
 	return R"(
 			SELECT
+				t.database_name AS database,
+				t.schema_name AS schema,
 				t.table_name,
 				LIST(c.column_name order by c.column_index) AS column_names,
 				LIST(c.data_type order by c.column_index) AS column_types,
-				FIRST(t.temporary) AS temporary
+				FIRST(t.temporary) AS temporary,
 			FROM duckdb_tables t
 			JOIN duckdb_columns c
 			USING (table_oid)
-			GROUP BY t.table_name
-			ORDER BY t.table_name;
+			GROUP BY t.database_name, t.schema_name, t.table_name
+			ORDER BY t.database_name, t.schema_name, t.table_name;
 	)";
 }
 

package/src/duckdb/src/function/scalar/strftime_format.cpp

@@ -417,6 +417,7 @@ string StrTimeFormat::ParseFormatSpecifier(const string &format_string, StrTimeF
 	if (format_string.empty()) {
 		return "Empty format string";
 	}
+	format.format_specifier = format_string;
 	format.specifiers.clear();
 	format.literals.clear();
 	format.numeric_width.clear();

package/src/duckdb/src/function/scalar_macro_function.cpp

@@ -49,4 +49,14 @@ string ScalarMacroFunction::ToSQL(const string &schema, const string &name) cons
 	return MacroFunction::ToSQL(schema, name) + StringUtil::Format("(%s);", expression_copy->ToString());
 }
 
+void ScalarMacroFunction::SerializeInternal(FieldWriter &writer) const {
+	writer.WriteSerializable(*expression);
+}
+
+unique_ptr<MacroFunction> ScalarMacroFunction::Deserialize(FieldReader &reader) {
+	auto result = make_uniq<ScalarMacroFunction>();
+	result->expression = reader.ReadRequiredSerializable<ParsedExpression>();
+	return std::move(result);
+}
+
 } // namespace duckdb

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -1,15 +1,16 @@
 #include "duckdb/common/bind_helpers.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/common/serializer/buffered_serializer.hpp"
 #include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types/column/column_data_collection.hpp"
 #include "duckdb/common/types/string_type.hpp"
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/function/copy_function.hpp"
 #include "duckdb/function/scalar/string_functions.hpp"
 #include "duckdb/function/table/read_csv.hpp"
 #include "duckdb/parser/parsed_data/copy_info.hpp"
-#include "duckdb/common/multi_file_reader.hpp"
-#include "duckdb/common/types/column/column_data_collection.hpp"
+
 #include <limits>
 
 namespace duckdb {
@@ -57,6 +58,15 @@ void BaseCSVData::Finalize() {
 			SubstringDetection(options.escape, options.null_str, "ESCAPE", "NULL");
 		}
 	}
+
+	if (!options.prefix.empty() || !options.suffix.empty()) {
+		if (options.prefix.empty() || options.suffix.empty()) {
+			throw BinderException("COPY ... (FORMAT CSV) must have both PREFIX and SUFFIX, or none at all");
+		}
+		if (options.header) {
+			throw BinderException("COPY ... (FORMAT CSV)'s HEADER cannot be combined with PREFIX/SUFFIX");
+		}
+	}
 }
 
 static unique_ptr<FunctionData> WriteCSVBind(ClientContext &context, CopyInfo &info, vector<string> &names,
@@ -85,6 +95,9 @@ static unique_ptr<FunctionData> WriteCSVBind(ClientContext &context, CopyInfo &i
 		bind_data->requires_quotes[bind_data->options.delimiter[0]] = true;
 		bind_data->requires_quotes[bind_data->options.quote[0]] = true;
 	}
+	if (!bind_data->options.write_newline.empty()) {
+		bind_data->newline = bind_data->options.write_newline;
+	}
 	return std::move(bind_data);
 }
 
@@ -251,24 +264,41 @@ struct LocalWriteCSVData : public LocalFunctionData {
 	BufferedSerializer serializer;
 	//! A chunk with VARCHAR columns to cast intermediates into
 	DataChunk cast_chunk;
+	//! If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY
+	bool written_anything = false;
 };
 
 struct GlobalWriteCSVData : public GlobalFunctionData {
-	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileCompressionType compression) : fs(fs) {
+	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileCompressionType compression)
+	    : fs(fs), written_anything(false) {
 		handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
 		                     FileLockType::WRITE_LOCK, compression);
 	}
 
+	//! Write generic data, e.g., CSV header
 	void WriteData(const_data_ptr_t data, idx_t size) {
 		lock_guard<mutex> flock(lock);
 		handle->Write((void *)data, size);
 	}
 
+	//! Write rows
+	void WriteRows(const_data_ptr_t data, idx_t size, const string &newline) {
+		lock_guard<mutex> flock(lock);
+		if (written_anything) {
+			handle->Write((void *)newline.c_str(), newline.length());
+		} else {
+			written_anything = true;
+		}
+		handle->Write((void *)data, size);
+	}
+
 	FileSystem &fs;
 	//! The mutex for writing to the physical file
 	mutex lock;
 	//! The file handle to write to
 	unique_ptr<FileHandle> handle;
+	//! If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY
+	bool written_anything;
 };
 
 static unique_ptr<LocalFunctionData> WriteCSVInitializeLocal(ExecutionContext &context, FunctionData &bind_data) {
@@ -290,6 +320,10 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 	auto global_data =
 	    make_uniq<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), file_path, options.compression);
 
+	if (!options.prefix.empty()) {
+		global_data->WriteData((const_data_ptr_t)options.prefix.c_str(), options.prefix.size());
+	}
+
 	if (options.header) {
 		BufferedSerializer serializer;
 		// write the header line to the file
@@ -304,11 +338,12 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 
 		global_data->WriteData(serializer.blob.data.get(), serializer.blob.size);
 	}
+
 	return std::move(global_data);
 }
 
 static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_data, DataChunk &cast_chunk,
-                                  BufferedSerializer &writer, DataChunk &input) {
+                                  BufferedSerializer &writer, DataChunk &input, bool &written_anything) {
 	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &options = csv_data.options;
 
@@ -324,15 +359,11 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat
 			csv_data.options.write_date_format[LogicalTypeId::DATE].ConvertDateVector(
 			    input.data[col_idx], cast_chunk.data[col_idx], input.size());
 		} else if (options.has_format[LogicalTypeId::TIMESTAMP] &&
-		           csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP) {
+		           (csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP ||
+		            csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP_TZ)) {
 			// use the timestamp format to cast the chunk
 			csv_data.options.write_date_format[LogicalTypeId::TIMESTAMP].ConvertTimestampVector(
 			    input.data[col_idx], cast_chunk.data[col_idx], input.size());
-		} else if (options.has_format[LogicalTypeId::TIMESTAMP_TZ] &&
-		           csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP_TZ) {
-			// use the timestamp format to cast the chunk
-			csv_data.options.write_date_format[LogicalTypeId::TIMESTAMP_TZ].ConvertTimestampVector(
-			    input.data[col_idx], cast_chunk.data[col_idx], input.size());
 		} else {
 			// non varchar column, perform the cast
 			VectorOperations::Cast(context, input.data[col_idx], cast_chunk.data[col_idx], input.size());
@@ -342,6 +373,11 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat
 	cast_chunk.Flatten();
 	// now loop over the vectors and output the values
 	for (idx_t row_idx = 0; row_idx < cast_chunk.size(); row_idx++) {
+		if (row_idx == 0 && !written_anything) {
+			written_anything = true;
+		} else {
+			writer.WriteBufferData(csv_data.newline);
+		}
 		// write values
 		for (idx_t col_idx = 0; col_idx < cast_chunk.ColumnCount(); col_idx++) {
 			if (col_idx != 0) {
@@ -361,7 +397,6 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat
 			WriteQuotedString(writer, csv_data, str_data[row_idx].GetData(), str_data[row_idx].GetSize(),
 			                  csv_data.options.force_quote[col_idx]);
 		}
-		writer.WriteBufferData(csv_data.newline);
 	}
 }
 
@@ -372,13 +407,15 @@ static void WriteCSVSink(ExecutionContext &context, FunctionData &bind_data, Glo
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
 
 	// write data into the local buffer
-	WriteCSVChunkInternal(context.client, bind_data, local_data.cast_chunk, local_data.serializer, input);
+	WriteCSVChunkInternal(context.client, bind_data, local_data.cast_chunk, local_data.serializer, input,
+	                      local_data.written_anything);
 
 	// check if we should flush what we have currently written
 	auto &writer = local_data.serializer;
 	if (writer.blob.size >= csv_data.flush_size) {
-		global_state.WriteData(writer.blob.data.get(), writer.blob.size);
+		global_state.WriteRows(writer.blob.data.get(), writer.blob.size, csv_data.newline);
 		writer.Reset();
+		local_data.written_anything = false;
 	}
 }
 
@@ -389,10 +426,11 @@ static void WriteCSVCombine(ExecutionContext &context, FunctionData &bind_data,
                             LocalFunctionData &lstate) {
 	auto &local_data = lstate.Cast<LocalWriteCSVData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
+	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &writer = local_data.serializer;
 	// flush the local writer
-	if (writer.blob.size > 0) {
-		global_state.WriteData(writer.blob.data.get(), writer.blob.size);
+	if (local_data.written_anything) {
+		global_state.WriteRows(writer.blob.data.get(), writer.blob.size, csv_data.newline);
 		writer.Reset();
 	}
 }
@@ -402,6 +440,16 @@ static void WriteCSVCombine(ExecutionContext &context, FunctionData &bind_data,
 //===--------------------------------------------------------------------===//
 void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate) {
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
+	auto &csv_data = bind_data.Cast<WriteCSVData>();
+	auto &options = csv_data.options;
+
+	BufferedSerializer serializer;
+	if (!options.suffix.empty()) {
+		serializer.WriteBufferData(options.suffix);
+	} else if (global_state.written_anything) {
+		serializer.WriteBufferData(csv_data.newline);
+	}
+	global_state.WriteData(serializer.blob.data.get(), serializer.blob.size);
 
 	global_state.handle->Close();
 	global_state.handle.reset();
@@ -438,10 +486,11 @@ unique_ptr<PreparedBatchData> WriteCSVPrepareBatch(ClientContext &context, Funct
 	DataChunk cast_chunk;
 	cast_chunk.Initialize(Allocator::Get(context), types);
 
-	auto batch = make_uniq<WriteCSVBatchData>();
 	// write CSV chunks to the batch data
+	bool written_anything = false;
+	auto batch = make_uniq<WriteCSVBatchData>();
 	for (auto &chunk : collection->Chunks()) {
-		WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->serializer, chunk);
+		WriteCSVChunkInternal(context, bind_data, cast_chunk, batch->serializer, chunk, written_anything);
 	}
 	return std::move(batch);
 }
@@ -453,8 +502,9 @@ void WriteCSVFlushBatch(ClientContext &context, FunctionData &bind_data, GlobalF
                         PreparedBatchData &batch) {
 	auto &csv_batch = batch.Cast<WriteCSVBatchData>();
 	auto &global_state = gstate.Cast<GlobalWriteCSVData>();
+	auto &csv_data = bind_data.Cast<WriteCSVData>();
 	auto &writer = csv_batch.serializer;
-	global_state.WriteData(writer.blob.data.get(), writer.blob.size);
+	global_state.WriteRows(writer.blob.data.get(), writer.blob.size, csv_data.newline);
 	writer.Reset();
 }