duckdb 0.7.2-dev3441.0 → 0.7.2-dev3546.0

package/src/duckdb/src/function/scalar/system/aggregate_export.cpp

@@ -36,12 +36,12 @@ struct ExportAggregateBindData : public FunctionData {
 struct CombineState : public FunctionLocalState {
 	idx_t state_size;
 
-	unique_ptr<data_t[]> state_buffer0, state_buffer1;
+	unsafe_array_ptr<data_t> state_buffer0, state_buffer1;
 	Vector state_vector0, state_vector1;
 
 	explicit CombineState(idx_t state_size_p)
-	    : state_size(state_size_p), state_buffer0(unique_ptr<data_t[]>(new data_t[state_size_p])),
-	      state_buffer1(unique_ptr<data_t[]>(new data_t[state_size_p])),
+	    : state_size(state_size_p), state_buffer0(make_unsafe_array<data_t>(state_size_p)),
+	      state_buffer1(make_unsafe_array<data_t>(state_size_p)),
 	      state_vector0(Value::POINTER((uintptr_t)state_buffer0.get())),
 	      state_vector1(Value::POINTER((uintptr_t)state_buffer1.get())) {
 	}
@@ -55,12 +55,12 @@ static unique_ptr<FunctionLocalState> InitCombineState(ExpressionState &state, c
 
 struct FinalizeState : public FunctionLocalState {
 	idx_t state_size;
-	unique_ptr<data_t[]> state_buffer;
+	unsafe_array_ptr<data_t> state_buffer;
 	Vector addresses;
 
 	explicit FinalizeState(idx_t state_size_p)
 	    : state_size(state_size_p),
-	      state_buffer(unique_ptr<data_t[]>(new data_t[STANDARD_VECTOR_SIZE * AlignValue(state_size_p)])),
+	      state_buffer(make_unsafe_array<data_t>(STANDARD_VECTOR_SIZE * AlignValue(state_size_p))),
 	      addresses(LogicalType::POINTER) {
 	}
 };

package/src/duckdb/src/function/scalar_macro_function.cpp

@@ -49,4 +49,14 @@ string ScalarMacroFunction::ToSQL(const string &schema, const string &name) cons
 	return MacroFunction::ToSQL(schema, name) + StringUtil::Format("(%s);", expression_copy->ToString());
 }
 
+void ScalarMacroFunction::SerializeInternal(FieldWriter &writer) const {
+	writer.WriteSerializable(*expression);
+}
+
+unique_ptr<MacroFunction> ScalarMacroFunction::Deserialize(FieldReader &reader) {
+	auto result = make_uniq<ScalarMacroFunction>();
+	result->expression = reader.ReadRequiredSerializable<ParsedExpression>();
+	return std::move(result);
+}
+
 } // namespace duckdb

package/src/duckdb/src/function/table/copy_csv.cpp

@@ -78,7 +78,7 @@ static unique_ptr<FunctionData> WriteCSVBind(ClientContext &context, CopyInfo &i
 	bind_data->is_simple = bind_data->options.delimiter.size() == 1 && bind_data->options.escape.size() == 1 &&
 	                       bind_data->options.quote.size() == 1;
 	if (bind_data->is_simple) {
-		bind_data->requires_quotes = unique_ptr<bool[]>(new bool[256]);
+		bind_data->requires_quotes = make_unsafe_array<bool>(256);
 		memset(bind_data->requires_quotes.get(), 0, sizeof(bool) * 256);
 		bind_data->requires_quotes['\n'] = true;
 		bind_data->requires_quotes['\r'] = true;
@@ -324,15 +324,11 @@ static void WriteCSVChunkInternal(ClientContext &context, FunctionData &bind_dat
 			csv_data.options.write_date_format[LogicalTypeId::DATE].ConvertDateVector(
 			    input.data[col_idx], cast_chunk.data[col_idx], input.size());
 		} else if (options.has_format[LogicalTypeId::TIMESTAMP] &&
-		           csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP) {
+		           (csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP ||
+		            csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP_TZ)) {
 			// use the timestamp format to cast the chunk
 			csv_data.options.write_date_format[LogicalTypeId::TIMESTAMP].ConvertTimestampVector(
 			    input.data[col_idx], cast_chunk.data[col_idx], input.size());
-		} else if (options.has_format[LogicalTypeId::TIMESTAMP_TZ] &&
-		           csv_data.sql_types[col_idx].id() == LogicalTypeId::TIMESTAMP_TZ) {
-			// use the timestamp format to cast the chunk
-			csv_data.options.write_date_format[LogicalTypeId::TIMESTAMP_TZ].ConvertTimestampVector(
-			    input.data[col_idx], cast_chunk.data[col_idx], input.size());
 		} else {
 			// non varchar column, perform the cast
 			VectorOperations::Cast(context, input.data[col_idx], cast_chunk.data[col_idx], input.size());

package/src/duckdb/src/function/table/read_csv.cpp

@@ -4,7 +4,6 @@
 #include "duckdb/main/database.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/common/enum_util.hpp"
-#include "duckdb/common/hive_partitioning.hpp"
 #include "duckdb/common/union_by_name.hpp"
 #include "duckdb/main/config.hpp"
 #include "duckdb/parser/expression/constant_expression.hpp"
@@ -15,7 +14,6 @@
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/execution/operator/persistent/csv_line_info.hpp"
-
 #include <limits>
 
 namespace duckdb {
@@ -23,11 +21,8 @@ namespace duckdb {
 unique_ptr<CSVFileHandle> ReadCSV::OpenCSV(const string &file_path, FileCompressionType compression,
                                            ClientContext &context) {
 	auto &fs = FileSystem::GetFileSystem(context);
-	auto file_handle = fs.OpenFile(file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
-	if (file_handle->CanSeek()) {
-		file_handle->Reset();
-	}
-	return make_uniq<CSVFileHandle>(std::move(file_handle), false);
+	auto &allocator = BufferAllocator::Get(context);
+	return CSVFileHandle::OpenFile(fs, allocator, file_path, compression, false);
 }
 
 void ReadCSVData::FinalizeRead(ClientContext &context) {
@@ -238,14 +233,6 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 	} else {
 		result->reader_bind = MultiFileReader::BindOptions(options.file_options, result->files, return_types, names);
 	}
-	auto &fs = FileSystem::GetFileSystem(context);
-	for (auto &file : result->files) {
-		if (fs.IsPipe(file)) {
-			result->is_pipe = true;
-			result->single_threaded = true;
-			break;
-		}
-	}
 	result->return_types = return_types;
 	result->return_names = names;
 	result->FinalizeRead(context);
@@ -265,7 +252,7 @@ static unique_ptr<FunctionData> ReadCSVAutoBind(ClientContext &context, TableFun
 struct ParallelCSVGlobalState : public GlobalTableFunctionState {
 public:
 	ParallelCSVGlobalState(ClientContext &context, unique_ptr<CSVFileHandle> file_handle_p,
-	                       vector<string> &files_path_p, idx_t system_threads_p, idx_t buffer_size_p,
+	                       const vector<string> &files_path_p, idx_t system_threads_p, idx_t buffer_size_p,
 	                       idx_t rows_to_skip, bool force_parallelism_p, vector<column_t> column_ids_p, bool has_header)
 	    : file_handle(std::move(file_handle_p)), system_threads(system_threads_p), buffer_size(buffer_size_p),
 	      force_parallelism(force_parallelism_p), column_ids(std::move(column_ids_p)),
@@ -278,6 +265,7 @@ public:
 		}
 		file_size = file_handle->FileSize();
 		first_file_size = file_size;
+		on_disk_file = file_handle->OnDiskFile();
 		bytes_read = 0;
 		if (buffer_size < file_size || file_size == 0) {
 			bytes_per_local_state = buffer_size / ParallelCSVGlobalState::MaxThreads();
@@ -335,7 +323,7 @@ public:
 
 	bool Finished();
 
-	double GetProgress(ReadCSVData &bind_data) const {
+	double GetProgress(const ReadCSVData &bind_data) const {
 		idx_t total_files = bind_data.files.size();
 
 		// get the progress WITHIN the current file
@@ -369,6 +357,8 @@ private:
 	idx_t bytes_per_local_state;
 	//! Size of first file
 	idx_t first_file_size;
+	//! Whether or not this is an on-disk file
+	bool on_disk_file = true;
 	//! Basically max number of threads in DuckDB
 	idx_t system_threads;
 	//! Size of the buffers
@@ -402,7 +392,7 @@ private:
 };
 
 idx_t ParallelCSVGlobalState::MaxThreads() const {
-	if (force_parallelism) {
+	if (force_parallelism || !on_disk_file) {
 		return system_threads;
 	}
 	idx_t one_mb = 1000000; // We initialize max one thread per Mb
@@ -628,7 +618,7 @@ idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t
 
 static unique_ptr<GlobalTableFunctionState> ParallelCSVInitGlobal(ClientContext &context,
                                                                   TableFunctionInitInput &input) {
-	auto &bind_data = (ReadCSVData &)*input.bind_data;
+	auto &bind_data = input.bind_data->CastNoConst<ReadCSVData>();
 	if (bind_data.files.empty()) {
 		// This can happen when a filename based filter pushdown has eliminated all possible files for this scan.
 		return make_uniq<ParallelCSVGlobalState>();
@@ -636,7 +626,15 @@ static unique_ptr<GlobalTableFunctionState> ParallelCSVInitGlobal(ClientContext
 	unique_ptr<CSVFileHandle> file_handle;
 
 	bind_data.options.file_path = bind_data.files[0];
-	file_handle = ReadCSV::OpenCSV(bind_data.options.file_path, bind_data.options.compression, context);
+
+	if (bind_data.initial_reader) {
+		file_handle = std::move(bind_data.initial_reader->file_handle);
+		file_handle->Reset();
+		file_handle->DisableReset();
+		bind_data.initial_reader.reset();
+	} else {
+		file_handle = ReadCSV::OpenCSV(bind_data.options.file_path, bind_data.options.compression, context);
+	}
 	return make_uniq<ParallelCSVGlobalState>(
 	    context, std::move(file_handle), bind_data.files, context.db->NumberOfThreads(), bind_data.options.buffer_size,
 	    bind_data.options.skip_rows, ClientConfig::GetConfig(context).verify_parallelism, input.column_ids,
@@ -738,7 +736,7 @@ struct SingleThreadedCSVState : public GlobalTableFunctionState {
 		return total_files;
 	}
 
-	double GetProgress(ReadCSVData &bind_data) const {
+	double GetProgress(const ReadCSVData &bind_data) const {
 		D_ASSERT(total_files == bind_data.files.size());
 		D_ASSERT(progress_in_files <= total_files * 100);
 		return (double(progress_in_files) / double(total_files));
@@ -746,6 +744,16 @@ struct SingleThreadedCSVState : public GlobalTableFunctionState {
 
 	unique_ptr<BufferedCSVReader> GetCSVReader(ClientContext &context, ReadCSVData &bind_data, idx_t &file_index,
 	                                           idx_t &total_size) {
+		auto reader = GetCSVReaderInternal(context, bind_data, file_index, total_size);
+		if (reader) {
+			reader->file_handle->DisableReset();
+		}
+		return reader;
+	}
+
+private:
+	unique_ptr<BufferedCSVReader> GetCSVReaderInternal(ClientContext &context, ReadCSVData &bind_data,
+	                                                   idx_t &file_index, idx_t &total_size) {
 		BufferedCSVReaderOptions options;
 		{
 			lock_guard<mutex> l(csv_lock);
@@ -799,14 +807,14 @@ public:
 
 static unique_ptr<GlobalTableFunctionState> SingleThreadedCSVInit(ClientContext &context,
                                                                   TableFunctionInitInput &input) {
-	auto &bind_data = (ReadCSVData &)*input.bind_data;
+	auto &bind_data = input.bind_data->CastNoConst<ReadCSVData>();
 	auto result = make_uniq<SingleThreadedCSVState>(bind_data.files.size());
 	if (bind_data.files.empty()) {
 		// This can happen when a filename based filter pushdown has eliminated all possible files for this scan.
 		return std::move(result);
 	} else {
 		bind_data.options.file_path = bind_data.files[0];
-		if (bind_data.initial_reader && bind_data.is_pipe) {
+		if (bind_data.initial_reader) {
 			// If this is a pipe and an initial reader already exists due to read_csv_auto
 			// We must re-use it, since we can't restart the reader due for it being a pipe.
 			result->initial_reader = std::move(bind_data.initial_reader);
@@ -904,7 +912,7 @@ static void SingleThreadedCSVFunction(ClientContext &context, TableFunctionInput
 // Read CSV Functions
 //===--------------------------------------------------------------------===//
 static unique_ptr<GlobalTableFunctionState> ReadCSVInitGlobal(ClientContext &context, TableFunctionInitInput &input) {
-	auto &bind_data = (ReadCSVData &)*input.bind_data;
+	auto &bind_data = input.bind_data->Cast<ReadCSVData>();
 	if (bind_data.single_threaded) {
 		return SingleThreadedCSVInit(context, input);
 	} else {
@@ -914,7 +922,7 @@ static unique_ptr<GlobalTableFunctionState> ReadCSVInitGlobal(ClientContext &con
 
 unique_ptr<LocalTableFunctionState> ReadCSVInitLocal(ExecutionContext &context, TableFunctionInitInput &input,
                                                      GlobalTableFunctionState *global_state_p) {
-	auto &csv_data = (ReadCSVData &)*input.bind_data;
+	auto &csv_data = input.bind_data->Cast<ReadCSVData>();
 	if (csv_data.single_threaded) {
 		return SingleThreadedReadCSVInitLocal(context, input, global_state_p);
 	} else {
@@ -923,7 +931,7 @@ unique_ptr<LocalTableFunctionState> ReadCSVInitLocal(ExecutionContext &context,
 }
 
 static void ReadCSVFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
-	auto &bind_data = (ReadCSVData &)*data_p.bind_data;
+	auto &bind_data = data_p.bind_data->Cast<ReadCSVData>();
 	if (bind_data.single_threaded) {
 		SingleThreadedCSVFunction(context, data_p, output);
 	} else {
@@ -933,7 +941,7 @@ static void ReadCSVFunction(ClientContext &context, TableFunctionInput &data_p,
 
 static idx_t CSVReaderGetBatchIndex(ClientContext &context, const FunctionData *bind_data_p,
                                     LocalTableFunctionState *local_state, GlobalTableFunctionState *global_state) {
-	auto &bind_data = (ReadCSVData &)*bind_data_p;
+	auto &bind_data = bind_data_p->Cast<ReadCSVData>();
 	if (bind_data.single_threaded) {
 		auto &data = local_state->Cast<SingleThreadedCSVLocalState>();
 		return data.file_index;
@@ -980,28 +988,28 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
 
 double CSVReaderProgress(ClientContext &context, const FunctionData *bind_data_p,
                          const GlobalTableFunctionState *global_state) {
-	auto &bind_data = (ReadCSVData &)*bind_data_p;
+	auto &bind_data = bind_data_p->Cast<ReadCSVData>();
 	if (bind_data.single_threaded) {
-		auto &data = (SingleThreadedCSVState &)*global_state;
+		auto &data = global_state->Cast<SingleThreadedCSVState>();
 		return data.GetProgress(bind_data);
 	} else {
-		auto &data = (const ParallelCSVGlobalState &)*global_state;
+		auto &data = global_state->Cast<ParallelCSVGlobalState>();
 		return data.GetProgress(bind_data);
 	}
 }
 
 void CSVComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionData *bind_data_p,
                               vector<unique_ptr<Expression>> &filters) {
-	auto data = (ReadCSVData *)bind_data_p;
+	auto &data = bind_data_p->Cast<ReadCSVData>();
 	auto reset_reader =
-	    MultiFileReader::ComplexFilterPushdown(context, data->files, data->options.file_options, get, filters);
+	    MultiFileReader::ComplexFilterPushdown(context, data.files, data.options.file_options, get, filters);
 	if (reset_reader) {
-		MultiFileReader::PruneReaders(*data);
+		MultiFileReader::PruneReaders(data);
 	}
 }
 
 unique_ptr<NodeStatistics> CSVReaderCardinality(ClientContext &context, const FunctionData *bind_data_p) {
-	auto &bind_data = (ReadCSVData &)*bind_data_p;
+	auto &bind_data = bind_data_p->Cast<ReadCSVData>();
 	idx_t per_file_cardinality = 0;
 	if (bind_data.initial_reader && bind_data.initial_reader->file_handle) {
 		auto estimated_row_width = (bind_data.csv_types.size() * 5);
@@ -1047,6 +1055,12 @@ void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
 	writer.WriteSerializable(file_options);
 	// write options
 	writer.WriteListNoReference<bool>(force_quote);
+	// FIXME: serialize date_format / has_format
+	vector<string> csv_formats;
+	for (auto &format : date_format) {
+		csv_formats.push_back(format.second.format_specifier);
+	}
+	writer.WriteList<string>(csv_formats);
 }
 
 void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
@@ -1083,10 +1097,21 @@ void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
 	file_options = reader.ReadRequiredSerializable<MultiFileReaderOptions, MultiFileReaderOptions>();
 	// write options
 	force_quote = reader.ReadRequiredList<bool>();
+	auto formats = reader.ReadRequiredList<string>();
+	vector<LogicalTypeId> format_types {LogicalTypeId::DATE, LogicalTypeId::TIMESTAMP};
+	for (idx_t f_idx = 0; f_idx < formats.size(); f_idx++) {
+		auto &format = formats[f_idx];
+		auto &type = format_types[f_idx];
+		if (format.empty()) {
+			continue;
+		}
+		has_format[type] = true;
+		StrTimeFormat::ParseFormatSpecifier(format, date_format[type]);
+	}
 }
 
 static void CSVReaderSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {
-	auto &bind_data = (ReadCSVData &)*bind_data_p;
+	auto &bind_data = bind_data_p->Cast<ReadCSVData>();
 	writer.WriteList<string>(bind_data.files);
 	writer.WriteRegularSerializableList<LogicalType>(bind_data.csv_types);
 	writer.WriteList<string>(bind_data.csv_names);

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev3441"
+#define DUCKDB_VERSION "0.7.2-dev3546"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e97702367a"
+#define DUCKDB_SOURCE_ID "bd8adef6d1"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/function/table_macro_function.cpp

@@ -31,4 +31,14 @@ string TableMacroFunction::ToSQL(const string &schema, const string &name) const
 	return MacroFunction::ToSQL(schema, name) + StringUtil::Format("TABLE (%s);", query_node->ToString());
 }
 
+void TableMacroFunction::SerializeInternal(FieldWriter &writer) const {
+	writer.WriteSerializable(*query_node);
+}
+
+unique_ptr<MacroFunction> TableMacroFunction::Deserialize(FieldReader &reader) {
+	auto result = make_uniq<TableMacroFunction>();
+	result->query_node = reader.ReadRequiredSerializable<QueryNode>();
+	return std::move(result);
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/macro_catalog_entry.hpp

@@ -24,8 +24,10 @@ public:
 	unique_ptr<MacroFunction> function;
 
 public:
+	virtual unique_ptr<CreateMacroInfo> GetInfoForSerialization() const;
 	//! Serialize the meta information
-	virtual void Serialize(Serializer &serializer) const = 0;
+	virtual void Serialize(Serializer &serializer) const;
+	static unique_ptr<CreateMacroInfo> Deserialize(Deserializer &main_source, ClientContext &context);
 
 	string ToSQL() const override {
 		return function->ToSQL(schema.name, name);

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp

@@ -23,11 +23,5 @@ public:
 
 public:
 	ScalarMacroCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateMacroInfo &info);
-
-public:
-	//! Serialize the meta information of the ScalarMacroCatalogEntry
-	void Serialize(Serializer &serializer) const override;
-	//! Deserializes to a CreateMacroInfo
-	static unique_ptr<CreateMacroInfo> Deserialize(Deserializer &source, ClientContext &context);
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_macro_catalog_entry.hpp

@@ -22,12 +22,6 @@ public:
 
 public:
 	TableMacroCatalogEntry(Catalog &catalog, SchemaCatalogEntry &schema, CreateMacroInfo &info);
-
-public:
-	//! Serialize the meta information of the ScalarMacroCatalogEntry
-	void Serialize(Serializer &serializer) const override;
-	//! Deserializes to a CreateMacroInfo
-	static unique_ptr<CreateMacroInfo> Deserialize(Deserializer &source, ClientContext &context);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/allocator.hpp

@@ -54,6 +54,9 @@ public:
 	idx_t GetSize() const {
 		return allocated_size;
 	}
+	bool IsSet() {
+		return pointer;
+	}
 	void Reset();
 
 private:

package/src/duckdb/src/include/duckdb/common/compressed_file_system.hpp

@@ -18,8 +18,8 @@ struct StreamData {
 	// various buffers & pointers
 	bool write = false;
 	bool refresh = false;
-	unique_ptr<data_t[]> in_buff;
-	unique_ptr<data_t[]> out_buff;
+	unsafe_array_ptr<data_t> in_buff;
+	unsafe_array_ptr<data_t> out_buff;
 	data_ptr_t out_buff_start = nullptr;
 	data_ptr_t out_buff_end = nullptr;
 	data_ptr_t in_buff_start = nullptr;
@@ -59,7 +59,7 @@ public:
 class CompressedFile : public FileHandle {
 public:
 	DUCKDB_API CompressedFile(CompressedFileSystem &fs, unique_ptr<FileHandle> child_handle_p, const string &path);
-	DUCKDB_API virtual ~CompressedFile() override;
+	DUCKDB_API ~CompressedFile() override;
 
 	CompressedFileSystem &compressed_fs;
 	unique_ptr<FileHandle> child_handle;

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -15,17 +15,14 @@
 #include "duckdb/common/typedefs.hpp"
 
 namespace duckdb {
+class Serializer;
+class Deserializer;
 
 //! inline std directives that we use frequently
 #ifndef DUCKDB_DEBUG_MOVE
 using std::move;
 #endif
 
-// template <class _Tp, class _Dp = std::default_delete<_Tp>>
-// class unique_ptr;
-
-// using data_ptr = unique_ptr<char[]>;
-
 // NOTE: there is a copy of this in the Postgres' parser grammar (gram.y)
 #define DEFAULT_SCHEMA  "main"
 #define INVALID_SCHEMA  ""

package/src/duckdb/src/include/duckdb/common/field_writer.hpp

@@ -263,15 +263,15 @@ public:
 		return ReadRequiredGenericList<T, idx_t, IndexReadOperation>();
 	}
 
-	template <class T>
-	set<T> ReadRequiredSet() {
+	template <class T, class CONTAINER_TYPE = set<T>>
+	CONTAINER_TYPE ReadRequiredSet() {
 		if (field_count >= max_field_count) {
 			// field is not there, throw an exception
 			throw SerializationException("Attempting to read a required field, but field is missing");
 		}
 		AddField();
 		auto result_count = source.Read<uint32_t>();
-		set<T> result;
+		CONTAINER_TYPE result;
 		for (idx_t i = 0; i < result_count; i++) {
 			result.insert(source.Read<T>());
 		}

package/src/duckdb/src/include/duckdb/common/helper.hpp

@@ -37,10 +37,10 @@ namespace duckdb {
 #define DUCKDB_EXPLICIT_FALLTHROUGH
 #endif
 
-template<class _Tp>
+template<class _Tp, bool SAFE = true>
 struct __unique_if
 {
-    typedef unique_ptr<_Tp> __unique_single;
+    typedef unique_ptr<_Tp, SAFE> __unique_single;
 };
 
 template<class _Tp>
@@ -57,19 +57,32 @@ struct __unique_if<_Tp[_Np]>
 
 template<class _Tp, class... _Args>
 inline 
-typename __unique_if<_Tp>::__unique_single
+typename __unique_if<_Tp, true>::__unique_single
 make_uniq(_Args&&... __args)
 {
-    return unique_ptr<_Tp>(new _Tp(std::forward<_Args>(__args)...));
+    return unique_ptr<_Tp, true>(new _Tp(std::forward<_Args>(__args)...));
 }
 
-template<class _Tp>
+template<class _Tp, class... _Args>
 inline 
-typename __unique_if<_Tp>::__unique_array_unknown_bound
-make_uniq(size_t __n)
+typename __unique_if<_Tp, false>::__unique_single
+make_unsafe_uniq(_Args&&... __args)
+{
+    return unique_ptr<_Tp, false>(new _Tp(std::forward<_Args>(__args)...));
+}
+
+template<class _Tp>
+inline unique_ptr<_Tp[], true>
+make_array(size_t __n)
+{
+    return unique_ptr<_Tp[], true>(new _Tp[__n]());
+}
+
+template<class _Tp>
+inline unique_ptr<_Tp[], false>
+make_unsafe_array(size_t __n)
 {
-    typedef typename std::remove_extent<_Tp>::type _Up;
-    return unique_ptr<_Tp>(new _Up[__n]());
+    return unique_ptr<_Tp[], false>(new _Tp[__n]());
 }
 
 template<class _Tp, class... _Args>

package/src/duckdb/src/include/duckdb/common/memory_safety.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+namespace duckdb {
+
+template <bool ENABLED>
+struct MemorySafety {
+#ifdef DEBUG
+	// In DEBUG mode safety is always on
+	static constexpr bool enabled = true;
+#else
+	static constexpr bool enabled = ENABLED;
+#endif
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/optional_ptr.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp

@@ -18,7 +18,7 @@ public:
 	                   FileLockType lock_type = FileLockType::READ_LOCK, optional_ptr<FileOpener> opener = nullptr);
 
 	FileSystem &fs;
-	unique_ptr<data_t[]> data;
+	unsafe_array_ptr<data_t> data;
 	idx_t offset;
 	idx_t read_data;
 	unique_ptr<FileHandle> handle;

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp

@@ -25,7 +25,7 @@ public:
 
 	FileSystem &fs;
 	string path;
-	unique_ptr<data_t[]> data;
+	unsafe_array_ptr<data_t> data;
 	idx_t offset;
 	idx_t total_written;
 	unique_ptr<FileHandle> handle;

package/src/duckdb/src/include/duckdb/common/serializer/buffered_serializer.hpp

@@ -9,13 +9,14 @@
 #pragma once
 
 #include "duckdb/common/serializer.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 
 namespace duckdb {
 
 #define SERIALIZER_DEFAULT_SIZE 1024
 
 struct BinaryData {
-	unique_ptr<data_t[]> data;
+	unsafe_array_ptr<data_t> data;
 	idx_t size;
 };
 
@@ -25,7 +26,7 @@ public:
 	//! writing past the initial threshold
 	DUCKDB_API explicit BufferedSerializer(idx_t maximum_size = SERIALIZER_DEFAULT_SIZE);
 	//! Serializes to a provided (owned) data pointer
-	BufferedSerializer(unique_ptr<data_t[]> data, idx_t size);
+	BufferedSerializer(unsafe_array_ptr<data_t> data, idx_t size);
 	BufferedSerializer(data_ptr_t data, idx_t size);
 
 	idx_t maximum_size;

package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp

@@ -65,10 +65,9 @@ struct is_unordered_map<typename std::unordered_map<Args...>> : std::true_type {
 template <typename T>
 struct is_unique_ptr : std::false_type {};
 
-template <typename T, typename D>
-struct is_unique_ptr<unique_ptr<T, D>> : std::true_type {
+template <typename T>
+struct is_unique_ptr<unique_ptr<T>> : std::true_type {
 	typedef T ELEMENT_TYPE;
-	typedef D DELETER_TYPE;
 };
 
 template <typename T>

package/src/duckdb/src/include/duckdb/common/sort/duckdb_pdqsort.hpp

@@ -25,6 +25,7 @@ applications, and to alter it and redistribute it freely, subject to the followi
 #include "duckdb/common/fast_mem.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 
 #include <algorithm>
 #include <cstddef>
@@ -38,6 +39,10 @@ using duckdb::idx_t;
 using duckdb::data_t;
 using duckdb::data_ptr_t;
 using duckdb::unique_ptr;
+using duckdb::array_ptr;
+using duckdb::unsafe_array_ptr;
+using duckdb::make_array;
+using duckdb::make_unsafe_array;
 using duckdb::FastMemcpy;
 using duckdb::FastMemcmp;
 
@@ -73,9 +78,9 @@ inline int log2(T n) {
 struct PDQConstants {
 	PDQConstants(idx_t entry_size, idx_t comp_offset, idx_t comp_size, data_ptr_t end)
 	    : entry_size(entry_size), comp_offset(comp_offset), comp_size(comp_size),
-	      tmp_buf_ptr(unique_ptr<data_t[]>(new data_t[entry_size])), tmp_buf(tmp_buf_ptr.get()),
-	      iter_swap_buf_ptr(unique_ptr<data_t[]>(new data_t[entry_size])), iter_swap_buf(iter_swap_buf_ptr.get()),
-	      swap_offsets_buf_ptr(unique_ptr<data_t[]>(new data_t[entry_size])),
+	      tmp_buf_ptr(make_unsafe_array<data_t>(entry_size)), tmp_buf(tmp_buf_ptr.get()),
+	      iter_swap_buf_ptr(make_unsafe_array<data_t>(entry_size)), iter_swap_buf(iter_swap_buf_ptr.get()),
+	      swap_offsets_buf_ptr(make_unsafe_array<data_t>(entry_size)),
 	      swap_offsets_buf(swap_offsets_buf_ptr.get()), end(end) {
 	}
 
@@ -83,13 +88,13 @@ struct PDQConstants {
 	const idx_t comp_offset;
 	const idx_t comp_size;
 
-	unique_ptr<data_t[]> tmp_buf_ptr;
+	unsafe_array_ptr<data_t> tmp_buf_ptr;
 	const data_ptr_t tmp_buf;
 
-	unique_ptr<data_t[]> iter_swap_buf_ptr;
+	unsafe_array_ptr<data_t> iter_swap_buf_ptr;
 	const data_ptr_t iter_swap_buf;
 
-	unique_ptr<data_t[]> swap_offsets_buf_ptr;
+	unsafe_array_ptr<data_t> swap_offsets_buf_ptr;
 	const data_ptr_t swap_offsets_buf;
 
 	const data_ptr_t end;