duckdb 0.7.2-dev3441.0 → 0.7.2-dev3515.0

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.7.2-dev3441"
+#define DUCKDB_VERSION "0.7.2-dev3515"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "e97702367a"
+#define DUCKDB_SOURCE_ID "12638bd156"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/allocator.hpp

@@ -54,6 +54,9 @@ public:
 	idx_t GetSize() const {
 		return allocated_size;
 	}
+	bool IsSet() {
+		return pointer;
+	}
 	void Reset();
 
 private:

package/src/duckdb/src/include/duckdb/common/compressed_file_system.hpp

@@ -18,8 +18,8 @@ struct StreamData {
 	// various buffers & pointers
 	bool write = false;
 	bool refresh = false;
-	unique_ptr<data_t[]> in_buff;
-	unique_ptr<data_t[]> out_buff;
+	unsafe_array_ptr<data_t> in_buff;
+	unsafe_array_ptr<data_t> out_buff;
 	data_ptr_t out_buff_start = nullptr;
 	data_ptr_t out_buff_end = nullptr;
 	data_ptr_t in_buff_start = nullptr;
@@ -59,7 +59,7 @@ public:
 class CompressedFile : public FileHandle {
 public:
 	DUCKDB_API CompressedFile(CompressedFileSystem &fs, unique_ptr<FileHandle> child_handle_p, const string &path);
-	DUCKDB_API virtual ~CompressedFile() override;
+	DUCKDB_API ~CompressedFile() override;
 
 	CompressedFileSystem &compressed_fs;
 	unique_ptr<FileHandle> child_handle;

package/src/duckdb/src/include/duckdb/common/constants.hpp

@@ -21,11 +21,6 @@ namespace duckdb {
 using std::move;
 #endif
 
-// template <class _Tp, class _Dp = std::default_delete<_Tp>>
-// class unique_ptr;
-
-// using data_ptr = unique_ptr<char[]>;
-
 // NOTE: there is a copy of this in the Postgres' parser grammar (gram.y)
 #define DEFAULT_SCHEMA  "main"
 #define INVALID_SCHEMA  ""

package/src/duckdb/src/include/duckdb/common/helper.hpp

@@ -37,10 +37,10 @@ namespace duckdb {
 #define DUCKDB_EXPLICIT_FALLTHROUGH
 #endif
 
-template<class _Tp>
+template<class _Tp, bool SAFE = true>
 struct __unique_if
 {
-    typedef unique_ptr<_Tp> __unique_single;
+    typedef unique_ptr<_Tp, SAFE> __unique_single;
 };
 
 template<class _Tp>
@@ -57,19 +57,32 @@ struct __unique_if<_Tp[_Np]>
 
 template<class _Tp, class... _Args>
 inline 
-typename __unique_if<_Tp>::__unique_single
+typename __unique_if<_Tp, true>::__unique_single
 make_uniq(_Args&&... __args)
 {
-    return unique_ptr<_Tp>(new _Tp(std::forward<_Args>(__args)...));
+    return unique_ptr<_Tp, true>(new _Tp(std::forward<_Args>(__args)...));
 }
 
-template<class _Tp>
+template<class _Tp, class... _Args>
 inline 
-typename __unique_if<_Tp>::__unique_array_unknown_bound
-make_uniq(size_t __n)
+typename __unique_if<_Tp, false>::__unique_single
+make_unsafe_uniq(_Args&&... __args)
+{
+    return unique_ptr<_Tp, false>(new _Tp(std::forward<_Args>(__args)...));
+}
+
+template<class _Tp>
+inline unique_ptr<_Tp[], true>
+make_array(size_t __n)
+{
+    return unique_ptr<_Tp[], true>(new _Tp[__n]());
+}
+
+template<class _Tp>
+inline unique_ptr<_Tp[], false>
+make_unsafe_array(size_t __n)
 {
-    typedef typename std::remove_extent<_Tp>::type _Up;
-    return unique_ptr<_Tp>(new _Up[__n]());
+    return unique_ptr<_Tp[], false>(new _Tp[__n]());
 }
 
 template<class _Tp, class... _Args>

package/src/duckdb/src/include/duckdb/common/memory_safety.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+namespace duckdb {
+
+template <bool ENABLED>
+struct MemorySafety {
+#ifdef DEBUG
+	// In DEBUG mode safety is always on
+	static constexpr bool enabled = true;
+#else
+	static constexpr bool enabled = ENABLED;
+#endif
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/optional_ptr.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 
 namespace duckdb {
 

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp

@@ -18,7 +18,7 @@ public:
 	                   FileLockType lock_type = FileLockType::READ_LOCK, optional_ptr<FileOpener> opener = nullptr);
 
 	FileSystem &fs;
-	unique_ptr<data_t[]> data;
+	unsafe_array_ptr<data_t> data;
 	idx_t offset;
 	idx_t read_data;
 	unique_ptr<FileHandle> handle;

package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp

@@ -25,7 +25,7 @@ public:
 
 	FileSystem &fs;
 	string path;
-	unique_ptr<data_t[]> data;
+	unsafe_array_ptr<data_t> data;
 	idx_t offset;
 	idx_t total_written;
 	unique_ptr<FileHandle> handle;

package/src/duckdb/src/include/duckdb/common/serializer/buffered_serializer.hpp

@@ -9,13 +9,14 @@
 #pragma once
 
 #include "duckdb/common/serializer.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 
 namespace duckdb {
 
 #define SERIALIZER_DEFAULT_SIZE 1024
 
 struct BinaryData {
-	unique_ptr<data_t[]> data;
+	unsafe_array_ptr<data_t> data;
 	idx_t size;
 };
 
@@ -25,7 +26,7 @@ public:
 	//! writing past the initial threshold
 	DUCKDB_API explicit BufferedSerializer(idx_t maximum_size = SERIALIZER_DEFAULT_SIZE);
 	//! Serializes to a provided (owned) data pointer
-	BufferedSerializer(unique_ptr<data_t[]> data, idx_t size);
+	BufferedSerializer(unsafe_array_ptr<data_t> data, idx_t size);
 	BufferedSerializer(data_ptr_t data, idx_t size);
 
 	idx_t maximum_size;

package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp

@@ -65,10 +65,9 @@ struct is_unordered_map<typename std::unordered_map<Args...>> : std::true_type {
 template <typename T>
 struct is_unique_ptr : std::false_type {};
 
-template <typename T, typename D>
-struct is_unique_ptr<unique_ptr<T, D>> : std::true_type {
+template <typename T>
+struct is_unique_ptr<unique_ptr<T>> : std::true_type {
 	typedef T ELEMENT_TYPE;
-	typedef D DELETER_TYPE;
 };
 
 template <typename T>

package/src/duckdb/src/include/duckdb/common/sort/duckdb_pdqsort.hpp

@@ -25,6 +25,7 @@ applications, and to alter it and redistribute it freely, subject to the followi
 #include "duckdb/common/fast_mem.hpp"
 #include "duckdb/common/helper.hpp"
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/unique_ptr.hpp"
 
 #include <algorithm>
 #include <cstddef>
@@ -38,6 +39,10 @@ using duckdb::idx_t;
 using duckdb::data_t;
 using duckdb::data_ptr_t;
 using duckdb::unique_ptr;
+using duckdb::array_ptr;
+using duckdb::unsafe_array_ptr;
+using duckdb::make_array;
+using duckdb::make_unsafe_array;
 using duckdb::FastMemcpy;
 using duckdb::FastMemcmp;
 
@@ -73,9 +78,9 @@ inline int log2(T n) {
 struct PDQConstants {
 	PDQConstants(idx_t entry_size, idx_t comp_offset, idx_t comp_size, data_ptr_t end)
 	    : entry_size(entry_size), comp_offset(comp_offset), comp_size(comp_size),
-	      tmp_buf_ptr(unique_ptr<data_t[]>(new data_t[entry_size])), tmp_buf(tmp_buf_ptr.get()),
-	      iter_swap_buf_ptr(unique_ptr<data_t[]>(new data_t[entry_size])), iter_swap_buf(iter_swap_buf_ptr.get()),
-	      swap_offsets_buf_ptr(unique_ptr<data_t[]>(new data_t[entry_size])),
+	      tmp_buf_ptr(make_unsafe_array<data_t>(entry_size)), tmp_buf(tmp_buf_ptr.get()),
+	      iter_swap_buf_ptr(make_unsafe_array<data_t>(entry_size)), iter_swap_buf(iter_swap_buf_ptr.get()),
+	      swap_offsets_buf_ptr(make_unsafe_array<data_t>(entry_size)),
 	      swap_offsets_buf(swap_offsets_buf_ptr.get()), end(end) {
 	}
 
@@ -83,13 +88,13 @@ struct PDQConstants {
 	const idx_t comp_offset;
 	const idx_t comp_size;
 
-	unique_ptr<data_t[]> tmp_buf_ptr;
+	unsafe_array_ptr<data_t> tmp_buf_ptr;
 	const data_ptr_t tmp_buf;
 
-	unique_ptr<data_t[]> iter_swap_buf_ptr;
+	unsafe_array_ptr<data_t> iter_swap_buf_ptr;
 	const data_ptr_t iter_swap_buf;
 
-	unique_ptr<data_t[]> swap_offsets_buf_ptr;
+	unsafe_array_ptr<data_t> swap_offsets_buf_ptr;
 	const data_ptr_t swap_offsets_buf;
 
 	const data_ptr_t end;

package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp

@@ -124,7 +124,8 @@ public:
 	//! Turn all the vectors from the chunk into flat vectors
 	DUCKDB_API void Flatten();
 
-	DUCKDB_API unique_ptr<UnifiedVectorFormat[]> ToUnifiedFormat();
+	// FIXME: this is DUCKDB_API, might need conversion back to regular unique ptr?
+	DUCKDB_API unsafe_array_ptr<UnifiedVectorFormat> ToUnifiedFormat();
 
 	DUCKDB_API void Slice(const SelectionVector &sel_vector, idx_t count);
 

package/src/duckdb/src/include/duckdb/common/types/selection_vector.hpp

@@ -18,7 +18,7 @@ class VectorBuffer;
 struct SelectionData {
 	DUCKDB_API explicit SelectionData(idx_t count);
 
-	unique_ptr<sel_t[]> owned_data;
+	unsafe_array_ptr<sel_t> owned_data;
 };
 
 struct SelectionVector {

package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp

@@ -24,7 +24,7 @@ struct TemplatedValidityData {
 public:
 	inline explicit TemplatedValidityData(idx_t count) {
 		auto entry_count = EntryCount(count);
-		owned_data = unique_ptr<V[]>(new V[entry_count]);
+		owned_data = make_unsafe_array<V>(entry_count);
 		for (idx_t entry_idx = 0; entry_idx < entry_count; entry_idx++) {
 			owned_data[entry_idx] = MAX_ENTRY;
 		}
@@ -32,13 +32,13 @@ public:
 	inline TemplatedValidityData(const V *validity_mask, idx_t count) {
 		D_ASSERT(validity_mask);
 		auto entry_count = EntryCount(count);
-		owned_data = unique_ptr<V[]>(new V[entry_count]);
+		owned_data = make_unsafe_array<V>(entry_count);
 		for (idx_t entry_idx = 0; entry_idx < entry_count; entry_idx++) {
 			owned_data[entry_idx] = validity_mask[entry_idx];
 		}
 	}
 
-	unique_ptr<V[]> owned_data;
+	unsafe_array_ptr<V> owned_data;
 
 public:
 	static inline idx_t EntryCount(idx_t count) {

package/src/duckdb/src/include/duckdb/common/types/vector_buffer.hpp

@@ -71,10 +71,10 @@ public:
 	}
 	explicit VectorBuffer(idx_t data_size) : buffer_type(VectorBufferType::STANDARD_BUFFER) {
 		if (data_size > 0) {
-			data = unique_ptr<data_t[]>(new data_t[data_size]);
+			data = make_unsafe_array<data_t>(data_size);
 		}
 	}
-	explicit VectorBuffer(unique_ptr<data_t[]> data_p)
+	explicit VectorBuffer(unsafe_array_ptr<data_t> data_p)
 	    : buffer_type(VectorBufferType::STANDARD_BUFFER), data(std::move(data_p)) {
 	}
 	virtual ~VectorBuffer() {
@@ -87,7 +87,7 @@ public:
 		return data.get();
 	}
 
-	void SetData(unique_ptr<data_t[]> new_data) {
+	void SetData(unsafe_array_ptr<data_t> new_data) {
 		data = std::move(new_data);
 	}
 
@@ -120,7 +120,7 @@ public:
 protected:
 	VectorBufferType buffer_type;
 	unique_ptr<VectorAuxiliaryData> aux_data;
-	unique_ptr<data_t[]> data;
+	unsafe_array_ptr<data_t> data;
 };
 
 //! The DictionaryBuffer holds a selection vector

package/src/duckdb/src/include/duckdb/common/unique_ptr.hpp

@@ -2,38 +2,45 @@
 
 #include "duckdb/common/exception.hpp"
 #include "duckdb/common/likely.hpp"
+#include "duckdb/common/memory_safety.hpp"
 
 #include <memory>
 #include <type_traits>
 
 namespace duckdb {
 
-namespace {
-struct __unique_ptr_utils {
-	static inline void AssertNotNull(void *ptr) {
-#ifdef DEBUG
-		if (DUCKDB_UNLIKELY(!ptr)) {
-			throw InternalException("Attempted to dereference unique_ptr that is NULL!");
+template <class _Tp, bool SAFE = true>
+class unique_ptr : public std::unique_ptr<_Tp, std::default_delete<_Tp>> {
+public:
+	using original = std::unique_ptr<_Tp, std::default_delete<_Tp>>;
+	using original::original;
+
+private:
+	static inline void AssertNotNull(const bool null) {
+#if defined(DUCKDB_DEBUG_NO_SAFETY) || defined(DUCKDB_CLANG_TIDY)
+		return;
+#else
+		if (DUCKDB_UNLIKELY(null)) {
+			throw duckdb::InternalException("Attempted to dereference unique_ptr that is NULL!");
 		}
 #endif
 	}
-};
-} // namespace
 
-template <class _Tp, class _Dp = std::default_delete<_Tp>>
-class unique_ptr : public std::unique_ptr<_Tp, _Dp> {
 public:
-	using original = std::unique_ptr<_Tp, _Dp>;
-	using original::original;
-
 	typename std::add_lvalue_reference<_Tp>::type operator*() const {
-		__unique_ptr_utils::AssertNotNull((void *)original::get());
-		return *(original::get());
+		const auto ptr = original::get();
+		if (MemorySafety<SAFE>::enabled) {
+			AssertNotNull(!ptr);
+		}
+		return *ptr;
 	}
 
 	typename original::pointer operator->() const {
-		__unique_ptr_utils::AssertNotNull((void *)original::get());
-		return original::get();
+		const auto ptr = original::get();
+		if (MemorySafety<SAFE>::enabled) {
+			AssertNotNull(!ptr);
+		}
+		return ptr;
 	}
 
 #ifdef DUCKDB_CLANG_TIDY
@@ -46,16 +53,40 @@ public:
 	}
 };
 
-template <class _Tp, class _Dp>
-class unique_ptr<_Tp[], _Dp> : public std::unique_ptr<_Tp[], _Dp> {
+template <class _Tp, bool SAFE>
+class unique_ptr<_Tp[], SAFE> : public std::unique_ptr<_Tp[], std::default_delete<_Tp[]>> {
 public:
-	using original = std::unique_ptr<_Tp[], _Dp>;
+	using original = std::unique_ptr<_Tp[], std::default_delete<_Tp[]>>;
 	using original::original;
 
+private:
+	static inline void AssertNotNull(const bool null) {
+#if defined(DUCKDB_DEBUG_NO_SAFETY) || defined(DUCKDB_CLANG_TIDY)
+		return;
+#else
+		if (DUCKDB_UNLIKELY(null)) {
+			throw duckdb::InternalException("Attempted to dereference unique_ptr that is NULL!");
+		}
+#endif
+	}
+
+public:
 	typename std::add_lvalue_reference<_Tp>::type operator[](size_t __i) const {
-		__unique_ptr_utils::AssertNotNull((void *)original::get());
-		return (original::get())[__i];
+		const auto ptr = original::get();
+		if (MemorySafety<SAFE>::enabled) {
+			AssertNotNull(!ptr);
+		}
+		return ptr[__i];
 	}
 };
 
+template <typename T>
+using array_ptr = unique_ptr<T[], true>;
+
+template <typename T>
+using unsafe_array_ptr = unique_ptr<T[], false>;
+
+template <typename T>
+using unsafe_unique_ptr = unique_ptr<T, false>;
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/vector.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/typedefs.hpp"
 #include "duckdb/common/likely.hpp"
 #include "duckdb/common/exception.hpp"
+#include "duckdb/common/memory_safety.hpp"
 #include <vector>
 
 namespace duckdb {
@@ -25,6 +26,7 @@ public:
 	using const_reference = typename original::const_reference;
 	using reference = typename original::reference;
 
+private:
 	static inline void AssertIndexInBounds(idx_t index, idx_t size) {
 #if defined(DUCKDB_DEBUG_NO_SAFETY) || defined(DUCKDB_CLANG_TIDY)
 		return;
@@ -35,6 +37,7 @@ public:
 #endif
 	}
 
+public:
 #ifdef DUCKDB_CLANG_TIDY
 	// This is necessary to tell clang-tidy that it reinitializes the variable after a move
 	[[clang::reinitializes]]
@@ -55,7 +58,7 @@ public:
 
 	template <bool _SAFE = false>
 	inline typename original::reference get(typename original::size_type __n) {
-		if (_SAFE) {
+		if (MemorySafety<_SAFE>::enabled) {
 			AssertIndexInBounds(__n, original::size());
 		}
 		return original::operator[](__n);
@@ -63,7 +66,7 @@ public:
 
 	template <bool _SAFE = false>
 	inline typename original::const_reference get(typename original::size_type __n) const {
-		if (_SAFE) {
+		if (MemorySafety<_SAFE>::enabled) {
 			AssertIndexInBounds(__n, original::size());
 		}
 		return original::operator[](__n);

package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp

@@ -73,7 +73,7 @@ struct AggregateHTAppendState {
 	SelectionVector empty_vector;
 	SelectionVector new_groups;
 	Vector addresses;
-	unique_ptr<UnifiedVectorFormat[]> group_data;
+	unsafe_array_ptr<UnifiedVectorFormat> group_data;
 	DataChunk group_chunk;
 
 	TupleDataChunkState chunk_state;

package/src/duckdb/src/include/duckdb/execution/join_hashtable.hpp

@@ -65,12 +65,12 @@ public:
 	//! returned by the JoinHashTable::Scan function and can be used to resume a
 	//! probe.
 	struct ScanStructure {
-		unique_ptr<UnifiedVectorFormat[]> key_data;
+		unsafe_array_ptr<UnifiedVectorFormat> key_data;
 		Vector pointers;
 		idx_t count;
 		SelectionVector sel_vector;
 		// whether or not the given tuple has found a match
-		unique_ptr<bool[]> found_match;
+		unsafe_array_ptr<bool> found_match;
 		JoinHashTable &ht;
 		bool finished;
 
@@ -212,8 +212,8 @@ private:
 	//! Insert the given set of locations into the HT with the given set of hashes
 	void InsertHashes(Vector &hashes, idx_t count, data_ptr_t key_locations[], bool parallel);
 
-	idx_t PrepareKeys(DataChunk &keys, unique_ptr<UnifiedVectorFormat[]> &key_data, const SelectionVector *&current_sel,
-	                  SelectionVector &sel, bool build_side);
+	idx_t PrepareKeys(DataChunk &keys, unsafe_array_ptr<UnifiedVectorFormat> &key_data,
+	                  const SelectionVector *&current_sel, SelectionVector &sel, bool build_side);
 
 	//! Lock for combining data_collection when merging HTs
 	mutex data_lock;

package/src/duckdb/src/include/duckdb/execution/operator/join/outer_join_marker.hpp

@@ -67,7 +67,7 @@ public:
 
 private:
 	bool enabled;
-	unique_ptr<bool[]> found_match;
+	unsafe_array_ptr<bool> found_match;
 	idx_t count;
 };
 

package/src/duckdb/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp

@@ -68,7 +68,7 @@ private:
 	//! Build and probe statistics
 	PerfectHashJoinStats perfect_join_statistics;
 	//! Stores the occurences of each value in the build side
-	unique_ptr<bool[]> bitmap_build_idx;
+	unsafe_array_ptr<bool> bitmap_build_idx;
 	//! Stores the number of unique keys in the build side
 	idx_t unique_keys = 0;
 };

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_range_join.hpp

@@ -83,7 +83,7 @@ public:
 		//! The total number of rows in the RHS
 		atomic<idx_t> count;
 		//! A bool indicating for each tuple in the RHS if they found a match (only used in FULL OUTER JOIN)
-		unique_ptr<bool[]> found_match;
+		unsafe_array_ptr<bool> found_match;
 		//! Memory usage per thread
 		idx_t memory_per_thread;
 	};

package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp

@@ -61,8 +61,6 @@ public:
 
 	DataChunk parse_chunk;
 
-	std::queue<unique_ptr<DataChunk>> cached_chunks;
-
 	ParserMode mode;
 
 public:

package/src/duckdb/src/include/duckdb/execution/operator/persistent/buffered_csv_reader.hpp

@@ -60,12 +60,12 @@ public:
 	virtual ~BufferedCSVReader() {
 	}
 
-	unique_ptr<char[]> buffer;
+	unsafe_array_ptr<char> buffer;
 	idx_t buffer_size;
 	idx_t position;
 	idx_t start = 0;
 
-	vector<unique_ptr<char[]>> cached_buffers;
+	vector<unsafe_array_ptr<char>> cached_buffers;
 
 	unique_ptr<CSVFileHandle> file_handle;