duckdb 1.1.2-dev4.0 → 1.1.3

package/src/duckdb/src/include/duckdb/common/extra_type_info/enum_type_info.hpp

@@ -0,0 +1,53 @@
+#pragma once
+
+#include "duckdb/common/extra_type_info.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/string_map_set.hpp"
+
+namespace duckdb {
+
+template <class T>
+struct EnumTypeInfoTemplated : public EnumTypeInfo {
+	explicit EnumTypeInfoTemplated(Vector &values_insert_order_p, idx_t size_p)
+	    : EnumTypeInfo(values_insert_order_p, size_p) {
+		D_ASSERT(values_insert_order_p.GetType().InternalType() == PhysicalType::VARCHAR);
+
+		UnifiedVectorFormat vdata;
+		values_insert_order.ToUnifiedFormat(size_p, vdata);
+
+		auto data = UnifiedVectorFormat::GetData<string_t>(vdata);
+		for (idx_t i = 0; i < size_p; i++) {
+			auto idx = vdata.sel->get_index(i);
+			if (!vdata.validity.RowIsValid(idx)) {
+				throw InternalException("Attempted to create ENUM type with NULL value");
+			}
+			if (values.count(data[idx]) > 0) {
+				throw InvalidInputException("Attempted to create ENUM type with duplicate value %s",
+				                            data[idx].GetString());
+			}
+			values[data[idx]] = UnsafeNumericCast<T>(i);
+		}
+	}
+
+	static shared_ptr<EnumTypeInfoTemplated> Deserialize(Deserializer &deserializer, uint32_t size) {
+		Vector values_insert_order(LogicalType::VARCHAR, size);
+		auto strings = FlatVector::GetData<string_t>(values_insert_order);
+
+		deserializer.ReadList(201, "values", [&](Deserializer::List &list, idx_t i) {
+			strings[i] = StringVector::AddStringOrBlob(values_insert_order, list.ReadElement<string>());
+		});
+		return make_shared_ptr<EnumTypeInfoTemplated>(values_insert_order, size);
+	}
+
+	const string_map_t<T> &GetValues() const {
+		return values;
+	}
+
+	EnumTypeInfoTemplated(const EnumTypeInfoTemplated &) = delete;
+	EnumTypeInfoTemplated &operator=(const EnumTypeInfoTemplated &) = delete;
+
+private:
+	string_map_t<T> values;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/insertion_order_preserving_map.hpp

@@ -95,13 +95,13 @@ public:
 		map.resize(nz);
 	}
 
-	void insert(const string &key, V &value) { // NOLINT: match stl API
-		map.push_back(make_pair(key, std::move(value)));
+	void insert(const string &key, V &&value) { // NOLINT: match stl API
+		map.emplace_back(key, std::move(value));
 		map_idx[key] = map.size() - 1;
 	}
 
-	void insert(const string &key, V &&value) { // NOLINT: match stl API
-		map.push_back(make_pair(key, std::move(value)));
+	void insert(const string &key, const V &value) { // NOLINT: match stl API
+		map.emplace_back(key, value);
 		map_idx[key] = map.size() - 1;
 	}
 
@@ -133,7 +133,7 @@ public:
 	V &operator[](const string &key) {
 		if (!contains(key)) {
 			auto v = V();
-			insert(key, v);
+			insert(key, std::move(v));
 		}
 		return map[map_idx[key]].second;
 	}

package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp

@@ -255,6 +255,11 @@ struct MultiFileReader {
 	static void PruneReaders(BIND_DATA &data, MultiFileList &file_list) {
 		unordered_set<string> file_set;
 
+		// Avoid materializing the file list if there's nothing to prune
+		if (!data.initial_reader && data.union_readers.empty()) {
+			return;
+		}
+
 		for (const auto &file : file_list.Files()) {
 			file_set.insert(file);
 		}

package/src/duckdb/src/include/duckdb/common/types/bit.hpp

@@ -17,85 +17,88 @@
 
 namespace duckdb {
 
+using bitstring_t = duckdb::string_t;
+
 //! The Bit class is a static class that holds helper functions for the BIT type.
 class Bit {
 public:
 	//! Returns the number of bits in the bit string
-	DUCKDB_API static idx_t BitLength(string_t bits);
+	DUCKDB_API static idx_t BitLength(bitstring_t bits);
 	//! Returns the number of set bits in the bit string
-	DUCKDB_API static idx_t BitCount(string_t bits);
+	DUCKDB_API static idx_t BitCount(bitstring_t bits);
 	//! Returns the number of bytes in the bit string
-	DUCKDB_API static idx_t OctetLength(string_t bits);
+	DUCKDB_API static idx_t OctetLength(bitstring_t bits);
 	//! Extracts the nth bit from bit string; the first (leftmost) bit is indexed 0
-	DUCKDB_API static idx_t GetBit(string_t bit_string, idx_t n);
+	DUCKDB_API static idx_t GetBit(bitstring_t bit_string, idx_t n);
 	//! Sets the nth bit in bit string to newvalue; the first (leftmost) bit is indexed 0
-	DUCKDB_API static void SetBit(string_t &bit_string, idx_t n, idx_t new_value);
+	DUCKDB_API static void SetBit(bitstring_t &bit_string, idx_t n, idx_t new_value);
 	//! Returns first starting index of the specified substring within bits, or zero if it's not present.
-	DUCKDB_API static idx_t BitPosition(string_t substring, string_t bits);
+	DUCKDB_API static idx_t BitPosition(bitstring_t substring, bitstring_t bits);
 	//! Converts bits to a string, writing the output to the designated output string.
 	//! The string needs to have space for at least GetStringSize(bits) bytes.
-	DUCKDB_API static void ToString(string_t bits, char *output);
-	DUCKDB_API static string ToString(string_t str);
+	DUCKDB_API static void ToString(bitstring_t bits, char *output);
+	DUCKDB_API static string ToString(bitstring_t bits);
 	//! Returns the bit size of a string -> bit conversion
 	DUCKDB_API static bool TryGetBitStringSize(string_t str, idx_t &result_size, string *error_message);
 	//! Convert a string to a bit. This function should ONLY be called after calling GetBitSize, since it does NOT
 	//! perform data validation.
-	DUCKDB_API static void ToBit(string_t str, string_t &output);
+	DUCKDB_API static void ToBit(string_t str, bitstring_t &output);
 
 	DUCKDB_API static string ToBit(string_t str);
 
 	//! output needs to have enough space allocated before calling this function (blob size + 1)
-	DUCKDB_API static void BlobToBit(string_t blob, string_t &output);
+	DUCKDB_API static void BlobToBit(string_t blob, bitstring_t &output);
 
 	DUCKDB_API static string BlobToBit(string_t blob);
 
 	//! output_str needs to have enough space allocated before calling this function (sizeof(T) + 1)
 	template <class T>
-	static void NumericToBit(T numeric, string_t &output_str);
+	static void NumericToBit(T numeric, bitstring_t &output_str);
 
 	template <class T>
 	static string NumericToBit(T numeric);
 
 	//! bit is expected to fit inside of output num (bit size <= sizeof(T) + 1)
 	template <class T>
-	static void BitToNumeric(string_t bit, T &output_num);
+	static void BitToNumeric(bitstring_t bit, T &output_num);
 
 	template <class T>
-	static T BitToNumeric(string_t bit);
+	static T BitToNumeric(bitstring_t bit);
 
 	//! bit is expected to fit inside of output_blob (bit size = output_blob + 1)
-	static void BitToBlob(string_t bit, string_t &output_blob);
+	static void BitToBlob(bitstring_t bit, string_t &output_blob);
 
-	static string BitToBlob(string_t bit);
+	static string BitToBlob(bitstring_t bit);
 
 	//! Creates a new bitstring of determined length
-	DUCKDB_API static void BitString(const string_t &input, const idx_t &len, string_t &result);
-	DUCKDB_API static void SetEmptyBitString(string_t &target, string_t &input);
-	DUCKDB_API static void SetEmptyBitString(string_t &target, idx_t len);
+	DUCKDB_API static void BitString(const string_t &input, idx_t len, bitstring_t &result);
+	DUCKDB_API static void ExtendBitString(const bitstring_t &input, idx_t bit_length, bitstring_t &result);
+	DUCKDB_API static void SetEmptyBitString(bitstring_t &target, string_t &input);
+	DUCKDB_API static void SetEmptyBitString(bitstring_t &target, idx_t len);
 	DUCKDB_API static idx_t ComputeBitstringLen(idx_t len);
 
-	DUCKDB_API static void RightShift(const string_t &bit_string, const idx_t &shif, string_t &result);
-	DUCKDB_API static void LeftShift(const string_t &bit_string, const idx_t &shift, string_t &result);
-	DUCKDB_API static void BitwiseAnd(const string_t &rhs, const string_t &lhs, string_t &result);
-	DUCKDB_API static void BitwiseOr(const string_t &rhs, const string_t &lhs, string_t &result);
-	DUCKDB_API static void BitwiseXor(const string_t &rhs, const string_t &lhs, string_t &result);
-	DUCKDB_API static void BitwiseNot(const string_t &rhs, string_t &result);
+	DUCKDB_API static void RightShift(const bitstring_t &bit_string, idx_t shift, bitstring_t &result);
+	DUCKDB_API static void LeftShift(const bitstring_t &bit_string, idx_t shift, bitstring_t &result);
+	DUCKDB_API static void BitwiseAnd(const bitstring_t &rhs, const bitstring_t &lhs, bitstring_t &result);
+	DUCKDB_API static void BitwiseOr(const bitstring_t &rhs, const bitstring_t &lhs, bitstring_t &result);
+	DUCKDB_API static void BitwiseXor(const bitstring_t &rhs, const bitstring_t &lhs, bitstring_t &result);
+	DUCKDB_API static void BitwiseNot(const bitstring_t &rhs, bitstring_t &result);
 
-	DUCKDB_API static void Verify(const string_t &input);
+	DUCKDB_API static void Verify(const bitstring_t &input);
 
 private:
-	static void Finalize(string_t &str);
-	static idx_t GetBitInternal(string_t bit_string, idx_t n);
-	static void SetBitInternal(string_t &bit_string, idx_t n, idx_t new_value);
+	static void Finalize(bitstring_t &str);
+	static idx_t GetBitInternal(bitstring_t bit_string, idx_t n);
+	static void SetBitInternal(bitstring_t &bit_string, idx_t n, idx_t new_value);
 	static idx_t GetBitIndex(idx_t n);
-	static uint8_t GetFirstByte(const string_t &str);
+	static uint8_t GetFirstByte(const bitstring_t &str);
 };
 
 //===--------------------------------------------------------------------===//
 // Bit Template definitions
 //===--------------------------------------------------------------------===//
 template <class T>
-void Bit::NumericToBit(T numeric, string_t &output_str) {
+void Bit::NumericToBit(T numeric, bitstring_t &output_str) {
 	D_ASSERT(output_str.GetSize() >= sizeof(T) + 1);
 
 	auto output = output_str.GetDataWriteable();
@@ -113,20 +116,20 @@ template <class T>
 string Bit::NumericToBit(T numeric) {
 	auto bit_len = sizeof(T) + 1;
 	auto buffer = make_unsafe_uniq_array_uninitialized<char>(bit_len);
-	string_t output_str(buffer.get(), UnsafeNumericCast<uint32_t>(bit_len));
+	bitstring_t output_str(buffer.get(), UnsafeNumericCast<uint32_t>(bit_len));
 	Bit::NumericToBit(numeric, output_str);
 	return output_str.GetString();
 }
 
 template <class T>
-T Bit::BitToNumeric(string_t bit) {
+T Bit::BitToNumeric(bitstring_t bit) {
 	T output;
 	Bit::BitToNumeric(bit, output);
 	return (output);
 }
 
 template <class T>
-void Bit::BitToNumeric(string_t bit, T &output_num) {
+void Bit::BitToNumeric(bitstring_t bit, T &output_num) {
 	D_ASSERT(bit.GetSize() <= sizeof(T) + 1);
 
 	output_num = 0;

package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp

@@ -83,22 +83,19 @@ public:
 	//! Set the DataChunk to own the data of data chunk, destroying the other chunk in the process
 	DUCKDB_API void Move(DataChunk &chunk);
 
-	//! Initializes the DataChunk with the specified types to an empty DataChunk
-	//! This will create one vector of the specified type for each LogicalType in the
-	//! types list. The vector will be referencing vector to the data owned by
-	//! the DataChunk.
+	//! Initializes a DataChunk with the given types and without any vector data allocation.
+	DUCKDB_API void InitializeEmpty(const vector<LogicalType> &types);
+
+	//! Initializes a DataChunk with the given types. Then, if the corresponding boolean in the initialize-vector is
+	//! true, it initializes the vector for that data type.
+	DUCKDB_API void Initialize(ClientContext &context, const vector<LogicalType> &types,
+	                           idx_t capacity = STANDARD_VECTOR_SIZE);
 	DUCKDB_API void Initialize(Allocator &allocator, const vector<LogicalType> &types,
 	                           idx_t capacity = STANDARD_VECTOR_SIZE);
-	DUCKDB_API void Initialize(ClientContext &context, const vector<LogicalType> &types,
+	DUCKDB_API void Initialize(ClientContext &context, const vector<LogicalType> &types, const vector<bool> &initialize,
+	                           idx_t capacity = STANDARD_VECTOR_SIZE);
+	DUCKDB_API void Initialize(Allocator &allocator, const vector<LogicalType> &types, const vector<bool> &initialize,
 	                           idx_t capacity = STANDARD_VECTOR_SIZE);
-	//! Initializes an empty DataChunk with the given types. The vectors will *not* have any data allocated for them.
-	DUCKDB_API void InitializeEmpty(const vector<LogicalType> &types);
-
-	DUCKDB_API void InitializeEmpty(vector<LogicalType>::const_iterator begin, vector<LogicalType>::const_iterator end);
-	DUCKDB_API void Initialize(Allocator &allocator, vector<LogicalType>::const_iterator begin,
-	                           vector<LogicalType>::const_iterator end, idx_t capacity = STANDARD_VECTOR_SIZE);
-	DUCKDB_API void Initialize(ClientContext &context, vector<LogicalType>::const_iterator begin,
-	                           vector<LogicalType>::const_iterator end, idx_t capacity = STANDARD_VECTOR_SIZE);
 
 	//! Append the other DataChunk to this one. The column count and types of
 	//! the two DataChunks have to match exactly. Throws an exception if there

package/src/duckdb/src/include/duckdb/common/types/uhugeint.hpp

@@ -76,7 +76,7 @@ public:
 
 	template <bool CHECK_OVERFLOW = true>
 	inline static uhugeint_t Divide(uhugeint_t lhs, uhugeint_t rhs) {
-		// division between two same-size unsigned intergers can only go wrong with division by zero
+		// division between two same-size unsigned integers can only go wrong with division by zero
 		if (rhs == 0) {
 			throw OutOfRangeException("Division of UHUGEINT by zero!");
 		}

package/src/duckdb/src/include/duckdb/common/types/vector_cache.hpp

@@ -16,18 +16,20 @@ namespace duckdb {
 class Allocator;
 class Vector;
 
-//! The VectorCache holds cached data that allows for re-use of the same memory by vectors
+//! The VectorCache holds cached vector data.
+//! It enables re-using the same memory for different vectors.
 class VectorCache {
 public:
-	//! Instantiate a vector cache with the given type and capacity
-	DUCKDB_API explicit VectorCache(Allocator &allocator, const LogicalType &type,
-	                                idx_t capacity = STANDARD_VECTOR_SIZE);
+	//! Instantiate an empty vector cache.
+	DUCKDB_API VectorCache();
+	//! Instantiate a vector cache with the given type and capacity.
+	DUCKDB_API VectorCache(Allocator &allocator, const LogicalType &type, const idx_t capacity = STANDARD_VECTOR_SIZE);
 
+public:
 	buffer_ptr<VectorBuffer> buffer;
 
 public:
 	void ResetFromCache(Vector &result) const;
-
 	const LogicalType &GetType() const;
 };
 

package/src/duckdb/src/include/duckdb/common/windows_undefs.hpp

@@ -6,7 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#pragma once
+// Do not add a header inclusion guard to this file. Otherwise these Win32 macros
+// may get defined and stomp on DuckDB symbols
 
 #ifdef WIN32
 

package/src/duckdb/src/include/duckdb/core_functions/aggregate/minmax_n_helpers.hpp

@@ -40,6 +40,8 @@ struct HeapEntry<string_t> {
 	HeapEntry(HeapEntry &&other) noexcept {
 		if (other.value.IsInlined()) {
 			value = other.value;
+			capacity = 0;
+			allocated_data = nullptr;
 		} else {
 			capacity = other.capacity;
 			allocated_data = other.allocated_data;

package/src/duckdb/src/include/duckdb/core_functions/scalar/bit_functions.hpp

@@ -48,7 +48,7 @@ struct BitStringFun {
 	static constexpr const char *Description = "Pads the bitstring until the specified length";
 	static constexpr const char *Example = "bitstring('1010'::BIT, 7)";
 
-	static ScalarFunction GetFunction();
+	static ScalarFunctionSet GetFunctions();
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp

@@ -330,12 +330,6 @@ struct ListNegativeDotProductFun {
 	static constexpr const char *Name = "list_negative_dot_product";
 };
 
-struct ListNegativeInnerProductFunAlias {
-	using ALIAS = ListNegativeInnerProductFun;
-
-	static constexpr const char *Name = "<#>";
-};
-
 struct UnpivotListFun {
 	static constexpr const char *Name = "unpivot_list";
 	static constexpr const char *Parameters = "any,...";

package/src/duckdb/src/include/duckdb/core_functions/scalar/math_functions.hpp

@@ -427,7 +427,7 @@ struct AcoshFun {
 	static constexpr const char *Name = "acosh";
 	static constexpr const char *Parameters = "x";
 	static constexpr const char *Description = "Computes the inverse hyperbolic cos of x";
-	static constexpr const char *Example = "acosh(0.5)";
+	static constexpr const char *Example = "acosh(2.3)";
 
 	static ScalarFunction GetFunction();
 };

package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp

@@ -28,10 +28,11 @@ struct ExpressionState {
 	vector<unique_ptr<ExpressionState>> child_states;
 	vector<LogicalType> types;
 	DataChunk intermediate_chunk;
+	vector<bool> initialize;
 
 public:
-	void AddChild(Expression *expr);
-	void Finalize(bool empty = false);
+	void AddChild(Expression &child_expr);
+	void Finalize();
 	Allocator &GetAllocator();
 	bool HasContext();
 	DUCKDB_API ClientContext &GetContext();

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -55,6 +55,9 @@ public:
 		return std::move(art);
 	}
 
+	//! Plan index construction.
+	static unique_ptr<PhysicalOperator> CreatePlan(PlanIndexInput &input);
+
 	//! Root of the tree.
 	Node tree = Node();
 	//! Fixed-size allocators holding the ART nodes.

package/src/duckdb/src/include/duckdb/execution/index/index_type.hpp

@@ -18,6 +18,8 @@
 namespace duckdb {
 
 class BoundIndex;
+class PhysicalOperator;
+class LogicalCreateIndex;
 enum class IndexConstraintType : uint8_t;
 class Expression;
 class TableIOManager;
@@ -43,7 +45,19 @@ struct CreateIndexInput {
 	      options(options) {};
 };
 
+struct PlanIndexInput {
+	ClientContext &context;
+	LogicalCreateIndex &op;
+	unique_ptr<PhysicalOperator> &table_scan;
+
+	PlanIndexInput(ClientContext &context_p, LogicalCreateIndex &op_p, unique_ptr<PhysicalOperator> &table_scan_p)
+	    : context(context_p), op(op_p), table_scan(table_scan_p) {
+	}
+};
+
 typedef unique_ptr<BoundIndex> (*index_create_function_t)(CreateIndexInput &input);
+typedef unique_ptr<PhysicalOperator> (*index_plan_function_t)(PlanIndexInput &input);
+
 //! A index "type"
 class IndexType {
 public:
@@ -51,7 +65,8 @@ public:
 	string name;
 
 	// Callbacks
-	index_create_function_t create_instance;
+	index_plan_function_t create_plan = nullptr;
+	index_create_function_t create_instance = nullptr;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp

@@ -35,15 +35,15 @@ public:
 
 	void UnpinBuffer(const idx_t cache_idx);
 	//! Returns the buffer size set for this CSV buffer manager
-	idx_t GetBufferSize();
+	idx_t GetBufferSize() const;
 	//! Returns the number of buffers in the cached_buffers cache
-	idx_t BufferCount();
+	idx_t BufferCount() const;
 	//! If this buffer manager is done. In the context of a buffer manager it means that it read all buffers at least
 	//! once.
-	bool Done();
+	bool Done() const;
 
 	void ResetBufferManager();
-	string GetFilePath();
+	string GetFilePath() const;
 
 	ClientContext &context;
 	idx_t skip_rows = 0;

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp

@@ -168,8 +168,10 @@ struct CSVReaderOptions {
 	void SetReadOption(const string &loption, const Value &value, vector<string> &expected_names);
 	void SetWriteOption(const string &loption, const Value &value);
 	void SetDateFormat(LogicalTypeId type, const string &format, bool read_format);
-	void ToNamedParameters(named_parameter_map_t &out);
-	void FromNamedParameters(named_parameter_map_t &in, ClientContext &context);
+	void ToNamedParameters(named_parameter_map_t &out) const;
+	void FromNamedParameters(const named_parameter_map_t &in, ClientContext &context);
+	//! Verify options are not conflicting
+	void Verify();
 
 	string ToString(const string &current_file_path) const;
 	//! If the type for column with idx i was manually set

package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp

@@ -9,6 +9,7 @@
 #pragma once
 
 #include "duckdb/common/types.hpp"
+#include "duckdb/execution/operator/csv_scanner/sniffer/sniff_result.hpp"
 
 namespace duckdb {
 //! Basic CSV Column Info
@@ -23,8 +24,8 @@ struct CSVColumnInfo {
 struct CSVSchema {
 	void Initialize(vector<string> &names, vector<LogicalType> &types, const string &file_path);
 	bool Empty() const;
-	bool SchemasMatch(string &error_message, vector<string> &names, vector<LogicalType> &types,
-	                  const string &cur_file_path);
+	bool SchemasMatch(string &error_message, SnifferResult &sniffer_result, const string &cur_file_path,
+	                  bool is_minimal_sniffer) const;
 
 private:
 	static bool CanWeCastIt(LogicalTypeId source, LogicalTypeId destination);