duckdb 1.3.2 → 1.3.3-dev4.0

package/src/duckdb/src/include/duckdb/common/sorting/sort_projection_column.hpp

@@ -0,0 +1,21 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/sorting/sort_projection_column.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/typedefs.hpp"
+
+namespace duckdb {
+
+struct SortProjectionColumn {
+	bool is_payload;
+	idx_t layout_col_idx;
+	idx_t output_col_idx;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/sorting/sorted_run.hpp

@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/sorting/sorted_run.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types/row/tuple_data_states.hpp"
+
+namespace duckdb {
+
+class BufferManager;
+class DataChunk;
+class TupleDataCollection;
+class TupleDataLayout;
+
+class SortedRun {
+public:
+	SortedRun(ClientContext &context, shared_ptr<TupleDataLayout> key_layout,
+	          shared_ptr<TupleDataLayout> payload_layout, bool is_index_sort);
+	unique_ptr<SortedRun> CreateRunForMaterialization() const;
+	~SortedRun();
+
+public:
+	//! Appends data to key/data collections
+	void Sink(DataChunk &key, DataChunk &payload);
+	//! Sorts the data (physically reorder data if external)
+	void Finalize(bool external);
+	//! Destroy data between these tuple indices
+	void DestroyData(idx_t tuple_idx_begin, idx_t tuple_idx_end);
+	//! Number of tuples
+	idx_t Count() const;
+	//! Size of this sorted run
+	idx_t SizeInBytes() const;
+
+public:
+	ClientContext &context;
+
+	//! Key and payload collections (and associated append states)
+	unique_ptr<TupleDataCollection> key_data;
+	unique_ptr<TupleDataCollection> payload_data;
+	TupleDataAppendState key_append_state;
+	TupleDataAppendState payload_append_state;
+
+	//! Whether this is an (approximate) index sort
+	const bool is_index_sort;
+
+	//! Whether this run has been finalized
+	bool finalized;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/sorting/sorted_run_merger.hpp

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/sorting/sorted_run_merger.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/execution/physical_operator_states.hpp"
+#include "duckdb/common/sorting/sort_projection_column.hpp"
+
+namespace duckdb {
+
+class TupleDataLayout;
+struct BoundOrderByNode;
+struct ProgressData;
+class SortedRun;
+enum class SortKeyType : uint8_t;
+
+class SortedRunMerger {
+	friend class SortedRunMergerLocalState;
+	friend class SortedRunMergerGlobalState;
+
+public:
+	SortedRunMerger(const Expression &decode_sort_key, shared_ptr<TupleDataLayout> key_layout,
+	                vector<unique_ptr<SortedRun>> &&sorted_runs,
+	                const vector<SortProjectionColumn> &output_projection_columns, idx_t partition_size, bool external,
+	                bool is_index_sort);
+
+public:
+	//===--------------------------------------------------------------------===//
+	// Source Interface
+	//===--------------------------------------------------------------------===//
+	unique_ptr<LocalSourceState> GetLocalSourceState(ExecutionContext &context, GlobalSourceState &gstate) const;
+	unique_ptr<GlobalSourceState> GetGlobalSourceState(ClientContext &context) const;
+	SourceResultType GetData(ExecutionContext &context, DataChunk &chunk, OperatorSourceInput &input) const;
+	OperatorPartitionData GetPartitionData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
+	                                       LocalSourceState &lstate, const OperatorPartitionInfo &partition_info) const;
+	ProgressData GetProgress(ClientContext &context, GlobalSourceState &gstate) const;
+
+public:
+	//===--------------------------------------------------------------------===//
+	// Non-Standard Interface
+	//===--------------------------------------------------------------------===//
+	SourceResultType MaterializeMerge(ExecutionContext &context, OperatorSourceInput &input) const;
+	unique_ptr<SortedRun> GetMaterialized(GlobalSourceState &global_state);
+
+public:
+	const Expression &decode_sort_key;
+	shared_ptr<TupleDataLayout> key_layout;
+	vector<unique_ptr<SortedRun>> sorted_runs;
+	const vector<SortProjectionColumn> &output_projection_columns;
+	const idx_t total_count;
+
+	const idx_t partition_size;
+	const bool external;
+	const bool is_index_sort;
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/tree_renderer/text_tree_renderer.hpp

@@ -37,6 +37,10 @@ struct TextTreeRendererConfig {
 	idx_t max_extra_lines = 30;
 	bool detailed = false;
 
+	// Formatting options
+	char thousand_separator = ',';
+	char decimal_separator = '.';
+
 #ifndef DUCKDB_ASCII_TREE_RENDERER
 	const char *LTCORNER = "\342\224\214"; // NOLINT "┌";
 	const char *RTCORNER = "\342\224\220"; // NOLINT "┐";
@@ -115,6 +119,7 @@ private:
 	void SplitUpExtraInfo(const InsertionOrderPreservingMap<string> &extra_info, vector<string> &result,
 	                      idx_t max_lines);
 	void SplitStringBuffer(const string &source, vector<string> &result);
+	string FormatNumber(const string &input);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/tree_renderer/yaml_tree_renderer.hpp

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "duckdb/main/profiling_node.hpp"
+#include "duckdb/common/tree_renderer.hpp"
+#include "duckdb/common/render_tree.hpp"
+
+namespace duckdb {
+class LogicalOperator;
+class PhysicalOperator;
+class Pipeline;
+struct PipelineRenderNode;
+
+class YAMLTreeRenderer : public TreeRenderer {
+public:
+	explicit YAMLTreeRenderer() {
+	}
+	~YAMLTreeRenderer() override {
+	}
+
+public:
+	string ToString(const LogicalOperator &op);
+	string ToString(const PhysicalOperator &op);
+	string ToString(const ProfilingNode &op);
+	string ToString(const Pipeline &op);
+
+	void Render(const LogicalOperator &op, std::ostream &ss);
+	void Render(const PhysicalOperator &op, std::ostream &ss);
+	void Render(const ProfilingNode &op, std::ostream &ss) override;
+	void Render(const Pipeline &op, std::ostream &ss);
+
+	void ToStreamInternal(RenderTree &root, std::ostream &ss) override;
+	bool UsesRawKeyNames() override {
+		return false;
+	}
+
+private:
+	void RenderRecursive(RenderTree &node, std::ostream &ss, idx_t depth, idx_t x, idx_t y);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/type_util.hpp

@@ -14,8 +14,10 @@
 #include "duckdb/common/types/timestamp.hpp"
 #include "duckdb/common/types/interval.hpp"
 #include "duckdb/common/uhugeint.hpp"
+#include "duckdb/common/types/double_na_equal.hpp"
 
 namespace duckdb {
+struct bignum_t;
 
 //! Returns the PhysicalType for the given type
 template <class T>
@@ -38,6 +40,8 @@ PhysicalType GetTypeId() {
 		return PhysicalType::UINT32;
 	} else if (std::is_same<T, uint64_t>()) {
 		return PhysicalType::UINT64;
+	} else if (std::is_same<T, idx_t>() || std::is_same<T, const idx_t>()) {
+		return PhysicalType::UINT64;
 	} else if (std::is_same<T, hugeint_t>()) {
 		return PhysicalType::INT128;
 	} else if (std::is_same<T, uhugeint_t>()) {
@@ -46,6 +50,10 @@ PhysicalType GetTypeId() {
 		return PhysicalType::INT32;
 	} else if (std::is_same<T, dtime_t>()) {
 		return PhysicalType::INT64;
+	} else if (std::is_same<T, dtime_tz_t>()) {
+		return PhysicalType::INT64;
+	} else if (std::is_same<T, dtime_ns_t>()) {
+		return PhysicalType::INT64;
 	} else if (std::is_same<T, timestamp_t>()) {
 		return PhysicalType::INT64;
 	} else if (std::is_same<T, timestamp_sec_t>()) {
@@ -56,17 +64,39 @@ PhysicalType GetTypeId() {
 		return PhysicalType::INT64;
 	} else if (std::is_same<T, timestamp_tz_t>()) {
 		return PhysicalType::INT64;
-	} else if (std::is_same<T, float>()) {
+	} else if (std::is_same<T, float>() || std::is_same<T, float_na_equal>()) {
 		return PhysicalType::FLOAT;
-	} else if (std::is_same<T, double>()) {
+	} else if (std::is_same<T, double>() || std::is_same<T, double_na_equal>()) {
 		return PhysicalType::DOUBLE;
-	} else if (std::is_same<T, const char *>() || std::is_same<T, char *>() || std::is_same<T, string_t>()) {
+	} else if (std::is_same<T, const char *>() || std::is_same<T, char *>() || std::is_same<T, string_t>() ||
+	           std::is_same<T, bignum_t>()) {
 		return PhysicalType::VARCHAR;
 	} else if (std::is_same<T, interval_t>()) {
 		return PhysicalType::INTERVAL;
+	} else if (std::is_same<T, list_entry_t>()) {
+		return PhysicalType::LIST;
+	} else if (std::is_pointer<T>() || std::is_same<T, uintptr_t>()) {
+		if (sizeof(uintptr_t) == sizeof(uint32_t)) {
+			return PhysicalType::UINT32;
+		} else if (sizeof(uintptr_t) == sizeof(uint64_t)) {
+			return PhysicalType::UINT64;
+		} else {
+			throw InternalException("Unsupported pointer size in GetTypeId");
+		}
 	} else {
-		return PhysicalType::INVALID;
+		throw InternalException("Unsupported type in GetTypeId");
+	}
+}
+
+template <class T>
+bool StorageTypeCompatible(PhysicalType type) {
+	if (std::is_same<T, int8_t>()) {
+		return type == PhysicalType::INT8 || type == PhysicalType::BOOL;
+	}
+	if (std::is_same<T, uint8_t>()) {
+		return type == PhysicalType::UINT8 || type == PhysicalType::BOOL;
 	}
+	return type == GetTypeId<T>();
 }
 
 template <class T>

package/src/duckdb/src/include/duckdb/common/type_visitor.hpp

@@ -26,6 +26,9 @@ template <class F>
 inline LogicalType TypeVisitor::VisitReplace(const LogicalType &type, F &&func) {
 	switch (type.id()) {
 	case LogicalTypeId::STRUCT: {
+		if (!type.AuxInfo()) {
+			return func(type);
+		}
 		auto children = StructType::GetChildTypes(type);
 		for (auto &child : children) {
 			child.second = VisitReplace(child.second, func);
@@ -33,6 +36,9 @@ inline LogicalType TypeVisitor::VisitReplace(const LogicalType &type, F &&func)
 		return func(LogicalType::STRUCT(children));
 	}
 	case LogicalTypeId::UNION: {
+		if (!type.AuxInfo()) {
+			return func(type);
+		}
 		auto children = UnionType::CopyMemberTypes(type);
 		for (auto &child : children) {
 			child.second = VisitReplace(child.second, func);
@@ -40,16 +46,25 @@ inline LogicalType TypeVisitor::VisitReplace(const LogicalType &type, F &&func)
 		return func(LogicalType::UNION(children));
 	}
 	case LogicalTypeId::LIST: {
-		auto child = ListType::GetChildType(type);
+		if (!type.AuxInfo()) {
+			return func(type);
+		}
+		const auto &child = ListType::GetChildType(type);
 		return func(LogicalType::LIST(VisitReplace(child, func)));
 	}
 	case LogicalTypeId::ARRAY: {
-		auto child = ArrayType::GetChildType(type);
+		if (!type.AuxInfo()) {
+			return func(type);
+		}
+		const auto &child = ArrayType::GetChildType(type);
 		return func(LogicalType::ARRAY(VisitReplace(child, func), ArrayType::GetSize(type)));
 	}
 	case LogicalTypeId::MAP: {
-		auto key = MapType::KeyType(type);
-		auto value = MapType::ValueType(type);
+		if (!type.AuxInfo()) {
+			return func(type);
+		}
+		const auto &key = MapType::KeyType(type);
+		const auto &value = MapType::ValueType(type);
 		return func(LogicalType::MAP(VisitReplace(key, func), VisitReplace(value, func)));
 	}
 	default:
@@ -64,6 +79,9 @@ inline bool TypeVisitor::Contains(const LogicalType &type, F &&predicate) {
 	}
 	switch (type.id()) {
 	case LogicalTypeId::STRUCT: {
+		if (!type.AuxInfo()) {
+			return false;
+		}
 		for (const auto &child : StructType::GetChildTypes(type)) {
 			if (Contains(child.second, predicate)) {
 				return true;
@@ -72,17 +90,29 @@ inline bool TypeVisitor::Contains(const LogicalType &type, F &&predicate) {
 		return false;
 	}
 	case LogicalTypeId::UNION:
-		for (const auto &child : UnionType::CopyMemberTypes(type)) {
-			if (Contains(child.second, predicate)) {
+		if (!type.AuxInfo()) {
+			return false;
+		}
+		for (idx_t i = 0; i < UnionType::GetMemberCount(type); i++) {
+			if (Contains(UnionType::GetMemberType(type, i), predicate)) {
 				return true;
 			}
 		}
 		return false;
 	case LogicalTypeId::LIST:
+		if (!type.AuxInfo()) {
+			return false;
+		}
 		return Contains(ListType::GetChildType(type), predicate);
 	case LogicalTypeId::ARRAY:
+		if (!type.AuxInfo()) {
+			return false;
+		}
 		return Contains(ArrayType::GetChildType(type), predicate);
 	case LogicalTypeId::MAP:
+		if (!type.AuxInfo()) {
+			return false;
+		}
 		return Contains(MapType::KeyType(type), predicate) || Contains(MapType::ValueType(type), predicate);
 	default:
 		return false;

package/src/duckdb/src/include/duckdb/common/types/bignum.hpp

@@ -0,0 +1,159 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/types/bignum.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/limits.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/types.hpp"
+#include "duckdb/common/winapi.hpp"
+#include "duckdb/function/cast/default_casts.hpp"
+#include "duckdb/common/bignum.hpp"
+
+namespace duckdb {
+using digit_t = uint32_t;
+using twodigit_t = uint64_t;
+
+//! The Bignum class is a static class that holds helper functions for the Bignum type.
+class Bignum {
+public:
+	//! This is the maximum number of bytes a BIGNUM can have on it's data size
+	//! i.e., 2^(8*3-1) - 1.
+	DUCKDB_API static constexpr uint32_t MAX_DATA_SIZE = 8388607;
+	//! Header size of a Bignum is always 3 bytes.
+	DUCKDB_API static constexpr uint8_t BIGNUM_HEADER_SIZE = 3;
+	//! Max(e such that 10**e fits in a digit_t)
+	DUCKDB_API static constexpr uint8_t DECIMAL_SHIFT = 9;
+	//! 10 ** DECIMAL_SHIFT
+	DUCKDB_API static constexpr digit_t DECIMAL_BASE = 1000000000;
+	//! Bytes of a digit_t
+	DUCKDB_API static constexpr uint8_t DIGIT_BYTES = sizeof(digit_t);
+	//! Bits of a digit_t
+	DUCKDB_API static constexpr uint8_t DIGIT_BITS = DIGIT_BYTES * 8;
+	//! Verifies if a Bignum is valid. i.e., if it has 3 header bytes. The header correctly represents the number of
+	//! data bytes, and the data bytes has no leading zero bytes.
+	DUCKDB_API static void Verify(const bignum_t &input);
+
+	//! Sets the header of a bignum (i.e., char* blob), depending on the number of bytes that bignum needs and if it's a
+	//! negative number
+	DUCKDB_API static void SetHeader(char *blob, uint64_t number_of_bytes, bool is_negative);
+	//! Initializes and returns a blob with value 0, allocated in Vector& result
+	DUCKDB_API static bignum_t InitializeBignumZero(Vector &result);
+	DUCKDB_API static string InitializeBignumZero();
+
+	//! Switch Case of To Bignum Convertion
+	DUCKDB_API static BoundCastInfo NumericToBignumCastSwitch(const LogicalType &source);
+
+	//! ----------------------------------- Varchar Cast ----------------------------------- //
+	//! Function to prepare a varchar for conversion. We trim zero's, check for negative values, and what-not
+	//! Returns false if this is an invalid varchar
+	DUCKDB_API static bool VarcharFormatting(const string_t &value, idx_t &start_pos, idx_t &end_pos, bool &is_negative,
+	                                         bool &is_zero);
+
+	//! Converts a char to a Digit
+	DUCKDB_API static int CharToDigit(char c);
+	//! Converts a Digit to a char
+	DUCKDB_API static char DigitToChar(int digit);
+	//! Function to convert a string_t into a vector of bytes
+	DUCKDB_API static void GetByteArray(vector<uint8_t> &byte_array, bool &is_negative, const string_t &blob);
+	//! Function to create a BIGNUM blob from a byte array containing the absolute value, plus an is_negative bool
+	DUCKDB_API static string FromByteArray(uint8_t *data, idx_t size, bool is_negative);
+	//! Function to convert BIGNUM blob to a VARCHAR
+	DUCKDB_API static string BignumToVarchar(const bignum_t &blob);
+	//! Function to convert Varchar to BIGNUM blob
+	DUCKDB_API static string VarcharToBignum(const string_t &value);
+	//! ----------------------------------- Double Cast ----------------------------------- //
+	DUCKDB_API static bool BignumToDouble(const bignum_t &blob, double &result, bool &strict);
+	template <class T>
+	static bool BignumToInt(const bignum_t &blob, T &result, bool &strict) {
+		auto data_byte_size = blob.data.GetSize() - BIGNUM_HEADER_SIZE;
+		auto data = blob.data.GetData();
+		bool is_negative = (data[0] & 0x80) == 0;
+
+		uhugeint_t abs_value = 0;
+		for (idx_t i = 0; i < data_byte_size; ++i) {
+			uint8_t byte = static_cast<uint8_t>(data[Bignum::BIGNUM_HEADER_SIZE + i]);
+			if (is_negative) {
+				byte = ~byte;
+			}
+			abs_value = (abs_value << 8) | byte;
+		}
+
+		if (is_negative) {
+			if (abs_value > static_cast<uhugeint_t>(std::numeric_limits<T>::max()) + 1) {
+				throw OutOfRangeException("Negative bignum too small for type");
+			}
+			result = static_cast<T>(-static_cast<hugeint_t>(abs_value));
+		} else {
+			if (abs_value > static_cast<uhugeint_t>(std::numeric_limits<T>::max())) {
+				throw OutOfRangeException("Positive bignum too large for type");
+			}
+			result = static_cast<T>(abs_value);
+		}
+		return true;
+	}
+};
+
+//! ----------------------------------- (u)Integral Cast ----------------------------------- //
+struct IntCastToBignum {
+	template <class SRC>
+	static inline bignum_t Operation(SRC input, Vector &result) {
+		return IntToBignum(result, input);
+	}
+};
+
+//! ----------------------------------- (u)HugeInt Cast ----------------------------------- //
+struct HugeintCastToBignum {
+	template <class SRC>
+	static inline bignum_t Operation(SRC input, Vector &result) {
+		throw InternalException("Unsupported type for cast to BIGNUM");
+	}
+};
+
+struct TryCastToBignum {
+	template <class SRC, class DST>
+	static inline bool Operation(SRC input, DST &result, Vector &result_vector, CastParameters &parameters) {
+		throw InternalException("Unsupported type for try cast to BIGNUM");
+	}
+};
+
+template <>
+DUCKDB_API bool TryCastToBignum::Operation(double double_value, bignum_t &result_value, Vector &result,
+                                           CastParameters &parameters);
+
+template <>
+DUCKDB_API bool TryCastToBignum::Operation(float float_value, bignum_t &result_value, Vector &result,
+                                           CastParameters &parameters);
+
+template <>
+DUCKDB_API bool TryCastToBignum::Operation(string_t input_value, bignum_t &result_value, Vector &result,
+                                           CastParameters &parameters);
+
+struct BignumCastToVarchar {
+	template <class SRC>
+	DUCKDB_API static inline string_t Operation(SRC input, Vector &result) {
+		return StringVector::AddStringOrBlob(result, Bignum::BignumToVarchar(input));
+	}
+};
+
+struct BignumToDoubleCast {
+	template <class SRC, class DST>
+	DUCKDB_API static inline bool Operation(SRC input, DST &result, bool strict = false) {
+		return Bignum::BignumToDouble(input, result, strict);
+	}
+};
+
+struct BignumToIntCast {
+	template <class SRC, class DST>
+	DUCKDB_API static inline bool Operation(SRC input, DST &result, bool strict = false) {
+		return Bignum::BignumToInt(input, result, strict);
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp

@@ -14,6 +14,7 @@ namespace duckdb {
 
 struct ChunkMetaData;
 struct VectorMetaData;
+struct SwizzleMetaData;
 
 struct BlockMetaData {
 	//! The underlying block handle
@@ -75,8 +76,8 @@ public:
 	void Initialize(ColumnDataAllocator &other);
 	void InitializeChunkState(ChunkManagementState &state, ChunkMetaData &meta_data);
 	data_ptr_t GetDataPointer(ChunkManagementState &state, uint32_t block_id, uint32_t offset);
-	void UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
-	                       uint32_t block_id, uint32_t offset);
+	void UnswizzlePointers(ChunkManagementState &state, Vector &result, SwizzleMetaData &swizzle_segment,
+	                       const VectorMetaData &string_heap_segment, const idx_t &v_offset, const bool &copied);
 
 	//! Prevents the block with the given id from being added to the eviction queue
 	void SetDestroyBufferUponUnpin(uint32_t block_id);

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp

@@ -36,11 +36,13 @@ struct VectorDataIndex {
 };
 
 struct SwizzleMetaData {
-	SwizzleMetaData(VectorDataIndex child_index_p, uint16_t offset_p, uint16_t count_p)
-	    : child_index(child_index_p), offset(offset_p), count(count_p) {
+	SwizzleMetaData(VectorDataIndex child_index_p, data_ptr_t ptr_p, uint16_t offset_p, uint16_t count_p)
+	    : child_index(child_index_p), ptr(ptr_p), offset(offset_p), count(count_p) {
 	}
 	//! Index of block storing heap
 	VectorDataIndex child_index;
+	//! Last-known pointer
+	data_ptr_t ptr;
 	//! Offset into the string_t vector
 	uint16_t offset;
 	//! Number of strings starting at 'offset' that have strings stored in the block with index 'child_index'