duckdb 0.7.2-dev0.0 → 0.7.2-dev1138.0

package/src/duckdb/src/include/duckdb/common/types/string_type.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/assert.hpp"
 #include "duckdb/common/constants.hpp"
+#include "duckdb/common/helper.hpp"
 
 #include <cstring>
 
@@ -117,10 +118,79 @@ public:
 
 	void Verify() const;
 	void VerifyNull() const;
+
+	struct StringComparisonOperators {
+		static inline bool Equals(const string_t &a, const string_t &b) {
+#ifdef DUCKDB_DEBUG_NO_INLINE
+			if (a.GetSize() != b.GetSize())
+				return false;
+			return (memcmp(a.GetDataUnsafe(), b.GetDataUnsafe(), a.GetSize()) == 0);
+#endif
+			uint64_t A = Load<uint64_t>((const_data_ptr_t)&a);
+			uint64_t B = Load<uint64_t>((const_data_ptr_t)&b);
+			if (A != B) {
+				// Either length or prefix are different -> not equal
+				return false;
+			}
+			// they have the same length and same prefix!
+			A = Load<uint64_t>((const_data_ptr_t)&a + 8u);
+			B = Load<uint64_t>((const_data_ptr_t)&b + 8u);
+			if (A == B) {
+				// either they are both inlined (so compare equal) or point to the same string (so compare equal)
+				return true;
+			}
+			if (!a.IsInlined()) {
+				// 'long' strings of the same length -> compare pointed value
+				if (memcmp(a.value.pointer.ptr, b.value.pointer.ptr, a.GetSize()) == 0) {
+					return true;
+				}
+			}
+			// either they are short string of same length but different content
+			//     or they point to string with different content
+			//     either way, they can't represent the same underlying string
+			return false;
+		}
+		// compare up to shared length. if still the same, compare lengths
+		static bool GreaterThan(const string_t &left, const string_t &right) {
+			const uint32_t left_length = left.GetSize();
+			const uint32_t right_length = right.GetSize();
+			const uint32_t min_length = std::min<uint32_t>(left_length, right_length);
+
+#ifndef DUCKDB_DEBUG_NO_INLINE
+			uint32_t A = Load<uint32_t>((const_data_ptr_t)left.GetPrefix());
+			uint32_t B = Load<uint32_t>((const_data_ptr_t)right.GetPrefix());
+
+			// Utility to move 0xa1b2c3d4 into 0xd4c3b2a1, basically inverting the order byte-a-byte
+			auto bswap = [](uint32_t v) -> uint32_t {
+				uint32_t t1 = (v >> 16u) | (v << 16u);
+				uint32_t t2 = t1 & 0x00ff00ff;
+				uint32_t t3 = t1 & 0xff00ff00;
+				return (t2 << 8u) | (t3 >> 8u);
+			};
+
+			// Check on prefix -----
+			// We dont' need to mask since:
+			//	if the prefix is greater(after bswap), it will stay greater regardless of the extra bytes
+			// 	if the prefix is smaller(after bswap), it will stay smaller regardless of the extra bytes
+			//	if the prefix is equal, the extra bytes are guaranteed to be /0 for the shorter one
+
+			if (A != B)
+				return bswap(A) > bswap(B);
+#endif
+			auto memcmp_res = memcmp(left.GetDataUnsafe(), right.GetDataUnsafe(), min_length);
+			return memcmp_res > 0 || (memcmp_res == 0 && left_length > right_length);
+		}
+	};
+
+	bool operator==(const string_t &r) const {
+		return StringComparisonOperators::Equals(*this, r);
+	}
+
+	bool operator>(const string_t &r) const {
+		return StringComparisonOperators::GreaterThan(*this, r);
+	}
 	bool operator<(const string_t &r) const {
-		auto this_str = this->GetString();
-		auto r_str = r.GetString();
-		return this_str < r_str;
+		return r > *this;
 	}
 
 private:

package/src/duckdb/src/include/duckdb/common/types/time.hpp

@@ -39,6 +39,9 @@ public:
 	DUCKDB_API static string ConversionError(const string &str);
 	DUCKDB_API static string ConversionError(string_t str);
 
+	DUCKDB_API static dtime_t FromTimeMs(int64_t time_ms);
+	DUCKDB_API static dtime_t FromTimeNs(int64_t time_ns);
+
 private:
 	static bool TryConvertInternal(const char *buf, idx_t len, idx_t &pos, dtime_t &result, bool strict);
 };

package/src/duckdb/src/include/duckdb/common/types/value.hpp

@@ -23,6 +23,7 @@ class CastFunctionSet;
 class Deserializer;
 class Serializer;
 struct GetCastFunctionInput;
+struct ExtraValueInfo;
 
 //! The Value object holds a single arbitrary value of any type that can be
 //! stored in the database.
@@ -66,7 +67,7 @@ public:
 	inline LogicalType &GetTypeMutable() {
 		return type_;
 	}
-	inline const LogicalType &type() const {
+	inline const LogicalType &type() const { // NOLINT
 		return type_;
 	}
 	inline bool IsNull() const {
@@ -77,6 +78,10 @@ public:
 	DUCKDB_API static Value MinimumValue(const LogicalType &type);
 	//! Create the highest possible value of a given type (numeric only)
 	DUCKDB_API static Value MaximumValue(const LogicalType &type);
+	//! Create the negative infinite value of a given type (numeric only)
+	DUCKDB_API static Value NegativeInfinity(const LogicalType &type);
+	//! Create the positive infinite value of a given type (numeric only)
+	DUCKDB_API static Value Infinity(const LogicalType &type);
 	//! Create a Numeric value of the specified type with the specified value
 	DUCKDB_API static Value Numeric(const LogicalType &type, int64_t value);
 	DUCKDB_API static Value Numeric(const LogicalType &type, hugeint_t value);
@@ -150,17 +155,17 @@ public:
 	//! Cannot be called with an empty list, use either EMPTYLIST or LIST with a type instead
 	DUCKDB_API static Value LIST(vector<Value> values);
 	//! Create a list value with the given entries
-	DUCKDB_API static Value LIST(LogicalType child_type, vector<Value> values);
+	DUCKDB_API static Value LIST(const LogicalType &child_type, vector<Value> values);
 	//! Create an empty list with the specified child-type
-	DUCKDB_API static Value EMPTYLIST(LogicalType child_type);
+	DUCKDB_API static Value EMPTYLIST(const LogicalType &child_type);
 	//! Create a map value with the given entries
-	DUCKDB_API static Value MAP(LogicalType child_type, vector<Value> values);
+	DUCKDB_API static Value MAP(const LogicalType &child_type, vector<Value> values);
 	//! Create a union value from a selected value and a tag from a set of alternatives.
 	DUCKDB_API static Value UNION(child_list_t<LogicalType> members, uint8_t tag, Value value);
 
 	//! Create a blob Value from a data pointer and a length: no bytes are interpreted
 	DUCKDB_API static Value BLOB(const_data_ptr_t data, idx_t len);
-	DUCKDB_API static Value BLOB_RAW(const string &data) {
+	DUCKDB_API static Value BLOB_RAW(const string &data) { // NOLINT
 		return Value::BLOB((const_data_ptr_t)data.c_str(), data.size());
 	}
 	//! Creates a blob by casting a specified string to a blob (i.e. interpreting \x characters)
@@ -225,6 +230,8 @@ public:
 
 	//! Serializes a Value to a stand-alone binary blob
 	DUCKDB_API void Serialize(Serializer &serializer) const;
+	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
+	DUCKDB_API static Value FormatDeserialize(FormatDeserializer &deserializer);
 	//! Deserializes a Value from a blob
 	DUCKDB_API static Value Deserialize(Deserializer &source);
 
@@ -277,11 +284,8 @@ public:
 
 private:
 	//! The logical of the value
-	LogicalType type_;
+	LogicalType type_; // NOLINT
 
-#if DUCKDB_API_VERSION < DUCKDB_API_0_3_2
-public:
-#endif
 	//! Whether or not the value is NULL
 	bool is_null;
 
@@ -297,21 +301,17 @@ public:
 		uint32_t uinteger;
 		uint64_t ubigint;
 		hugeint_t hugeint;
-		float float_;
-		double double_;
+		float float_;   // NOLINT
+		double double_; // NOLINT
 		uintptr_t pointer;
 		uint64_t hash;
 		date_t date;
 		dtime_t time;
 		timestamp_t timestamp;
 		interval_t interval;
-	} value_;
-
-	//! The value of the object, if it is of a variable size type
-	string str_value;
+	} value_; // NOLINT
 
-	vector<Value> struct_value;
-	vector<Value> list_value;
+	shared_ptr<ExtraValueInfo> value_info_; // NOLINT
 
 private:
 	template <class T>
@@ -535,37 +535,6 @@ DUCKDB_API timestamp_t Value::GetValueUnsafe() const;
 template <>
 DUCKDB_API interval_t Value::GetValueUnsafe() const;
 
-template <>
-DUCKDB_API int8_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API int16_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API int32_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API int64_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API hugeint_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint8_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint16_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint32_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API uint64_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API float &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API double &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API date_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API dtime_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API timestamp_t &Value::GetReferenceUnsafe();
-template <>
-DUCKDB_API interval_t &Value::GetReferenceUnsafe();
-
 template <>
 DUCKDB_API bool Value::IsNan(float input);
 template <>

package/src/duckdb/src/include/duckdb/common/types/value_map.hpp

@@ -1,7 +1,7 @@
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
-// duckdb/parser/value_map.hpp
+// duckdb/common/types/value_map.hpp
 //
 //
 //===----------------------------------------------------------------------===//

package/src/duckdb/src/include/duckdb/common/types/vector.hpp

@@ -162,6 +162,9 @@ public:
 	//! Deserializes a blob back into a Vector
 	DUCKDB_API void Deserialize(idx_t count, Deserializer &source);
 
+	DUCKDB_API void FormatSerialize(FormatSerializer &serializer, idx_t count);
+	DUCKDB_API void FormatDeserialize(FormatDeserializer &deserializer, idx_t count);
+
 	// Getters
 	inline VectorType GetVectorType() const {
 		return vector_type;
@@ -312,7 +315,6 @@ struct FlatVector {
 		return !vector.validity.RowIsValid(idx);
 	}
 	DUCKDB_API static const SelectionVector *IncrementalSelectionVector();
-	static Value GetValuesFromOffsets(Vector &values, vector<idx_t> &offsets);
 };
 
 struct ListVector {

package/src/duckdb/src/include/duckdb/common/types.hpp

@@ -17,12 +17,28 @@
 
 namespace duckdb {
 
+class FormatSerializer;
+class FormatDeserializer;
 class Serializer;
 class Deserializer;
 class Value;
 class TypeCatalogEntry;
 class Vector;
 class ClientContext;
+class FieldWriter;
+
+//! Extra Type Info Type
+enum class ExtraTypeInfoType : uint8_t {
+	INVALID_TYPE_INFO = 0,
+	GENERIC_TYPE_INFO = 1,
+	DECIMAL_TYPE_INFO = 2,
+	STRING_TYPE_INFO = 3,
+	LIST_TYPE_INFO = 4,
+	STRUCT_TYPE_INFO = 5,
+	ENUM_TYPE_INFO = 6,
+	USER_TYPE_INFO = 7,
+	AGGREGATE_STATE_TYPE_INFO = 8
+};
 
 struct hugeint_t {
 public:
@@ -32,7 +48,7 @@ public:
 public:
 	DUCKDB_API hugeint_t() = default;
 	DUCKDB_API hugeint_t(int64_t value); // NOLINT: Allow implicit conversion from `int64_t`
-	DUCKDB_API constexpr hugeint_t(int64_t upper, uint64_t lower): lower(lower), upper(upper) {};
+	DUCKDB_API constexpr hugeint_t(int64_t upper, uint64_t lower): lower(lower), upper(upper) {}
 	DUCKDB_API constexpr hugeint_t(const hugeint_t &rhs) = default;
 	DUCKDB_API constexpr hugeint_t(hugeint_t &&rhs) = default;
 	DUCKDB_API hugeint_t &operator=(const hugeint_t &rhs) = default;
@@ -94,6 +110,12 @@ struct list_entry_t {
 	list_entry_t() = default;
 	list_entry_t(uint64_t offset, uint64_t length) : offset(offset), length(length) {
 	}
+	inline constexpr bool operator != (const list_entry_t &other) const {
+		return !(*this == other);
+	}
+	inline constexpr bool operator == (const list_entry_t &other) const {
+		return offset == other.offset && length == other.length;
+	}
 
 	uint64_t offset;
 	uint64_t length;
@@ -261,7 +283,6 @@ enum class LogicalTypeId : uint8_t {
 
 	HUGEINT = 50,
 	POINTER = 51,
-	// HASH = 52, // deprecated, uses UBIGINT instead
 	VALIDITY = 53,
 	UUID = 54,
 
@@ -275,8 +296,8 @@ enum class LogicalTypeId : uint8_t {
 	UNION = 107
 };
 
-struct ExtraTypeInfo;
 
+struct ExtraTypeInfo;
 
 struct aggregate_state_t;
 
@@ -298,6 +319,11 @@ struct LogicalType {
 	inline const ExtraTypeInfo *AuxInfo() const {
 		return type_info_.get();
 	}
+
+	inline shared_ptr<ExtraTypeInfo> GetAuxInfoShrPtr() const {
+		return type_info_;
+	}
+
 	inline void CopyAuxInfo(const LogicalType& other) {
 		type_info_ = other.type_info_;
 	}
@@ -314,7 +340,7 @@ struct LogicalType {
 	inline LogicalType& operator=(LogicalType&& other) noexcept {
 		id_ = other.id_;
 		physical_type_ = other.physical_type_;
-		type_info_ = std::move(other.type_info_);
+		std::swap(type_info_, other.type_info_);
 		return *this;
 	}
 
@@ -325,9 +351,16 @@ struct LogicalType {
 
 	//! Serializes a LogicalType to a stand-alone binary blob
 	DUCKDB_API void Serialize(Serializer &serializer) const;
+
+	DUCKDB_API void SerializeEnumType(Serializer &serializer) const;
+
 	//! Deserializes a blob back into an LogicalType
 	DUCKDB_API static LogicalType Deserialize(Deserializer &source);
 
+	DUCKDB_API void FormatSerialize(FormatSerializer &serializer) const;
+	DUCKDB_API static LogicalType FormatDeserialize(FormatDeserializer &deserializer);
+
+
 	DUCKDB_API static bool TypeIsTimestamp(LogicalTypeId id) {
 		return (id == LogicalTypeId::TIMESTAMP ||
 				id == LogicalTypeId::TIMESTAMP_MS ||
@@ -350,6 +383,8 @@ struct LogicalType {
 	DUCKDB_API static void SetCatalog(LogicalType &type, TypeCatalogEntry* catalog_entry);
 	DUCKDB_API static TypeCatalogEntry* GetCatalog(const LogicalType &type);
 
+	DUCKDB_API static ExtraTypeInfoType GetExtraTypeInfoType(const ExtraTypeInfo &type);
+
 	//! Gets the decimal properties of a numeric type. Fails if the type is not numeric.
 	DUCKDB_API bool GetDecimalProperties(uint8_t &width, uint8_t &scale) const;
 
@@ -404,10 +439,10 @@ public:
 	// explicitly allowing these functions to be capitalized to be in-line with the remaining functions
 	DUCKDB_API static LogicalType DECIMAL(int width, int scale);                 // NOLINT
 	DUCKDB_API static LogicalType VARCHAR_COLLATION(string collation);           // NOLINT
-	DUCKDB_API static LogicalType LIST( LogicalType child);                       // NOLINT
-	DUCKDB_API static LogicalType STRUCT( child_list_t<LogicalType> children);    // NOLINT
+	DUCKDB_API static LogicalType LIST(const LogicalType &child);                       // NOLINT
+	DUCKDB_API static LogicalType STRUCT(const child_list_t<LogicalType> &children);    // NOLINT
 	DUCKDB_API static LogicalType AGGREGATE_STATE(aggregate_state_t state_type);    // NOLINT
-	DUCKDB_API static LogicalType MAP(LogicalType child);				// NOLINT
+	DUCKDB_API static LogicalType MAP(const LogicalType &child);				// NOLINT
 	DUCKDB_API static LogicalType MAP( child_list_t<LogicalType> children);       // NOLINT
 	DUCKDB_API static LogicalType MAP(LogicalType key, LogicalType value); // NOLINT
 	DUCKDB_API static LogicalType UNION( child_list_t<LogicalType> members);     // NOLINT
@@ -442,12 +477,14 @@ struct UserType{
 struct EnumType{
 	DUCKDB_API static const string &GetTypeName(const LogicalType &type);
 	DUCKDB_API static int64_t GetPos(const LogicalType &type, const string_t& key);
-	DUCKDB_API static Vector &GetValuesInsertOrder(const LogicalType &type);
+	DUCKDB_API static const Vector &GetValuesInsertOrder(const LogicalType &type);
 	DUCKDB_API static idx_t GetSize(const LogicalType &type);
 	DUCKDB_API static const string GetValue(const Value &val);
 	DUCKDB_API static void SetCatalog(LogicalType &type, TypeCatalogEntry* catalog_entry);
 	DUCKDB_API static TypeCatalogEntry* GetCatalog(const LogicalType &type);
+	DUCKDB_API static string GetSchemaName(const LogicalType &type);
 	DUCKDB_API static PhysicalType GetPhysicalType(const LogicalType &type);
+	DUCKDB_API static void Serialize(FieldWriter& writer, const ExtraTypeInfo& type_info, bool serialize_internals);
 };
 
 struct StructType {

package/src/duckdb/src/include/duckdb/common/vector_operations/unary_executor.hpp

@@ -209,8 +209,8 @@ public:
 
 	template <class INPUT_TYPE, class RESULT_TYPE, class OP>
 	static void ExecuteString(Vector &input, Vector &result, idx_t count) {
-		UnaryExecutor::GenericExecute<string_t, string_t, UnaryStringOperator<OP>>(input, result, count,
-		                                                                           (void *)&result);
+		UnaryExecutor::GenericExecute<INPUT_TYPE, RESULT_TYPE, UnaryStringOperator<OP>>(input, result, count,
+		                                                                                (void *)&result);
 	}
 };
 

package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp

@@ -113,6 +113,7 @@ public:
 	}
 
 	idx_t MaxCapacity();
+	static idx_t GetMaxCapacity(HtEntryType entry_type, idx_t tuple_size);
 
 	void Partition(vector<GroupedAggregateHashTable *> &partition_hts, hash_t mask, idx_t shift);
 

package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp

@@ -27,18 +27,7 @@
 namespace duckdb {
 
 class ConflictManager;
-
-struct ARTIndexScanState : public IndexScanState {
-
-	//! Scan predicates (single predicate scan or range scan)
-	Value values[2];
-	//! Expressions of the scan predicates
-	ExpressionType expressions[2];
-	bool checked = false;
-	//! All scanned row IDs
-	vector<row_t> result_ids;
-	Iterator iterator;
-};
+struct ARTIndexScanState;
 
 enum class VerifyExistenceType : uint8_t {
 	APPEND = 0,    // appends to a table
@@ -72,7 +61,7 @@ public:
 	          vector<row_t> &result_ids) override;
 
 	//! Called when data is appended to the index. The lock obtained from InitializeLock must be held
-	bool Append(IndexLock &lock, DataChunk &entries, Vector &row_identifiers) override;
+	PreservedError Append(IndexLock &lock, DataChunk &entries, Vector &row_identifiers) override;
 	//! Verify that data can be appended to the index without a constraint violation
 	void VerifyAppend(DataChunk &chunk) override;
 	//! Verify that data can be appended to the index without a constraint violation using the conflict manager
@@ -80,7 +69,7 @@ public:
 	//! Delete a chunk of entries from the index. The lock obtained from InitializeLock must be held
 	void Delete(IndexLock &lock, DataChunk &entries, Vector &row_identifiers) override;
 	//! Insert a chunk of entries into the index
-	bool Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;
+	PreservedError Insert(IndexLock &lock, DataChunk &data, Vector &row_ids) override;
 
 	//! Construct an ART from a vector of sorted keys
 	bool ConstructFromSorted(idx_t count, vector<Key> &keys, Vector &row_identifiers);

package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp

@@ -20,7 +20,7 @@ class PhysicalPerfectHashAggregate : public PhysicalOperator {
 public:
 	PhysicalPerfectHashAggregate(ClientContext &context, vector<LogicalType> types,
 	                             vector<unique_ptr<Expression>> aggregates, vector<unique_ptr<Expression>> groups,
-	                             vector<unique_ptr<BaseStatistics>> group_stats, vector<idx_t> required_bits,
+	                             const vector<unique_ptr<BaseStatistics>> &group_stats, vector<idx_t> required_bits,
 	                             idx_t estimated_cardinality);
 
 	//! The groups

package/src/duckdb/src/include/duckdb/execution/operator/join/physical_cross_product.hpp

@@ -60,10 +60,12 @@ public:
 
 	OperatorResultType Execute(DataChunk &input, DataChunk &output);
 
+	// returns if the left side is scanned as a constant vector
 	bool ScanLHS() {
 		return scan_input_chunk;
 	}
 
+	// returns the position in the chunk of chunk scanned as a constant input vector
 	idx_t PositionInChunk() {
 		return position_in_chunk;
 	}

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_file_handle.hpp

@@ -87,6 +87,7 @@ public:
 			// we have data left to read from the file
 			// read directly into the buffer
 			auto bytes_read = file_handle->Read((char *)buffer + result_offset, nr_bytes - result_offset);
+			file_size = file_handle->GetFileSize();
 			read_position += bytes_read;
 			if (reset_enabled) {
 				// if reset caching is enabled, we need to cache the bytes that we have read

package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/types.hpp"
 
 namespace duckdb {
 
@@ -77,12 +78,19 @@ struct BufferedCSVReaderOptions {
 	vector<LogicalType> sql_type_list;
 	//! User-defined name list
 	vector<string> name_list;
+	//! Types considered as candidates for auto detection ordered by descending specificity (~ from high to low)
+	vector<LogicalType> auto_type_candidates = {LogicalType::VARCHAR, LogicalType::TIMESTAMP, LogicalType::DATE,
+	                                            LogicalType::TIME,    LogicalType::DOUBLE,    LogicalType::BIGINT,
+	                                            LogicalType::BOOLEAN, LogicalType::SQLNULL};
+
 	//===--------------------------------------------------------------------===//
 	// ReadCSVOptions
 	//===--------------------------------------------------------------------===//
 
 	//! How many leading rows to skip
 	idx_t skip_rows = 0;
+	//! Whether or not the skip_rows is set by the user
+	bool skip_rows_set = false;
 	//! Maximum CSV line size: specified because if we reach this amount, we likely have wrong delimiters (default: 2MB)
 	//! note that this is the guaranteed line length that will succeed, longer lines may be accepted if slightly above
 	idx_t maximum_line_size = 2097152;
@@ -110,6 +118,8 @@ struct BufferedCSVReaderOptions {
 	idx_t buffer_size = CSVBuffer::INITIAL_BUFFER_SIZE_COLOSSAL;
 	//! Decimal separator when reading as numeric
 	string decimal_separator = ".";
+	//! Whether or not to pad rows that do not have enough columns with NULL values
+	bool null_padding = true;
 
 	//===--------------------------------------------------------------------===//
 	// WriteCSVOptions

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_projection.hpp

@@ -30,6 +30,11 @@ public:
 	}
 
 	string ParamsToString() const override;
+
+	static unique_ptr<PhysicalOperator>
+	CreateJoinProjection(vector<LogicalType> proj_types, const vector<LogicalType> &lhs_types,
+	                     const vector<LogicalType> &rhs_types, const vector<idx_t> &left_projection_map,
+	                     const vector<idx_t> &right_projection_map, const idx_t estimated_cardinality);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp

@@ -57,9 +57,12 @@ private:
 
 	HashTableList unpartitioned_hts;
 	unordered_map<hash_t, HashTableList> radix_partitioned_hts;
+	idx_t tuple_size;
 
 private:
 	idx_t ListAddChunk(HashTableList &list, DataChunk &groups, Vector &group_hashes, DataChunk &payload,
 	                   const vector<idx_t> &filter);
+	//! Returns the HT entry size used for intermediate hash tables
+	HtEntryType GetHTEntrySize();
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp

@@ -48,6 +48,7 @@ protected:
 
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalAggregate &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalAnyJoin &op);
+	unique_ptr<PhysicalOperator> CreatePlan(LogicalAsOfJoin &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalColumnDataGet &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalComparisonJoin &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalCreate &op);
@@ -88,9 +89,6 @@ protected:
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalRecursiveCTE &op);
 	unique_ptr<PhysicalOperator> CreatePlan(LogicalCTERef &op);
 
-	unique_ptr<PhysicalOperator> CreateDistinctOn(unique_ptr<PhysicalOperator> child,
-	                                              vector<unique_ptr<Expression>> distinct_targets);
-
 	unique_ptr<PhysicalOperator> ExtractAggregateExpressions(unique_ptr<PhysicalOperator> child,
 	                                                         vector<unique_ptr<Expression>> &expressions,
 	                                                         vector<unique_ptr<Expression>> &groups);

package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp

@@ -15,6 +15,60 @@
 
 namespace duckdb {
 
+class WindowAggregateState {
+public:
+	WindowAggregateState(AggregateFunction &aggregate, FunctionData *bind_info, const LogicalType &result_type_p);
+	virtual ~WindowAggregateState();
+
+	virtual void Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered);
+	virtual void Finalize();
+	virtual void Compute(Vector &result, idx_t rid, idx_t start, idx_t end);
+
+protected:
+	void AggregateInit();
+	void AggegateFinal(Vector &result, idx_t rid);
+
+	//! The aggregate that the window function is computed over
+	AggregateFunction aggregate;
+	//! The bind info of the aggregate
+	FunctionData *bind_info;
+	//! The result type of the window function
+	LogicalType result_type;
+
+	//! Data pointer that contains a single state, used for intermediate window segment aggregation
+	vector<data_t> state;
+	//! Reused result state container for the window functions
+	Vector statev;
+	//! A vector of pointers to "state", used for intermediate window segment aggregation
+	Vector statep;
+	//! Input data chunk, used for intermediate window segment aggregation
+	DataChunk inputs;
+};
+
+class WindowConstantAggregate : public WindowAggregateState {
+public:
+	static bool IsConstantAggregate(const BoundWindowExpression &wexpr);
+
+	WindowConstantAggregate(AggregateFunction &aggregate, FunctionData *bind_info, const LogicalType &result_type_p,
+	                        const ValidityMask &partition_mask, const idx_t count);
+	~WindowConstantAggregate() override {
+	}
+
+	void Sink(DataChunk &payload_chunk, SelectionVector *filter_sel, idx_t filtered) override;
+	void Finalize() override;
+	void Compute(Vector &result, idx_t rid, idx_t start, idx_t end) override;
+
+private:
+	//! Partition starts
+	vector<idx_t> partition_offsets;
+	//! Aggregate results
+	unique_ptr<Vector> results;
+	//! The current result partition being built/read
+	idx_t partition;
+	//! The current input row being built/read
+	idx_t row;
+};
+
 class WindowSegmentTree {
 public:
 	using FrameBounds = std::pair<idx_t, idx_t>;

package/src/duckdb/src/include/duckdb/function/aggregate/distributive_functions.hpp

@@ -27,6 +27,11 @@ struct BitXorFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct BitStringAggFun {
+	static void GetBitStringAggregate(const LogicalType &type, AggregateFunctionSet &bitstring_agg);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct CountStarFun {
 	static AggregateFunction GetFunction();