duckdb 0.8.2-dev150.0 → 0.8.2-dev1549.0

package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp

@@ -22,6 +22,9 @@ struct SelectionVector;
 //! Generic radix partitioning functions
 struct RadixPartitioning {
 public:
+	//! 4096 partitions ought to be enough to go out-of-core properly
+	static constexpr const idx_t MAX_RADIX_BITS = 12;
+
 	//! The number of partitions for a given number of radix bits
 	static inline constexpr idx_t NumberOfPartitions(idx_t radix_bits) {
 		return idx_t(1) << radix_bits;
@@ -38,10 +41,12 @@ public:
 		throw InternalException("RadixPartitioning::RadixBits unable to find partition count!");
 	}
 
+	//! Radix bits begin after uint16_t because these bits are used as salt in the aggregate HT
 	static inline constexpr idx_t Shift(idx_t radix_bits) {
-		return 48 - radix_bits;
+		return (sizeof(hash_t) - sizeof(uint16_t)) * 8 - radix_bits;
 	}
 
+	//! Mask of the radix bits of the hash
 	static inline constexpr hash_t Mask(idx_t radix_bits) {
 		return (hash_t(1 << radix_bits) - 1) << Shift(radix_bits);
 	}
@@ -49,26 +54,6 @@ public:
 	//! Select using a cutoff on the radix bits of the hash
 	static idx_t Select(Vector &hashes, const SelectionVector *sel, idx_t count, idx_t radix_bits, idx_t cutoff,
 	                    SelectionVector *true_sel, SelectionVector *false_sel);
-
-	//! Convert hashes to bins
-	static void HashesToBins(Vector &hashes, idx_t radix_bits, Vector &bins, idx_t count);
-};
-
-//! Templated radix partitioning constants, can be templated to the number of radix bits
-template <idx_t radix_bits>
-struct RadixPartitioningConstants {
-public:
-	//! Bitmask of the upper bits of the 5th byte
-	static constexpr const idx_t NUM_PARTITIONS = RadixPartitioning::NumberOfPartitions(radix_bits);
-	static constexpr const idx_t SHIFT = RadixPartitioning::Shift(radix_bits);
-	static constexpr const hash_t MASK = RadixPartitioning::Mask(radix_bits);
-
-public:
-	//! Apply bitmask and right shift to get a number between 0 and NUM_PARTITIONS
-	static inline hash_t ApplyMask(hash_t hash) {
-		D_ASSERT((hash & MASK) >> SHIFT < NUM_PARTITIONS);
-		return (hash & MASK) >> SHIFT;
-	}
 };
 
 //! RadixPartitionedColumnData is a PartitionedColumnData that partitions input based on the radix of a hash

package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp

@@ -13,7 +13,7 @@
 
 namespace duckdb {
 
-class Allocator;
+class ArenaAllocator;
 struct AggregateObject;
 struct AggregateFilterData;
 class DataChunk;
@@ -26,10 +26,10 @@ class Vector;
 struct UnifiedVectorFormat;
 
 struct RowOperationsState {
-	RowOperationsState(Allocator &allocator) : allocator(allocator) {
+	explicit RowOperationsState(ArenaAllocator &allocator) : allocator(allocator) {
 	}
 
-	Allocator &allocator;
+	ArenaAllocator &allocator;
 };
 
 // RowOperations contains a set of operations that operate on data using a RowLayout

package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp

@@ -53,9 +53,12 @@ public:
 	                         const vector<BoundOrderByNode> &order_bys, const Types &payload_types,
 	                         const vector<unique_ptr<BaseStatistics>> &partitions_stats, idx_t estimated_cardinality);
 
+	void SyncPartitioning(const PartitionGlobalSinkState &other);
+
 	void UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
 	void CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
 
+	void BuildSortState(ColumnDataCollection &group_data, GlobalSortState &global_sort) const;
 	void BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &global_sort);
 
 	ClientContext &context;
@@ -67,6 +70,8 @@ public:
 	unique_ptr<RadixPartitionedColumnData> grouping_data;
 	//! Payload plus hash column
 	Types grouping_types;
+	//! The number of radix bits if this partition is being synced with another
+	idx_t fixed_bits;
 
 	// OVER(...) (sorting)
 	Orders partitions;
@@ -175,10 +180,18 @@ public:
 
 class PartitionGlobalMergeStates {
 public:
+	struct Callback {
+		virtual bool HasError() const {
+			return false;
+		}
+	};
+
 	using PartitionGlobalMergeStatePtr = unique_ptr<PartitionGlobalMergeState>;
 
 	explicit PartitionGlobalMergeStates(PartitionGlobalSinkState &sink);
 
+	bool ExecuteTask(PartitionLocalMergeState &local_state, Callback &callback);
+
 	vector<PartitionGlobalMergeStatePtr> states;
 };
 

package/src/duckdb/src/include/duckdb/common/types/batched_data_collection.hpp

@@ -24,7 +24,7 @@ struct BatchedChunkScanState {
 //! Scans over a BatchedDataCollection are ordered by batch index
 class BatchedDataCollection {
 public:
-	DUCKDB_API BatchedDataCollection(vector<LogicalType> types);
+	DUCKDB_API BatchedDataCollection(ClientContext &context, vector<LogicalType> types, bool buffer_managed = false);
 
 	//! Appends a datachunk with the given batch index to the batched collection
 	DUCKDB_API void Append(DataChunk &input, idx_t batch_index);
@@ -51,7 +51,9 @@ private:
 		ColumnDataAppendState append_state;
 	};
 
+	ClientContext &context;
 	vector<LogicalType> types;
+	bool buffer_managed;
 	//! The data of the batched chunk collection - a set of batch_index -> ColumnDataCollection pointers
 	map<idx_t, unique_ptr<ColumnDataCollection>> data;
 	//! The last batch collection that was inserted into

package/src/duckdb/src/include/duckdb/common/types/bit.hpp

@@ -8,8 +8,12 @@
 
 #pragma once
 
+#include "duckdb/common/assert.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/hugeint.hpp"
+#include "duckdb/common/limits.hpp"
 #include "duckdb/common/types.hpp"
+#include "duckdb/common/types/string_type.hpp"
 
 namespace duckdb {
 
@@ -37,7 +41,33 @@ public:
 	//! Convert a string to a bit. This function should ONLY be called after calling GetBitSize, since it does NOT
 	//! perform data validation.
 	DUCKDB_API static void ToBit(string_t str, string_t &output);
+
 	DUCKDB_API static string ToBit(string_t str);
+
+	//! output needs to have enough space allocated before calling this function (blob size + 1)
+	DUCKDB_API static void BlobToBit(string_t blob, string_t &output);
+
+	DUCKDB_API static string BlobToBit(string_t blob);
+
+	//! output_str needs to have enough space allocated before calling this function (sizeof(T) + 1)
+	template <class T>
+	static void NumericToBit(T numeric, string_t &output_str);
+
+	template <class T>
+	static string NumericToBit(T numeric);
+
+	//! bit is expected to fit inside of output num (bit size <= sizeof(T) + 1)
+	template <class T>
+	static void BitToNumeric(string_t bit, T &output_num);
+
+	template <class T>
+	static T BitToNumeric(string_t bit);
+
+	//! bit is expected to fit inside of output_blob (bit size = output_blob + 1)
+	static void BitToBlob(string_t bit, string_t &output_blob);
+
+	static string BitToBlob(string_t bit);
+
 	//! Creates a new bitstring of determined length
 	DUCKDB_API static void BitString(const string_t &input, const idx_t &len, string_t &result);
 	DUCKDB_API static void SetEmptyBitString(string_t &target, string_t &input);
@@ -58,5 +88,56 @@ private:
 	static idx_t GetBitInternal(string_t bit_string, idx_t n);
 	static void SetBitInternal(string_t &bit_string, idx_t n, idx_t new_value);
 	static idx_t GetBitIndex(idx_t n);
+	static uint8_t GetFirstByte(const string_t &str);
 };
+
+//===--------------------------------------------------------------------===//
+// Bit Template definitions
+//===--------------------------------------------------------------------===//
+template <class T>
+void Bit::NumericToBit(T numeric, string_t &output_str) {
+	D_ASSERT(output_str.GetSize() >= sizeof(T) + 1);
+
+	auto output = output_str.GetDataWriteable();
+	auto data = const_data_ptr_cast(&numeric);
+
+	*output = 0; // set padding to 0
+	++output;
+	for (idx_t idx = 0; idx < sizeof(T); ++idx) {
+		output[idx] = data[sizeof(T) - idx - 1];
+	}
+	Bit::Finalize(output_str);
+}
+
+template <class T>
+string Bit::NumericToBit(T numeric) {
+	auto bit_len = sizeof(T) + 1;
+	auto buffer = make_unsafe_uniq_array<char>(bit_len);
+	string_t output_str(buffer.get(), bit_len);
+	Bit::NumericToBit(numeric, output_str);
+	return output_str.GetString();
+}
+
+template <class T>
+T Bit::BitToNumeric(string_t bit) {
+	T output;
+	Bit::BitToNumeric(bit, output);
+	return (output);
+}
+
+template <class T>
+void Bit::BitToNumeric(string_t bit, T &output_num) {
+	D_ASSERT(bit.GetSize() <= sizeof(T) + 1);
+
+	output_num = 0;
+	auto data = const_data_ptr_cast(bit.GetData());
+	auto output = data_ptr_cast(&output_num);
+
+	idx_t padded_byte_idx = sizeof(T) - bit.GetSize() + 1;
+	output[sizeof(T) - 1 - padded_byte_idx] = GetFirstByte(bit);
+	for (idx_t idx = padded_byte_idx + 1; idx < sizeof(T); ++idx) {
+		output[sizeof(T) - 1 - idx] = data[1 + idx - padded_byte_idx];
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp

@@ -53,7 +53,7 @@ public:
 	void Initialize(ColumnDataAllocator &other);
 	void InitializeChunkState(ChunkManagementState &state, ChunkMetaData &meta_data);
 	data_ptr_t GetDataPointer(ChunkManagementState &state, uint32_t block_id, uint32_t offset);
-	void UnswizzlePointers(ChunkManagementState &state, Vector &result, uint16_t v_offset, uint16_t count,
+	void UnswizzlePointers(ChunkManagementState &state, Vector &result, idx_t v_offset, uint16_t count,
 	                       uint32_t block_id, uint32_t offset);
 
 	//! Deletes the block with the given id

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp

@@ -143,7 +143,12 @@ public:
 	//! Initialize the column data collection
 	void Initialize(vector<LogicalType> types);
 
-	//! Get a vector of references to every chunk (segment, index in segment), and optionally sort by block id
+	//! Get references to the string heaps in this ColumnDataCollection
+	vector<shared_ptr<StringHeap>> GetHeapReferences();
+	//! Get the allocator type of this ColumnDataCollection
+	ColumnDataAllocatorType GetAllocatorType() const;
+
+	//! Get a vector of the segments in this ColumnDataCollection
 	const vector<unique_ptr<ColumnDataCollectionSegment>> &GetSegments() const;
 
 private:

package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp

@@ -94,7 +94,7 @@ public:
 	//! The set of child indices
 	vector<VectorDataIndex> child_indices;
 	//! The string heap for the column data collection (only used for IN_MEMORY_ALLOCATOR)
-	StringHeap heap;
+	shared_ptr<StringHeap> heap;
 
 public:
 	void AllocateNewChunk();

package/src/duckdb/src/include/duckdb/common/types/column/column_data_scan_states.hpp

@@ -20,7 +20,9 @@ enum class ColumnDataAllocatorType : uint8_t {
 	BUFFER_MANAGER_ALLOCATOR,
 	//! Use an in-memory allocator, allocating data for every chunk
 	//! This causes the column data collection to allocate blocks that are not tied to a buffer manager
-	IN_MEMORY_ALLOCATOR
+	IN_MEMORY_ALLOCATOR,
+	//! Use a buffer manager to allocate vectors, but use a StringHeap for strings
+	HYBRID
 };
 
 enum class ColumnDataScanProperties : uint8_t {

package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp

@@ -153,7 +153,7 @@ public:
 
 	//! Converts this DataChunk to a printable string representation
 	DUCKDB_API string ToString() const;
-	DUCKDB_API void Print();
+	DUCKDB_API void Print() const;
 
 	DataChunk(const DataChunk &) = delete;
 

package/src/duckdb/src/include/duckdb/common/types/date.hpp

@@ -72,13 +72,13 @@ struct date_t { // NOLINT
 	};
 
 	// special values
-	static inline date_t infinity() {
+	static inline date_t infinity() { // NOLINT
 		return date_t(NumericLimits<int32_t>::Maximum());
-	} // NOLINT
-	static inline date_t ninfinity() {
+	}                                  // NOLINT
+	static inline date_t ninfinity() { // NOLINT
 		return date_t(-NumericLimits<int32_t>::Maximum());
-	} // NOLINT
-	static inline date_t epoch() {
+	}                              // NOLINT
+	static inline date_t epoch() { // NOLINT
 		return date_t(0);
 	} // NOLINT
 };
@@ -158,6 +158,8 @@ public:
 	DUCKDB_API static int64_t EpochNanoseconds(date_t date);
 	//! Extract the epoch from the date (microseconds since 1970-01-01)
 	DUCKDB_API static int64_t EpochMicroseconds(date_t date);
+	//! Extract the epoch from the date (milliseconds since 1970-01-01)
+	DUCKDB_API static int64_t EpochMilliseconds(date_t date);
 	//! Convert the epoch (seconds since 1970-01-01) to a date_t
 	DUCKDB_API static date_t EpochToDate(int64_t epoch);
 

package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp

@@ -34,28 +34,26 @@ struct LinkedList {
 
 // forward declarations
 struct ListSegmentFunctions;
-typedef ListSegment *(*create_segment_t)(const ListSegmentFunctions &functions, Allocator &allocator,
+typedef ListSegment *(*create_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
                                          uint16_t capacity);
-typedef void (*write_data_to_segment_t)(const ListSegmentFunctions &functions, Allocator &allocator,
+typedef void (*write_data_to_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
                                         ListSegment *segment, Vector &input, idx_t &entry_idx, idx_t &count);
 typedef void (*read_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *segment,
                                          Vector &result, idx_t &total_count);
 typedef ListSegment *(*copy_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *source,
-                                                 Allocator &allocator);
-typedef void (*destroy_segment_t)(const ListSegmentFunctions &functions, ListSegment *segment, Allocator &allocator);
+                                                 ArenaAllocator &allocator);
 
 struct ListSegmentFunctions {
 	create_segment_t create_segment;
 	write_data_to_segment_t write_data;
 	read_data_from_segment_t read_data;
 	copy_data_from_segment_t copy_data;
-	destroy_segment_t destroy;
 	vector<ListSegmentFunctions> child_functions;
 
-	void AppendRow(Allocator &allocator, LinkedList &linked_list, Vector &input, idx_t &entry_idx, idx_t &count) const;
+	void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, Vector &input, idx_t &entry_idx,
+	               idx_t &count) const;
 	void BuildListVector(const LinkedList &linked_list, Vector &result, idx_t &initial_total_count) const;
-	void CopyLinkedList(const LinkedList &source_list, LinkedList &target_list, Allocator &allocator) const;
-	void Destroy(Allocator &allocator, LinkedList &linked_list) const;
+	void CopyLinkedList(const LinkedList &source_list, LinkedList &target_list, ArenaAllocator &allocator) const;
 };
 
 void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type);

package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp

@@ -48,7 +48,6 @@ struct PartitionTupleDataAllocators {
 //! partitioning, e.g., radix, hive
 class PartitionedTupleData {
 public:
-	unique_ptr<PartitionedTupleData> CreateShared();
 	virtual ~PartitionedTupleData();
 
 public:

package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp

@@ -17,6 +17,7 @@ namespace duckdb {
 class TupleDataAllocator;
 struct TupleDataScatterFunction;
 struct TupleDataGatherFunction;
+struct RowOperationsState;
 
 typedef void (*tuple_data_scatter_function_t)(const Vector &source, const TupleDataVectorFormat &source_format,
                                               const SelectionVector &append_sel, const idx_t append_count,

package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_states.hpp

@@ -38,6 +38,9 @@ struct CombinedListData {
 };
 
 struct TupleDataVectorFormat {
+	const SelectionVector *original_sel;
+	SelectionVector original_owned_sel;
+
 	UnifiedVectorFormat data;
 	vector<TupleDataVectorFormat> child_formats;
 	unique_ptr<CombinedListData> combined_list_data;

package/src/duckdb/src/include/duckdb/common/types/string_type.hpp

@@ -83,6 +83,10 @@ public:
 		return value.pointer.prefix;
 	}
 
+	char *GetPrefixWriteable() const {
+		return (char *)value.pointer.prefix;
+	}
+
 	idx_t GetSize() const {
 		return value.inlined.length;
 	}
@@ -95,6 +99,11 @@ public:
 		return GetString();
 	}
 
+	char *GetPointer() const {
+		D_ASSERT(!IsInlined());
+		return value.pointer.ptr;
+	}
+
 	void SetPointer(char *new_ptr) {
 		D_ASSERT(!IsInlined());
 		value.pointer.ptr = new_ptr;

package/src/duckdb/src/include/duckdb/common/types.hpp

@@ -27,19 +27,6 @@ class Vector;
 class ClientContext;
 class FieldWriter;
 
-//! Extra Type Info Type
-enum class ExtraTypeInfoType : uint8_t {
-	INVALID_TYPE_INFO = 0,
-	GENERIC_TYPE_INFO = 1,
-	DECIMAL_TYPE_INFO = 2,
-	STRING_TYPE_INFO = 3,
-	LIST_TYPE_INFO = 4,
-	STRUCT_TYPE_INFO = 5,
-	ENUM_TYPE_INFO = 6,
-	USER_TYPE_INFO = 7,
-	AGGREGATE_STATE_TYPE_INFO = 8
-};
-
 struct string_t;
 
 template <class T>
@@ -328,8 +315,6 @@ struct LogicalType {
 
 	DUCKDB_API static LogicalType MaxLogicalType(const LogicalType &left, const LogicalType &right);
 
-	DUCKDB_API static ExtraTypeInfoType GetExtraTypeInfoType(const ExtraTypeInfo &type);
-
 	//! Gets the decimal properties of a numeric type. Fails if the type is not numeric.
 	DUCKDB_API bool GetDecimalProperties(uint8_t &width, uint8_t &scale) const;
 
@@ -480,6 +465,7 @@ bool ApproxEqual(float l, float r);
 bool ApproxEqual(double l, double r);
 
 struct aggregate_state_t {
+	aggregate_state_t() {}
 	aggregate_state_t(string function_name_p, LogicalType return_type_p, vector<LogicalType> bound_argument_types_p) : function_name(std::move(function_name_p)), return_type(std::move(return_type_p)), bound_argument_types(std::move(bound_argument_types_p)) {
 	}
 

package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/file_system.hpp"
 #include "duckdb/common/map.hpp"
+#include "duckdb/common/unordered_set.hpp"
 
 namespace duckdb {
 
@@ -22,120 +23,60 @@ public:
 	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
 	                                FileOpener *opener = nullptr) override;
 
-	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
-		handle.file_system.Read(handle, buffer, nr_bytes, location);
-	};
+	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
+	void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
 
-	void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
-		handle.file_system.Write(handle, buffer, nr_bytes, location);
-	}
+	int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
 
-	int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
-		return handle.file_system.Read(handle, buffer, nr_bytes);
-	}
+	int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
 
-	int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
-		return handle.file_system.Write(handle, buffer, nr_bytes);
-	}
+	int64_t GetFileSize(FileHandle &handle) override;
+	time_t GetLastModifiedTime(FileHandle &handle) override;
+	FileType GetFileType(FileHandle &handle) override;
 
-	int64_t GetFileSize(FileHandle &handle) override {
-		return handle.file_system.GetFileSize(handle);
-	}
-	time_t GetLastModifiedTime(FileHandle &handle) override {
-		return handle.file_system.GetLastModifiedTime(handle);
-	}
-	FileType GetFileType(FileHandle &handle) override {
-		return handle.file_system.GetFileType(handle);
-	}
+	void Truncate(FileHandle &handle, int64_t new_size) override;
 
-	void Truncate(FileHandle &handle, int64_t new_size) override {
-		handle.file_system.Truncate(handle, new_size);
-	}
-
-	void FileSync(FileHandle &handle) override {
-		handle.file_system.FileSync(handle);
-	}
+	void FileSync(FileHandle &handle) override;
 
 	// need to look up correct fs for this
-	bool DirectoryExists(const string &directory) override {
-		return FindFileSystem(directory)->DirectoryExists(directory);
-	}
-	void CreateDirectory(const string &directory) override {
-		FindFileSystem(directory)->CreateDirectory(directory);
-	}
+	bool DirectoryExists(const string &directory) override;
+	void CreateDirectory(const string &directory) override;
 
-	void RemoveDirectory(const string &directory) override {
-		FindFileSystem(directory)->RemoveDirectory(directory);
-	}
+	void RemoveDirectory(const string &directory) override;
 
 	bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
-	               FileOpener *opener = nullptr) override {
-		return FindFileSystem(directory)->ListFiles(directory, callback, opener);
-	}
-
-	void MoveFile(const string &source, const string &target) override {
-		FindFileSystem(source)->MoveFile(source, target);
-	}
-
-	bool FileExists(const string &filename) override {
-		return FindFileSystem(filename)->FileExists(filename);
-	}
-
-	bool IsPipe(const string &filename) override {
-		return FindFileSystem(filename)->IsPipe(filename);
-	}
-	virtual void RemoveFile(const string &filename) override {
-		FindFileSystem(filename)->RemoveFile(filename);
-	}
-
-	virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override {
-		return FindFileSystem(path)->Glob(path, opener);
-	}
-
-	void RegisterSubSystem(unique_ptr<FileSystem> fs) override {
-		sub_systems.push_back(std::move(fs));
-	}
-
-	void UnregisterSubSystem(const string &name) override {
-		for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
-			if (sub_system->get()->GetName() == name) {
-				sub_systems.erase(sub_system);
-				return;
-			}
-		}
-		throw InvalidInputException("Could not find filesystem with name %s", name);
-	}
-
-	void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override {
-		compressed_fs[compression_type] = std::move(fs);
-	}
-
-	vector<string> ListSubSystems() override {
-		vector<string> names(sub_systems.size());
-		for (idx_t i = 0; i < sub_systems.size(); i++) {
-			names[i] = sub_systems[i]->GetName();
-		}
-		return names;
-	}
-
-	std::string GetName() const override {
-		return "VirtualFileSystem";
-	}
+	               FileOpener *opener = nullptr) override;
+
+	void MoveFile(const string &source, const string &target) override;
+
+	bool FileExists(const string &filename) override;
+
+	bool IsPipe(const string &filename) override;
+	virtual void RemoveFile(const string &filename) override;
+
+	virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override;
+
+	void RegisterSubSystem(unique_ptr<FileSystem> fs) override;
+
+	void UnregisterSubSystem(const string &name) override;
+
+	void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override;
+
+	vector<string> ListSubSystems() override;
+
+	std::string GetName() const override;
+
+	void SetDisabledFileSystems(const vector<string> &names) override;
 
 private:
-	FileSystem *FindFileSystem(const string &path) {
-		for (auto &sub_system : sub_systems) {
-			if (sub_system->CanHandleFile(path)) {
-				return sub_system.get();
-			}
-		}
-		return default_fs.get();
-	}
+	FileSystem &FindFileSystem(const string &path);
+	FileSystem &FindFileSystemInternal(const string &path);
 
 private:
 	vector<unique_ptr<FileSystem>> sub_systems;
 	map<FileCompressionType, unique_ptr<FileSystem>> compressed_fs;
 	const unique_ptr<FileSystem> default_fs;
+	unordered_set<string> disabled_file_systems;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/corr.hpp

@@ -31,10 +31,10 @@ struct CorrOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x_input, const B_TYPE &y_input, AggregateBinaryInput &idata) {
-		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, x_input, y_input, idata);
-		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.dev_pop_x, x_input);
-		STDDevBaseOperation::Execute<B_TYPE, StddevState>(state.dev_pop_y, y_input);
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
+		CovarOperation::Operation<A_TYPE, B_TYPE, CovarState, OP>(state.cov_pop, y, x, idata);
+		STDDevBaseOperation::Execute<A_TYPE, StddevState>(state.dev_pop_x, x);
+		STDDevBaseOperation::Execute<B_TYPE, StddevState>(state.dev_pop_y, y);
 	}
 
 	template <class STATE, class OP>

package/src/duckdb/src/include/duckdb/core_functions/aggregate/algebraic/covar.hpp

@@ -5,6 +5,7 @@
 //
 //
 //===----------------------------------------------------------------------===//
+// COVAR_POP(y,x)
 
 #pragma once
 
@@ -29,7 +30,7 @@ struct CovarOperation {
 	}
 
 	template <class A_TYPE, class B_TYPE, class STATE, class OP>
-	static void Operation(STATE &state, const A_TYPE &x, const B_TYPE &y, AggregateBinaryInput &idata) {
+	static void Operation(STATE &state, const A_TYPE &y, const B_TYPE &x, AggregateBinaryInput &idata) {
 		// update running mean and d^2
 		const uint64_t n = ++(state.count);
 
@@ -39,6 +40,7 @@ struct CovarOperation {
 		const double dy = (y - state.meany);
 		const double meany = state.meany + dy / n;
 
+		//  Schubert and Gertz SSDBM 2018 (4.3)
 		const double C = state.co_moment + dx * (y - meany);
 
 		state.meanx = meanx;