duckdb 0.10.3-dev0.0 → 0.10.3-dev6.0

package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp

@@ -36,12 +36,14 @@ struct ExtraTypeInfo {
 
 	ExtraTypeInfoType type;
 	string alias;
+	vector<Value> modifiers;
 
 public:
 	bool Equals(ExtraTypeInfo *other_p) const;
 
 	virtual void Serialize(Serializer &serializer) const;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	virtual shared_ptr<ExtraTypeInfo> Copy() const;
 
 	template <class TARGET>
 	TARGET &Cast() {
@@ -67,6 +69,7 @@ struct DecimalTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -83,6 +86,7 @@ struct StringTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -99,6 +103,7 @@ struct ListTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -115,6 +120,7 @@ struct StructTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &deserializer);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -131,6 +137,7 @@ struct AggregateStateTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -141,15 +148,18 @@ private:
 
 struct UserTypeInfo : public ExtraTypeInfo {
 	explicit UserTypeInfo(string name_p);
-	UserTypeInfo(string catalog_p, string schema_p, string name_p);
+	UserTypeInfo(string name_p, vector<Value> modifiers_p);
+	UserTypeInfo(string catalog_p, string schema_p, string name_p, vector<Value> modifiers_p);
 
 	string catalog;
 	string schema;
 	string user_type_name;
+	vector<Value> user_type_modifiers;
 
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -176,6 +186,7 @@ public:
 
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	// Equalities are only used in enums with different catalog entries
@@ -196,6 +207,7 @@ struct ArrayTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &reader);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -210,6 +222,7 @@ struct AnyTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -226,6 +239,7 @@ struct IntegerLiteralTypeInfo : public ExtraTypeInfo {
 public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;

package/src/duckdb/src/include/duckdb/common/file_opener.hpp

@@ -14,6 +14,8 @@
 
 namespace duckdb {
 
+struct CatalogTransaction;
+class SecretManager;
 class ClientContext;
 class Value;
 
@@ -33,8 +35,10 @@ public:
 	virtual optional_ptr<ClientContext> TryGetClientContext() = 0;
 	virtual optional_ptr<DatabaseInstance> TryGetDatabase() = 0;
 
+	DUCKDB_API static unique_ptr<CatalogTransaction> TryGetCatalogTransaction(optional_ptr<FileOpener> opener);
 	DUCKDB_API static optional_ptr<ClientContext> TryGetClientContext(optional_ptr<FileOpener> opener);
 	DUCKDB_API static optional_ptr<DatabaseInstance> TryGetDatabase(optional_ptr<FileOpener> opener);
+	DUCKDB_API static optional_ptr<SecretManager> TryGetSecretManager(optional_ptr<FileOpener> opener);
 	DUCKDB_API static SettingLookupResult TryGetCurrentSetting(optional_ptr<FileOpener> opener, const string &key,
 	                                                           Value &result);
 	DUCKDB_API static SettingLookupResult TryGetCurrentSetting(optional_ptr<FileOpener> opener, const string &key,

package/src/duckdb/src/include/duckdb/common/file_system.hpp

@@ -16,6 +16,7 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/enums/file_glob_options.hpp"
 #include "duckdb/common/optional_ptr.hpp"
+#include "duckdb/common/optional_idx.hpp"
 #include "duckdb/common/error_data.hpp"
 #include "duckdb/common/file_open_flags.hpp"
 #include <functional>
@@ -172,6 +173,8 @@ public:
 	DUCKDB_API virtual string ExpandPath(const string &path);
 	//! Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails.
 	DUCKDB_API static optional_idx GetAvailableMemory();
+	//! Returns the space available on the disk. Returns DConstants::INVALID_INDEX if the information was not available.
+	DUCKDB_API static optional_idx GetAvailableDiskSpace(const string &path);
 	//! Path separator for path
 	DUCKDB_API virtual string PathSeparator(const string &path);
 	//! Checks if path is starts with separator (i.e., '/' on UNIX '\\' on Windows)

package/src/duckdb/src/include/duckdb/common/gzip_file_system.hpp

@@ -25,8 +25,11 @@ public:
 
 	//! Verifies that a buffer contains a valid GZIP header
 	static void VerifyGZIPHeader(uint8_t gzip_hdr[], idx_t read_count);
+	static bool CheckIsZip(const char *length, idx_t size);
+
 	//! Consumes a byte stream as a gzip string, returning the decompressed string
 	static string UncompressGZIPString(const string &in);
+	static string UncompressGZIPString(const char *length, idx_t size);
 
 	unique_ptr<StreamWrapper> CreateStream() override;
 	idx_t InBufferSize() override;

package/src/duckdb/src/include/duckdb/common/helper.hpp

@@ -65,6 +65,14 @@ make_uniq(ARGS&&... args) // NOLINT: mimic std style
     return unique_ptr<DATA_TYPE, std::default_delete<DATA_TYPE>, true>(new DATA_TYPE(std::forward<ARGS>(args)...));
 }
 
+template<class DATA_TYPE, class... ARGS>
+inline 
+shared_ptr<DATA_TYPE>
+make_shared_ptr(ARGS&&... args) // NOLINT: mimic std style
+{
+	return shared_ptr<DATA_TYPE>(std::make_shared<DATA_TYPE>(std::forward<ARGS>(args)...));
+}
+
 template<class DATA_TYPE, class... ARGS>
 inline 
 typename TemplatedUniqueIf<DATA_TYPE, false>::templated_unique_single_t
@@ -109,10 +117,15 @@ unique_ptr<S> unique_ptr_cast(unique_ptr<T> src) { // NOLINT: mimic std style
 	return unique_ptr<S>(static_cast<S *>(src.release()));
 }
 
+template <typename T, typename S>
+shared_ptr<S> shared_ptr_cast(shared_ptr<T> src) { // NOLINT: mimic std style
+	return shared_ptr<S>(std::static_pointer_cast<S, T>(src.internal));
+}
+
 struct SharedConstructor {
 	template <class T, typename... ARGS>
 	static shared_ptr<T> Create(ARGS &&...args) {
-		return make_shared<T>(std::forward<ARGS>(args)...);
+		return make_shared_ptr<T>(std::forward<ARGS>(args)...);
 	}
 };
 
@@ -139,6 +152,14 @@ static duckdb::unique_ptr<T> make_unique(ARGS&&... __args) { // NOLINT: mimic st
 	return unique_ptr<T>(new T(std::forward<ARGS>(__args)...));
 }
 
+template <class T, class... ARGS>
+static duckdb::shared_ptr<T> make_shared(ARGS&&... __args) { // NOLINT: mimic std style
+#ifndef DUCKDB_ENABLE_DEPRECATED_API
+	static_assert(sizeof(T) == 0, "Use make_shared_ptr instead of make_shared!");
+#endif // DUCKDB_ENABLE_DEPRECATED_API
+	return shared_ptr<T>(new T(std::forward<ARGS>(__args)...));
+}
+
 template <typename T>
 constexpr T MaxValue(T a, T b) {
 	return a > b ? a : b;

package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp

@@ -74,36 +74,22 @@ class GlobalHivePartitionState {
 public:
 	mutex lock;
 	hive_partition_map_t partition_map;
-	//! Used for incremental updating local copies of the partition map;
-	vector<hive_partition_map_t::const_iterator> partitions;
 };
 
 class HivePartitionedColumnData : public PartitionedColumnData {
 public:
 	HivePartitionedColumnData(ClientContext &context, vector<LogicalType> types, vector<idx_t> partition_by_cols,
-	                          shared_ptr<GlobalHivePartitionState> global_state = nullptr)
-	    : PartitionedColumnData(PartitionedColumnDataType::HIVE, context, std::move(types)),
-	      global_state(std::move(global_state)), group_by_columns(std::move(partition_by_cols)),
-	      hashes_v(LogicalType::HASH) {
-		InitializeKeys();
-	}
-	HivePartitionedColumnData(const HivePartitionedColumnData &other);
+	                          shared_ptr<GlobalHivePartitionState> global_state = nullptr);
 	void ComputePartitionIndices(PartitionedColumnDataAppendState &state, DataChunk &input) override;
 
 	//! Reverse lookup map to reconstruct keys from a partition id
 	std::map<idx_t, const HivePartitionKey *> GetReverseMap();
 
 protected:
-	//! Create allocators for all currently registered partitions
-	void GrowAllocators();
-	//! Create append states for all currently registered partitions
-	void GrowAppendState(PartitionedColumnDataAppendState &state);
-	//! Create and initialize partitions for all currently registered partitions
-	void GrowPartitions(PartitionedColumnDataAppendState &state);
 	//! Register a newly discovered partition
 	idx_t RegisterNewPartition(HivePartitionKey key, PartitionedColumnDataAppendState &state);
-	//! Copy the newly added entries in the global_state.map to the local_partition_map (requires lock!)
-	void SynchronizeLocalMap();
+	//! Add a new partition with the given partition id
+	void AddNewPartition(HivePartitionKey key, idx_t partition_id, PartitionedColumnDataAppendState &state);
 
 private:
 	void InitializeKeys();

package/src/duckdb/src/include/duckdb/common/http_state.hpp

@@ -20,7 +20,7 @@ namespace duckdb {
 class CachedFileHandle;
 
 //! Represents a file that is intended to be fully downloaded, then used in parallel by multiple threads
-class CachedFile : public std::enable_shared_from_this<CachedFile> {
+class CachedFile : public enable_shared_from_this<CachedFile> {
 	friend class CachedFileHandle;
 
 public:

package/src/duckdb/src/include/duckdb/common/insertion_order_preserving_map.hpp

@@ -0,0 +1,129 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/insertion_order_preserving_map.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/unordered_set.hpp"
+#include "duckdb/common/string.hpp"
+#include "duckdb/common/string_util.hpp"
+#include "duckdb/common/helper.hpp"
+#include "duckdb/common/case_insensitive_map.hpp"
+#include "duckdb/common/pair.hpp"
+
+namespace duckdb {
+
+template <typename V>
+class InsertionOrderPreservingMap {
+public:
+	typedef vector<pair<string, V>> VECTOR_TYPE; // NOLINT: matching name of std
+	typedef string key_type;                     // NOLINT: matching name of std
+
+public:
+	InsertionOrderPreservingMap() {
+	}
+
+private:
+	VECTOR_TYPE map;
+	case_insensitive_map_t<idx_t> map_idx;
+
+public:
+	vector<string> Keys() const {
+		vector<string> keys;
+		keys.resize(this->size());
+		for (auto &kv : map_idx) {
+			keys[kv.second] = kv.first;
+		}
+
+		return keys;
+	}
+
+	typename VECTOR_TYPE::iterator begin() { // NOLINT: match stl API
+		return map.begin();
+	}
+
+	typename VECTOR_TYPE::iterator end() { // NOLINT: match stl API
+		return map.end();
+	}
+
+	typename VECTOR_TYPE::const_iterator begin() const { // NOLINT: match stl API
+		return map.begin();
+	}
+
+	typename VECTOR_TYPE::const_iterator end() const { // NOLINT: match stl API
+		return map.end();
+	}
+
+	typename VECTOR_TYPE::iterator find(const string &key) { // NOLINT: match stl API
+		auto entry = map_idx.find(key);
+		if (entry == map_idx.end()) {
+			return map.end();
+		}
+		return map.begin() + static_cast<typename VECTOR_TYPE::difference_type>(entry->second);
+	}
+
+	typename VECTOR_TYPE::const_iterator find(const string &key) const { // NOLINT: match stl API
+		auto entry = map_idx.find(key);
+		if (entry == map_idx.end()) {
+			return map.end();
+		}
+		return map.begin() + static_cast<typename VECTOR_TYPE::difference_type>(entry->second);
+	}
+
+	idx_t size() const { // NOLINT: match stl API
+		return map_idx.size();
+	}
+
+	bool empty() const { // NOLINT: match stl API
+		return map_idx.empty();
+	}
+
+	void resize(idx_t nz) { // NOLINT: match stl API
+		map.resize(nz);
+	}
+
+	void insert(const string &key, V &value) { // NOLINT: match stl API
+		map.push_back(make_pair(key, std::move(value)));
+		map_idx[key] = map.size() - 1;
+	}
+
+	void insert(pair<string, V> &&value) { // NOLINT: match stl API
+		map.push_back(std::move(value));
+		map_idx[value.first] = map.size() - 1;
+	}
+
+	void erase(typename VECTOR_TYPE::iterator it) { // NOLINT: match stl API
+		auto key = it->first;
+		auto idx = map_idx[it->first];
+		map.erase(it);
+		map_idx.erase(key);
+		for (auto &kv : map_idx) {
+			if (kv.second > idx) {
+				kv.second--;
+			}
+		}
+	}
+
+	bool contains(const string &key) const { // NOLINT: match stl API
+		return map_idx.find(key) != map_idx.end();
+	}
+
+	const V &at(const string &key) const { // NOLINT: match stl API
+		return map[map_idx.at(key)].second;
+	}
+
+	V &operator[](const string &key) {
+		if (!contains(key)) {
+			auto v = V();
+			insert(key, v);
+		}
+		return map[map_idx[key]].second;
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/common/multi_file_list.hpp

@@ -0,0 +1,151 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/multi_file_list.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/common.hpp"
+#include "duckdb/common/multi_file_reader_options.hpp"
+
+namespace duckdb {
+class MultiFileList;
+
+enum class FileExpandResult : uint8_t { NO_FILES, SINGLE_FILE, MULTIPLE_FILES };
+
+struct MultiFileListScanData {
+	idx_t current_file_idx = DConstants::INVALID_INDEX;
+};
+
+class MultiFileListIterationHelper {
+public:
+	DUCKDB_API explicit MultiFileListIterationHelper(MultiFileList &collection);
+
+private:
+	MultiFileList &file_list;
+
+private:
+	class MultiFileListIterator;
+
+	class MultiFileListIterator {
+	public:
+		DUCKDB_API explicit MultiFileListIterator(MultiFileList *file_list);
+
+		optional_ptr<MultiFileList> file_list;
+		MultiFileListScanData file_scan_data;
+		string current_file;
+
+	public:
+		DUCKDB_API void Next();
+
+		DUCKDB_API MultiFileListIterator &operator++();
+		DUCKDB_API bool operator!=(const MultiFileListIterator &other) const;
+		DUCKDB_API const string &operator*() const;
+	};
+
+public:
+	MultiFileListIterator begin(); // NOLINT: match stl API
+	MultiFileListIterator end();   // NOLINT: match stl API
+};
+
+//! Abstract class for lazily generated list of file paths/globs
+//! NOTE: subclasses are responsible for ensuring thread-safety
+class MultiFileList {
+public:
+	explicit MultiFileList(vector<string> paths, FileGlobOptions options);
+	virtual ~MultiFileList();
+
+	//! Returns the raw, unexpanded paths, pre-filter
+	const vector<string> GetPaths() const;
+
+	//! Get Iterator over the files for pretty for loops
+	MultiFileListIterationHelper Files();
+
+	//! Initialize a sequential scan over a file list
+	void InitializeScan(MultiFileListScanData &iterator);
+	//! Scan the next file into result_file, returns false when out of files
+	bool Scan(MultiFileListScanData &iterator, string &result_file);
+
+	//! Returns the first file or an empty string if GetTotalFileCount() == 0
+	string GetFirstFile();
+	//! Syntactic sugar for GetExpandResult() == FileExpandResult::NO_FILES
+	bool IsEmpty();
+
+	//! Virtual functions for subclasses
+public:
+	virtual unique_ptr<MultiFileList> ComplexFilterPushdown(ClientContext &context,
+	                                                        const MultiFileReaderOptions &options, LogicalGet &get,
+	                                                        vector<unique_ptr<Expression>> &filters);
+	virtual vector<string> GetAllFiles() = 0;
+	virtual FileExpandResult GetExpandResult() = 0;
+	virtual idx_t GetTotalFileCount() = 0;
+
+protected:
+	//! Get the i-th expanded file
+	virtual string GetFile(idx_t i) = 0;
+
+protected:
+	//! The unexpanded input paths
+	const vector<string> paths;
+	//! Whether paths can expand to 0 files
+	const FileGlobOptions glob_options;
+};
+
+//! MultiFileList that takes a list of files and produces the same list of paths. Useful for quickly wrapping
+//! existing vectors of paths in a MultiFileList without changing any code
+class SimpleMultiFileList : public MultiFileList {
+public:
+	//! Construct a SimpleMultiFileList from a list of already expanded files
+	explicit SimpleMultiFileList(vector<string> paths);
+	//! Copy `paths` to `filtered_files` and apply the filters
+	unique_ptr<MultiFileList> ComplexFilterPushdown(ClientContext &context, const MultiFileReaderOptions &options,
+	                                                LogicalGet &get, vector<unique_ptr<Expression>> &filters) override;
+
+	//! Main MultiFileList API
+	vector<string> GetAllFiles() override;
+	FileExpandResult GetExpandResult() override;
+	idx_t GetTotalFileCount() override;
+
+protected:
+	//! Main MultiFileList API
+	string GetFile(idx_t i) override;
+};
+
+//! MultiFileList that takes a list of paths and produces a list of files with all globs expanded
+class GlobMultiFileList : public MultiFileList {
+public:
+	GlobMultiFileList(ClientContext &context, vector<string> paths, FileGlobOptions options);
+	//! Calls ExpandAll, then prunes the expanded_files using the hive/filename filters
+	unique_ptr<MultiFileList> ComplexFilterPushdown(ClientContext &context, const MultiFileReaderOptions &options,
+	                                                LogicalGet &get, vector<unique_ptr<Expression>> &filters) override;
+
+	//! Main MultiFileList API
+	vector<string> GetAllFiles() override;
+	FileExpandResult GetExpandResult() override;
+	idx_t GetTotalFileCount() override;
+
+protected:
+	//! Main MultiFileList API
+	string GetFile(idx_t i) override;
+
+	//! Get the i-th expanded file
+	string GetFileInternal(idx_t i);
+	//! Grabs the next path and expands it into Expanded paths: returns false if no more files to expand
+	bool ExpandPathInternal();
+	//! Whether all files have been expanded
+	bool IsFullyExpanded();
+
+	//! The ClientContext for globbing
+	ClientContext &context;
+	//! The current path to expand
+	idx_t current_path;
+	//! The expanded files
+	vector<string> expanded_files;
+
+	mutex lock;
+};
+
+} // namespace duckdb