duckdb 1.3.3-dev0.0 → 1.3.3-dev4.0

package/src/duckdb/src/include/duckdb/common/exception_format_value.hpp

@@ -15,6 +15,8 @@
 
 namespace duckdb {
 
+class String;
+
 // Helper class to support custom overloading
 // Escaping " and quoting the value with "
 class SQLIdentifier {
@@ -49,14 +51,16 @@ enum class ExceptionFormatValueType : uint8_t {
 struct ExceptionFormatValue {
 	DUCKDB_API ExceptionFormatValue(double dbl_val);     // NOLINT
 	DUCKDB_API ExceptionFormatValue(int64_t int_val);    // NOLINT
+	DUCKDB_API ExceptionFormatValue(idx_t uint_val);     // NOLINT
 	DUCKDB_API ExceptionFormatValue(string str_val);     // NOLINT
+	DUCKDB_API ExceptionFormatValue(String str_val);     // NOLINT
 	DUCKDB_API ExceptionFormatValue(hugeint_t hg_val);   // NOLINT
 	DUCKDB_API ExceptionFormatValue(uhugeint_t uhg_val); // NOLINT
 
 	ExceptionFormatValueType type;
 
 	double dbl_val = 0;
-	int64_t int_val = 0;
+	hugeint_t int_val = 0;
 	string str_val;
 
 public:
@@ -82,10 +86,14 @@ DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(double v
 template <>
 DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(string value);
 template <>
+DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(String value);
+template <>
 DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(const char *value);
 template <>
 DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(char *value);
 template <>
+DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(idx_t value);
+template <>
 DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(hugeint_t value);
 template <>
 DUCKDB_API ExceptionFormatValue ExceptionFormatValue::CreateFormatValue(uhugeint_t value);

package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp

@@ -27,7 +27,8 @@ enum class ExtraTypeInfoType : uint8_t {
 	AGGREGATE_STATE_TYPE_INFO = 8,
 	ARRAY_TYPE_INFO = 9,
 	ANY_TYPE_INFO = 10,
-	INTEGER_LITERAL_TYPE_INFO = 11
+	INTEGER_LITERAL_TYPE_INFO = 11,
+	TEMPLATE_TYPE_INFO = 12
 };
 
 struct ExtraTypeInfo {
@@ -50,6 +51,7 @@ public:
 	virtual void Serialize(Serializer &serializer) const;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 	virtual shared_ptr<ExtraTypeInfo> Copy() const;
+	virtual shared_ptr<ExtraTypeInfo> DeepCopy() const;
 
 	template <class TARGET>
 	TARGET &Cast() {
@@ -110,6 +112,7 @@ public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 	shared_ptr<ExtraTypeInfo> Copy() const override;
+	shared_ptr<ExtraTypeInfo> DeepCopy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -127,6 +130,7 @@ public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &deserializer);
 	shared_ptr<ExtraTypeInfo> Copy() const override;
+	shared_ptr<ExtraTypeInfo> DeepCopy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -214,6 +218,7 @@ public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &reader);
 	shared_ptr<ExtraTypeInfo> Copy() const override;
+	shared_ptr<ExtraTypeInfo> DeepCopy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -229,6 +234,7 @@ public:
 	void Serialize(Serializer &serializer) const override;
 	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
 	shared_ptr<ExtraTypeInfo> Copy() const override;
+	shared_ptr<ExtraTypeInfo> DeepCopy() const override;
 
 protected:
 	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
@@ -254,4 +260,22 @@ private:
 	IntegerLiteralTypeInfo();
 };
 
+struct TemplateTypeInfo : public ExtraTypeInfo {
+
+	explicit TemplateTypeInfo(string name_p);
+
+	// The name of the template, e.g. `T`, or `KEY_TYPE`. Used to distinguish between different template types within
+	// the same function. The binder tries to resolve all templates with the same name to the same concrete type.
+	string name;
+
+public:
+	void Serialize(Serializer &serializer) const override;
+	static shared_ptr<ExtraTypeInfo> Deserialize(Deserializer &source);
+	shared_ptr<ExtraTypeInfo> Copy() const override;
+
+protected:
+	bool EqualsInternal(ExtraTypeInfo *other_p) const override;
+	TemplateTypeInfo();
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/file_buffer.hpp

@@ -12,8 +12,11 @@
 #include "duckdb/common/enums/debug_initialize.hpp"
 
 namespace duckdb {
+
 class Allocator;
 class BlockManager;
+class QueryContext;
+
 struct FileHandle;
 
 enum class FileBufferType : uint8_t { BLOCK = 1, MANAGED_BUFFER = 2, TINY_BUFFER = 3, EXTERNAL_FILE = 4 };
@@ -29,7 +32,7 @@ public:
 	//! DIRECT_IO
 	FileBuffer(Allocator &allocator, FileBufferType type, uint64_t user_size, idx_t block_header_size);
 	FileBuffer(Allocator &allocator, FileBufferType type, BlockManager &block_manager);
-	FileBuffer(FileBuffer &source, FileBufferType type);
+	FileBuffer(FileBuffer &source, FileBufferType type, idx_t block_header_size);
 
 	virtual ~FileBuffer();
 
@@ -41,10 +44,10 @@ public:
 	uint64_t size;
 
 public:
-	//! Read into the FileBuffer from the specified location.
-	void Read(FileHandle &handle, uint64_t location);
-	//! Write the contents of the FileBuffer to the specified location.
-	void Write(FileHandle &handle, uint64_t location);
+	//! Read into the FileBuffer from the location.
+	void Read(QueryContext context, FileHandle &handle, uint64_t location);
+	//! Write the FileBuffer to the location.
+	void Write(QueryContext context, FileHandle &handle, const uint64_t location);
 
 	void Clear();
 
@@ -54,10 +57,13 @@ public:
 
 	// Same rules as the constructor. We add room for a header, in addition to
 	// the requested user bytes. We then sector-align the result.
-	void ResizeInternal(uint64_t user_size, uint64_t block_header_size);
 	void Resize(uint64_t user_size, BlockManager &block_manager);
 	void Resize(BlockManager &block_manager);
 
+	idx_t GetHeaderSize() const {
+		return internal_size - size;
+	}
+
 	uint64_t AllocSize() const {
 		return internal_size;
 	}
@@ -88,6 +94,9 @@ protected:
 
 	void ReallocBuffer(idx_t new_size);
 	void Init();
+
+private:
+	void ResizeInternal(uint64_t user_size, uint64_t block_header_size);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/file_open_flags.hpp

@@ -28,6 +28,8 @@ public:
 	static constexpr idx_t FILE_FLAGS_PARALLEL_ACCESS = idx_t(1 << 8);
 	static constexpr idx_t FILE_FLAGS_EXCLUSIVE_CREATE = idx_t(1 << 9);
 	static constexpr idx_t FILE_FLAGS_NULL_IF_EXISTS = idx_t(1 << 10);
+	static constexpr idx_t FILE_FLAGS_MULTI_CLIENT_ACCESS = idx_t(1 << 11);
+	static constexpr idx_t FILE_FLAGS_DISABLE_LOGGING = idx_t(1 << 12);
 
 public:
 	FileOpenFlags() = default;
@@ -107,6 +109,12 @@ public:
 	inline bool ReturnNullIfExists() const {
 		return flags & FILE_FLAGS_NULL_IF_EXISTS;
 	}
+	inline bool MultiClientAccess() const {
+		return flags & FILE_FLAGS_MULTI_CLIENT_ACCESS;
+	}
+	inline bool DisableLogging() const {
+		return flags & FILE_FLAGS_DISABLE_LOGGING;
+	}
 	inline idx_t GetFlagsInternal() const {
 		return flags;
 	}
@@ -145,6 +153,12 @@ public:
 	    FileOpenFlags(FileOpenFlags::FILE_FLAGS_EXCLUSIVE_CREATE);
 	//!  Return NULL if the file exist instead of throwing an error
 	static constexpr FileOpenFlags FILE_FLAGS_NULL_IF_EXISTS = FileOpenFlags(FileOpenFlags::FILE_FLAGS_NULL_IF_EXISTS);
+	//! Multiple clients may access the file at the same time
+	static constexpr FileOpenFlags FILE_FLAGS_MULTI_CLIENT_ACCESS =
+	    FileOpenFlags(FileOpenFlags::FILE_FLAGS_MULTI_CLIENT_ACCESS);
+	//! Disables logging to avoid infinite loops when using FileHandle-backed log storage
+	static constexpr FileOpenFlags FILE_FLAGS_DISABLE_LOGGING =
+	    FileOpenFlags(FileOpenFlags::FILE_FLAGS_DISABLE_LOGGING);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/file_opener.hpp

@@ -10,13 +10,14 @@
 
 #include "duckdb/common/string.hpp"
 #include "duckdb/common/winapi.hpp"
-#include "duckdb/main/settings.hpp"
+#include "duckdb/main/setting_info.hpp"
 
 namespace duckdb {
 
 struct CatalogTransaction;
 class SecretManager;
 class ClientContext;
+class HTTPUtil;
 class Value;
 class Logger;
 

package/src/duckdb/src/include/duckdb/common/file_system.hpp

@@ -27,13 +27,14 @@
 #undef RemoveDirectory
 
 namespace duckdb {
+
 class AttachedDatabase;
-class ClientContext;
 class DatabaseInstance;
 class FileOpener;
 class FileSystem;
-class FileHandleLogger;
 class Logger;
+class ClientContext;
+class QueryContext;
 
 enum class FileType {
 	//! Regular file
@@ -63,17 +64,20 @@ public:
 	// Read at [nr_bytes] bytes into [buffer], and return the bytes actually read.
 	// File offset will be changed, which advances for number of bytes read.
 	DUCKDB_API int64_t Read(void *buffer, idx_t nr_bytes);
+	DUCKDB_API int64_t Read(QueryContext context, void *buffer, idx_t nr_bytes);
 	DUCKDB_API int64_t Write(void *buffer, idx_t nr_bytes);
 	// Read at [nr_bytes] bytes into [buffer].
 	// File offset will not be changed.
 	DUCKDB_API void Read(void *buffer, idx_t nr_bytes, idx_t location);
-	DUCKDB_API void Write(void *buffer, idx_t nr_bytes, idx_t location);
+	DUCKDB_API void Read(QueryContext context, void *buffer, idx_t nr_bytes, idx_t location);
+	DUCKDB_API void Write(QueryContext context, void *buffer, idx_t nr_bytes, idx_t location);
 	DUCKDB_API void Seek(idx_t location);
 	DUCKDB_API void Reset();
 	DUCKDB_API idx_t SeekPosition();
 	DUCKDB_API void Sync();
 	DUCKDB_API void Truncate(int64_t new_size);
 	DUCKDB_API string ReadLine();
+	DUCKDB_API string ReadLine(QueryContext context);
 	DUCKDB_API bool Trim(idx_t offset_bytes, idx_t length_bytes);
 	DUCKDB_API virtual idx_t GetProgress();
 	DUCKDB_API virtual FileCompressionType GetFileCompressionType();
@@ -148,7 +152,7 @@ public:
 	//! Returns the file size of a file handle, returns -1 on error
 	DUCKDB_API virtual int64_t GetFileSize(FileHandle &handle);
 	//! Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error
-	DUCKDB_API virtual time_t GetLastModifiedTime(FileHandle &handle);
+	DUCKDB_API virtual timestamp_t GetLastModifiedTime(FileHandle &handle);
 	//! Returns a tag that uniquely identifies the version of the file,
 	//! used for checking cache invalidation for CachingFileSystem httpfs files
 	DUCKDB_API virtual string GetVersionTag(FileHandle &handle);
@@ -217,6 +221,8 @@ public:
 	DUCKDB_API string ConvertSeparators(const string &path);
 	//! Extract the base name of a file (e.g. if the input is lib/example.dll the base name is 'example')
 	DUCKDB_API string ExtractBaseName(const string &path);
+	//! Extract the extension of a file (e.g. if the input is lib/example.dll the extension is 'dll')
+	DUCKDB_API string ExtractExtension(const string &path);
 	//! Extract the name of a file (e.g if the input is lib/example.dll the name is 'example.dll')
 	DUCKDB_API string ExtractName(const string &path);
 
@@ -228,7 +234,7 @@ public:
 	//! Runs a glob on the file system, returning a list of matching files
 	DUCKDB_API virtual vector<OpenFileInfo> Glob(const string &path, FileOpener *opener = nullptr);
 	DUCKDB_API vector<OpenFileInfo> GlobFiles(const string &path, ClientContext &context,
-	                                          FileGlobOptions options = FileGlobOptions::DISALLOW_EMPTY);
+	                                          const FileGlobInput &input = FileGlobOptions::DISALLOW_EMPTY);
 
 	//! registers a sub-file system to handle certain file name prefixes, e.g. http:// etc.
 	DUCKDB_API virtual void RegisterSubSystem(unique_ptr<FileSystem> sub_fs);
@@ -261,7 +267,8 @@ public:
 	//! in a file on-disk are much cheaper than e.g. random reads in a file over the network
 	DUCKDB_API virtual bool OnDiskFile(FileHandle &handle);
 
-	DUCKDB_API virtual unique_ptr<FileHandle> OpenCompressedFile(unique_ptr<FileHandle> handle, bool write);
+	DUCKDB_API virtual unique_ptr<FileHandle> OpenCompressedFile(QueryContext context, unique_ptr<FileHandle> handle,
+	                                                             bool write);
 
 	//! Create a LocalFileSystem.
 	DUCKDB_API static unique_ptr<FileSystem> CreateLocal();
@@ -274,6 +281,7 @@ public:
 	DUCKDB_API static bool IsRemoteFile(const string &path, string &extension);
 
 	DUCKDB_API virtual void SetDisabledFileSystems(const vector<string> &names);
+	DUCKDB_API virtual bool SubSystemIsDisabled(const string &name);
 
 	DUCKDB_API static bool IsDirectory(const OpenFileInfo &info);
 

package/src/duckdb/src/include/duckdb/common/fsst.hpp

@@ -28,7 +28,8 @@ private:
 	// The decode will overflow a bit into "extra_space", but "str" will contain the full string
 	struct StringWithExtraSpace {
 		string_t str;
-		uint64_t extra_space[string_t::INLINE_LENGTH];
+		// INLINE_BYTES instead of INLINE_LENGTH so this isn't 0-length array when building with DUCKDB_DEBUG_NO_INLINE
+		uint64_t extra_space[string_t::INLINE_BYTES];
 	};
 
 public:
@@ -56,7 +57,7 @@ public:
 			                  decompressed_string_size, string_t::INLINE_LENGTH);
 		}
 		D_ASSERT(decompressed_string_size <= string_t::INLINE_LENGTH);
-		result.str.SetSizeAndFinalize(UnsafeNumericCast<uint32_t>(decompressed_string_size));
+		result.str.SetSizeAndFinalize(UnsafeNumericCast<uint32_t>(decompressed_string_size), string_t::INLINE_LENGTH);
 		return result.str;
 	}
 	static string DecompressValue(void *duckdb_fsst_decoder, const char *compressed_string,

package/src/duckdb/src/include/duckdb/common/gzip_file_system.hpp

@@ -17,7 +17,7 @@ class GZipFileSystem : public CompressedFileSystem {
 	static constexpr const idx_t BUFFER_SIZE = 1u << 15;
 
 public:
-	unique_ptr<FileHandle> OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) override;
+	unique_ptr<FileHandle> OpenCompressedFile(QueryContext context, unique_ptr<FileHandle> handle, bool write) override;
 
 	std::string GetName() const override {
 		return "GZipFileSystem";

package/src/duckdb/src/include/duckdb/common/helper.hpp

@@ -58,7 +58,7 @@ struct TemplatedUniqueIf<DATA_TYPE[N]>
 };
 
 template<class DATA_TYPE, class... ARGS>
-inline 
+inline
 typename TemplatedUniqueIf<DATA_TYPE, true>::templated_unique_single_t
 make_uniq(ARGS&&... args) // NOLINT: mimic std style
 {
@@ -66,7 +66,7 @@ make_uniq(ARGS&&... args) // NOLINT: mimic std style
 }
 
 template<class DATA_TYPE, class... ARGS>
-inline 
+inline
 shared_ptr<DATA_TYPE>
 make_shared_ptr(ARGS&&... args) // NOLINT: mimic std style
 {
@@ -74,7 +74,7 @@ make_shared_ptr(ARGS&&... args) // NOLINT: mimic std style
 }
 
 template<class DATA_TYPE, class... ARGS>
-inline 
+inline
 typename TemplatedUniqueIf<DATA_TYPE, false>::templated_unique_single_t
 make_unsafe_uniq(ARGS&&... args) // NOLINT: mimic std style
 {
@@ -201,6 +201,11 @@ static inline T AlignValue(T n) {
 	return ((n + (val - 1)) / val) * val;
 }
 
+template <class T>
+static T AlignValue(T n, T val) {
+	return ((n + (val - 1)) / val) * val;
+}
+
 template<uintptr_t alignment, class T>
 inline T *AlignPointer(T *addr) {
 	static_assert((alignment & (alignment - 1)) == 0, "'alignment' has to be a power of 2");
@@ -269,4 +274,17 @@ void DynamicCastCheck(const SRC *source) {
 #endif
 }
 
+//! Used to increment counters that need to be exception-proof
+template<typename T>
+class PostIncrement {
+public:
+	explicit PostIncrement(T &t) : t(t) {
+	}
+	~PostIncrement() {
+		++t;
+	}
+private:
+	T &t;
+};
+
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/local_file_system.hpp

@@ -37,7 +37,7 @@ public:
 	//! Returns the file size of a file handle, returns -1 on error
 	int64_t GetFileSize(FileHandle &handle) override;
 	//! Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error
-	time_t GetLastModifiedTime(FileHandle &handle) override;
+	timestamp_t GetLastModifiedTime(FileHandle &handle) override;
 	//! Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error
 	FileType GetFileType(FileHandle &handle) override;
 	//! Truncate a file to a maximum size of new_size, new_size should be smaller than or equal to the current size of

package/src/duckdb/src/include/duckdb/common/multi_file/base_file_reader.hpp

@@ -18,6 +18,8 @@
 #include "duckdb/common/open_file_info.hpp"
 
 namespace duckdb {
+
+class BaseStatistics;
 class BaseUnionData;
 struct GlobalTableFunctionState;
 struct LocalTableFunctionState;
@@ -134,6 +136,8 @@ public:
 		return file.path;
 	}
 
+	virtual unique_ptr<BaseStatistics> GetStatistics(ClientContext &context, const string &name);
+
 public:
 	template <class TARGET>
 	TARGET &Cast() {

package/src/duckdb/src/include/duckdb/common/multi_file/multi_file_function.hpp

@@ -20,6 +20,7 @@ namespace duckdb {
 struct MultiFileReaderInterface {
 	virtual ~MultiFileReaderInterface();
 
+	virtual void InitializeInterface(ClientContext &context, MultiFileReader &reader, MultiFileList &file_list);
 	virtual unique_ptr<BaseFileReaderOptions> InitializeOptions(ClientContext &context,
 	                                                            optional_ptr<TableFunctionInfo> info) = 0;
 	virtual bool ParseCopyOption(ClientContext &context, const string &key, const vector<Value> &values,
@@ -55,6 +56,7 @@ struct MultiFileReaderInterface {
 	virtual unique_ptr<NodeStatistics> GetCardinality(const MultiFileBindData &bind_data, idx_t file_count) = 0;
 	virtual void GetVirtualColumns(ClientContext &context, MultiFileBindData &bind_data, virtual_column_map_t &result);
 	virtual unique_ptr<MultiFileReaderInterface> Copy();
+	virtual FileGlobInput GetGlobInput();
 };
 
 template <class OP>
@@ -159,10 +161,13 @@ public:
 
 	static unique_ptr<FunctionData> MultiFileBind(ClientContext &context, TableFunctionBindInput &input,
 	                                              vector<LogicalType> &return_types, vector<string> &names) {
+		auto interface = OP::CreateInterface(context);
 		auto multi_file_reader = MultiFileReader::Create(input.table_function);
-		auto file_list = multi_file_reader->CreateFileList(context, input.inputs[0]);
 
-		auto interface = OP::InitializeInterface(context, *multi_file_reader, *file_list);
+		auto glob_input = multi_file_reader->GetGlobInput(*interface);
+		auto file_list = multi_file_reader->CreateFileList(context, input.inputs[0], glob_input);
+
+		interface->InitializeInterface(context, *multi_file_reader, *file_list);
 
 		MultiFileOptions file_options;
 
@@ -181,19 +186,22 @@ public:
 		                             std::move(file_options), std::move(options), std::move(interface));
 	}
 
-	static unique_ptr<FunctionData> MultiFileBindCopy(ClientContext &context, CopyInfo &info,
+	static unique_ptr<FunctionData> MultiFileBindCopy(ClientContext &context, CopyFromFunctionBindInput &input,
 	                                                  vector<string> &expected_names,
 	                                                  vector<LogicalType> &expected_types) {
+		auto interface = OP::CreateInterface(context);
 		auto multi_file_reader = MultiFileReader::CreateDefault("COPY");
-		vector<string> paths = {info.file_path};
-		auto file_list = multi_file_reader->CreateFileList(context, paths);
+		vector<string> paths = {input.info.file_path};
+		auto glob_input = multi_file_reader->GetGlobInput(*interface);
+		auto file_list = multi_file_reader->CreateFileList(context, paths, glob_input);
 
-		auto interface = OP::InitializeInterface(context, *multi_file_reader, *file_list);
+		interface->InitializeInterface(context, *multi_file_reader, *file_list);
 
 		auto options = interface->InitializeOptions(context, nullptr);
 		MultiFileOptions file_options;
+		file_options.auto_detect_hive_partitioning = false;
 
-		for (auto &option : info.options) {
+		for (auto &option : input.info.options) {
 			auto loption = StringUtil::Lower(option.first);
 			if (interface->ParseCopyOption(context, loption, option.second, *options, expected_names, expected_types)) {
 				continue;
@@ -369,7 +377,7 @@ public:
 			}
 		}
 		lstate.scan_chunk.Destroy();
-		lstate.scan_chunk.Initialize(context, intermediate_chunk_types);
+		lstate.scan_chunk.Initialize(BufferAllocator::Get(context), intermediate_chunk_types);
 
 		auto &executor = lstate.executor;
 		executor.ClearExpressions();
@@ -501,6 +509,7 @@ public:
 		result->file_index = 0;
 		result->column_indexes = input.column_indexes;
 		result->filters = input.filters.get();
+		result->op = input.op;
 		result->global_state = bind_data.interface->InitializeGlobalState(context, bind_data, *result);
 		result->max_threads = NumericCast<idx_t>(TaskScheduler::GetScheduler(context).NumberOfThreads());
 
@@ -610,11 +619,6 @@ public:
 	                                                     column_t column_index) {
 		auto &bind_data = bind_data_p->Cast<MultiFileBindData>();
 
-		// NOTE: we do not want to parse the file metadata for the sole purpose of getting column statistics
-		if (bind_data.file_list->GetExpandResult() == FileExpandResult::MULTIPLE_FILES) {
-			// multiple files, no luck!
-			return nullptr;
-		}
 		if (!bind_data.initial_reader) {
 			// no reader
 			return nullptr;
@@ -623,7 +627,32 @@ public:
 		if (IsVirtualColumn(column_index)) {
 			return nullptr;
 		}
-		return bind_data.initial_reader->GetStatistics(context, bind_data.names[column_index]);
+
+		const auto &col_name = bind_data.names[column_index];
+
+		// NOTE: we do not want to parse the file metadata for the sole purpose of getting column statistics
+		if (bind_data.file_list->GetExpandResult() == FileExpandResult::MULTIPLE_FILES) {
+			if (!bind_data.file_options.union_by_name) {
+				// multiple files, but no union_by_name: no luck!
+				return nullptr;
+			}
+
+			auto merged_stats = bind_data.initial_reader->GetStatistics(context, col_name);
+			if (!merged_stats) {
+				return nullptr;
+			}
+
+			for (idx_t i = 1; i < bind_data.union_readers.size(); i++) {
+				auto &union_reader = *bind_data.union_readers[i];
+				auto stats = union_reader.GetStatistics(context, col_name);
+				if (!stats || merged_stats->GetType() != stats->GetType()) {
+					return nullptr;
+				}
+				merged_stats->Merge(*stats);
+			}
+			return merged_stats;
+		}
+		return bind_data.initial_reader->GetStatistics(context, col_name);
 	}
 
 	static double MultiFileProgress(ClientContext &context, const FunctionData *bind_data_p,

package/src/duckdb/src/include/duckdb/common/multi_file/multi_file_list.hpp

@@ -69,7 +69,8 @@ struct MultiFilePushdownInfo {
 //! NOTE: subclasses are responsible for ensuring thread-safety
 class MultiFileList {
 public:
-	explicit MultiFileList(vector<OpenFileInfo> paths, FileGlobOptions options);
+	MultiFileList(vector<OpenFileInfo> paths, FileGlobOptions options);
+	MultiFileList(vector<OpenFileInfo> paths, FileGlobInput input);
 	virtual ~MultiFileList();
 
 	//! Returns the raw, unexpanded paths, pre-filter
@@ -114,7 +115,7 @@ protected:
 	//! The unexpanded input paths
 	const vector<OpenFileInfo> paths;
 	//! Whether paths can expand to 0 files
-	const FileGlobOptions glob_options;
+	const FileGlobInput glob_input;
 
 public:
 	template <class TARGET>
@@ -157,7 +158,7 @@ protected:
 //! MultiFileList that takes a list of paths and produces a list of files with all globs expanded
 class GlobMultiFileList : public MultiFileList {
 public:
-	GlobMultiFileList(ClientContext &context, vector<OpenFileInfo> paths, FileGlobOptions options);
+	GlobMultiFileList(ClientContext &context, vector<OpenFileInfo> paths, FileGlobInput glob_input);
 	//! Calls ExpandAll, then prunes the expanded_files using the hive/filename filters
 	unique_ptr<MultiFileList> ComplexFilterPushdown(ClientContext &context, const MultiFileOptions &options,
 	                                                MultiFilePushdownInfo &info,

package/src/duckdb/src/include/duckdb/common/multi_file/multi_file_reader.hpp

@@ -37,6 +37,10 @@ public:
 	static constexpr int32_t ORDINAL_FIELD_ID = 2147483645;
 	// Reserved field id used for the "_pos" field according to the iceberg spec (used for file_row_number)
 	static constexpr int32_t FILENAME_FIELD_ID = 2147483646;
+	// Reserved field id used for the "file_path" field for iceberg positional deletes
+	static constexpr int32_t DELETE_FILE_PATH_FIELD_ID = 2147483546;
+	// Reserved field id used for the "pos" field for iceberg positional deletes
+	static constexpr int32_t DELETE_POS_FIELD_ID = 2147483545;
 	// Reserved field id used for the "_row_id" field according to the iceberg spec
 	static constexpr int32_t ROW_ID_FIELD_ID = 2147483540;
 	// Reserved field id used for the "_last_updated_sequence_number" field according to the iceberg spec
@@ -63,10 +67,11 @@ public:
 	//! Create a MultiFileList from a vector of paths. Any globs will be expanded using the default filesystem
 	DUCKDB_API virtual shared_ptr<MultiFileList>
 	CreateFileList(ClientContext &context, const vector<string> &paths,
-	               FileGlobOptions options = FileGlobOptions::DISALLOW_EMPTY);
+	               const FileGlobInput &glob_input = FileGlobOptions::DISALLOW_EMPTY);
 	//! Shorthand for ParsePaths + CreateFileList
-	DUCKDB_API shared_ptr<MultiFileList> CreateFileList(ClientContext &context, const Value &input,
-	                                                    FileGlobOptions options = FileGlobOptions::DISALLOW_EMPTY);
+	DUCKDB_API shared_ptr<MultiFileList>
+	CreateFileList(ClientContext &context, const Value &input,
+	               const FileGlobInput &glob_input = FileGlobOptions::DISALLOW_EMPTY);
 
 	//! Parse the named parameters of a multi-file reader
 	DUCKDB_API virtual bool ParseOption(const string &key, const Value &val, MultiFileOptions &options,
@@ -182,6 +187,8 @@ public:
 
 	DUCKDB_API virtual unique_ptr<MultiFileReader> Copy() const;
 
+	DUCKDB_API virtual FileGlobInput GetGlobInput(MultiFileReaderInterface &interface);
+
 protected:
 	//! Used in errors to report which function is using this MultiFileReader
 	string function_name;

package/src/duckdb/src/include/duckdb/common/multi_file/multi_file_states.hpp

@@ -169,6 +169,8 @@ struct MultiFileGlobalState : public GlobalTableFunctionState {
 
 	unique_ptr<GlobalTableFunctionState> global_state;
 
+	optional_ptr<const PhysicalOperator> op;
+
 	idx_t MaxThreads() const override {
 		return max_threads;
 	}