duckdb 0.8.2-dev37.0 → 0.8.2-dev3949.0

package/src/duckdb/src/common/file_system.cpp

@@ -23,6 +23,14 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+
+#ifdef __MVS__
+#define _XOPEN_SOURCE_EXTENDED 1
+#include <sys/resource.h>
+// enjoy - https://reviews.llvm.org/D92110
+#define PATH_MAX _XOPEN_PATH_MAX
+#endif
+
 #else
 #include <string>
 #include <sysinfoapi.h>
@@ -63,11 +71,11 @@ string FileSystem::GetEnvVariable(const string &name) {
 }
 
 bool FileSystem::IsPathAbsolute(const string &path) {
-	auto path_separator = FileSystem::PathSeparator();
+	auto path_separator = PathSeparator(path);
 	return PathMatched(path, path_separator);
 }
 
-string FileSystem::PathSeparator() {
+string FileSystem::PathSeparator(const string &path) {
 	return "/";
 }
 
@@ -79,7 +87,14 @@ void FileSystem::SetWorkingDirectory(const string &path) {
 
 idx_t FileSystem::GetAvailableMemory() {
 	errno = 0;
+
+#ifdef __MVS__
+	struct rlimit limit;
+	int rlim_rc = getrlimit(RLIMIT_AS, &limit);
+	idx_t max_memory = MinValue<idx_t>(limit.rlim_max, UINTPTR_MAX);
+#else
 	idx_t max_memory = MinValue<idx_t>((idx_t)sysconf(_SC_PHYS_PAGES) * (idx_t)sysconf(_SC_PAGESIZE), UINTPTR_MAX);
+#endif
 	if (errno != 0) {
 		return DConstants::INVALID_INDEX;
 	}
@@ -152,7 +167,7 @@ string FileSystem::NormalizeAbsolutePath(const string &path) {
 	return result;
 }
 
-string FileSystem::PathSeparator() {
+string FileSystem::PathSeparator(const string &path) {
 	return "\\";
 }
 
@@ -195,11 +210,11 @@ string FileSystem::GetWorkingDirectory() {
 
 string FileSystem::JoinPath(const string &a, const string &b) {
 	// FIXME: sanitize paths
-	return a + PathSeparator() + b;
+	return a + PathSeparator(a) + b;
 }
 
 string FileSystem::ConvertSeparators(const string &path) {
-	auto separator_str = PathSeparator();
+	auto separator_str = PathSeparator(path);
 	char separator = separator_str[0];
 	if (separator == '/') {
 		// on unix-based systems we only accept / as a separator
@@ -214,7 +229,7 @@ string FileSystem::ExtractName(const string &path) {
 		return string();
 	}
 	auto normalized_path = ConvertSeparators(path);
-	auto sep = PathSeparator();
+	auto sep = PathSeparator(path);
 	auto splits = StringUtil::Split(normalized_path, sep);
 	D_ASSERT(!splits.empty());
 	return splits.back();
@@ -370,6 +385,10 @@ void FileSystem::UnregisterSubSystem(const string &name) {
 	throw NotImplementedException("%s: Can't unregister a sub system on a non-virtual file system", GetName());
 }
 
+void FileSystem::SetDisabledFileSystems(const vector<string> &names) {
+	throw NotImplementedException("%s: Can't disable file systems on a non-virtual file system", GetName());
+}
+
 vector<string> FileSystem::ListSubSystems() {
 	throw NotImplementedException("%s: Can't list sub systems on a non-virtual file system", GetName());
 }
@@ -378,16 +397,31 @@ bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
 
+static string LookupExtensionForPattern(const string &pattern) {
+	for (const auto &entry : EXTENSION_FILE_PREFIXES) {
+		if (StringUtil::StartsWith(pattern, entry.name)) {
+			return entry.extension;
+		}
+	}
+	return "";
+}
+
 vector<string> FileSystem::GlobFiles(const string &pattern, ClientContext &context, FileGlobOptions options) {
 	auto result = Glob(pattern);
 	if (result.empty()) {
-		string required_extension;
-		if (FileSystem::IsRemoteFile(pattern)) {
-			required_extension = "httpfs";
-		}
+		string required_extension = LookupExtensionForPattern(pattern);
 		if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
-			// an extension is required to read this file but it is not loaded - try to load it
-			ExtensionHelper::LoadExternalExtension(context, required_extension);
+			auto &dbconfig = DBConfig::GetConfig(context);
+			if (!ExtensionHelper::CanAutoloadExtension(required_extension) ||
+			    !dbconfig.options.autoload_known_extensions) {
+				auto error_message =
+				    "File " + pattern + " requires the extension " + required_extension + " to be loaded";
+				error_message =
+				    ExtensionHelper::AddExtensionInstallHintToErrorMsg(context, error_message, required_extension);
+				throw MissingExtensionException(error_message);
+			}
+			// an extension is required to read this file, but it is not loaded - try to load it
+			ExtensionHelper::AutoLoadExtension(context, required_extension);
 			// success! glob again
 			// check the extension is loaded just in case to prevent an infinite loop here
 			if (!context.db->ExtensionIsLoaded(required_extension)) {

package/src/duckdb/src/common/filename_pattern.cpp

@@ -24,7 +24,7 @@ void FilenamePattern::SetFilenamePattern(const string &pattern) {
 	_pos = std::min(_pos, (idx_t)_base.length());
 }
 
-string FilenamePattern::CreateFilename(const FileSystem &fs, const string &path, const string &extension,
+string FilenamePattern::CreateFilename(FileSystem &fs, const string &path, const string &extension,
                                        idx_t offset) const {
 	string result(_base);
 	string replacement;

package/src/duckdb/src/common/gzip_file_system.cpp

@@ -145,6 +145,13 @@ void MiniZStreamWrapper::Initialize(CompressedFile &file, bool write) {
 bool MiniZStreamWrapper::Read(StreamData &sd) {
 	// Handling for the concatenated files
 	if (sd.refresh) {
+		auto available = (uint32_t)(sd.in_buff_end - sd.in_buff_start);
+		if (available <= GZIP_FOOTER_SIZE) {
+			// Only footer is available so we just close and return finished
+			Close();
+			return true;
+		}
+
 		sd.refresh = false;
 		auto body_ptr = sd.in_buff_start + GZIP_FOOTER_SIZE;
 		uint8_t gzip_hdr[GZIP_HEADER_MINSIZE];
@@ -200,18 +207,6 @@ bool MiniZStreamWrapper::Read(StreamData &sd) {
 
 	// if stream ended, deallocate inflator
 	if (ret == duckdb_miniz::MZ_STREAM_END) {
-		// Last read from file done and remaining bytes only for footer or less
-		if ((sd.in_buff_end < sd.in_buff.get() + sd.in_buf_size) && mz_stream_ptr->avail_in <= GZIP_FOOTER_SIZE) {
-			Close();
-			return true;
-		}
-		if (mz_stream_ptr->avail_in > GZIP_FOOTER_SIZE) {
-			// Definitely not concatenated gzip
-			if (*(sd.in_buff_start + GZIP_FOOTER_SIZE) != 0x1F) {
-				Close();
-				return true;
-			}
-		}
 		// Concatenated GZIP potentially coming up - refresh input buffer
 		sd.refresh = true;
 	}

package/src/duckdb/src/common/hive_partitioning.cpp

@@ -6,6 +6,7 @@
 #include "duckdb/planner/expression/bound_constant_expression.hpp"
 #include "duckdb/planner/expression/bound_reference_expression.hpp"
 #include "duckdb/planner/expression_iterator.hpp"
+#include "duckdb/planner/operator/logical_get.hpp"
 #include "duckdb/planner/table_filter.hpp"
 #include "re2/re2.h"
 
@@ -86,12 +87,15 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
 //		 currently, only expressions that cannot be evaluated during pushdown are removed.
 void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
                                               vector<unique_ptr<Expression>> &filters,
-                                              unordered_map<string, column_t> &column_map, idx_t table_index,
+                                              unordered_map<string, column_t> &column_map, LogicalGet &get,
                                               bool hive_enabled, bool filename_enabled) {
+
 	vector<string> pruned_files;
 	vector<bool> have_preserved_filter(filters.size(), false);
 	vector<unique_ptr<Expression>> pruned_filters;
+	unordered_set<idx_t> filters_applied_to_files;
 	duckdb_re2::RE2 regex(REGEX_STRING);
+	auto table_index = get.table_index;
 
 	if ((!filename_enabled && !hive_enabled) || filters.empty()) {
 		return;
@@ -121,11 +125,11 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
 			} else if (!result_value.GetValue<bool>()) {
 				// filter evaluates to false
 				should_prune_file = true;
-			}
-
-			// Use filter combiner to determine that this filter makes
-			if (!should_prune_file && combiner.AddFilter(std::move(filter_copy)) == FilterResult::UNSATISFIABLE) {
-				should_prune_file = true;
+				// convert the filter to a table filter.
+				if (filters_applied_to_files.find(j) == filters_applied_to_files.end()) {
+					get.extra_info.file_filters += filter->ToString();
+					filters_applied_to_files.insert(j);
+				}
 			}
 		}
 

package/src/duckdb/src/common/http_state.cpp

@@ -0,0 +1,78 @@
+#include "duckdb/common/http_state.hpp"
+
+namespace duckdb {
+
+CachedFileHandle::CachedFileHandle(shared_ptr<CachedFile> &file_p) {
+	// If the file was not yet initialized, we need to grab a lock.
+	if (!file_p->initialized) {
+		lock = make_uniq<lock_guard<mutex>>(file_p->lock);
+	}
+	file = file_p;
+}
+
+void CachedFileHandle::SetInitialized() {
+	if (file->initialized) {
+		throw InternalException("Cannot set initialized on cached file that was already initialized");
+	}
+	if (!lock) {
+		throw InternalException("Cannot set initialized on cached file without lock");
+	}
+	file->initialized = true;
+	lock = nullptr;
+}
+
+void CachedFileHandle::AllocateBuffer(idx_t size) {
+	if (file->initialized) {
+		throw InternalException("Cannot allocate a buffer for a cached file that was already initialized");
+	}
+	file->data = std::shared_ptr<char>(new char[size], std::default_delete<char[]>());
+	file->capacity = size;
+}
+
+void CachedFileHandle::GrowBuffer(idx_t new_capacity, idx_t bytes_to_copy) {
+	// copy shared ptr to old data
+	auto old_data = file->data;
+	// allocate new buffer that can hold the new capacity
+	AllocateBuffer(new_capacity);
+	// copy the old data
+	Write(old_data.get(), bytes_to_copy);
+}
+
+void CachedFileHandle::Write(const char *buffer, idx_t length, idx_t offset) {
+	//! Only write to non-initialized files with a lock;
+	D_ASSERT(!file->initialized && lock);
+	memcpy(file->data.get() + offset, buffer, length);
+}
+
+void HTTPState::Reset() {
+	// Reset Counters
+	head_count = 0;
+	get_count = 0;
+	put_count = 0;
+	post_count = 0;
+	total_bytes_received = 0;
+	total_bytes_sent = 0;
+
+	// Reset cached files
+	cached_files.clear();
+}
+
+shared_ptr<HTTPState> HTTPState::TryGetState(FileOpener *opener) {
+	auto client_context = FileOpener::TryGetClientContext(opener);
+	if (client_context) {
+		return client_context->client_data->http_state;
+	}
+	return nullptr;
+}
+
+//! Get cache entry, create if not exists
+shared_ptr<CachedFile> &HTTPState::GetCachedFile(const string &path) {
+	lock_guard<mutex> lock(cached_files_mutex);
+	auto &cache_entry_ref = cached_files[path];
+	if (!cache_entry_ref) {
+		cache_entry_ref = make_shared<CachedFile>();
+	}
+	return cache_entry_ref;
+}
+
+} // namespace duckdb

package/src/duckdb/src/common/local_file_system.cpp

@@ -261,13 +261,19 @@ idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
 
 void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
-	int64_t bytes_read = pread(fd, buffer, nr_bytes, location);
-	if (bytes_read == -1) {
-		throw IOException("Could not read from file \"%s\": %s", handle.path, strerror(errno));
-	}
-	if (bytes_read != nr_bytes) {
-		throw IOException("Could not read all bytes from file \"%s\": wanted=%lld read=%lld", handle.path, nr_bytes,
-		                  bytes_read);
+	auto read_buffer = char_ptr_cast(buffer);
+	while (nr_bytes > 0) {
+		int64_t bytes_read = pread(fd, read_buffer, nr_bytes, location);
+		if (bytes_read == -1) {
+			throw IOException("Could not read from file \"%s\": %s", handle.path, strerror(errno));
+		}
+		if (bytes_read == 0) {
+			throw IOException(
+			    "Could not read enough bytes from file \"%s\": attempted to read %llu bytes from location %llu",
+			    handle.path, nr_bytes, location);
+		}
+		read_buffer += bytes_read;
+		nr_bytes -= bytes_read;
 	}
 }
 
@@ -282,13 +288,15 @@ int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes
 
 void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
 	int fd = handle.Cast<UnixFileHandle>().fd;
-	int64_t bytes_written = pwrite(fd, buffer, nr_bytes, location);
-	if (bytes_written == -1) {
-		throw IOException("Could not write file \"%s\": %s", handle.path, strerror(errno));
-	}
-	if (bytes_written != nr_bytes) {
-		throw IOException("Could not write all bytes to file \"%s\": wanted=%lld wrote=%lld", handle.path, nr_bytes,
-		                  bytes_written);
+	auto write_buffer = char_ptr_cast(buffer);
+	while (nr_bytes > 0) {
+		int64_t bytes_written = pwrite(fd, write_buffer, nr_bytes, location);
+		if (bytes_written < 0) {
+			throw IOException("Could not write file \"%s\": %s", handle.path, strerror(errno));
+		}
+		D_ASSERT(bytes_written >= 0 && bytes_written);
+		write_buffer += bytes_written;
+		nr_bytes -= bytes_written;
 	}
 }
 
@@ -375,7 +383,7 @@ int RemoveDirectoryRecursive(const char *path) {
 				continue;
 			}
 			len = path_len + (idx_t)strlen(p->d_name) + 2;
-			buf = new char[len];
+			buf = new (std::nothrow) char[len];
 			if (buf) {
 				struct stat statbuf;
 				snprintf(buf, len, "%s/%s", path, p->d_name);
@@ -559,7 +567,7 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path_p, uint8_t f
 }
 
 void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
-	auto &whandle = (WindowsFileHandle &)handle;
+	auto &whandle = handle.Cast<WindowsFileHandle>();
 	whandle.position = location;
 	LARGE_INTEGER wlocation;
 	wlocation.QuadPart = location;
@@ -567,7 +575,7 @@ void LocalFileSystem::SetFilePointer(FileHandle &handle, idx_t location) {
 }
 
 idx_t LocalFileSystem::GetFilePointer(FileHandle &handle) {
-	return ((WindowsFileHandle &)handle).position;
+	return handle.Cast<WindowsFileHandle>().position;
 }
 
 static DWORD FSInternalRead(FileHandle &handle, HANDLE hFile, void *buffer, int64_t nr_bytes, idx_t location) {
@@ -597,8 +605,8 @@ void LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, i
 }
 
 int64_t LocalFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	auto &pos = ((WindowsFileHandle &)handle).position;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
+	auto &pos = handle.Cast<WindowsFileHandle>().position;
 	auto n = std::min<idx_t>(std::max<idx_t>(GetFileSize(handle), pos) - pos, nr_bytes);
 	auto bytes_read = FSInternalRead(handle, hFile, buffer, n, pos);
 	pos += bytes_read;
@@ -622,7 +630,7 @@ static DWORD FSInternalWrite(FileHandle &handle, HANDLE hFile, void *buffer, int
 }
 
 void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, location);
 	if (bytes_written != nr_bytes) {
 		throw IOException("Could not write all bytes from file \"%s\": wanted=%lld wrote=%lld", handle.path, nr_bytes,
@@ -631,15 +639,15 @@ void LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes,
 }
 
 int64_t LocalFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
-	auto &pos = ((WindowsFileHandle &)handle).position;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
+	auto &pos = handle.Cast<WindowsFileHandle>().position;
 	auto bytes_written = FSInternalWrite(handle, hFile, buffer, nr_bytes, pos);
 	pos += bytes_written;
 	return bytes_written;
 }
 
 int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	LARGE_INTEGER result;
 	if (!GetFileSizeEx(hFile, &result)) {
 		return -1;
@@ -648,7 +656,7 @@ int64_t LocalFileSystem::GetFileSize(FileHandle &handle) {
 }
 
 time_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 
 	// https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfiletime
 	FILETIME last_write;
@@ -674,7 +682,7 @@ time_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) {
 }
 
 void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	// seek to the location
 	SetFilePointer(handle, new_size);
 	// now set the end of file position
@@ -700,7 +708,7 @@ void LocalFileSystem::CreateDirectory(const string &directory) {
 	}
 	auto unicode_path = WindowsUtil::UTF8ToUnicode(directory.c_str());
 	if (directory.empty() || !CreateDirectoryW(unicode_path.c_str(), NULL) || !DirectoryExists(directory)) {
-		throw IOException("Could not create directory!");
+		throw IOException("Could not create directory: \'%s\'", directory.c_str());
 	}
 }
 
@@ -767,7 +775,7 @@ bool LocalFileSystem::ListFiles(const string &directory, const std::function<voi
 }
 
 void LocalFileSystem::FileSync(FileHandle &handle) {
-	HANDLE hFile = ((WindowsFileHandle &)handle).fd;
+	HANDLE hFile = handle.Cast<WindowsFileHandle>().fd;
 	if (FlushFileBuffers(hFile) == 0) {
 		throw IOException("Could not flush file handle to disk!");
 	}
@@ -782,7 +790,7 @@ void LocalFileSystem::MoveFile(const string &source, const string &target) {
 }
 
 FileType LocalFileSystem::GetFileType(FileHandle &handle) {
-	auto path = ((WindowsFileHandle &)handle).path;
+	auto path = handle.Cast<WindowsFileHandle>().path;
 	// pipes in windows are just files in '\\.\pipe\' folder
 	if (strncmp(path.c_str(), PIPE_PREFIX, strlen(PIPE_PREFIX)) == 0) {
 		return FileType::FILE_TYPE_FIFO;