duckdb 0.2.10-dev325.0 → 0.2.10-dev337.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.2.10-dev325.0",
+  "version": "0.2.10-dev337.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -18182,6 +18182,10 @@ FileSystem &FileSystem::GetFileSystem(ClientContext &context) {
 	return *context.db->config.file_system;
 }
 
+FileOpener *FileSystem::GetFileOpener(ClientContext &context) {
+	return context.file_opener.get();
+}
+
 #ifndef _WIN32
 string FileSystem::PathSeparator() {
 	return "/";
@@ -18283,7 +18287,7 @@ string FileSystem::GetHomeDirectory() {
 
 // LCOV_EXCL_START
 unique_ptr<FileHandle> FileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock,
-                                            FileCompressionType compression) {
+                                            FileCompressionType compression, FileOpener *opener) {
 	throw NotImplementedException("%s: OpenFile is not implemented!", GetName());
 }
 
@@ -20215,7 +20219,8 @@ namespace duckdb {
 class LocalFileSystem : public FileSystem {
 public:
 	unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = FileLockType::NO_LOCK,
-	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED) override;
+	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
+	                                FileOpener *opener = nullptr) override;
 
 	//! Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Read().
@@ -20322,14 +20327,18 @@ extern "C" WINBASEAPI BOOL WINAPI GetPhysicallyInstalledSystemMemory(PULONGLONG)
 namespace duckdb {
 
 static void AssertValidFileFlags(uint8_t flags) {
-	// cannot combine Read and Write flags
-	D_ASSERT(!(flags & FileFlags::FILE_FLAGS_READ && flags & FileFlags::FILE_FLAGS_WRITE));
-	// cannot combine Read and CREATE/Append flags
-	D_ASSERT(!(flags & FileFlags::FILE_FLAGS_READ && flags & FileFlags::FILE_FLAGS_APPEND));
-	D_ASSERT(!(flags & FileFlags::FILE_FLAGS_READ && flags & FileFlags::FILE_FLAGS_FILE_CREATE));
-	D_ASSERT(!(flags & FileFlags::FILE_FLAGS_READ && flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW));
+#ifdef DEBUG
+	bool is_read = flags & FileFlags::FILE_FLAGS_READ;
+	bool is_write = flags & FileFlags::FILE_FLAGS_WRITE;
+	// require either READ or WRITE (or both)
+	D_ASSERT(is_read || is_write);
+	// CREATE/Append flags require writing
+	D_ASSERT(is_write || !(flags & FileFlags::FILE_FLAGS_APPEND));
+	D_ASSERT(is_write || !(flags & FileFlags::FILE_FLAGS_FILE_CREATE));
+	D_ASSERT(is_write || !(flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW));
 	// cannot combine CREATE and CREATE_NEW flags
 	D_ASSERT(!(flags & FileFlags::FILE_FLAGS_FILE_CREATE && flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW));
+#endif
 }
 
 #ifndef _WIN32
@@ -20362,8 +20371,33 @@ public:
 	int fd;
 };
 
+static FileType GetFileTypeInternal(int fd) {
+	struct stat s;
+	if (fstat(fd, &s) == -1) {
+		return FileType::FILE_TYPE_INVALID;
+	}
+	switch (s.st_mode & S_IFMT) {
+	case S_IFBLK:
+		return FileType::FILE_TYPE_BLOCKDEV;
+	case S_IFCHR:
+		return FileType::FILE_TYPE_CHARDEV;
+	case S_IFIFO:
+		return FileType::FILE_TYPE_FIFO;
+	case S_IFDIR:
+		return FileType::FILE_TYPE_DIR;
+	case S_IFLNK:
+		return FileType::FILE_TYPE_LINK;
+	case S_IFREG:
+		return FileType::FILE_TYPE_REGULAR;
+	case S_IFSOCK:
+		return FileType::FILE_TYPE_SOCKET;
+	default:
+		return FileType::FILE_TYPE_INVALID;
+	}
+}
+
 unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock_type,
-                                                 FileCompressionType compression) {
+                                                 FileCompressionType compression, FileOpener *opener) {
 	if (compression != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
@@ -20372,12 +20406,21 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t fla
 
 	int open_flags = 0;
 	int rc;
-	if (flags & FileFlags::FILE_FLAGS_READ) {
+	bool open_read = flags & FileFlags::FILE_FLAGS_READ;
+	bool open_write = flags & FileFlags::FILE_FLAGS_WRITE;
+	if (open_read && open_write) {
+		open_flags = O_RDWR;
+	} else if (open_read) {
 		open_flags = O_RDONLY;
+	} else if (open_write) {
+		open_flags = O_WRONLY;
 	} else {
+		throw InternalException("READ, WRITE or both should be specified when opening a file");
+	}
+	if (open_write) {
 		// need Read or Write
 		D_ASSERT(flags & FileFlags::FILE_FLAGS_WRITE);
-		open_flags = O_RDWR | O_CLOEXEC;
+		open_flags |= O_CLOEXEC;
 		if (flags & FileFlags::FILE_FLAGS_FILE_CREATE) {
 			open_flags |= O_CREAT;
 		} else if (flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW) {
@@ -20413,15 +20456,19 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t fla
 	// #endif
 	if (lock_type != FileLockType::NO_LOCK) {
 		// set lock on file
-		struct flock fl;
-		memset(&fl, 0, sizeof fl);
-		fl.l_type = lock_type == FileLockType::READ_LOCK ? F_RDLCK : F_WRLCK;
-		fl.l_whence = SEEK_SET;
-		fl.l_start = 0;
-		fl.l_len = 0;
-		rc = fcntl(fd, F_SETLK, &fl);
-		if (rc == -1) {
-			throw IOException("Could not set lock on file \"%s\": %s", path, strerror(errno));
+		// but only if it is not an input/output stream
+		auto file_type = GetFileTypeInternal(fd);
+		if (file_type != FileType::FILE_TYPE_FIFO && file_type != FileType::FILE_TYPE_SOCKET) {
+			struct flock fl;
+			memset(&fl, 0, sizeof fl);
+			fl.l_type = lock_type == FileLockType::READ_LOCK ? F_RDLCK : F_WRLCK;
+			fl.l_whence = SEEK_SET;
+			fl.l_start = 0;
+			fl.l_len = 0;
+			rc = fcntl(fd, F_SETLK, &fl);
+			if (rc == -1) {
+				throw IOException("Could not set lock on file \"%s\": %s", path, strerror(errno));
+			}
 		}
 	}
 	return make_unique<UnixFileHandle>(*this, path, fd);
@@ -20507,28 +20554,7 @@ time_t LocalFileSystem::GetLastModifiedTime(FileHandle &handle) {
 
 FileType LocalFileSystem::GetFileType(FileHandle &handle) {
 	int fd = ((UnixFileHandle &)handle).fd;
-	struct stat s;
-	if (fstat(fd, &s) == -1) {
-		return FileType::FILE_TYPE_INVALID;
-	}
-	switch (s.st_mode & S_IFMT) {
-	case S_IFBLK:
-		return FileType::FILE_TYPE_BLOCKDEV;
-	case S_IFCHR:
-		return FileType::FILE_TYPE_CHARDEV;
-	case S_IFIFO:
-		return FileType::FILE_TYPE_FIFO;
-	case S_IFDIR:
-		return FileType::FILE_TYPE_DIR;
-	case S_IFLNK:
-		return FileType::FILE_TYPE_LINK;
-	case S_IFREG:
-		return FileType::FILE_TYPE_REGULAR;
-	case S_IFSOCK:
-		return FileType::FILE_TYPE_SOCKET;
-	default:
-		return FileType::FILE_TYPE_INVALID;
-	}
+	return GetFileTypeInternal(fd);
 }
 
 void LocalFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
@@ -20721,7 +20747,7 @@ public:
 };
 
 unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock_type,
-                                                 FileCompressionType compression) {
+                                                 FileCompressionType compression, FileOpener *opener) {
 	if (compression != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
@@ -20731,14 +20757,21 @@ unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t fla
 	DWORD share_mode;
 	DWORD creation_disposition = OPEN_EXISTING;
 	DWORD flags_and_attributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED;
-	if (flags & FileFlags::FILE_FLAGS_READ) {
+	bool open_read = flags & FileFlags::FILE_FLAGS_READ;
+	bool open_write = flags & FileFlags::FILE_FLAGS_WRITE;
+	if (open_read && open_write) {
+		desired_access = GENERIC_READ | GENERIC_WRITE;
+		share_mode = 0;
+	} else if (open_read) {
 		desired_access = GENERIC_READ;
 		share_mode = FILE_SHARE_READ;
-	} else {
-		// need Read or Write
-		D_ASSERT(flags & FileFlags::FILE_FLAGS_WRITE);
-		desired_access = GENERIC_READ | GENERIC_WRITE;
+	} else if (open_write) {
+		desired_access = GENERIC_WRITE;
 		share_mode = 0;
+	} else {
+		throw InternalException("READ, WRITE or both should be specified when opening a file");
+	}
+	if (open_write) {
 		if (flags & FileFlags::FILE_FLAGS_FILE_CREATE) {
 			creation_disposition = OPEN_ALWAYS;
 		} else if (flags & FileFlags::FILE_FLAGS_FILE_CREATE_NEW) {
@@ -28380,7 +28413,7 @@ namespace duckdb {
 
 class BufferedFileReader : public Deserializer {
 public:
-	BufferedFileReader(FileSystem &fs, const char *path);
+	BufferedFileReader(FileSystem &fs, const char *path, FileOpener *opener = nullptr);
 
 	FileSystem &fs;
 	unique_ptr<data_t[]> data;
@@ -28412,9 +28445,10 @@ private:
 
 namespace duckdb {
 
-BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path)
+BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, FileOpener *opener)
     : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), read_data(0), total_read(0) {
-	handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::READ_LOCK);
+	handle =
+	    fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::READ_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 	file_size = fs.GetFileSize(*handle);
 }
 
@@ -28467,9 +28501,12 @@ bool BufferedFileReader::Finished() {
 
 namespace duckdb {
 
-BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags)
+// Remove this when we switch C++17: https://stackoverflow.com/a/53350948
+constexpr uint8_t BufferedFileWriter::DEFAULT_OPEN_FLAGS;
+
+BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags, FileOpener *opener)
     : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), total_written(0) {
-	handle = fs.OpenFile(path, open_flags, FileLockType::WRITE_LOCK);
+	handle = fs.OpenFile(path, open_flags, FileLockType::WRITE_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 }
 
 int64_t BufferedFileWriter::GetFileSize() {
@@ -45310,7 +45347,8 @@ public:
 	VirtualFileSystem();
 
 	unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = FileLockType::NO_LOCK,
-	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED) override;
+	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
+	                                FileOpener *opener = nullptr) override;
 
 	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
 		handle.file_system.Read(handle, buffer, nr_bytes, location);
@@ -45414,7 +45452,7 @@ VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) {
 }
 
 unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock,
-                                                   FileCompressionType compression) {
+                                                   FileCompressionType compression, FileOpener *opener) {
 	if (compression == FileCompressionType::AUTO_DETECT) {
 		// auto detect compression settings based on file name
 		auto lower_path = StringUtil::Lower(path);
@@ -45425,7 +45463,7 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 		}
 	}
 	// open the base file handle
-	auto file_handle = FindFileSystem(path)->OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED);
+	auto file_handle = FindFileSystem(path)->OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
 	if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
 		file_handle = PipeFileSystem::OpenPipe(move(file_handle));
 	} else if (compression != FileCompressionType::UNCOMPRESSED) {
@@ -59462,6 +59500,7 @@ struct CopyInfo;
 struct FileHandle;
 struct StrpTimeFormat;
 
+class FileOpener;
 class FileSystem;
 
 //! The shifts array allows for linear searching of multi-byte values. For each position, it determines the next
@@ -59562,10 +59601,11 @@ public:
 	BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options,
 	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
 
-	BufferedCSVReader(FileSystem &fs, BufferedCSVReaderOptions options,
+	BufferedCSVReader(FileSystem &fs, FileOpener *opener, BufferedCSVReaderOptions options,
 	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
 
 	FileSystem &fs;
+	FileOpener *opener;
 	BufferedCSVReaderOptions options;
 	vector<LogicalType> sql_types;
 	vector<string> col_names;
@@ -59798,16 +59838,17 @@ TextSearchShiftArray::TextSearchShiftArray(string search_term) : length(search_t
 	}
 }
 
-BufferedCSVReader::BufferedCSVReader(FileSystem &fs_p, BufferedCSVReaderOptions options_p,
+BufferedCSVReader::BufferedCSVReader(FileSystem &fs_p, FileOpener *opener_p, BufferedCSVReaderOptions options_p,
                                      const vector<LogicalType> &requested_types)
-    : fs(fs_p), options(move(options_p)), buffer_size(0), position(0), start(0) {
+    : fs(fs_p), opener(opener_p), options(move(options_p)), buffer_size(0), position(0), start(0) {
 	file_handle = OpenCSV(options);
 	Initialize(requested_types);
 }
 
 BufferedCSVReader::BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
                                      const vector<LogicalType> &requested_types)
-    : BufferedCSVReader(FileSystem::GetFileSystem(context), move(options_p), requested_types) {
+    : BufferedCSVReader(FileSystem::GetFileSystem(context), FileSystem::GetFileOpener(context), move(options_p),
+                        requested_types) {
 }
 
 void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
@@ -59839,8 +59880,8 @@ unique_ptr<FileHandle> BufferedCSVReader::OpenCSV(const BufferedCSVReaderOptions
 		this->compression = FileCompressionType::GZIP;
 	}
 
-	auto result =
-	    fs.OpenFile(options.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, this->compression);
+	auto result = fs.OpenFile(options.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
+	                          this->compression, this->opener);
 	plain_file_source = result->OnDiskFile() && result->CanSeek();
 	file_size = result->GetFileSize();
 	return result;
@@ -59850,7 +59891,7 @@ unique_ptr<FileHandle> BufferedCSVReader::OpenCSV(const BufferedCSVReaderOptions
 static string GenerateColumnName(const idx_t total_cols, const idx_t col_number, const string &prefix = "column") {
 	int max_digits = NumericHelper::UnsignedLength(total_cols - 1);
 	int digits = NumericHelper::UnsignedLength(col_number);
-	string leading_zeros = string("0", max_digits - digits);
+	string leading_zeros = string(max_digits - digits, '0');
 	string value = to_string(col_number);
 	return string(prefix + leading_zeros + value);
 }
@@ -61584,10 +61625,10 @@ static void WriteCatalogEntries(stringstream &ss, vector<CatalogEntry *> &entrie
 	ss << std::endl;
 }
 
-static void WriteStringStreamToFile(FileSystem &fs, stringstream &ss, const string &path) {
+static void WriteStringStreamToFile(FileSystem &fs, FileOpener *opener, stringstream &ss, const string &path) {
 	auto ss_string = ss.str();
 	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
-	                          FileLockType::WRITE_LOCK);
+	                          FileLockType::WRITE_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 	fs.Write(*handle, (void *)ss_string.c_str(), ss_string.size());
 	handle.reset();
 }
@@ -61641,6 +61682,7 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, TableCatalogEnt
 void PhysicalExport::GetChunkInternal(ExecutionContext &context, DataChunk &chunk, PhysicalOperatorState *state) const {
 	auto &ccontext = context.client;
 	auto &fs = FileSystem::GetFileSystem(ccontext);
+	auto *opener = FileSystem::GetFileOpener(ccontext);
 
 	// gather all catalog types to export
 	vector<CatalogEntry *> schemas;
@@ -61680,7 +61722,7 @@ void PhysicalExport::GetChunkInternal(ExecutionContext &context, DataChunk &chun
 	WriteCatalogEntries(ss, views);
 	WriteCatalogEntries(ss, indexes);
 
-	WriteStringStreamToFile(fs, ss, fs.JoinPath(info->file_path, "schema.sql"));
+	WriteStringStreamToFile(fs, opener, ss, fs.JoinPath(info->file_path, "schema.sql"));
 
 	// write the load.sql file
 	// for every table, we write COPY INTO statement with the specified options
@@ -61690,7 +61732,7 @@ void PhysicalExport::GetChunkInternal(ExecutionContext &context, DataChunk &chun
 		auto exported_table_info = kv.second;
 		WriteCopyStatement(fs, load_ss, table, *info, exported_table_info, function);
 	}
-	WriteStringStreamToFile(fs, load_ss, fs.JoinPath(info->file_path, "load.sql"));
+	WriteStringStreamToFile(fs, opener, load_ss, fs.JoinPath(info->file_path, "load.sql"));
 	state->finished = true;
 }
 
@@ -76035,7 +76077,9 @@ static void PragmaLogQueryPath(ClientContext &context, const FunctionParameters
 		// empty path: clean up query writer
 		context.log_query_writer = nullptr;
 	} else {
-		context.log_query_writer = make_unique<BufferedFileWriter>(FileSystem::GetFileSystem(context), str_val);
+		context.log_query_writer =
+		    make_unique<BufferedFileWriter>(FileSystem::GetFileSystem(context), str_val,
+		                                    BufferedFileWriter::DEFAULT_OPEN_FLAGS, context.file_opener.get());
 	}
 }
 
@@ -76243,12 +76287,15 @@ string PragmaVersion(ClientContext &context, const FunctionParameters &parameter
 
 string PragmaImportDatabase(ClientContext &context, const FunctionParameters &parameters) {
 	auto &fs = FileSystem::GetFileSystem(context);
+	auto *opener = FileSystem::GetFileOpener(context);
+
 	string query;
 	// read the "shema.sql" and "load.sql" files
 	vector<string> files = {"schema.sql", "load.sql"};
 	for (auto &file : files) {
 		auto file_path = fs.JoinPath(parameters.values[0].ToString(), file);
-		auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ);
+		auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK,
+		                          FileSystem::DEFAULT_COMPRESSION, opener);
 		auto fsize = fs.GetFileSize(*handle);
 		auto buffer = unique_ptr<char[]>(new char[fsize]);
 		fs.Read(*handle, buffer.get(), fsize);
@@ -91223,9 +91270,9 @@ struct LocalReadCSVData : public LocalFunctionData {
 };
 
 struct GlobalWriteCSVData : public GlobalFunctionData {
-	GlobalWriteCSVData(FileSystem &fs, const string &file_path) : fs(fs) {
+	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileOpener *opener) : fs(fs) {
 		handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
-		                     FileLockType::WRITE_LOCK);
+		                     FileLockType::WRITE_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 	}
 
 	void WriteData(const_data_ptr_t data, idx_t size) {
@@ -91255,7 +91302,8 @@ static unique_ptr<LocalFunctionData> WriteCSVInitializeLocal(ClientContext &cont
 static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &context, FunctionData &bind_data) {
 	auto &csv_data = (WriteCSVData &)bind_data;
 	auto &options = csv_data.options;
-	auto global_data = make_unique<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), csv_data.files[0]);
+	auto global_data = make_unique<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), csv_data.files[0],
+	                                                   FileSystem::GetFileOpener(context));
 
 	if (options.header) {
 		BufferedSerializer serializer;
@@ -96978,6 +97026,61 @@ bool duckdb_value_is_null(duckdb_result *result, idx_t col, idx_t row) {
 	return result->columns[col].nullmask[row];
 }
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/client_context_file_opener.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/file_opener.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+
+class Value;
+
+//! Abstact type that provide client-spcific context to FileSystem.
+class FileOpener {
+public:
+	virtual ~FileOpener() {};
+
+	virtual bool TryGetCurrentSetting(const string &key, Value &result) = 0;
+};
+
+} // namespace duckdb
+
+namespace duckdb {
+
+class ClientContext;
+
+//! ClientContext-specific FileOpener implemenation.
+//! This object is owned by ClientContext and never outlives it.
+class ClientContextFileOpener : public FileOpener {
+public:
+	explicit ClientContextFileOpener(ClientContext &context_p) : context(context_p) {
+	}
+
+	bool TryGetCurrentSetting(const string &key, Value &result) override;
+
+private:
+	ClientContext &context;
+};
+
+} // namespace duckdb
+
 
 
 
@@ -97428,7 +97531,8 @@ private:
 ClientContext::ClientContext(shared_ptr<DatabaseInstance> database)
     : profiler(make_unique<QueryProfiler>()), query_profiler_history(make_unique<QueryProfilerHistory>()),
       db(move(database)), transaction(db->GetTransactionManager(), *this), interrupted(false), executor(*this),
-      temporary_objects(make_unique<SchemaCatalogEntry>(&db->GetCatalog(), TEMP_SCHEMA, true)), open_result(nullptr) {
+      temporary_objects(make_unique<SchemaCatalogEntry>(&db->GetCatalog(), TEMP_SCHEMA, true)),
+      file_opener(make_unique<ClientContextFileOpener>(*this)), open_result(nullptr) {
 	std::random_device rd;
 	random_engine.seed(rd());
 
@@ -97895,7 +97999,8 @@ void ClientContext::LogQueryInternal(ClientContextLock &, const string &query) {
 #ifdef DUCKDB_FORCE_QUERY_LOG
 		try {
 			string log_path(DUCKDB_FORCE_QUERY_LOG);
-			log_query_writer = make_unique<BufferedFileWriter>(FileSystem::GetFileSystem(*this), log_path);
+			log_query_writer = make_unique<BufferedFileWriter>(
+			    FileSystem::GetFileSystem(*this), log_path, BufferedFileWriter::DEFAULT_OPEN_FLAGS, file_opener.get());
 		} catch (...) {
 			return;
 		}
@@ -98315,6 +98420,16 @@ bool ClientContext::TryGetCurrentSetting(const std::string &key, Value &result)
 
 
 
+namespace duckdb {
+
+bool ClientContextFileOpener::TryGetCurrentSetting(const string &key, Value &result) {
+	return context.TryGetCurrentSetting(key, result);
+}
+
+} // namespace duckdb
+
+
+
 namespace duckdb {
 
 static ConfigurationOption internal_options[] = {
@@ -145876,6 +145991,29 @@ idx_t LocalStorage::Delete(DataTable *table, Vector &row_ids, idx_t count) {
 	idx_t chunk_idx = GetChunk(row_ids);
 	D_ASSERT(chunk_idx < storage->collection.ChunkCount());
 
+	// delete from unique indices (if any)
+	if (!storage->indexes.empty()) {
+		// Index::Delete assumes that ALL rows are being deleted, so
+		// Slice out the rows that are being deleted from the storage Chunk
+		auto &chunk = storage->collection.GetChunk(chunk_idx);
+
+		VectorData row_ids_data;
+		row_ids.Orrify(count, row_ids_data);
+		auto row_identifiers = (const row_t *)row_ids_data.data;
+		SelectionVector sel(count);
+		for (idx_t i = 0; i < count; ++i) {
+			const auto idx = row_ids_data.sel->get_index(i);
+			sel.set_index(i, row_identifiers[idx] - MAX_ROW_ID);
+		}
+
+		DataChunk deleted;
+		deleted.InitializeEmpty(chunk.GetTypes());
+		deleted.Slice(chunk, sel, count);
+		for (auto &index : storage->indexes) {
+			index->Delete(deleted, row_ids);
+		}
+	}
+
 	// get a pointer to the deleted entries for this chunk
 	bool *deleted;
 	auto entry = storage->deleted_entries.find(chunk_idx);
@@ -146392,7 +146530,7 @@ SingleFileBlockManager::SingleFileBlockManager(DatabaseInstance &db, string path
 		flags = FileFlags::FILE_FLAGS_READ;
 		lock = FileLockType::READ_LOCK;
 	} else {
-		flags = FileFlags::FILE_FLAGS_WRITE;
+		flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_READ;
 		lock = FileLockType::WRITE_LOCK;
 		if (create_new) {
 			flags |= FileFlags::FILE_FLAGS_FILE_CREATE;

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "57a9aff40"
-#define DUCKDB_VERSION "0.2.10-dev325"
+#define DUCKDB_SOURCE_ID "ee3e405a9"
+#define DUCKDB_VERSION "0.2.10-dev337"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -1710,6 +1710,7 @@ enum class FileCompressionType : uint8_t { AUTO_DETECT = 0, UNCOMPRESSED = 1, GZ
 namespace duckdb {
 class ClientContext;
 class DatabaseInstance;
+class FileOpener;
 class FileSystem;
 
 enum class FileType {
@@ -1787,12 +1788,15 @@ public:
 	}
 
 public:
+	static constexpr FileLockType DEFAULT_LOCK = FileLockType::NO_LOCK;
+	static constexpr FileCompressionType DEFAULT_COMPRESSION = FileCompressionType::UNCOMPRESSED;
 	static FileSystem &GetFileSystem(ClientContext &context);
 	static FileSystem &GetFileSystem(DatabaseInstance &db);
+	static FileOpener *GetFileOpener(ClientContext &context);
 
-	virtual unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags,
-	                                        FileLockType lock = FileLockType::NO_LOCK,
-	                                        FileCompressionType compression = FileCompressionType::UNCOMPRESSED);
+	virtual unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = DEFAULT_LOCK,
+	                                        FileCompressionType compression = DEFAULT_COMPRESSION,
+	                                        FileOpener *opener = nullptr);
 
 	//! Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Read().
@@ -1889,10 +1893,12 @@ namespace duckdb {
 
 class BufferedFileWriter : public Serializer {
 public:
+	static constexpr uint8_t DEFAULT_OPEN_FLAGS = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE;
+
 	//! Serializes to a buffer allocated by the serializer, will expand when
 	//! writing past the initial threshold
-	BufferedFileWriter(FileSystem &fs, const string &path,
-	                   uint8_t open_flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE);
+	BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags = DEFAULT_OPEN_FLAGS,
+	                   FileOpener *opener = nullptr);
 
 	FileSystem &fs;
 	unique_ptr<data_t[]> data;
@@ -13058,6 +13064,7 @@ class Appender;
 class Catalog;
 class ChunkCollection;
 class DatabaseInstance;
+class FileOpener;
 class LogicalOperator;
 class PreparedStatementData;
 class Relation;
@@ -13129,6 +13136,8 @@ public:
 	//! The schema search path, in order by which entries are searched if no schema entry is provided
 	vector<string> catalog_search_path = {TEMP_SCHEMA, DEFAULT_SCHEMA, "pg_catalog"};
 
+	unique_ptr<FileOpener> file_opener;
+
 public:
 	DUCKDB_API Transaction &ActiveTransaction() {
 		return transaction.ActiveTransaction();