duckdb 0.2.10-dev320.0 → 0.2.10-dev331.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.2.10-dev320.0",
+  "version": "0.2.10-dev331.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -18182,6 +18182,10 @@ FileSystem &FileSystem::GetFileSystem(ClientContext &context) {
 	return *context.db->config.file_system;
 }
 
+FileOpener *FileSystem::GetFileOpener(ClientContext &context) {
+	return context.file_opener.get();
+}
+
 #ifndef _WIN32
 string FileSystem::PathSeparator() {
 	return "/";
@@ -18283,7 +18287,7 @@ string FileSystem::GetHomeDirectory() {
 
 // LCOV_EXCL_START
 unique_ptr<FileHandle> FileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock,
-                                            FileCompressionType compression) {
+                                            FileCompressionType compression, FileOpener *opener) {
 	throw NotImplementedException("%s: OpenFile is not implemented!", GetName());
 }
 
@@ -20215,7 +20219,8 @@ namespace duckdb {
 class LocalFileSystem : public FileSystem {
 public:
 	unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = FileLockType::NO_LOCK,
-	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED) override;
+	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
+	                                FileOpener *opener = nullptr) override;
 
 	//! Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Read().
@@ -20363,7 +20368,7 @@ public:
 };
 
 unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock_type,
-                                                 FileCompressionType compression) {
+                                                 FileCompressionType compression, FileOpener *opener) {
 	if (compression != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
@@ -20721,7 +20726,7 @@ public:
 };
 
 unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock_type,
-                                                 FileCompressionType compression) {
+                                                 FileCompressionType compression, FileOpener *opener) {
 	if (compression != FileCompressionType::UNCOMPRESSED) {
 		throw NotImplementedException("Unsupported compression type for default file system");
 	}
@@ -28380,7 +28385,7 @@ namespace duckdb {
 
 class BufferedFileReader : public Deserializer {
 public:
-	BufferedFileReader(FileSystem &fs, const char *path);
+	BufferedFileReader(FileSystem &fs, const char *path, FileOpener *opener = nullptr);
 
 	FileSystem &fs;
 	unique_ptr<data_t[]> data;
@@ -28412,9 +28417,10 @@ private:
 
 namespace duckdb {
 
-BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path)
+BufferedFileReader::BufferedFileReader(FileSystem &fs, const char *path, FileOpener *opener)
     : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), read_data(0), total_read(0) {
-	handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::READ_LOCK);
+	handle =
+	    fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::READ_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 	file_size = fs.GetFileSize(*handle);
 }
 
@@ -28467,9 +28473,12 @@ bool BufferedFileReader::Finished() {
 
 namespace duckdb {
 
-BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags)
+// Remove this when we switch C++17: https://stackoverflow.com/a/53350948
+constexpr uint8_t BufferedFileWriter::DEFAULT_OPEN_FLAGS;
+
+BufferedFileWriter::BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags, FileOpener *opener)
     : fs(fs), data(unique_ptr<data_t[]>(new data_t[FILE_BUFFER_SIZE])), offset(0), total_written(0) {
-	handle = fs.OpenFile(path, open_flags, FileLockType::WRITE_LOCK);
+	handle = fs.OpenFile(path, open_flags, FileLockType::WRITE_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 }
 
 int64_t BufferedFileWriter::GetFileSize() {
@@ -45310,7 +45319,8 @@ public:
 	VirtualFileSystem();
 
 	unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = FileLockType::NO_LOCK,
-	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED) override;
+	                                FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
+	                                FileOpener *opener = nullptr) override;
 
 	void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
 		handle.file_system.Read(handle, buffer, nr_bytes, location);
@@ -45414,7 +45424,7 @@ VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) {
 }
 
 unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock,
-                                                   FileCompressionType compression) {
+                                                   FileCompressionType compression, FileOpener *opener) {
 	if (compression == FileCompressionType::AUTO_DETECT) {
 		// auto detect compression settings based on file name
 		auto lower_path = StringUtil::Lower(path);
@@ -45425,7 +45435,7 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 		}
 	}
 	// open the base file handle
-	auto file_handle = FindFileSystem(path)->OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED);
+	auto file_handle = FindFileSystem(path)->OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
 	if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
 		file_handle = PipeFileSystem::OpenPipe(move(file_handle));
 	} else if (compression != FileCompressionType::UNCOMPRESSED) {
@@ -59462,6 +59472,7 @@ struct CopyInfo;
 struct FileHandle;
 struct StrpTimeFormat;
 
+class FileOpener;
 class FileSystem;
 
 //! The shifts array allows for linear searching of multi-byte values. For each position, it determines the next
@@ -59562,10 +59573,11 @@ public:
 	BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options,
 	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
 
-	BufferedCSVReader(FileSystem &fs, BufferedCSVReaderOptions options,
+	BufferedCSVReader(FileSystem &fs, FileOpener *opener, BufferedCSVReaderOptions options,
 	                  const vector<LogicalType> &requested_types = vector<LogicalType>());
 
 	FileSystem &fs;
+	FileOpener *opener;
 	BufferedCSVReaderOptions options;
 	vector<LogicalType> sql_types;
 	vector<string> col_names;
@@ -59798,16 +59810,17 @@ TextSearchShiftArray::TextSearchShiftArray(string search_term) : length(search_t
 	}
 }
 
-BufferedCSVReader::BufferedCSVReader(FileSystem &fs_p, BufferedCSVReaderOptions options_p,
+BufferedCSVReader::BufferedCSVReader(FileSystem &fs_p, FileOpener *opener_p, BufferedCSVReaderOptions options_p,
                                      const vector<LogicalType> &requested_types)
-    : fs(fs_p), options(move(options_p)), buffer_size(0), position(0), start(0) {
+    : fs(fs_p), opener(opener_p), options(move(options_p)), buffer_size(0), position(0), start(0) {
 	file_handle = OpenCSV(options);
 	Initialize(requested_types);
 }
 
 BufferedCSVReader::BufferedCSVReader(ClientContext &context, BufferedCSVReaderOptions options_p,
                                      const vector<LogicalType> &requested_types)
-    : BufferedCSVReader(FileSystem::GetFileSystem(context), move(options_p), requested_types) {
+    : BufferedCSVReader(FileSystem::GetFileSystem(context), FileSystem::GetFileOpener(context), move(options_p),
+                        requested_types) {
 }
 
 void BufferedCSVReader::Initialize(const vector<LogicalType> &requested_types) {
@@ -59839,8 +59852,8 @@ unique_ptr<FileHandle> BufferedCSVReader::OpenCSV(const BufferedCSVReaderOptions
 		this->compression = FileCompressionType::GZIP;
 	}
 
-	auto result =
-	    fs.OpenFile(options.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, this->compression);
+	auto result = fs.OpenFile(options.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
+	                          this->compression, this->opener);
 	plain_file_source = result->OnDiskFile() && result->CanSeek();
 	file_size = result->GetFileSize();
 	return result;
@@ -61584,10 +61597,10 @@ static void WriteCatalogEntries(stringstream &ss, vector<CatalogEntry *> &entrie
 	ss << std::endl;
 }
 
-static void WriteStringStreamToFile(FileSystem &fs, stringstream &ss, const string &path) {
+static void WriteStringStreamToFile(FileSystem &fs, FileOpener *opener, stringstream &ss, const string &path) {
 	auto ss_string = ss.str();
 	auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
-	                          FileLockType::WRITE_LOCK);
+	                          FileLockType::WRITE_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 	fs.Write(*handle, (void *)ss_string.c_str(), ss_string.size());
 	handle.reset();
 }
@@ -61641,6 +61654,7 @@ static void WriteCopyStatement(FileSystem &fs, stringstream &ss, TableCatalogEnt
 void PhysicalExport::GetChunkInternal(ExecutionContext &context, DataChunk &chunk, PhysicalOperatorState *state) const {
 	auto &ccontext = context.client;
 	auto &fs = FileSystem::GetFileSystem(ccontext);
+	auto *opener = FileSystem::GetFileOpener(ccontext);
 
 	// gather all catalog types to export
 	vector<CatalogEntry *> schemas;
@@ -61680,7 +61694,7 @@ void PhysicalExport::GetChunkInternal(ExecutionContext &context, DataChunk &chun
 	WriteCatalogEntries(ss, views);
 	WriteCatalogEntries(ss, indexes);
 
-	WriteStringStreamToFile(fs, ss, fs.JoinPath(info->file_path, "schema.sql"));
+	WriteStringStreamToFile(fs, opener, ss, fs.JoinPath(info->file_path, "schema.sql"));
 
 	// write the load.sql file
 	// for every table, we write COPY INTO statement with the specified options
@@ -61690,7 +61704,7 @@ void PhysicalExport::GetChunkInternal(ExecutionContext &context, DataChunk &chun
 		auto exported_table_info = kv.second;
 		WriteCopyStatement(fs, load_ss, table, *info, exported_table_info, function);
 	}
-	WriteStringStreamToFile(fs, load_ss, fs.JoinPath(info->file_path, "load.sql"));
+	WriteStringStreamToFile(fs, opener, load_ss, fs.JoinPath(info->file_path, "load.sql"));
 	state->finished = true;
 }
 
@@ -76035,7 +76049,9 @@ static void PragmaLogQueryPath(ClientContext &context, const FunctionParameters
 		// empty path: clean up query writer
 		context.log_query_writer = nullptr;
 	} else {
-		context.log_query_writer = make_unique<BufferedFileWriter>(FileSystem::GetFileSystem(context), str_val);
+		context.log_query_writer =
+		    make_unique<BufferedFileWriter>(FileSystem::GetFileSystem(context), str_val,
+		                                    BufferedFileWriter::DEFAULT_OPEN_FLAGS, context.file_opener.get());
 	}
 }
 
@@ -76243,12 +76259,15 @@ string PragmaVersion(ClientContext &context, const FunctionParameters &parameter
 
 string PragmaImportDatabase(ClientContext &context, const FunctionParameters &parameters) {
 	auto &fs = FileSystem::GetFileSystem(context);
+	auto *opener = FileSystem::GetFileOpener(context);
+
 	string query;
 	// read the "shema.sql" and "load.sql" files
 	vector<string> files = {"schema.sql", "load.sql"};
 	for (auto &file : files) {
 		auto file_path = fs.JoinPath(parameters.values[0].ToString(), file);
-		auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ);
+		auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK,
+		                          FileSystem::DEFAULT_COMPRESSION, opener);
 		auto fsize = fs.GetFileSize(*handle);
 		auto buffer = unique_ptr<char[]>(new char[fsize]);
 		fs.Read(*handle, buffer.get(), fsize);
@@ -91223,9 +91242,9 @@ struct LocalReadCSVData : public LocalFunctionData {
 };
 
 struct GlobalWriteCSVData : public GlobalFunctionData {
-	GlobalWriteCSVData(FileSystem &fs, const string &file_path) : fs(fs) {
+	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileOpener *opener) : fs(fs) {
 		handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
-		                     FileLockType::WRITE_LOCK);
+		                     FileLockType::WRITE_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
 	}
 
 	void WriteData(const_data_ptr_t data, idx_t size) {
@@ -91255,7 +91274,8 @@ static unique_ptr<LocalFunctionData> WriteCSVInitializeLocal(ClientContext &cont
 static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &context, FunctionData &bind_data) {
 	auto &csv_data = (WriteCSVData &)bind_data;
 	auto &options = csv_data.options;
-	auto global_data = make_unique<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), csv_data.files[0]);
+	auto global_data = make_unique<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), csv_data.files[0],
+	                                                   FileSystem::GetFileOpener(context));
 
 	if (options.header) {
 		BufferedSerializer serializer;
@@ -96978,6 +96998,61 @@ bool duckdb_value_is_null(duckdb_result *result, idx_t col, idx_t row) {
 	return result->columns[col].nullmask[row];
 }
 
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/main/client_context_file_opener.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/file_opener.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+namespace duckdb {
+
+class Value;
+
+//! Abstact type that provide client-spcific context to FileSystem.
+class FileOpener {
+public:
+	virtual ~FileOpener() {};
+
+	virtual bool TryGetCurrentSetting(const string &key, Value &result) = 0;
+};
+
+} // namespace duckdb
+
+namespace duckdb {
+
+class ClientContext;
+
+//! ClientContext-specific FileOpener implemenation.
+//! This object is owned by ClientContext and never outlives it.
+class ClientContextFileOpener : public FileOpener {
+public:
+	explicit ClientContextFileOpener(ClientContext &context_p) : context(context_p) {
+	}
+
+	bool TryGetCurrentSetting(const string &key, Value &result) override;
+
+private:
+	ClientContext &context;
+};
+
+} // namespace duckdb
+
 
 
 
@@ -97428,7 +97503,8 @@ private:
 ClientContext::ClientContext(shared_ptr<DatabaseInstance> database)
     : profiler(make_unique<QueryProfiler>()), query_profiler_history(make_unique<QueryProfilerHistory>()),
       db(move(database)), transaction(db->GetTransactionManager(), *this), interrupted(false), executor(*this),
-      temporary_objects(make_unique<SchemaCatalogEntry>(&db->GetCatalog(), TEMP_SCHEMA, true)), open_result(nullptr) {
+      temporary_objects(make_unique<SchemaCatalogEntry>(&db->GetCatalog(), TEMP_SCHEMA, true)),
+      file_opener(make_unique<ClientContextFileOpener>(*this)), open_result(nullptr) {
 	std::random_device rd;
 	random_engine.seed(rd());
 
@@ -97895,7 +97971,8 @@ void ClientContext::LogQueryInternal(ClientContextLock &, const string &query) {
 #ifdef DUCKDB_FORCE_QUERY_LOG
 		try {
 			string log_path(DUCKDB_FORCE_QUERY_LOG);
-			log_query_writer = make_unique<BufferedFileWriter>(FileSystem::GetFileSystem(*this), log_path);
+			log_query_writer = make_unique<BufferedFileWriter>(
+			    FileSystem::GetFileSystem(*this), log_path, BufferedFileWriter::DEFAULT_OPEN_FLAGS, file_opener.get());
 		} catch (...) {
 			return;
 		}
@@ -98315,6 +98392,16 @@ bool ClientContext::TryGetCurrentSetting(const std::string &key, Value &result)
 
 
 
+namespace duckdb {
+
+bool ClientContextFileOpener::TryGetCurrentSetting(const string &key, Value &result) {
+	return context.TryGetCurrentSetting(key, result);
+}
+
+} // namespace duckdb
+
+
+
 namespace duckdb {
 
 static ConfigurationOption internal_options[] = {
@@ -145876,6 +145963,29 @@ idx_t LocalStorage::Delete(DataTable *table, Vector &row_ids, idx_t count) {
 	idx_t chunk_idx = GetChunk(row_ids);
 	D_ASSERT(chunk_idx < storage->collection.ChunkCount());
 
+	// delete from unique indices (if any)
+	if (!storage->indexes.empty()) {
+		// Index::Delete assumes that ALL rows are being deleted, so
+		// Slice out the rows that are being deleted from the storage Chunk
+		auto &chunk = storage->collection.GetChunk(chunk_idx);
+
+		VectorData row_ids_data;
+		row_ids.Orrify(count, row_ids_data);
+		auto row_identifiers = (const row_t *)row_ids_data.data;
+		SelectionVector sel(count);
+		for (idx_t i = 0; i < count; ++i) {
+			const auto idx = row_ids_data.sel->get_index(i);
+			sel.set_index(i, row_identifiers[idx] - MAX_ROW_ID);
+		}
+
+		DataChunk deleted;
+		deleted.InitializeEmpty(chunk.GetTypes());
+		deleted.Slice(chunk, sel, count);
+		for (auto &index : storage->indexes) {
+			index->Delete(deleted, row_ids);
+		}
+	}
+
 	// get a pointer to the deleted entries for this chunk
 	bool *deleted;
 	auto entry = storage->deleted_entries.find(chunk_idx);

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "153b23b3a"
-#define DUCKDB_VERSION "0.2.10-dev320"
+#define DUCKDB_SOURCE_ID "7dd852e1d"
+#define DUCKDB_VERSION "0.2.10-dev331"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -1710,6 +1710,7 @@ enum class FileCompressionType : uint8_t { AUTO_DETECT = 0, UNCOMPRESSED = 1, GZ
 namespace duckdb {
 class ClientContext;
 class DatabaseInstance;
+class FileOpener;
 class FileSystem;
 
 enum class FileType {
@@ -1787,12 +1788,15 @@ public:
 	}
 
 public:
+	static constexpr FileLockType DEFAULT_LOCK = FileLockType::NO_LOCK;
+	static constexpr FileCompressionType DEFAULT_COMPRESSION = FileCompressionType::UNCOMPRESSED;
 	static FileSystem &GetFileSystem(ClientContext &context);
 	static FileSystem &GetFileSystem(DatabaseInstance &db);
+	static FileOpener *GetFileOpener(ClientContext &context);
 
-	virtual unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags,
-	                                        FileLockType lock = FileLockType::NO_LOCK,
-	                                        FileCompressionType compression = FileCompressionType::UNCOMPRESSED);
+	virtual unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = DEFAULT_LOCK,
+	                                        FileCompressionType compression = DEFAULT_COMPRESSION,
+	                                        FileOpener *opener = nullptr);
 
 	//! Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is
 	//! equivalent to calling SetFilePointer(location) followed by calling Read().
@@ -1889,10 +1893,12 @@ namespace duckdb {
 
 class BufferedFileWriter : public Serializer {
 public:
+	static constexpr uint8_t DEFAULT_OPEN_FLAGS = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE;
+
 	//! Serializes to a buffer allocated by the serializer, will expand when
 	//! writing past the initial threshold
-	BufferedFileWriter(FileSystem &fs, const string &path,
-	                   uint8_t open_flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE);
+	BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags = DEFAULT_OPEN_FLAGS,
+	                   FileOpener *opener = nullptr);
 
 	FileSystem &fs;
 	unique_ptr<data_t[]> data;
@@ -13058,6 +13064,7 @@ class Appender;
 class Catalog;
 class ChunkCollection;
 class DatabaseInstance;
+class FileOpener;
 class LogicalOperator;
 class PreparedStatementData;
 class Relation;
@@ -13129,6 +13136,8 @@ public:
 	//! The schema search path, in order by which entries are searched if no schema entry is provided
 	vector<string> catalog_search_path = {TEMP_SCHEMA, DEFAULT_SCHEMA, "pg_catalog"};
 
+	unique_ptr<FileOpener> file_opener;
+
 public:
 	DUCKDB_API Transaction &ActiveTransaction() {
 		return transaction.ActiveTransaction();