duckdb 0.3.2-dev300.0 → 0.3.2-dev324.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.3.2-dev300.0",
+  "version": "0.3.2-dev324.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -10187,6 +10187,217 @@ uint64_t Checksum(uint8_t *buffer, size_t size) {
 }
 
 } // namespace duckdb
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/common/compressed_file_system.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+
+
+namespace duckdb {
+class CompressedFile;
+
+struct StreamData {
+	// various buffers & pointers
+	bool write = false;
+	unique_ptr<data_t[]> in_buff;
+	unique_ptr<data_t[]> out_buff;
+	data_ptr_t out_buff_start = nullptr;
+	data_ptr_t out_buff_end = nullptr;
+	data_ptr_t in_buff_start = nullptr;
+	data_ptr_t in_buff_end = nullptr;
+
+	idx_t in_buf_size = 0;
+	idx_t out_buf_size = 0;
+};
+
+struct StreamWrapper {
+	DUCKDB_API virtual ~StreamWrapper();
+
+	DUCKDB_API virtual void Initialize(CompressedFile &file, bool write) = 0;
+	DUCKDB_API virtual bool Read(StreamData &stream_data) = 0;
+	DUCKDB_API virtual void Write(CompressedFile &file, StreamData &stream_data, data_ptr_t buffer,
+	                              int64_t nr_bytes) = 0;
+	DUCKDB_API virtual void Close() = 0;
+};
+
+class CompressedFileSystem : public FileSystem {
+public:
+	DUCKDB_API int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
+	DUCKDB_API int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
+
+	DUCKDB_API void Reset(FileHandle &handle) override;
+
+	DUCKDB_API int64_t GetFileSize(FileHandle &handle) override;
+
+	DUCKDB_API bool OnDiskFile(FileHandle &handle) override;
+	DUCKDB_API bool CanSeek() override;
+
+	DUCKDB_API virtual unique_ptr<StreamWrapper> CreateStream() = 0;
+	DUCKDB_API virtual idx_t InBufferSize() = 0;
+	DUCKDB_API virtual idx_t OutBufferSize() = 0;
+};
+
+class CompressedFile : public FileHandle {
+public:
+	DUCKDB_API CompressedFile(CompressedFileSystem &fs, unique_ptr<FileHandle> child_handle_p, const string &path);
+	DUCKDB_API virtual ~CompressedFile() override;
+
+	CompressedFileSystem &compressed_fs;
+	unique_ptr<FileHandle> child_handle;
+	//! Whether the file is opened for reading or for writing
+	bool write = false;
+	StreamData stream_data;
+
+public:
+	DUCKDB_API void Initialize(bool write);
+	DUCKDB_API int64_t ReadData(void *buffer, int64_t nr_bytes);
+	DUCKDB_API int64_t WriteData(data_ptr_t buffer, int64_t nr_bytes);
+	DUCKDB_API void Close() override;
+
+private:
+	unique_ptr<StreamWrapper> stream_wrapper;
+};
+
+} // namespace duckdb
+
+
+namespace duckdb {
+
+StreamWrapper::~StreamWrapper() {
+}
+
+CompressedFile::CompressedFile(CompressedFileSystem &fs, unique_ptr<FileHandle> child_handle_p, const string &path)
+    : FileHandle(fs, path), compressed_fs(fs), child_handle(move(child_handle_p)) {
+}
+
+CompressedFile::~CompressedFile() {
+	Close();
+}
+
+void CompressedFile::Initialize(bool write) {
+	Close();
+
+	this->write = write;
+	stream_data.in_buf_size = compressed_fs.InBufferSize();
+	stream_data.out_buf_size = compressed_fs.OutBufferSize();
+	stream_data.in_buff = unique_ptr<data_t[]>(new data_t[stream_data.in_buf_size]);
+	stream_data.in_buff_start = stream_data.in_buff.get();
+	stream_data.in_buff_end = stream_data.in_buff.get();
+	stream_data.out_buff = unique_ptr<data_t[]>(new data_t[stream_data.out_buf_size]);
+	stream_data.out_buff_start = stream_data.out_buff.get();
+	stream_data.out_buff_end = stream_data.out_buff.get();
+
+	stream_wrapper = compressed_fs.CreateStream();
+	stream_wrapper->Initialize(*this, write);
+}
+
+int64_t CompressedFile::ReadData(void *buffer, int64_t remaining) {
+	idx_t total_read = 0;
+	while (true) {
+		// first check if there are input bytes available in the output buffers
+		if (stream_data.out_buff_start != stream_data.out_buff_end) {
+			// there is! copy it into the output buffer
+			idx_t available = MinValue<idx_t>(remaining, stream_data.out_buff_end - stream_data.out_buff_start);
+			memcpy(data_ptr_t(buffer) + total_read, stream_data.out_buff_start, available);
+
+			// increment the total read variables as required
+			stream_data.out_buff_start += available;
+			total_read += available;
+			remaining -= available;
+			if (remaining == 0) {
+				// done! read enough
+				return total_read;
+			}
+		}
+		if (!stream_wrapper) {
+			return total_read;
+		}
+
+		// ran out of buffer: read more data from the child stream
+		stream_data.out_buff_start = stream_data.out_buff.get();
+		stream_data.out_buff_end = stream_data.out_buff.get();
+		D_ASSERT(stream_data.in_buff_start <= stream_data.in_buff_end);
+		D_ASSERT(stream_data.in_buff_end <= stream_data.in_buff_start + stream_data.in_buf_size);
+
+		// read more input if none available
+		if (stream_data.in_buff_start == stream_data.in_buff_end) {
+			// empty input buffer: refill from the start
+			stream_data.in_buff_start = stream_data.in_buff.get();
+			stream_data.in_buff_end = stream_data.in_buff_start;
+			auto sz = child_handle->Read(stream_data.in_buff.get(), stream_data.in_buf_size);
+			if (sz <= 0) {
+				stream_wrapper.reset();
+				break;
+			}
+			stream_data.in_buff_end = stream_data.in_buff_start + sz;
+		}
+
+		auto finished = stream_wrapper->Read(stream_data);
+		if (finished) {
+			stream_wrapper.reset();
+		}
+	}
+	return total_read;
+}
+
+int64_t CompressedFile::WriteData(data_ptr_t buffer, int64_t nr_bytes) {
+	stream_wrapper->Write(*this, stream_data, buffer, nr_bytes);
+	return nr_bytes;
+}
+
+void CompressedFile::Close() {
+	if (stream_wrapper) {
+		stream_wrapper->Close();
+		stream_wrapper.reset();
+	}
+	stream_data.in_buff.reset();
+	stream_data.out_buff.reset();
+	stream_data.out_buff_start = nullptr;
+	stream_data.out_buff_end = nullptr;
+	stream_data.in_buff_start = nullptr;
+	stream_data.in_buff_end = nullptr;
+	stream_data.in_buf_size = 0;
+	stream_data.out_buf_size = 0;
+}
+
+int64_t CompressedFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
+	auto &compressed_file = (CompressedFile &)handle;
+	return compressed_file.ReadData(buffer, nr_bytes);
+}
+
+int64_t CompressedFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
+	auto &compressed_file = (CompressedFile &)handle;
+	return compressed_file.WriteData((data_ptr_t)buffer, nr_bytes);
+}
+
+void CompressedFileSystem::Reset(FileHandle &handle) {
+	auto &compressed_file = (CompressedFile &)handle;
+	compressed_file.child_handle->Reset();
+	compressed_file.Initialize(compressed_file.write);
+}
+
+int64_t CompressedFileSystem::GetFileSize(FileHandle &handle) {
+	auto &compressed_file = (CompressedFile &)handle;
+	return compressed_file.child_handle->GetFileSize();
+}
+
+bool CompressedFileSystem::OnDiskFile(FileHandle &handle) {
+	auto &compressed_file = (CompressedFile &)handle;
+	return compressed_file.child_handle->OnDiskFile();
+}
+
+bool CompressedFileSystem::CanSeek() {
+	return false;
+}
+
+} // namespace duckdb
 
 
 
@@ -10931,6 +11142,27 @@ ExpressionType FlipComparisionExpression(ExpressionType type) {
 } // namespace duckdb
 
 
+
+namespace duckdb {
+
+FileCompressionType FileCompressionTypeFromString(const string &input) {
+	auto parameter = StringUtil::Lower(input);
+	if (parameter == "infer" || parameter == "auto") {
+		return FileCompressionType::AUTO_DETECT;
+	} else if (parameter == "gzip") {
+		return FileCompressionType::GZIP;
+	} else if (parameter == "zstd") {
+		return FileCompressionType::ZSTD;
+	} else if (parameter == "uncompressed" || parameter == "none" || parameter.empty()) {
+		return FileCompressionType::UNCOMPRESSED;
+	} else {
+		throw ParserException("Unrecognized file compression type \"%s\"", input);
+	}
+}
+
+} // namespace duckdb
+
+
 namespace duckdb {
 
 string JoinTypeToString(JoinType type) {
@@ -18692,6 +18924,10 @@ void FileSystem::RegisterSubSystem(unique_ptr<FileSystem> sub_fs) {
 	throw NotImplementedException("%s: Can't register a sub system on a non-virtual file system", GetName());
 }
 
+void FileSystem::RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> sub_fs) {
+	throw NotImplementedException("%s: Can't register a sub system on a non-virtual file system", GetName());
+}
+
 bool FileSystem::CanHandleFile(const string &fpath) {
 	throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
 }
@@ -18712,6 +18948,10 @@ bool FileSystem::CanSeek() {
 	throw NotImplementedException("%s: CanSeek is not implemented!", GetName());
 }
 
+unique_ptr<FileHandle> FileSystem::OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) {
+	throw NotImplementedException("%s: OpenCompressedFile is not implemented!", GetName());
+}
+
 bool FileSystem::OnDiskFile(FileHandle &handle) {
 	throw NotImplementedException("%s: OnDiskFile is not implemented!", GetName());
 }
@@ -18798,19 +19038,15 @@ FileType FileHandle::GetType() {
 
 namespace duckdb {
 
-class GZipFileSystem : public FileSystem {
-public:
-	static unique_ptr<FileHandle> OpenCompressedFile(unique_ptr<FileHandle> handle);
-
-	int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
-
-	void Reset(FileHandle &handle) override;
+class GZipFileSystem : public CompressedFileSystem {
+	// 32 KB
+	static constexpr const idx_t BUFFER_SIZE = 1 << 15;
 
-	int64_t GetFileSize(FileHandle &handle) override;
+public:
+	unique_ptr<FileHandle> OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) override;
 
-	bool OnDiskFile(FileHandle &handle) override;
-	bool CanSeek() override {
-		return false;
+	std::string GetName() const override {
+		return "GZipFileSystem";
 	}
 
 	//! Verifies that a buffer contains a valid GZIP header
@@ -18818,9 +19054,9 @@ public:
 	//! Consumes a byte stream as a gzip string, returning the decompressed string
 	static string UncompressGZIPString(const string &in);
 
-	std::string GetName() const override {
-		return "GZipFileSystem";
-	}
+	unique_ptr<StreamWrapper> CreateStream() override;
+	idx_t InBufferSize() override;
+	idx_t OutBufferSize() override;
 };
 
 static constexpr const uint8_t GZIP_COMPRESSION_DEFLATE = 0x08;
@@ -20142,6 +20378,155 @@ void *mz_zip_extract_archive_file_to_heap_v2(const char *pZip_filename, const ch
 
 
 
+// LICENSE_CHANGE_BEGIN
+// The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #3
+// See the end of this file for a list
+
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// miniz_wrapper.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+
+
+
+#include <string>
+#include <stdexcept>
+
+namespace duckdb {
+
+enum class MiniZStreamType {
+	MINIZ_TYPE_NONE,
+	MINIZ_TYPE_INFLATE,
+	MINIZ_TYPE_DEFLATE
+};
+
+struct MiniZStream {
+	static constexpr uint8_t GZIP_HEADER_MINSIZE = 10;
+	static constexpr uint8_t GZIP_FOOTER_SIZE = 8;
+	static constexpr uint8_t GZIP_COMPRESSION_DEFLATE = 0x08;
+	static constexpr unsigned char GZIP_FLAG_UNSUPPORTED = 0x1 | 0x2 | 0x4 | 0x10 | 0x20;
+
+public:
+	MiniZStream() : type(MiniZStreamType::MINIZ_TYPE_NONE) {
+		memset(&stream, 0, sizeof(duckdb_miniz::mz_stream));
+	}
+	~MiniZStream() {
+		switch(type) {
+		case MiniZStreamType::MINIZ_TYPE_INFLATE:
+			duckdb_miniz::mz_inflateEnd(&stream);
+			break;
+		case MiniZStreamType::MINIZ_TYPE_DEFLATE:
+			duckdb_miniz::mz_deflateEnd(&stream);
+			break;
+		default:
+			break;
+		}
+	}
+	void FormatException(std::string error_msg) {
+		throw std::runtime_error(error_msg);
+	}
+	void FormatException(const char *error_msg, int mz_ret) {
+		auto err = duckdb_miniz::mz_error(mz_ret);
+		FormatException(error_msg + std::string(": ") + (err ? err : "Unknown error code"));
+	}
+	void Decompress(const char *compressed_data, size_t compressed_size, char *out_data, size_t out_size) {
+		auto mz_ret = mz_inflateInit2(&stream, -MZ_DEFAULT_WINDOW_BITS);
+		if (mz_ret != duckdb_miniz::MZ_OK) {
+			FormatException("Failed to initialize miniz", mz_ret);
+		}
+		type = MiniZStreamType::MINIZ_TYPE_INFLATE;
+
+		if (compressed_size < GZIP_HEADER_MINSIZE) {
+			FormatException("Failed to decompress GZIP block: compressed size is less than gzip header size");
+		}
+		auto gzip_hdr = (const unsigned char *)compressed_data;
+		if (gzip_hdr[0] != 0x1F || gzip_hdr[1] != 0x8B || gzip_hdr[2] != GZIP_COMPRESSION_DEFLATE ||
+		    gzip_hdr[3] & GZIP_FLAG_UNSUPPORTED) {
+			FormatException("Input is invalid/unsupported GZIP stream");
+		}
+
+		stream.next_in = (const unsigned char *)compressed_data + GZIP_HEADER_MINSIZE;
+		stream.avail_in = compressed_size - GZIP_HEADER_MINSIZE;
+		stream.next_out = (unsigned char *)out_data;
+		stream.avail_out = out_size;
+
+		mz_ret = mz_inflate(&stream, duckdb_miniz::MZ_FINISH);
+		if (mz_ret != duckdb_miniz::MZ_OK && mz_ret != duckdb_miniz::MZ_STREAM_END) {
+			FormatException("Failed to decompress GZIP block", mz_ret);
+		}
+	}
+	size_t MaxCompressedLength(size_t input_size) {
+		return duckdb_miniz::mz_compressBound(input_size) + GZIP_HEADER_MINSIZE + GZIP_FOOTER_SIZE;
+	}
+	static void InitializeGZIPHeader(unsigned char *gzip_header) {
+		memset(gzip_header, 0, GZIP_HEADER_MINSIZE);
+		gzip_header[0] = 0x1F;
+		gzip_header[1] = 0x8B;
+		gzip_header[2] = GZIP_COMPRESSION_DEFLATE;
+		gzip_header[3] = 0;
+		gzip_header[4] = 0;
+		gzip_header[5] = 0;
+		gzip_header[6] = 0;
+		gzip_header[7] = 0;
+		gzip_header[8] = 0;
+		gzip_header[9] = 0xFF;
+	}
+
+	static void InitializeGZIPFooter(unsigned char *gzip_footer, duckdb_miniz::mz_ulong crc, idx_t uncompressed_size) {
+		gzip_footer[0] = crc & 0xFF;
+		gzip_footer[1] = (crc >> 8) & 0xFF;
+		gzip_footer[2] = (crc >> 16) & 0xFF;
+		gzip_footer[3] = (crc >> 24) & 0xFF;
+		gzip_footer[4] = uncompressed_size & 0xFF;
+		gzip_footer[5] = (uncompressed_size >> 8) & 0xFF;
+		gzip_footer[6] = (uncompressed_size >> 16) & 0xFF;
+		gzip_footer[7] = (uncompressed_size >> 24) & 0xFF;
+	}
+
+	void Compress(const char *uncompressed_data, size_t uncompressed_size, char *out_data, size_t *out_size) {
+		auto mz_ret = mz_deflateInit2(&stream, duckdb_miniz::MZ_DEFAULT_LEVEL, MZ_DEFLATED, -MZ_DEFAULT_WINDOW_BITS, 1, 0);
+		if (mz_ret != duckdb_miniz::MZ_OK) {
+			FormatException("Failed to initialize miniz", mz_ret);
+		}
+		type = MiniZStreamType::MINIZ_TYPE_DEFLATE;
+
+		auto gzip_header = (unsigned char*) out_data;
+		InitializeGZIPHeader(gzip_header);
+
+		auto gzip_body = gzip_header + GZIP_HEADER_MINSIZE;
+
+		stream.next_in = (const unsigned char*) uncompressed_data;
+		stream.avail_in = uncompressed_size;
+		stream.next_out = gzip_body;
+		stream.avail_out = *out_size - GZIP_HEADER_MINSIZE;
+
+		mz_ret = mz_deflate(&stream, duckdb_miniz::MZ_FINISH);
+		if (mz_ret != duckdb_miniz::MZ_OK && mz_ret != duckdb_miniz::MZ_STREAM_END) {
+			FormatException("Failed to compress GZIP block", mz_ret);
+		}
+		auto gzip_footer = gzip_body + stream.total_out;
+		auto crc = duckdb_miniz::mz_crc32(MZ_CRC32_INIT, (const unsigned char*) uncompressed_data, uncompressed_size);
+		InitializeGZIPFooter(gzip_footer, crc, uncompressed_size);
+
+		*out_size = stream.total_out + GZIP_HEADER_MINSIZE + GZIP_FOOTER_SIZE;
+	}
+
+private:
+	duckdb_miniz::mz_stream stream;
+	MiniZStreamType type;
+};
+
+}
+
+
+// LICENSE_CHANGE_END
+
+
+
 
 namespace duckdb {
 
@@ -20200,206 +20585,202 @@ static idx_t GZipConsumeString(FileHandle &input) {
 	return size;
 }
 
-struct MiniZStreamWrapper {
-	~MiniZStreamWrapper() {
-		Close();
-	}
+struct MiniZStreamWrapper : public StreamWrapper {
+	~MiniZStreamWrapper() override;
 
+	CompressedFile *file = nullptr;
 	duckdb_miniz::mz_stream *mz_stream_ptr = nullptr;
+	bool writing = false;
+	duckdb_miniz::mz_ulong crc;
+	idx_t total_size;
 
 public:
-	void Initialize() {
-		Close();
-		mz_stream_ptr = new duckdb_miniz::mz_stream();
-		memset(mz_stream_ptr, 0, sizeof(duckdb_miniz::mz_stream));
-	}
+	void Initialize(CompressedFile &file, bool write) override;
 
-	void Close() {
-		if (!mz_stream_ptr) {
-			return;
-		}
-		duckdb_miniz::mz_inflateEnd(mz_stream_ptr);
-		delete mz_stream_ptr;
-		mz_stream_ptr = nullptr;
-	}
-};
-
-class GZipFile : public FileHandle {
-	static constexpr const idx_t BUFFER_SIZE = 1024;
-
-public:
-	GZipFile(unique_ptr<FileHandle> child_handle_p, const string &path)
-	    : FileHandle(gzip_fs, path), child_handle(move(child_handle_p)) {
-		Initialize();
-	}
-	~GZipFile() override {
-		Close();
-	}
+	bool Read(StreamData &stream_data) override;
+	void Write(CompressedFile &file, StreamData &stream_data, data_ptr_t buffer, int64_t nr_bytes) override;
 
-	void Initialize();
-	int64_t ReadData(void *buffer, int64_t nr_bytes);
+	void Close() override;
 
-	GZipFileSystem gzip_fs;
-	unique_ptr<FileHandle> child_handle;
-
-protected:
-	void Close() override {
-		miniz_stream.reset();
-		in_buff.reset();
-		out_buff.reset();
-	}
-
-private:
-	idx_t data_start = 0;
-	unique_ptr<MiniZStreamWrapper> miniz_stream;
-	// various buffers & pointers
-	unique_ptr<data_t[]> in_buff;
-	unique_ptr<data_t[]> out_buff;
-	data_ptr_t out_buff_start = nullptr;
-	data_ptr_t out_buff_end = nullptr;
-	data_ptr_t in_buff_start = nullptr;
-	data_ptr_t in_buff_end = nullptr;
+	void FlushStream();
 };
 
-void GZipFileSystem::VerifyGZIPHeader(uint8_t gzip_hdr[], idx_t read_count) {
-	// check for incorrectly formatted files
-	if (read_count != GZIP_HEADER_MINSIZE) {
-		throw IOException("Input is not a GZIP stream");
-	}
-	if (gzip_hdr[0] != 0x1F || gzip_hdr[1] != 0x8B) { // magic header
-		throw IOException("Input is not a GZIP stream");
-	}
-	if (gzip_hdr[2] != GZIP_COMPRESSION_DEFLATE) { // compression method
-		throw IOException("Unsupported GZIP compression method");
+MiniZStreamWrapper::~MiniZStreamWrapper() {
+	// avoid closing if destroyed during stack unwinding
+	if (std::uncaught_exception()) {
+		return;
 	}
-	if (gzip_hdr[3] & GZIP_FLAG_UNSUPPORTED) {
-		throw IOException("Unsupported GZIP archive");
+	try {
+		Close();
+	} catch (...) {
 	}
 }
 
-void GZipFile::Initialize() {
+void MiniZStreamWrapper::Initialize(CompressedFile &file, bool write) {
 	Close();
+	this->file = &file;
+	mz_stream_ptr = new duckdb_miniz::mz_stream();
+	memset(mz_stream_ptr, 0, sizeof(duckdb_miniz::mz_stream));
+	this->writing = write;
 
-	D_ASSERT(BUFFER_SIZE >= 3); // found to work fine with 3
+	// TODO use custom alloc/free methods in miniz to throw exceptions on OOM
 	uint8_t gzip_hdr[GZIP_HEADER_MINSIZE];
-	data_start = GZIP_HEADER_MINSIZE;
-
-	in_buff = unique_ptr<data_t[]>(new data_t[BUFFER_SIZE]);
-	in_buff_start = in_buff.get();
-	in_buff_end = in_buff.get();
-	out_buff = unique_ptr<data_t[]>(new data_t[BUFFER_SIZE]);
-	out_buff_start = out_buff.get();
-	out_buff_end = out_buff.get();
-
-	miniz_stream = make_unique<MiniZStreamWrapper>();
-	miniz_stream->Initialize();
+	if (write) {
+		crc = MZ_CRC32_INIT;
+		total_size = 0;
 
-	auto &mz_stream_ptr = miniz_stream->mz_stream_ptr;
+		MiniZStream::InitializeGZIPHeader(gzip_hdr);
+		file.child_handle->Write(gzip_hdr, GZIP_HEADER_MINSIZE);
 
-	// TODO use custom alloc/free methods in miniz to throw exceptions on OOM
-	auto read_count = child_handle->Read(gzip_hdr, GZIP_HEADER_MINSIZE);
-	GZipFileSystem::VerifyGZIPHeader(gzip_hdr, read_count);
-
-	if (gzip_hdr[3] & GZIP_FLAG_NAME) {
-		child_handle->Seek(data_start);
-		data_start += GZipConsumeString(*child_handle);
-	}
-	child_handle->Seek(data_start);
-	// stream is now set to beginning of payload data
-	auto ret = duckdb_miniz::mz_inflateInit2((duckdb_miniz::mz_streamp)mz_stream_ptr, -MZ_DEFAULT_WINDOW_BITS);
-	if (ret != duckdb_miniz::MZ_OK) {
-		throw InternalException("Failed to initialize miniz");
+		auto ret = mz_deflateInit2((duckdb_miniz::mz_streamp)mz_stream_ptr, duckdb_miniz::MZ_DEFAULT_LEVEL, MZ_DEFLATED,
+		                           -MZ_DEFAULT_WINDOW_BITS, 1, 0);
+		if (ret != duckdb_miniz::MZ_OK) {
+			throw InternalException("Failed to initialize miniz");
+		}
+	} else {
+		idx_t data_start = GZIP_HEADER_MINSIZE;
+		auto read_count = file.child_handle->Read(gzip_hdr, GZIP_HEADER_MINSIZE);
+		GZipFileSystem::VerifyGZIPHeader(gzip_hdr, read_count);
+
+		if (gzip_hdr[3] & GZIP_FLAG_NAME) {
+			file.child_handle->Seek(data_start);
+			data_start += GZipConsumeString(*file.child_handle);
+		}
+		file.child_handle->Seek(data_start);
+		// stream is now set to beginning of payload data
+		auto ret = duckdb_miniz::mz_inflateInit2((duckdb_miniz::mz_streamp)mz_stream_ptr, -MZ_DEFAULT_WINDOW_BITS);
+		if (ret != duckdb_miniz::MZ_OK) {
+			throw InternalException("Failed to initialize miniz");
+		}
+	}
+}
+
+bool MiniZStreamWrapper::Read(StreamData &sd) {
+	// actually decompress
+	mz_stream_ptr->next_in = (data_ptr_t)sd.in_buff_start;
+	D_ASSERT(sd.in_buff_end - sd.in_buff_start < NumericLimits<int32_t>::Maximum());
+	mz_stream_ptr->avail_in = (uint32_t)(sd.in_buff_end - sd.in_buff_start);
+	mz_stream_ptr->next_out = (data_ptr_t)sd.out_buff_end;
+	mz_stream_ptr->avail_out = (uint32_t)((sd.out_buff.get() + sd.out_buf_size) - sd.out_buff_end);
+	auto ret = duckdb_miniz::mz_inflate(mz_stream_ptr, duckdb_miniz::MZ_NO_FLUSH);
+	if (ret != duckdb_miniz::MZ_OK && ret != duckdb_miniz::MZ_STREAM_END) {
+		throw IOException("Failed to decode gzip stream: %s", duckdb_miniz::mz_error(ret));
+	}
+	// update pointers following inflate()
+	sd.in_buff_start = (data_ptr_t)mz_stream_ptr->next_in;
+	sd.in_buff_end = sd.in_buff_start + mz_stream_ptr->avail_in;
+	sd.out_buff_end = (data_ptr_t)mz_stream_ptr->next_out;
+	D_ASSERT(sd.out_buff_end + mz_stream_ptr->avail_out == sd.out_buff.get() + sd.out_buf_size);
+	// if stream ended, deallocate inflator
+	if (ret == duckdb_miniz::MZ_STREAM_END) {
+		Close();
+		return true;
 	}
+	return false;
 }
 
-unique_ptr<FileHandle> GZipFileSystem::OpenCompressedFile(unique_ptr<FileHandle> handle) {
-	auto path = handle->path;
-	return make_unique<GZipFile>(move(handle), path);
-}
+void MiniZStreamWrapper::Write(CompressedFile &file, StreamData &sd, data_ptr_t uncompressed_data,
+                               int64_t uncompressed_size) {
+	// update the src and the total size
+	crc = duckdb_miniz::mz_crc32(crc, (const unsigned char *)uncompressed_data, uncompressed_size);
+	total_size += uncompressed_size;
 
-int64_t GZipFile::ReadData(void *buffer, int64_t remaining) {
-	auto &mz_stream_ptr = miniz_stream->mz_stream_ptr;
-	idx_t total_read = 0;
-	while (true) {
-		// first check if there are input bytes available in the output buffers
-		if (out_buff_start != out_buff_end) {
-			// there is! copy it into the output buffer
-			idx_t available = MinValue<idx_t>(remaining, out_buff_end - out_buff_start);
-			memcpy(data_ptr_t(buffer) + total_read, out_buff_start, available);
+	auto remaining = uncompressed_size;
+	while (remaining > 0) {
+		idx_t output_remaining = (sd.out_buff.get() + sd.out_buf_size) - sd.out_buff_start;
 
-			// increment the total read variables as required
-			out_buff_start += available;
-			total_read += available;
-			remaining -= available;
-			if (remaining == 0) {
-				// done! read enough
-				return total_read;
-			}
+		mz_stream_ptr->next_in = (const unsigned char *)uncompressed_data;
+		mz_stream_ptr->avail_in = remaining;
+		mz_stream_ptr->next_out = sd.out_buff_start;
+		mz_stream_ptr->avail_out = output_remaining;
+
+		auto res = mz_deflate(mz_stream_ptr, duckdb_miniz::MZ_NO_FLUSH);
+		if (res != duckdb_miniz::MZ_OK) {
+			D_ASSERT(res != duckdb_miniz::MZ_STREAM_END);
+			throw InternalException("Failed to compress GZIP block");
 		}
-		if (!mz_stream_ptr) {
-			return total_read;
+		sd.out_buff_start += output_remaining - mz_stream_ptr->avail_out;
+		if (mz_stream_ptr->avail_out == 0) {
+			// no more output buffer available: flush
+			file.child_handle->Write(sd.out_buff.get(), sd.out_buff_start - sd.out_buff.get());
+			sd.out_buff_start = sd.out_buff.get();
 		}
+		idx_t written = remaining - mz_stream_ptr->avail_in;
+		uncompressed_data += written;
+		remaining = mz_stream_ptr->avail_in;
+	}
+}
 
-		// ran out of buffer: read more data from the child stream
-		out_buff_start = out_buff.get();
-		out_buff_end = out_buff.get();
-		D_ASSERT(in_buff_start <= in_buff_end);
-		D_ASSERT(in_buff_end <= in_buff_start + GZipFile::BUFFER_SIZE);
+void MiniZStreamWrapper::FlushStream() {
+	auto &sd = file->stream_data;
+	mz_stream_ptr->next_in = nullptr;
+	mz_stream_ptr->avail_in = 0;
+	while (true) {
+		auto output_remaining = (sd.out_buff.get() + sd.out_buf_size) - sd.out_buff_start;
+		mz_stream_ptr->next_out = sd.out_buff_start;
+		mz_stream_ptr->avail_out = output_remaining;
 
-		// read more input if none available
-		if (in_buff_start == in_buff_end) {
-			// empty input buffer: refill from the start
-			in_buff_start = in_buff.get();
-			auto sz = child_handle->Read(in_buff.get(), BUFFER_SIZE);
-			if (sz <= 0) {
-				break;
-			}
-			in_buff_end = in_buff_start + sz;
+		auto res = mz_deflate(mz_stream_ptr, duckdb_miniz::MZ_FINISH);
+		sd.out_buff_start += (output_remaining - mz_stream_ptr->avail_out);
+		if (sd.out_buff_start > sd.out_buff.get()) {
+			file->child_handle->Write(sd.out_buff.get(), sd.out_buff_start - sd.out_buff.get());
+			sd.out_buff_start = sd.out_buff.get();
 		}
-
-		// actually decompress
-		mz_stream_ptr->next_in = (data_ptr_t)in_buff_start;
-		D_ASSERT(in_buff_end - in_buff_start < NumericLimits<int32_t>::Maximum());
-		mz_stream_ptr->avail_in = (uint32_t)(in_buff_end - in_buff_start);
-		mz_stream_ptr->next_out = (data_ptr_t)out_buff_end;
-		D_ASSERT((out_buff.get() + BUFFER_SIZE) - out_buff_end < NumericLimits<int32_t>::Maximum());
-		mz_stream_ptr->avail_out = (uint32_t)((out_buff.get() + BUFFER_SIZE) - out_buff_end);
-		auto ret = duckdb_miniz::mz_inflate(mz_stream_ptr, duckdb_miniz::MZ_NO_FLUSH);
-		if (ret != duckdb_miniz::MZ_OK && ret != duckdb_miniz::MZ_STREAM_END) {
-			throw Exception(duckdb_miniz::mz_error(ret));
+		if (res == duckdb_miniz::MZ_STREAM_END) {
+			break;
 		}
-		// update pointers following inflate()
-		in_buff_start = (data_ptr_t)mz_stream_ptr->next_in;
-		in_buff_end = in_buff_start + mz_stream_ptr->avail_in;
-		out_buff_end = (data_ptr_t)mz_stream_ptr->next_out;
-		D_ASSERT(out_buff_end + mz_stream_ptr->avail_out == out_buff.get() + BUFFER_SIZE);
-		// if stream ended, deallocate inflator
-		if (ret == duckdb_miniz::MZ_STREAM_END) {
-			miniz_stream->Close();
+		if (res != duckdb_miniz::MZ_OK) {
+			throw InternalException("Failed to compress GZIP block");
 		}
 	}
-	return total_read;
 }
 
-int64_t GZipFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
-	auto &gzip_file = (GZipFile &)handle;
-	return gzip_file.ReadData(buffer, nr_bytes);
-}
+void MiniZStreamWrapper::Close() {
+	if (!mz_stream_ptr) {
+		return;
+	}
+	if (writing) {
+		// flush anything remaining in the stream
+		FlushStream();
 
-void GZipFileSystem::Reset(FileHandle &handle) {
-	auto &gzip_file = (GZipFile &)handle;
-	gzip_file.child_handle->Reset();
-	gzip_file.Initialize();
-}
+		// write the footer
+		unsigned char gzip_footer[MiniZStream::GZIP_FOOTER_SIZE];
+		MiniZStream::InitializeGZIPFooter(gzip_footer, crc, total_size);
+		file->child_handle->Write(gzip_footer, MiniZStream::GZIP_FOOTER_SIZE);
 
-int64_t GZipFileSystem::GetFileSize(FileHandle &handle) {
-	auto &gzip_file = (GZipFile &)handle;
-	return gzip_file.child_handle->GetFileSize();
+		duckdb_miniz::mz_deflateEnd(mz_stream_ptr);
+	} else {
+		duckdb_miniz::mz_inflateEnd(mz_stream_ptr);
+	}
+	delete mz_stream_ptr;
+	mz_stream_ptr = nullptr;
+	file = nullptr;
 }
 
-bool GZipFileSystem::OnDiskFile(FileHandle &handle) {
-	auto &gzip_file = (GZipFile &)handle;
-	return gzip_file.child_handle->OnDiskFile();
+class GZipFile : public CompressedFile {
+public:
+	GZipFile(unique_ptr<FileHandle> child_handle_p, const string &path, bool write)
+	    : CompressedFile(gzip_fs, move(child_handle_p), path) {
+		Initialize(write);
+	}
+
+	GZipFileSystem gzip_fs;
+};
+
+void GZipFileSystem::VerifyGZIPHeader(uint8_t gzip_hdr[], idx_t read_count) {
+	// check for incorrectly formatted files
+	if (read_count != GZIP_HEADER_MINSIZE) {
+		throw IOException("Input is not a GZIP stream");
+	}
+	if (gzip_hdr[0] != 0x1F || gzip_hdr[1] != 0x8B) { // magic header
+		throw IOException("Input is not a GZIP stream");
+	}
+	if (gzip_hdr[2] != GZIP_COMPRESSION_DEFLATE) { // compression method
+		throw IOException("Unsupported GZIP compression method");
+	}
+	if (gzip_hdr[3] & GZIP_FLAG_UNSUPPORTED) {
+		throw IOException("Unsupported GZIP archive");
+	}
 }
 
 string GZipFileSystem::UncompressGZIPString(const string &in) {
@@ -20458,6 +20839,23 @@ string GZipFileSystem::UncompressGZIPString(const string &in) {
 	return decompressed;
 }
 
+unique_ptr<FileHandle> GZipFileSystem::OpenCompressedFile(unique_ptr<FileHandle> handle, bool write) {
+	auto path = handle->path;
+	return make_unique<GZipFile>(move(handle), path, write);
+}
+
+unique_ptr<StreamWrapper> GZipFileSystem::CreateStream() {
+	return make_unique<MiniZStreamWrapper>();
+}
+
+idx_t GZipFileSystem::InBufferSize() {
+	return BUFFER_SIZE;
+}
+
+idx_t GZipFileSystem::OutBufferSize() {
+	return BUFFER_SIZE;
+}
+
 } // namespace duckdb
 
 
@@ -20793,15 +21191,14 @@ public:
 		Close();
 	}
 
-protected:
+	int fd;
+
+public:
 	void Close() override {
 		if (fd != -1) {
 			close(fd);
 		}
 	};
-
-public:
-	int fd;
 };
 
 static FileType GetFileTypeInternal(int fd) { // LCOV_EXCL_START
@@ -21169,14 +21566,13 @@ public:
 		Close();
 	}
 
-protected:
+	idx_t position;
+	HANDLE fd;
+
+public:
 	void Close() override {
 		CloseHandle(fd);
 	};
-
-public:
-	idx_t position;
-	HANDLE fd;
 };
 
 unique_ptr<FileHandle> LocalFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock_type,
@@ -26351,13 +26747,13 @@ public:
 	    : FileHandle(pipe_fs, path), child_handle(move(child_handle_p)) {
 	}
 
-	int64_t ReadChunk(void *buffer, int64_t nr_bytes);
-	int64_t WriteChunk(void *buffer, int64_t nr_bytes);
-
 	PipeFileSystem pipe_fs;
 	unique_ptr<FileHandle> child_handle;
 
-protected:
+public:
+	int64_t ReadChunk(void *buffer, int64_t nr_bytes);
+	int64_t WriteChunk(void *buffer, int64_t nr_bytes);
+
 	void Close() override {
 	}
 };
@@ -48084,6 +48480,7 @@ void VectorOperations::ReadFromStorage(data_ptr_t source, idx_t count, Vector &r
 
 
 
+
 namespace duckdb {
 
 // bunch of wrappers to allow registering protocol handlers
@@ -48165,6 +48562,10 @@ public:
 		sub_systems.push_back(move(fs));
 	}
 
+	void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override {
+		compressed_fs[compression_type] = move(fs);
+	}
+
 	std::string GetName() const override {
 		return "VirtualFileSystem";
 	}
@@ -48181,6 +48582,7 @@ private:
 
 private:
 	vector<unique_ptr<FileSystem>> sub_systems;
+	map<FileCompressionType, unique_ptr<FileSystem>> compressed_fs;
 	const unique_ptr<FileSystem> default_fs;
 };
 
@@ -48194,6 +48596,7 @@ private:
 namespace duckdb {
 
 VirtualFileSystem::VirtualFileSystem() : default_fs(FileSystem::CreateLocal()) {
+	RegisterSubSystem(FileCompressionType::GZIP, make_unique<GZipFileSystem>());
 }
 
 unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t flags, FileLockType lock,
@@ -48203,6 +48606,8 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 		auto lower_path = StringUtil::Lower(path);
 		if (StringUtil::EndsWith(lower_path, ".gz")) {
 			compression = FileCompressionType::GZIP;
+		} else if (StringUtil::EndsWith(lower_path, ".zst")) {
+			compression = FileCompressionType::ZSTD;
 		} else {
 			compression = FileCompressionType::UNCOMPRESSED;
 		}
@@ -48212,13 +48617,12 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
 	if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
 		file_handle = PipeFileSystem::OpenPipe(move(file_handle));
 	} else if (compression != FileCompressionType::UNCOMPRESSED) {
-		switch (compression) {
-		case FileCompressionType::GZIP:
-			file_handle = GZipFileSystem::OpenCompressedFile(move(file_handle));
-			break;
-		default:
-			throw NotImplementedException("Unimplemented compression type");
+		auto entry = compressed_fs.find(compression);
+		if (entry == compressed_fs.end()) {
+			throw NotImplementedException(
+			    "Attempting to open a compressed file, but the compression type is not supported");
 		}
+		file_handle = entry->second->OpenCompressedFile(move(file_handle), flags & FileFlags::FILE_FLAGS_WRITE);
 	}
 	return file_handle;
 }
@@ -62229,8 +62633,8 @@ struct BufferedCSVReaderOptions {
 	//! The file path of the CSV file to read
 	string file_path;
 	//! Whether file is compressed or not, and if so which compression type
-	//! ("infer" (default; infer from file extention), "gzip", "none")
-	string compression = "infer";
+	//! AUTO_DETECT (default; infer from file extension)
+	FileCompressionType compression = FileCompressionType::AUTO_DETECT;
 	//! Whether or not to automatically detect dialect and datatypes
 	bool auto_detect = false;
 	//! Whether or not a delimiter was defined by the user
@@ -62302,7 +62706,6 @@ public:
 	unique_ptr<FileHandle> file_handle;
 	bool plain_file_source = false;
 	idx_t file_size = 0;
-	FileCompressionType compression = FileCompressionType::UNCOMPRESSED;
 
 	unique_ptr<char[]> buffer;
 	idx_t buffer_size;
@@ -62584,15 +62987,8 @@ void BufferedCSVReader::PrepareComplexParser() {
 }
 
 unique_ptr<FileHandle> BufferedCSVReader::OpenCSV(const BufferedCSVReaderOptions &options) {
-	this->compression = FileCompressionType::UNCOMPRESSED;
-	if (options.compression == "infer" || options.compression == "auto") {
-		this->compression = FileCompressionType::AUTO_DETECT;
-	} else if (options.compression == "gzip") {
-		this->compression = FileCompressionType::GZIP;
-	}
-
 	auto result = fs.OpenFile(options.file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK,
-	                          this->compression, this->opener);
+	                          options.compression, this->opener);
 	plain_file_source = result->OnDiskFile() && result->CanSeek();
 	file_size = result->GetFileSize();
 	return result;
@@ -63235,9 +63631,8 @@ void BufferedCSVReader::DetectHeader(const vector<vector<LogicalType>> &best_sql
 
 	} else {
 		options.header = false;
-		idx_t total_columns = parse_chunk.ColumnCount();
-		for (idx_t col = 0; col < total_columns; col++) {
-			string column_name = GenerateColumnName(total_columns, col);
+		for (idx_t col = 0; col < options.num_cols; col++) {
+			string column_name = GenerateColumnName(options.num_cols, col);
 			col_names.push_back(column_name);
 		}
 	}
@@ -63389,6 +63784,7 @@ vector<LogicalType> BufferedCSVReader::SniffCSV(const vector<LogicalType> &reque
 	// #######
 	// ### header detection
 	// #######
+	options.num_cols = best_num_cols;
 	DetectHeader(best_sql_types_candidates, best_header_row);
 
 	// #######
@@ -95536,11 +95932,7 @@ static bool ParseBaseOption(BufferedCSVReaderOptions &options, string &loption,
 			throw BinderException("Copy is only supported for UTF-8 encoded files, ENCODING 'UTF-8'");
 		}
 	} else if (loption == "compression") {
-		options.compression = ParseString(set);
-		if (!(options.compression == "infer" || options.compression == "gzip" || options.compression == "none" ||
-		      options.compression.empty())) {
-			throw BinderException("read_csv currently only supports 'gzip' compression.");
-		}
+		options.compression = FileCompressionTypeFromString(ParseString(set));
 	} else if (loption == "skip") {
 		options.skip_rows = ParseInteger(set);
 	} else {
@@ -95856,14 +96248,15 @@ struct LocalReadCSVData : public LocalFunctionData {
 };
 
 struct GlobalWriteCSVData : public GlobalFunctionData {
-	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileOpener *opener) : fs(fs) {
+	GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileOpener *opener, FileCompressionType compression)
+	    : fs(fs) {
 		handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
-		                     FileLockType::WRITE_LOCK, FileSystem::DEFAULT_COMPRESSION, opener);
+		                     FileLockType::WRITE_LOCK, compression, opener);
 	}
 
 	void WriteData(const_data_ptr_t data, idx_t size) {
 		lock_guard<mutex> flock(lock);
-		fs.Write(*handle, (void *)data, size);
+		handle->Write((void *)data, size);
 	}
 
 	FileSystem &fs;
@@ -95889,7 +96282,7 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
 	auto &csv_data = (WriteCSVData &)bind_data;
 	auto &options = csv_data.options;
 	auto global_data = make_unique<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), csv_data.files[0],
-	                                                   FileSystem::GetFileOpener(context));
+	                                                   FileSystem::GetFileOpener(context), options.compression);
 
 	if (options.header) {
 		BufferedSerializer serializer;
@@ -95977,6 +96370,15 @@ static void WriteCSVCombine(ClientContext &context, FunctionData &bind_data, Glo
 	}
 }
 
+//===--------------------------------------------------------------------===//
+// Finalize
+//===--------------------------------------------------------------------===//
+void WriteCSVFinalize(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate) {
+	auto &global_state = (GlobalWriteCSVData &)gstate;
+	global_state.handle->Close();
+	global_state.handle.reset();
+}
+
 void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 	CopyFunction info("csv");
 	info.copy_to_bind = WriteCSVBind;
@@ -95984,6 +96386,7 @@ void CSVCopyFunction::RegisterFunction(BuiltinFunctions &set) {
 	info.copy_to_initialize_global = WriteCSVInitializeGlobal;
 	info.copy_to_sink = WriteCSVSink;
 	info.copy_to_combine = WriteCSVCombine;
+	info.copy_to_finalize = WriteCSVFinalize;
 
 	info.copy_from_bind = ReadCSVBind;
 	info.copy_from_function = ReadCSVTableFunction::GetFunction();
@@ -96887,7 +97290,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
 				throw BinderException("read_csv requires at least a single column as input!");
 			}
 		} else if (kv.first == "compression") {
-			options.compression = kv.second.str_value;
+			options.compression = FileCompressionTypeFromString(kv.second.str_value);
 		} else if (kv.first == "filename") {
 			result->include_file_name = kv.second.value_.boolean;
 		} else if (kv.first == "skip") {
@@ -96898,10 +97301,6 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
 		throw BinderException("read_csv requires columns to be specified. Use read_csv_auto or set read_csv(..., "
 		                      "AUTO_DETECT=TRUE) to automatically guess columns.");
 	}
-	if (!(options.compression == "infer" || options.compression == "gzip" || options.compression == "none" ||
-	      options.compression.empty())) {
-		throw BinderException("read_csv currently only supports 'gzip' compression.");
-	}
 	if (options.auto_detect) {
 		options.file_path = result->files[0];
 		auto initial_reader = make_unique<BufferedCSVReader>(context, options);
@@ -97029,8 +97428,13 @@ void ReadCSVTableFunction::RegisterFunction(BuiltinFunctions &set) {
 
 unique_ptr<TableFunctionRef> ReadCSVReplacement(const string &table_name, void *data) {
 	auto lower_name = StringUtil::Lower(table_name);
-	if (!StringUtil::EndsWith(lower_name, ".csv") && !StringUtil::EndsWith(lower_name, ".tsv") &&
-	    !StringUtil::EndsWith(lower_name, ".csv.gz")) {
+	// remove any compression
+	if (StringUtil::EndsWith(lower_name, ".gz")) {
+		lower_name = lower_name.substr(0, lower_name.size() - 3);
+	} else if (StringUtil::EndsWith(lower_name, ".zst")) {
+		lower_name = lower_name.substr(0, lower_name.size() - 4);
+	}
+	if (!StringUtil::EndsWith(lower_name, ".csv") && !StringUtil::EndsWith(lower_name, ".tsv")) {
 		return nullptr;
 	}
 	auto table_function = make_unique<TableFunctionRef>();

package/src/duckdb.hpp

@@ -10,8 +10,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 #pragma once
 #define DUCKDB_AMALGAMATION 1
-#define DUCKDB_SOURCE_ID "c9a9053a2"
-#define DUCKDB_VERSION "0.3.2-dev300"
+#define DUCKDB_SOURCE_ID "5d577b5d9"
+#define DUCKDB_VERSION "0.3.2-dev324"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -1719,7 +1719,9 @@ using std::unordered_map;
 
 namespace duckdb {
 
-enum class FileCompressionType : uint8_t { AUTO_DETECT = 0, UNCOMPRESSED = 1, GZIP = 2 };
+enum class FileCompressionType : uint8_t { AUTO_DETECT = 0, UNCOMPRESSED = 1, GZIP = 2, ZSTD = 3 };
+
+FileCompressionType FileCompressionTypeFromString(const string &input);
 
 } // namespace duckdb
 
@@ -1779,7 +1781,7 @@ public:
 	idx_t GetFileSize();
 	FileType GetType();
 
-protected:
+	//! Closes the file handle.
 	virtual void Close() = 0;
 
 public:
@@ -1883,6 +1885,7 @@ public:
 
 	//! registers a sub-file system to handle certain file name prefixes, e.g. http:// etc.
 	virtual void RegisterSubSystem(unique_ptr<FileSystem> sub_fs);
+	virtual void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs);
 
 	//! Whether or not a sub-system can handle a specific file path
 	virtual bool CanHandleFile(const string &fpath);
@@ -1899,6 +1902,8 @@ public:
 	//! in a file on-disk are much cheaper than e.g. random reads in a file over the network
 	virtual bool OnDiskFile(FileHandle &handle);
 
+	virtual unique_ptr<FileHandle> OpenCompressedFile(unique_ptr<FileHandle> handle, bool write);
+
 	//! Create a LocalFileSystem.
 	static unique_ptr<FileSystem> CreateLocal();