duckdb 0.10.1 → 0.10.2-dev3.0

package/src/duckdb/extension/json/json_scan.cpp

@@ -148,9 +148,6 @@ JSONScanLocalState::JSONScanLocalState(ClientContext &context, JSONScanGlobalSta
     : scan_count(0), batch_index(DConstants::INVALID_INDEX), total_read_size(0), total_tuple_count(0),
       bind_data(gstate.bind_data), allocator(BufferAllocator::Get(context)), is_last(false),
       fs(FileSystem::GetFileSystem(context)), buffer_size(0), buffer_offset(0), prev_buffer_remainder(0) {
-
-	// Buffer to reconstruct JSON values when they cross a buffer boundary
-	reconstruct_buffer = gstate.allocator.Allocate(gstate.buffer_capacity);
 }
 
 JSONGlobalTableFunctionState::JSONGlobalTableFunctionState(ClientContext &context, TableFunctionInitInput &input)
@@ -276,13 +273,13 @@ idx_t JSONScanLocalState::ReadNext(JSONScanGlobalState &gstate) {
 			}
 			if (current_buffer_handle->buffer_index != 0 &&
 			    current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
-				if (ReconstructFirstObject()) {
+				if (ReconstructFirstObject(gstate)) {
 					scan_count++;
 				}
 			}
 		}
 
-		ParseNextChunk();
+		ParseNextChunk(gstate);
 	}
 
 	return scan_count;
@@ -540,16 +537,12 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 		}
 	}
 
-	// If we cannot re-use a buffer we create a new one
-	if (!buffer.IsSet()) {
-		buffer = gstate.allocator.Allocate(gstate.buffer_capacity);
-	}
-
-	buffer_ptr = char_ptr_cast(buffer.get());
-
 	// Copy last bit of previous buffer
 	if (current_reader && current_reader->GetFormat() != JSONFormat::NEWLINE_DELIMITED && !is_last) {
-		memcpy(buffer_ptr, reconstruct_buffer.get(), prev_buffer_remainder);
+		if (!buffer.IsSet()) {
+			buffer = AllocateBuffer(gstate);
+		}
+		memcpy(buffer_ptr, GetReconstructBuffer(gstate), prev_buffer_remainder);
 	}
 
 	optional_idx buffer_index;
@@ -558,7 +551,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 		if (current_reader) {
 			// Try to read (if we were not the last read in the previous iteration)
 			bool file_done = false;
-			bool read_success = ReadNextBufferInternal(gstate, buffer_index, file_done);
+			bool read_success = ReadNextBufferInternal(gstate, buffer, buffer_index, file_done);
 			if (!is_last && read_success) {
 				// We read something
 				if (buffer_index.GetIndex() == 0 && current_reader->GetFormat() == JSONFormat::ARRAY) {
@@ -603,7 +596,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 		// Open the file if it is not yet open
 		if (!current_reader->IsOpen()) {
 			current_reader->OpenJSONFile();
-			if (current_reader->GetFileHandle().FileSize() == 0) {
+			if (current_reader->GetFileHandle().FileSize() == 0 && !current_reader->GetFileHandle().IsPipe()) {
 				current_reader->GetFileHandle().Close();
 				// Skip over empty files
 				if (gstate.enable_parallel_scans) {
@@ -616,7 +609,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 		// Auto-detect if we haven't yet done this during the bind
 		if (gstate.bind_data.options.record_type == JSONRecordType::AUTO_DETECT ||
 		    current_reader->GetFormat() == JSONFormat::AUTO_DETECT) {
-			ReadAndAutoDetect(gstate, buffer_index);
+			ReadAndAutoDetect(gstate, buffer, buffer_index);
 		}
 
 		if (gstate.enable_parallel_scans) {
@@ -655,10 +648,11 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 	return true;
 }
 
-void JSONScanLocalState::ReadAndAutoDetect(JSONScanGlobalState &gstate, optional_idx &buffer_index) {
+void JSONScanLocalState::ReadAndAutoDetect(JSONScanGlobalState &gstate, AllocatedData &buffer,
+                                           optional_idx &buffer_index) {
 	// We have to detect the JSON format - hold the gstate lock while we do this
 	bool file_done = false;
-	if (!ReadNextBufferInternal(gstate, buffer_index, file_done)) {
+	if (!ReadNextBufferInternal(gstate, buffer, buffer_index, file_done)) {
 		return;
 	}
 	if (buffer_size == 0) {
@@ -676,21 +670,21 @@ void JSONScanLocalState::ReadAndAutoDetect(JSONScanGlobalState &gstate, optional
 		SkipOverArrayStart();
 	}
 
-	if (bind_data.options.record_type == JSONRecordType::RECORDS &&
+	if (!bind_data.ignore_errors && bind_data.options.record_type == JSONRecordType::RECORDS &&
 	    current_reader->GetRecordType() != JSONRecordType::RECORDS) {
 		throw InvalidInputException("Expected file \"%s\" to contain records, detected non-record JSON instead.",
 		                            current_reader->GetFileName());
 	}
 }
 
-bool JSONScanLocalState::ReadNextBufferInternal(JSONScanGlobalState &gstate, optional_idx &buffer_index,
-                                                bool &file_done) {
+bool JSONScanLocalState::ReadNextBufferInternal(JSONScanGlobalState &gstate, AllocatedData &buffer,
+                                                optional_idx &buffer_index, bool &file_done) {
 	if (current_reader->GetFileHandle().CanSeek()) {
-		if (!ReadNextBufferSeek(gstate, buffer_index, file_done)) {
+		if (!ReadNextBufferSeek(gstate, buffer, buffer_index, file_done)) {
 			return false;
 		}
 	} else {
-		if (!ReadNextBufferNoSeek(gstate, buffer_index, file_done)) {
+		if (!ReadNextBufferNoSeek(gstate, buffer, buffer_index, file_done)) {
 			return false;
 		}
 	}
@@ -700,7 +694,8 @@ bool JSONScanLocalState::ReadNextBufferInternal(JSONScanGlobalState &gstate, opt
 	return true;
 }
 
-bool JSONScanLocalState::ReadNextBufferSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done) {
+bool JSONScanLocalState::ReadNextBufferSeek(JSONScanGlobalState &gstate, AllocatedData &buffer,
+                                            optional_idx &buffer_index, bool &file_done) {
 	auto &file_handle = current_reader->GetFileHandle();
 
 	idx_t request_size = gstate.buffer_capacity - prev_buffer_remainder - YYJSON_PADDING_SIZE;
@@ -709,6 +704,12 @@ bool JSONScanLocalState::ReadNextBufferSeek(JSONScanGlobalState &gstate, optiona
 
 	{
 		lock_guard<mutex> reader_guard(current_reader->lock);
+		if (file_handle.LastReadRequested()) {
+			return false;
+		}
+		if (!buffer.IsSet()) {
+			buffer = AllocateBuffer(gstate);
+		}
 		if (!file_handle.GetPositionAndSize(read_position, read_size, request_size)) {
 			return false; // We weren't able to read
 		}
@@ -742,16 +743,25 @@ bool JSONScanLocalState::ReadNextBufferSeek(JSONScanGlobalState &gstate, optiona
 	return true;
 }
 
-bool JSONScanLocalState::ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index,
-                                              bool &file_done) {
+bool JSONScanLocalState::ReadNextBufferNoSeek(JSONScanGlobalState &gstate, AllocatedData &buffer,
+                                              optional_idx &buffer_index, bool &file_done) {
 	idx_t request_size = gstate.buffer_capacity - prev_buffer_remainder - YYJSON_PADDING_SIZE;
 	idx_t read_size;
 
 	{
 		lock_guard<mutex> reader_guard(current_reader->lock);
-		if (!current_reader->HasFileHandle() || !current_reader->IsOpen() ||
-		    !current_reader->GetFileHandle().Read(buffer_ptr + prev_buffer_remainder, read_size, request_size,
-		                                          file_done, gstate.bind_data.type == JSONScanType::SAMPLE)) {
+		if (!current_reader->HasFileHandle() || !current_reader->IsOpen()) {
+			return false; // Couldn't read anything
+		}
+		auto &file_handle = current_reader->GetFileHandle();
+		if (file_handle.LastReadRequested()) {
+			return false;
+		}
+		if (!buffer.IsSet()) {
+			buffer = AllocateBuffer(gstate);
+		}
+		if (!file_handle.Read(buffer_ptr + prev_buffer_remainder, read_size, request_size, file_done,
+		                      gstate.bind_data.type == JSONScanType::SAMPLE)) {
 			return false; // Couldn't read anything
 		}
 		buffer_index = current_reader->GetBufferIndex();
@@ -766,6 +776,19 @@ bool JSONScanLocalState::ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optio
 	return true;
 }
 
+AllocatedData JSONScanLocalState::AllocateBuffer(JSONScanGlobalState &gstate) {
+	auto buffer = gstate.allocator.Allocate(gstate.buffer_capacity);
+	buffer_ptr = char_ptr_cast(buffer.get());
+	return buffer;
+}
+
+data_ptr_t JSONScanLocalState::GetReconstructBuffer(JSONScanGlobalState &gstate) {
+	if (!reconstruct_buffer.IsSet()) {
+		reconstruct_buffer = gstate.allocator.Allocate(gstate.buffer_capacity);
+	}
+	return reconstruct_buffer.get();
+}
+
 void JSONScanLocalState::SkipOverArrayStart() {
 	// First read of this buffer, check if it's actually an array and skip over the bytes
 	SkipWhitespace(buffer_ptr, buffer_offset, buffer_size);
@@ -795,7 +818,7 @@ void JSONScanLocalState::SkipOverArrayStart() {
 	}
 }
 
-bool JSONScanLocalState::ReconstructFirstObject() {
+bool JSONScanLocalState::ReconstructFirstObject(JSONScanGlobalState &gstate) {
 	D_ASSERT(current_buffer_handle->buffer_index != 0);
 	D_ASSERT(current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED);
 
@@ -811,7 +834,7 @@ bool JSONScanLocalState::ReconstructFirstObject() {
 	auto part1_size = prev_buffer_ptr - part1_ptr;
 
 	// Now copy the data to our reconstruct buffer
-	const auto reconstruct_ptr = reconstruct_buffer.get();
+	const auto reconstruct_ptr = GetReconstructBuffer(gstate);
 	memcpy(reconstruct_ptr, part1_ptr, part1_size);
 
 	// We copied the object, so we are no longer reading the previous buffer
@@ -851,7 +874,7 @@ bool JSONScanLocalState::ReconstructFirstObject() {
 	return true;
 }
 
-void JSONScanLocalState::ParseNextChunk() {
+void JSONScanLocalState::ParseNextChunk(JSONScanGlobalState &gstate) {
 	auto buffer_offset_before = buffer_offset;
 
 	const auto format = current_reader->GetFormat();
@@ -862,6 +885,7 @@ void JSONScanLocalState::ParseNextChunk() {
 		if (remaining == 0) {
 			break;
 		}
+		D_ASSERT(format != JSONFormat::AUTO_DETECT);
 		const char *json_end = format == JSONFormat::NEWLINE_DELIMITED ? NextNewline(json_start, remaining)
 		                                                               : NextJSON(json_start, remaining);
 		if (json_end == nullptr) {
@@ -872,7 +896,7 @@ void JSONScanLocalState::ParseNextChunk() {
 					if (remaining > bind_data.maximum_object_size) {
 						ThrowObjectSizeError(remaining);
 					}
-					memcpy(reconstruct_buffer.get(), json_start, remaining);
+					memcpy(GetReconstructBuffer(gstate), json_start, remaining);
 					prev_buffer_remainder = remaining;
 				}
 				buffer_offset = buffer_size;

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -16,6 +16,7 @@
 #include "templated_column_reader.hpp"
 #include "utf8proc_wrapper.hpp"
 #include "zstd.h"
+#include "lz4.hpp"
 
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/types/bit.hpp"
@@ -339,6 +340,13 @@ void ColumnReader::DecompressInternal(CompressionCodec::type codec, const_data_p
 		s.Decompress(const_char_ptr_cast(src), src_size, char_ptr_cast(dst), dst_size);
 		break;
 	}
+	case CompressionCodec::LZ4_RAW: {
+		auto res = duckdb_lz4::LZ4_decompress_safe(const_char_ptr_cast(src), char_ptr_cast(dst), src_size, dst_size);
+		if (res != NumericCast<int>(dst_size)) {
+			throw std::runtime_error("LZ4 decompression failure");
+		}
+		break;
+	}
 	case CompressionCodec::SNAPPY: {
 		{
 			size_t uncompressed_size = 0;
@@ -346,7 +354,7 @@ void ColumnReader::DecompressInternal(CompressionCodec::type codec, const_data_p
 			if (!res) {
 				throw std::runtime_error("Snappy decompression failure");
 			}
-			if (uncompressed_size != (size_t)dst_size) {
+			if (uncompressed_size != dst_size) {
 				throw std::runtime_error("Snappy decompression failure: Uncompressed data size mismatch");
 			}
 		}
@@ -358,16 +366,17 @@ void ColumnReader::DecompressInternal(CompressionCodec::type codec, const_data_p
 	}
 	case CompressionCodec::ZSTD: {
 		auto res = duckdb_zstd::ZSTD_decompress(dst, dst_size, src, src_size);
-		if (duckdb_zstd::ZSTD_isError(res) || res != (size_t)dst_size) {
+		if (duckdb_zstd::ZSTD_isError(res) || res != dst_size) {
 			throw std::runtime_error("ZSTD Decompression failure");
 		}
 		break;
 	}
+
 	default: {
 		std::stringstream codec_name;
 		codec_name << codec;
 		throw std::runtime_error("Unsupported compression codec \"" + codec_name.str() +
-		                         "\". Supported options are uncompressed, gzip, snappy or zstd");
+		                         "\". Supported options are uncompressed, gzip, lz4_raw, snappy or zstd");
 	}
 	}
 }

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -15,12 +15,13 @@
 #include "duckdb/common/string_map_set.hpp"
 #include "duckdb/common/types/date.hpp"
 #include "duckdb/common/types/hugeint.hpp"
-#include "duckdb/common/types/uhugeint.hpp"
 #include "duckdb/common/types/string_heap.hpp"
 #include "duckdb/common/types/time.hpp"
 #include "duckdb/common/types/timestamp.hpp"
+#include "duckdb/common/types/uhugeint.hpp"
 #endif
 
+#include "lz4.hpp"
 #include "miniz_wrapper.hpp"
 #include "snappy.h"
 #include "zstd.h"
@@ -191,6 +192,7 @@ void ColumnWriter::CompressPage(MemoryStream &temp_writer, size_t &compressed_si
 		compressed_size = temp_writer.GetPosition();
 		compressed_data = temp_writer.GetData();
 		break;
+
 	case CompressionCodec::SNAPPY: {
 		compressed_size = duckdb_snappy::MaxCompressedLength(temp_writer.GetPosition());
 		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
@@ -200,6 +202,15 @@ void ColumnWriter::CompressPage(MemoryStream &temp_writer, size_t &compressed_si
 		D_ASSERT(compressed_size <= duckdb_snappy::MaxCompressedLength(temp_writer.GetPosition()));
 		break;
 	}
+	case CompressionCodec::LZ4_RAW: {
+		compressed_size = duckdb_lz4::LZ4_compressBound(temp_writer.GetPosition());
+		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
+		compressed_size = duckdb_lz4::LZ4_compress_default(const_char_ptr_cast(temp_writer.GetData()),
+		                                                   char_ptr_cast(compressed_buf.get()),
+		                                                   temp_writer.GetPosition(), compressed_size);
+		compressed_data = compressed_buf.get();
+		break;
+	}
 	case CompressionCodec::GZIP: {
 		MiniZStream s;
 		compressed_size = s.MaxCompressedLength(temp_writer.GetPosition());
@@ -344,15 +355,18 @@ public:
 	~BasicColumnWriter() override = default;
 
 	//! We limit the uncompressed page size to 100MB
-	// The max size in Parquet is 2GB, but we choose a more conservative limit
+	//! The max size in Parquet is 2GB, but we choose a more conservative limit
 	static constexpr const idx_t MAX_UNCOMPRESSED_PAGE_SIZE = 100000000;
 	//! Dictionary pages must be below 2GB. Unlike data pages, there's only one dictionary page.
-	//  For this reason we go with a much higher, but still a conservative upper bound of 1GB;
+	//! For this reason we go with a much higher, but still a conservative upper bound of 1GB;
 	static constexpr const idx_t MAX_UNCOMPRESSED_DICT_PAGE_SIZE = 1e9;
+	//! If the dictionary has this many entries, but the compression ratio is still below 1,
+	//! we stop creating the dictionary
+	static constexpr const idx_t DICTIONARY_ANALYZE_THRESHOLD = 1e4;
 
-	// the maximum size a key entry in an RLE page takes
+	//! The maximum size a key entry in an RLE page takes
 	static constexpr const idx_t MAX_DICTIONARY_KEY_SIZE = sizeof(uint32_t);
-	// the size of encoding the string length
+	//! The size of encoding the string length
 	static constexpr const idx_t STRING_LENGTH_SIZE = sizeof(uint32_t);
 
 public:
@@ -1282,6 +1296,12 @@ public:
 
 	void Analyze(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count) override {
 		auto &state = state_p.Cast<StringColumnWriterState>();
+		if (writer.DictionaryCompressionRatioThreshold() == NumericLimits<double>::Maximum() ||
+		    (state.dictionary.size() > DICTIONARY_ANALYZE_THRESHOLD && WontUseDictionary(state))) {
+			// Early out: compression ratio is less than the specified parameter
+			// after seeing more entries than the threshold
+			return;
+		}
 
 		idx_t vcount = parent ? parent->definition_levels.size() - state.definition_levels.size() : count;
 		idx_t parent_index = state.definition_levels.size();
@@ -1308,6 +1328,7 @@ public:
 					new_value_index++;
 					state.estimated_dict_page_size += value.GetSize() + MAX_DICTIONARY_KEY_SIZE;
 				}
+
 				// if the value changed, we will encode it in the page
 				if (last_value_index != found.first->second) {
 					// we will add the value index size later, when we know the total number of keys
@@ -1329,8 +1350,7 @@ public:
 
 		// check if a dictionary will require more space than a plain write, or if the dictionary page is going to
 		// be too large
-		if (state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE ||
-		    state.estimated_rle_pages_size + state.estimated_dict_page_size > state.estimated_plain_size) {
+		if (WontUseDictionary(state)) {
 			// clearing the dictionary signals a plain write
 			state.dictionary.clear();
 			state.key_bit_width = 0;
@@ -1446,6 +1466,23 @@ public:
 			return strings[index].GetSize();
 		}
 	}
+
+private:
+	bool WontUseDictionary(StringColumnWriterState &state) const {
+		return state.estimated_dict_page_size > MAX_UNCOMPRESSED_DICT_PAGE_SIZE ||
+		       DictionaryCompressionRatio(state) < writer.DictionaryCompressionRatioThreshold();
+	}
+
+	static double DictionaryCompressionRatio(StringColumnWriterState &state) {
+		// If any are 0, we just return a compression ratio of 1
+		if (state.estimated_plain_size == 0 || state.estimated_rle_pages_size == 0 ||
+		    state.estimated_dict_page_size == 0) {
+			return 1;
+		}
+		// Otherwise, plain size divided by compressed size
+		return double(state.estimated_plain_size) /
+		       double(state.estimated_rle_pages_size + state.estimated_dict_page_size);
+	}
 };
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -64,7 +64,7 @@ public:
 	ParquetWriter(FileSystem &fs, string file_name, vector<LogicalType> types, vector<string> names,
 	              duckdb_parquet::format::CompressionCodec::type codec, ChildFieldIDs field_ids,
 	              const vector<pair<string, string>> &kv_metadata,
-	              shared_ptr<ParquetEncryptionConfig> encryption_config);
+	              shared_ptr<ParquetEncryptionConfig> encryption_config, double dictionary_compression_ratio_threshold);
 
 public:
 	void PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result);
@@ -91,6 +91,9 @@ public:
 		lock_guard<mutex> glock(lock);
 		return writer->total_written;
 	}
+	double DictionaryCompressionRatioThreshold() const {
+		return dictionary_compression_ratio_threshold;
+	}
 
 	static CopyTypeSupport TypeIsSupported(const LogicalType &type);
 
@@ -106,6 +109,7 @@ private:
 	duckdb_parquet::format::CompressionCodec::type codec;
 	ChildFieldIDs field_ids;
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
+	double dictionary_compression_ratio_threshold;
 
 	unique_ptr<BufferedFileWriter> writer;
 	shared_ptr<duckdb_apache::thrift::protocol::TProtocol> protocol;

package/src/duckdb/extension/parquet/parquet_extension.cpp

@@ -130,6 +130,9 @@ struct ParquetWriteBindData : public TableFunctionData {
 	//! How/Whether to encrypt the data
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
 
+	//! Dictionary compression is applied only if the compression ratio exceeds this threshold
+	double dictionary_compression_ratio_threshold = 1.0;
+
 	ChildFieldIDs field_ids;
 };
 
@@ -947,6 +950,10 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
 				bind_data->codec = duckdb_parquet::format::CompressionCodec::GZIP;
 			} else if (roption == "zstd") {
 				bind_data->codec = duckdb_parquet::format::CompressionCodec::ZSTD;
+			} else if (roption == "lz4" || roption == "lz4_raw") {
+				/* LZ4 is technically another compression scheme, but deprecated and arrow also uses them
+				 * interchangeably */
+				bind_data->codec = duckdb_parquet::format::CompressionCodec::LZ4_RAW;
 			} else {
 				throw BinderException("Expected %s argument to be either [uncompressed, snappy, gzip or zstd]",
 				                      loption);
@@ -988,6 +995,15 @@ unique_ptr<FunctionData> ParquetWriteBind(ClientContext &context, CopyFunctionBi
 			}
 		} else if (loption == "encryption_config") {
 			bind_data->encryption_config = ParquetEncryptionConfig::Create(context, option.second[0]);
+		} else if (loption == "dictionary_compression_ratio_threshold") {
+			auto val = option.second[0].GetValue<double>();
+			if (val == -1) {
+				val = NumericLimits<double>::Maximum();
+			} else if (val < 0) {
+				throw BinderException("dictionary_compression_ratio_threshold must be greater than 0, or -1 to disable "
+				                      "dictionary compression");
+			}
+			bind_data->dictionary_compression_ratio_threshold = val;
 		} else {
 			throw NotImplementedException("Unrecognized option for PARQUET: %s", option.first.c_str());
 		}
@@ -1013,9 +1029,10 @@ unique_ptr<GlobalFunctionData> ParquetWriteInitializeGlobal(ClientContext &conte
 	auto &parquet_bind = bind_data.Cast<ParquetWriteBindData>();
 
 	auto &fs = FileSystem::GetFileSystem(context);
-	global_state->writer = make_uniq<ParquetWriter>(fs, file_path, parquet_bind.sql_types, parquet_bind.column_names,
-	                                                parquet_bind.codec, parquet_bind.field_ids.Copy(),
-	                                                parquet_bind.kv_metadata, parquet_bind.encryption_config);
+	global_state->writer =
+	    make_uniq<ParquetWriter>(fs, file_path, parquet_bind.sql_types, parquet_bind.column_names, parquet_bind.codec,
+	                             parquet_bind.field_ids.Copy(), parquet_bind.kv_metadata,
+	                             parquet_bind.encryption_config, parquet_bind.dictionary_compression_ratio_threshold);
 	return std::move(global_state);
 }
 
@@ -1081,6 +1098,9 @@ const char *EnumUtil::ToChars<duckdb_parquet::format::CompressionCodec::type>(
 	case CompressionCodec::LZ4:
 		return "LZ4";
 		break;
+	case CompressionCodec::LZ4_RAW:
+		return "LZ4_RAW";
+		break;
 	case CompressionCodec::ZSTD:
 		return "ZSTD";
 		break;
@@ -1110,6 +1130,9 @@ EnumUtil::FromString<duckdb_parquet::format::CompressionCodec::type>(const char
 	if (StringUtil::Equals(value, "LZ4")) {
 		return CompressionCodec::LZ4;
 	}
+	if (StringUtil::Equals(value, "LZ4_RAW")) {
+		return CompressionCodec::LZ4_RAW;
+	}
 	if (StringUtil::Equals(value, "ZSTD")) {
 		return CompressionCodec::ZSTD;
 	}
@@ -1128,6 +1151,8 @@ static void ParquetCopySerialize(Serializer &serializer, const FunctionData &bin
 	serializer.WriteProperty(106, "field_ids", bind_data.field_ids);
 	serializer.WritePropertyWithDefault<shared_ptr<ParquetEncryptionConfig>>(107, "encryption_config",
 	                                                                         bind_data.encryption_config, nullptr);
+	serializer.WriteProperty(108, "dictionary_compression_ratio_threshold",
+	                         bind_data.dictionary_compression_ratio_threshold);
 }
 
 static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserializer, CopyFunction &function) {
@@ -1141,6 +1166,8 @@ static unique_ptr<FunctionData> ParquetCopyDeserialize(Deserializer &deserialize
 	data->field_ids = deserializer.ReadProperty<ChildFieldIDs>(106, "field_ids");
 	deserializer.ReadPropertyWithDefault<shared_ptr<ParquetEncryptionConfig>>(107, "encryption_config",
 	                                                                          data->encryption_config, nullptr);
+	deserializer.ReadPropertyWithDefault<double>(108, "dictionary_compression_ratio_threshold",
+	                                             data->dictionary_compression_ratio_threshold, 1.0);
 	return std::move(data);
 }
 // LCOV_EXCL_STOP

package/src/duckdb/extension/parquet/parquet_metadata.cpp

@@ -24,7 +24,7 @@ public:
 	}
 };
 
-enum class ParquetMetadataOperatorType { META_DATA, SCHEMA, KEY_VALUE_META_DATA, FILE_META_DATA };
+enum class ParquetMetadataOperatorType : uint8_t { META_DATA, SCHEMA, KEY_VALUE_META_DATA, FILE_META_DATA };
 
 struct ParquetMetaDataOperatorData : public GlobalTableFunctionState {
 	explicit ParquetMetaDataOperatorData(ClientContext &context, const vector<LogicalType> &types)

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -350,9 +350,11 @@ void VerifyUniqueNames(const vector<string> &names) {
 ParquetWriter::ParquetWriter(FileSystem &fs, string file_name_p, vector<LogicalType> types_p, vector<string> names_p,
                              CompressionCodec::type codec, ChildFieldIDs field_ids_p,
                              const vector<pair<string, string>> &kv_metadata,
-                             shared_ptr<ParquetEncryptionConfig> encryption_config_p)
+                             shared_ptr<ParquetEncryptionConfig> encryption_config_p,
+                             double dictionary_compression_ratio_threshold_p)
     : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
-      field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)) {
+      field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)),
+      dictionary_compression_ratio_threshold(dictionary_compression_ratio_threshold_p) {
 	// initialize the file writer
 	writer = make_uniq<BufferedFileWriter>(fs, file_name.c_str(),
 	                                       FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);

package/src/duckdb/extension/parquet/zstd_file_system.cpp

@@ -27,7 +27,7 @@ ZstdStreamWrapper::~ZstdStreamWrapper() {
 	}
 	try {
 		Close();
-	} catch (...) {
+	} catch (...) { // NOLINT: swallow exceptions in destructor
 	}
 }
 

package/src/duckdb/src/catalog/catalog.cpp

@@ -55,7 +55,11 @@ AttachedDatabase &Catalog::GetAttached() {
 	return db;
 }
 
-const string &Catalog::GetName() {
+const AttachedDatabase &Catalog::GetAttached() const {
+	return db;
+}
+
+const string &Catalog::GetName() const {
 	return GetAttached().GetName();
 }
 

package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp

@@ -36,8 +36,8 @@
 
 namespace duckdb {
 
-void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType alter_fk_type,
-                               vector<unique_ptr<AlterForeignKeyInfo>> &fk_arrays) {
+static void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType alter_fk_type,
+                                      vector<unique_ptr<AlterForeignKeyInfo>> &fk_arrays) {
 	if (entry.type != CatalogType::TABLE_ENTRY) {
 		return;
 	}
@@ -63,6 +63,19 @@ void FindForeignKeyInformation(CatalogEntry &entry, AlterForeignKeyType alter_fk
 	}
 }
 
+static void LazyLoadIndexes(ClientContext &context, CatalogEntry &entry) {
+	if (entry.type == CatalogType::TABLE_ENTRY) {
+		auto &table_entry = entry.Cast<TableCatalogEntry>();
+		table_entry.GetStorage().info->InitializeIndexes(context);
+	} else if (entry.type == CatalogType::INDEX_ENTRY) {
+		auto &index_entry = entry.Cast<IndexCatalogEntry>();
+		auto &table_entry = Catalog::GetEntry(context, CatalogType::TABLE_ENTRY, index_entry.catalog.GetName(),
+		                                      index_entry.GetSchemaName(), index_entry.GetTableName())
+		                        .Cast<TableCatalogEntry>();
+		table_entry.GetStorage().info->InitializeIndexes(context);
+	}
+}
+
 DuckSchemaEntry::DuckSchemaEntry(Catalog &catalog, CreateSchemaInfo &info)
     : SchemaCatalogEntry(catalog, info), tables(catalog, make_uniq<DefaultViewGenerator>(catalog, *this)),
       indexes(catalog), table_functions(catalog), copy_functions(catalog), pragma_functions(catalog),
@@ -82,7 +95,7 @@ unique_ptr<CatalogEntry> DuckSchemaEntry::Copy(ClientContext &context) const {
 optional_ptr<CatalogEntry> DuckSchemaEntry::AddEntryInternal(CatalogTransaction transaction,
                                                              unique_ptr<StandardEntry> entry,
                                                              OnCreateConflict on_conflict,
-                                                             DependencyList dependencies) {
+                                                             LogicalDependencyList dependencies) {
 	auto entry_name = entry->name;
 	auto entry_type = entry->type;
 	auto result = entry.get();
@@ -184,7 +197,7 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::CreateFunction(CatalogTransaction tr
 
 optional_ptr<CatalogEntry> DuckSchemaEntry::AddEntry(CatalogTransaction transaction, unique_ptr<StandardEntry> entry,
                                                      OnCreateConflict on_conflict) {
-	DependencyList dependencies;
+	LogicalDependencyList dependencies;
 	return AddEntryInternal(transaction, std::move(entry), on_conflict, dependencies);
 }
 
@@ -205,7 +218,7 @@ optional_ptr<CatalogEntry> DuckSchemaEntry::CreateView(CatalogTransaction transa
 
 optional_ptr<CatalogEntry> DuckSchemaEntry::CreateIndex(ClientContext &context, CreateIndexInfo &info,
                                                         TableCatalogEntry &table) {
-	DependencyList dependencies;
+	LogicalDependencyList dependencies;
 	dependencies.AddDependency(table);
 
 	// currently, we can not alter PK/FK/UNIQUE constraints
@@ -287,6 +300,9 @@ void DuckSchemaEntry::DropEntry(ClientContext &context, DropInfo &info) {
 		                       CatalogTypeToString(existing_entry->type), CatalogTypeToString(info.type));
 	}
 
+	// if this is a index or table with indexes, initialize any unknown index instances
+	LazyLoadIndexes(context, *existing_entry);
+
 	// if there is a foreign key constraint, get that information
 	vector<unique_ptr<AlterForeignKeyInfo>> fk_arrays;
 	FindForeignKeyInformation(*existing_entry, AlterForeignKeyType::AFT_DELETE, fk_arrays);