duckdb 0.10.3-dev3.0 → 0.10.3-dev6.0

package/src/duckdb/extension/json/json_functions/json_structure.cpp

@@ -388,22 +388,28 @@ static inline void ExtractStructureObject(yyjson_val *obj, JSONStructureNode &no
 	auto &description = node.GetOrCreateDescription(LogicalTypeId::STRUCT);
 
 	// Keep track of keys so we can detect duplicates
-	case_insensitive_set_t obj_keys;
+	unordered_set<string> obj_keys;
+	case_insensitive_set_t ci_obj_keys;
 
 	size_t idx, max;
 	yyjson_val *key, *val;
 	yyjson_obj_foreach(obj, idx, max, key, val) {
-		auto key_ptr = unsafe_yyjson_get_str(key);
-		auto key_len = unsafe_yyjson_get_len(key);
-		auto insert_result = obj_keys.insert(string(key_ptr, key_len));
-		if (!ignore_errors && !insert_result.second) {
-			JSONCommon::ThrowValFormatError("Duplicate key \"" + string(key_ptr, key_len) + "\" in object %s", obj);
+		const string obj_key(unsafe_yyjson_get_str(key), unsafe_yyjson_get_len(key));
+		auto insert_result = obj_keys.insert(obj_key);
+		if (!ignore_errors && !insert_result.second) { // Exact match
+			JSONCommon::ThrowValFormatError("Duplicate key \"" + obj_key + "\" in object %s", obj);
+		}
+		insert_result = ci_obj_keys.insert(obj_key);
+		if (!ignore_errors && !insert_result.second) { // Case-insensitive match
+			JSONCommon::ThrowValFormatError("Duplicate key (different case) \"" + obj_key + "\" and \"" +
+			                                    *insert_result.first + "\" in object %s",
+			                                obj);
 		}
 		description.GetOrCreateChild(key, val, ignore_errors);
 	}
 }
 
-static inline void ExtractStructureVal(yyjson_val *val, JSONStructureNode &node, const bool ignore_errors) {
+static inline void ExtractStructureVal(yyjson_val *val, JSONStructureNode &node) {
 	D_ASSERT(!yyjson_is_arr(val) && !yyjson_is_obj(val));
 	node.GetOrCreateDescription(JSONCommon::ValTypeToLogicalTypeId(val));
 }
@@ -416,7 +422,7 @@ void JSONStructure::ExtractStructure(yyjson_val *val, JSONStructureNode &node, c
 	case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE:
 		return ExtractStructureObject(val, node, ignore_errors);
 	default:
-		return ExtractStructureVal(val, node, ignore_errors);
+		return ExtractStructureVal(val, node);
 	}
 }
 
@@ -475,9 +481,9 @@ static inline yyjson_mut_val *ConvertStructure(const JSONStructureNode &node, yy
 	}
 }
 
-static inline string_t JSONStructureFunction(yyjson_val *val, yyjson_alc *alc, Vector &result) {
+static inline string_t JSONStructureFunction(yyjson_val *val, yyjson_alc *alc, Vector &) {
 	return JSONCommon::WriteVal<yyjson_mut_val>(
-	    ConvertStructure(ExtractStructureInternal(val, false), yyjson_mut_doc_new(alc)), alc);
+	    ConvertStructure(ExtractStructureInternal(val, true), yyjson_mut_doc_new(alc)), alc);
 }
 
 static void StructureFunction(DataChunk &args, ExpressionState &state, Vector &result) {
@@ -497,8 +503,7 @@ ScalarFunctionSet JSONFunctions::GetStructureFunction() {
 }
 
 static LogicalType StructureToTypeArray(ClientContext &context, const JSONStructureNode &node, const idx_t max_depth,
-                                        const double field_appearance_threshold, idx_t depth,
-                                        const idx_t sample_count) {
+                                        const double field_appearance_threshold, idx_t depth) {
 	D_ASSERT(node.descriptions.size() == 1 && node.descriptions[0].type == LogicalTypeId::LIST);
 	const auto &desc = node.descriptions[0];
 	D_ASSERT(desc.children.size() == 1);
@@ -565,7 +570,7 @@ LogicalType JSONStructure::StructureToType(ClientContext &context, const JSONStr
 	D_ASSERT(desc.type != LogicalTypeId::INVALID);
 	switch (desc.type) {
 	case LogicalTypeId::LIST:
-		return StructureToTypeArray(context, node, max_depth, field_appearance_threshold, depth, sample_count);
+		return StructureToTypeArray(context, node, max_depth, field_appearance_threshold, depth);
 	case LogicalTypeId::STRUCT:
 		return StructureToTypeObject(context, node, max_depth, field_appearance_threshold, depth, sample_count);
 	case LogicalTypeId::VARCHAR:

package/src/duckdb/extension/json/json_functions/json_transform.cpp

@@ -59,6 +59,7 @@ static LogicalType StructureStringToType(yyjson_val *val, ClientContext &context
 		return StructureStringToTypeArray(val, context);
 	case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE:
 		return StructureToTypeObject(val, context);
+	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		return TransformStringToLogicalType(unsafe_yyjson_get_str(val), context);
 	default:
@@ -99,6 +100,7 @@ static inline bool GetValueNumerical(yyjson_val *val, T &result, JSONTransformOp
 	D_ASSERT(unsafe_yyjson_get_tag(val) != (YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE));
 	bool success;
 	switch (unsafe_yyjson_get_tag(val)) {
+	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		success = OP::template Operation<string_t, T>(GetString(val), result, options.strict_cast);
 		break;
@@ -134,6 +136,7 @@ static inline bool GetValueDecimal(yyjson_val *val, T &result, uint8_t w, uint8_
 	D_ASSERT(unsafe_yyjson_get_tag(val) != (YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE));
 	bool success;
 	switch (unsafe_yyjson_get_tag(val)) {
+	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		success = OP::template Operation<string_t, T>(GetString(val), result, options.parameters, w, s);
 		break;
@@ -167,6 +170,7 @@ static inline bool GetValueDecimal(yyjson_val *val, T &result, uint8_t w, uint8_
 static inline bool GetValueString(yyjson_val *val, yyjson_alc *alc, string_t &result, Vector &vector) {
 	D_ASSERT(unsafe_yyjson_get_tag(val) != (YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE));
 	switch (unsafe_yyjson_get_tag(val)) {
+	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NOESC:
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
 		result = string_t(unsafe_yyjson_get_str(val), unsafe_yyjson_get_len(val));
 		return true;

package/src/duckdb/extension/json/json_functions/json_type.cpp

@@ -11,11 +11,11 @@ static void UnaryTypeFunction(DataChunk &args, ExpressionState &state, Vector &r
 }
 
 static void BinaryTypeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
-	JSONExecutors::BinaryExecute<string_t>(args, state, result, GetType);
+	JSONExecutors::BinaryExecute<string_t, false>(args, state, result, GetType);
 }
 
 static void ManyTypeFunction(DataChunk &args, ExpressionState &state, Vector &result) {
-	JSONExecutors::ExecuteMany<string_t>(args, state, result, GetType);
+	JSONExecutors::ExecuteMany<string_t, false>(args, state, result, GetType);
 }
 
 static void GetTypeFunctionsInternal(ScalarFunctionSet &set, const LogicalType &input_type) {

package/src/duckdb/extension/json/json_functions/read_json.cpp

@@ -3,6 +3,7 @@
 #include "json_scan.hpp"
 #include "json_structure.hpp"
 #include "json_transform.hpp"
+#include "duckdb/common/helper.hpp"
 
 namespace duckdb {
 
@@ -276,8 +277,10 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
 		D_ASSERT(return_types.size() == names.size());
 	}
 
-	bind_data->reader_bind =
-	    MultiFileReader::BindOptions(bind_data->options.file_options, bind_data->files, return_types, names);
+	SimpleMultiFileList file_list(std::move(bind_data->files));
+	MultiFileReader().BindOptions(bind_data->options.file_options, file_list, return_types, names,
+	                              bind_data->reader_bind);
+	bind_data->files = file_list.GetAllFiles();
 
 	auto &transform_options = bind_data->transform_options;
 	transform_options.strict_cast = !bind_data->ignore_errors;
@@ -344,7 +347,7 @@ static void ReadJSONFunction(ClientContext &context, TableFunctionInput &data_p,
 	}
 
 	if (output.size() != 0) {
-		MultiFileReader::FinalizeChunk(gstate.bind_data.reader_bind, lstate.GetReaderData(), output);
+		MultiFileReader().FinalizeChunk(context, gstate.bind_data.reader_bind, lstate.GetReaderData(), output, nullptr);
 	}
 }
 
@@ -381,26 +384,26 @@ TableFunctionSet CreateJSONFunctionInfo(string name, shared_ptr<JSONScanInfo> in
 }
 
 TableFunctionSet JSONFunctions::GetReadJSONFunction() {
-	auto info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true);
+	auto info = make_shared_ptr<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT,
+	                                          JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_json", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadNDJSONFunction() {
-	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
-	                                      JSONRecordType::AUTO_DETECT, true);
+	auto info = make_shared_ptr<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
+	                                          JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_ndjson", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadJSONAutoFunction() {
-	auto info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true);
+	auto info = make_shared_ptr<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT,
+	                                          JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_json_auto", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadNDJSONAutoFunction() {
-	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
-	                                      JSONRecordType::AUTO_DETECT, true);
+	auto info = make_shared_ptr<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
+	                                          JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_ndjson_auto", std::move(info));
 }
 

package/src/duckdb/extension/json/json_functions/read_json_objects.cpp

@@ -1,6 +1,7 @@
 #include "json_common.hpp"
 #include "json_functions.hpp"
 #include "json_scan.hpp"
+#include "duckdb/common/helper.hpp"
 
 namespace duckdb {
 
@@ -13,8 +14,10 @@ unique_ptr<FunctionData> ReadJSONObjectsBind(ClientContext &context, TableFuncti
 	return_types.push_back(LogicalType::JSON());
 	names.emplace_back("json");
 
-	bind_data->reader_bind =
-	    MultiFileReader::BindOptions(bind_data->options.file_options, bind_data->files, return_types, names);
+	SimpleMultiFileList file_list(std::move(bind_data->files));
+	MultiFileReader().BindOptions(bind_data->options.file_options, file_list, return_types, names,
+	                              bind_data->reader_bind);
+	bind_data->files = file_list.GetAllFiles();
 
 	return std::move(bind_data);
 }
@@ -44,7 +47,7 @@ static void ReadJSONObjectsFunction(ClientContext &context, TableFunctionInput &
 	output.SetCardinality(count);
 
 	if (output.size() != 0) {
-		MultiFileReader::FinalizeChunk(gstate.bind_data.reader_bind, lstate.GetReaderData(), output);
+		MultiFileReader().FinalizeChunk(context, gstate.bind_data.reader_bind, lstate.GetReaderData(), output, nullptr);
 	}
 }
 
@@ -61,7 +64,7 @@ TableFunction GetReadJSONObjectsTableFunction(bool list_parameter, shared_ptr<JS
 TableFunctionSet JSONFunctions::GetReadJSONObjectsFunction() {
 	TableFunctionSet function_set("read_json_objects");
 	auto function_info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::ARRAY, JSONRecordType::RECORDS);
+	    make_shared_ptr<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::ARRAY, JSONRecordType::RECORDS);
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
 	return function_set;
@@ -69,8 +72,8 @@ TableFunctionSet JSONFunctions::GetReadJSONObjectsFunction() {
 
 TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() {
 	TableFunctionSet function_set("read_ndjson_objects");
-	auto function_info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED,
-	                                               JSONRecordType::RECORDS);
+	auto function_info = make_shared_ptr<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED,
+	                                                   JSONRecordType::RECORDS);
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
 	return function_set;
@@ -78,8 +81,8 @@ TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() {
 
 TableFunctionSet JSONFunctions::GetReadJSONObjectsAutoFunction() {
 	TableFunctionSet function_set("read_json_objects_auto");
-	auto function_info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::AUTO_DETECT, JSONRecordType::RECORDS);
+	auto function_info = make_shared_ptr<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::AUTO_DETECT,
+	                                                   JSONRecordType::RECORDS);
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
 	return function_set;

package/src/duckdb/extension/json/json_functions.cpp

@@ -16,7 +16,7 @@ using JSONPathType = JSONCommon::JSONPathType;
 
 static JSONPathType CheckPath(const Value &path_val, string &path, size_t &len) {
 	if (path_val.IsNull()) {
-		throw InternalException("JSON path cannot be NULL");
+		throw BinderException("JSON path cannot be NULL");
 	}
 	const auto path_str_val = path_val.DefaultCastAs(LogicalType::VARCHAR);
 	auto path_str = path_str_val.GetValueUnsafe<string_t>();
@@ -194,8 +194,9 @@ vector<TableFunctionSet> JSONFunctions::GetTableFunctions() {
 	return functions;
 }
 
-unique_ptr<TableRef> JSONFunctions::ReadJSONReplacement(ClientContext &context, const string &table_name,
-                                                        ReplacementScanData *data) {
+unique_ptr<TableRef> JSONFunctions::ReadJSONReplacement(ClientContext &context, ReplacementScanInput &input,
+                                                        optional_ptr<ReplacementScanData> data) {
+	auto &table_name = input.table_name;
 	if (!ReplacementScan::CanReplace(table_name, {"json", "jsonl", "ndjson"})) {
 		return nullptr;
 	}

package/src/duckdb/extension/json/json_scan.cpp

@@ -29,7 +29,7 @@ void JSONScanData::Bind(ClientContext &context, TableFunctionBindInput &input) {
 	auto_detect = info.auto_detect;
 
 	for (auto &kv : input.named_parameters) {
-		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options, context)) {
+		if (MultiFileReader().ParseOption(kv.first, kv.second, options.file_options, context)) {
 			continue;
 		}
 		auto loption = StringUtil::Lower(kv.first);
@@ -60,8 +60,12 @@ void JSONScanData::Bind(ClientContext &context, TableFunctionBindInput &input) {
 		}
 	}
 
-	files = MultiFileReader::GetFileList(context, input.inputs[0], "JSON");
-	options.file_options.AutoDetectHivePartitioning(files, context);
+	auto multi_file_reader = MultiFileReader::Create(input.table_function);
+	auto file_list = multi_file_reader->CreateFileList(context, input.inputs[0]);
+	options.file_options.AutoDetectHivePartitioning(*file_list, context);
+
+	// TODO: store the MultiFilelist instead
+	files = file_list->GetAllFiles();
 
 	InitializeReaders(context);
 }
@@ -201,9 +205,9 @@ unique_ptr<GlobalTableFunctionState> JSONGlobalTableFunctionState::Init(ClientCo
 
 	vector<LogicalType> dummy_types(input.column_ids.size(), LogicalType::ANY);
 	for (auto &reader : gstate.json_readers) {
-		MultiFileReader::FinalizeBind(reader->GetOptions().file_options, gstate.bind_data.reader_bind,
-		                              reader->GetFileName(), gstate.names, dummy_types, bind_data.names,
-		                              input.column_ids, reader->reader_data, context);
+		MultiFileReader().FinalizeBind(reader->GetOptions().file_options, gstate.bind_data.reader_bind,
+		                               reader->GetFileName(), gstate.names, dummy_types, bind_data.names,
+		                               input.column_ids, reader->reader_data, context, nullptr);
 	}
 
 	return std::move(result);
@@ -973,10 +977,16 @@ unique_ptr<NodeStatistics> JSONScan::Cardinality(ClientContext &, const Function
 void JSONScan::ComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionData *bind_data_p,
                                      vector<unique_ptr<Expression>> &filters) {
 	auto &data = bind_data_p->Cast<JSONScanData>();
-	auto reset_reader =
-	    MultiFileReader::ComplexFilterPushdown(context, data.files, data.options.file_options, get, filters);
-	if (reset_reader) {
-		MultiFileReader::PruneReaders(data);
+
+	SimpleMultiFileList file_list(std::move(data.files));
+
+	auto filtered_list =
+	    MultiFileReader().ComplexFilterPushdown(context, file_list, data.options.file_options, get, filters);
+	if (filtered_list) {
+		MultiFileReader().PruneReaders(data, *filtered_list);
+		data.files = filtered_list->GetAllFiles();
+	} else {
+		data.files = file_list.GetAllFiles();
 	}
 }
 
@@ -995,7 +1005,7 @@ unique_ptr<FunctionData> JSONScan::Deserialize(Deserializer &deserializer, Table
 }
 
 void JSONScan::TableFunctionDefaults(TableFunction &table_function) {
-	MultiFileReader::AddParameters(table_function);
+	MultiFileReader().AddParameters(table_function);
 
 	table_function.named_parameters["maximum_object_size"] = LogicalType::UINTEGER;
 	table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN;

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -12,6 +12,7 @@
 #include "row_number_column_reader.hpp"
 #include "snappy.h"
 #include "string_column_reader.hpp"
+#include "null_column_reader.hpp"
 #include "struct_column_reader.hpp"
 #include "templated_column_reader.hpp"
 #include "utf8proc_wrapper.hpp"
@@ -19,6 +20,7 @@
 #include "lz4.hpp"
 
 #ifndef DUCKDB_AMALGAMATION
+#include "duckdb/common/helper.hpp"
 #include "duckdb/common/types/bit.hpp"
 #include "duckdb/common/types/blob.hpp"
 #endif
@@ -303,7 +305,7 @@ void ColumnReader::PreparePageV2(PageHeader &page_hdr) {
 
 void ColumnReader::AllocateBlock(idx_t size) {
 	if (!block) {
-		block = make_shared<ResizeableBuffer>(GetAllocator(), size);
+		block = make_shared_ptr<ResizeableBuffer>(GetAllocator(), size);
 	} else {
 		block->resize(GetAllocator(), size);
 	}
@@ -515,7 +517,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr
 			        result);
 		} else if (dbp_decoder) {
 			// TODO keep this in the state
-			auto read_buf = make_shared<ResizeableBuffer>();
+			auto read_buf = make_shared_ptr<ResizeableBuffer>();
 
 			switch (schema.type) {
 			case duckdb_parquet::format::Type::INT32:
@@ -536,7 +538,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr
 		} else if (rle_decoder) {
 			// RLE encoding for boolean
 			D_ASSERT(type.id() == LogicalTypeId::BOOLEAN);
-			auto read_buf = make_shared<ResizeableBuffer>();
+			auto read_buf = make_shared_ptr<ResizeableBuffer>();
 			read_buf->resize(reader.allocator, sizeof(bool) * (read_now - null_count));
 			rle_decoder->GetBatch<uint8_t>(read_buf->ptr, read_now - null_count);
 			PlainTemplated<bool, TemplatedParquetValueConversion<bool>>(read_buf, define_out, read_now, filter,
@@ -545,7 +547,7 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr
 			// DELTA_BYTE_ARRAY or DELTA_LENGTH_BYTE_ARRAY
 			DeltaByteArray(define_out, read_now, filter, result_offset, result);
 		} else if (bss_decoder) {
-			auto read_buf = make_shared<ResizeableBuffer>();
+			auto read_buf = make_shared_ptr<ResizeableBuffer>();
 
 			switch (schema.type) {
 			case duckdb_parquet::format::Type::FLOAT:
@@ -661,7 +663,7 @@ void StringColumnReader::Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num
 static shared_ptr<ResizeableBuffer> ReadDbpData(Allocator &allocator, ResizeableBuffer &buffer, idx_t &value_count) {
 	auto decoder = make_uniq<DbpDecoder>(buffer.ptr, buffer.len);
 	value_count = decoder->TotalValues();
-	auto result = make_shared<ResizeableBuffer>();
+	auto result = make_shared_ptr<ResizeableBuffer>();
 	result->resize(allocator, sizeof(uint32_t) * value_count);
 	decoder->GetBatch<uint32_t>(result->ptr, value_count);
 	decoder->Finalize();
@@ -1533,6 +1535,8 @@ unique_ptr<ColumnReader> ColumnReader::CreateReader(ParquetReader &reader, const
 		return make_uniq<UUIDColumnReader>(reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
 	case LogicalTypeId::INTERVAL:
 		return make_uniq<IntervalColumnReader>(reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
+	case LogicalTypeId::SQLNULL:
+		return make_uniq<NullColumnReader>(reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
 	default:
 		break;
 	}

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -69,6 +69,10 @@ static uint8_t GetVarintSize(uint32_t val) {
 ColumnWriterStatistics::~ColumnWriterStatistics() {
 }
 
+bool ColumnWriterStatistics::HasStats() {
+	return false;
+}
+
 string ColumnWriterStatistics::GetMin() {
 	return string();
 }
@@ -221,11 +225,16 @@ void ColumnWriter::CompressPage(MemoryStream &temp_writer, size_t &compressed_si
 		break;
 	}
 	case CompressionCodec::ZSTD: {
+		auto configured_compression = writer.CompressionLevel();
+		int compress_level = ZSTD_CLEVEL_DEFAULT;
+		if (configured_compression.IsValid()) {
+			compress_level = static_cast<int>(configured_compression.GetIndex());
+		}
 		compressed_size = duckdb_zstd::ZSTD_compressBound(temp_writer.GetPosition());
 		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
-		compressed_size = duckdb_zstd::ZSTD_compress((void *)compressed_buf.get(), compressed_size,
-		                                             (const void *)temp_writer.GetData(), temp_writer.GetPosition(),
-		                                             ZSTD_CLEVEL_DEFAULT);
+		compressed_size =
+		    duckdb_zstd::ZSTD_compress((void *)compressed_buf.get(), compressed_size,
+		                               (const void *)temp_writer.GetData(), temp_writer.GetPosition(), compress_level);
 		compressed_data = compressed_buf.get();
 		break;
 	}
@@ -480,7 +489,7 @@ void BasicColumnWriter::BeginWrite(ColumnWriterState &state_p) {
 		auto &page_info = state.page_info[page_idx];
 		if (page_info.row_count == 0) {
 			D_ASSERT(page_idx + 1 == state.page_info.size());
-			state.page_info.erase(state.page_info.begin() + page_idx);
+			state.page_info.erase_at(page_idx);
 			break;
 		}
 		PageWriteInformation write_info;
@@ -585,7 +594,7 @@ void BasicColumnWriter::FlushPage(BasicColumnWriterState &state) {
 	D_ASSERT(hdr.compressed_page_size > 0);
 
 	if (write_info.compressed_buf) {
-		// if the data has been compressed, we no longer need the compressed data
+		// if the data has been compressed, we no longer need the uncompressed data
 		D_ASSERT(write_info.compressed_buf.get() == write_info.compressed_data);
 		write_info.temp_writer.reset();
 	}
@@ -646,15 +655,12 @@ void BasicColumnWriter::SetParquetStatistics(BasicColumnWriterState &state,
 		column_chunk.meta_data.statistics.__isset.max = true;
 		column_chunk.meta_data.__isset.statistics = true;
 	}
-	auto min_value = state.stats_state->GetMinValue();
-	if (!min_value.empty()) {
-		column_chunk.meta_data.statistics.min_value = std::move(min_value);
+	if (state.stats_state->HasStats()) {
+		column_chunk.meta_data.statistics.min_value = state.stats_state->GetMinValue();
 		column_chunk.meta_data.statistics.__isset.min_value = true;
 		column_chunk.meta_data.__isset.statistics = true;
-	}
-	auto max_value = state.stats_state->GetMaxValue();
-	if (!max_value.empty()) {
-		column_chunk.meta_data.statistics.max_value = std::move(max_value);
+
+		column_chunk.meta_data.statistics.max_value = state.stats_state->GetMaxValue();
 		column_chunk.meta_data.statistics.__isset.max_value = true;
 		column_chunk.meta_data.__isset.statistics = true;
 	}
@@ -681,18 +687,24 @@ void BasicColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 	if (HasDictionary(state)) {
 		column_chunk.meta_data.statistics.distinct_count = DictionarySize(state);
 		column_chunk.meta_data.statistics.__isset.distinct_count = true;
-		column_chunk.meta_data.dictionary_page_offset = start_offset;
+		column_chunk.meta_data.dictionary_page_offset = column_writer.GetTotalWritten();
 		column_chunk.meta_data.__isset.dictionary_page_offset = true;
 		FlushDictionary(state, state.stats_state.get());
 	}
 
 	// record the start position of the pages for this column
-	column_chunk.meta_data.data_page_offset = column_writer.GetTotalWritten();
+	column_chunk.meta_data.data_page_offset = 0;
 	SetParquetStatistics(state, column_chunk);
 
 	// write the individual pages to disk
 	idx_t total_uncompressed_size = 0;
 	for (auto &write_info : state.write_info) {
+		// set the data page offset whenever we see the *first* data page
+		if (column_chunk.meta_data.data_page_offset == 0 && (write_info.page_header.type == PageType::DATA_PAGE ||
+		                                                     write_info.page_header.type == PageType::DATA_PAGE_V2)) {
+			column_chunk.meta_data.data_page_offset = column_writer.GetTotalWritten();
+			;
+		}
 		D_ASSERT(write_info.page_header.uncompressed_page_size > 0);
 		auto header_start_offset = column_writer.GetTotalWritten();
 		writer.Write(write_info.page_header);
@@ -756,7 +768,7 @@ public:
 	T max;
 
 public:
-	bool HasStats() {
+	bool HasStats() override {
 		return min <= max;
 	}
 
@@ -902,7 +914,7 @@ public:
 	bool max;
 
 public:
-	bool HasStats() {
+	bool HasStats() override {
 		return !(min && !max);
 	}
 
@@ -1024,7 +1036,7 @@ public:
 		return string(const_char_ptr_cast(buffer), 16);
 	}
 
-	bool HasStats() {
+	bool HasStats() override {
 		return min <= max;
 	}
 
@@ -1190,7 +1202,7 @@ public:
 	string max;
 
 public:
-	bool HasStats() {
+	bool HasStats() override {
 		return has_stats;
 	}
 
@@ -1205,6 +1217,7 @@ public:
 			// ideally we avoid placing several mega or giga-byte long strings there
 			// we put a threshold of 10KB, if we see strings that exceed this threshold we avoid gathering stats
 			values_too_big = true;
+			has_stats = false;
 			min = string();
 			max = string();
 			return;

package/src/duckdb/extension/parquet/include/column_writer.hpp

@@ -44,6 +44,7 @@ class ColumnWriterStatistics {
 public:
 	virtual ~ColumnWriterStatistics();
 
+	virtual bool HasStats();
 	virtual string GetMin();
 	virtual string GetMax();
 	virtual string GetMinValue();

package/src/duckdb/extension/parquet/include/null_column_reader.hpp

@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// null_column_reader.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "column_reader.hpp"
+#include "duckdb/common/helper.hpp"
+
+namespace duckdb {
+
+class NullColumnReader : public ColumnReader {
+public:
+	static constexpr const PhysicalType TYPE = PhysicalType::INVALID;
+
+public:
+	NullColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p, idx_t schema_idx_p,
+	                 idx_t max_define_p, idx_t max_repeat_p)
+	    : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p) {};
+
+	shared_ptr<ResizeableBuffer> dict;
+
+public:
+	void Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num_entries) override {
+		dict = std::move(data);
+	}
+
+	void Offsets(uint32_t *offsets, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
+	             idx_t result_offset, Vector &result) override {
+		auto &result_mask = FlatVector::Validity(result);
+
+		for (idx_t row_idx = 0; row_idx < num_values; row_idx++) {
+			result_mask.SetInvalid(row_idx + result_offset);
+		}
+	}
+
+	void Plain(shared_ptr<ByteBuffer> plain_data, uint8_t *defines, uint64_t num_values, parquet_filter_t &filter,
+	           idx_t result_offset, Vector &result) override {
+		(void)defines;
+		(void)plain_data;
+		(void)filter;
+
+		auto &result_mask = FlatVector::Validity(result);
+		for (idx_t row_idx = 0; row_idx < num_values; row_idx++) {
+			result_mask.SetInvalid(row_idx + result_offset);
+		}
+	}
+};
+
+} // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_extension.hpp

@@ -8,6 +8,7 @@ class ParquetExtension : public Extension {
 public:
 	void Load(DuckDB &db) override;
 	std::string Name() override;
+	std::string Version() const override;
 };
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/include/parquet_reader.hpp

@@ -53,7 +53,7 @@ struct ParquetReaderScanState {
 	idx_t group_offset;
 	unique_ptr<FileHandle> file_handle;
 	unique_ptr<ColumnReader> root_reader;
-	unique_ptr<duckdb_apache::thrift::protocol::TProtocol> thrift_file_proto;
+	std::unique_ptr<duckdb_apache::thrift::protocol::TProtocol> thrift_file_proto;
 
 	bool finished;
 	SelectionVector sel;

package/src/duckdb/extension/parquet/include/parquet_writer.hpp

@@ -64,7 +64,8 @@ public:
 	ParquetWriter(FileSystem &fs, string file_name, vector<LogicalType> types, vector<string> names,
 	              duckdb_parquet::format::CompressionCodec::type codec, ChildFieldIDs field_ids,
 	              const vector<pair<string, string>> &kv_metadata,
-	              shared_ptr<ParquetEncryptionConfig> encryption_config, double dictionary_compression_ratio_threshold);
+	              shared_ptr<ParquetEncryptionConfig> encryption_config, double dictionary_compression_ratio_threshold,
+	              optional_idx compression_level);
 
 public:
 	void PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGroup &result);
@@ -94,6 +95,9 @@ public:
 	double DictionaryCompressionRatioThreshold() const {
 		return dictionary_compression_ratio_threshold;
 	}
+	optional_idx CompressionLevel() const {
+		return compression_level;
+	}
 
 	static CopyTypeSupport TypeIsSupported(const LogicalType &type);
 
@@ -110,9 +114,10 @@ private:
 	ChildFieldIDs field_ids;
 	shared_ptr<ParquetEncryptionConfig> encryption_config;
 	double dictionary_compression_ratio_threshold;
+	optional_idx compression_level;
 
 	unique_ptr<BufferedFileWriter> writer;
-	shared_ptr<duckdb_apache::thrift::protocol::TProtocol> protocol;
+	std::shared_ptr<duckdb_apache::thrift::protocol::TProtocol> protocol;
 	duckdb_parquet::format::FileMetaData file_meta_data;
 	std::mutex lock;