duckdb 1.1.4-dev2.0 → 1.1.4-dev9.0

package/src/duckdb/extension/parquet/parquet_statistics.cpp

@@ -5,21 +5,24 @@
 #include "parquet_timestamp.hpp"
 #include "string_column_reader.hpp"
 #include "struct_column_reader.hpp"
+#include "zstd/common/xxhash.hpp"
+
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/time.hpp"
 #include "duckdb/common/types/value.hpp"
 #include "duckdb/storage/statistics/struct_stats.hpp"
+#include "duckdb/planner/filter/constant_filter.hpp"
 #endif
 
 namespace duckdb {
 
-using duckdb_parquet::format::ConvertedType;
-using duckdb_parquet::format::Type;
+using duckdb_parquet::ConvertedType;
+using duckdb_parquet::Type;
 
 static unique_ptr<BaseStatistics> CreateNumericStats(const LogicalType &type,
-                                                     const duckdb_parquet::format::SchemaElement &schema_ele,
-                                                     const duckdb_parquet::format::Statistics &parquet_stats) {
+                                                     const duckdb_parquet::SchemaElement &schema_ele,
+                                                     const duckdb_parquet::Statistics &parquet_stats) {
 	auto stats = NumericStats::CreateUnknown(type);
 
 	// for reasons unknown to science, Parquet defines *both* `min` and `min_value` as well as `max` and
@@ -27,16 +30,16 @@ static unique_ptr<BaseStatistics> CreateNumericStats(const LogicalType &type,
 	Value min;
 	Value max;
 	if (parquet_stats.__isset.min_value) {
-		min = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.min_value).DefaultCastAs(type);
+		min = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.min_value);
 	} else if (parquet_stats.__isset.min) {
-		min = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.min).DefaultCastAs(type);
+		min = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.min);
 	} else {
 		min = Value(type);
 	}
 	if (parquet_stats.__isset.max_value) {
-		max = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.max_value).DefaultCastAs(type);
+		max = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.max_value);
 	} else if (parquet_stats.__isset.max) {
-		max = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.max).DefaultCastAs(type);
+		max = ParquetStatisticsUtils::ConvertValue(type, schema_ele, parquet_stats.max);
 	} else {
 		max = Value(type);
 	}
@@ -45,14 +48,24 @@ static unique_ptr<BaseStatistics> CreateNumericStats(const LogicalType &type,
 	return stats.ToUnique();
 }
 
-Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
-                                           const duckdb_parquet::format::SchemaElement &schema_ele,
+Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type, const duckdb_parquet::SchemaElement &schema_ele,
                                            const std::string &stats) {
+	Value result;
+	string error;
+	auto stats_val = ConvertValueInternal(type, schema_ele, stats);
+	if (!stats_val.DefaultTryCastAs(type, result, &error)) {
+		return Value(type);
+	}
+	return result;
+}
+Value ParquetStatisticsUtils::ConvertValueInternal(const LogicalType &type,
+                                                   const duckdb_parquet::SchemaElement &schema_ele,
+                                                   const std::string &stats) {
 	auto stats_data = const_data_ptr_cast(stats.c_str());
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN: {
 		if (stats.size() != sizeof(bool)) {
-			throw InternalException("Incorrect stats size for type BOOLEAN");
+			throw InvalidInputException("Incorrect stats size for type BOOLEAN");
 		}
 		return Value::BOOLEAN(Load<bool>(stats_data));
 	}
@@ -60,29 +73,29 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 	case LogicalTypeId::USMALLINT:
 	case LogicalTypeId::UINTEGER:
 		if (stats.size() != sizeof(uint32_t)) {
-			throw InternalException("Incorrect stats size for type UINTEGER");
+			throw InvalidInputException("Incorrect stats size for type UINTEGER");
 		}
 		return Value::UINTEGER(Load<uint32_t>(stats_data));
 	case LogicalTypeId::UBIGINT:
 		if (stats.size() != sizeof(uint64_t)) {
-			throw InternalException("Incorrect stats size for type UBIGINT");
+			throw InvalidInputException("Incorrect stats size for type UBIGINT");
 		}
 		return Value::UBIGINT(Load<uint64_t>(stats_data));
 	case LogicalTypeId::TINYINT:
 	case LogicalTypeId::SMALLINT:
 	case LogicalTypeId::INTEGER:
 		if (stats.size() != sizeof(int32_t)) {
-			throw InternalException("Incorrect stats size for type INTEGER");
+			throw InvalidInputException("Incorrect stats size for type INTEGER");
 		}
 		return Value::INTEGER(Load<int32_t>(stats_data));
 	case LogicalTypeId::BIGINT:
 		if (stats.size() != sizeof(int64_t)) {
-			throw InternalException("Incorrect stats size for type BIGINT");
+			throw InvalidInputException("Incorrect stats size for type BIGINT");
 		}
 		return Value::BIGINT(Load<int64_t>(stats_data));
 	case LogicalTypeId::FLOAT: {
 		if (stats.size() != sizeof(float)) {
-			throw InternalException("Incorrect stats size for type FLOAT");
+			throw InvalidInputException("Incorrect stats size for type FLOAT");
 		}
 		auto val = Load<float>(stats_data);
 		if (!Value::FloatIsFinite(val)) {
@@ -100,7 +113,7 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 			break;
 		}
 		if (stats.size() != sizeof(double)) {
-			throw InternalException("Incorrect stats size for type DOUBLE");
+			throw InvalidInputException("Incorrect stats size for type DOUBLE");
 		}
 		auto val = Load<double>(stats_data);
 		if (!Value::DoubleIsFinite(val)) {
@@ -114,13 +127,13 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 		switch (schema_ele.type) {
 		case Type::INT32: {
 			if (stats.size() != sizeof(int32_t)) {
-				throw InternalException("Incorrect stats size for type %s", type.ToString());
+				throw InvalidInputException("Incorrect stats size for type %s", type.ToString());
 			}
 			return Value::DECIMAL(Load<int32_t>(stats_data), width, scale);
 		}
 		case Type::INT64: {
 			if (stats.size() != sizeof(int64_t)) {
-				throw InternalException("Incorrect stats size for type %s", type.ToString());
+				throw InvalidInputException("Incorrect stats size for type %s", type.ToString());
 			}
 			return Value::DECIMAL(Load<int64_t>(stats_data), width, scale);
 		}
@@ -141,22 +154,21 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 				    ParquetDecimalUtils::ReadDecimalValue<hugeint_t>(stats_data, stats.size(), schema_ele), width,
 				    scale);
 			default:
-				throw InternalException("Unsupported internal type for decimal");
+				throw InvalidInputException("Unsupported internal type for decimal");
 			}
 		default:
 			throw InternalException("Unsupported internal type for decimal?..");
 		}
 	}
-	case LogicalType::VARCHAR:
-	case LogicalType::BLOB:
-		if (Value::StringIsValid(stats)) {
-			return Value(stats);
-		} else {
+	case LogicalTypeId::VARCHAR:
+	case LogicalTypeId::BLOB:
+		if (type.id() == LogicalTypeId::BLOB || !Value::StringIsValid(stats)) {
 			return Value(Blob::ToString(string_t(stats)));
 		}
+		return Value(stats);
 	case LogicalTypeId::DATE:
 		if (stats.size() != sizeof(int32_t)) {
-			throw InternalException("Incorrect stats size for type DATE");
+			throw InvalidInputException("Incorrect stats size for type DATE");
 		}
 		return Value::DATE(date_t(Load<int32_t>(stats_data)));
 	case LogicalTypeId::TIME: {
@@ -166,7 +178,7 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 		} else if (stats.size() == sizeof(int64_t)) {
 			val = Load<int64_t>(stats_data);
 		} else {
-			throw InternalException("Incorrect stats size for type TIME");
+			throw InvalidInputException("Incorrect stats size for type TIME");
 		}
 		if (schema_ele.__isset.logicalType && schema_ele.logicalType.__isset.TIME) {
 			// logical type
@@ -180,7 +192,7 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 				throw InternalException("Time logicalType is set but unit is not defined");
 			}
 		}
-		if (schema_ele.converted_type == duckdb_parquet::format::ConvertedType::TIME_MILLIS) {
+		if (schema_ele.converted_type == duckdb_parquet::ConvertedType::TIME_MILLIS) {
 			return Value::TIME(Time::FromTimeMs(val));
 		} else {
 			return Value::TIME(dtime_t(val));
@@ -193,7 +205,7 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 		} else if (stats.size() == sizeof(int64_t)) {
 			val = Load<int64_t>(stats_data);
 		} else {
-			throw InternalException("Incorrect stats size for type TIMETZ");
+			throw InvalidInputException("Incorrect stats size for type TIMETZ");
 		}
 		if (schema_ele.__isset.logicalType && schema_ele.logicalType.__isset.TIME) {
 			// logical type
@@ -214,13 +226,13 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 		timestamp_t timestamp_value;
 		if (schema_ele.type == Type::INT96) {
 			if (stats.size() != sizeof(Int96)) {
-				throw InternalException("Incorrect stats size for type TIMESTAMP");
+				throw InvalidInputException("Incorrect stats size for type TIMESTAMP");
 			}
 			timestamp_value = ImpalaTimestampToTimestamp(Load<Int96>(stats_data));
 		} else {
 			D_ASSERT(schema_ele.type == Type::INT64);
 			if (stats.size() != sizeof(int64_t)) {
-				throw InternalException("Incorrect stats size for type TIMESTAMP");
+				throw InvalidInputException("Incorrect stats size for type TIMESTAMP");
 			}
 			auto val = Load<int64_t>(stats_data);
 			if (schema_ele.__isset.logicalType && schema_ele.logicalType.__isset.TIMESTAMP) {
@@ -234,29 +246,28 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 				} else {
 					throw InternalException("Timestamp logicalType is set but unit is not defined");
 				}
-			} else if (schema_ele.converted_type == duckdb_parquet::format::ConvertedType::TIMESTAMP_MILLIS) {
+			} else if (schema_ele.converted_type == duckdb_parquet::ConvertedType::TIMESTAMP_MILLIS) {
 				timestamp_value = Timestamp::FromEpochMs(val);
 			} else {
 				timestamp_value = timestamp_t(val);
 			}
 		}
 		if (type.id() == LogicalTypeId::TIMESTAMP_TZ) {
-			return Value::TIMESTAMPTZ(timestamp_value);
-		} else {
-			return Value::TIMESTAMP(timestamp_value);
+			return Value::TIMESTAMPTZ(timestamp_tz_t(timestamp_value));
 		}
+		return Value::TIMESTAMP(timestamp_value);
 	}
 	case LogicalTypeId::TIMESTAMP_NS: {
 		timestamp_ns_t timestamp_value;
 		if (schema_ele.type == Type::INT96) {
 			if (stats.size() != sizeof(Int96)) {
-				throw InternalException("Incorrect stats size for type TIMESTAMP_NS");
+				throw InvalidInputException("Incorrect stats size for type TIMESTAMP_NS");
 			}
 			timestamp_value = ImpalaTimestampToTimestampNS(Load<Int96>(stats_data));
 		} else {
 			D_ASSERT(schema_ele.type == Type::INT64);
 			if (stats.size() != sizeof(int64_t)) {
-				throw InternalException("Incorrect stats size for type TIMESTAMP_NS");
+				throw InvalidInputException("Incorrect stats size for type TIMESTAMP_NS");
 			}
 			auto val = Load<int64_t>(stats_data);
 			if (schema_ele.__isset.logicalType && schema_ele.logicalType.__isset.TIMESTAMP) {
@@ -270,7 +281,7 @@ Value ParquetStatisticsUtils::ConvertValue(const LogicalType &type,
 				} else {
 					throw InternalException("Timestamp (NS) logicalType is set but unit is unknown");
 				}
-			} else if (schema_ele.converted_type == duckdb_parquet::format::ConvertedType::TIMESTAMP_MILLIS) {
+			} else if (schema_ele.converted_type == duckdb_parquet::ConvertedType::TIMESTAMP_MILLIS) {
 				timestamp_value = ParquetTimestampMsToTimestampNs(val);
 			} else {
 				timestamp_value = ParquetTimestampUsToTimestampNs(val);
@@ -300,6 +311,9 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 		auto &struct_reader = reader.Cast<StructColumnReader>();
 		// Recurse into child readers
 		for (idx_t i = 0; i < struct_reader.child_readers.size(); i++) {
+			if (!struct_reader.child_readers[i]) {
+				continue;
+			}
 			auto &child_reader = *struct_reader.child_readers[i];
 			auto child_stats = ParquetStatisticsUtils::TransformColumnStatistics(child_reader, columns);
 			StructStats::SetChildStats(struct_stats, i, std::move(child_stats));
@@ -387,4 +401,210 @@ unique_ptr<BaseStatistics> ParquetStatisticsUtils::TransformColumnStatistics(con
 	return row_group_stats;
 }
 
+static bool HasFilterConstants(const TableFilter &duckdb_filter) {
+	switch (duckdb_filter.filter_type) {
+	case TableFilterType::CONSTANT_COMPARISON: {
+		auto &constant_filter = duckdb_filter.Cast<ConstantFilter>();
+		return (constant_filter.comparison_type == ExpressionType::COMPARE_EQUAL && !constant_filter.constant.IsNull());
+	}
+	case TableFilterType::CONJUNCTION_AND: {
+		auto &conjunction_and_filter = duckdb_filter.Cast<ConjunctionAndFilter>();
+		bool child_has_constant = false;
+		for (auto &child_filter : conjunction_and_filter.child_filters) {
+			child_has_constant |= HasFilterConstants(*child_filter);
+		}
+		return child_has_constant;
+	}
+	case TableFilterType::CONJUNCTION_OR: {
+		auto &conjunction_or_filter = duckdb_filter.Cast<ConjunctionOrFilter>();
+		bool child_has_constant = false;
+		for (auto &child_filter : conjunction_or_filter.child_filters) {
+			child_has_constant |= HasFilterConstants(*child_filter);
+		}
+		return child_has_constant;
+	}
+	default:
+		return false;
+	}
+}
+
+template <class T>
+uint64_t ValueXH64FixedWidth(const Value &constant) {
+	T val = constant.GetValue<T>();
+	return duckdb_zstd::XXH64(&val, sizeof(val), 0);
+}
+
+// TODO we can only this if the parquet representation of the type exactly matches the duckdb rep!
+// TODO TEST THIS!
+// TODO perhaps we can re-use some writer infra here
+static uint64_t ValueXXH64(const Value &constant) {
+	switch (constant.type().InternalType()) {
+	case PhysicalType::UINT8:
+		return ValueXH64FixedWidth<int32_t>(constant);
+	case PhysicalType::INT8:
+		return ValueXH64FixedWidth<int32_t>(constant);
+	case PhysicalType::UINT16:
+		return ValueXH64FixedWidth<int32_t>(constant);
+	case PhysicalType::INT16:
+		return ValueXH64FixedWidth<int32_t>(constant);
+	case PhysicalType::UINT32:
+		return ValueXH64FixedWidth<uint32_t>(constant);
+	case PhysicalType::INT32:
+		return ValueXH64FixedWidth<int32_t>(constant);
+	case PhysicalType::UINT64:
+		return ValueXH64FixedWidth<uint64_t>(constant);
+	case PhysicalType::INT64:
+		return ValueXH64FixedWidth<int64_t>(constant);
+	case PhysicalType::FLOAT:
+		return ValueXH64FixedWidth<float>(constant);
+	case PhysicalType::DOUBLE:
+		return ValueXH64FixedWidth<double>(constant);
+	case PhysicalType::VARCHAR: {
+		auto val = constant.GetValue<string>();
+		return duckdb_zstd::XXH64(val.c_str(), val.length(), 0);
+	}
+	default:
+		return 0;
+	}
+}
+
+static bool ApplyBloomFilter(const TableFilter &duckdb_filter, ParquetBloomFilter &bloom_filter) {
+	switch (duckdb_filter.filter_type) {
+	case TableFilterType::CONSTANT_COMPARISON: {
+		auto &constant_filter = duckdb_filter.Cast<ConstantFilter>();
+		auto is_compare_equal = constant_filter.comparison_type == ExpressionType::COMPARE_EQUAL;
+		D_ASSERT(!constant_filter.constant.IsNull());
+		auto hash = ValueXXH64(constant_filter.constant);
+		return hash > 0 && !bloom_filter.FilterCheck(hash) && is_compare_equal;
+	}
+	case TableFilterType::CONJUNCTION_AND: {
+		auto &conjunction_and_filter = duckdb_filter.Cast<ConjunctionAndFilter>();
+		bool any_children_true = false;
+		for (auto &child_filter : conjunction_and_filter.child_filters) {
+			any_children_true |= ApplyBloomFilter(*child_filter, bloom_filter);
+		}
+		return any_children_true;
+	}
+	case TableFilterType::CONJUNCTION_OR: {
+		auto &conjunction_or_filter = duckdb_filter.Cast<ConjunctionOrFilter>();
+		bool all_children_true = true;
+		for (auto &child_filter : conjunction_or_filter.child_filters) {
+			all_children_true &= ApplyBloomFilter(*child_filter, bloom_filter);
+		}
+		return all_children_true;
+	}
+	default:
+		return false;
+	}
+}
+
+bool ParquetStatisticsUtils::BloomFilterSupported(const LogicalTypeId &type_id) {
+	switch (type_id) {
+	case LogicalTypeId::TINYINT:
+	case LogicalTypeId::UTINYINT:
+	case LogicalTypeId::SMALLINT:
+	case LogicalTypeId::USMALLINT:
+	case LogicalTypeId::INTEGER:
+	case LogicalTypeId::UINTEGER:
+	case LogicalTypeId::BIGINT:
+	case LogicalTypeId::UBIGINT:
+	case LogicalTypeId::FLOAT:
+	case LogicalTypeId::DOUBLE:
+	case LogicalTypeId::VARCHAR:
+	case LogicalTypeId::BLOB:
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool ParquetStatisticsUtils::BloomFilterExcludes(const TableFilter &duckdb_filter,
+                                                 const duckdb_parquet::ColumnMetaData &column_meta_data,
+                                                 TProtocol &file_proto, Allocator &allocator) {
+	if (!HasFilterConstants(duckdb_filter) || !column_meta_data.__isset.bloom_filter_offset ||
+	    column_meta_data.bloom_filter_offset <= 0) {
+		return false;
+	}
+	// TODO check length against file length!
+
+	auto &transport = reinterpret_cast<ThriftFileTransport &>(*file_proto.getTransport());
+	transport.SetLocation(column_meta_data.bloom_filter_offset);
+	if (column_meta_data.__isset.bloom_filter_length && column_meta_data.bloom_filter_length > 0) {
+		transport.Prefetch(column_meta_data.bloom_filter_offset, column_meta_data.bloom_filter_length);
+	}
+
+	duckdb_parquet::BloomFilterHeader filter_header;
+	// TODO the bloom filter could be encrypted, too, so need to double check that this is NOT the case
+	filter_header.read(&file_proto);
+	if (!filter_header.algorithm.__isset.BLOCK || !filter_header.compression.__isset.UNCOMPRESSED ||
+	    !filter_header.hash.__isset.XXHASH) {
+		return false;
+	}
+
+	auto new_buffer = make_uniq<ResizeableBuffer>(allocator, filter_header.numBytes);
+	transport.read(new_buffer->ptr, filter_header.numBytes);
+	ParquetBloomFilter bloom_filter(std::move(new_buffer));
+	return ApplyBloomFilter(duckdb_filter, bloom_filter);
+}
+
+ParquetBloomFilter::ParquetBloomFilter(idx_t num_entries, double bloom_filter_false_positive_ratio) {
+
+	// aim for hit ratio of 0.01%
+	// see http://tfk.mit.edu/pdf/bloom.pdf
+	double f = bloom_filter_false_positive_ratio;
+	double k = 8.0;
+	double n = LossyNumericCast<double>(num_entries);
+	double m = -k * n / std::log(1 - std::pow(f, 1 / k));
+	auto b = MaxValue<idx_t>(NextPowerOfTwo(LossyNumericCast<idx_t>(m / k)) / 32, 1);
+
+	D_ASSERT(b > 0 && IsPowerOfTwo(b));
+
+	data = make_uniq<ResizeableBuffer>(Allocator::DefaultAllocator(), sizeof(ParquetBloomBlock) * b);
+	data->zero();
+	block_count = data->len / sizeof(ParquetBloomBlock);
+	D_ASSERT(data->len % sizeof(ParquetBloomBlock) == 0);
+}
+
+ParquetBloomFilter::ParquetBloomFilter(unique_ptr<ResizeableBuffer> data_p) {
+	D_ASSERT(data_p->len % sizeof(ParquetBloomBlock) == 0);
+	data = std::move(data_p);
+	block_count = data->len / sizeof(ParquetBloomBlock);
+	D_ASSERT(data->len % sizeof(ParquetBloomBlock) == 0);
+}
+
+void ParquetBloomFilter::FilterInsert(uint64_t x) {
+	auto blocks = reinterpret_cast<ParquetBloomBlock *>(data->ptr);
+	uint64_t i = ((x >> 32) * block_count) >> 32;
+	auto &b = blocks[i];
+	ParquetBloomBlock::BlockInsert(b, x);
+}
+
+bool ParquetBloomFilter::FilterCheck(uint64_t x) {
+	auto blocks = reinterpret_cast<ParquetBloomBlock *>(data->ptr);
+	auto i = ((x >> 32) * block_count) >> 32;
+	return ParquetBloomBlock::BlockCheck(blocks[i], x);
+}
+
+// compiler optimizes this into a single instruction (popcnt)
+static uint8_t PopCnt64(uint64_t n) {
+	uint8_t c = 0;
+	for (; n; ++c) {
+		n &= n - 1;
+	}
+	return c;
+}
+
+double ParquetBloomFilter::OneRatio() {
+	auto bloom_ptr = reinterpret_cast<uint64_t *>(data->ptr);
+	idx_t one_count = 0;
+	for (idx_t b_idx = 0; b_idx < data->len / sizeof(uint64_t); ++b_idx) {
+		one_count += PopCnt64(bloom_ptr[b_idx]);
+	}
+	return LossyNumericCast<double>(one_count) / (LossyNumericCast<double>(data->len) * 8.0);
+}
+
+ResizeableBuffer *ParquetBloomFilter::Get() {
+	return data.get();
+}
+
 } // namespace duckdb

package/src/duckdb/extension/parquet/parquet_timestamp.cpp

@@ -105,6 +105,16 @@ date_t ParquetIntToDate(const int32_t &raw_date) {
 	return date_t(raw_date);
 }
 
+template <typename T>
+static T ParquetWrapTime(const T &raw, const T day) {
+	// Special case 24:00:00
+	if (raw == day) {
+		return raw;
+	}
+	const auto modulus = raw % day;
+	return modulus + (modulus < 0) * day;
+}
+
 dtime_t ParquetIntToTimeMs(const int32_t &raw_millis) {
 	return Time::FromTimeMs(raw_millis);
 }
@@ -118,15 +128,19 @@ dtime_t ParquetIntToTimeNs(const int64_t &raw_nanos) {
 }
 
 dtime_tz_t ParquetIntToTimeMsTZ(const int32_t &raw_millis) {
-	return dtime_tz_t(Time::FromTimeMs(raw_millis), 0);
+	const int32_t MSECS_PER_DAY = Interval::MSECS_PER_SEC * Interval::SECS_PER_DAY;
+	const auto millis = ParquetWrapTime(raw_millis, MSECS_PER_DAY);
+	return dtime_tz_t(Time::FromTimeMs(millis), 0);
 }
 
 dtime_tz_t ParquetIntToTimeTZ(const int64_t &raw_micros) {
-	return dtime_tz_t(dtime_t(raw_micros), 0);
+	const auto micros = ParquetWrapTime(raw_micros, Interval::MICROS_PER_DAY);
+	return dtime_tz_t(dtime_t(micros), 0);
 }
 
 dtime_tz_t ParquetIntToTimeNsTZ(const int64_t &raw_nanos) {
-	return dtime_tz_t(Time::FromTimeNs(raw_nanos), 0);
+	const auto nanos = ParquetWrapTime(raw_nanos, Interval::NANOS_PER_DAY);
+	return dtime_tz_t(Time::FromTimeNs(nanos), 0);
 }
 
 } // namespace duckdb

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -4,6 +4,7 @@
 #include "mbedtls_wrapper.hpp"
 #include "parquet_crypto.hpp"
 #include "parquet_timestamp.hpp"
+#include "resizable_buffer.hpp"
 
 #ifndef DUCKDB_AMALGAMATION
 #include "duckdb/common/file_system.hpp"
@@ -25,16 +26,16 @@ using namespace duckdb_apache::thrift;            // NOLINT
 using namespace duckdb_apache::thrift::protocol;  // NOLINT
 using namespace duckdb_apache::thrift::transport; // NOLINT
 
-using duckdb_parquet::format::CompressionCodec;
-using duckdb_parquet::format::ConvertedType;
-using duckdb_parquet::format::Encoding;
-using duckdb_parquet::format::FieldRepetitionType;
-using duckdb_parquet::format::FileCryptoMetaData;
-using duckdb_parquet::format::FileMetaData;
-using duckdb_parquet::format::PageHeader;
-using duckdb_parquet::format::PageType;
-using ParquetRowGroup = duckdb_parquet::format::RowGroup;
-using duckdb_parquet::format::Type;
+using duckdb_parquet::CompressionCodec;
+using duckdb_parquet::ConvertedType;
+using duckdb_parquet::Encoding;
+using duckdb_parquet::FieldRepetitionType;
+using duckdb_parquet::FileCryptoMetaData;
+using duckdb_parquet::FileMetaData;
+using duckdb_parquet::PageHeader;
+using duckdb_parquet::PageType;
+using ParquetRowGroup = duckdb_parquet::RowGroup;
+using duckdb_parquet::Type;
 
 ChildFieldIDs::ChildFieldIDs() : ids(make_uniq<case_insensitive_map_t<FieldID>>()) {
 }
@@ -167,11 +168,12 @@ Type::type ParquetWriter::DuckDBTypeToParquetType(const LogicalType &duckdb_type
 	throw NotImplementedException("Unimplemented type for Parquet \"%s\"", duckdb_type.ToString());
 }
 
-void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type,
-                                        duckdb_parquet::format::SchemaElement &schema_ele) {
+void ParquetWriter::SetSchemaProperties(const LogicalType &duckdb_type, duckdb_parquet::SchemaElement &schema_ele) {
 	if (duckdb_type.IsJSONType()) {
 		schema_ele.converted_type = ConvertedType::JSON;
 		schema_ele.__isset.converted_type = true;
+		schema_ele.__isset.logicalType = true;
+		schema_ele.logicalType.__set_JSON(duckdb_parquet::JsonType());
 		return;
 	}
 	switch (duckdb_type.id()) {
@@ -317,13 +319,15 @@ void VerifyUniqueNames(const vector<string> &names) {
 ParquetWriter::ParquetWriter(ClientContext &context, FileSystem &fs, string file_name_p, vector<LogicalType> types_p,
                              vector<string> names_p, CompressionCodec::type codec, ChildFieldIDs field_ids_p,
                              const vector<pair<string, string>> &kv_metadata,
-                             shared_ptr<ParquetEncryptionConfig> encryption_config_p,
-                             double dictionary_compression_ratio_threshold_p, optional_idx compression_level_p,
+                             shared_ptr<ParquetEncryptionConfig> encryption_config_p, idx_t dictionary_size_limit_p,
+                             double bloom_filter_false_positive_ratio_p, int64_t compression_level_p,
                              bool debug_use_openssl_p)
     : file_name(std::move(file_name_p)), sql_types(std::move(types_p)), column_names(std::move(names_p)), codec(codec),
       field_ids(std::move(field_ids_p)), encryption_config(std::move(encryption_config_p)),
-      dictionary_compression_ratio_threshold(dictionary_compression_ratio_threshold_p),
+      dictionary_size_limit(dictionary_size_limit_p),
+      bloom_filter_false_positive_ratio(bloom_filter_false_positive_ratio_p), compression_level(compression_level_p),
       debug_use_openssl(debug_use_openssl_p) {
+
 	// initialize the file writer
 	writer = make_uniq<BufferedFileWriter>(fs, file_name.c_str(),
 	                                       FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
@@ -356,31 +360,18 @@ ParquetWriter::ParquetWriter(ClientContext &context, FileSystem &fs, string file
 	file_meta_data.schema.resize(1);
 
 	for (auto &kv_pair : kv_metadata) {
-		duckdb_parquet::format::KeyValue kv;
+		duckdb_parquet::KeyValue kv;
 		kv.__set_key(kv_pair.first);
 		kv.__set_value(kv_pair.second);
 		file_meta_data.key_value_metadata.push_back(kv);
 		file_meta_data.__isset.key_value_metadata = true;
 	}
-	if (compression_level_p.IsValid()) {
-		idx_t level = compression_level_p.GetIndex();
-		switch (codec) {
-		case CompressionCodec::ZSTD:
-			if (level < 1 || level > 22) {
-				throw BinderException("Compression level for ZSTD must be between 1 and 22");
-			}
-			break;
-		default:
-			throw NotImplementedException("Compression level is only supported for the ZSTD compression codec");
-		}
-		compression_level = level;
-	}
 
 	// populate root schema object
 	file_meta_data.schema[0].name = "duckdb_schema";
 	file_meta_data.schema[0].num_children = NumericCast<int32_t>(sql_types.size());
 	file_meta_data.schema[0].__isset.num_children = true;
-	file_meta_data.schema[0].repetition_type = duckdb_parquet::format::FieldRepetitionType::REQUIRED;
+	file_meta_data.schema[0].repetition_type = duckdb_parquet::FieldRepetitionType::REQUIRED;
 	file_meta_data.schema[0].__isset.repetition_type = true;
 
 	auto &unique_names = column_names;
@@ -535,12 +526,41 @@ void ParquetWriter::Flush(ColumnDataCollection &buffer) {
 }
 
 void ParquetWriter::Finalize() {
-	const auto start_offset = writer->GetTotalWritten();
+
+	// dump the bloom filters right before footer, not if stuff is encrypted
+
+	for (auto &bloom_filter_entry : bloom_filters) {
+		D_ASSERT(!encryption_config);
+		// write nonsense bloom filter header
+		duckdb_parquet::BloomFilterHeader filter_header;
+		auto bloom_filter_bytes = bloom_filter_entry.bloom_filter->Get();
+		filter_header.numBytes = NumericCast<int32_t>(bloom_filter_bytes->len);
+		filter_header.algorithm.__set_BLOCK(duckdb_parquet::SplitBlockAlgorithm());
+		filter_header.compression.__set_UNCOMPRESSED(duckdb_parquet::Uncompressed());
+		filter_header.hash.__set_XXHASH(duckdb_parquet::XxHash());
+
+		// set metadata flags
+		auto &column_chunk =
+		    file_meta_data.row_groups[bloom_filter_entry.row_group_idx].columns[bloom_filter_entry.column_idx];
+
+		column_chunk.meta_data.__isset.bloom_filter_offset = true;
+		column_chunk.meta_data.bloom_filter_offset = NumericCast<int64_t>(writer->GetTotalWritten());
+
+		auto bloom_filter_header_size = Write(filter_header);
+		// write actual data
+		WriteData(bloom_filter_bytes->ptr, bloom_filter_bytes->len);
+
+		column_chunk.meta_data.__isset.bloom_filter_length = true;
+		column_chunk.meta_data.bloom_filter_length =
+		    NumericCast<int32_t>(bloom_filter_header_size + bloom_filter_bytes->len);
+	}
+
+	const auto metadata_start_offset = writer->GetTotalWritten();
 	if (encryption_config) {
 		// Crypto metadata is written unencrypted
 		FileCryptoMetaData crypto_metadata;
-		duckdb_parquet::format::AesGcmV1 aes_gcm_v1;
-		duckdb_parquet::format::EncryptionAlgorithm alg;
+		duckdb_parquet::AesGcmV1 aes_gcm_v1;
+		duckdb_parquet::EncryptionAlgorithm alg;
 		alg.__set_AES_GCM_V1(aes_gcm_v1);
 		crypto_metadata.__set_encryption_algorithm(alg);
 		crypto_metadata.write(protocol.get());
@@ -553,7 +573,7 @@ void ParquetWriter::Finalize() {
 
 	Write(file_meta_data);
 
-	writer->Write<uint32_t>(writer->GetTotalWritten() - start_offset);
+	writer->Write<uint32_t>(writer->GetTotalWritten() - metadata_start_offset);
 
 	if (encryption_config) {
 		// encrypted parquet files also end with the string "PARE"
@@ -575,4 +595,15 @@ GeoParquetFileMetadata &ParquetWriter::GetGeoParquetData() {
 	return *geoparquet_data;
 }
 
+void ParquetWriter::BufferBloomFilter(idx_t col_idx, unique_ptr<ParquetBloomFilter> bloom_filter) {
+	if (encryption_config) {
+		return;
+	}
+	ParquetBloomFilterEntry new_entry;
+	new_entry.bloom_filter = std::move(bloom_filter);
+	new_entry.column_idx = col_idx;
+	new_entry.row_group_idx = file_meta_data.row_groups.size();
+	bloom_filters.push_back(std::move(new_entry));
+}
+
 } // namespace duckdb