duckdb 0.8.2-dev37.0 → 0.8.2-dev3949.0

package/src/duckdb/extension/json/serialize_json.cpp

@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+// This file is automatically generated by scripts/generate_serialization.py
+// Do not edit this file manually, your changes will be overwritten
+//===----------------------------------------------------------------------===//
+
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+#include "buffered_json_reader.hpp"
+#include "json_transform.hpp"
+#include "json_scan.hpp"
+
+namespace duckdb {
+
+void BufferedJSONReaderOptions::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "format", format);
+	serializer.WriteProperty(101, "record_type", record_type);
+	serializer.WriteProperty(102, "compression", compression);
+	serializer.WriteProperty(103, "file_options", file_options);
+}
+
+BufferedJSONReaderOptions BufferedJSONReaderOptions::FormatDeserialize(FormatDeserializer &deserializer) {
+	BufferedJSONReaderOptions result;
+	deserializer.ReadProperty(100, "format", result.format);
+	deserializer.ReadProperty(101, "record_type", result.record_type);
+	deserializer.ReadProperty(102, "compression", result.compression);
+	deserializer.ReadProperty(103, "file_options", result.file_options);
+	return result;
+}
+
+void JSONScanData::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "json_type", type);
+	serializer.WriteProperty(101, "options", options);
+	serializer.WriteProperty(102, "reader_bind", reader_bind);
+	serializer.WriteProperty(103, "files", files);
+	serializer.WriteProperty(104, "ignore_errors", ignore_errors);
+	serializer.WriteProperty(105, "maximum_object_size", maximum_object_size);
+	serializer.WriteProperty(106, "auto_detect", auto_detect);
+	serializer.WriteProperty(107, "sample_size", sample_size);
+	serializer.WriteProperty(108, "max_depth", max_depth);
+	serializer.WriteProperty(109, "transform_options", transform_options);
+	serializer.WriteProperty(110, "names", names);
+	serializer.WriteProperty(111, "date_format", GetDateFormat());
+	serializer.WriteProperty(112, "timestamp_format", GetTimestampFormat());
+}
+
+unique_ptr<JSONScanData> JSONScanData::FormatDeserialize(FormatDeserializer &deserializer) {
+	auto type = deserializer.ReadProperty<JSONScanType>(100, "json_type");
+	auto options = deserializer.ReadProperty<BufferedJSONReaderOptions>(101, "options");
+	auto reader_bind = deserializer.ReadProperty<MultiFileReaderBindData>(102, "reader_bind");
+	auto files = deserializer.ReadProperty<vector<string>>(103, "files");
+	auto ignore_errors = deserializer.ReadProperty<bool>(104, "ignore_errors");
+	auto maximum_object_size = deserializer.ReadProperty<idx_t>(105, "maximum_object_size");
+	auto auto_detect = deserializer.ReadProperty<bool>(106, "auto_detect");
+	auto sample_size = deserializer.ReadProperty<idx_t>(107, "sample_size");
+	auto max_depth = deserializer.ReadProperty<idx_t>(108, "max_depth");
+	auto transform_options = deserializer.ReadProperty<JSONTransformOptions>(109, "transform_options");
+	auto names = deserializer.ReadProperty<vector<string>>(110, "names");
+	auto date_format = deserializer.ReadProperty<string>(111, "date_format");
+	auto timestamp_format = deserializer.ReadProperty<string>(112, "timestamp_format");
+	auto result = duckdb::unique_ptr<JSONScanData>(new JSONScanData(deserializer.Get<ClientContext &>(), std::move(files), std::move(date_format), std::move(timestamp_format)));
+	result->type = type;
+	result->options = options;
+	result->reader_bind = reader_bind;
+	result->ignore_errors = ignore_errors;
+	result->maximum_object_size = maximum_object_size;
+	result->auto_detect = auto_detect;
+	result->sample_size = sample_size;
+	result->max_depth = max_depth;
+	result->transform_options = transform_options;
+	result->names = std::move(names);
+	return result;
+}
+
+void JSONTransformOptions::FormatSerialize(FormatSerializer &serializer) const {
+	serializer.WriteProperty(100, "strict_cast", strict_cast);
+	serializer.WriteProperty(101, "error_duplicate_key", error_duplicate_key);
+	serializer.WriteProperty(102, "error_missing_key", error_missing_key);
+	serializer.WriteProperty(103, "error_unknown_key", error_unknown_key);
+	serializer.WriteProperty(104, "delay_error", delay_error);
+}
+
+JSONTransformOptions JSONTransformOptions::FormatDeserialize(FormatDeserializer &deserializer) {
+	JSONTransformOptions result;
+	deserializer.ReadProperty(100, "strict_cast", result.strict_cast);
+	deserializer.ReadProperty(101, "error_duplicate_key", result.error_duplicate_key);
+	deserializer.ReadProperty(102, "error_missing_key", result.error_missing_key);
+	deserializer.ReadProperty(103, "error_unknown_key", result.error_unknown_key);
+	deserializer.ReadProperty(104, "delay_error", result.delay_error);
+	return result;
+}
+
+} // namespace duckdb

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -1,27 +1,25 @@
 #include "column_reader.hpp"
-#include "parquet_timestamp.hpp"
-#include "utf8proc_wrapper.hpp"
-#include "parquet_reader.hpp"
 
 #include "boolean_column_reader.hpp"
-#include "cast_column_reader.hpp"
-#include "row_number_column_reader.hpp"
 #include "callback_column_reader.hpp"
-#include "parquet_decimal_utils.hpp"
+#include "cast_column_reader.hpp"
+#include "duckdb.hpp"
 #include "list_column_reader.hpp"
+#include "miniz_wrapper.hpp"
+#include "parquet_decimal_utils.hpp"
+#include "parquet_reader.hpp"
+#include "parquet_timestamp.hpp"
+#include "row_number_column_reader.hpp"
+#include "snappy.h"
 #include "string_column_reader.hpp"
 #include "struct_column_reader.hpp"
 #include "templated_column_reader.hpp"
-
-#include "snappy.h"
-#include "miniz_wrapper.hpp"
+#include "utf8proc_wrapper.hpp"
 #include "zstd.h"
-#include <iostream>
 
-#include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
-#include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/bit.hpp"
+#include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
 #endif
 
@@ -509,13 +507,13 @@ idx_t ColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data_ptr
 			// TODO keep this in the state
 			auto read_buf = make_shared<ResizeableBuffer>();
 
-			switch (type.InternalType()) {
-			case PhysicalType::INT32:
+			switch (schema.type) {
+			case duckdb_parquet::format::Type::INT32:
 				read_buf->resize(reader.allocator, sizeof(int32_t) * (read_now - null_count));
 				dbp_decoder->GetBatch<int32_t>(read_buf->ptr, read_now - null_count);
 
 				break;
-			case PhysicalType::INT64:
+			case duckdb_parquet::format::Type::INT64:
 				read_buf->resize(reader.allocator, sizeof(int64_t) * (read_now - null_count));
 				dbp_decoder->GetBatch<int64_t>(read_buf->ptr, read_now - null_count);
 				break;
@@ -614,7 +612,7 @@ uint32_t StringColumnReader::VerifyString(const char *str_data, uint32_t str_len
 
 void StringColumnReader::Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num_entries) {
 	dict = std::move(data);
-	dict_strings = duckdb::unique_ptr<string_t[]>(new string_t[num_entries]);
+	dict_strings = unique_ptr<string_t[]>(new string_t[num_entries]);
 	for (idx_t dict_idx = 0; dict_idx < num_entries; dict_idx++) {
 		uint32_t str_len;
 		if (fixed_width_string_length == 0) {
@@ -873,7 +871,7 @@ idx_t ListColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data
 
 ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
                                    idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
-                                   duckdb::unique_ptr<ColumnReader> child_column_reader_p)
+                                   unique_ptr<ColumnReader> child_column_reader_p)
     : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
       child_column_reader(std::move(child_column_reader_p)),
       read_cache(reader.allocator, ListType::GetChildType(Type())), read_vector(read_cache), overflow_child_count(0) {
@@ -889,8 +887,8 @@ ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, co
 void ListColumnReader::ApplyPendingSkips(idx_t num_values) {
 	pending_skips -= num_values;
 
-	auto define_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
-	auto repeat_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
+	auto define_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);
+	auto repeat_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);
 
 	idx_t remaining = num_values;
 	idx_t read = 0;
@@ -953,7 +951,7 @@ idx_t RowNumberColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
 //===--------------------------------------------------------------------===//
 // Cast Column Reader
 //===--------------------------------------------------------------------===//
-CastColumnReader::CastColumnReader(duckdb::unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
+CastColumnReader::CastColumnReader(unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
     : ColumnReader(child_reader_p->Reader(), std::move(target_type_p), child_reader_p->Schema(),
                    child_reader_p->FileIdx(), child_reader_p->MaxDefine(), child_reader_p->MaxRepeat()),
       child_reader(std::move(child_reader_p)) {
@@ -1005,7 +1003,7 @@ idx_t CastColumnReader::GroupRowsAvailable() {
 //===--------------------------------------------------------------------===//
 StructColumnReader::StructColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
                                        idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
-                                       vector<duckdb::unique_ptr<ColumnReader>> child_readers_p)
+                                       vector<unique_ptr<ColumnReader>> child_readers_p)
     : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
       child_readers(std::move(child_readers_p)) {
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
@@ -1155,9 +1153,9 @@ protected:
 };
 
 template <bool FIXED_LENGTH>
-static duckdb::unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
-                                                                    const SchemaElement &schema_p, idx_t file_idx_p,
-                                                                    idx_t max_define, idx_t max_repeat) {
+static unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
+                                                            const SchemaElement &schema_p, idx_t file_idx_p,
+                                                            idx_t max_define, idx_t max_repeat) {
 	switch (type_p.InternalType()) {
 	case PhysicalType::INT16:
 		return make_uniq<DecimalColumnReader<int16_t, FIXED_LENGTH>>(reader, type_p, schema_p, file_idx_p, max_define,
@@ -1391,7 +1389,6 @@ unique_ptr<ColumnReader> ColumnReader::CreateReader(ParquetReader &reader, const
 		return make_uniq<CallbackColumnReader<int32_t, date_t, ParquetIntToDate>>(reader, type_p, schema_p, file_idx_p,
 		                                                                          max_define, max_repeat);
 	case LogicalTypeId::TIME:
-	case LogicalTypeId::TIME_TZ:
 		if (schema_p.__isset.logicalType && schema_p.logicalType.__isset.TIME) {
 			if (schema_p.logicalType.TIME.unit.__isset.MILLIS) {
 				return make_uniq<CallbackColumnReader<int32_t, dtime_t, ParquetIntToTimeMs>>(
@@ -1415,6 +1412,21 @@ unique_ptr<ColumnReader> ColumnReader::CreateReader(ParquetReader &reader, const
 				break;
 			}
 		}
+	case LogicalTypeId::TIME_TZ:
+		if (schema_p.__isset.logicalType && schema_p.logicalType.__isset.TIME) {
+			if (schema_p.logicalType.TIME.unit.__isset.MICROS) {
+				return make_uniq<CallbackColumnReader<int64_t, dtime_tz_t, ParquetIntToTimeTZ>>(
+				    reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
+			}
+		} else if (schema_p.__isset.converted_type) {
+			switch (schema_p.converted_type) {
+			case ConvertedType::TIME_MICROS:
+				return make_uniq<CallbackColumnReader<int64_t, dtime_tz_t, ParquetIntToTimeTZ>>(
+				    reader, type_p, schema_p, file_idx_p, max_define, max_repeat);
+			default:
+				break;
+			}
+		}
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::VARCHAR:
 		return make_uniq<StringColumnReader>(reader, type_p, schema_p, file_idx_p, max_define, max_repeat);

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -184,7 +184,7 @@ ColumnWriterState::~ColumnWriterState() {
 }
 
 void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
-                                duckdb::unique_ptr<data_t[]> &compressed_buf) {
+                                unique_ptr<data_t[]> &compressed_buf) {
 	switch (writer.GetCodec()) {
 	case CompressionCodec::UNCOMPRESSED:
 		compressed_size = temp_writer.blob.size;
@@ -192,7 +192,7 @@ void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compres
 		break;
 	case CompressionCodec::SNAPPY: {
 		compressed_size = duckdb_snappy::MaxCompressedLength(temp_writer.blob.size);
-		compressed_buf = duckdb::unique_ptr<data_t[]>(new data_t[compressed_size]);
+		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
 		duckdb_snappy::RawCompress(const_char_ptr_cast(temp_writer.blob.data.get()), temp_writer.blob.size,
 		                           char_ptr_cast(compressed_buf.get()), &compressed_size);
 		compressed_data = compressed_buf.get();
@@ -202,7 +202,7 @@ void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compres
 	case CompressionCodec::GZIP: {
 		MiniZStream s;
 		compressed_size = s.MaxCompressedLength(temp_writer.blob.size);
-		compressed_buf = duckdb::unique_ptr<data_t[]>(new data_t[compressed_size]);
+		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
 		s.Compress(const_char_ptr_cast(temp_writer.blob.data.get()), temp_writer.blob.size,
 		           char_ptr_cast(compressed_buf.get()), &compressed_size);
 		compressed_data = compressed_buf.get();
@@ -210,7 +210,7 @@ void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compres
 	}
 	case CompressionCodec::ZSTD: {
 		compressed_size = duckdb_zstd::ZSTD_compressBound(temp_writer.blob.size);
-		compressed_buf = duckdb::unique_ptr<data_t[]>(new data_t[compressed_size]);
+		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
 		compressed_size = duckdb_zstd::ZSTD_compress((void *)compressed_buf.get(), compressed_size,
 		                                             (const void *)temp_writer.blob.data.get(), temp_writer.blob.size,
 		                                             ZSTD_CLEVEL_DEFAULT);
@@ -303,14 +303,14 @@ struct PageInformation {
 
 struct PageWriteInformation {
 	PageHeader page_header;
-	duckdb::unique_ptr<BufferedSerializer> temp_writer;
-	duckdb::unique_ptr<ColumnWriterPageState> page_state;
+	unique_ptr<BufferedSerializer> temp_writer;
+	unique_ptr<ColumnWriterPageState> page_state;
 	idx_t write_page_idx = 0;
 	idx_t write_count = 0;
 	idx_t max_write_count = 0;
 	size_t compressed_size;
 	data_ptr_t compressed_data;
-	duckdb::unique_ptr<data_t[]> compressed_buf;
+	unique_ptr<data_t[]> compressed_buf;
 };
 
 class BasicColumnWriterState : public ColumnWriterState {
@@ -325,7 +325,7 @@ public:
 	idx_t col_idx;
 	vector<PageInformation> page_info;
 	vector<PageWriteInformation> write_info;
-	duckdb::unique_ptr<ColumnWriterStatistics> stats_state;
+	unique_ptr<ColumnWriterStatistics> stats_state;
 	idx_t current_page = 0;
 };
 
@@ -355,8 +355,7 @@ public:
 	static constexpr const idx_t STRING_LENGTH_SIZE = sizeof(uint32_t);
 
 public:
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override;
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override;
 	void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) override;
 	void BeginWrite(ColumnWriterState &state) override;
 	void Write(ColumnWriterState &state, Vector &vector, idx_t count) override;
@@ -372,10 +371,10 @@ protected:
 	void FlushPage(BasicColumnWriterState &state);
 
 	//! Initializes the state used to track statistics during writing. Only used for scalar types.
-	virtual duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState();
+	virtual unique_ptr<ColumnWriterStatistics> InitializeStatsState();
 
 	//! Initialize the writer for a specific page. Only used for scalar types.
-	virtual duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state);
+	virtual unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state);
 
 	//! Flushes the writer for a specific page. Only used for scalar types.
 	virtual void FlushPageState(Serializer &temp_writer, ColumnWriterPageState *state);
@@ -391,16 +390,14 @@ protected:
 	}
 	//! The number of elements in the dictionary
 	virtual idx_t DictionarySize(BasicColumnWriterState &state_p);
-	void WriteDictionary(BasicColumnWriterState &state, duckdb::unique_ptr<BufferedSerializer> temp_writer,
-	                     idx_t row_count);
+	void WriteDictionary(BasicColumnWriterState &state, unique_ptr<BufferedSerializer> temp_writer, idx_t row_count);
 	virtual void FlushDictionary(BasicColumnWriterState &state, ColumnWriterStatistics *stats);
 
 	void SetParquetStatistics(BasicColumnWriterState &state, duckdb_parquet::format::ColumnChunk &column);
 	void RegisterToRowGroup(duckdb_parquet::format::RowGroup &row_group);
 };
 
-unique_ptr<ColumnWriterState> BasicColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-                                                                      Allocator &allocator) {
+unique_ptr<ColumnWriterState> BasicColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) {
 	auto result = make_uniq<BasicColumnWriterState>(row_group, row_group.columns.size());
 	RegisterToRowGroup(row_group);
 	return std::move(result);
@@ -698,8 +695,8 @@ idx_t BasicColumnWriter::DictionarySize(BasicColumnWriterState &state) {
 	throw InternalException("This page does not have a dictionary");
 }
 
-void BasicColumnWriter::WriteDictionary(BasicColumnWriterState &state,
-                                        duckdb::unique_ptr<BufferedSerializer> temp_writer, idx_t row_count) {
+void BasicColumnWriter::WriteDictionary(BasicColumnWriterState &state, unique_ptr<BufferedSerializer> temp_writer,
+                                        idx_t row_count) {
 	D_ASSERT(temp_writer);
 	D_ASSERT(temp_writer->blob.size > 0);
 
@@ -761,7 +758,7 @@ public:
 
 struct BaseParquetOperator {
 	template <class SRC, class TGT>
-	static duckdb::unique_ptr<ColumnWriterStatistics> InitializeStats() {
+	static unique_ptr<ColumnWriterStatistics> InitializeStats() {
 		return make_uniq<NumericStatisticsState<SRC, TGT, BaseParquetOperator>>();
 	}
 
@@ -805,7 +802,7 @@ struct ParquetHugeintOperator {
 	}
 
 	template <class SRC, class TGT>
-	static duckdb::unique_ptr<ColumnWriterStatistics> InitializeStats() {
+	static unique_ptr<ColumnWriterStatistics> InitializeStats() {
 		return make_uniq<ColumnWriterStatistics>();
 	}
 
@@ -837,7 +834,7 @@ public:
 	~StandardColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return OP::template InitializeStats<SRC, TGT>();
 	}
 
@@ -897,7 +894,7 @@ public:
 	~BooleanColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<BooleanStatisticsState>();
 	}
 
@@ -928,7 +925,7 @@ public:
 		}
 	}
 
-	duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
+	unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
 		return make_uniq<BooleanWriterPageState>();
 	}
 
@@ -1022,7 +1019,7 @@ public:
 	~FixedDecimalColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<FixedDecimalStatistics>();
 	}
 
@@ -1196,8 +1193,8 @@ public:
 
 class StringColumnWriterState : public BasicColumnWriterState {
 public:
-	StringColumnWriterState(duckdb_parquet::format::RowGroup &row_group, Allocator &allocator, idx_t col_idx)
-	    : BasicColumnWriterState(row_group, col_idx), dictionary_heap(allocator) {
+	StringColumnWriterState(duckdb_parquet::format::RowGroup &row_group, idx_t col_idx)
+	    : BasicColumnWriterState(row_group, col_idx) {
 	}
 	~StringColumnWriterState() override = default;
 
@@ -1208,7 +1205,6 @@ public:
 
 	// Dictionary and accompanying string heap
 	string_map_t<uint32_t> dictionary;
-	StringHeap dictionary_heap;
 	// key_bit_width== 0 signifies the chunk is written in plain encoding
 	uint32_t key_bit_width;
 
@@ -1243,13 +1239,12 @@ public:
 	~StringColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<StringStatisticsState>();
 	}
 
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override {
-		auto result = make_uniq<StringColumnWriterState>(row_group, allocator, row_group.columns.size());
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override {
+		auto result = make_uniq<StringColumnWriterState>(row_group, row_group.columns.size());
 		RegisterToRowGroup(row_group);
 		return std::move(result);
 	}
@@ -1279,11 +1274,8 @@ public:
 			if (validity.RowIsValid(vector_index)) {
 				run_length++;
 				const auto &value = strings[vector_index];
-				// If the value did not yet exist in the dictionary we add it to the StringHeap
-				auto found = !value.IsInlined() && state.dictionary.find(value) == state.dictionary.end()
-				                 ? state.dictionary.insert(string_map_t<uint32_t>::value_type(
-				                       state.dictionary_heap.AddBlob(value), new_value_index))
-				                 : state.dictionary.insert(string_map_t<uint32_t>::value_type(value, new_value_index));
+				// Try to insert into the dictionary. If it's already there, we get back the value index
+				auto found = state.dictionary.insert(string_map_t<uint32_t>::value_type(value, new_value_index));
 				state.estimated_plain_size += value.GetSize() + STRING_LENGTH_SIZE;
 				if (found.second) {
 					// string didn't exist yet in the dictionary
@@ -1359,7 +1351,7 @@ public:
 		}
 	}
 
-	duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state_p) override {
+	unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state_p) override {
 		auto &state = state_p.Cast<StringColumnWriterState>();
 		return make_uniq<StringWriterPageState>(state.key_bit_width, state.dictionary);
 	}
@@ -1456,7 +1448,7 @@ public:
 	uint32_t bit_width;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<StringStatisticsState>();
 	}
 
@@ -1499,7 +1491,7 @@ public:
 		}
 	}
 
-	duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
+	unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
 		return make_uniq<EnumWriterPageState>(bit_width);
 	}
 
@@ -1557,17 +1549,16 @@ public:
 class StructColumnWriter : public ColumnWriter {
 public:
 	StructColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path_p, idx_t max_repeat,
-	                   idx_t max_define, vector<duckdb::unique_ptr<ColumnWriter>> child_writers_p, bool can_have_nulls)
+	                   idx_t max_define, vector<unique_ptr<ColumnWriter>> child_writers_p, bool can_have_nulls)
 	    : ColumnWriter(writer, schema_idx, std::move(schema_path_p), max_repeat, max_define, can_have_nulls),
 	      child_writers(std::move(child_writers_p)) {
 	}
 	~StructColumnWriter() override = default;
 
-	vector<duckdb::unique_ptr<ColumnWriter>> child_writers;
+	vector<unique_ptr<ColumnWriter>> child_writers;
 
 public:
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override;
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override;
 	bool HasAnalyze() override;
 	void Analyze(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) override;
 	void FinalizeAnalyze(ColumnWriterState &state) override;
@@ -1587,16 +1578,15 @@ public:
 
 	duckdb_parquet::format::RowGroup &row_group;
 	idx_t col_idx;
-	vector<duckdb::unique_ptr<ColumnWriterState>> child_states;
+	vector<unique_ptr<ColumnWriterState>> child_states;
 };
 
-unique_ptr<ColumnWriterState> StructColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-                                                                       Allocator &allocator) {
+unique_ptr<ColumnWriterState> StructColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) {
 	auto result = make_uniq<StructColumnWriterState>(row_group, row_group.columns.size());
 
 	result->child_states.reserve(child_writers.size());
 	for (auto &child_writer : child_writers) {
-		result->child_states.push_back(child_writer->InitializeWriteState(row_group, allocator));
+		result->child_states.push_back(child_writer->InitializeWriteState(row_group));
 	}
 	return std::move(result);
 }
@@ -1680,17 +1670,16 @@ void StructColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 class ListColumnWriter : public ColumnWriter {
 public:
 	ListColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path_p, idx_t max_repeat,
-	                 idx_t max_define, duckdb::unique_ptr<ColumnWriter> child_writer_p, bool can_have_nulls)
+	                 idx_t max_define, unique_ptr<ColumnWriter> child_writer_p, bool can_have_nulls)
 	    : ColumnWriter(writer, schema_idx, std::move(schema_path_p), max_repeat, max_define, can_have_nulls),
 	      child_writer(std::move(child_writer_p)) {
 	}
 	~ListColumnWriter() override = default;
 
-	duckdb::unique_ptr<ColumnWriter> child_writer;
+	unique_ptr<ColumnWriter> child_writer;
 
 public:
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override;
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override;
 	bool HasAnalyze() override;
 	void Analyze(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) override;
 	void FinalizeAnalyze(ColumnWriterState &state) override;
@@ -1710,14 +1699,13 @@ public:
 
 	duckdb_parquet::format::RowGroup &row_group;
 	idx_t col_idx;
-	duckdb::unique_ptr<ColumnWriterState> child_state;
+	unique_ptr<ColumnWriterState> child_state;
 	idx_t parent_index = 0;
 };
 
-unique_ptr<ColumnWriterState> ListColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-                                                                     Allocator &allocator) {
+unique_ptr<ColumnWriterState> ListColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) {
 	auto result = make_uniq<ListColumnWriterState>(row_group, row_group.columns.size());
-	result->child_state = child_writer->InitializeWriteState(row_group, allocator);
+	result->child_state = child_writer->InitializeWriteState(row_group);
 	return std::move(result);
 }
 
@@ -1818,12 +1806,24 @@ void ListColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parquet::format::SchemaElement> &schemas,
                                                              ParquetWriter &writer, const LogicalType &type,
                                                              const string &name, vector<string> schema_path,
+                                                             optional_ptr<const ChildFieldIDs> field_ids,
                                                              idx_t max_repeat, idx_t max_define, bool can_have_nulls) {
 	auto null_type = can_have_nulls ? FieldRepetitionType::OPTIONAL : FieldRepetitionType::REQUIRED;
 	if (!can_have_nulls) {
 		max_define--;
 	}
 	idx_t schema_idx = schemas.size();
+
+	optional_ptr<const FieldID> field_id;
+	optional_ptr<const ChildFieldIDs> child_field_ids;
+	if (field_ids) {
+		auto field_id_it = field_ids->ids->find(name);
+		if (field_id_it != field_ids->ids->end()) {
+			field_id = &field_id_it->second;
+			child_field_ids = &field_id->child_field_ids;
+		}
+	}
+
 	if (type.id() == LogicalTypeId::STRUCT) {
 		auto &child_types = StructType::GetChildTypes(type);
 		// set up the schema element for this struct
@@ -1834,15 +1834,19 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		schema_element.__isset.type = false;
 		schema_element.__isset.repetition_type = true;
 		schema_element.name = name;
+		if (field_id && field_id->set) {
+			schema_element.__isset.field_id = true;
+			schema_element.field_id = field_id->field_id;
+		}
 		schemas.push_back(std::move(schema_element));
 		schema_path.push_back(name);
 
 		// construct the child types recursively
-		vector<duckdb::unique_ptr<ColumnWriter>> child_writers;
+		vector<unique_ptr<ColumnWriter>> child_writers;
 		child_writers.reserve(child_types.size());
 		for (auto &child_type : child_types) {
 			child_writers.push_back(CreateWriterRecursive(schemas, writer, child_type.second, child_type.first,
-			                                              schema_path, max_repeat, max_define + 1));
+			                                              schema_path, child_field_ids, max_repeat, max_define + 1));
 		}
 		return make_uniq<StructColumnWriter>(writer, schema_idx, std::move(schema_path), max_repeat, max_define,
 		                                     std::move(child_writers), can_have_nulls);
@@ -1861,6 +1865,10 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		optional_element.__isset.repetition_type = true;
 		optional_element.__isset.converted_type = true;
 		optional_element.name = name;
+		if (field_id && field_id->set) {
+			optional_element.__isset.field_id = true;
+			optional_element.field_id = field_id->field_id;
+		}
 		schemas.push_back(std::move(optional_element));
 		schema_path.push_back(name);
 
@@ -1875,8 +1883,8 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		schemas.push_back(std::move(repeated_element));
 		schema_path.emplace_back("list");
 
-		auto child_writer =
-		    CreateWriterRecursive(schemas, writer, child_type, "element", schema_path, max_repeat + 1, max_define + 2);
+		auto child_writer = CreateWriterRecursive(schemas, writer, child_type, "element", schema_path, child_field_ids,
+		                                          max_repeat + 1, max_define + 2);
 		return make_uniq<ListColumnWriter>(writer, schema_idx, std::move(schema_path), max_repeat, max_define,
 		                                   std::move(child_writer), can_have_nulls);
 	}
@@ -1899,6 +1907,10 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		top_element.__isset.converted_type = true;
 		top_element.__isset.type = false;
 		top_element.name = name;
+		if (field_id && field_id->set) {
+			top_element.__isset.field_id = true;
+			top_element.field_id = field_id->field_id;
+		}
 		schemas.push_back(std::move(top_element));
 		schema_path.push_back(name);
 
@@ -1916,13 +1928,13 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		// construct the child types recursively
 		vector<LogicalType> kv_types {MapType::KeyType(type), MapType::ValueType(type)};
 		vector<string> kv_names {"key", "value"};
-		vector<duckdb::unique_ptr<ColumnWriter>> child_writers;
+		vector<unique_ptr<ColumnWriter>> child_writers;
 		child_writers.reserve(2);
 		for (idx_t i = 0; i < 2; i++) {
 			// key needs to be marked as REQUIRED
 			bool is_key = i == 0;
 			auto child_writer = CreateWriterRecursive(schemas, writer, kv_types[i], kv_names[i], schema_path,
-			                                          max_repeat + 1, max_define + 2, !is_key);
+			                                          child_field_ids, max_repeat + 1, max_define + 2, !is_key);
 
 			child_writers.push_back(std::move(child_writer));
 		}
@@ -1938,6 +1950,10 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 	schema_element.__isset.type = true;
 	schema_element.__isset.repetition_type = true;
 	schema_element.name = name;
+	if (field_id && field_id->set) {
+		schema_element.__isset.field_id = true;
+		schema_element.field_id = field_id->field_id;
+	}
 	ParquetWriter::SetSchemaProperties(type, schema_element);
 	schemas.push_back(std::move(schema_element));
 	schema_path.push_back(name);