duckdb 0.8.2-dev161.0 → 0.8.2-dev1764.0

package/src/duckdb/extension/json/json_functions.cpp

@@ -12,24 +12,23 @@
 
 namespace duckdb {
 
-static void CheckPath(const Value &path_val, string &path, size_t &len) {
-	string error;
-	Value path_str_val;
+using JSONPathType = JSONCommon::JSONPathType;
+
+static JSONPathType CheckPath(const Value &path_val, string &path, size_t &len) {
 	if (path_val.IsNull()) {
 		throw InvalidInputException("JSON path cannot be NULL");
 	}
-	if (!path_val.DefaultTryCastAs(LogicalType::VARCHAR, path_str_val, &error)) {
-		throw InvalidInputException(error);
-	}
+	const auto path_str_val = path_val.DefaultCastAs(LogicalType::VARCHAR);
 	auto path_str = path_str_val.GetValueUnsafe<string_t>();
 	len = path_str.GetSize();
 	auto ptr = path_str.GetData();
 	// Empty strings and invalid $ paths yield an error
 	if (len == 0) {
-		throw InvalidInputException("Empty JSON path");
+		throw BinderException("Empty JSON path");
 	}
+	JSONPathType path_type = JSONPathType::REGULAR;
 	if (*ptr == '$') {
-		JSONCommon::ValidatePathDollar(ptr, len);
+		path_type = JSONCommon::ValidatePath(ptr, len, true);
 	}
 	// Copy over string to the bind data
 	if (*ptr == '/' || *ptr == '$') {
@@ -38,19 +37,20 @@ static void CheckPath(const Value &path_val, string &path, size_t &len) {
 		path = "/" + string(ptr, len);
 		len++;
 	}
+	return path_type;
 }
 
-JSONReadFunctionData::JSONReadFunctionData(bool constant, string path_p, idx_t len)
-    : constant(constant), path(std::move(path_p)), ptr(path.c_str()), len(len) {
+JSONReadFunctionData::JSONReadFunctionData(bool constant, string path_p, idx_t len, JSONPathType path_type_p)
+    : constant(constant), path(std::move(path_p)), path_type(path_type_p), ptr(path.c_str()), len(len) {
 }
 
 unique_ptr<FunctionData> JSONReadFunctionData::Copy() const {
-	return make_uniq<JSONReadFunctionData>(constant, path, len);
+	return make_uniq<JSONReadFunctionData>(constant, path, len, path_type);
 }
 
 bool JSONReadFunctionData::Equals(const FunctionData &other_p) const {
 	auto &other = (const JSONReadFunctionData &)other_p;
-	return constant == other.constant && path == other.path && len == other.len;
+	return constant == other.constant && path == other.path && len == other.len && path_type == other.path_type;
 }
 
 unique_ptr<FunctionData> JSONReadFunctionData::Bind(ClientContext &context, ScalarFunction &bound_function,
@@ -59,12 +59,16 @@ unique_ptr<FunctionData> JSONReadFunctionData::Bind(ClientContext &context, Scal
 	bool constant = false;
 	string path = "";
 	size_t len = 0;
+	JSONPathType path_type = JSONPathType::REGULAR;
 	if (arguments[1]->return_type.id() != LogicalTypeId::SQLNULL && arguments[1]->IsFoldable()) {
 		constant = true;
 		const auto path_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
-		CheckPath(path_val, path, len);
+		path_type = CheckPath(path_val, path, len);
+	}
+	if (path_type == JSONCommon::JSONPathType::WILDCARD) {
+		bound_function.return_type = LogicalType::LIST(bound_function.return_type);
 	}
-	return make_uniq<JSONReadFunctionData>(constant, std::move(path), len);
+	return make_uniq<JSONReadFunctionData>(constant, std::move(path), len, path_type);
 }
 
 JSONReadManyFunctionData::JSONReadManyFunctionData(vector<string> paths_p, vector<size_t> lens_p)
@@ -90,10 +94,7 @@ unique_ptr<FunctionData> JSONReadManyFunctionData::Bind(ClientContext &context,
 		throw ParameterNotResolvedException();
 	}
 	if (!arguments[1]->IsFoldable()) {
-		throw InvalidInputException("List of paths must be constant");
-	}
-	if (arguments[1]->return_type.id() == LogicalTypeId::SQLNULL) {
-		return make_uniq<JSONReadManyFunctionData>(vector<string>(), vector<size_t>());
+		throw BinderException("List of paths must be constant");
 	}
 
 	vector<string> paths;
@@ -102,7 +103,9 @@ unique_ptr<FunctionData> JSONReadManyFunctionData::Bind(ClientContext &context,
 	for (auto &path_val : ListValue::GetChildren(paths_val)) {
 		paths.emplace_back("");
 		lens.push_back(0);
-		CheckPath(path_val, paths.back(), lens.back());
+		if (CheckPath(path_val, paths.back(), lens.back()) == JSONPathType::WILDCARD) {
+			throw BinderException("Cannot have wildcards in JSON path when supplying multiple paths");
+		}
 	}
 
 	return make_uniq<JSONReadManyFunctionData>(std::move(paths), std::move(lens));
@@ -120,11 +123,8 @@ unique_ptr<FunctionLocalState> JSONFunctionLocalState::Init(ExpressionState &sta
 }
 
 unique_ptr<FunctionLocalState> JSONFunctionLocalState::InitCastLocalState(CastLocalStateParameters &parameters) {
-	if (parameters.context) {
-		return make_uniq<JSONFunctionLocalState>(*parameters.context);
-	} else {
-		return make_uniq<JSONFunctionLocalState>(Allocator::DefaultAllocator());
-	}
+	return parameters.context ? make_uniq<JSONFunctionLocalState>(*parameters.context)
+	                          : make_uniq<JSONFunctionLocalState>(Allocator::DefaultAllocator());
 }
 
 JSONFunctionLocalState &JSONFunctionLocalState::ResetAndGet(ExpressionState &state) {

package/src/duckdb/extension/json/json_scan.cpp

@@ -19,7 +19,7 @@ void JSONScanData::Bind(ClientContext &context, TableFunctionBindInput &input) {
 	auto_detect = info.auto_detect;
 
 	for (auto &kv : input.named_parameters) {
-		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options)) {
+		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options, context)) {
 			continue;
 		}
 		auto loption = StringUtil::Lower(kv.first);
@@ -51,10 +51,7 @@ void JSONScanData::Bind(ClientContext &context, TableFunctionBindInput &input) {
 	}
 
 	files = MultiFileReader::GetFileList(context, input.inputs[0], "JSON");
-
-	if (options.file_options.auto_detect_hive_partitioning) {
-		options.file_options.hive_partitioning = MultiFileReaderOptions::AutoDetectHivePartitioning(files);
-	}
+	options.file_options.AutoDetectHivePartitioning(files, context);
 
 	InitializeReaders(context);
 }
@@ -236,7 +233,7 @@ unique_ptr<GlobalTableFunctionState> JSONGlobalTableFunctionState::Init(ClientCo
 	for (auto &reader : gstate.json_readers) {
 		MultiFileReader::FinalizeBind(reader->GetOptions().file_options, gstate.bind_data.reader_bind,
 		                              reader->GetFileName(), gstate.names, dummy_types, bind_data.names,
-		                              input.column_ids, reader->reader_data);
+		                              input.column_ids, reader->reader_data, context);
 	}
 
 	return std::move(result);

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -1,27 +1,25 @@
 #include "column_reader.hpp"
-#include "parquet_timestamp.hpp"
-#include "utf8proc_wrapper.hpp"
-#include "parquet_reader.hpp"
 
 #include "boolean_column_reader.hpp"
-#include "cast_column_reader.hpp"
-#include "row_number_column_reader.hpp"
 #include "callback_column_reader.hpp"
-#include "parquet_decimal_utils.hpp"
+#include "cast_column_reader.hpp"
+#include "duckdb.hpp"
 #include "list_column_reader.hpp"
+#include "miniz_wrapper.hpp"
+#include "parquet_decimal_utils.hpp"
+#include "parquet_reader.hpp"
+#include "parquet_timestamp.hpp"
+#include "row_number_column_reader.hpp"
+#include "snappy.h"
 #include "string_column_reader.hpp"
 #include "struct_column_reader.hpp"
 #include "templated_column_reader.hpp"
-
-#include "snappy.h"
-#include "miniz_wrapper.hpp"
+#include "utf8proc_wrapper.hpp"
 #include "zstd.h"
-#include <iostream>
 
-#include "duckdb.hpp"
 #ifndef DUCKDB_AMALGAMATION
-#include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/bit.hpp"
+#include "duckdb/common/types/blob.hpp"
 #include "duckdb/common/types/chunk_collection.hpp"
 #endif
 
@@ -614,7 +612,7 @@ uint32_t StringColumnReader::VerifyString(const char *str_data, uint32_t str_len
 
 void StringColumnReader::Dictionary(shared_ptr<ResizeableBuffer> data, idx_t num_entries) {
 	dict = std::move(data);
-	dict_strings = duckdb::unique_ptr<string_t[]>(new string_t[num_entries]);
+	dict_strings = unique_ptr<string_t[]>(new string_t[num_entries]);
 	for (idx_t dict_idx = 0; dict_idx < num_entries; dict_idx++) {
 		uint32_t str_len;
 		if (fixed_width_string_length == 0) {
@@ -873,7 +871,7 @@ idx_t ListColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data
 
 ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
                                    idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
-                                   duckdb::unique_ptr<ColumnReader> child_column_reader_p)
+                                   unique_ptr<ColumnReader> child_column_reader_p)
     : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
       child_column_reader(std::move(child_column_reader_p)),
       read_cache(reader.allocator, ListType::GetChildType(Type())), read_vector(read_cache), overflow_child_count(0) {
@@ -889,8 +887,8 @@ ListColumnReader::ListColumnReader(ParquetReader &reader, LogicalType type_p, co
 void ListColumnReader::ApplyPendingSkips(idx_t num_values) {
 	pending_skips -= num_values;
 
-	auto define_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
-	auto repeat_out = duckdb::unique_ptr<uint8_t[]>(new uint8_t[num_values]);
+	auto define_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);
+	auto repeat_out = unique_ptr<uint8_t[]>(new uint8_t[num_values]);
 
 	idx_t remaining = num_values;
 	idx_t read = 0;
@@ -953,7 +951,7 @@ idx_t RowNumberColumnReader::Read(uint64_t num_values, parquet_filter_t &filter,
 //===--------------------------------------------------------------------===//
 // Cast Column Reader
 //===--------------------------------------------------------------------===//
-CastColumnReader::CastColumnReader(duckdb::unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
+CastColumnReader::CastColumnReader(unique_ptr<ColumnReader> child_reader_p, LogicalType target_type_p)
     : ColumnReader(child_reader_p->Reader(), std::move(target_type_p), child_reader_p->Schema(),
                    child_reader_p->FileIdx(), child_reader_p->MaxDefine(), child_reader_p->MaxRepeat()),
       child_reader(std::move(child_reader_p)) {
@@ -1005,7 +1003,7 @@ idx_t CastColumnReader::GroupRowsAvailable() {
 //===--------------------------------------------------------------------===//
 StructColumnReader::StructColumnReader(ParquetReader &reader, LogicalType type_p, const SchemaElement &schema_p,
                                        idx_t schema_idx_p, idx_t max_define_p, idx_t max_repeat_p,
-                                       vector<duckdb::unique_ptr<ColumnReader>> child_readers_p)
+                                       vector<unique_ptr<ColumnReader>> child_readers_p)
     : ColumnReader(reader, std::move(type_p), schema_p, schema_idx_p, max_define_p, max_repeat_p),
       child_readers(std::move(child_readers_p)) {
 	D_ASSERT(type.InternalType() == PhysicalType::STRUCT);
@@ -1155,9 +1153,9 @@ protected:
 };
 
 template <bool FIXED_LENGTH>
-static duckdb::unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
-                                                                    const SchemaElement &schema_p, idx_t file_idx_p,
-                                                                    idx_t max_define, idx_t max_repeat) {
+static unique_ptr<ColumnReader> CreateDecimalReaderInternal(ParquetReader &reader, const LogicalType &type_p,
+                                                            const SchemaElement &schema_p, idx_t file_idx_p,
+                                                            idx_t max_define, idx_t max_repeat) {
 	switch (type_p.InternalType()) {
 	case PhysicalType::INT16:
 		return make_uniq<DecimalColumnReader<int16_t, FIXED_LENGTH>>(reader, type_p, schema_p, file_idx_p, max_define,

package/src/duckdb/extension/parquet/column_writer.cpp

@@ -184,7 +184,7 @@ ColumnWriterState::~ColumnWriterState() {
 }
 
 void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compressed_size, data_ptr_t &compressed_data,
-                                duckdb::unique_ptr<data_t[]> &compressed_buf) {
+                                unique_ptr<data_t[]> &compressed_buf) {
 	switch (writer.GetCodec()) {
 	case CompressionCodec::UNCOMPRESSED:
 		compressed_size = temp_writer.blob.size;
@@ -192,7 +192,7 @@ void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compres
 		break;
 	case CompressionCodec::SNAPPY: {
 		compressed_size = duckdb_snappy::MaxCompressedLength(temp_writer.blob.size);
-		compressed_buf = duckdb::unique_ptr<data_t[]>(new data_t[compressed_size]);
+		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
 		duckdb_snappy::RawCompress(const_char_ptr_cast(temp_writer.blob.data.get()), temp_writer.blob.size,
 		                           char_ptr_cast(compressed_buf.get()), &compressed_size);
 		compressed_data = compressed_buf.get();
@@ -202,7 +202,7 @@ void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compres
 	case CompressionCodec::GZIP: {
 		MiniZStream s;
 		compressed_size = s.MaxCompressedLength(temp_writer.blob.size);
-		compressed_buf = duckdb::unique_ptr<data_t[]>(new data_t[compressed_size]);
+		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
 		s.Compress(const_char_ptr_cast(temp_writer.blob.data.get()), temp_writer.blob.size,
 		           char_ptr_cast(compressed_buf.get()), &compressed_size);
 		compressed_data = compressed_buf.get();
@@ -210,7 +210,7 @@ void ColumnWriter::CompressPage(BufferedSerializer &temp_writer, size_t &compres
 	}
 	case CompressionCodec::ZSTD: {
 		compressed_size = duckdb_zstd::ZSTD_compressBound(temp_writer.blob.size);
-		compressed_buf = duckdb::unique_ptr<data_t[]>(new data_t[compressed_size]);
+		compressed_buf = unique_ptr<data_t[]>(new data_t[compressed_size]);
 		compressed_size = duckdb_zstd::ZSTD_compress((void *)compressed_buf.get(), compressed_size,
 		                                             (const void *)temp_writer.blob.data.get(), temp_writer.blob.size,
 		                                             ZSTD_CLEVEL_DEFAULT);
@@ -303,14 +303,14 @@ struct PageInformation {
 
 struct PageWriteInformation {
 	PageHeader page_header;
-	duckdb::unique_ptr<BufferedSerializer> temp_writer;
-	duckdb::unique_ptr<ColumnWriterPageState> page_state;
+	unique_ptr<BufferedSerializer> temp_writer;
+	unique_ptr<ColumnWriterPageState> page_state;
 	idx_t write_page_idx = 0;
 	idx_t write_count = 0;
 	idx_t max_write_count = 0;
 	size_t compressed_size;
 	data_ptr_t compressed_data;
-	duckdb::unique_ptr<data_t[]> compressed_buf;
+	unique_ptr<data_t[]> compressed_buf;
 };
 
 class BasicColumnWriterState : public ColumnWriterState {
@@ -325,7 +325,7 @@ public:
 	idx_t col_idx;
 	vector<PageInformation> page_info;
 	vector<PageWriteInformation> write_info;
-	duckdb::unique_ptr<ColumnWriterStatistics> stats_state;
+	unique_ptr<ColumnWriterStatistics> stats_state;
 	idx_t current_page = 0;
 };
 
@@ -355,8 +355,7 @@ public:
 	static constexpr const idx_t STRING_LENGTH_SIZE = sizeof(uint32_t);
 
 public:
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override;
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override;
 	void Prepare(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) override;
 	void BeginWrite(ColumnWriterState &state) override;
 	void Write(ColumnWriterState &state, Vector &vector, idx_t count) override;
@@ -372,10 +371,10 @@ protected:
 	void FlushPage(BasicColumnWriterState &state);
 
 	//! Initializes the state used to track statistics during writing. Only used for scalar types.
-	virtual duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState();
+	virtual unique_ptr<ColumnWriterStatistics> InitializeStatsState();
 
 	//! Initialize the writer for a specific page. Only used for scalar types.
-	virtual duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state);
+	virtual unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state);
 
 	//! Flushes the writer for a specific page. Only used for scalar types.
 	virtual void FlushPageState(Serializer &temp_writer, ColumnWriterPageState *state);
@@ -391,16 +390,14 @@ protected:
 	}
 	//! The number of elements in the dictionary
 	virtual idx_t DictionarySize(BasicColumnWriterState &state_p);
-	void WriteDictionary(BasicColumnWriterState &state, duckdb::unique_ptr<BufferedSerializer> temp_writer,
-	                     idx_t row_count);
+	void WriteDictionary(BasicColumnWriterState &state, unique_ptr<BufferedSerializer> temp_writer, idx_t row_count);
 	virtual void FlushDictionary(BasicColumnWriterState &state, ColumnWriterStatistics *stats);
 
 	void SetParquetStatistics(BasicColumnWriterState &state, duckdb_parquet::format::ColumnChunk &column);
 	void RegisterToRowGroup(duckdb_parquet::format::RowGroup &row_group);
 };
 
-unique_ptr<ColumnWriterState> BasicColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-                                                                      Allocator &allocator) {
+unique_ptr<ColumnWriterState> BasicColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) {
 	auto result = make_uniq<BasicColumnWriterState>(row_group, row_group.columns.size());
 	RegisterToRowGroup(row_group);
 	return std::move(result);
@@ -698,8 +695,8 @@ idx_t BasicColumnWriter::DictionarySize(BasicColumnWriterState &state) {
 	throw InternalException("This page does not have a dictionary");
 }
 
-void BasicColumnWriter::WriteDictionary(BasicColumnWriterState &state,
-                                        duckdb::unique_ptr<BufferedSerializer> temp_writer, idx_t row_count) {
+void BasicColumnWriter::WriteDictionary(BasicColumnWriterState &state, unique_ptr<BufferedSerializer> temp_writer,
+                                        idx_t row_count) {
 	D_ASSERT(temp_writer);
 	D_ASSERT(temp_writer->blob.size > 0);
 
@@ -761,7 +758,7 @@ public:
 
 struct BaseParquetOperator {
 	template <class SRC, class TGT>
-	static duckdb::unique_ptr<ColumnWriterStatistics> InitializeStats() {
+	static unique_ptr<ColumnWriterStatistics> InitializeStats() {
 		return make_uniq<NumericStatisticsState<SRC, TGT, BaseParquetOperator>>();
 	}
 
@@ -805,7 +802,7 @@ struct ParquetHugeintOperator {
 	}
 
 	template <class SRC, class TGT>
-	static duckdb::unique_ptr<ColumnWriterStatistics> InitializeStats() {
+	static unique_ptr<ColumnWriterStatistics> InitializeStats() {
 		return make_uniq<ColumnWriterStatistics>();
 	}
 
@@ -837,7 +834,7 @@ public:
 	~StandardColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return OP::template InitializeStats<SRC, TGT>();
 	}
 
@@ -897,7 +894,7 @@ public:
 	~BooleanColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<BooleanStatisticsState>();
 	}
 
@@ -928,7 +925,7 @@ public:
 		}
 	}
 
-	duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
+	unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
 		return make_uniq<BooleanWriterPageState>();
 	}
 
@@ -1022,7 +1019,7 @@ public:
 	~FixedDecimalColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<FixedDecimalStatistics>();
 	}
 
@@ -1196,8 +1193,8 @@ public:
 
 class StringColumnWriterState : public BasicColumnWriterState {
 public:
-	StringColumnWriterState(duckdb_parquet::format::RowGroup &row_group, Allocator &allocator, idx_t col_idx)
-	    : BasicColumnWriterState(row_group, col_idx), dictionary_heap(allocator) {
+	StringColumnWriterState(duckdb_parquet::format::RowGroup &row_group, idx_t col_idx)
+	    : BasicColumnWriterState(row_group, col_idx) {
 	}
 	~StringColumnWriterState() override = default;
 
@@ -1208,7 +1205,6 @@ public:
 
 	// Dictionary and accompanying string heap
 	string_map_t<uint32_t> dictionary;
-	StringHeap dictionary_heap;
 	// key_bit_width== 0 signifies the chunk is written in plain encoding
 	uint32_t key_bit_width;
 
@@ -1243,13 +1239,12 @@ public:
 	~StringColumnWriter() override = default;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<StringStatisticsState>();
 	}
 
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override {
-		auto result = make_uniq<StringColumnWriterState>(row_group, allocator, row_group.columns.size());
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override {
+		auto result = make_uniq<StringColumnWriterState>(row_group, row_group.columns.size());
 		RegisterToRowGroup(row_group);
 		return std::move(result);
 	}
@@ -1279,11 +1274,8 @@ public:
 			if (validity.RowIsValid(vector_index)) {
 				run_length++;
 				const auto &value = strings[vector_index];
-				// If the value did not yet exist in the dictionary we add it to the StringHeap
-				auto found = !value.IsInlined() && state.dictionary.find(value) == state.dictionary.end()
-				                 ? state.dictionary.insert(string_map_t<uint32_t>::value_type(
-				                       state.dictionary_heap.AddBlob(value), new_value_index))
-				                 : state.dictionary.insert(string_map_t<uint32_t>::value_type(value, new_value_index));
+				// Try to insert into the dictionary. If it's already there, we get back the value index
+				auto found = state.dictionary.insert(string_map_t<uint32_t>::value_type(value, new_value_index));
 				state.estimated_plain_size += value.GetSize() + STRING_LENGTH_SIZE;
 				if (found.second) {
 					// string didn't exist yet in the dictionary
@@ -1359,7 +1351,7 @@ public:
 		}
 	}
 
-	duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state_p) override {
+	unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state_p) override {
 		auto &state = state_p.Cast<StringColumnWriterState>();
 		return make_uniq<StringWriterPageState>(state.key_bit_width, state.dictionary);
 	}
@@ -1456,7 +1448,7 @@ public:
 	uint32_t bit_width;
 
 public:
-	duckdb::unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
+	unique_ptr<ColumnWriterStatistics> InitializeStatsState() override {
 		return make_uniq<StringStatisticsState>();
 	}
 
@@ -1499,7 +1491,7 @@ public:
 		}
 	}
 
-	duckdb::unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
+	unique_ptr<ColumnWriterPageState> InitializePageState(BasicColumnWriterState &state) override {
 		return make_uniq<EnumWriterPageState>(bit_width);
 	}
 
@@ -1557,17 +1549,16 @@ public:
 class StructColumnWriter : public ColumnWriter {
 public:
 	StructColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path_p, idx_t max_repeat,
-	                   idx_t max_define, vector<duckdb::unique_ptr<ColumnWriter>> child_writers_p, bool can_have_nulls)
+	                   idx_t max_define, vector<unique_ptr<ColumnWriter>> child_writers_p, bool can_have_nulls)
 	    : ColumnWriter(writer, schema_idx, std::move(schema_path_p), max_repeat, max_define, can_have_nulls),
 	      child_writers(std::move(child_writers_p)) {
 	}
 	~StructColumnWriter() override = default;
 
-	vector<duckdb::unique_ptr<ColumnWriter>> child_writers;
+	vector<unique_ptr<ColumnWriter>> child_writers;
 
 public:
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override;
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override;
 	bool HasAnalyze() override;
 	void Analyze(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) override;
 	void FinalizeAnalyze(ColumnWriterState &state) override;
@@ -1587,16 +1578,15 @@ public:
 
 	duckdb_parquet::format::RowGroup &row_group;
 	idx_t col_idx;
-	vector<duckdb::unique_ptr<ColumnWriterState>> child_states;
+	vector<unique_ptr<ColumnWriterState>> child_states;
 };
 
-unique_ptr<ColumnWriterState> StructColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-                                                                       Allocator &allocator) {
+unique_ptr<ColumnWriterState> StructColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) {
 	auto result = make_uniq<StructColumnWriterState>(row_group, row_group.columns.size());
 
 	result->child_states.reserve(child_writers.size());
 	for (auto &child_writer : child_writers) {
-		result->child_states.push_back(child_writer->InitializeWriteState(row_group, allocator));
+		result->child_states.push_back(child_writer->InitializeWriteState(row_group));
 	}
 	return std::move(result);
 }
@@ -1680,17 +1670,16 @@ void StructColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 class ListColumnWriter : public ColumnWriter {
 public:
 	ListColumnWriter(ParquetWriter &writer, idx_t schema_idx, vector<string> schema_path_p, idx_t max_repeat,
-	                 idx_t max_define, duckdb::unique_ptr<ColumnWriter> child_writer_p, bool can_have_nulls)
+	                 idx_t max_define, unique_ptr<ColumnWriter> child_writer_p, bool can_have_nulls)
 	    : ColumnWriter(writer, schema_idx, std::move(schema_path_p), max_repeat, max_define, can_have_nulls),
 	      child_writer(std::move(child_writer_p)) {
 	}
 	~ListColumnWriter() override = default;
 
-	duckdb::unique_ptr<ColumnWriter> child_writer;
+	unique_ptr<ColumnWriter> child_writer;
 
 public:
-	duckdb::unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-	                                                           Allocator &allocator) override;
+	unique_ptr<ColumnWriterState> InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) override;
 	bool HasAnalyze() override;
 	void Analyze(ColumnWriterState &state, ColumnWriterState *parent, Vector &vector, idx_t count) override;
 	void FinalizeAnalyze(ColumnWriterState &state) override;
@@ -1710,14 +1699,13 @@ public:
 
 	duckdb_parquet::format::RowGroup &row_group;
 	idx_t col_idx;
-	duckdb::unique_ptr<ColumnWriterState> child_state;
+	unique_ptr<ColumnWriterState> child_state;
 	idx_t parent_index = 0;
 };
 
-unique_ptr<ColumnWriterState> ListColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group,
-                                                                     Allocator &allocator) {
+unique_ptr<ColumnWriterState> ListColumnWriter::InitializeWriteState(duckdb_parquet::format::RowGroup &row_group) {
 	auto result = make_uniq<ListColumnWriterState>(row_group, row_group.columns.size());
-	result->child_state = child_writer->InitializeWriteState(row_group, allocator);
+	result->child_state = child_writer->InitializeWriteState(row_group);
 	return std::move(result);
 }
 
@@ -1818,12 +1806,24 @@ void ListColumnWriter::FinalizeWrite(ColumnWriterState &state_p) {
 unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parquet::format::SchemaElement> &schemas,
                                                              ParquetWriter &writer, const LogicalType &type,
                                                              const string &name, vector<string> schema_path,
+                                                             optional_ptr<const ChildFieldIDs> field_ids,
                                                              idx_t max_repeat, idx_t max_define, bool can_have_nulls) {
 	auto null_type = can_have_nulls ? FieldRepetitionType::OPTIONAL : FieldRepetitionType::REQUIRED;
 	if (!can_have_nulls) {
 		max_define--;
 	}
 	idx_t schema_idx = schemas.size();
+
+	optional_ptr<const FieldID> field_id;
+	optional_ptr<const ChildFieldIDs> child_field_ids;
+	if (field_ids) {
+		auto field_id_it = field_ids->ids->find(name);
+		if (field_id_it != field_ids->ids->end()) {
+			field_id = &field_id_it->second;
+			child_field_ids = &field_id->child_field_ids;
+		}
+	}
+
 	if (type.id() == LogicalTypeId::STRUCT) {
 		auto &child_types = StructType::GetChildTypes(type);
 		// set up the schema element for this struct
@@ -1834,15 +1834,19 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		schema_element.__isset.type = false;
 		schema_element.__isset.repetition_type = true;
 		schema_element.name = name;
+		if (field_id && field_id->set) {
+			schema_element.__isset.field_id = true;
+			schema_element.field_id = field_id->field_id;
+		}
 		schemas.push_back(std::move(schema_element));
 		schema_path.push_back(name);
 
 		// construct the child types recursively
-		vector<duckdb::unique_ptr<ColumnWriter>> child_writers;
+		vector<unique_ptr<ColumnWriter>> child_writers;
 		child_writers.reserve(child_types.size());
 		for (auto &child_type : child_types) {
 			child_writers.push_back(CreateWriterRecursive(schemas, writer, child_type.second, child_type.first,
-			                                              schema_path, max_repeat, max_define + 1));
+			                                              schema_path, child_field_ids, max_repeat, max_define + 1));
 		}
 		return make_uniq<StructColumnWriter>(writer, schema_idx, std::move(schema_path), max_repeat, max_define,
 		                                     std::move(child_writers), can_have_nulls);
@@ -1861,6 +1865,10 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		optional_element.__isset.repetition_type = true;
 		optional_element.__isset.converted_type = true;
 		optional_element.name = name;
+		if (field_id && field_id->set) {
+			optional_element.__isset.field_id = true;
+			optional_element.field_id = field_id->field_id;
+		}
 		schemas.push_back(std::move(optional_element));
 		schema_path.push_back(name);
 
@@ -1875,8 +1883,8 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		schemas.push_back(std::move(repeated_element));
 		schema_path.emplace_back("list");
 
-		auto child_writer =
-		    CreateWriterRecursive(schemas, writer, child_type, "element", schema_path, max_repeat + 1, max_define + 2);
+		auto child_writer = CreateWriterRecursive(schemas, writer, child_type, "element", schema_path, child_field_ids,
+		                                          max_repeat + 1, max_define + 2);
 		return make_uniq<ListColumnWriter>(writer, schema_idx, std::move(schema_path), max_repeat, max_define,
 		                                   std::move(child_writer), can_have_nulls);
 	}
@@ -1899,6 +1907,10 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		top_element.__isset.converted_type = true;
 		top_element.__isset.type = false;
 		top_element.name = name;
+		if (field_id && field_id->set) {
+			top_element.__isset.field_id = true;
+			top_element.field_id = field_id->field_id;
+		}
 		schemas.push_back(std::move(top_element));
 		schema_path.push_back(name);
 
@@ -1916,13 +1928,13 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 		// construct the child types recursively
 		vector<LogicalType> kv_types {MapType::KeyType(type), MapType::ValueType(type)};
 		vector<string> kv_names {"key", "value"};
-		vector<duckdb::unique_ptr<ColumnWriter>> child_writers;
+		vector<unique_ptr<ColumnWriter>> child_writers;
 		child_writers.reserve(2);
 		for (idx_t i = 0; i < 2; i++) {
 			// key needs to be marked as REQUIRED
 			bool is_key = i == 0;
 			auto child_writer = CreateWriterRecursive(schemas, writer, kv_types[i], kv_names[i], schema_path,
-			                                          max_repeat + 1, max_define + 2, !is_key);
+			                                          child_field_ids, max_repeat + 1, max_define + 2, !is_key);
 
 			child_writers.push_back(std::move(child_writer));
 		}
@@ -1938,6 +1950,10 @@ unique_ptr<ColumnWriter> ColumnWriter::CreateWriterRecursive(vector<duckdb_parqu
 	schema_element.__isset.type = true;
 	schema_element.__isset.repetition_type = true;
 	schema_element.name = name;
+	if (field_id && field_id->set) {
+		schema_element.__isset.field_id = true;
+		schema_element.field_id = field_id->field_id;
+	}
 	ParquetWriter::SetSchemaProperties(type, schema_element);
 	schemas.push_back(std::move(schema_element));
 	schema_path.push_back(name);