duckdb 0.8.2-dev77.0 → 0.9.0

package/src/duckdb/src/include/duckdb/function/compression_function.hpp

@@ -14,6 +14,7 @@
 #include "duckdb/common/map.hpp"
 #include "duckdb/storage/storage_info.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/storage/data_pointer.hpp"
 
 namespace duckdb {
 class DatabaseInstance;
@@ -21,6 +22,7 @@ class ColumnData;
 class ColumnDataCheckpointer;
 class ColumnSegment;
 class SegmentStatistics;
+struct ColumnSegmentState;
 
 struct ColumnFetchState;
 struct ColumnScanState;
@@ -62,6 +64,11 @@ struct CompressedSegmentState {
 	virtual ~CompressedSegmentState() {
 	}
 
+	//! Display info for PRAGMA storage_info
+	virtual string GetSegmentInfo() const { // LCOV_EXCL_START
+		return "";
+	} // LCOV_EXCL_STOP
+
 	template <class TARGET>
 	TARGET &Cast() {
 		D_ASSERT(dynamic_cast<TARGET *>(this));
@@ -75,7 +82,7 @@ struct CompressedSegmentState {
 };
 
 struct CompressionAppendState {
-	CompressionAppendState(BufferHandle handle_p) : handle(std::move(handle_p)) {
+	explicit CompressionAppendState(BufferHandle handle_p) : handle(std::move(handle_p)) {
 	}
 	virtual ~CompressionAppendState() {
 	}
@@ -139,13 +146,24 @@ typedef void (*compression_skip_t)(ColumnSegment &segment, ColumnScanState &stat
 //===--------------------------------------------------------------------===//
 // Append (optional)
 //===--------------------------------------------------------------------===//
-typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(ColumnSegment &segment, block_id_t block_id);
+typedef unique_ptr<CompressedSegmentState> (*compression_init_segment_t)(
+    ColumnSegment &segment, block_id_t block_id, optional_ptr<ColumnSegmentState> segment_state);
 typedef unique_ptr<CompressionAppendState> (*compression_init_append_t)(ColumnSegment &segment);
 typedef idx_t (*compression_append_t)(CompressionAppendState &append_state, ColumnSegment &segment,
                                       SegmentStatistics &stats, UnifiedVectorFormat &data, idx_t offset, idx_t count);
 typedef idx_t (*compression_finalize_append_t)(ColumnSegment &segment, SegmentStatistics &stats);
 typedef void (*compression_revert_append_t)(ColumnSegment &segment, idx_t start_row);
 
+//===--------------------------------------------------------------------===//
+// Serialization (optional)
+//===--------------------------------------------------------------------===//
+//! Function prototype for serializing the segment state
+typedef unique_ptr<ColumnSegmentState> (*compression_serialize_state_t)(ColumnSegment &segment);
+//! Function prototype for deserializing the segment state
+typedef unique_ptr<ColumnSegmentState> (*compression_deserialize_state_t)(Deserializer &deserializer);
+//! Function prototype for cleaning up the segment state when the column data is dropped
+typedef void (*compression_cleanup_state_t)(ColumnSegment &segment);
+
 class CompressionFunction {
 public:
 	CompressionFunction(CompressionType type, PhysicalType data_type, compression_init_analyze_t init_analyze,
@@ -157,12 +175,16 @@ public:
 	                    compression_init_segment_t init_segment = nullptr,
 	                    compression_init_append_t init_append = nullptr, compression_append_t append = nullptr,
 	                    compression_finalize_append_t finalize_append = nullptr,
-	                    compression_revert_append_t revert_append = nullptr)
+	                    compression_revert_append_t revert_append = nullptr,
+	                    compression_serialize_state_t serialize_state = nullptr,
+	                    compression_deserialize_state_t deserialize_state = nullptr,
+	                    compression_cleanup_state_t cleanup_state = nullptr)
 	    : type(type), data_type(data_type), init_analyze(init_analyze), analyze(analyze), final_analyze(final_analyze),
 	      init_compression(init_compression), compress(compress), compress_finalize(compress_finalize),
 	      init_scan(init_scan), scan_vector(scan_vector), scan_partial(scan_partial), fetch_row(fetch_row), skip(skip),
 	      init_segment(init_segment), init_append(init_append), append(append), finalize_append(finalize_append),
-	      revert_append(revert_append) {
+	      revert_append(revert_append), serialize_state(serialize_state), deserialize_state(deserialize_state),
+	      cleanup_state(cleanup_state) {
 	}
 
 	//! Compression type
@@ -218,6 +240,16 @@ public:
 	compression_finalize_append_t finalize_append;
 	//! Revert append (optional)
 	compression_revert_append_t revert_append;
+
+	// State serialize functions
+	//! This is only necessary if the segment state has information that must be written to disk in the metadata
+
+	//! Serialize the segment state to the metadata (optional)
+	compression_serialize_state_t serialize_state;
+	//! Deserialize the segment state to the metadata (optional)
+	compression_deserialize_state_t deserialize_state;
+	//! Cleanup the segment state (optional)
+	compression_cleanup_state_t cleanup_state;
 };
 
 //! The set of compression functions

package/src/duckdb/src/include/duckdb/function/copy_function.hpp

@@ -82,10 +82,10 @@ typedef void (*copy_to_combine_t)(ExecutionContext &context, FunctionData &bind_
                                   LocalFunctionData &lstate);
 typedef void (*copy_to_finalize_t)(ClientContext &context, FunctionData &bind_data, GlobalFunctionData &gstate);
 
-typedef void (*copy_to_serialize_t)(FieldWriter &writer, const FunctionData &bind_data, const CopyFunction &function);
+typedef void (*copy_to_serialize_t)(Serializer &serializer, const FunctionData &bind_data,
+                                    const CopyFunction &function);
 
-typedef unique_ptr<FunctionData> (*copy_to_deserialize_t)(ClientContext &context, FieldReader &reader,
-                                                          CopyFunction &function);
+typedef unique_ptr<FunctionData> (*copy_to_deserialize_t)(Deserializer &deserializer, CopyFunction &function);
 
 typedef unique_ptr<FunctionData> (*copy_from_bind_t)(ClientContext &context, CopyInfo &info,
                                                      vector<string> &expected_names,
@@ -99,13 +99,18 @@ typedef void (*copy_flush_batch_t)(ClientContext &context, FunctionData &bind_da
                                    PreparedBatchData &batch);
 typedef idx_t (*copy_desired_batch_size_t)(ClientContext &context, FunctionData &bind_data);
 
+enum class CopyTypeSupport { SUPPORTED, LOSSY, UNSUPPORTED };
+
+typedef CopyTypeSupport (*copy_supports_type_t)(const LogicalType &type);
+
 class CopyFunction : public Function {
 public:
 	explicit CopyFunction(string name)
 	    : Function(name), plan(nullptr), copy_to_bind(nullptr), copy_to_initialize_local(nullptr),
 	      copy_to_initialize_global(nullptr), copy_to_sink(nullptr), copy_to_combine(nullptr),
 	      copy_to_finalize(nullptr), execution_mode(nullptr), prepare_batch(nullptr), flush_batch(nullptr),
-	      desired_batch_size(nullptr), serialize(nullptr), deserialize(nullptr), copy_from_bind(nullptr) {
+	      desired_batch_size(nullptr), serialize(nullptr), deserialize(nullptr), supports_type(nullptr),
+	      copy_from_bind(nullptr) {
 	}
 
 	//! Plan rewrite copy function
@@ -126,6 +131,8 @@ public:
 	copy_to_serialize_t serialize;
 	copy_to_deserialize_t deserialize;
 
+	copy_supports_type_t supports_type;
+
 	copy_from_bind_t copy_from_bind;
 	TableFunction copy_from_function;
 

package/src/duckdb/src/include/duckdb/function/function_serialization.hpp

@@ -8,94 +8,91 @@
 
 #pragma once
 
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
 class FunctionSerializer {
 public:
 	template <class FUNC>
-	static void SerializeBase(FieldWriter &writer, const FUNC &function, FunctionData *bind_info) {
+	static void Serialize(Serializer &serializer, const FUNC &function, optional_ptr<FunctionData> bind_info) {
 		D_ASSERT(!function.name.empty());
-		writer.WriteString(function.name);
-		writer.WriteRegularSerializableList(function.arguments);
-		writer.WriteRegularSerializableList(function.original_arguments);
-		bool serialize = function.serialize;
-		writer.WriteField(serialize);
-		if (serialize) {
-			function.serialize(writer, bind_info, function);
-			// First check if serialize throws a NotImplementedException, in which case it doesn't require a deserialize
-			// function
+		serializer.WriteProperty(500, "name", function.name);
+		serializer.WriteProperty(501, "arguments", function.arguments);
+		serializer.WriteProperty(502, "original_arguments", function.original_arguments);
+		bool has_serialize = function.serialize;
+		serializer.WriteProperty(503, "has_serialize", has_serialize);
+		if (has_serialize) {
+			serializer.WriteObject(504, "function_data",
+			                       [&](Serializer &obj) { function.serialize(obj, bind_info, function); });
 			D_ASSERT(function.deserialize);
 		}
 	}
 
-	template <class FUNC>
-	static void Serialize(FieldWriter &writer, const FUNC &function, const LogicalType &return_type,
-	                      const vector<unique_ptr<Expression>> &children, FunctionData *bind_info) {
-		SerializeBase(writer, function, bind_info);
-		writer.WriteSerializable(return_type);
-		writer.WriteSerializableList(children);
-	}
-
 	template <class FUNC, class CATALOG_ENTRY>
-	static FUNC DeserializeBaseInternal(FieldReader &reader, PlanDeserializationState &state, CatalogType type,
-	                                    unique_ptr<FunctionData> &bind_info, bool &has_deserialize) {
-		auto &context = state.context;
-		auto name = reader.ReadRequired<string>();
-		auto arguments = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
-		// note: original_arguments are optional (can be list of size 0)
-		auto original_arguments = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
-
-		auto &func_catalog = Catalog::GetEntry(context, type, SYSTEM_CATALOG, DEFAULT_SCHEMA, name);
-		if (func_catalog.type != type) {
-			throw InternalException("Cant find catalog entry for function %s", name);
+	static FUNC DeserializeFunction(ClientContext &context, CatalogType catalog_type, const string &name,
+	                                vector<LogicalType> arguments, vector<LogicalType> original_arguments) {
+		auto &func_catalog = Catalog::GetEntry(context, catalog_type, SYSTEM_CATALOG, DEFAULT_SCHEMA, name);
+		if (func_catalog.type != catalog_type) {
+			throw InternalException("DeserializeFunction - cant find catalog entry for function %s", name);
 		}
-
 		auto &functions = func_catalog.Cast<CATALOG_ENTRY>();
 		auto function = functions.functions.GetFunctionByArguments(
-		    state.context, original_arguments.empty() ? arguments : original_arguments);
+		    context, original_arguments.empty() ? arguments : original_arguments);
 		function.arguments = std::move(arguments);
 		function.original_arguments = std::move(original_arguments);
-
-		has_deserialize = reader.ReadRequired<bool>();
-		if (has_deserialize) {
-			if (!function.deserialize) {
-				throw SerializationException("Function requires deserialization but no deserialization function for %s",
-				                             function.name);
-			}
-			bind_info = function.deserialize(state, reader, function);
-		} else {
-			D_ASSERT(!function.serialize);
-			D_ASSERT(!function.deserialize);
-		}
 		return function;
 	}
+
 	template <class FUNC, class CATALOG_ENTRY>
-	static FUNC DeserializeBase(FieldReader &reader, PlanDeserializationState &state, CatalogType type,
-	                            unique_ptr<FunctionData> &bind_info) {
-		bool has_deserialize;
-		return DeserializeBaseInternal<FUNC, CATALOG_ENTRY>(reader, state, type, bind_info, has_deserialize);
+	static pair<FUNC, bool> DeserializeBase(Deserializer &deserializer, CatalogType catalog_type) {
+		auto &context = deserializer.Get<ClientContext &>();
+		auto name = deserializer.ReadProperty<string>(500, "name");
+		auto arguments = deserializer.ReadProperty<vector<LogicalType>>(501, "arguments");
+		auto original_arguments = deserializer.ReadProperty<vector<LogicalType>>(502, "original_arguments");
+		auto function = DeserializeFunction<FUNC, CATALOG_ENTRY>(context, catalog_type, name, std::move(arguments),
+		                                                         std::move(original_arguments));
+		auto has_serialize = deserializer.ReadProperty<bool>(503, "has_serialize");
+		return make_pair(std::move(function), has_serialize);
+	}
+
+	template <class FUNC>
+	static unique_ptr<FunctionData> FunctionDeserialize(Deserializer &deserializer, FUNC &function) {
+		if (!function.deserialize) {
+			throw SerializationException("Function requires deserialization but no deserialization function for %s",
+			                             function.name);
+		}
+		unique_ptr<FunctionData> result;
+		deserializer.ReadObject(504, "function_data",
+		                        [&](Deserializer &obj) { result = function.deserialize(obj, function); });
+		return result;
 	}
 
 	template <class FUNC, class CATALOG_ENTRY>
-	static FUNC Deserialize(FieldReader &reader, ExpressionDeserializationState &state, CatalogType type,
-	                        vector<unique_ptr<Expression>> &children, unique_ptr<FunctionData> &bind_info) {
-		bool has_deserialize;
-		auto function =
-		    DeserializeBaseInternal<FUNC, CATALOG_ENTRY>(reader, state.gstate, type, bind_info, has_deserialize);
-		auto return_type = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
-		children = reader.ReadRequiredSerializableList<Expression>(state.gstate);
+	static pair<FUNC, unique_ptr<FunctionData>> Deserialize(Deserializer &deserializer, CatalogType catalog_type,
+	                                                        vector<unique_ptr<Expression>> &children,
+	                                                        LogicalType return_type) {
+		auto &context = deserializer.Get<ClientContext &>();
+		auto entry = DeserializeBase<FUNC, CATALOG_ENTRY>(deserializer, catalog_type);
+		auto &function = entry.first;
+		auto has_serialize = entry.second;
 
-		// we re-bind the function only if the function did not have an explicit deserialize method
-		auto &context = state.gstate.context;
-		if (!has_deserialize && function.bind) {
-			bind_info = function.bind(context, function, children);
+		unique_ptr<FunctionData> bind_data;
+		if (has_serialize) {
+			bind_data = FunctionDeserialize<FUNC>(deserializer, function);
+		} else if (function.bind) {
+			try {
+				bind_data = function.bind(context, function, children);
+			} catch (Exception &ex) {
+				// FIXME
+				throw SerializationException("Error during bind of function in deserialization: %s", ex.what());
+			}
 		}
-		function.return_type = return_type;
-		return function;
+		function.return_type = std::move(return_type);
+		return make_pair(std::move(function), std::move(bind_data));
 	}
 };
 

package/src/duckdb/src/include/duckdb/function/macro_function.hpp

@@ -45,12 +45,9 @@ public:
 
 	virtual string ToSQL(const string &schema, const string &name) const;
 
-	void Serialize(Serializer &serializer) const;
+	virtual void Serialize(Serializer &serializer) const;
 	static unique_ptr<MacroFunction> Deserialize(Deserializer &deserializer);
 
-protected:
-	virtual void SerializeInternal(FieldWriter &writer) const = 0;
-
 public:
 	template <class TARGET>
 	TARGET &Cast() {

package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp

@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/scalar/compressed_materialization_functions.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/function/built_in_functions.hpp"
+#include "duckdb/function/function_set.hpp"
+
+namespace duckdb {
+
+struct CompressedMaterializationFunctions {
+	//! The types we compress integral types to
+	static const vector<LogicalType> IntegralTypes();
+	//! The types we compress strings to
+	static const vector<LogicalType> StringTypes();
+
+	static unique_ptr<FunctionData> Bind(ClientContext &context, ScalarFunction &bound_function,
+	                                     vector<unique_ptr<Expression>> &arguments);
+};
+
+//! Needed for (de)serialization without binding
+enum class CompressedMaterializationDirection : uint8_t { INVALID = 0, COMPRESS = 1, DECOMPRESS = 2 };
+
+struct CMIntegralCompressFun {
+	static ScalarFunction GetFunction(const LogicalType &input_type, const LogicalType &result_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct CMIntegralDecompressFun {
+	static ScalarFunction GetFunction(const LogicalType &input_type, const LogicalType &result_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct CMStringCompressFun {
+	static ScalarFunction GetFunction(const LogicalType &result_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+struct CMStringDecompressFun {
+	static ScalarFunction GetFunction(const LogicalType &input_type);
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp

@@ -130,7 +130,7 @@ void ListContainsOrPosition(DataChunk &args, Vector &result) {
 		TemplatedContainsOrPosition<int8_t, T, OP, LIST_ACCESSOR>(args, result, true);
 		break;
 	default:
-		throw NotImplementedException("This function has not been implemented for physical type %s",
+		throw NotImplementedException("This function has not been implemented for logical type %s",
 		                              TypeIdToString(physical_type));
 	}
 }

package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp

@@ -12,9 +12,10 @@
 #include "duckdb/function/scalar_function.hpp"
 #include "duckdb/common/map.hpp"
 #include "duckdb/common/unordered_map.hpp"
-#include "duckdb/common/field_writer.hpp"
 #include "duckdb/function/built_in_functions.hpp"
 #include "duckdb/function/scalar/list/contains_or_position.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -58,19 +59,17 @@ struct VariableReturnBindData : public FunctionData {
 		return make_uniq<VariableReturnBindData>(stype);
 	}
 	bool Equals(const FunctionData &other_p) const override {
-		auto &other = (const VariableReturnBindData &)other_p;
+		auto &other = other_p.Cast<VariableReturnBindData>();
 		return stype == other.stype;
 	}
-
-	static void Serialize(FieldWriter &writer, const FunctionData *bind_data_p, const ScalarFunction &function) {
-		D_ASSERT(bind_data_p);
-		auto &info = bind_data_p->Cast<VariableReturnBindData>();
-		writer.WriteSerializable(info.stype);
+	static void Serialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data,
+	                      const ScalarFunction &function) {
+		auto &info = bind_data->Cast<VariableReturnBindData>();
+		serializer.WriteProperty(100, "variable_return_type", info.stype);
 	}
 
-	static unique_ptr<FunctionData> Deserialize(PlanDeserializationState &context, FieldReader &reader,
-	                                            ScalarFunction &bound_function) {
-		auto stype = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
+	static unique_ptr<FunctionData> Deserialize(Deserializer &deserializer, ScalarFunction &bound_function) {
+		auto stype = deserializer.ReadProperty<LogicalType>(100, "variable_return_type");
 		return make_uniq<VariableReturnBindData>(std::move(stype));
 	}
 };
@@ -99,6 +98,11 @@ struct ListPositionFun {
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 
+struct ListResizeFun {
+	static ScalarFunction GetFunction();
+	static void RegisterFunction(BuiltinFunctions &set);
+};
+
 struct StructExtractFun {
 	static ScalarFunction GetFunction();
 	static void RegisterFunction(BuiltinFunctions &set);

package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp

@@ -52,7 +52,8 @@ enum class StrTimeSpecifier : uint8_t {
 	LOCALE_APPROPRIATE_DATE_AND_TIME =
 	    29, // %c - Locale’s appropriate date and time representation. (Mon Sep 30 07:06:05 2013)
 	LOCALE_APPROPRIATE_DATE = 30, // %x - Locale’s appropriate date representation. (09/30/13)
-	LOCALE_APPROPRIATE_TIME = 31  // %X - Locale’s appropriate time representation. (07:06:05)
+	LOCALE_APPROPRIATE_TIME = 31, // %X - Locale’s appropriate time representation. (07:06:05)
+	NANOSECOND_PADDED = 32 // %n - Nanosecond as a decimal number, zero-padded on the left. (000000000 - 999999999)
 };
 
 struct StrTimeFormat {
@@ -121,6 +122,8 @@ protected:
 
 struct StrpTimeFormat : public StrTimeFormat {
 public:
+	StrpTimeFormat();
+
 	//! Type-safe parsing argument
 	struct ParseResult {
 		int32_t data[8]; // year, month, day, hour, min, sec, µs, offset
@@ -140,20 +143,26 @@ public:
 public:
 	DUCKDB_API static ParseResult Parse(const string &format, const string &text);
 
-	DUCKDB_API bool Parse(string_t str, ParseResult &result);
+	DUCKDB_API bool Parse(string_t str, ParseResult &result) const;
 
-	DUCKDB_API bool TryParseDate(string_t str, date_t &result, string &error_message);
-	DUCKDB_API bool TryParseTimestamp(string_t str, timestamp_t &result, string &error_message);
+	DUCKDB_API bool TryParseDate(string_t str, date_t &result, string &error_message) const;
+	DUCKDB_API bool TryParseTimestamp(string_t str, timestamp_t &result, string &error_message) const;
 
 	date_t ParseDate(string_t str);
 	timestamp_t ParseTimestamp(string_t str);
 
+	void Serialize(Serializer &serializer) const;
+	static StrpTimeFormat Deserialize(Deserializer &deserializer);
+
 protected:
 	static string FormatStrpTimeError(const string &input, idx_t position);
 	DUCKDB_API void AddFormatSpecifier(string preceding_literal, StrTimeSpecifier specifier) override;
 	int NumericSpecifierWidth(StrTimeSpecifier specifier);
 	int32_t TryParseCollection(const char *data, idx_t &pos, idx_t size, const string_t collection[],
-	                           idx_t collection_count);
+	                           idx_t collection_count) const;
+
+private:
+	explicit StrpTimeFormat(const string &format_string);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp

@@ -83,12 +83,14 @@ struct LengthFun {
 };
 
 struct LikeFun {
+	static ScalarFunction GetLikeFunction();
 	static void RegisterFunction(BuiltinFunctions &set);
 	DUCKDB_API static bool Glob(const char *s, idx_t slen, const char *pattern, idx_t plen,
 	                            bool allow_question_mark = true);
 };
 
 struct LikeEscapeFun {
+	static ScalarFunction GetLikeEscapeFun();
 	static void RegisterFunction(BuiltinFunctions &set);
 };
 

package/src/duckdb/src/include/duckdb/function/scalar_function.hpp

@@ -14,7 +14,6 @@
 #include "duckdb/common/vector_operations/vector_operations.hpp"
 #include "duckdb/execution/expression_executor_state.hpp"
 #include "duckdb/function/function.hpp"
-#include "duckdb/planner/plan_serialization.hpp"
 #include "duckdb/storage/statistics/base_statistics.hpp"
 #include "duckdb/common/optional_ptr.hpp"
 
@@ -64,10 +63,9 @@ typedef unique_ptr<BaseStatistics> (*function_statistics_t)(ClientContext &conte
 //! Adds the dependencies of this BoundFunctionExpression to the set of dependencies
 typedef void (*dependency_function_t)(BoundFunctionExpression &expr, DependencyList &dependencies);
 
-typedef void (*function_serialize_t)(FieldWriter &writer, const FunctionData *bind_data,
+typedef void (*function_serialize_t)(Serializer &serializer, const optional_ptr<FunctionData> bind_data,
                                      const ScalarFunction &function);
-typedef unique_ptr<FunctionData> (*function_deserialize_t)(PlanDeserializationState &state, FieldReader &reader,
-                                                           ScalarFunction &function);
+typedef unique_ptr<FunctionData> (*function_deserialize_t)(Deserializer &deserializer, ScalarFunction &function);
 
 class ScalarFunction : public BaseScalarFunction {
 public:
@@ -105,9 +103,6 @@ public:
 
 	DUCKDB_API bool Equal(const ScalarFunction &rhs) const;
 
-private:
-	bool CompareScalarFunctionT(const scalar_function_t &other) const;
-
 public:
 	DUCKDB_API static void NopFunction(DataChunk &input, ExpressionState &state, Vector &result);
 

package/src/duckdb/src/include/duckdb/function/scalar_macro_function.hpp

@@ -34,10 +34,8 @@ public:
 
 	string ToSQL(const string &schema, const string &name) const override;
 
-	static unique_ptr<MacroFunction> Deserialize(FieldReader &reader);
-
-protected:
-	void SerializeInternal(FieldWriter &writer) const override;
+	void Serialize(Serializer &serializer) const override;
+	static unique_ptr<MacroFunction> Deserialize(Deserializer &deserializer);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp

@@ -0,0 +1,99 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/function/table/arrow_duck_schema.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb/common/types.hpp"
+#include "duckdb/common/unordered_map.hpp"
+#include "duckdb/common/vector.hpp"
+#include "duckdb/common/unique_ptr.hpp"
+
+namespace duckdb {
+//===--------------------------------------------------------------------===//
+// Arrow Variable Size Types
+//===--------------------------------------------------------------------===//
+enum class ArrowVariableSizeType : uint8_t { FIXED_SIZE = 0, NORMAL = 1, SUPER_SIZE = 2 };
+
+//===--------------------------------------------------------------------===//
+// Arrow Time/Date Types
+//===--------------------------------------------------------------------===//
+enum class ArrowDateTimeType : uint8_t {
+	MILLISECONDS = 0,
+	MICROSECONDS = 1,
+	NANOSECONDS = 2,
+	SECONDS = 3,
+	DAYS = 4,
+	MONTHS = 5,
+	MONTH_DAY_NANO = 6
+};
+
+class ArrowType {
+public:
+	//! From a DuckDB type
+	ArrowType(LogicalType type_p)
+	    : type(std::move(type_p)), size_type(ArrowVariableSizeType::NORMAL),
+	      date_time_precision(ArrowDateTimeType::DAYS) {};
+
+	//! From a DuckDB type + fixed_size
+	ArrowType(LogicalType type_p, idx_t fixed_size_p)
+	    : type(std::move(type_p)), size_type(ArrowVariableSizeType::FIXED_SIZE),
+	      date_time_precision(ArrowDateTimeType::DAYS), fixed_size(fixed_size_p) {};
+
+	//! From a DuckDB type + variable size type
+	ArrowType(LogicalType type_p, ArrowVariableSizeType size_type_p)
+	    : type(std::move(type_p)), size_type(size_type_p), date_time_precision(ArrowDateTimeType::DAYS) {};
+
+	//! From a DuckDB type + datetime type
+	ArrowType(LogicalType type_p, ArrowDateTimeType date_time_precision_p)
+	    : type(std::move(type_p)), size_type(ArrowVariableSizeType::NORMAL),
+	      date_time_precision(date_time_precision_p) {};
+
+	void AddChild(unique_ptr<ArrowType> child);
+
+	void AssignChildren(vector<unique_ptr<ArrowType>> children);
+
+	const LogicalType &GetDuckType() const;
+
+	ArrowVariableSizeType GetSizeType() const;
+
+	idx_t FixedSize() const;
+
+	void SetDictionary(unique_ptr<ArrowType> dictionary);
+
+	ArrowDateTimeType GetDateTimeType() const;
+
+	const ArrowType &GetDictionary() const;
+
+	const ArrowType &operator[](idx_t index) const;
+
+private:
+	LogicalType type;
+	//! If we have a nested type, their children's type.
+	vector<unique_ptr<ArrowType>> children;
+	//! If its a variable size type (e.g., strings, blobs, lists) holds which type it is
+	ArrowVariableSizeType size_type;
+	//! If this is a date/time holds its precision
+	ArrowDateTimeType date_time_precision;
+	//! Only for size types with fixed size
+	idx_t fixed_size = 0;
+	//! Hold the optional type if the array is a dictionary
+	unique_ptr<ArrowType> dictionary_type;
+};
+
+using arrow_column_map_t = unordered_map<idx_t, unique_ptr<ArrowType>>;
+
+struct ArrowTableType {
+public:
+	void AddColumn(idx_t index, unique_ptr<ArrowType> type);
+	const arrow_column_map_t &GetColumns() const;
+
+private:
+	arrow_column_map_t arrow_convert_data;
+};
+
+} // namespace duckdb