duckdb 0.8.2-dev2068.0 → 0.8.2-dev2133.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/binding.gyp +4 -0
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/json/buffered_json_reader.cpp +2 -0
  4. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +5 -19
  5. package/src/duckdb/extension/json/include/json_enums.hpp +60 -0
  6. package/src/duckdb/extension/json/include/json_scan.hpp +14 -10
  7. package/src/duckdb/extension/json/include/json_transform.hpp +3 -0
  8. package/src/duckdb/extension/json/json_enums.cpp +105 -0
  9. package/src/duckdb/extension/json/json_functions/json_transform.cpp +2 -0
  10. package/src/duckdb/extension/json/json_scan.cpp +44 -0
  11. package/src/duckdb/extension/json/serialize_json.cpp +92 -0
  12. package/src/duckdb/extension/parquet/include/parquet_reader.hpp +3 -0
  13. package/src/duckdb/extension/parquet/parquet_extension.cpp +23 -0
  14. package/src/duckdb/extension/parquet/parquet_reader.cpp +3 -0
  15. package/src/duckdb/extension/parquet/serialize_parquet.cpp +26 -0
  16. package/src/duckdb/src/common/arrow/appender/bool_data.cpp +44 -0
  17. package/src/duckdb/src/common/arrow/appender/list_data.cpp +78 -0
  18. package/src/duckdb/src/common/arrow/appender/map_data.cpp +86 -0
  19. package/src/duckdb/src/common/arrow/appender/struct_data.cpp +45 -0
  20. package/src/duckdb/src/common/arrow/appender/union_data.cpp +70 -0
  21. package/src/duckdb/src/common/arrow/arrow_appender.cpp +89 -727
  22. package/src/duckdb/src/common/arrow/arrow_wrapper.cpp +2 -1
  23. package/src/duckdb/src/common/local_file_system.cpp +17 -14
  24. package/src/duckdb/src/common/serializer/format_serializer.cpp +15 -0
  25. package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +26 -0
  26. package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +47 -0
  27. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +28 -0
  28. package/src/duckdb/src/core_functions/scalar/date/strftime.cpp +10 -0
  29. package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +22 -3
  30. package/src/duckdb/src/function/aggregate/distributive/count.cpp +0 -11
  31. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +1 -9
  32. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +27 -0
  33. package/src/duckdb/src/function/scalar_function.cpp +2 -1
  34. package/src/duckdb/src/function/table/read_csv.cpp +18 -0
  35. package/src/duckdb/src/function/table/table_scan.cpp +35 -0
  36. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  37. package/src/duckdb/src/function/table_function.cpp +4 -3
  38. package/src/duckdb/src/include/duckdb/common/arrow/appender/append_data.hpp +109 -0
  39. package/src/duckdb/src/include/duckdb/common/arrow/appender/bool_data.hpp +15 -0
  40. package/src/duckdb/src/include/duckdb/common/arrow/appender/enum_data.hpp +69 -0
  41. package/src/duckdb/src/include/duckdb/common/arrow/appender/list.hpp +8 -0
  42. package/src/duckdb/src/include/duckdb/common/arrow/appender/list_data.hpp +18 -0
  43. package/src/duckdb/src/include/duckdb/common/arrow/appender/map_data.hpp +18 -0
  44. package/src/duckdb/src/include/duckdb/common/arrow/appender/scalar_data.hpp +88 -0
  45. package/src/duckdb/src/include/duckdb/common/arrow/appender/struct_data.hpp +18 -0
  46. package/src/duckdb/src/include/duckdb/common/arrow/appender/union_data.hpp +21 -0
  47. package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp +105 -0
  48. package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp +5 -0
  49. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +5 -1
  50. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +2 -0
  51. package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +32 -0
  52. package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +45 -15
  53. package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +10 -0
  54. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +2 -0
  55. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +11 -2
  56. package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +81 -0
  57. package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp +8 -0
  58. package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +8 -0
  59. package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp +7 -0
  60. package/src/duckdb/src/include/duckdb/function/table_function.hpp +8 -0
  61. package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp +3 -0
  62. package/src/duckdb/src/include/duckdb/planner/expression/bound_function_expression.hpp +4 -0
  63. package/src/duckdb/src/include/duckdb/planner/expression/bound_window_expression.hpp +3 -0
  64. package/src/duckdb/src/include/duckdb/planner/filter/conjunction_filter.hpp +4 -0
  65. package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +2 -0
  66. package/src/duckdb/src/include/duckdb/planner/filter/null_filter.hpp +4 -0
  67. package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +2 -0
  68. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +7 -1
  69. package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +7 -1
  70. package/src/duckdb/src/main/extension/extension_helper.cpp +13 -0
  71. package/src/duckdb/src/parallel/executor.cpp +1 -1
  72. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +23 -0
  73. package/src/duckdb/src/planner/expression/bound_function_expression.cpp +22 -0
  74. package/src/duckdb/src/planner/expression/bound_window_expression.cpp +47 -0
  75. package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp +8 -0
  76. package/src/duckdb/src/planner/operator/logical_get.cpp +69 -0
  77. package/src/duckdb/src/storage/serialization/serialize_expression.cpp +9 -0
  78. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +6 -0
  79. package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +190 -0
  80. package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +97 -0
  81. package/src/duckdb/ub_src_common_arrow_appender.cpp +10 -0
  82. package/src/duckdb/ub_src_common_serializer.cpp +2 -0
  83. package/src/duckdb/ub_src_storage_serialization.cpp +2 -0
@@ -28,25 +28,11 @@ protected:
28
28
  public:
29
29
  // Serialize a value
30
30
  template <class T>
31
- typename std::enable_if<!std::is_enum<T>::value, void>::type WriteProperty(const char *tag, const T &value) {
31
+ void WriteProperty(const char *tag, const T &value) {
32
32
  SetTag(tag);
33
33
  WriteValue(value);
34
34
  }
35
35
 
36
- // Serialize an enum
37
- template <class T>
38
- typename std::enable_if<std::is_enum<T>::value, void>::type WriteProperty(const char *tag, T value) {
39
- SetTag(tag);
40
- if (serialize_enum_as_string) {
41
- // Use the enum serializer to lookup tostring function
42
- auto str = EnumUtil::ToChars(value);
43
- WriteValue(str);
44
- } else {
45
- // Use the underlying type
46
- WriteValue(static_cast<typename std::underlying_type<T>::type>(value));
47
- }
48
- }
49
-
50
36
  // Optional pointer
51
37
  template <class POINTER>
52
38
  void WriteOptionalProperty(const char *tag, POINTER &&ptr) {
@@ -67,7 +53,29 @@ public:
67
53
  WriteDataPtr(ptr, count);
68
54
  }
69
55
 
56
+ // Manually begin an object - should be followed by EndObject
57
+ void BeginObject(const char *tag) {
58
+ SetTag(tag);
59
+ OnObjectBegin();
60
+ }
61
+
62
+ void EndObject() {
63
+ OnObjectEnd();
64
+ }
65
+
70
66
  protected:
67
+ template <typename T>
68
+ typename std::enable_if<std::is_enum<T>::value, void>::type WriteValue(const T value) {
69
+ if (serialize_enum_as_string) {
70
+ // Use the enum serializer to lookup tostring function
71
+ auto str = EnumUtil::ToChars(value);
72
+ WriteValue(str);
73
+ } else {
74
+ // Use the underlying type
75
+ WriteValue(static_cast<typename std::underlying_type<T>::type>(value));
76
+ }
77
+ }
78
+
71
79
  // Unique Pointer Ref
72
80
  template <typename T>
73
81
  void WriteValue(const unique_ptr<T> &ptr) {
@@ -160,6 +168,24 @@ protected:
160
168
  OnMapEnd(count);
161
169
  }
162
170
 
171
+ // Map
172
+ template <class K, class V, class HASH, class CMP>
173
+ void WriteValue(const duckdb::map<K, V, HASH, CMP> &map) {
174
+ auto count = map.size();
175
+ OnMapBegin(count);
176
+ for (auto &item : map) {
177
+ OnMapEntryBegin();
178
+ OnMapKeyBegin();
179
+ WriteValue(item.first);
180
+ OnMapKeyEnd();
181
+ OnMapValueBegin();
182
+ WriteValue(item.second);
183
+ OnMapValueEnd();
184
+ OnMapEntryEnd();
185
+ }
186
+ OnMapEnd(count);
187
+ }
188
+
163
189
  // class or struct implementing `FormatSerialize(FormatSerializer& FormatSerializer)`;
164
190
  template <typename T>
165
191
  typename std::enable_if<has_serialize<T>::value>::type WriteValue(const T &value) {
@@ -247,4 +273,8 @@ protected:
247
273
  }
248
274
  };
249
275
 
276
+ // We need to special case vector<bool> because elements of vector<bool> cannot be referenced
277
+ template <>
278
+ void FormatSerializer::WriteValue(const vector<bool> &vec);
279
+
250
280
  } // namespace duckdb
@@ -70,6 +70,16 @@ struct is_unordered_map<typename duckdb::unordered_map<Args...>> : std::true_typ
70
70
  typedef typename std::tuple_element<3, std::tuple<Args...>>::type EQUAL_TYPE;
71
71
  };
72
72
 
73
+ template <typename T>
74
+ struct is_map : std::false_type {};
75
+ template <typename... Args>
76
+ struct is_map<typename duckdb::map<Args...>> : std::true_type {
77
+ typedef typename std::tuple_element<0, std::tuple<Args...>>::type KEY_TYPE;
78
+ typedef typename std::tuple_element<1, std::tuple<Args...>>::type VALUE_TYPE;
79
+ typedef typename std::tuple_element<2, std::tuple<Args...>>::type HASH_TYPE;
80
+ typedef typename std::tuple_element<3, std::tuple<Args...>>::type EQUAL_TYPE;
81
+ };
82
+
73
83
  template <typename T>
74
84
  struct is_unique_ptr : std::false_type {};
75
85
 
@@ -147,6 +147,8 @@ struct BufferedCSVReaderOptions {
147
147
 
148
148
  void Serialize(FieldWriter &writer) const;
149
149
  void Deserialize(FieldReader &reader);
150
+ void FormatSerialize(FormatSerializer &serializer) const;
151
+ static BufferedCSVReaderOptions FormatDeserialize(FormatDeserializer &deserializer);
150
152
 
151
153
  void SetCompression(const string &compression);
152
154
  void SetHeader(bool has_header);
@@ -52,6 +52,11 @@ typedef void (*aggregate_serialize_t)(FieldWriter &writer, const FunctionData *b
52
52
  typedef unique_ptr<FunctionData> (*aggregate_deserialize_t)(PlanDeserializationState &context, FieldReader &reader,
53
53
  AggregateFunction &function);
54
54
 
55
+ typedef void (*aggregate_format_serialize_t)(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data,
56
+ const AggregateFunction &function);
57
+ typedef unique_ptr<FunctionData> (*aggregate_format_deserialize_t)(FormatDeserializer &deserializer,
58
+ AggregateFunction &function);
59
+
55
60
  class AggregateFunction : public BaseScalarFunction {
56
61
  public:
57
62
  AggregateFunction(const string &name, const vector<LogicalType> &arguments, const LogicalType &return_type,
@@ -66,7 +71,8 @@ public:
66
71
  LogicalType(LogicalTypeId::INVALID), null_handling),
67
72
  state_size(state_size), initialize(initialize), update(update), combine(combine), finalize(finalize),
68
73
  simple_update(simple_update), window(window), bind(bind), destructor(destructor), statistics(statistics),
69
- serialize(serialize), deserialize(deserialize), order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
74
+ serialize(serialize), deserialize(deserialize), format_serialize(nullptr), format_deserialize(nullptr),
75
+ order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
70
76
  }
71
77
 
72
78
  AggregateFunction(const string &name, const vector<LogicalType> &arguments, const LogicalType &return_type,
@@ -80,7 +86,8 @@ public:
80
86
  LogicalType(LogicalTypeId::INVALID)),
81
87
  state_size(state_size), initialize(initialize), update(update), combine(combine), finalize(finalize),
82
88
  simple_update(simple_update), window(window), bind(bind), destructor(destructor), statistics(statistics),
83
- serialize(serialize), deserialize(deserialize), order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
89
+ serialize(serialize), deserialize(deserialize), format_serialize(nullptr), format_deserialize(nullptr),
90
+ order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
84
91
  }
85
92
 
86
93
  AggregateFunction(const vector<LogicalType> &arguments, const LogicalType &return_type, aggregate_size_t state_size,
@@ -131,6 +138,8 @@ public:
131
138
 
132
139
  aggregate_serialize_t serialize;
133
140
  aggregate_deserialize_t deserialize;
141
+ aggregate_format_serialize_t format_serialize;
142
+ aggregate_format_deserialize_t format_deserialize;
134
143
  //! Whether or not the aggregate is order dependent
135
144
  AggregateOrderDependent order_dependent;
136
145
 
@@ -11,6 +11,8 @@
11
11
  #include "duckdb/common/field_writer.hpp"
12
12
  #include "duckdb/main/client_context.hpp"
13
13
  #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
14
+ #include "duckdb/common/serializer/format_serializer.hpp"
15
+ #include "duckdb/common/serializer/format_deserializer.hpp"
14
16
 
15
17
  namespace duckdb {
16
18
 
@@ -97,6 +99,85 @@ public:
97
99
  function.return_type = return_type;
98
100
  return function;
99
101
  }
102
+
103
+ template <class FUNC>
104
+ static void FormatSerialize(FormatSerializer &serializer, const FUNC &function,
105
+ optional_ptr<FunctionData> bind_info) {
106
+ D_ASSERT(!function.name.empty());
107
+ serializer.WriteProperty("name", function.name);
108
+ serializer.WriteProperty("arguments", function.arguments);
109
+ serializer.WriteProperty("original_arguments", function.original_arguments);
110
+ bool has_serialize = function.format_serialize;
111
+ serializer.WriteProperty("has_serialize", has_serialize);
112
+ if (has_serialize) {
113
+ serializer.BeginObject("function_data");
114
+ function.format_serialize(serializer, bind_info, function);
115
+ serializer.EndObject();
116
+ D_ASSERT(function.format_deserialize);
117
+ }
118
+ }
119
+
120
+ template <class FUNC, class CATALOG_ENTRY>
121
+ static FUNC DeserializeFunction(ClientContext &context, CatalogType catalog_type, const string &name,
122
+ vector<LogicalType> arguments, vector<LogicalType> original_arguments) {
123
+ auto &func_catalog = Catalog::GetEntry(context, catalog_type, SYSTEM_CATALOG, DEFAULT_SCHEMA, name);
124
+ if (func_catalog.type != catalog_type) {
125
+ throw InternalException("DeserializeFunction - cant find catalog entry for function %s", name);
126
+ }
127
+ auto &functions = func_catalog.Cast<CATALOG_ENTRY>();
128
+ auto function = functions.functions.GetFunctionByArguments(
129
+ context, original_arguments.empty() ? arguments : original_arguments);
130
+ function.arguments = std::move(arguments);
131
+ function.original_arguments = std::move(original_arguments);
132
+ return function;
133
+ }
134
+
135
+ template <class FUNC, class CATALOG_ENTRY>
136
+ static pair<FUNC, bool> FormatDeserializeBase(FormatDeserializer &deserializer, CatalogType catalog_type) {
137
+ auto &context = deserializer.Get<ClientContext &>();
138
+ auto name = deserializer.ReadProperty<string>("name");
139
+ auto arguments = deserializer.ReadProperty<vector<LogicalType>>("arguments");
140
+ auto original_arguments = deserializer.ReadProperty<vector<LogicalType>>("original_arguments");
141
+ auto function = DeserializeFunction<FUNC, CATALOG_ENTRY>(context, catalog_type, name, std::move(arguments),
142
+ std::move(original_arguments));
143
+ auto has_serialize = deserializer.ReadProperty<bool>("has_serialize");
144
+ return make_pair(std::move(function), has_serialize);
145
+ }
146
+
147
+ template <class FUNC>
148
+ static unique_ptr<FunctionData> FunctionDeserialize(FormatDeserializer &deserializer, FUNC &function) {
149
+ if (!function.format_deserialize) {
150
+ throw SerializationException("Function requires deserialization but no deserialization function for %s",
151
+ function.name);
152
+ }
153
+ deserializer.BeginObject("function_data");
154
+ auto result = function.format_deserialize(deserializer, function);
155
+ deserializer.EndObject();
156
+ return result;
157
+ }
158
+
159
+ template <class FUNC, class CATALOG_ENTRY>
160
+ static pair<FUNC, unique_ptr<FunctionData>> FormatDeserialize(FormatDeserializer &deserializer,
161
+ CatalogType catalog_type,
162
+ vector<unique_ptr<Expression>> &children) {
163
+ auto &context = deserializer.Get<ClientContext &>();
164
+ auto entry = FormatDeserializeBase<FUNC, CATALOG_ENTRY>(deserializer, catalog_type);
165
+ auto &function = entry.first;
166
+ auto has_serialize = entry.second;
167
+
168
+ unique_ptr<FunctionData> bind_data;
169
+ if (has_serialize) {
170
+ bind_data = FunctionDeserialize<FUNC>(deserializer, function);
171
+ } else if (function.bind) {
172
+ try {
173
+ bind_data = function.bind(context, function, children);
174
+ } catch (Exception &ex) {
175
+ // FIXME
176
+ throw SerializationException("Error during bind of function in deserialization: %s", ex.what());
177
+ }
178
+ }
179
+ return make_pair(std::move(function), std::move(bind_data));
180
+ }
100
181
  };
101
182
 
102
183
  } // namespace duckdb
@@ -121,6 +121,8 @@ protected:
121
121
 
122
122
  struct StrpTimeFormat : public StrTimeFormat {
123
123
  public:
124
+ StrpTimeFormat();
125
+
124
126
  //! Type-safe parsing argument
125
127
  struct ParseResult {
126
128
  int32_t data[8]; // year, month, day, hour, min, sec, µs, offset
@@ -148,12 +150,18 @@ public:
148
150
  date_t ParseDate(string_t str);
149
151
  timestamp_t ParseTimestamp(string_t str);
150
152
 
153
+ void FormatSerialize(FormatSerializer &serializer) const;
154
+ static StrpTimeFormat FormatDeserialize(FormatDeserializer &deserializer);
155
+
151
156
  protected:
152
157
  static string FormatStrpTimeError(const string &input, idx_t position);
153
158
  DUCKDB_API void AddFormatSpecifier(string preceding_literal, StrTimeSpecifier specifier) override;
154
159
  int NumericSpecifierWidth(StrTimeSpecifier specifier);
155
160
  int32_t TryParseCollection(const char *data, idx_t &pos, idx_t size, const string_t collection[],
156
161
  idx_t collection_count);
162
+
163
+ private:
164
+ explicit StrpTimeFormat(const string &format_string);
157
165
  };
158
166
 
159
167
  } // namespace duckdb
@@ -69,6 +69,11 @@ typedef void (*function_serialize_t)(FieldWriter &writer, const FunctionData *bi
69
69
  typedef unique_ptr<FunctionData> (*function_deserialize_t)(PlanDeserializationState &state, FieldReader &reader,
70
70
  ScalarFunction &function);
71
71
 
72
+ typedef void (*function_format_serialize_t)(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data,
73
+ const ScalarFunction &function);
74
+ typedef unique_ptr<FunctionData> (*function_format_deserialize_t)(FormatDeserializer &deserializer,
75
+ ScalarFunction &function);
76
+
72
77
  class ScalarFunction : public BaseScalarFunction {
73
78
  public:
74
79
  DUCKDB_API ScalarFunction(string name, vector<LogicalType> arguments, LogicalType return_type,
@@ -100,6 +105,9 @@ public:
100
105
  function_serialize_t serialize;
101
106
  function_deserialize_t deserialize;
102
107
 
108
+ function_format_serialize_t format_serialize;
109
+ function_format_deserialize_t format_deserialize;
110
+
103
111
  DUCKDB_API bool operator==(const ScalarFunction &rhs) const;
104
112
  DUCKDB_API bool operator!=(const ScalarFunction &rhs) const;
105
113
 
@@ -76,6 +76,10 @@ struct ColumnInfo {
76
76
  info.types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
77
77
  return info;
78
78
  }
79
+
80
+ void FormatSerialize(FormatSerializer &serializer) const;
81
+ static ColumnInfo FormatDeserialize(FormatDeserializer &deserializer);
82
+
79
83
  vector<std::string> names;
80
84
  vector<LogicalType> types;
81
85
  };
@@ -105,6 +109,9 @@ struct ReadCSVData : public BaseCSVData {
105
109
  this->initial_reader = std::move(reader);
106
110
  }
107
111
  void FinalizeRead(ClientContext &context);
112
+
113
+ void FormatSerialize(FormatSerializer &serializer) const;
114
+ static unique_ptr<ReadCSVData> FormatDeserialize(FormatDeserializer &deserializer);
108
115
  };
109
116
 
110
117
  struct CSVCopyFunction {
@@ -207,6 +207,12 @@ typedef void (*table_function_serialize_t)(FieldWriter &writer, const FunctionDa
207
207
  typedef unique_ptr<FunctionData> (*table_function_deserialize_t)(PlanDeserializationState &context, FieldReader &reader,
208
208
  TableFunction &function);
209
209
 
210
+ typedef void (*table_function_format_serialize_t)(FormatSerializer &serializer,
211
+ const optional_ptr<FunctionData> bind_data,
212
+ const TableFunction &function);
213
+ typedef unique_ptr<FunctionData> (*table_function_format_deserialize_t)(FormatDeserializer &deserializer,
214
+ TableFunction &function);
215
+
210
216
  class TableFunction : public SimpleNamedParameterFunction {
211
217
  public:
212
218
  DUCKDB_API
@@ -265,6 +271,8 @@ public:
265
271
 
266
272
  table_function_serialize_t serialize;
267
273
  table_function_deserialize_t deserialize;
274
+ table_function_format_serialize_t format_serialize;
275
+ table_function_format_deserialize_t format_deserialize;
268
276
  bool verify_serialization = true;
269
277
 
270
278
  //! Whether or not the table function supports projection pushdown. If not supported a projection will be added
@@ -57,5 +57,8 @@ public:
57
57
  unique_ptr<Expression> Copy() override;
58
58
  void Serialize(FieldWriter &writer) const override;
59
59
  static unique_ptr<Expression> Deserialize(ExpressionDeserializationState &state, FieldReader &reader);
60
+
61
+ void FormatSerialize(FormatSerializer &serializer) const override;
62
+ static unique_ptr<Expression> FormatDeserialize(FormatDeserializer &deserializer);
60
63
  };
61
64
  } // namespace duckdb
@@ -46,5 +46,9 @@ public:
46
46
 
47
47
  void Serialize(FieldWriter &writer) const override;
48
48
  static unique_ptr<Expression> Deserialize(ExpressionDeserializationState &state, FieldReader &reader);
49
+
50
+ void FormatSerialize(FormatSerializer &serializer) const override;
51
+ static unique_ptr<Expression> FormatDeserialize(FormatDeserializer &deserializer);
49
52
  };
53
+
50
54
  } // namespace duckdb
@@ -67,5 +67,8 @@ public:
67
67
 
68
68
  void Serialize(FieldWriter &writer) const override;
69
69
  static unique_ptr<Expression> Deserialize(ExpressionDeserializationState &state, FieldReader &reader);
70
+
71
+ void FormatSerialize(FormatSerializer &serializer) const override;
72
+ static unique_ptr<Expression> FormatDeserialize(FormatDeserializer &deserializer);
70
73
  };
71
74
  } // namespace duckdb
@@ -45,6 +45,8 @@ public:
45
45
  bool Equals(const TableFilter &other) const override;
46
46
  void Serialize(FieldWriter &writer) const override;
47
47
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
48
+ void FormatSerialize(FormatSerializer &serializer) const override;
49
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
48
50
  };
49
51
 
50
52
  class ConjunctionAndFilter : public ConjunctionFilter {
@@ -60,6 +62,8 @@ public:
60
62
  bool Equals(const TableFilter &other) const override;
61
63
  void Serialize(FieldWriter &writer) const override;
62
64
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
65
+ void FormatSerialize(FormatSerializer &serializer) const override;
66
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
63
67
  };
64
68
 
65
69
  } // namespace duckdb
@@ -32,6 +32,8 @@ public:
32
32
  bool Equals(const TableFilter &other) const override;
33
33
  void Serialize(FieldWriter &writer) const override;
34
34
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
35
+ void FormatSerialize(FormatSerializer &serializer) const override;
36
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
35
37
  };
36
38
 
37
39
  } // namespace duckdb
@@ -24,6 +24,8 @@ public:
24
24
  string ToString(const string &column_name) override;
25
25
  void Serialize(FieldWriter &writer) const override;
26
26
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
27
+ void FormatSerialize(FormatSerializer &serializer) const override;
28
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
27
29
  };
28
30
 
29
31
  class IsNotNullFilter : public TableFilter {
@@ -38,6 +40,8 @@ public:
38
40
  string ToString(const string &column_name) override;
39
41
  void Serialize(FieldWriter &writer) const override;
40
42
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
43
+ void FormatSerialize(FormatSerializer &serializer) const override;
44
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
41
45
  };
42
46
 
43
47
  } // namespace duckdb
@@ -45,6 +45,8 @@ public:
45
45
  bool SupportSerialization() const override {
46
46
  return false;
47
47
  }
48
+ void FormatSerialize(FormatSerializer &serializer) const override;
49
+ static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
48
50
 
49
51
  protected:
50
52
  void ResolveTypes() override {
@@ -69,9 +69,15 @@ public:
69
69
  //! Skips the serialization check in VerifyPlan
70
70
  bool SupportSerialization() const override {
71
71
  return function.verify_serialization;
72
- };
72
+ }
73
+
74
+ void FormatSerialize(FormatSerializer &serializer) const override;
75
+ static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
73
76
 
74
77
  protected:
75
78
  void ResolveTypes() override;
79
+
80
+ private:
81
+ LogicalGet();
76
82
  };
77
83
  } // namespace duckdb
@@ -29,7 +29,7 @@ enum class TableFilterType : uint8_t {
29
29
  //! TableFilter represents a filter pushed down into the table scan.
30
30
  class TableFilter {
31
31
  public:
32
- TableFilter(TableFilterType filter_type_p) : filter_type(filter_type_p) {
32
+ explicit TableFilter(TableFilterType filter_type_p) : filter_type(filter_type_p) {
33
33
  }
34
34
  virtual ~TableFilter() {
35
35
  }
@@ -48,6 +48,9 @@ public:
48
48
  virtual void Serialize(FieldWriter &writer) const = 0;
49
49
  static unique_ptr<TableFilter> Deserialize(Deserializer &source);
50
50
 
51
+ virtual void FormatSerialize(FormatSerializer &serializer) const;
52
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
53
+
51
54
  public:
52
55
  template <class TARGET>
53
56
  TARGET &Cast() {
@@ -100,6 +103,9 @@ public:
100
103
 
101
104
  void Serialize(Serializer &serializer) const;
102
105
  static unique_ptr<TableFilterSet> Deserialize(Deserializer &source);
106
+
107
+ void FormatSerialize(FormatSerializer &serializer) const;
108
+ static TableFilterSet FormatDeserialize(FormatDeserializer &deserializer);
103
109
  };
104
110
 
105
111
  } // namespace duckdb
@@ -179,6 +179,19 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
179
179
  }
180
180
  #endif
181
181
 
182
+ #ifdef DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE
183
+ if (!initial_load && StringUtil::Contains(DUCKDB_EXTENSIONS_TEST_WITH_LOADABLE, extension)) {
184
+ Connection con(db);
185
+ auto result = con.Query((string) "LOAD '" + DUCKDB_EXTENSIONS_BUILD_PATH + "/" + extension + "/" + extension +
186
+ ".duckdb_extension'");
187
+ if (result->HasError()) {
188
+ result->Print();
189
+ return ExtensionLoadResult::EXTENSION_UNKNOWN;
190
+ }
191
+ return ExtensionLoadResult::LOADED_EXTENSION;
192
+ }
193
+ #endif
194
+
182
195
  // This is the main extension loading mechanism that loads the extension that are statically linked.
183
196
  #if defined(GENERATED_EXTENSION_HEADERS) && GENERATED_EXTENSION_HEADERS
184
197
  if (TryLoadLinkedExtension(db, extension)) {
@@ -107,7 +107,7 @@ void Executor::SchedulePipeline(const shared_ptr<MetaPipeline> &meta_pipeline, S
107
107
  group_stack.pipeline_finish_event, base_stack.pipeline_complete_event);
108
108
 
109
109
  // dependencies: base_finish -> pipeline_event -> group_finish
110
- pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_event);
110
+ pipeline_stack.pipeline_event.AddDependency(base_stack.pipeline_finish_event);
111
111
  group_stack.pipeline_finish_event.AddDependency(pipeline_stack.pipeline_event);
112
112
 
113
113
  // add pipeline stack to event map
@@ -104,4 +104,27 @@ unique_ptr<Expression> BoundAggregateExpression::Deserialize(ExpressionDeseriali
104
104
  return std::move(x);
105
105
  }
106
106
 
107
+ void BoundAggregateExpression::FormatSerialize(FormatSerializer &serializer) const {
108
+ Expression::FormatSerialize(serializer);
109
+ serializer.WriteProperty("return_type", return_type);
110
+ serializer.WriteProperty("children", children);
111
+ FunctionSerializer::FormatSerialize(serializer, function, bind_info.get());
112
+ serializer.WriteProperty("aggregate_type", aggr_type);
113
+ serializer.WriteOptionalProperty("filter", filter);
114
+ serializer.WriteOptionalProperty("order_bys", order_bys);
115
+ }
116
+
117
+ unique_ptr<Expression> BoundAggregateExpression::FormatDeserialize(FormatDeserializer &deserializer) {
118
+ auto return_type = deserializer.ReadProperty<LogicalType>("return_type");
119
+ auto children = deserializer.ReadProperty<vector<unique_ptr<Expression>>>("children");
120
+ auto entry = FunctionSerializer::FormatDeserialize<AggregateFunction, AggregateFunctionCatalogEntry>(
121
+ deserializer, CatalogType::AGGREGATE_FUNCTION_ENTRY, children);
122
+ auto aggregate_type = deserializer.ReadProperty<AggregateType>("aggregate_type");
123
+ auto filter = deserializer.ReadOptionalProperty<unique_ptr<Expression>>("filter");
124
+ auto result = make_uniq<BoundAggregateExpression>(std::move(entry.first), std::move(children), std::move(filter),
125
+ std::move(entry.second), aggregate_type);
126
+ deserializer.ReadOptionalProperty("order_bys", result->order_bys);
127
+ return std::move(result);
128
+ }
129
+
107
130
  } // namespace duckdb
@@ -3,6 +3,8 @@
3
3
  #include "duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp"
4
4
  #include "duckdb/common/types/hash.hpp"
5
5
  #include "duckdb/function/function_serialization.hpp"
6
+ #include "duckdb/common/serializer/format_serializer.hpp"
7
+ #include "duckdb/common/serializer/format_deserializer.hpp"
6
8
 
7
9
  namespace duckdb {
8
10
 
@@ -92,4 +94,24 @@ unique_ptr<Expression> BoundFunctionExpression::Deserialize(ExpressionDeserializ
92
94
  return make_uniq<BoundFunctionExpression>(std::move(return_type), std::move(function), std::move(children),
93
95
  std::move(bind_info), is_operator);
94
96
  }
97
+
98
+ void BoundFunctionExpression::FormatSerialize(FormatSerializer &serializer) const {
99
+ Expression::FormatSerialize(serializer);
100
+ serializer.WriteProperty("return_type", return_type);
101
+ serializer.WriteProperty("children", children);
102
+ FunctionSerializer::FormatSerialize(serializer, function, bind_info.get());
103
+ serializer.WriteProperty("is_operator", is_operator);
104
+ }
105
+
106
+ unique_ptr<Expression> BoundFunctionExpression::FormatDeserialize(FormatDeserializer &deserializer) {
107
+ auto return_type = deserializer.ReadProperty<LogicalType>("return_type");
108
+ auto children = deserializer.ReadProperty<vector<unique_ptr<Expression>>>("children");
109
+ auto entry = FunctionSerializer::FormatDeserialize<ScalarFunction, ScalarFunctionCatalogEntry>(
110
+ deserializer, CatalogType::SCALAR_FUNCTION_ENTRY, children);
111
+ auto result = make_uniq<BoundFunctionExpression>(std::move(return_type), std::move(entry.first),
112
+ std::move(children), std::move(entry.second));
113
+ deserializer.ReadProperty("is_operator", result->is_operator);
114
+ return std::move(result);
115
+ }
116
+
95
117
  } // namespace duckdb
@@ -162,4 +162,51 @@ unique_ptr<Expression> BoundWindowExpression::Deserialize(ExpressionDeserializat
162
162
  return std::move(result);
163
163
  }
164
164
 
165
+ void BoundWindowExpression::FormatSerialize(FormatSerializer &serializer) const {
166
+ Expression::FormatSerialize(serializer);
167
+ serializer.WriteProperty("return_type", return_type);
168
+ serializer.WriteProperty("children", children);
169
+ if (type == ExpressionType::WINDOW_AGGREGATE) {
170
+ D_ASSERT(aggregate);
171
+ FunctionSerializer::FormatSerialize(serializer, *aggregate, bind_info.get());
172
+ }
173
+ serializer.WriteProperty("partitions", partitions);
174
+ serializer.WriteProperty("orders", orders);
175
+ serializer.WriteOptionalProperty("filters", filter_expr);
176
+ serializer.WriteProperty("ignore_nulls", ignore_nulls);
177
+ serializer.WriteProperty("start", start);
178
+ serializer.WriteProperty("end", end);
179
+ serializer.WriteOptionalProperty("start_expr", start_expr);
180
+ serializer.WriteOptionalProperty("end_expr", end_expr);
181
+ serializer.WriteOptionalProperty("offset_expr", offset_expr);
182
+ serializer.WriteOptionalProperty("default_expr", default_expr);
183
+ }
184
+
185
+ unique_ptr<Expression> BoundWindowExpression::FormatDeserialize(FormatDeserializer &deserializer) {
186
+ auto expression_type = deserializer.Get<ExpressionType>();
187
+ auto return_type = deserializer.ReadProperty<LogicalType>("return_type");
188
+ auto children = deserializer.ReadProperty<vector<unique_ptr<Expression>>>("children");
189
+ unique_ptr<AggregateFunction> aggregate;
190
+ unique_ptr<FunctionData> bind_info;
191
+ if (expression_type == ExpressionType::WINDOW_AGGREGATE) {
192
+ auto entry = FunctionSerializer::FormatDeserialize<AggregateFunction, AggregateFunctionCatalogEntry>(
193
+ deserializer, CatalogType::AGGREGATE_FUNCTION_ENTRY, children);
194
+ aggregate = make_uniq<AggregateFunction>(std::move(entry.first));
195
+ bind_info = std::move(entry.second);
196
+ }
197
+ auto result =
198
+ make_uniq<BoundWindowExpression>(expression_type, return_type, std::move(aggregate), std::move(bind_info));
199
+ deserializer.ReadProperty("partitions", result->partitions);
200
+ deserializer.ReadProperty("orders", result->orders);
201
+ deserializer.ReadOptionalProperty("filters", result->filter_expr);
202
+ deserializer.ReadProperty("ignore_nulls", result->ignore_nulls);
203
+ deserializer.ReadProperty("start", result->start);
204
+ deserializer.ReadProperty("end", result->end);
205
+ deserializer.ReadOptionalProperty("start_expr", result->start_expr);
206
+ deserializer.ReadOptionalProperty("end_expr", result->end_expr);
207
+ deserializer.ReadOptionalProperty("offset_expr", result->offset_expr);
208
+ deserializer.ReadOptionalProperty("default_expr", result->default_expr);
209
+ return std::move(result);
210
+ }
211
+
165
212
  } // namespace duckdb
@@ -59,6 +59,14 @@ unique_ptr<LogicalOperator> LogicalCopyToFile::Deserialize(LogicalDeserializatio
59
59
  return std::move(result);
60
60
  }
61
61
 
62
+ void LogicalCopyToFile::FormatSerialize(FormatSerializer &serializer) const {
63
+ throw SerializationException("LogicalCopyToFile not implemented yet");
64
+ }
65
+
66
+ unique_ptr<LogicalOperator> LogicalCopyToFile::FormatDeserialize(FormatDeserializer &deserializer) {
67
+ throw SerializationException("LogicalCopyToFile not implemented yet");
68
+ }
69
+
62
70
  idx_t LogicalCopyToFile::EstimateCardinality(ClientContext &context) {
63
71
  return 1;
64
72
  }