duckdb 0.8.2-dev2090.0 → 0.8.2-dev2208.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/binding.gyp +3 -0
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/json/buffered_json_reader.cpp +2 -0
  4. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +5 -19
  5. package/src/duckdb/extension/json/include/json_enums.hpp +60 -0
  6. package/src/duckdb/extension/json/include/json_scan.hpp +14 -10
  7. package/src/duckdb/extension/json/include/json_transform.hpp +3 -0
  8. package/src/duckdb/extension/json/json_enums.cpp +105 -0
  9. package/src/duckdb/extension/json/json_functions/json_transform.cpp +2 -0
  10. package/src/duckdb/extension/json/json_functions.cpp +2 -1
  11. package/src/duckdb/extension/json/json_scan.cpp +44 -0
  12. package/src/duckdb/extension/json/serialize_json.cpp +92 -0
  13. package/src/duckdb/extension/parquet/include/parquet_reader.hpp +3 -0
  14. package/src/duckdb/extension/parquet/parquet_extension.cpp +25 -1
  15. package/src/duckdb/extension/parquet/parquet_reader.cpp +3 -0
  16. package/src/duckdb/extension/parquet/serialize_parquet.cpp +26 -0
  17. package/src/duckdb/src/catalog/catalog.cpp +2 -6
  18. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -4
  19. package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +0 -4
  20. package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp +7 -26
  21. package/src/duckdb/src/catalog/catalog_set.cpp +0 -63
  22. package/src/duckdb/src/catalog/dependency_manager.cpp +0 -36
  23. package/src/duckdb/src/common/extra_type_info.cpp +24 -46
  24. package/src/duckdb/src/common/field_writer.cpp +0 -1
  25. package/src/duckdb/src/common/file_system.cpp +6 -6
  26. package/src/duckdb/src/common/filename_pattern.cpp +1 -1
  27. package/src/duckdb/src/common/gzip_file_system.cpp +7 -12
  28. package/src/duckdb/src/common/local_file_system.cpp +17 -14
  29. package/src/duckdb/src/common/multi_file_reader.cpp +8 -5
  30. package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +0 -9
  31. package/src/duckdb/src/common/serializer/format_serializer.cpp +15 -0
  32. package/src/duckdb/src/common/types.cpp +12 -56
  33. package/src/duckdb/src/common/virtual_file_system.cpp +4 -0
  34. package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +26 -0
  35. package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +47 -0
  36. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +28 -0
  37. package/src/duckdb/src/core_functions/scalar/date/strftime.cpp +10 -0
  38. package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +22 -3
  39. package/src/duckdb/src/core_functions/scalar/union/union_tag.cpp +1 -1
  40. package/src/duckdb/src/execution/column_binding_resolver.cpp +3 -7
  41. package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +1 -1
  42. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +0 -10
  43. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +2 -1
  44. package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp +2 -6
  45. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +1 -2
  46. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +14 -1
  47. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +3 -17
  48. package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp +2 -5
  49. package/src/duckdb/src/execution/physical_plan_generator.cpp +1 -5
  50. package/src/duckdb/src/function/aggregate/distributive/count.cpp +0 -11
  51. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +1 -9
  52. package/src/duckdb/src/function/cast/string_cast.cpp +0 -1
  53. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +27 -0
  54. package/src/duckdb/src/function/scalar_function.cpp +5 -20
  55. package/src/duckdb/src/function/table/read_csv.cpp +20 -1
  56. package/src/duckdb/src/function/table/system/test_all_types.cpp +4 -4
  57. package/src/duckdb/src/function/table/table_scan.cpp +35 -0
  58. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  59. package/src/duckdb/src/function/table_function.cpp +4 -3
  60. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +0 -2
  61. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp +1 -4
  62. package/src/duckdb/src/include/duckdb/catalog/catalog_set.hpp +0 -6
  63. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +2 -6
  64. package/src/duckdb/src/include/duckdb/common/field_writer.hpp +0 -4
  65. package/src/duckdb/src/include/duckdb/common/file_system.hpp +8 -8
  66. package/src/duckdb/src/include/duckdb/common/filename_pattern.hpp +1 -1
  67. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +5 -1
  68. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +3 -1
  69. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +4 -0
  70. package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +0 -4
  71. package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +32 -0
  72. package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +45 -15
  73. package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +10 -0
  74. package/src/duckdb/src/include/duckdb/common/serializer.hpp +0 -7
  75. package/src/duckdb/src/include/duckdb/common/types/timestamp.hpp +14 -10
  76. package/src/duckdb/src/include/duckdb/common/types.hpp +5 -10
  77. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +2 -0
  78. package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_reader_options.hpp +2 -0
  79. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +0 -4
  80. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -2
  81. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +11 -2
  82. package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +81 -0
  83. package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp +8 -0
  84. package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +8 -3
  85. package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp +7 -0
  86. package/src/duckdb/src/include/duckdb/function/table_function.hpp +8 -0
  87. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +1 -1
  88. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +2 -2
  89. package/src/duckdb/src/include/duckdb/planner/expression/bound_aggregate_expression.hpp +3 -0
  90. package/src/duckdb/src/include/duckdb/planner/expression/bound_function_expression.hpp +4 -0
  91. package/src/duckdb/src/include/duckdb/planner/expression/bound_window_expression.hpp +3 -0
  92. package/src/duckdb/src/include/duckdb/planner/filter/conjunction_filter.hpp +4 -0
  93. package/src/duckdb/src/include/duckdb/planner/filter/constant_filter.hpp +2 -0
  94. package/src/duckdb/src/include/duckdb/planner/filter/null_filter.hpp +4 -0
  95. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +0 -2
  96. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +1 -2
  97. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +2 -4
  98. package/src/duckdb/src/include/duckdb/planner/operator/logical_copy_to_file.hpp +2 -0
  99. package/src/duckdb/src/include/duckdb/planner/operator/logical_create_index.hpp +10 -7
  100. package/src/duckdb/src/include/duckdb/planner/operator/logical_extension_operator.hpp +5 -0
  101. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +7 -1
  102. package/src/duckdb/src/include/duckdb/planner/operator_extension.hpp +1 -0
  103. package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +7 -1
  104. package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -3
  105. package/src/duckdb/src/main/attached_database.cpp +2 -2
  106. package/src/duckdb/src/main/database.cpp +1 -1
  107. package/src/duckdb/src/main/db_instance_cache.cpp +14 -6
  108. package/src/duckdb/src/main/extension/extension_helper.cpp +13 -0
  109. package/src/duckdb/src/main/extension/extension_install.cpp +1 -1
  110. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  111. package/src/duckdb/src/optimizer/compressed_materialization.cpp +0 -1
  112. package/src/duckdb/src/optimizer/deliminator.cpp +7 -7
  113. package/src/duckdb/src/optimizer/unnest_rewriter.cpp +3 -5
  114. package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +16 -1
  115. package/src/duckdb/src/parser/transform/statement/transform_create_type.cpp +1 -1
  116. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +3 -3
  117. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +7 -23
  118. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +0 -24
  119. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +3 -5
  120. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +23 -0
  121. package/src/duckdb/src/planner/expression/bound_function_expression.cpp +22 -0
  122. package/src/duckdb/src/planner/expression/bound_window_expression.cpp +47 -0
  123. package/src/duckdb/src/planner/logical_operator.cpp +1 -6
  124. package/src/duckdb/src/planner/logical_operator_visitor.cpp +3 -6
  125. package/src/duckdb/src/planner/operator/logical_comparison_join.cpp +2 -14
  126. package/src/duckdb/src/planner/operator/logical_copy_to_file.cpp +8 -0
  127. package/src/duckdb/src/planner/operator/logical_create_index.cpp +21 -12
  128. package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +24 -0
  129. package/src/duckdb/src/planner/operator/logical_get.cpp +69 -0
  130. package/src/duckdb/src/planner/planner.cpp +0 -1
  131. package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -5
  132. package/src/duckdb/src/storage/meta_block_reader.cpp +0 -9
  133. package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +2 -0
  134. package/src/duckdb/src/storage/serialization/serialize_expression.cpp +9 -0
  135. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +33 -0
  136. package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +190 -0
  137. package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +97 -0
  138. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  139. package/src/duckdb/src/storage/wal_replay.cpp +1 -3
  140. package/src/duckdb/ub_src_common_serializer.cpp +2 -0
  141. package/src/duckdb/ub_src_planner_operator.cpp +0 -4
  142. package/src/duckdb/ub_src_storage_serialization.cpp +2 -0
  143. package/src/statement.cpp +0 -2
  144. package/test/columns.test.ts +1 -2
  145. package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp +0 -27
  146. package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_join.hpp +0 -32
  147. package/src/duckdb/src/planner/operator/logical_asof_join.cpp +0 -14
  148. package/src/duckdb/src/planner/operator/logical_delim_join.cpp +0 -25
@@ -22,8 +22,6 @@ private:
22
22
  uint64_t version = 0L;
23
23
 
24
24
  public:
25
- bool is_query_plan = false;
26
-
27
25
  virtual ~Serializer() {
28
26
  }
29
27
 
@@ -119,11 +117,6 @@ public:
119
117
  throw InternalException("This deserializer does not have a client-context");
120
118
  };
121
119
 
122
- //! Gets the catalog for the deserializer
123
- virtual optional_ptr<Catalog> GetCatalog() {
124
- return nullptr;
125
- };
126
-
127
120
  template <class T>
128
121
  T Read() {
129
122
  T value;
@@ -26,7 +26,7 @@ struct timestamp_t { // NOLINT
26
26
  int64_t value;
27
27
 
28
28
  timestamp_t() = default;
29
- explicit inline timestamp_t(int64_t value_p) : value(value_p) {
29
+ explicit inline constexpr timestamp_t(int64_t value_p) : value(value_p) {
30
30
  }
31
31
  inline timestamp_t &operator=(int64_t value_p) {
32
32
  value = value_p;
@@ -67,21 +67,25 @@ struct timestamp_t { // NOLINT
67
67
  timestamp_t &operator-=(const int64_t &delta);
68
68
 
69
69
  // special values
70
- static timestamp_t infinity() { // NOLINT
70
+ static constexpr timestamp_t infinity() { // NOLINT
71
71
  return timestamp_t(NumericLimits<int64_t>::Maximum());
72
- } // NOLINT
73
- static timestamp_t ninfinity() { // NOLINT
72
+ } // NOLINT
73
+ static constexpr timestamp_t ninfinity() { // NOLINT
74
74
  return timestamp_t(-NumericLimits<int64_t>::Maximum());
75
- } // NOLINT
76
- static inline timestamp_t epoch() { // NOLINT
75
+ } // NOLINT
76
+ static constexpr inline timestamp_t epoch() { // NOLINT
77
77
  return timestamp_t(0);
78
78
  } // NOLINT
79
79
  };
80
80
 
81
- struct timestamp_tz_t : public timestamp_t {}; // NOLINT
82
- struct timestamp_ns_t : public timestamp_t {}; // NOLINT
83
- struct timestamp_ms_t : public timestamp_t {}; // NOLINT
84
- struct timestamp_sec_t : public timestamp_t {}; // NOLINT
81
+ struct timestamp_tz_t : public timestamp_t { // NOLINT
82
+ };
83
+ struct timestamp_ns_t : public timestamp_t { // NOLINT
84
+ };
85
+ struct timestamp_ms_t : public timestamp_t { // NOLINT
86
+ };
87
+ struct timestamp_sec_t : public timestamp_t { // NOLINT
88
+ };
85
89
 
86
90
  enum class TimestampCastResult : uint8_t { SUCCESS, ERROR_INCORRECT_FORMAT, ERROR_NON_UTC_TIMEZONE };
87
91
 
@@ -286,8 +286,6 @@ struct LogicalType {
286
286
  //! Serializes a LogicalType to a stand-alone binary blob
287
287
  DUCKDB_API void Serialize(Serializer &serializer) const;
288
288
 
289
- DUCKDB_API void SerializeEnumType(Serializer &serializer) const;
290
-
291
289
  //! Deserializes a blob back into an LogicalType
292
290
  DUCKDB_API static LogicalType Deserialize(Deserializer &source);
293
291
 
@@ -373,9 +371,10 @@ public:
373
371
  DUCKDB_API static LogicalType STRUCT(child_list_t<LogicalType> children); // NOLINT
374
372
  DUCKDB_API static LogicalType AGGREGATE_STATE(aggregate_state_t state_type); // NOLINT
375
373
  DUCKDB_API static LogicalType MAP(const LogicalType &child); // NOLINT
376
- DUCKDB_API static LogicalType MAP( child_list_t<LogicalType> children); // NOLINT
377
374
  DUCKDB_API static LogicalType MAP(LogicalType key, LogicalType value); // NOLINT
378
375
  DUCKDB_API static LogicalType UNION( child_list_t<LogicalType> members); // NOLINT
376
+ DUCKDB_API static LogicalType ENUM(Vector &ordered_data, idx_t size); // NOLINT
377
+ // DEPRECATED - provided for backwards compatibility
379
378
  DUCKDB_API static LogicalType ENUM(const string &enum_name, Vector &ordered_data, idx_t size); // NOLINT
380
379
  DUCKDB_API static LogicalType USER(const string &user_type_name); // NOLINT
381
380
  //! A list of all NUMERIC types (integral and floating point types)
@@ -400,21 +399,17 @@ struct ListType {
400
399
  DUCKDB_API static const LogicalType &GetChildType(const LogicalType &type);
401
400
  };
402
401
 
403
- struct UserType{
402
+ struct UserType {
404
403
  DUCKDB_API static const string &GetTypeName(const LogicalType &type);
405
404
  };
406
405
 
407
- struct EnumType{
408
- DUCKDB_API static const string &GetTypeName(const LogicalType &type);
406
+ struct EnumType {
409
407
  DUCKDB_API static int64_t GetPos(const LogicalType &type, const string_t& key);
410
408
  DUCKDB_API static const Vector &GetValuesInsertOrder(const LogicalType &type);
411
409
  DUCKDB_API static idx_t GetSize(const LogicalType &type);
412
410
  DUCKDB_API static const string GetValue(const Value &val);
413
- DUCKDB_API static void SetCatalog(LogicalType &type, optional_ptr<TypeCatalogEntry> catalog_entry);
414
- DUCKDB_API static optional_ptr<TypeCatalogEntry> GetCatalog(const LogicalType &type);
415
- DUCKDB_API static string GetSchemaName(const LogicalType &type);
416
411
  DUCKDB_API static PhysicalType GetPhysicalType(const LogicalType &type);
417
- DUCKDB_API static void Serialize(FieldWriter& writer, const ExtraTypeInfo& type_info, bool serialize_internals);
412
+ DUCKDB_API static string_t GetString(const LogicalType &type, idx_t pos);
418
413
  };
419
414
 
420
415
  struct StructType {
@@ -68,6 +68,8 @@ public:
68
68
 
69
69
  void SetDisabledFileSystems(const vector<string> &names) override;
70
70
 
71
+ string PathSeparator(const string &path) override;
72
+
71
73
  private:
72
74
  FileSystem &FindFileSystem(const string &path);
73
75
  FileSystem &FindFileSystemInternal(const string &path);
@@ -147,6 +147,8 @@ struct BufferedCSVReaderOptions {
147
147
 
148
148
  void Serialize(FieldWriter &writer) const;
149
149
  void Deserialize(FieldReader &reader);
150
+ void FormatSerialize(FormatSerializer &serializer) const;
151
+ static BufferedCSVReaderOptions FormatDeserialize(FormatDeserializer &deserializer);
150
152
 
151
153
  void SetCompression(const string &compression);
152
154
  void SetHeader(bool has_header);
@@ -22,10 +22,6 @@ public:
22
22
  static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::TABLE_SCAN;
23
23
 
24
24
  public:
25
- //! Regular Table Scan
26
- PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
27
- vector<column_t> column_ids, vector<string> names, unique_ptr<TableFilterSet> table_filters,
28
- idx_t estimated_cardinality);
29
25
  //! Table scan that immediately projects out filter columns that are unused in the remainder of the query plan
30
26
  PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
31
27
  vector<LogicalType> returned_types, vector<column_t> column_ids, vector<idx_t> projection_ids,
@@ -50,7 +50,6 @@ protected:
50
50
 
51
51
  unique_ptr<PhysicalOperator> CreatePlan(LogicalAggregate &op);
52
52
  unique_ptr<PhysicalOperator> CreatePlan(LogicalAnyJoin &op);
53
- unique_ptr<PhysicalOperator> CreatePlan(LogicalAsOfJoin &op);
54
53
  unique_ptr<PhysicalOperator> CreatePlan(LogicalColumnDataGet &op);
55
54
  unique_ptr<PhysicalOperator> CreatePlan(LogicalComparisonJoin &op);
56
55
  unique_ptr<PhysicalOperator> CreatePlan(LogicalCreate &op);
@@ -59,7 +58,6 @@ protected:
59
58
  unique_ptr<PhysicalOperator> CreatePlan(LogicalCrossProduct &op);
60
59
  unique_ptr<PhysicalOperator> CreatePlan(LogicalDelete &op);
61
60
  unique_ptr<PhysicalOperator> CreatePlan(LogicalDelimGet &op);
62
- unique_ptr<PhysicalOperator> CreatePlan(LogicalDelimJoin &op);
63
61
  unique_ptr<PhysicalOperator> CreatePlan(LogicalDistinct &op);
64
62
  unique_ptr<PhysicalOperator> CreatePlan(LogicalDummyScan &expr);
65
63
  unique_ptr<PhysicalOperator> CreatePlan(LogicalEmptyResult &op);
@@ -93,6 +91,9 @@ protected:
93
91
  unique_ptr<PhysicalOperator> CreatePlan(LogicalCTERef &op);
94
92
  unique_ptr<PhysicalOperator> CreatePlan(LogicalPivot &op);
95
93
 
94
+ unique_ptr<PhysicalOperator> PlanAsOfJoin(LogicalComparisonJoin &op);
95
+ unique_ptr<PhysicalOperator> PlanComparisonJoin(LogicalComparisonJoin &op);
96
+ unique_ptr<PhysicalOperator> PlanDelimJoin(LogicalComparisonJoin &op);
96
97
  unique_ptr<PhysicalOperator> ExtractAggregateExpressions(unique_ptr<PhysicalOperator> child,
97
98
  vector<unique_ptr<Expression>> &expressions,
98
99
  vector<unique_ptr<Expression>> &groups);
@@ -52,6 +52,11 @@ typedef void (*aggregate_serialize_t)(FieldWriter &writer, const FunctionData *b
52
52
  typedef unique_ptr<FunctionData> (*aggregate_deserialize_t)(PlanDeserializationState &context, FieldReader &reader,
53
53
  AggregateFunction &function);
54
54
 
55
+ typedef void (*aggregate_format_serialize_t)(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data,
56
+ const AggregateFunction &function);
57
+ typedef unique_ptr<FunctionData> (*aggregate_format_deserialize_t)(FormatDeserializer &deserializer,
58
+ AggregateFunction &function);
59
+
55
60
  class AggregateFunction : public BaseScalarFunction {
56
61
  public:
57
62
  AggregateFunction(const string &name, const vector<LogicalType> &arguments, const LogicalType &return_type,
@@ -66,7 +71,8 @@ public:
66
71
  LogicalType(LogicalTypeId::INVALID), null_handling),
67
72
  state_size(state_size), initialize(initialize), update(update), combine(combine), finalize(finalize),
68
73
  simple_update(simple_update), window(window), bind(bind), destructor(destructor), statistics(statistics),
69
- serialize(serialize), deserialize(deserialize), order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
74
+ serialize(serialize), deserialize(deserialize), format_serialize(nullptr), format_deserialize(nullptr),
75
+ order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
70
76
  }
71
77
 
72
78
  AggregateFunction(const string &name, const vector<LogicalType> &arguments, const LogicalType &return_type,
@@ -80,7 +86,8 @@ public:
80
86
  LogicalType(LogicalTypeId::INVALID)),
81
87
  state_size(state_size), initialize(initialize), update(update), combine(combine), finalize(finalize),
82
88
  simple_update(simple_update), window(window), bind(bind), destructor(destructor), statistics(statistics),
83
- serialize(serialize), deserialize(deserialize), order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
89
+ serialize(serialize), deserialize(deserialize), format_serialize(nullptr), format_deserialize(nullptr),
90
+ order_dependent(AggregateOrderDependent::ORDER_DEPENDENT) {
84
91
  }
85
92
 
86
93
  AggregateFunction(const vector<LogicalType> &arguments, const LogicalType &return_type, aggregate_size_t state_size,
@@ -131,6 +138,8 @@ public:
131
138
 
132
139
  aggregate_serialize_t serialize;
133
140
  aggregate_deserialize_t deserialize;
141
+ aggregate_format_serialize_t format_serialize;
142
+ aggregate_format_deserialize_t format_deserialize;
134
143
  //! Whether or not the aggregate is order dependent
135
144
  AggregateOrderDependent order_dependent;
136
145
 
@@ -11,6 +11,8 @@
11
11
  #include "duckdb/common/field_writer.hpp"
12
12
  #include "duckdb/main/client_context.hpp"
13
13
  #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
14
+ #include "duckdb/common/serializer/format_serializer.hpp"
15
+ #include "duckdb/common/serializer/format_deserializer.hpp"
14
16
 
15
17
  namespace duckdb {
16
18
 
@@ -97,6 +99,85 @@ public:
97
99
  function.return_type = return_type;
98
100
  return function;
99
101
  }
102
+
103
+ template <class FUNC>
104
+ static void FormatSerialize(FormatSerializer &serializer, const FUNC &function,
105
+ optional_ptr<FunctionData> bind_info) {
106
+ D_ASSERT(!function.name.empty());
107
+ serializer.WriteProperty("name", function.name);
108
+ serializer.WriteProperty("arguments", function.arguments);
109
+ serializer.WriteProperty("original_arguments", function.original_arguments);
110
+ bool has_serialize = function.format_serialize;
111
+ serializer.WriteProperty("has_serialize", has_serialize);
112
+ if (has_serialize) {
113
+ serializer.BeginObject("function_data");
114
+ function.format_serialize(serializer, bind_info, function);
115
+ serializer.EndObject();
116
+ D_ASSERT(function.format_deserialize);
117
+ }
118
+ }
119
+
120
+ template <class FUNC, class CATALOG_ENTRY>
121
+ static FUNC DeserializeFunction(ClientContext &context, CatalogType catalog_type, const string &name,
122
+ vector<LogicalType> arguments, vector<LogicalType> original_arguments) {
123
+ auto &func_catalog = Catalog::GetEntry(context, catalog_type, SYSTEM_CATALOG, DEFAULT_SCHEMA, name);
124
+ if (func_catalog.type != catalog_type) {
125
+ throw InternalException("DeserializeFunction - cant find catalog entry for function %s", name);
126
+ }
127
+ auto &functions = func_catalog.Cast<CATALOG_ENTRY>();
128
+ auto function = functions.functions.GetFunctionByArguments(
129
+ context, original_arguments.empty() ? arguments : original_arguments);
130
+ function.arguments = std::move(arguments);
131
+ function.original_arguments = std::move(original_arguments);
132
+ return function;
133
+ }
134
+
135
+ template <class FUNC, class CATALOG_ENTRY>
136
+ static pair<FUNC, bool> FormatDeserializeBase(FormatDeserializer &deserializer, CatalogType catalog_type) {
137
+ auto &context = deserializer.Get<ClientContext &>();
138
+ auto name = deserializer.ReadProperty<string>("name");
139
+ auto arguments = deserializer.ReadProperty<vector<LogicalType>>("arguments");
140
+ auto original_arguments = deserializer.ReadProperty<vector<LogicalType>>("original_arguments");
141
+ auto function = DeserializeFunction<FUNC, CATALOG_ENTRY>(context, catalog_type, name, std::move(arguments),
142
+ std::move(original_arguments));
143
+ auto has_serialize = deserializer.ReadProperty<bool>("has_serialize");
144
+ return make_pair(std::move(function), has_serialize);
145
+ }
146
+
147
+ template <class FUNC>
148
+ static unique_ptr<FunctionData> FunctionDeserialize(FormatDeserializer &deserializer, FUNC &function) {
149
+ if (!function.format_deserialize) {
150
+ throw SerializationException("Function requires deserialization but no deserialization function for %s",
151
+ function.name);
152
+ }
153
+ deserializer.BeginObject("function_data");
154
+ auto result = function.format_deserialize(deserializer, function);
155
+ deserializer.EndObject();
156
+ return result;
157
+ }
158
+
159
+ template <class FUNC, class CATALOG_ENTRY>
160
+ static pair<FUNC, unique_ptr<FunctionData>> FormatDeserialize(FormatDeserializer &deserializer,
161
+ CatalogType catalog_type,
162
+ vector<unique_ptr<Expression>> &children) {
163
+ auto &context = deserializer.Get<ClientContext &>();
164
+ auto entry = FormatDeserializeBase<FUNC, CATALOG_ENTRY>(deserializer, catalog_type);
165
+ auto &function = entry.first;
166
+ auto has_serialize = entry.second;
167
+
168
+ unique_ptr<FunctionData> bind_data;
169
+ if (has_serialize) {
170
+ bind_data = FunctionDeserialize<FUNC>(deserializer, function);
171
+ } else if (function.bind) {
172
+ try {
173
+ bind_data = function.bind(context, function, children);
174
+ } catch (Exception &ex) {
175
+ // FIXME
176
+ throw SerializationException("Error during bind of function in deserialization: %s", ex.what());
177
+ }
178
+ }
179
+ return make_pair(std::move(function), std::move(bind_data));
180
+ }
100
181
  };
101
182
 
102
183
  } // namespace duckdb
@@ -121,6 +121,8 @@ protected:
121
121
 
122
122
  struct StrpTimeFormat : public StrTimeFormat {
123
123
  public:
124
+ StrpTimeFormat();
125
+
124
126
  //! Type-safe parsing argument
125
127
  struct ParseResult {
126
128
  int32_t data[8]; // year, month, day, hour, min, sec, µs, offset
@@ -148,12 +150,18 @@ public:
148
150
  date_t ParseDate(string_t str);
149
151
  timestamp_t ParseTimestamp(string_t str);
150
152
 
153
+ void FormatSerialize(FormatSerializer &serializer) const;
154
+ static StrpTimeFormat FormatDeserialize(FormatDeserializer &deserializer);
155
+
151
156
  protected:
152
157
  static string FormatStrpTimeError(const string &input, idx_t position);
153
158
  DUCKDB_API void AddFormatSpecifier(string preceding_literal, StrTimeSpecifier specifier) override;
154
159
  int NumericSpecifierWidth(StrTimeSpecifier specifier);
155
160
  int32_t TryParseCollection(const char *data, idx_t &pos, idx_t size, const string_t collection[],
156
161
  idx_t collection_count);
162
+
163
+ private:
164
+ explicit StrpTimeFormat(const string &format_string);
157
165
  };
158
166
 
159
167
  } // namespace duckdb
@@ -69,6 +69,11 @@ typedef void (*function_serialize_t)(FieldWriter &writer, const FunctionData *bi
69
69
  typedef unique_ptr<FunctionData> (*function_deserialize_t)(PlanDeserializationState &state, FieldReader &reader,
70
70
  ScalarFunction &function);
71
71
 
72
+ typedef void (*function_format_serialize_t)(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data,
73
+ const ScalarFunction &function);
74
+ typedef unique_ptr<FunctionData> (*function_format_deserialize_t)(FormatDeserializer &deserializer,
75
+ ScalarFunction &function);
76
+
72
77
  class ScalarFunction : public BaseScalarFunction {
73
78
  public:
74
79
  DUCKDB_API ScalarFunction(string name, vector<LogicalType> arguments, LogicalType return_type,
@@ -100,14 +105,14 @@ public:
100
105
  function_serialize_t serialize;
101
106
  function_deserialize_t deserialize;
102
107
 
108
+ function_format_serialize_t format_serialize;
109
+ function_format_deserialize_t format_deserialize;
110
+
103
111
  DUCKDB_API bool operator==(const ScalarFunction &rhs) const;
104
112
  DUCKDB_API bool operator!=(const ScalarFunction &rhs) const;
105
113
 
106
114
  DUCKDB_API bool Equal(const ScalarFunction &rhs) const;
107
115
 
108
- private:
109
- bool CompareScalarFunctionT(const scalar_function_t &other) const;
110
-
111
116
  public:
112
117
  DUCKDB_API static void NopFunction(DataChunk &input, ExpressionState &state, Vector &result);
113
118
 
@@ -76,6 +76,10 @@ struct ColumnInfo {
76
76
  info.types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
77
77
  return info;
78
78
  }
79
+
80
+ void FormatSerialize(FormatSerializer &serializer) const;
81
+ static ColumnInfo FormatDeserialize(FormatDeserializer &deserializer);
82
+
79
83
  vector<std::string> names;
80
84
  vector<LogicalType> types;
81
85
  };
@@ -105,6 +109,9 @@ struct ReadCSVData : public BaseCSVData {
105
109
  this->initial_reader = std::move(reader);
106
110
  }
107
111
  void FinalizeRead(ClientContext &context);
112
+
113
+ void FormatSerialize(FormatSerializer &serializer) const;
114
+ static unique_ptr<ReadCSVData> FormatDeserialize(FormatDeserializer &deserializer);
108
115
  };
109
116
 
110
117
  struct CSVCopyFunction {
@@ -207,6 +207,12 @@ typedef void (*table_function_serialize_t)(FieldWriter &writer, const FunctionDa
207
207
  typedef unique_ptr<FunctionData> (*table_function_deserialize_t)(PlanDeserializationState &context, FieldReader &reader,
208
208
  TableFunction &function);
209
209
 
210
+ typedef void (*table_function_format_serialize_t)(FormatSerializer &serializer,
211
+ const optional_ptr<FunctionData> bind_data,
212
+ const TableFunction &function);
213
+ typedef unique_ptr<FunctionData> (*table_function_format_deserialize_t)(FormatDeserializer &deserializer,
214
+ TableFunction &function);
215
+
210
216
  class TableFunction : public SimpleNamedParameterFunction {
211
217
  public:
212
218
  DUCKDB_API
@@ -265,6 +271,8 @@ public:
265
271
 
266
272
  table_function_serialize_t serialize;
267
273
  table_function_deserialize_t deserialize;
274
+ table_function_format_serialize_t format_serialize;
275
+ table_function_format_deserialize_t format_deserialize;
268
276
  bool verify_serialization = true;
269
277
 
270
278
  //! Whether or not the table function supports projection pushdown. If not supported a projection will be added
@@ -60,7 +60,7 @@ public:
60
60
  bool IsInitialDatabase() const;
61
61
  void SetInitialDatabase();
62
62
 
63
- static string ExtractDatabaseName(const string &dbpath);
63
+ static string ExtractDatabaseName(const string &dbpath, FileSystem &fs);
64
64
 
65
65
  private:
66
66
  DatabaseInstance &db;
@@ -27,9 +27,9 @@ private:
27
27
  void FindCandidates(unique_ptr<LogicalOperator> &op, vector<DelimCandidate> &candidates);
28
28
  void FindJoinWithDelimGet(unique_ptr<LogicalOperator> &op, DelimCandidate &candidate);
29
29
  //! Remove joins with a DelimGet
30
- bool RemoveJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
30
+ bool RemoveJoinWithDelimGet(LogicalComparisonJoin &delim_join, const idx_t delim_get_count,
31
31
  unique_ptr<LogicalOperator> &join, bool &all_equality_conditions);
32
- bool RemoveInequalityJoinWithDelimGet(LogicalDelimJoin &delim_join, const idx_t delim_get_count,
32
+ bool RemoveInequalityJoinWithDelimGet(LogicalComparisonJoin &delim_join, const idx_t delim_get_count,
33
33
  unique_ptr<LogicalOperator> &join,
34
34
  const vector<ReplacementBinding> &replacement_bindings);
35
35
 
@@ -57,5 +57,8 @@ public:
57
57
  unique_ptr<Expression> Copy() override;
58
58
  void Serialize(FieldWriter &writer) const override;
59
59
  static unique_ptr<Expression> Deserialize(ExpressionDeserializationState &state, FieldReader &reader);
60
+
61
+ void FormatSerialize(FormatSerializer &serializer) const override;
62
+ static unique_ptr<Expression> FormatDeserialize(FormatDeserializer &deserializer);
60
63
  };
61
64
  } // namespace duckdb
@@ -46,5 +46,9 @@ public:
46
46
 
47
47
  void Serialize(FieldWriter &writer) const override;
48
48
  static unique_ptr<Expression> Deserialize(ExpressionDeserializationState &state, FieldReader &reader);
49
+
50
+ void FormatSerialize(FormatSerializer &serializer) const override;
51
+ static unique_ptr<Expression> FormatDeserialize(FormatDeserializer &deserializer);
49
52
  };
53
+
50
54
  } // namespace duckdb
@@ -67,5 +67,8 @@ public:
67
67
 
68
68
  void Serialize(FieldWriter &writer) const override;
69
69
  static unique_ptr<Expression> Deserialize(ExpressionDeserializationState &state, FieldReader &reader);
70
+
71
+ void FormatSerialize(FormatSerializer &serializer) const override;
72
+ static unique_ptr<Expression> FormatDeserialize(FormatDeserializer &deserializer);
70
73
  };
71
74
  } // namespace duckdb
@@ -45,6 +45,8 @@ public:
45
45
  bool Equals(const TableFilter &other) const override;
46
46
  void Serialize(FieldWriter &writer) const override;
47
47
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
48
+ void FormatSerialize(FormatSerializer &serializer) const override;
49
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
48
50
  };
49
51
 
50
52
  class ConjunctionAndFilter : public ConjunctionFilter {
@@ -60,6 +62,8 @@ public:
60
62
  bool Equals(const TableFilter &other) const override;
61
63
  void Serialize(FieldWriter &writer) const override;
62
64
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
65
+ void FormatSerialize(FormatSerializer &serializer) const override;
66
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
63
67
  };
64
68
 
65
69
  } // namespace duckdb
@@ -32,6 +32,8 @@ public:
32
32
  bool Equals(const TableFilter &other) const override;
33
33
  void Serialize(FieldWriter &writer) const override;
34
34
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
35
+ void FormatSerialize(FormatSerializer &serializer) const override;
36
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
35
37
  };
36
38
 
37
39
  } // namespace duckdb
@@ -24,6 +24,8 @@ public:
24
24
  string ToString(const string &column_name) override;
25
25
  void Serialize(FieldWriter &writer) const override;
26
26
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
27
+ void FormatSerialize(FormatSerializer &serializer) const override;
28
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
27
29
  };
28
30
 
29
31
  class IsNotNullFilter : public TableFilter {
@@ -38,6 +40,8 @@ public:
38
40
  string ToString(const string &column_name) override;
39
41
  void Serialize(FieldWriter &writer) const override;
40
42
  static unique_ptr<TableFilter> Deserialize(FieldReader &source);
43
+ void FormatSerialize(FormatSerializer &serializer) const override;
44
+ static unique_ptr<TableFilter> FormatDeserialize(FormatDeserializer &deserializer);
41
45
  };
42
46
 
43
47
  } // namespace duckdb
@@ -14,7 +14,6 @@ class LogicalOperator;
14
14
 
15
15
  class LogicalAggregate;
16
16
  class LogicalAnyJoin;
17
- class LogicalAsOfJoin;
18
17
  class LogicalColumnDataGet;
19
18
  class LogicalComparisonJoin;
20
19
  class LogicalCopyToFile;
@@ -26,7 +25,6 @@ class LogicalCrossProduct;
26
25
  class LogicalCTERef;
27
26
  class LogicalDelete;
28
27
  class LogicalDelimGet;
29
- class LogicalDelimJoin;
30
28
  class LogicalDistinct;
31
29
  class LogicalDummyScan;
32
30
  class LogicalEmptyResult;
@@ -1,6 +1,5 @@
1
1
  #include "duckdb/planner/operator/logical_aggregate.hpp"
2
2
  #include "duckdb/planner/operator/logical_any_join.hpp"
3
- #include "duckdb/planner/operator/logical_asof_join.hpp"
4
3
  #include "duckdb/planner/operator/logical_column_data_get.hpp"
5
4
  #include "duckdb/planner/operator/logical_comparison_join.hpp"
6
5
  #include "duckdb/planner/operator/logical_copy_to_file.hpp"
@@ -11,7 +10,6 @@
11
10
  #include "duckdb/planner/operator/logical_cteref.hpp"
12
11
  #include "duckdb/planner/operator/logical_delete.hpp"
13
12
  #include "duckdb/planner/operator/logical_delim_get.hpp"
14
- #include "duckdb/planner/operator/logical_delim_join.hpp"
15
13
  #include "duckdb/planner/operator/logical_distinct.hpp"
16
14
  #include "duckdb/planner/operator/logical_dummy_scan.hpp"
17
15
  #include "duckdb/planner/operator/logical_empty_result.hpp"
@@ -19,6 +17,7 @@
19
17
  #include "duckdb/planner/operator/logical_explain.hpp"
20
18
  #include "duckdb/planner/operator/logical_export.hpp"
21
19
  #include "duckdb/planner/operator/logical_expression_get.hpp"
20
+ #include "duckdb/planner/operator/logical_extension_operator.hpp"
22
21
  #include "duckdb/planner/operator/logical_filter.hpp"
23
22
  #include "duckdb/planner/operator/logical_get.hpp"
24
23
  #include "duckdb/planner/operator/logical_insert.hpp"
@@ -15,7 +15,6 @@
15
15
  #include "duckdb/planner/operator/logical_join.hpp"
16
16
 
17
17
  namespace duckdb {
18
- class LogicalDelimJoin;
19
18
 
20
19
  //! LogicalComparisonJoin represents a join that involves comparisons between the LHS and RHS
21
20
  class LogicalComparisonJoin : public LogicalJoin {
@@ -30,6 +29,8 @@ public:
30
29
  vector<JoinCondition> conditions;
31
30
  //! Used for duplicate-eliminated MARK joins
32
31
  vector<LogicalType> mark_types;
32
+ //! The set of columns that will be duplicate eliminated from the LHS and pushed into the RHS
33
+ vector<unique_ptr<Expression>> duplicate_eliminated_columns;
33
34
 
34
35
  public:
35
36
  string ParamsToString() const override;
@@ -41,9 +42,6 @@ public:
41
42
  void FormatSerialize(FormatSerializer &serializer) const override;
42
43
  static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
43
44
 
44
- //! Turn a delim join into a regular comparison join (after all required delim scans have been pruned)
45
- static unique_ptr<LogicalOperator> FromDelimJoin(LogicalDelimJoin &join);
46
-
47
45
  public:
48
46
  static unique_ptr<LogicalOperator> CreateJoin(ClientContext &context, JoinType type, JoinRefType ref_type,
49
47
  unique_ptr<LogicalOperator> left_child,
@@ -45,6 +45,8 @@ public:
45
45
  bool SupportSerialization() const override {
46
46
  return false;
47
47
  }
48
+ void FormatSerialize(FormatSerializer &serializer) const override;
49
+ static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
48
50
 
49
51
  protected:
50
52
  void ResolveTypes() override {
@@ -19,19 +19,14 @@ public:
19
19
  static constexpr const LogicalOperatorType TYPE = LogicalOperatorType::LOGICAL_CREATE_INDEX;
20
20
 
21
21
  public:
22
- LogicalCreateIndex(unique_ptr<FunctionData> bind_data_p, unique_ptr<CreateIndexInfo> info_p,
23
- vector<unique_ptr<Expression>> expressions_p, TableCatalogEntry &table_p,
24
- TableFunction function_p);
22
+ LogicalCreateIndex(unique_ptr<CreateIndexInfo> info_p, vector<unique_ptr<Expression>> expressions_p,
23
+ TableCatalogEntry &table_p);
25
24
 
26
- //! The bind data of the function
27
- unique_ptr<FunctionData> bind_data;
28
25
  // Info for index creation
29
26
  unique_ptr<CreateIndexInfo> info;
30
27
 
31
28
  //! The table to create the index for
32
29
  TableCatalogEntry &table;
33
- //! The function that is called
34
- TableFunction function;
35
30
 
36
31
  //! Unbound expressions to be used in the optimizer
37
32
  vector<unique_ptr<Expression>> unbound_expressions;
@@ -40,7 +35,15 @@ public:
40
35
  void Serialize(FieldWriter &writer) const override;
41
36
  static unique_ptr<LogicalOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
42
37
 
38
+ void FormatSerialize(FormatSerializer &serializer) const override;
39
+ static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
40
+
43
41
  protected:
44
42
  void ResolveTypes() override;
43
+
44
+ private:
45
+ LogicalCreateIndex(ClientContext &context, unique_ptr<CreateInfo> info, vector<unique_ptr<Expression>> expressions);
46
+
47
+ TableCatalogEntry &BindTable(ClientContext &context, CreateIndexInfo &info);
45
48
  };
46
49
  } // namespace duckdb
@@ -26,6 +26,11 @@ public:
26
26
 
27
27
  static unique_ptr<LogicalExtensionOperator> Deserialize(LogicalDeserializationState &state, FieldReader &reader);
28
28
 
29
+ virtual void FormatSerialize(FormatSerializer &serializer) const override;
30
+ static unique_ptr<LogicalOperator> FormatDeserialize(FormatDeserializer &deserializer);
31
+
29
32
  virtual unique_ptr<PhysicalOperator> CreatePlan(ClientContext &context, PhysicalPlanGenerator &generator) = 0;
33
+
34
+ virtual string GetExtensionName() const;
30
35
  };
31
36
  } // namespace duckdb