duckdb 0.8.2-dev2700.0 → 0.8.2-dev2842.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/extension/icu/icu-makedate.cpp +12 -6
  3. package/src/duckdb/extension/json/include/json_deserializer.hpp +1 -1
  4. package/src/duckdb/extension/json/include/json_serializer.hpp +1 -1
  5. package/src/duckdb/extension/json/json_deserializer.cpp +10 -10
  6. package/src/duckdb/extension/json/json_scan.cpp +2 -2
  7. package/src/duckdb/extension/json/json_serializer.cpp +11 -10
  8. package/src/duckdb/extension/json/serialize_json.cpp +44 -44
  9. package/src/duckdb/extension/parquet/parquet_extension.cpp +11 -10
  10. package/src/duckdb/extension/parquet/serialize_parquet.cpp +6 -6
  11. package/src/duckdb/src/common/adbc/adbc.cpp +52 -21
  12. package/src/duckdb/src/common/adbc/driver_manager.cpp +12 -2
  13. package/src/duckdb/src/common/enum_util.cpp +5 -0
  14. package/src/duckdb/src/common/extra_type_info.cpp +2 -2
  15. package/src/duckdb/src/common/serializer/binary_deserializer.cpp +5 -3
  16. package/src/duckdb/src/common/serializer/binary_serializer.cpp +10 -5
  17. package/src/duckdb/src/common/types/column/column_data_collection.cpp +4 -4
  18. package/src/duckdb/src/common/types/row/row_data_collection_scanner.cpp +35 -5
  19. package/src/duckdb/src/common/types/value.cpp +33 -33
  20. package/src/duckdb/src/common/types/vector.cpp +20 -20
  21. package/src/duckdb/src/core_functions/aggregate/holistic/approximate_quantile.cpp +2 -2
  22. package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +6 -6
  23. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +4 -4
  24. package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +4 -4
  25. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +283 -91
  26. package/src/duckdb/src/execution/operator/filter/physical_filter.cpp +1 -1
  27. package/src/duckdb/src/execution/operator/join/physical_comparison_join.cpp +1 -2
  28. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +1 -1
  29. package/src/duckdb/src/execution/physical_plan_generator.cpp +1 -6
  30. package/src/duckdb/src/execution/window_executor.cpp +10 -1
  31. package/src/duckdb/src/function/table/read_csv.cpp +4 -4
  32. package/src/duckdb/src/function/table/table_scan.cpp +14 -14
  33. package/src/duckdb/src/function/table/version/pragma_version.cpp +5 -2
  34. package/src/duckdb/src/include/duckdb/common/adbc/adbc.hpp +2 -0
  35. package/src/duckdb/src/include/duckdb/common/enums/pending_execution_result.hpp +1 -1
  36. package/src/duckdb/src/include/duckdb/common/index_vector.hpp +2 -2
  37. package/src/duckdb/src/include/duckdb/common/serializer/binary_deserializer.hpp +7 -3
  38. package/src/duckdb/src/include/duckdb/common/serializer/binary_serializer.hpp +2 -1
  39. package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +18 -17
  40. package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +10 -9
  41. package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +4 -0
  42. package/src/duckdb/src/include/duckdb/common/types/row/row_data_collection_scanner.hpp +5 -1
  43. package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +0 -2
  44. package/src/duckdb/src/include/duckdb/function/function_serialization.hpp +10 -10
  45. package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp +5 -0
  46. package/src/duckdb/src/include/duckdb/main/relation/aggregate_relation.hpp +4 -1
  47. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +37 -63
  48. package/src/duckdb/src/include/duckdb/optimizer/join_order/cost_model.hpp +37 -0
  49. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_node.hpp +14 -29
  50. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_order_optimizer.hpp +7 -21
  51. package/src/duckdb/src/include/duckdb/optimizer/join_order/join_relation.hpp +0 -11
  52. package/src/duckdb/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +89 -0
  53. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph.hpp +17 -31
  54. package/src/duckdb/src/include/duckdb/optimizer/join_order/query_graph_manager.hpp +113 -0
  55. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp +73 -0
  56. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp +73 -0
  57. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +4 -1
  58. package/src/duckdb/src/include/duckdb/parser/group_by_node.hpp +11 -0
  59. package/src/duckdb/src/include/duckdb/parser/parser.hpp +4 -0
  60. package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +0 -2
  61. package/src/duckdb/src/include/duckdb.h +11 -1
  62. package/src/duckdb/src/main/capi/pending-c.cpp +17 -0
  63. package/src/duckdb/src/main/pending_query_result.cpp +9 -1
  64. package/src/duckdb/src/main/relation/aggregate_relation.cpp +20 -10
  65. package/src/duckdb/src/main/relation.cpp +4 -4
  66. package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +79 -325
  67. package/src/duckdb/src/optimizer/join_order/cost_model.cpp +19 -0
  68. package/src/duckdb/src/optimizer/join_order/join_node.cpp +5 -37
  69. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +48 -1078
  70. package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp +552 -0
  71. package/src/duckdb/src/optimizer/join_order/query_graph.cpp +32 -29
  72. package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +409 -0
  73. package/src/duckdb/src/optimizer/join_order/relation_manager.cpp +356 -0
  74. package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +351 -0
  75. package/src/duckdb/src/parallel/executor.cpp +6 -0
  76. package/src/duckdb/src/parallel/task_scheduler.cpp +7 -0
  77. package/src/duckdb/src/parser/parser.cpp +18 -3
  78. package/src/duckdb/src/parser/tableref/pivotref.cpp +6 -6
  79. package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +1 -1
  80. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +10 -10
  81. package/src/duckdb/src/planner/expression/bound_function_expression.cpp +6 -6
  82. package/src/duckdb/src/planner/expression/bound_window_expression.cpp +24 -24
  83. package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +2 -2
  84. package/src/duckdb/src/planner/operator/logical_get.cpp +26 -22
  85. package/src/duckdb/src/storage/serialization/serialize_constraint.cpp +26 -26
  86. package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +66 -66
  87. package/src/duckdb/src/storage/serialization/serialize_expression.cpp +78 -78
  88. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +250 -250
  89. package/src/duckdb/src/storage/serialization/serialize_macro_function.cpp +10 -10
  90. package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +206 -206
  91. package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +116 -116
  92. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +110 -110
  93. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +48 -48
  94. package/src/duckdb/src/storage/serialization/serialize_result_modifier.cpp +16 -16
  95. package/src/duckdb/src/storage/serialization/serialize_statement.cpp +2 -2
  96. package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +10 -10
  97. package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +54 -54
  98. package/src/duckdb/src/storage/serialization/serialize_types.cpp +22 -22
  99. package/src/duckdb/src/storage/table/update_segment.cpp +1 -1
  100. package/src/duckdb/ub_src_optimizer_join_order.cpp +10 -0
@@ -1,8 +1,8 @@
1
1
  #ifndef DUCKDB_VERSION
2
- #define DUCKDB_VERSION "0.8.2-dev2700"
2
+ #define DUCKDB_VERSION "0.8.2-dev2842"
3
3
  #endif
4
4
  #ifndef DUCKDB_SOURCE_ID
5
- #define DUCKDB_SOURCE_ID "acbbfe0e79"
5
+ #define DUCKDB_SOURCE_ID "6421a36e94"
6
6
  #endif
7
7
  #include "duckdb/function/table/system_functions.hpp"
8
8
  #include "duckdb/main/database.hpp"
@@ -95,6 +95,9 @@ string DuckDB::Platform() {
95
95
  postfix = "_gcc4";
96
96
  }
97
97
  #endif
98
+ #if defined(__ANDROID__)
99
+ postfix += "_android"; // using + because it may also be gcc4
100
+ #endif
98
101
  #ifdef __MINGW32__
99
102
  postfix = "_mingw";
100
103
  #endif
@@ -88,4 +88,6 @@ void SetError(struct AdbcError *error, const std::string &message);
88
88
 
89
89
  void InitiliazeADBCError(AdbcError *error);
90
90
 
91
+ AdbcStatusCode SetErrorMaybe(const void *result, AdbcError *error, const std::string &error_message);
92
+
91
93
  } // namespace duckdb_adbc
@@ -12,6 +12,6 @@
12
12
 
13
13
  namespace duckdb {
14
14
 
15
- enum class PendingExecutionResult : uint8_t { RESULT_READY, RESULT_NOT_READY, EXECUTION_ERROR };
15
+ enum class PendingExecutionResult : uint8_t { RESULT_READY, RESULT_NOT_READY, EXECUTION_ERROR, NO_TASKS_AVAILABLE };
16
16
 
17
17
  } // namespace duckdb
@@ -62,12 +62,12 @@ public:
62
62
  }
63
63
 
64
64
  void FormatSerialize(FormatSerializer &serializer) const {
65
- serializer.WriteProperty("internal_vector", internal_vector);
65
+ serializer.WriteProperty(100, "internal_vector", internal_vector);
66
66
  }
67
67
 
68
68
  static IndexVector<T, INDEX_TYPE> FormatDeserialize(FormatDeserializer &deserializer) {
69
69
  IndexVector<T, INDEX_TYPE> result;
70
- deserializer.ReadProperty("internal_vector", result.internal_vector);
70
+ deserializer.ReadProperty(100, "internal_vector", result.internal_vector);
71
71
  return result;
72
72
  }
73
73
 
@@ -45,13 +45,17 @@ private:
45
45
  struct State {
46
46
  uint32_t expected_field_count;
47
47
  idx_t expected_size;
48
+ field_id_t expected_field_id;
48
49
  uint32_t read_field_count;
49
- State(uint32_t expected_field_count, idx_t expected_size)
50
- : expected_field_count(expected_field_count), expected_size(expected_size), read_field_count(0) {
50
+
51
+ State(uint32_t expected_field_count, idx_t expected_size, field_id_t expected_field_id)
52
+ : expected_field_count(expected_field_count), expected_size(expected_size),
53
+ expected_field_id(expected_field_id), read_field_count(0) {
51
54
  }
52
55
  };
53
56
 
54
57
  const char *current_tag = nullptr;
58
+ field_id_t current_field_id = 0;
55
59
  data_ptr_t ptr;
56
60
  data_ptr_t end_ptr;
57
61
  vector<State> stack;
@@ -72,7 +76,7 @@ private:
72
76
  }
73
77
 
74
78
  // Set the 'tag' of the property to read
75
- void SetTag(const char *tag) final;
79
+ void SetTag(const field_id_t field_id, const char *tag) final;
76
80
 
77
81
  //===--------------------------------------------------------------------===//
78
82
  // Nested Types Hooks
@@ -24,6 +24,7 @@ private:
24
24
  };
25
25
 
26
26
  const char *current_tag;
27
+ field_id_t current_field_id = 0;
27
28
 
28
29
  vector<data_t> data;
29
30
  vector<State> stack;
@@ -55,7 +56,7 @@ public:
55
56
  return std::move(serializer.data);
56
57
  }
57
58
 
58
- void SetTag(const char *tag) final;
59
+ void SetTag(const field_id_t field_id, const char *tag) final;
59
60
 
60
61
  //===--------------------------------------------------------------------===//
61
62
  // Nested Types Hooks
@@ -30,22 +30,22 @@ protected:
30
30
  public:
31
31
  // Read into an existing value
32
32
  template <typename T>
33
- inline void ReadProperty(const char *tag, T &ret) {
34
- SetTag(tag);
33
+ inline void ReadProperty(const field_id_t field_id, const char *tag, T &ret) {
34
+ SetTag(field_id, tag);
35
35
  ret = Read<T>();
36
36
  }
37
37
 
38
38
  // Read and return a value
39
39
  template <typename T>
40
- inline T ReadProperty(const char *tag) {
41
- SetTag(tag);
40
+ inline T ReadProperty(const field_id_t field_id, const char *tag) {
41
+ SetTag(field_id, tag);
42
42
  return Read<T>();
43
43
  }
44
44
 
45
45
  // Read optional property and return a value, or forward a default value
46
46
  template <typename T>
47
- inline T ReadOptionalPropertyOrDefault(const char *tag, T &&default_value) {
48
- SetTag(tag);
47
+ inline T ReadOptionalPropertyOrDefault(const field_id_t field_id, const char *tag, T &&default_value) {
48
+ SetTag(field_id, tag);
49
49
  auto present = OnOptionalBegin();
50
50
  if (present) {
51
51
  auto item = Read<T>();
@@ -59,8 +59,8 @@ public:
59
59
 
60
60
  // Read optional property into an existing value, or use a default value
61
61
  template <typename T>
62
- inline void ReadOptionalPropertyOrDefault(const char *tag, T &ret, T &&default_value) {
63
- SetTag(tag);
62
+ inline void ReadOptionalPropertyOrDefault(const field_id_t field_id, const char *tag, T &ret, T &&default_value) {
63
+ SetTag(field_id, tag);
64
64
  auto present = OnOptionalBegin();
65
65
  if (present) {
66
66
  ret = Read<T>();
@@ -74,8 +74,8 @@ public:
74
74
  // Read optional property and return a value, or default construct it
75
75
  template <typename T>
76
76
  inline typename std::enable_if<std::is_default_constructible<T>::value, T>::type
77
- ReadOptionalProperty(const char *tag) {
78
- SetTag(tag);
77
+ ReadOptionalProperty(const field_id_t field_id, const char *tag) {
78
+ SetTag(field_id, tag);
79
79
  auto present = OnOptionalBegin();
80
80
  if (present) {
81
81
  auto item = Read<T>();
@@ -90,8 +90,8 @@ public:
90
90
  // Read optional property into an existing value, or default construct it
91
91
  template <typename T>
92
92
  inline typename std::enable_if<std::is_default_constructible<T>::value, void>::type
93
- ReadOptionalProperty(const char *tag, T &ret) {
94
- SetTag(tag);
93
+ ReadOptionalProperty(const field_id_t field_id, const char *tag, T &ret) {
94
+ SetTag(field_id, tag);
95
95
  auto present = OnOptionalBegin();
96
96
  if (present) {
97
97
  ret = Read<T>();
@@ -104,8 +104,8 @@ public:
104
104
 
105
105
  // Special case:
106
106
  // Read into an existing data_ptr_t
107
- inline void ReadProperty(const char *tag, data_ptr_t ret, idx_t count) {
108
- SetTag(tag);
107
+ inline void ReadProperty(const field_id_t field_id, const char *tag, data_ptr_t ret, idx_t count) {
108
+ SetTag(field_id, tag);
109
109
  ReadDataPtr(ret, count);
110
110
  }
111
111
 
@@ -128,8 +128,8 @@ public:
128
128
  }
129
129
 
130
130
  // Manually begin an object - should be followed by EndObject
131
- void BeginObject(const char *tag) {
132
- SetTag(tag);
131
+ void BeginObject(const field_id_t field_id, const char *tag) {
132
+ SetTag(field_id, tag);
133
133
  OnObjectBegin();
134
134
  }
135
135
 
@@ -392,7 +392,8 @@ private:
392
392
  }
393
393
 
394
394
  protected:
395
- virtual void SetTag(const char *tag) {
395
+ virtual void SetTag(const field_id_t field_id, const char *tag) {
396
+ (void)field_id;
396
397
  (void)tag;
397
398
  }
398
399
 
@@ -28,15 +28,15 @@ protected:
28
28
  public:
29
29
  // Serialize a value
30
30
  template <class T>
31
- void WriteProperty(const char *tag, const T &value) {
32
- SetTag(tag);
31
+ void WriteProperty(const field_id_t field_id, const char *tag, const T &value) {
32
+ SetTag(field_id, tag);
33
33
  WriteValue(value);
34
34
  }
35
35
 
36
36
  // Optional pointer
37
37
  template <class POINTER>
38
- void WriteOptionalProperty(const char *tag, POINTER &&ptr) {
39
- SetTag(tag);
38
+ void WriteOptionalProperty(const field_id_t field_id, const char *tag, POINTER &&ptr) {
39
+ SetTag(field_id, tag);
40
40
  if (ptr == nullptr) {
41
41
  OnOptionalBegin(false);
42
42
  OnOptionalEnd(false);
@@ -48,14 +48,14 @@ public:
48
48
  }
49
49
 
50
50
  // Special case: data_ptr_T
51
- void WriteProperty(const char *tag, const_data_ptr_t ptr, idx_t count) {
52
- SetTag(tag);
51
+ void WriteProperty(const field_id_t field_id, const char *tag, const_data_ptr_t ptr, idx_t count) {
52
+ SetTag(field_id, tag);
53
53
  WriteDataPtr(ptr, count);
54
54
  }
55
55
 
56
56
  // Manually begin an object - should be followed by EndObject
57
- void BeginObject(const char *tag) {
58
- SetTag(tag);
57
+ void BeginObject(const field_id_t field_id, const char *tag) {
58
+ SetTag(field_id, tag);
59
59
  OnObjectBegin();
60
60
  }
61
61
 
@@ -196,7 +196,8 @@ protected:
196
196
  }
197
197
 
198
198
  // Handle setting a "tag" (optional)
199
- virtual void SetTag(const char *tag) {
199
+ virtual void SetTag(const field_id_t field_id, const char *tag) {
200
+ (void)field_id;
200
201
  (void)tag;
201
202
  }
202
203
 
@@ -1,5 +1,7 @@
1
1
  #pragma once
2
2
  #include <type_traits>
3
+ #include <cstdint>
4
+
3
5
  #include "duckdb/common/vector.hpp"
4
6
  #include "duckdb/common/unordered_map.hpp"
5
7
  #include "duckdb/common/unordered_set.hpp"
@@ -10,6 +12,8 @@ namespace duckdb {
10
12
  class FormatSerializer; // Forward declare
11
13
  class FormatDeserializer; // Forward declare
12
14
 
15
+ typedef uint32_t field_id_t;
16
+
13
17
  // Backport to c++11
14
18
  template <class...>
15
19
  using void_t = void;
@@ -50,6 +50,10 @@ public:
50
50
  RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
51
51
  bool flush = true);
52
52
 
53
+ // Single block scan
54
+ RowDataCollectionScanner(RowDataCollection &rows, RowDataCollection &heap, const RowLayout &layout, bool external,
55
+ idx_t block_idx, bool flush);
56
+
53
57
  //! The type layout of the payload
54
58
  inline const vector<LogicalType> &GetTypes() const {
55
59
  return layout.GetTypes();
@@ -93,7 +97,7 @@ private:
93
97
  //! Read state
94
98
  ScanState read_state;
95
99
  //! The total count of sorted_data
96
- const idx_t total_count;
100
+ idx_t total_count;
97
101
  //! The number of rows scanned so far
98
102
  idx_t total_scanned;
99
103
  //! Addresses used to gather from the sorted data
@@ -36,7 +36,6 @@ public:
36
36
  public:
37
37
  PhysicalOperator(PhysicalOperatorType type, vector<LogicalType> types, idx_t estimated_cardinality)
38
38
  : type(type), types(std::move(types)), estimated_cardinality(estimated_cardinality) {
39
- estimated_props = make_uniq<EstimatedProperties>(estimated_cardinality, 0);
40
39
  }
41
40
 
42
41
  virtual ~PhysicalOperator() {
@@ -50,7 +49,6 @@ public:
50
49
  vector<LogicalType> types;
51
50
  //! The estimated cardinality of this physical operator
52
51
  idx_t estimated_cardinality;
53
- unique_ptr<EstimatedProperties> estimated_props;
54
52
 
55
53
  //! The global sink state of this operator
56
54
  unique_ptr<GlobalSinkState> sink_state;
@@ -104,13 +104,13 @@ public:
104
104
  static void FormatSerialize(FormatSerializer &serializer, const FUNC &function,
105
105
  optional_ptr<FunctionData> bind_info) {
106
106
  D_ASSERT(!function.name.empty());
107
- serializer.WriteProperty("name", function.name);
108
- serializer.WriteProperty("arguments", function.arguments);
109
- serializer.WriteProperty("original_arguments", function.original_arguments);
107
+ serializer.WriteProperty(500, "name", function.name);
108
+ serializer.WriteProperty(501, "arguments", function.arguments);
109
+ serializer.WriteProperty(502, "original_arguments", function.original_arguments);
110
110
  bool has_serialize = function.format_serialize;
111
- serializer.WriteProperty("has_serialize", has_serialize);
111
+ serializer.WriteProperty(503, "has_serialize", has_serialize);
112
112
  if (has_serialize) {
113
- serializer.BeginObject("function_data");
113
+ serializer.BeginObject(504, "function_data");
114
114
  function.format_serialize(serializer, bind_info, function);
115
115
  serializer.EndObject();
116
116
  D_ASSERT(function.format_deserialize);
@@ -135,12 +135,12 @@ public:
135
135
  template <class FUNC, class CATALOG_ENTRY>
136
136
  static pair<FUNC, bool> FormatDeserializeBase(FormatDeserializer &deserializer, CatalogType catalog_type) {
137
137
  auto &context = deserializer.Get<ClientContext &>();
138
- auto name = deserializer.ReadProperty<string>("name");
139
- auto arguments = deserializer.ReadProperty<vector<LogicalType>>("arguments");
140
- auto original_arguments = deserializer.ReadProperty<vector<LogicalType>>("original_arguments");
138
+ auto name = deserializer.ReadProperty<string>(500, "name");
139
+ auto arguments = deserializer.ReadProperty<vector<LogicalType>>(501, "arguments");
140
+ auto original_arguments = deserializer.ReadProperty<vector<LogicalType>>(502, "original_arguments");
141
141
  auto function = DeserializeFunction<FUNC, CATALOG_ENTRY>(context, catalog_type, name, std::move(arguments),
142
142
  std::move(original_arguments));
143
- auto has_serialize = deserializer.ReadProperty<bool>("has_serialize");
143
+ auto has_serialize = deserializer.ReadProperty<bool>(503, "has_serialize");
144
144
  return make_pair(std::move(function), has_serialize);
145
145
  }
146
146
 
@@ -150,7 +150,7 @@ public:
150
150
  throw SerializationException("Function requires deserialization but no deserialization function for %s",
151
151
  function.name);
152
152
  }
153
- deserializer.BeginObject("function_data");
153
+ deserializer.BeginObject(504, "function_data");
154
154
  auto result = function.format_deserialize(deserializer, function);
155
155
  deserializer.EndObject();
156
156
  return result;
@@ -34,6 +34,8 @@ public:
34
34
  //! If this returns RESULT_READY, the Execute function can be called to obtain a pointer to the result.
35
35
  //! If this returns RESULT_NOT_READY, the ExecuteTask function should be called again.
36
36
  //! If this returns EXECUTION_ERROR, an error occurred during execution.
37
+ //! If this returns NO_TASKS_AVAILABLE, this means currently no meaningful work can be done by the current executor,
38
+ //! but tasks may become available in the future.
37
39
  //! The error message can be obtained by calling GetError() on the PendingQueryResult.
38
40
  DUCKDB_API PendingExecutionResult ExecuteTask();
39
41
 
@@ -43,6 +45,9 @@ public:
43
45
 
44
46
  DUCKDB_API void Close();
45
47
 
48
+ //! Function to determine whether execution is considered finished
49
+ DUCKDB_API static bool IsFinished(PendingExecutionResult result);
50
+
46
51
  private:
47
52
  shared_ptr<ClientContext> context;
48
53
  bool allow_stream_result;
@@ -10,17 +10,20 @@
10
10
 
11
11
  #include "duckdb/main/relation.hpp"
12
12
  #include "duckdb/parser/parsed_expression.hpp"
13
+ #include "duckdb/parser/group_by_node.hpp"
13
14
 
14
15
  namespace duckdb {
15
16
 
16
17
  class AggregateRelation : public Relation {
17
18
  public:
18
19
  DUCKDB_API AggregateRelation(shared_ptr<Relation> child, vector<unique_ptr<ParsedExpression>> expressions);
20
+ DUCKDB_API AggregateRelation(shared_ptr<Relation> child, vector<unique_ptr<ParsedExpression>> expressions,
21
+ GroupByNode groups);
19
22
  DUCKDB_API AggregateRelation(shared_ptr<Relation> child, vector<unique_ptr<ParsedExpression>> expressions,
20
23
  vector<unique_ptr<ParsedExpression>> groups);
21
24
 
22
25
  vector<unique_ptr<ParsedExpression>> expressions;
23
- vector<unique_ptr<ParsedExpression>> groups;
26
+ GroupByNode groups;
24
27
  vector<ColumnDefinition> columns;
25
28
  shared_ptr<Relation> child;
26
29
 
@@ -7,21 +7,14 @@
7
7
  //===----------------------------------------------------------------------===//
8
8
  #pragma once
9
9
 
10
- #include "duckdb/optimizer/join_order/join_node.hpp"
11
- #include "duckdb/planner/column_binding.hpp"
12
10
  #include "duckdb/planner/column_binding_map.hpp"
13
- #include "duckdb/planner/filter/conjunction_filter.hpp"
14
- #include "duckdb/planner/filter/constant_filter.hpp"
11
+ #include "duckdb/optimizer/join_order/query_graph.hpp"
12
+
13
+ #include "duckdb/optimizer/join_order/relation_statistics_helper.hpp"
15
14
 
16
15
  namespace duckdb {
17
16
 
18
- struct RelationAttributes {
19
- string original_name;
20
- // the relation columns used in join filters
21
- // Needed when iterating over columns and initializing total domain values.
22
- unordered_set<idx_t> columns;
23
- double cardinality;
24
- };
17
+ struct FilterInfo;
25
18
 
26
19
  struct RelationsToTDom {
27
20
  //! column binding sets that are equivalent in a join plan.
@@ -33,19 +26,13 @@ struct RelationsToTDom {
33
26
  idx_t tdom_no_hll;
34
27
  bool has_tdom_hll;
35
28
  vector<FilterInfo *> filters;
29
+ vector<string> column_names;
36
30
 
37
31
  RelationsToTDom(const column_binding_set_t &column_binding_set)
38
32
  : equivalent_relations(column_binding_set), tdom_hll(0), tdom_no_hll(NumericLimits<idx_t>::Maximum()),
39
33
  has_tdom_hll(false) {};
40
34
  };
41
35
 
42
- struct NodeOp {
43
- unique_ptr<JoinNode> node;
44
- LogicalOperator &op;
45
-
46
- NodeOp(unique_ptr<JoinNode> node, LogicalOperator &op) : node(std::move(node)), op(op) {};
47
- };
48
-
49
36
  struct Subgraph2Denominator {
50
37
  unordered_set<idx_t> relations;
51
38
  double denom;
@@ -53,69 +40,56 @@ struct Subgraph2Denominator {
53
40
  Subgraph2Denominator() : relations(), denom(1) {};
54
41
  };
55
42
 
56
- class CardinalityEstimator {
43
+ class CardinalityHelper {
57
44
  public:
58
- explicit CardinalityEstimator(ClientContext &context) : context(context) {
45
+ CardinalityHelper() {
59
46
  }
47
+ CardinalityHelper(double cardinality_before_filters, double filter_string)
48
+ : cardinality_before_filters(cardinality_before_filters), filter_strength(filter_string) {};
60
49
 
61
- private:
62
- ClientContext &context;
50
+ public:
51
+ double cardinality_before_filters;
52
+ double filter_strength;
53
+
54
+ vector<string> table_names_joined;
55
+ vector<string> column_names;
56
+ };
63
57
 
64
- //! A mapping of relation id -> RelationAttributes
65
- unordered_map<idx_t, RelationAttributes> relation_attributes;
66
- //! A mapping of (relation, bound_column) -> (actual table, actual column)
67
- column_binding_map_t<ColumnBinding> relation_column_to_original_column;
58
+ class CardinalityEstimator {
59
+ public:
60
+ explicit CardinalityEstimator() {};
68
61
 
62
+ private:
69
63
  vector<RelationsToTDom> relations_to_tdoms;
64
+ unordered_map<string, CardinalityHelper> relation_set_2_cardinality;
65
+ JoinRelationSetManager set_manager;
66
+ vector<RelationStats> relation_stats;
70
67
 
71
68
  public:
72
- static constexpr double DEFAULT_SELECTIVITY = 0.2;
73
-
74
- static void VerifySymmetry(JoinNode &result, JoinNode &entry);
75
-
76
- //! given a binding of (relation, column) used for DP, and a (table, column) in that catalog
77
- //! Add the key value entry into the relation_column_to_original_column
78
- void AddRelationToColumnMapping(ColumnBinding key, ColumnBinding value);
79
- //! Add a column to the relation_to_columns map.
80
- void AddColumnToRelationMap(idx_t table_index, idx_t column_index);
81
- //! Dump all bindings in relation_column_to_original_column into the child_binding_map
82
- // If you have a non-reorderable join, this function is used to keep track of bindings
83
- // in the child join plan.
84
- void CopyRelationMap(column_binding_map_t<ColumnBinding> &child_binding_map);
85
- void MergeBindings(idx_t, idx_t relation_id, vector<column_binding_map_t<ColumnBinding>> &child_binding_maps);
86
- void AddRelationColumnMapping(LogicalGet &get, idx_t relation_id);
87
-
88
- void InitTotalDomains();
89
- void UpdateTotalDomains(JoinNode &node, LogicalOperator &op);
90
- void InitEquivalentRelations(vector<unique_ptr<FilterInfo>> &filter_infos);
91
-
92
- void InitCardinalityEstimatorProps(vector<NodeOp> &node_ops, vector<unique_ptr<FilterInfo>> &filter_infos);
93
- double EstimateCardinalityWithSet(JoinRelationSet &new_set);
94
- void EstimateBaseTableCardinality(JoinNode &node, LogicalOperator &op);
95
- double EstimateCrossProduct(const JoinNode &left, const JoinNode &right);
96
- static double ComputeCost(JoinNode &left, JoinNode &right, double expected_cardinality);
69
+ void RemoveEmptyTotalDomains();
70
+ void UpdateTotalDomains(optional_ptr<JoinRelationSet> set, RelationStats &stats);
71
+ void InitEquivalentRelations(const vector<unique_ptr<FilterInfo>> &filter_infos);
72
+
73
+ void InitCardinalityEstimatorProps(optional_ptr<JoinRelationSet> set, RelationStats &stats);
74
+
75
+ //! cost model needs estimated cardinalities to the fraction since the formula captures
76
+ //! distinct count selectivities and multiplicities. Hence the template
77
+ template <class T>
78
+ T EstimateCardinalityWithSet(JoinRelationSet &new_set);
79
+
80
+ //! used for debugging.
81
+ void AddRelationNamesToTdoms(vector<RelationStats> &stats);
82
+ void PrintRelationToTdomInfo();
97
83
 
98
84
  private:
99
85
  bool SingleColumnFilter(FilterInfo &filter_info);
100
- //! Filter & bindings -> list of indexes into the equivalent_relations array.
101
- // The column binding set at each index is an equivalence set.
102
86
  vector<idx_t> DetermineMatchingEquivalentSets(FilterInfo *filter_info);
103
-
104
87
  //! Given a filter, add the column bindings to the matching equivalent set at the index
105
88
  //! given in matching equivalent sets.
106
89
  //! If there are multiple equivalence sets, they are merged.
107
90
  void AddToEquivalenceSets(FilterInfo *filter_info, vector<idx_t> matching_equivalent_sets);
108
-
109
- optional_ptr<TableFilterSet> GetTableFilters(LogicalOperator &op, idx_t table_index);
110
-
111
91
  void AddRelationTdom(FilterInfo &filter_info);
112
92
  bool EmptyFilter(FilterInfo &filter_info);
113
-
114
- idx_t InspectConjunctionAND(idx_t cardinality, idx_t column_index, ConjunctionAndFilter &fil,
115
- unique_ptr<BaseStatistics> base_stats);
116
- idx_t InspectConjunctionOR(idx_t cardinality, idx_t column_index, ConjunctionOrFilter &fil,
117
- unique_ptr<BaseStatistics> base_stats);
118
- idx_t InspectTableFilters(idx_t cardinality, LogicalOperator &op, TableFilterSet &table_filters, idx_t table_index);
119
93
  };
120
94
 
121
95
  } // namespace duckdb
@@ -0,0 +1,37 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/optimizer/join_order/cost_model.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+ #pragma once
9
+
10
+ #include "duckdb/optimizer/join_order/join_node.hpp"
11
+ #include "duckdb/optimizer/join_order/cardinality_estimator.hpp"
12
+
13
+ namespace duckdb {
14
+
15
+ class QueryGraphManager;
16
+
17
+ class CostModel {
18
+ public:
19
+ CostModel(QueryGraphManager &query_graph_manager);
20
+
21
+ private:
22
+ //! query graph storing relation manager information
23
+ QueryGraphManager &query_graph_manager;
24
+
25
+ public:
26
+ void InitCostModel();
27
+
28
+ //! Compute cost of a join relation set
29
+ double ComputeCost(JoinNode &left, JoinNode &right);
30
+
31
+ //! Cardinality Estimator used to calculate cost
32
+ CardinalityEstimator cardinality_estimator;
33
+
34
+ private:
35
+ };
36
+
37
+ } // namespace duckdb
@@ -5,62 +5,47 @@
5
5
  //
6
6
  //
7
7
  //===----------------------------------------------------------------------===//
8
-
9
8
  #pragma once
10
9
 
11
- #include "duckdb/common/unordered_map.hpp"
12
- #include "duckdb/common/unordered_set.hpp"
13
- #include "duckdb/optimizer/join_order/estimated_properties.hpp"
14
10
  #include "duckdb/optimizer/join_order/join_relation.hpp"
15
11
  #include "duckdb/optimizer/join_order/query_graph.hpp"
16
- #include "duckdb/parser/expression_map.hpp"
17
- #include "duckdb/planner/logical_operator_visitor.hpp"
18
- #include "duckdb/planner/table_filter.hpp"
19
- #include "duckdb/storage/statistics/distinct_statistics.hpp"
20
12
 
21
13
  namespace duckdb {
22
14
 
23
- class JoinOrderOptimizer;
15
+ struct NeighborInfo;
24
16
 
25
17
  class JoinNode {
26
18
  public:
27
19
  //! Represents a node in the join plan
28
20
  JoinRelationSet &set;
21
+ //! information on how left and right are connected
29
22
  optional_ptr<NeighborInfo> info;
30
- //! If the JoinNode is a base table, then base_cardinality is the cardinality before filters
31
- //! estimated_props.cardinality will be the cardinality after filters. With no filters, the two are equal
32
- bool has_filter;
23
+ //! left and right plans
33
24
  optional_ptr<JoinNode> left;
34
25
  optional_ptr<JoinNode> right;
35
26
 
36
- unique_ptr<EstimatedProperties> estimated_props;
27
+ //! The cost of the join node. The cost is stored here so that the cost of
28
+ //! a join node stays in sync with how the join node is constructed. Storing the cost in an unordered_set
29
+ //! in the cost model is error prone. If the plan enumerator join node is updated and not the cost model
30
+ //! the whole Join Order Optimizer can start exhibiting undesired behavior.
31
+ double cost;
32
+ //! used only to populate logical operators with estimated caridnalities after the best join plan has been found.
33
+ idx_t cardinality;
34
+
35
+ //! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality
36
+ JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right, double cost);
37
37
 
38
38
  //! Create a leaf node in the join tree
39
39
  //! set cost to 0 for leaf nodes
40
40
  //! cost will be the cost to *produce* an intermediate table
41
- JoinNode(JoinRelationSet &set, const double base_cardinality);
42
-
43
- //! Create an intermediate node in the join tree. base_cardinality = estimated_props.cardinality
44
- JoinNode(JoinRelationSet &set, optional_ptr<NeighborInfo> info, JoinNode &left, JoinNode &right,
45
- const double base_cardinality, double cost);
41
+ JoinNode(JoinRelationSet &set);
46
42
 
47
43
  bool operator==(const JoinNode &other) {
48
44
  return other.set.ToString().compare(set.ToString()) == 0;
49
45
  }
50
46
 
51
47
  private:
52
- double base_cardinality;
53
-
54
48
  public:
55
- template <class CARDINALITY_TYPE>
56
- CARDINALITY_TYPE GetCardinality() const {
57
- return estimated_props->GetCardinality<CARDINALITY_TYPE>();
58
- }
59
- double GetCost();
60
- void SetCost(double cost);
61
- double GetBaseTableCardinality();
62
- void SetBaseTableCardinality(double base_card);
63
- void SetEstimatedCardinality(double estimated_card);
64
49
  void PrintJoinNode();
65
50
  string ToString();
66
51
  };