duckdb 0.7.2-dev717.0 → 0.7.2-dev865.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/binding.gyp +2 -0
  2. package/lib/duckdb.d.ts +12 -1
  3. package/lib/duckdb.js +19 -0
  4. package/package.json +1 -1
  5. package/src/duckdb/extension/json/include/json_common.hpp +1 -0
  6. package/src/duckdb/extension/json/include/json_functions.hpp +1 -0
  7. package/src/duckdb/extension/json/include/json_serializer.hpp +77 -0
  8. package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +147 -0
  9. package/src/duckdb/extension/json/json_functions.cpp +1 -0
  10. package/src/duckdb/extension/json/json_scan.cpp +2 -2
  11. package/src/duckdb/extension/json/json_serializer.cpp +217 -0
  12. package/src/duckdb/src/catalog/catalog.cpp +21 -5
  13. package/src/duckdb/src/common/enums/expression_type.cpp +8 -222
  14. package/src/duckdb/src/common/enums/join_type.cpp +3 -22
  15. package/src/duckdb/src/common/exception.cpp +2 -2
  16. package/src/duckdb/src/common/serializer/enum_serializer.cpp +1172 -0
  17. package/src/duckdb/src/common/types/value.cpp +117 -93
  18. package/src/duckdb/src/common/types/vector.cpp +140 -1
  19. package/src/duckdb/src/common/types.cpp +166 -89
  20. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +3 -0
  21. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +5 -8
  22. package/src/duckdb/src/function/scalar/date/date_part.cpp +2 -2
  23. package/src/duckdb/src/function/scalar/date/date_trunc.cpp +2 -2
  24. package/src/duckdb/src/function/scalar/list/list_aggregates.cpp +1 -1
  25. package/src/duckdb/src/function/scalar/list/list_lambdas.cpp +4 -0
  26. package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +8 -8
  27. package/src/duckdb/src/function/scalar/string/regexp/regexp_extract_all.cpp +243 -0
  28. package/src/duckdb/src/function/scalar/string/regexp/regexp_util.cpp +79 -0
  29. package/src/duckdb/src/function/scalar/string/regexp.cpp +21 -80
  30. package/src/duckdb/src/function/table/arrow_conversion.cpp +7 -1
  31. package/src/duckdb/src/function/table/table_scan.cpp +1 -1
  32. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  33. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +3 -0
  34. package/src/duckdb/src/include/duckdb/common/enums/aggregate_handling.hpp +2 -0
  35. package/src/duckdb/src/include/duckdb/common/enums/expression_type.hpp +2 -3
  36. package/src/duckdb/src/include/duckdb/common/enums/joinref_type.hpp +2 -0
  37. package/src/duckdb/src/include/duckdb/common/enums/order_type.hpp +2 -0
  38. package/src/duckdb/src/include/duckdb/common/enums/set_operation_type.hpp +2 -1
  39. package/src/duckdb/src/include/duckdb/common/exception.hpp +40 -9
  40. package/src/duckdb/src/include/duckdb/common/preserved_error.hpp +3 -0
  41. package/src/duckdb/src/include/duckdb/common/serializer/enum_serializer.hpp +113 -0
  42. package/src/duckdb/src/include/duckdb/common/serializer/format_deserializer.hpp +336 -0
  43. package/src/duckdb/src/include/duckdb/common/serializer/format_serializer.hpp +268 -0
  44. package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +126 -0
  45. package/src/duckdb/src/include/duckdb/common/string_util.hpp +12 -0
  46. package/src/duckdb/src/include/duckdb/common/types/value.hpp +2 -31
  47. package/src/duckdb/src/include/duckdb/common/types/vector.hpp +3 -0
  48. package/src/duckdb/src/include/duckdb/common/types.hpp +8 -2
  49. package/src/duckdb/src/include/duckdb/function/scalar/regexp.hpp +81 -1
  50. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +1 -0
  51. package/src/duckdb/src/include/duckdb/parser/common_table_expression_info.hpp +2 -0
  52. package/src/duckdb/src/include/duckdb/parser/expression/between_expression.hpp +3 -0
  53. package/src/duckdb/src/include/duckdb/parser/expression/bound_expression.hpp +2 -0
  54. package/src/duckdb/src/include/duckdb/parser/expression/case_expression.hpp +5 -0
  55. package/src/duckdb/src/include/duckdb/parser/expression/cast_expression.hpp +2 -0
  56. package/src/duckdb/src/include/duckdb/parser/expression/collate_expression.hpp +2 -0
  57. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +2 -0
  58. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +2 -0
  59. package/src/duckdb/src/include/duckdb/parser/expression/conjunction_expression.hpp +2 -0
  60. package/src/duckdb/src/include/duckdb/parser/expression/constant_expression.hpp +3 -0
  61. package/src/duckdb/src/include/duckdb/parser/expression/default_expression.hpp +1 -0
  62. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +2 -0
  63. package/src/duckdb/src/include/duckdb/parser/expression/lambda_expression.hpp +2 -0
  64. package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +2 -0
  65. package/src/duckdb/src/include/duckdb/parser/expression/parameter_expression.hpp +2 -0
  66. package/src/duckdb/src/include/duckdb/parser/expression/positional_reference_expression.hpp +2 -0
  67. package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +2 -0
  68. package/src/duckdb/src/include/duckdb/parser/expression/subquery_expression.hpp +2 -0
  69. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +5 -0
  70. package/src/duckdb/src/include/duckdb/parser/parsed_data/sample_options.hpp +2 -0
  71. package/src/duckdb/src/include/duckdb/parser/parsed_expression.hpp +5 -0
  72. package/src/duckdb/src/include/duckdb/parser/query_node/recursive_cte_node.hpp +3 -0
  73. package/src/duckdb/src/include/duckdb/parser/query_node/select_node.hpp +5 -0
  74. package/src/duckdb/src/include/duckdb/parser/query_node/set_operation_node.hpp +3 -0
  75. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +11 -1
  76. package/src/duckdb/src/include/duckdb/parser/result_modifier.hpp +24 -1
  77. package/src/duckdb/src/include/duckdb/parser/sql_statement.hpp +2 -1
  78. package/src/duckdb/src/include/duckdb/parser/statement/select_statement.hpp +6 -1
  79. package/src/duckdb/src/include/duckdb/parser/tableref/basetableref.hpp +4 -0
  80. package/src/duckdb/src/include/duckdb/parser/tableref/emptytableref.hpp +2 -0
  81. package/src/duckdb/src/include/duckdb/parser/tableref/expressionlistref.hpp +3 -0
  82. package/src/duckdb/src/include/duckdb/parser/tableref/joinref.hpp +3 -0
  83. package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp +9 -0
  84. package/src/duckdb/src/include/duckdb/parser/tableref/subqueryref.hpp +3 -0
  85. package/src/duckdb/src/include/duckdb/parser/tableref/table_function_ref.hpp +3 -0
  86. package/src/duckdb/src/include/duckdb/parser/tableref.hpp +3 -1
  87. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +9 -52
  88. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp +62 -0
  89. package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +2 -1
  90. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -3
  91. package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp +3 -2
  92. package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +5 -3
  93. package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +4 -1
  94. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +6 -3
  95. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
  96. package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +37 -0
  97. package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +8 -1
  98. package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp +4 -3
  99. package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +271 -26
  100. package/src/duckdb/src/main/extension/extension_install.cpp +7 -2
  101. package/src/duckdb/src/optimizer/deliminator.cpp +1 -1
  102. package/src/duckdb/src/optimizer/filter_combiner.cpp +1 -1
  103. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +3 -3
  104. package/src/duckdb/src/optimizer/rule/move_constants.cpp +2 -2
  105. package/src/duckdb/src/optimizer/statistics/operator/propagate_filter.cpp +1 -1
  106. package/src/duckdb/src/parser/common_table_expression_info.cpp +19 -0
  107. package/src/duckdb/src/parser/expression/between_expression.cpp +17 -0
  108. package/src/duckdb/src/parser/expression/case_expression.cpp +28 -0
  109. package/src/duckdb/src/parser/expression/cast_expression.cpp +17 -0
  110. package/src/duckdb/src/parser/expression/collate_expression.cpp +16 -0
  111. package/src/duckdb/src/parser/expression/columnref_expression.cpp +15 -0
  112. package/src/duckdb/src/parser/expression/comparison_expression.cpp +16 -0
  113. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +15 -0
  114. package/src/duckdb/src/parser/expression/constant_expression.cpp +14 -0
  115. package/src/duckdb/src/parser/expression/default_expression.cpp +7 -0
  116. package/src/duckdb/src/parser/expression/function_expression.cpp +35 -0
  117. package/src/duckdb/src/parser/expression/lambda_expression.cpp +16 -0
  118. package/src/duckdb/src/parser/expression/operator_expression.cpp +15 -0
  119. package/src/duckdb/src/parser/expression/parameter_expression.cpp +15 -0
  120. package/src/duckdb/src/parser/expression/positional_reference_expression.cpp +14 -0
  121. package/src/duckdb/src/parser/expression/star_expression.cpp +20 -0
  122. package/src/duckdb/src/parser/expression/subquery_expression.cpp +20 -0
  123. package/src/duckdb/src/parser/expression/window_expression.cpp +43 -0
  124. package/src/duckdb/src/parser/parsed_data/sample_options.cpp +22 -10
  125. package/src/duckdb/src/parser/parsed_expression.cpp +72 -0
  126. package/src/duckdb/src/parser/query_node/recursive_cte_node.cpp +21 -0
  127. package/src/duckdb/src/parser/query_node/select_node.cpp +31 -0
  128. package/src/duckdb/src/parser/query_node/set_operation_node.cpp +17 -0
  129. package/src/duckdb/src/parser/query_node.cpp +50 -0
  130. package/src/duckdb/src/parser/result_modifier.cpp +78 -0
  131. package/src/duckdb/src/parser/statement/select_statement.cpp +12 -0
  132. package/src/duckdb/src/parser/tableref/basetableref.cpp +21 -0
  133. package/src/duckdb/src/parser/tableref/emptytableref.cpp +4 -0
  134. package/src/duckdb/src/parser/tableref/expressionlistref.cpp +17 -0
  135. package/src/duckdb/src/parser/tableref/joinref.cpp +25 -0
  136. package/src/duckdb/src/parser/tableref/pivotref.cpp +53 -0
  137. package/src/duckdb/src/parser/tableref/subqueryref.cpp +15 -0
  138. package/src/duckdb/src/parser/tableref/table_function.cpp +17 -0
  139. package/src/duckdb/src/parser/tableref.cpp +46 -0
  140. package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +11 -0
  141. package/src/duckdb/src/parser/transform/expression/transform_bool_expr.cpp +1 -1
  142. package/src/duckdb/src/parser/transform/expression/transform_operator.cpp +1 -1
  143. package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp +1 -1
  144. package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +22 -4
  145. package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +4 -0
  146. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +1 -1
  147. package/src/duckdb/src/planner/expression/bound_expression.cpp +4 -0
  148. package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -11
  149. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +6 -0
  150. package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -0
  151. package/src/duckdb/src/storage/compression/numeric_constant.cpp +2 -2
  152. package/src/duckdb/src/storage/data_table.cpp +1 -1
  153. package/src/duckdb/src/storage/statistics/numeric_stats.cpp +145 -83
  154. package/src/duckdb/src/storage/statistics/numeric_stats_union.cpp +65 -0
  155. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  156. package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +1 -6
  157. package/src/duckdb/src/storage/table/column_data.cpp +29 -35
  158. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +5 -5
  159. package/src/duckdb/src/storage/table/column_segment.cpp +8 -7
  160. package/src/duckdb/src/storage/table/list_column_data.cpp +2 -1
  161. package/src/duckdb/src/storage/table/persistent_table_data.cpp +2 -1
  162. package/src/duckdb/src/storage/table/row_group.cpp +9 -9
  163. package/src/duckdb/src/storage/table/row_group_collection.cpp +82 -66
  164. package/src/duckdb/src/storage/table/scan_state.cpp +22 -3
  165. package/src/duckdb/src/storage/table/standard_column_data.cpp +1 -0
  166. package/src/duckdb/src/storage/table/struct_column_data.cpp +1 -0
  167. package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +0 -1
  168. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +11780 -11512
  169. package/src/duckdb/third_party/re2/re2/re2.cc +9 -0
  170. package/src/duckdb/third_party/re2/re2/re2.h +2 -0
  171. package/src/duckdb/ub_extension_json_json_functions.cpp +2 -0
  172. package/src/duckdb/ub_src_common_serializer.cpp +2 -0
  173. package/src/duckdb/ub_src_function_scalar_string_regexp.cpp +4 -0
  174. package/src/duckdb/ub_src_parser.cpp +2 -0
  175. package/src/duckdb/ub_src_storage_statistics.cpp +2 -0
  176. package/src/duckdb/ub_src_storage_table.cpp +0 -2
  177. package/src/utils.cpp +12 -0
  178. package/test/extension.test.ts +44 -26
  179. package/src/duckdb/src/storage/table/segment_tree.cpp +0 -179
@@ -17,7 +17,10 @@
17
17
 
18
18
  namespace duckdb {
19
19
 
20
- enum QueryNodeType : uint8_t {
20
+ class FormatDeserializer;
21
+ class FormatSerializer;
22
+
23
+ enum class QueryNodeType : uint8_t {
21
24
  SELECT_NODE = 1,
22
25
  SET_OPERATION_NODE = 2,
23
26
  BOUND_SUBQUERY_NODE = 3,
@@ -35,6 +38,10 @@ public:
35
38
  public:
36
39
  string ToString() const;
37
40
  CommonTableExpressionMap Copy() const;
41
+
42
+ void FormatSerialize(FormatSerializer &serializer) const;
43
+ // static void FormatDeserialize(FormatDeserializer &deserializer, CommonTableExpressionMap &ret);
44
+ static CommonTableExpressionMap FormatDeserialize(FormatDeserializer &deserializer);
38
45
  };
39
46
 
40
47
  class QueryNode {
@@ -73,6 +80,9 @@ public:
73
80
  //! Adds a distinct modifier to the query node
74
81
  void AddDistinct();
75
82
 
83
+ virtual void FormatSerialize(FormatSerializer &serializer) const;
84
+ static unique_ptr<QueryNode> FormatDeserialize(FormatDeserializer &deserializer);
85
+
76
86
  protected:
77
87
  //! Copy base QueryNode properties from another expression to this one,
78
88
  //! used in Copy method
@@ -16,14 +16,19 @@
16
16
  namespace duckdb {
17
17
  class FieldWriter;
18
18
  class FieldReader;
19
+ class FormatDeserializer;
20
+ class FormatSerializer;
19
21
 
20
- enum ResultModifierType : uint8_t {
22
+ enum class ResultModifierType : uint8_t {
21
23
  LIMIT_MODIFIER = 1,
22
24
  ORDER_MODIFIER = 2,
23
25
  DISTINCT_MODIFIER = 3,
24
26
  LIMIT_PERCENT_MODIFIER = 4
25
27
  };
26
28
 
29
+ const char *ToString(ResultModifierType value);
30
+ ResultModifierType ResultModifierFromString(const char *value);
31
+
27
32
  //! A ResultModifier
28
33
  class ResultModifier {
29
34
  public:
@@ -46,6 +51,9 @@ public:
46
51
  virtual void Serialize(FieldWriter &writer) const = 0;
47
52
  //! Deserializes a blob back into a ResultModifier
48
53
  static unique_ptr<ResultModifier> Deserialize(Deserializer &source);
54
+
55
+ virtual void FormatSerialize(FormatSerializer &serializer) const;
56
+ static unique_ptr<ResultModifier> FormatDeserialize(FormatDeserializer &deserializer);
49
57
  };
50
58
 
51
59
  //! Single node in ORDER BY statement
@@ -65,6 +73,9 @@ public:
65
73
  void Serialize(Serializer &serializer) const;
66
74
  string ToString() const;
67
75
  static OrderByNode Deserialize(Deserializer &source);
76
+
77
+ void FormatSerialize(FormatSerializer &serializer) const;
78
+ static OrderByNode FormatDeserialize(FormatDeserializer &deserializer);
68
79
  };
69
80
 
70
81
  class LimitModifier : public ResultModifier {
@@ -82,6 +93,9 @@ public:
82
93
  unique_ptr<ResultModifier> Copy() const override;
83
94
  void Serialize(FieldWriter &writer) const override;
84
95
  static unique_ptr<ResultModifier> Deserialize(FieldReader &reader);
96
+
97
+ void FormatSerialize(FormatSerializer &serializer) const override;
98
+ static unique_ptr<ResultModifier> FormatDeserialize(FormatDeserializer &deserializer);
85
99
  };
86
100
 
87
101
  class OrderModifier : public ResultModifier {
@@ -97,6 +111,9 @@ public:
97
111
  unique_ptr<ResultModifier> Copy() const override;
98
112
  void Serialize(FieldWriter &writer) const override;
99
113
  static unique_ptr<ResultModifier> Deserialize(FieldReader &reader);
114
+
115
+ void FormatSerialize(FormatSerializer &serializer) const override;
116
+ static unique_ptr<ResultModifier> FormatDeserialize(FormatDeserializer &deserializer);
100
117
  };
101
118
 
102
119
  class DistinctModifier : public ResultModifier {
@@ -112,6 +129,9 @@ public:
112
129
  unique_ptr<ResultModifier> Copy() const override;
113
130
  void Serialize(FieldWriter &writer) const override;
114
131
  static unique_ptr<ResultModifier> Deserialize(FieldReader &reader);
132
+
133
+ void FormatSerialize(FormatSerializer &serializer) const override;
134
+ static unique_ptr<ResultModifier> FormatDeserialize(FormatDeserializer &deserializer);
115
135
  };
116
136
 
117
137
  class LimitPercentModifier : public ResultModifier {
@@ -129,6 +149,9 @@ public:
129
149
  unique_ptr<ResultModifier> Copy() const override;
130
150
  void Serialize(FieldWriter &writer) const override;
131
151
  static unique_ptr<ResultModifier> Deserialize(FieldReader &reader);
152
+
153
+ void FormatSerialize(FormatSerializer &serializer) const override;
154
+ static unique_ptr<ResultModifier> FormatDeserialize(FormatDeserializer &deserializer);
132
155
  };
133
156
 
134
157
  } // namespace duckdb
@@ -19,7 +19,8 @@ namespace duckdb {
19
19
  //! SQLStatement is the base class of any type of SQL statement.
20
20
  class SQLStatement {
21
21
  public:
22
- explicit SQLStatement(StatementType type) : type(type) {};
22
+ explicit SQLStatement(StatementType type) : type(type) {
23
+ }
23
24
  virtual ~SQLStatement() {
24
25
  }
25
26
 
@@ -10,13 +10,15 @@
10
10
 
11
11
  #include "duckdb/common/unordered_map.hpp"
12
12
  #include "duckdb/parser/parsed_expression.hpp"
13
- #include "duckdb/parser/query_node.hpp"
14
13
  #include "duckdb/parser/sql_statement.hpp"
15
14
  #include "duckdb/parser/tableref.hpp"
15
+ #include "duckdb/parser/query_node.hpp"
16
16
 
17
17
  namespace duckdb {
18
18
 
19
19
  class QueryNode;
20
+ class FormatSerializer;
21
+ class FormatDeserializer;
20
22
 
21
23
  //! SelectStatement is a typical SELECT clause
22
24
  class SelectStatement : public SQLStatement {
@@ -42,5 +44,8 @@ public:
42
44
  static unique_ptr<SelectStatement> Deserialize(Deserializer &source);
43
45
  //! Whether or not the statements are equivalent
44
46
  bool Equals(const SQLStatement *other) const;
47
+
48
+ void FormatSerialize(FormatSerializer &serializer) const;
49
+ static unique_ptr<SelectStatement> FormatDeserialize(FormatDeserializer &deserializer);
45
50
  };
46
51
  } // namespace duckdb
@@ -38,5 +38,9 @@ public:
38
38
  void Serialize(FieldWriter &serializer) const override;
39
39
  //! Deserializes a blob back into a BaseTableRef
40
40
  static unique_ptr<TableRef> Deserialize(FieldReader &source);
41
+
42
+ void FormatSerialize(FormatSerializer &serializer) const override;
43
+
44
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
41
45
  };
42
46
  } // namespace duckdb
@@ -27,5 +27,7 @@ public:
27
27
  void Serialize(FieldWriter &serializer) const override;
28
28
  //! Deserializes a blob back into a DummyTableRef
29
29
  static unique_ptr<TableRef> Deserialize(FieldReader &source);
30
+
31
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
30
32
  };
31
33
  } // namespace duckdb
@@ -37,5 +37,8 @@ public:
37
37
  void Serialize(FieldWriter &serializer) const override;
38
38
  //! Deserializes a blob back into a ExpressionListRef
39
39
  static unique_ptr<TableRef> Deserialize(FieldReader &source);
40
+
41
+ void FormatSerialize(FormatSerializer &serializer) const override;
42
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
40
43
  };
41
44
  } // namespace duckdb
@@ -47,5 +47,8 @@ public:
47
47
  void Serialize(FieldWriter &serializer) const override;
48
48
  //! Deserializes a blob back into a JoinRef
49
49
  static unique_ptr<TableRef> Deserialize(FieldReader &source);
50
+
51
+ void FormatSerialize(FormatSerializer &serializer) const override;
52
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
50
53
  };
51
54
  } // namespace duckdb
@@ -24,6 +24,9 @@ struct PivotColumnEntry {
24
24
  void Serialize(Serializer &serializer) const;
25
25
  PivotColumnEntry Copy() const;
26
26
  static PivotColumnEntry Deserialize(Deserializer &source);
27
+
28
+ void FormatSerialize(FormatSerializer &serializer) const;
29
+ static PivotColumnEntry FormatDeserialize(FormatDeserializer &source);
27
30
  };
28
31
 
29
32
  struct PivotColumn {
@@ -39,6 +42,9 @@ struct PivotColumn {
39
42
  void Serialize(Serializer &serializer) const;
40
43
  PivotColumn Copy() const;
41
44
  static PivotColumn Deserialize(Deserializer &source);
45
+
46
+ void FormatSerialize(FormatSerializer &serializer) const;
47
+ static PivotColumn FormatDeserialize(FormatDeserializer &source);
42
48
  };
43
49
 
44
50
  //! Represents a PIVOT or UNPIVOT expression
@@ -72,5 +78,8 @@ public:
72
78
  void Serialize(FieldWriter &serializer) const override;
73
79
  //! Deserializes a blob back into a JoinRef
74
80
  static unique_ptr<TableRef> Deserialize(FieldReader &source);
81
+
82
+ void FormatSerialize(FormatSerializer &serializer) const override;
83
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
75
84
  };
76
85
  } // namespace duckdb
@@ -32,5 +32,8 @@ public:
32
32
  void Serialize(FieldWriter &serializer) const override;
33
33
  //! Deserializes a blob back into a SubqueryRef
34
34
  static unique_ptr<TableRef> Deserialize(FieldReader &source);
35
+
36
+ void FormatSerialize(FormatSerializer &serializer) const override;
37
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
35
38
  };
36
39
  } // namespace duckdb
@@ -40,5 +40,8 @@ public:
40
40
  void Serialize(FieldWriter &serializer) const override;
41
41
  //! Deserializes a blob back into a BaseTableRef
42
42
  static unique_ptr<TableRef> Deserialize(FieldReader &source);
43
+
44
+ void FormatSerialize(FormatSerializer &serializer) const override;
45
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &source);
43
46
  };
44
47
  } // namespace duckdb
@@ -48,8 +48,10 @@ public:
48
48
  DUCKDB_API virtual void Serialize(FieldWriter &writer) const = 0;
49
49
  //! Deserializes a blob back into a TableRef
50
50
  DUCKDB_API static unique_ptr<TableRef> Deserialize(Deserializer &source);
51
-
52
51
  //! Copy the properties of this table ref to the target
53
52
  void CopyProperties(TableRef &target) const;
53
+
54
+ virtual void FormatSerialize(FormatSerializer &serializer) const;
55
+ static unique_ptr<TableRef> FormatDeserialize(FormatDeserializer &deserializer);
54
56
  };
55
57
  } // namespace duckdb
@@ -8,9 +8,7 @@
8
8
 
9
9
  #pragma once
10
10
 
11
- #include "duckdb/common/common.hpp"
12
- #include "duckdb/common/exception.hpp"
13
- #include "duckdb/common/types/hugeint.hpp"
11
+ #include "duckdb/storage/statistics/numeric_stats_union.hpp"
14
12
  #include "duckdb/common/enums/filter_propagate_result.hpp"
15
13
  #include "duckdb/common/enums/expression_type.hpp"
16
14
  #include "duckdb/common/operator/comparison_operators.hpp"
@@ -23,28 +21,6 @@ class FieldReader;
23
21
  struct SelectionVector;
24
22
  class Vector;
25
23
 
26
- struct NumericValueUnion {
27
- union Val {
28
- int8_t boolean;
29
- int8_t tinyint;
30
- int16_t smallint;
31
- int32_t integer;
32
- int64_t bigint;
33
- uint8_t utinyint;
34
- uint16_t usmallint;
35
- uint32_t uinteger;
36
- uint64_t ubigint;
37
- hugeint_t hugeint;
38
- float float_;
39
- double double_;
40
- } value_;
41
-
42
- template <class T>
43
- T &GetReferenceUnsafe() {
44
- throw InternalException("NumericValueUnion::GetReferenceUnsafe called on unsupported type");
45
- }
46
- };
47
-
48
24
  struct NumericStatsData {
49
25
  //! Whether or not the value has a max value
50
26
  bool has_min;
@@ -109,13 +85,17 @@ struct NumericStats {
109
85
  static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
110
86
 
111
87
  template <class T>
112
- static T GetMinUnsafe(const BaseStatistics &stats) {
113
- return NumericStats::Min(stats).template GetValueUnsafe<T>();
88
+ static T GetMin(const BaseStatistics &stats) {
89
+ return NumericStats::Min(stats).GetValueUnsafe<T>();
114
90
  }
115
91
  template <class T>
116
- static T GetMaxUnsafe(const BaseStatistics &stats) {
117
- return NumericStats::Max(stats).template GetValueUnsafe<T>();
92
+ static T GetMax(const BaseStatistics &stats) {
93
+ return NumericStats::Max(stats).GetValueUnsafe<T>();
118
94
  }
95
+ template <class T>
96
+ static T GetMinUnsafe(const BaseStatistics &stats);
97
+ template <class T>
98
+ static T GetMaxUnsafe(const BaseStatistics &stats);
119
99
 
120
100
  private:
121
101
  static NumericStatsData &GetDataUnsafe(BaseStatistics &stats);
@@ -131,27 +111,4 @@ void NumericStats::Update<interval_t>(BaseStatistics &stats, interval_t new_valu
131
111
  template <>
132
112
  void NumericStats::Update<list_entry_t>(BaseStatistics &stats, list_entry_t new_value);
133
113
 
134
- template <>
135
- int8_t &NumericValueUnion::GetReferenceUnsafe();
136
- template <>
137
- int16_t &NumericValueUnion::GetReferenceUnsafe();
138
- template <>
139
- int32_t &NumericValueUnion::GetReferenceUnsafe();
140
- template <>
141
- int64_t &NumericValueUnion::GetReferenceUnsafe();
142
- template <>
143
- hugeint_t &NumericValueUnion::GetReferenceUnsafe();
144
- template <>
145
- uint8_t &NumericValueUnion::GetReferenceUnsafe();
146
- template <>
147
- uint16_t &NumericValueUnion::GetReferenceUnsafe();
148
- template <>
149
- uint32_t &NumericValueUnion::GetReferenceUnsafe();
150
- template <>
151
- uint64_t &NumericValueUnion::GetReferenceUnsafe();
152
- template <>
153
- float &NumericValueUnion::GetReferenceUnsafe();
154
- template <>
155
- double &NumericValueUnion::GetReferenceUnsafe();
156
-
157
114
  } // namespace duckdb
@@ -0,0 +1,62 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/storage/statistics/numeric_stats_union.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/common/common.hpp"
12
+ #include "duckdb/common/exception.hpp"
13
+ #include "duckdb/common/types/hugeint.hpp"
14
+
15
+ namespace duckdb {
16
+
17
+ struct NumericValueUnion {
18
+ union Val {
19
+ bool boolean;
20
+ int8_t tinyint;
21
+ int16_t smallint;
22
+ int32_t integer;
23
+ int64_t bigint;
24
+ uint8_t utinyint;
25
+ uint16_t usmallint;
26
+ uint32_t uinteger;
27
+ uint64_t ubigint;
28
+ hugeint_t hugeint;
29
+ float float_;
30
+ double double_;
31
+ } value_;
32
+
33
+ template <class T>
34
+ T &GetReferenceUnsafe();
35
+ };
36
+
37
+ template <>
38
+ DUCKDB_API bool &NumericValueUnion::GetReferenceUnsafe();
39
+ template <>
40
+ DUCKDB_API int8_t &NumericValueUnion::GetReferenceUnsafe();
41
+ template <>
42
+ DUCKDB_API int16_t &NumericValueUnion::GetReferenceUnsafe();
43
+ template <>
44
+ DUCKDB_API int32_t &NumericValueUnion::GetReferenceUnsafe();
45
+ template <>
46
+ DUCKDB_API int64_t &NumericValueUnion::GetReferenceUnsafe();
47
+ template <>
48
+ DUCKDB_API hugeint_t &NumericValueUnion::GetReferenceUnsafe();
49
+ template <>
50
+ DUCKDB_API uint8_t &NumericValueUnion::GetReferenceUnsafe();
51
+ template <>
52
+ DUCKDB_API uint16_t &NumericValueUnion::GetReferenceUnsafe();
53
+ template <>
54
+ DUCKDB_API uint32_t &NumericValueUnion::GetReferenceUnsafe();
55
+ template <>
56
+ DUCKDB_API uint64_t &NumericValueUnion::GetReferenceUnsafe();
57
+ template <>
58
+ DUCKDB_API float &NumericValueUnion::GetReferenceUnsafe();
59
+ template <>
60
+ DUCKDB_API double &NumericValueUnion::GetReferenceUnsafe();
61
+
62
+ } // namespace duckdb
@@ -13,6 +13,7 @@
13
13
  #include "duckdb/storage/data_pointer.hpp"
14
14
  #include "duckdb/storage/statistics/segment_statistics.hpp"
15
15
  #include "duckdb/storage/table/column_segment.hpp"
16
+ #include "duckdb/storage/table/column_data.hpp"
16
17
  #include "duckdb/common/unordered_set.hpp"
17
18
 
18
19
  namespace duckdb {
@@ -28,7 +29,7 @@ struct ColumnCheckpointState {
28
29
 
29
30
  RowGroup &row_group;
30
31
  ColumnData &column_data;
31
- SegmentTree new_tree;
32
+ ColumnSegmentTree new_tree;
32
33
  vector<DataPointer> data_pointers;
33
34
  unique_ptr<BaseStatistics> global_stats;
34
35
 
@@ -15,7 +15,8 @@
15
15
  #include "duckdb/storage/data_pointer.hpp"
16
16
  #include "duckdb/storage/table/persistent_table_data.hpp"
17
17
  #include "duckdb/storage/statistics/segment_statistics.hpp"
18
- #include "duckdb/storage/table/column_checkpoint_state.hpp"
18
+ #include "duckdb/storage/table/segment_tree.hpp"
19
+ #include "duckdb/storage/table/column_segment.hpp"
19
20
  #include "duckdb/common/mutex.hpp"
20
21
 
21
22
  namespace duckdb {
@@ -31,10 +32,12 @@ struct TransactionData;
31
32
  struct DataTableInfo;
32
33
 
33
34
  struct ColumnCheckpointInfo {
34
- ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
35
+ explicit ColumnCheckpointInfo(CompressionType compression_type_p) : compression_type(compression_type_p) {};
35
36
  CompressionType compression_type;
36
37
  };
37
38
 
39
+ class ColumnSegmentTree : public SegmentTree<ColumnSegment> {};
40
+
38
41
  class ColumnData {
39
42
  friend class ColumnDataCheckpointer;
40
43
 
@@ -148,7 +151,7 @@ protected:
148
151
 
149
152
  protected:
150
153
  //! The segments holding the data of this column segment
151
- SegmentTree data;
154
+ ColumnSegmentTree data;
152
155
  //! The lock for the updates
153
156
  mutex update_lock;
154
157
  //! The updates for this column segment
@@ -10,6 +10,7 @@
10
10
 
11
11
  #include "duckdb/storage/table/column_data.hpp"
12
12
  #include "duckdb/function/compression_function.hpp"
13
+ #include "duckdb/storage/table/column_checkpoint_state.hpp"
13
14
 
14
15
  namespace duckdb {
15
16
 
@@ -25,7 +26,7 @@ public:
25
26
  RowGroup &GetRowGroup();
26
27
  ColumnCheckpointState &GetCheckpointState();
27
28
 
28
- void Checkpoint(vector<SegmentNode> nodes);
29
+ void Checkpoint(vector<SegmentNode<ColumnSegment>> nodes);
29
30
 
30
31
  private:
31
32
  void ScanSegments(const std::function<void(Vector &, idx_t)> &callback);
@@ -40,7 +41,7 @@ private:
40
41
  ColumnCheckpointState &state;
41
42
  bool is_validity;
42
43
  Vector intermediate;
43
- vector<SegmentNode> nodes;
44
+ vector<SegmentNode<ColumnSegment>> nodes;
44
45
  vector<CompressionFunction *> compression_functions;
45
46
  ColumnCheckpointInfo &checkpoint_info;
46
47
  };
@@ -9,7 +9,6 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/storage/block.hpp"
12
- #include "duckdb/storage/table/segment_tree.hpp"
13
12
  #include "duckdb/common/types.hpp"
14
13
  #include "duckdb/common/types/vector.hpp"
15
14
  #include "duckdb/storage/buffer_manager.hpp"
@@ -17,6 +16,7 @@
17
16
  #include "duckdb/storage/storage_lock.hpp"
18
17
  #include "duckdb/storage/table/scan_state.hpp"
19
18
  #include "duckdb/function/compression_function.hpp"
19
+ #include "duckdb/storage/table/segment_base.hpp"
20
20
 
21
21
  namespace duckdb {
22
22
  class ColumnSegment;
@@ -35,10 +35,12 @@ struct ColumnAppendState;
35
35
  enum class ColumnSegmentType : uint8_t { TRANSIENT, PERSISTENT };
36
36
  //! TableFilter represents a filter pushed down into the table scan.
37
37
 
38
- class ColumnSegment : public SegmentBase {
38
+ class ColumnSegment : public SegmentBase<ColumnSegment> {
39
39
  public:
40
- ~ColumnSegment() override;
40
+ ~ColumnSegment();
41
41
 
42
+ //! The index within the segment tree
43
+ idx_t index;
42
44
  //! The database instance
43
45
  DatabaseInstance &db;
44
46
  //! The type stored in the column
@@ -22,8 +22,11 @@ public:
22
22
  explicit PersistentTableData(idx_t column_count);
23
23
  ~PersistentTableData();
24
24
 
25
- vector<RowGroupPointer> row_groups;
26
25
  TableStatistics table_stats;
26
+ idx_t total_rows;
27
+ idx_t row_group_count;
28
+ block_id_t block_id;
29
+ idx_t offset;
27
30
  };
28
31
 
29
32
  } // namespace duckdb
@@ -9,7 +9,6 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/common/vector_size.hpp"
12
- #include "duckdb/storage/table/segment_base.hpp"
13
12
  #include "duckdb/storage/table/chunk_info.hpp"
14
13
  #include "duckdb/storage/table/append_state.hpp"
15
14
  #include "duckdb/storage/table/scan_state.hpp"
@@ -17,6 +16,7 @@
17
16
  #include "duckdb/common/enums/scan_options.hpp"
18
17
  #include "duckdb/common/mutex.hpp"
19
18
  #include "duckdb/parser/column_list.hpp"
19
+ #include "duckdb/storage/table/segment_base.hpp"
20
20
 
21
21
  namespace duckdb {
22
22
  class AttachedDatabase;
@@ -42,7 +42,7 @@ struct RowGroupWriteData {
42
42
  vector<BaseStatistics> statistics;
43
43
  };
44
44
 
45
- class RowGroup : public SegmentBase {
45
+ class RowGroup : public SegmentBase<RowGroup> {
46
46
  public:
47
47
  friend class ColumnData;
48
48
  friend class VersionDeleteState;
@@ -58,6 +58,9 @@ public:
58
58
  RowGroup(RowGroup &row_group, idx_t start);
59
59
  ~RowGroup();
60
60
 
61
+ //! The index within the segment tree
62
+ idx_t index;
63
+
61
64
  private:
62
65
  //! The database instance
63
66
  AttachedDatabase &db;
@@ -135,7 +138,7 @@ public:
135
138
  RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
136
139
  RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
137
140
  static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
138
- static RowGroupPointer Deserialize(Deserializer &source, const ColumnList &columns);
141
+ static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
139
142
 
140
143
  void InitializeAppend(RowGroupAppendState &append_state);
141
144
  void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
@@ -23,6 +23,8 @@ class TableStatistics;
23
23
 
24
24
  class BoundConstraint;
25
25
 
26
+ class RowGroupSegmentTree;
27
+
26
28
  class RowGroupCollection {
27
29
  public:
28
30
  RowGroupCollection(shared_ptr<DataTableInfo> info, BlockManager &block_manager, vector<LogicalType> types,
@@ -46,8 +48,8 @@ public:
46
48
  void InitializeCreateIndexScan(CreateIndexScanState &state);
47
49
  void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
48
50
  idx_t end_row);
49
- static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroup *row_group, idx_t vector_index,
50
- idx_t max_row);
51
+ static bool InitializeScanInRowGroup(CollectionScanState &state, ParallelCollectionScanState &parallel_state,
52
+ idx_t vector_index, idx_t max_row);
51
53
  void InitializeParallelScan(ParallelCollectionScanState &state);
52
54
  bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
53
55
 
@@ -109,7 +111,7 @@ private:
109
111
  vector<LogicalType> types;
110
112
  idx_t row_start;
111
113
  //! The segment trees holding the various row_groups of the table
112
- shared_ptr<SegmentTree> row_groups;
114
+ shared_ptr<RowGroupSegmentTree> row_groups;
113
115
  //! Table statistics
114
116
  TableStatistics stats;
115
117
  };
@@ -0,0 +1,37 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/storage/table/row_group_segment_tree.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/storage/table/segment_tree.hpp"
12
+ #include "duckdb/storage/table/row_group.hpp"
13
+
14
+ namespace duckdb {
15
+ struct DataTableInfo;
16
+ class PersistentTableData;
17
+ class MetaBlockReader;
18
+
19
+ class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
20
+ public:
21
+ RowGroupSegmentTree(DataTableInfo &table_info_p, BlockManager &block_manager_p, vector<LogicalType> column_types_p);
22
+ ~RowGroupSegmentTree() override;
23
+
24
+ void Initialize(PersistentTableData &data);
25
+
26
+ protected:
27
+ unique_ptr<RowGroup> LoadSegment() override;
28
+
29
+ DataTableInfo &info;
30
+ BlockManager &block_manager;
31
+ vector<LogicalType> column_types;
32
+ idx_t current_row_group;
33
+ idx_t max_row_group;
34
+ unique_ptr<MetaBlockReader> reader;
35
+ };
36
+
37
+ } // namespace duckdb