duckdb 0.7.2-dev832.0 → 0.7.2-dev886.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/src/catalog/catalog.cpp +21 -5
  3. package/src/duckdb/src/catalog/default/default_functions.cpp +3 -0
  4. package/src/duckdb/src/catalog/duck_catalog.cpp +34 -7
  5. package/src/duckdb/src/common/box_renderer.cpp +109 -23
  6. package/src/duckdb/src/common/types/value.cpp +0 -93
  7. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +3 -0
  8. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +5 -8
  9. package/src/duckdb/src/function/scalar/date/date_part.cpp +2 -2
  10. package/src/duckdb/src/function/scalar/date/date_trunc.cpp +2 -2
  11. package/src/duckdb/src/function/scalar/list/list_aggregates.cpp +1 -1
  12. package/src/duckdb/src/function/scalar/list/list_lambdas.cpp +4 -0
  13. package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +8 -8
  14. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  15. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +3 -0
  16. package/src/duckdb/src/include/duckdb/catalog/duck_catalog.hpp +2 -1
  17. package/src/duckdb/src/include/duckdb/common/box_renderer.hpp +8 -2
  18. package/src/duckdb/src/include/duckdb/common/types/value.hpp +0 -31
  19. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +2 -0
  20. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +9 -52
  21. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp +62 -0
  22. package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +2 -1
  23. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -3
  24. package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp +3 -2
  25. package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +5 -3
  26. package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +4 -1
  27. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +6 -3
  28. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
  29. package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +37 -0
  30. package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +8 -1
  31. package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp +4 -3
  32. package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +271 -26
  33. package/src/duckdb/src/optimizer/filter_pushdown.cpp +11 -7
  34. package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp +1 -10
  35. package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp +1 -1
  36. package/src/duckdb/src/optimizer/pushdown/pushdown_single_join.cpp +1 -1
  37. package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +11 -0
  38. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +31 -6
  39. package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +22 -4
  40. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +1 -1
  41. package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -11
  42. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +6 -0
  43. package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -0
  44. package/src/duckdb/src/storage/compression/numeric_constant.cpp +2 -2
  45. package/src/duckdb/src/storage/data_table.cpp +1 -1
  46. package/src/duckdb/src/storage/statistics/numeric_stats.cpp +145 -83
  47. package/src/duckdb/src/storage/statistics/numeric_stats_union.cpp +65 -0
  48. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  49. package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +1 -6
  50. package/src/duckdb/src/storage/table/column_data.cpp +29 -35
  51. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +5 -5
  52. package/src/duckdb/src/storage/table/column_segment.cpp +8 -7
  53. package/src/duckdb/src/storage/table/list_column_data.cpp +2 -1
  54. package/src/duckdb/src/storage/table/persistent_table_data.cpp +2 -1
  55. package/src/duckdb/src/storage/table/row_group.cpp +9 -9
  56. package/src/duckdb/src/storage/table/row_group_collection.cpp +82 -66
  57. package/src/duckdb/src/storage/table/scan_state.cpp +22 -3
  58. package/src/duckdb/src/storage/table/standard_column_data.cpp +1 -0
  59. package/src/duckdb/src/storage/table/struct_column_data.cpp +1 -0
  60. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +11578 -11222
  61. package/src/duckdb/ub_src_storage_statistics.cpp +2 -0
  62. package/src/duckdb/ub_src_storage_table.cpp +0 -2
  63. package/src/duckdb/src/storage/table/segment_tree.cpp +0 -179
@@ -10,71 +10,316 @@
10
10
 
11
11
  #include "duckdb/common/constants.hpp"
12
12
  #include "duckdb/storage/storage_lock.hpp"
13
- #include "duckdb/storage/table/segment_base.hpp"
14
13
  #include "duckdb/storage/table/segment_lock.hpp"
15
14
  #include "duckdb/common/vector.hpp"
16
15
  #include "duckdb/common/mutex.hpp"
16
+ #include "duckdb/common/string_util.hpp"
17
17
 
18
18
  namespace duckdb {
19
19
 
20
+ template <class T>
20
21
  struct SegmentNode {
21
22
  idx_t row_start;
22
- unique_ptr<SegmentBase> node;
23
+ unique_ptr<T> node;
23
24
  };
24
25
 
25
26
  //! The SegmentTree maintains a list of all segments of a specific column in a table, and allows searching for a segment
26
27
  //! by row number
28
+ template <class T, bool SUPPORTS_LAZY_LOADING = false>
27
29
  class SegmentTree {
30
+ private:
31
+ class SegmentIterationHelper;
32
+
28
33
  public:
34
+ explicit SegmentTree() : finished_loading(true) {
35
+ }
36
+ virtual ~SegmentTree() {
37
+ }
38
+
29
39
  //! Locks the segment tree. All methods to the segment tree either lock the segment tree, or take an already
30
40
  //! obtained lock.
31
- SegmentLock Lock();
41
+ SegmentLock Lock() {
42
+ return SegmentLock(node_lock);
43
+ }
32
44
 
33
- bool IsEmpty(SegmentLock &);
45
+ bool IsEmpty(SegmentLock &l) {
46
+ return GetRootSegment(l) == nullptr;
47
+ }
34
48
 
35
49
  //! Gets a pointer to the first segment. Useful for scans.
36
- SegmentBase *GetRootSegment();
37
- SegmentBase *GetRootSegment(SegmentLock &);
50
+ T *GetRootSegment() {
51
+ auto l = Lock();
52
+ return GetRootSegment(l);
53
+ }
54
+
55
+ T *GetRootSegment(SegmentLock &l) {
56
+ if (nodes.empty()) {
57
+ LoadNextSegment(l);
58
+ }
59
+ return nodes.empty() ? nullptr : nodes[0].node.get();
60
+ }
38
61
  //! Obtains ownership of the data of the segment tree
39
- vector<SegmentNode> MoveSegments(SegmentLock &);
62
+ vector<SegmentNode<T>> MoveSegments(SegmentLock &l) {
63
+ LoadAllSegments(l);
64
+ return std::move(nodes);
65
+ }
40
66
  //! Gets a pointer to the nth segment. Negative numbers start from the back.
41
- SegmentBase *GetSegmentByIndex(int64_t index);
42
- SegmentBase *GetSegmentByIndex(SegmentLock &, int64_t index);
67
+ T *GetSegmentByIndex(int64_t index) {
68
+ auto l = Lock();
69
+ return GetSegmentByIndex(l, index);
70
+ }
71
+ T *GetSegmentByIndex(SegmentLock &l, int64_t index) {
72
+ if (index < 0) {
73
+ // load all segments
74
+ LoadAllSegments(l);
75
+ index = nodes.size() + index;
76
+ if (index < 0) {
77
+ return nullptr;
78
+ }
79
+ return nodes[index].node.get();
80
+ } else {
81
+ // lazily load segments until we reach the specific segment
82
+ while (idx_t(index) >= nodes.size() && LoadNextSegment(l)) {
83
+ }
84
+ if (idx_t(index) >= nodes.size()) {
85
+ return nullptr;
86
+ }
87
+ return nodes[index].node.get();
88
+ }
89
+ }
90
+ //! Gets the next segment
91
+ T *GetNextSegment(T *segment) {
92
+ if (!SUPPORTS_LAZY_LOADING) {
93
+ return segment->Next();
94
+ }
95
+ if (finished_loading) {
96
+ return segment->Next();
97
+ }
98
+ auto l = Lock();
99
+ return GetNextSegment(l, segment);
100
+ }
101
+ T *GetNextSegment(SegmentLock &l, T *segment) {
102
+ if (!segment) {
103
+ return nullptr;
104
+ }
105
+ #ifdef DEBUG
106
+ D_ASSERT(nodes[segment->index].node.get() == segment);
107
+ #endif
108
+ return GetSegmentByIndex(l, segment->index + 1);
109
+ }
43
110
 
44
111
  //! Gets a pointer to the last segment. Useful for appends.
45
- SegmentBase *GetLastSegment();
46
- SegmentBase *GetLastSegment(SegmentLock &);
112
+ T *GetLastSegment(SegmentLock &l) {
113
+ LoadAllSegments(l);
114
+ if (nodes.empty()) {
115
+ return nullptr;
116
+ }
117
+ return nodes.back().node.get();
118
+ }
47
119
  //! Gets a pointer to a specific column segment for the given row
48
- SegmentBase *GetSegment(idx_t row_number);
49
- SegmentBase *GetSegment(SegmentLock &, idx_t row_number);
120
+ T *GetSegment(idx_t row_number) {
121
+ auto l = Lock();
122
+ return GetSegment(l, row_number);
123
+ }
124
+ T *GetSegment(SegmentLock &l, idx_t row_number) {
125
+ return nodes[GetSegmentIndex(l, row_number)].node.get();
126
+ }
50
127
 
51
128
  //! Append a column segment to the tree
52
- void AppendSegment(unique_ptr<SegmentBase> segment);
53
- void AppendSegment(SegmentLock &, unique_ptr<SegmentBase> segment);
129
+ void AppendSegmentInternal(SegmentLock &l, unique_ptr<T> segment) {
130
+ D_ASSERT(segment);
131
+ // add the node to the list of nodes
132
+ if (!nodes.empty()) {
133
+ nodes.back().node->next = segment.get();
134
+ }
135
+ SegmentNode<T> node;
136
+ segment->index = nodes.size();
137
+ node.row_start = segment->start;
138
+ node.node = std::move(segment);
139
+ nodes.push_back(std::move(node));
140
+ }
141
+ void AppendSegment(unique_ptr<T> segment) {
142
+ auto l = Lock();
143
+ AppendSegment(l, std::move(segment));
144
+ }
145
+ void AppendSegment(SegmentLock &l, unique_ptr<T> segment) {
146
+ LoadAllSegments(l);
147
+ AppendSegmentInternal(l, std::move(segment));
148
+ }
54
149
  //! Debug method, check whether the segment is in the segment tree
55
- bool HasSegment(SegmentBase *segment);
56
- bool HasSegment(SegmentLock &, SegmentBase *segment);
150
+ bool HasSegment(T *segment) {
151
+ auto l = Lock();
152
+ return HasSegment(l, segment);
153
+ }
154
+ bool HasSegment(SegmentLock &, T *segment) {
155
+ return segment->index < nodes.size() && nodes[segment->index].node.get() == segment;
156
+ }
57
157
 
58
158
  //! Replace this tree with another tree, taking over its nodes in-place
59
- void Replace(SegmentTree &other);
60
- void Replace(SegmentLock &, SegmentTree &other);
159
+ void Replace(SegmentTree<T> &other) {
160
+ auto l = Lock();
161
+ Replace(l, other);
162
+ }
163
+ void Replace(SegmentLock &l, SegmentTree<T> &other) {
164
+ other.LoadAllSegments(l);
165
+ nodes = std::move(other.nodes);
166
+ }
61
167
 
62
168
  //! Erase all segments after a specific segment
63
- void EraseSegments(SegmentLock &, idx_t segment_start);
169
+ void EraseSegments(SegmentLock &l, idx_t segment_start) {
170
+ LoadAllSegments(l);
171
+ if (segment_start >= nodes.size() - 1) {
172
+ return;
173
+ }
174
+ nodes.erase(nodes.begin() + segment_start + 1, nodes.end());
175
+ }
64
176
 
65
177
  //! Get the segment index of the column segment for the given row
66
- idx_t GetSegmentIndex(idx_t row_number);
67
- idx_t GetSegmentIndex(SegmentLock &, idx_t row_number);
68
- bool TryGetSegmentIndex(SegmentLock &, idx_t row_number, idx_t &);
178
+ idx_t GetSegmentIndex(SegmentLock &l, idx_t row_number) {
179
+ idx_t segment_index;
180
+ if (TryGetSegmentIndex(l, row_number, segment_index)) {
181
+ return segment_index;
182
+ }
183
+ string error;
184
+ error = StringUtil::Format("Attempting to find row number \"%lld\" in %lld nodes\n", row_number, nodes.size());
185
+ for (idx_t i = 0; i < nodes.size(); i++) {
186
+ error += StringUtil::Format("Node %lld: Start %lld, Count %lld", i, nodes[i].row_start,
187
+ nodes[i].node->count.load());
188
+ }
189
+ throw InternalException("Could not find node in column segment tree!\n%s%s", error, Exception::GetStackTrace());
190
+ }
191
+
192
+ bool TryGetSegmentIndex(SegmentLock &l, idx_t row_number, idx_t &result) {
193
+ // load segments until the row number is within bounds
194
+ while (nodes.empty() || (row_number >= (nodes.back().row_start + nodes.back().node->count))) {
195
+ if (!LoadNextSegment(l)) {
196
+ break;
197
+ }
198
+ }
199
+ if (nodes.empty()) {
200
+ return false;
201
+ }
202
+ D_ASSERT(!nodes.empty());
203
+ D_ASSERT(row_number >= nodes[0].row_start);
204
+ D_ASSERT(row_number < nodes.back().row_start + nodes.back().node->count);
205
+ idx_t lower = 0;
206
+ idx_t upper = nodes.size() - 1;
207
+ // binary search to find the node
208
+ while (lower <= upper) {
209
+ idx_t index = (lower + upper) / 2;
210
+ D_ASSERT(index < nodes.size());
211
+ auto &entry = nodes[index];
212
+ D_ASSERT(entry.row_start == entry.node->start);
213
+ if (row_number < entry.row_start) {
214
+ upper = index - 1;
215
+ } else if (row_number >= entry.row_start + entry.node->count) {
216
+ lower = index + 1;
217
+ } else {
218
+ result = index;
219
+ return true;
220
+ }
221
+ }
222
+ return false;
223
+ }
69
224
 
70
- void Verify(SegmentLock &);
71
- void Verify();
225
+ void Verify(SegmentLock &) {
226
+ #ifdef DEBUG
227
+ idx_t base_start = nodes.empty() ? 0 : nodes[0].node->start;
228
+ for (idx_t i = 0; i < nodes.size(); i++) {
229
+ D_ASSERT(nodes[i].row_start == nodes[i].node->start);
230
+ D_ASSERT(nodes[i].node->start == base_start);
231
+ base_start += nodes[i].node->count;
232
+ }
233
+ #endif
234
+ }
235
+ void Verify() {
236
+ #ifdef DEBUG
237
+ auto l = Lock();
238
+ Verify(l);
239
+ #endif
240
+ }
241
+
242
+ SegmentIterationHelper Segments() {
243
+ return SegmentIterationHelper(*this);
244
+ }
245
+
246
+ protected:
247
+ atomic<bool> finished_loading;
248
+
249
+ //! Load the next segment - only used when lazily loading
250
+ virtual unique_ptr<T> LoadSegment() {
251
+ return nullptr;
252
+ }
72
253
 
73
254
  private:
74
255
  //! The nodes in the tree, can be binary searched
75
- vector<SegmentNode> nodes;
256
+ vector<SegmentNode<T>> nodes;
76
257
  //! Lock to access or modify the nodes
77
258
  mutex node_lock;
259
+
260
+ private:
261
+ class SegmentIterationHelper {
262
+ public:
263
+ explicit SegmentIterationHelper(SegmentTree &tree) : tree(tree) {
264
+ }
265
+
266
+ private:
267
+ SegmentTree &tree;
268
+
269
+ private:
270
+ class SegmentIterator {
271
+ public:
272
+ SegmentIterator(SegmentTree &tree_p, T *current_p) : tree(tree_p), current(current_p) {
273
+ }
274
+
275
+ SegmentTree &tree;
276
+ T *current;
277
+
278
+ public:
279
+ void Next() {
280
+ current = tree.GetNextSegment(current);
281
+ }
282
+
283
+ SegmentIterator &operator++() {
284
+ Next();
285
+ return *this;
286
+ }
287
+ bool operator!=(const SegmentIterator &other) const {
288
+ return current != other.current;
289
+ }
290
+ T &operator*() const {
291
+ D_ASSERT(current);
292
+ return *current;
293
+ }
294
+ };
295
+
296
+ public:
297
+ SegmentIterator begin() {
298
+ return SegmentIterator(tree, tree.GetRootSegment());
299
+ }
300
+ SegmentIterator end() {
301
+ return SegmentIterator(tree, nullptr);
302
+ }
303
+ };
304
+
305
+ //! Load the next segment, if there are any left to load
306
+ bool LoadNextSegment(SegmentLock &l) {
307
+ if (finished_loading) {
308
+ return false;
309
+ }
310
+ auto result = LoadSegment();
311
+ if (result) {
312
+ AppendSegmentInternal(l, std::move(result));
313
+ return true;
314
+ }
315
+ return false;
316
+ }
317
+
318
+ //! Load all segments, if there are any left to load
319
+ void LoadAllSegments(SegmentLock &l) {
320
+ while (LoadNextSegment(l))
321
+ ;
322
+ }
78
323
  };
79
324
 
80
325
  } // namespace duckdb
@@ -105,13 +105,7 @@ void FilterPushdown::GenerateFilters() {
105
105
  });
106
106
  }
107
107
 
108
- unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
109
- // unhandled type, first perform filter pushdown in its children
110
- for (auto &child : op->children) {
111
- FilterPushdown pushdown(optimizer);
112
- child = pushdown.Rewrite(std::move(child));
113
- }
114
- // now push any existing filters
108
+ unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
115
109
  if (filters.empty()) {
116
110
  // no filters to push
117
111
  return op;
@@ -124,6 +118,16 @@ unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOpe
124
118
  return std::move(filter);
125
119
  }
126
120
 
121
+ unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
122
+ // unhandled type, first perform filter pushdown in its children
123
+ for (auto &child : op->children) {
124
+ FilterPushdown pushdown(optimizer);
125
+ child = pushdown.Rewrite(std::move(child));
126
+ }
127
+ // now push any existing filters
128
+ return PushFinalFilters(std::move(op));
129
+ }
130
+
127
131
  void FilterPushdown::Filter::ExtractBindings() {
128
132
  bindings.clear();
129
133
  LogicalJoin::GetExpressionBindings(*filter, bindings);
@@ -122,16 +122,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
122
122
  right_pushdown.GenerateFilters();
123
123
  op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
124
124
  op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
125
- if (filters.empty()) {
126
- // no filters to push
127
- return op;
128
- }
129
- auto filter = make_unique<LogicalFilter>();
130
- for (auto &f : filters) {
131
- filter->expressions.push_back(std::move(f->filter));
132
- }
133
- filter->children.push_back(std::move(op));
134
- return std::move(filter);
125
+ return PushFinalFilters(std::move(op));
135
126
  }
136
127
 
137
128
  } // namespace duckdb
@@ -77,7 +77,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
77
77
  }
78
78
  op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
79
79
  op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
80
- return FinishPushdown(std::move(op));
80
+ return PushFinalFilters(std::move(op));
81
81
  }
82
82
 
83
83
  } // namespace duckdb
@@ -23,7 +23,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownSingleJoin(unique_ptr<Logica
23
23
  }
24
24
  op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
25
25
  op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
26
- return FinishPushdown(std::move(op));
26
+ return PushFinalFilters(std::move(op));
27
27
  }
28
28
 
29
29
  } // namespace duckdb
@@ -49,6 +49,17 @@ unique_ptr<ParsedExpression> Transformer::TransformArrayAccess(duckdb_libpgquery
49
49
  result = make_unique<OperatorExpression>(ExpressionType::STRUCT_EXTRACT, std::move(children));
50
50
  break;
51
51
  }
52
+ case duckdb_libpgquery::T_PGFuncCall: {
53
+ auto func = (duckdb_libpgquery::PGFuncCall *)target;
54
+ auto function = TransformFuncCall(func);
55
+ if (function->type != ExpressionType::FUNCTION) {
56
+ throw ParserException("%s.%s() call must be a function", result->ToString(), function->ToString());
57
+ }
58
+ auto &f = (FunctionExpression &)*function;
59
+ f.children.insert(f.children.begin(), std::move(result));
60
+ result = std::move(function);
61
+ break;
62
+ }
52
63
  default:
53
64
  throw NotImplementedException("Unimplemented subscript type");
54
65
  }
@@ -10,6 +10,7 @@
10
10
  #include "duckdb/parser/statement/drop_statement.hpp"
11
11
  #include "duckdb/parser/parsed_data/drop_info.hpp"
12
12
  #include "duckdb/parser/expression/cast_expression.hpp"
13
+ #include "duckdb/parser/expression/function_expression.hpp"
13
14
  #include "duckdb/parser/result_modifier.hpp"
14
15
  #include "duckdb/parser/tableref/subqueryref.hpp"
15
16
 
@@ -99,15 +100,35 @@ unique_ptr<SQLStatement> Transformer::CreatePivotStatement(unique_ptr<SQLStateme
99
100
  unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PGSelectStmt *stmt) {
100
101
  auto pivot = stmt->pivot;
101
102
  auto source = TransformTableRefNode(pivot->source);
102
- auto columns = TransformPivotList(pivot->columns);
103
103
 
104
104
  auto select_node = make_unique<SelectNode>();
105
105
  // handle the CTEs
106
106
  if (stmt->withClause) {
107
107
  TransformCTE(reinterpret_cast<duckdb_libpgquery::PGWithClause *>(stmt->withClause), select_node->cte_map);
108
108
  }
109
+ if (!pivot->columns) {
110
+ // no pivot columns - not actually a pivot
111
+ select_node->from_table = std::move(source);
112
+ if (pivot->groups) {
113
+ auto groups = TransformStringList(pivot->groups);
114
+ GroupingSet set;
115
+ for (idx_t gr = 0; gr < groups.size(); gr++) {
116
+ auto &group = groups[gr];
117
+ auto colref = make_unique<ColumnRefExpression>(group);
118
+ select_node->select_list.push_back(colref->Copy());
119
+ select_node->groups.group_expressions.push_back(std::move(colref));
120
+ set.insert(gr);
121
+ }
122
+ select_node->groups.grouping_sets.push_back(std::move(set));
123
+ }
124
+ if (pivot->aggrs) {
125
+ TransformExpressionList(*pivot->aggrs, select_node->select_list);
126
+ }
127
+ return std::move(select_node);
128
+ }
109
129
 
110
130
  // generate CREATE TYPE statements for each of the columns that do not have an IN list
131
+ auto columns = TransformPivotList(pivot->columns);
111
132
  auto pivot_idx = PivotEntryCount();
112
133
  for (idx_t c = 0; c < columns.size(); c++) {
113
134
  auto &col = columns[c];
@@ -131,13 +152,17 @@ unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PG
131
152
 
132
153
  auto pivot_ref = make_unique<PivotRef>();
133
154
  pivot_ref->source = std::move(source);
134
- if (pivot->aggrs) {
135
- TransformExpressionList(*pivot->aggrs, pivot_ref->aggregates);
155
+ if (pivot->unpivots) {
156
+ pivot_ref->unpivot_names = TransformStringList(pivot->unpivots);
136
157
  } else {
137
- if (!pivot->unpivots) {
138
- throw InternalException("No unpivots and no aggrs");
158
+ if (pivot->aggrs) {
159
+ TransformExpressionList(*pivot->aggrs, pivot_ref->aggregates);
160
+ } else {
161
+ // pivot but no aggregates specified - push a count star
162
+ vector<unique_ptr<ParsedExpression>> children;
163
+ auto function = make_unique<FunctionExpression>("count_star", std::move(children));
164
+ pivot_ref->aggregates.push_back(std::move(function));
139
165
  }
140
- pivot_ref->unpivot_names = TransformStringList(pivot->unpivots);
141
166
  }
142
167
  if (pivot->groups) {
143
168
  pivot_ref->groups = TransformStringList(pivot->groups);
@@ -38,10 +38,28 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
38
38
  "function has to be called in a FROM clause (similar to a table).",
39
39
  function.function_name)));
40
40
  }
41
- // not a table function - search again without if_exists to throw the error
42
- Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
43
- function.function_name, false, error_context);
44
- throw InternalException("Catalog::GetEntry for scalar function did not throw a second time");
41
+ // not a table function - check if the schema is set
42
+ if (!function.schema.empty()) {
43
+ // the schema is set - check if we can turn this the schema into a column ref
44
+ string error;
45
+ unique_ptr<ColumnRefExpression> colref;
46
+ if (function.catalog.empty()) {
47
+ colref = make_unique<ColumnRefExpression>(function.schema);
48
+ } else {
49
+ colref = make_unique<ColumnRefExpression>(function.schema, function.catalog);
50
+ }
51
+ auto new_colref = QualifyColumnName(*colref, error);
52
+ if (error.empty()) {
53
+ // we can! transform this into a function call on the column
54
+ // i.e. "x.lower()" becomes "lower(x)"
55
+ function.children.insert(function.children.begin(), std::move(colref));
56
+ function.catalog = INVALID_CATALOG;
57
+ function.schema = INVALID_SCHEMA;
58
+ }
59
+ }
60
+ // rebind the function
61
+ func = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
62
+ function.function_name, false, error_context);
45
63
  }
46
64
 
47
65
  if (func->type != CatalogType::AGGREGATE_FUNCTION_ENTRY &&
@@ -56,7 +56,7 @@ static void ConstructPivots(PivotRef &ref, idx_t pivot_idx, vector<unique_ptr<Pa
56
56
  auto &function = (FunctionExpression &)*copy;
57
57
  // add the filter and alias to the aggregate function
58
58
  function.filter = expr->Copy();
59
- if (ref.aggregates.size() > 1) {
59
+ if (ref.aggregates.size() > 1 || !function.alias.empty()) {
60
60
  // if there are multiple aggregates specified we add the name of the aggregate as well
61
61
  function.alias = name + "_" + function.GetName();
62
62
  } else {
@@ -1,7 +1,5 @@
1
1
  #include "duckdb/storage/checkpoint/table_data_reader.hpp"
2
2
  #include "duckdb/storage/meta_block_reader.hpp"
3
-
4
- #include "duckdb/common/vector_operations/vector_operations.hpp"
5
3
  #include "duckdb/common/types/null_value.hpp"
6
4
 
7
5
  #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
@@ -9,9 +7,6 @@
9
7
  #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
10
8
 
11
9
  #include "duckdb/main/database.hpp"
12
- #include "duckdb/main/client_context.hpp"
13
-
14
- #include "duckdb/storage/table/row_group.hpp"
15
10
 
16
11
  namespace duckdb {
17
12
 
@@ -27,12 +22,9 @@ void TableDataReader::ReadTableData() {
27
22
  info.data->table_stats.Deserialize(reader, columns);
28
23
 
29
24
  // deserialize each of the individual row groups
30
- auto row_group_count = reader.Read<uint64_t>();
31
- info.data->row_groups.reserve(row_group_count);
32
- for (idx_t i = 0; i < row_group_count; i++) {
33
- auto row_group_pointer = RowGroup::Deserialize(reader, columns);
34
- info.data->row_groups.push_back(std::move(row_group_pointer));
35
- }
25
+ info.data->row_group_count = reader.Read<uint64_t>();
26
+ info.data->block_id = reader.block->BlockId();
27
+ info.data->offset = reader.offset;
36
28
  }
37
29
 
38
30
  } // namespace duckdb
@@ -50,13 +50,19 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
50
50
 
51
51
  // now start writing the row group pointers to disk
52
52
  table_data_writer.Write<uint64_t>(row_group_pointers.size());
53
+ idx_t total_rows = 0;
53
54
  for (auto &row_group_pointer : row_group_pointers) {
55
+ auto row_group_count = row_group_pointer.row_start + row_group_pointer.tuple_count;
56
+ if (row_group_count > total_rows) {
57
+ total_rows = row_group_count;
58
+ }
54
59
  RowGroup::Serialize(row_group_pointer, table_data_writer);
55
60
  }
56
61
 
57
62
  // Pointer to the table itself goes to the metadata stream.
58
63
  meta_data_writer.Write<block_id_t>(pointer.block_id);
59
64
  meta_data_writer.Write<uint64_t>(pointer.offset);
65
+ meta_data_writer.Write<idx_t>(total_rows);
60
66
 
61
67
  // Now we serialize indexes in the table_metadata_writer
62
68
  std::vector<BlockPointer> index_pointers = info->indexes.SerializeIndexes(table_data_writer);
@@ -467,6 +467,7 @@ void CheckpointReader::ReadTableData(ClientContext &context, MetaBlockReader &re
467
467
  TableDataReader data_reader(table_data_reader, bound_info);
468
468
 
469
469
  data_reader.ReadTableData();
470
+ bound_info.data->total_rows = reader.Read<idx_t>();
470
471
 
471
472
  // Get any indexes block info
472
473
  idx_t num_indexes = reader.Read<idx_t>();
@@ -33,7 +33,7 @@ void ConstantFillFunction(ColumnSegment &segment, Vector &result, idx_t start_id
33
33
  auto &nstats = segment.stats.statistics;
34
34
 
35
35
  auto data = FlatVector::GetData<T>(result);
36
- auto constant_value = NumericStats::GetMinUnsafe<T>(nstats);
36
+ auto constant_value = NumericStats::GetMin<T>(nstats);
37
37
  for (idx_t i = 0; i < count; i++) {
38
38
  data[start_idx + i] = constant_value;
39
39
  }
@@ -71,7 +71,7 @@ void ConstantScanFunction(ColumnSegment &segment, ColumnScanState &state, idx_t
71
71
  auto &nstats = segment.stats.statistics;
72
72
 
73
73
  auto data = FlatVector::GetData<T>(result);
74
- data[0] = NumericStats::GetMinUnsafe<T>(nstats);
74
+ data[0] = NumericStats::GetMin<T>(nstats);
75
75
  result.SetVectorType(VectorType::CONSTANT_VECTOR);
76
76
  }
77
77
 
@@ -45,7 +45,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
45
45
  auto types = GetTypes();
46
46
  this->row_groups =
47
47
  make_shared<RowGroupCollection>(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0);
48
- if (data && !data->row_groups.empty()) {
48
+ if (data && data->row_group_count > 0) {
49
49
  this->row_groups->Initialize(*data);
50
50
  } else {
51
51
  this->row_groups->InitializeEmpty();