duckdb 0.7.2-dev832.0 → 0.7.2-dev886.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/src/catalog/catalog.cpp +21 -5
- package/src/duckdb/src/catalog/default/default_functions.cpp +3 -0
- package/src/duckdb/src/catalog/duck_catalog.cpp +34 -7
- package/src/duckdb/src/common/box_renderer.cpp +109 -23
- package/src/duckdb/src/common/types/value.cpp +0 -93
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +3 -0
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +5 -8
- package/src/duckdb/src/function/scalar/date/date_part.cpp +2 -2
- package/src/duckdb/src/function/scalar/date/date_trunc.cpp +2 -2
- package/src/duckdb/src/function/scalar/list/list_aggregates.cpp +1 -1
- package/src/duckdb/src/function/scalar/list/list_lambdas.cpp +4 -0
- package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +8 -8
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +3 -0
- package/src/duckdb/src/include/duckdb/catalog/duck_catalog.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/box_renderer.hpp +8 -2
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +0 -31
- package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +9 -52
- package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp +62 -0
- package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +2 -1
- package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -3
- package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +5 -3
- package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +4 -1
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +6 -3
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
- package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +37 -0
- package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +8 -1
- package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp +4 -3
- package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +271 -26
- package/src/duckdb/src/optimizer/filter_pushdown.cpp +11 -7
- package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp +1 -10
- package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp +1 -1
- package/src/duckdb/src/optimizer/pushdown/pushdown_single_join.cpp +1 -1
- package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +11 -0
- package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +31 -6
- package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +22 -4
- package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +1 -1
- package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -11
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +6 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -0
- package/src/duckdb/src/storage/compression/numeric_constant.cpp +2 -2
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/statistics/numeric_stats.cpp +145 -83
- package/src/duckdb/src/storage/statistics/numeric_stats_union.cpp +65 -0
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +1 -6
- package/src/duckdb/src/storage/table/column_data.cpp +29 -35
- package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +5 -5
- package/src/duckdb/src/storage/table/column_segment.cpp +8 -7
- package/src/duckdb/src/storage/table/list_column_data.cpp +2 -1
- package/src/duckdb/src/storage/table/persistent_table_data.cpp +2 -1
- package/src/duckdb/src/storage/table/row_group.cpp +9 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +82 -66
- package/src/duckdb/src/storage/table/scan_state.cpp +22 -3
- package/src/duckdb/src/storage/table/standard_column_data.cpp +1 -0
- package/src/duckdb/src/storage/table/struct_column_data.cpp +1 -0
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +11578 -11222
- package/src/duckdb/ub_src_storage_statistics.cpp +2 -0
- package/src/duckdb/ub_src_storage_table.cpp +0 -2
- package/src/duckdb/src/storage/table/segment_tree.cpp +0 -179
@@ -10,71 +10,316 @@
|
|
10
10
|
|
11
11
|
#include "duckdb/common/constants.hpp"
|
12
12
|
#include "duckdb/storage/storage_lock.hpp"
|
13
|
-
#include "duckdb/storage/table/segment_base.hpp"
|
14
13
|
#include "duckdb/storage/table/segment_lock.hpp"
|
15
14
|
#include "duckdb/common/vector.hpp"
|
16
15
|
#include "duckdb/common/mutex.hpp"
|
16
|
+
#include "duckdb/common/string_util.hpp"
|
17
17
|
|
18
18
|
namespace duckdb {
|
19
19
|
|
20
|
+
template <class T>
|
20
21
|
struct SegmentNode {
|
21
22
|
idx_t row_start;
|
22
|
-
unique_ptr<
|
23
|
+
unique_ptr<T> node;
|
23
24
|
};
|
24
25
|
|
25
26
|
//! The SegmentTree maintains a list of all segments of a specific column in a table, and allows searching for a segment
|
26
27
|
//! by row number
|
28
|
+
template <class T, bool SUPPORTS_LAZY_LOADING = false>
|
27
29
|
class SegmentTree {
|
30
|
+
private:
|
31
|
+
class SegmentIterationHelper;
|
32
|
+
|
28
33
|
public:
|
34
|
+
explicit SegmentTree() : finished_loading(true) {
|
35
|
+
}
|
36
|
+
virtual ~SegmentTree() {
|
37
|
+
}
|
38
|
+
|
29
39
|
//! Locks the segment tree. All methods to the segment tree either lock the segment tree, or take an already
|
30
40
|
//! obtained lock.
|
31
|
-
SegmentLock Lock()
|
41
|
+
SegmentLock Lock() {
|
42
|
+
return SegmentLock(node_lock);
|
43
|
+
}
|
32
44
|
|
33
|
-
bool IsEmpty(SegmentLock &)
|
45
|
+
bool IsEmpty(SegmentLock &l) {
|
46
|
+
return GetRootSegment(l) == nullptr;
|
47
|
+
}
|
34
48
|
|
35
49
|
//! Gets a pointer to the first segment. Useful for scans.
|
36
|
-
|
37
|
-
|
50
|
+
T *GetRootSegment() {
|
51
|
+
auto l = Lock();
|
52
|
+
return GetRootSegment(l);
|
53
|
+
}
|
54
|
+
|
55
|
+
T *GetRootSegment(SegmentLock &l) {
|
56
|
+
if (nodes.empty()) {
|
57
|
+
LoadNextSegment(l);
|
58
|
+
}
|
59
|
+
return nodes.empty() ? nullptr : nodes[0].node.get();
|
60
|
+
}
|
38
61
|
//! Obtains ownership of the data of the segment tree
|
39
|
-
vector<SegmentNode
|
62
|
+
vector<SegmentNode<T>> MoveSegments(SegmentLock &l) {
|
63
|
+
LoadAllSegments(l);
|
64
|
+
return std::move(nodes);
|
65
|
+
}
|
40
66
|
//! Gets a pointer to the nth segment. Negative numbers start from the back.
|
41
|
-
|
42
|
-
|
67
|
+
T *GetSegmentByIndex(int64_t index) {
|
68
|
+
auto l = Lock();
|
69
|
+
return GetSegmentByIndex(l, index);
|
70
|
+
}
|
71
|
+
T *GetSegmentByIndex(SegmentLock &l, int64_t index) {
|
72
|
+
if (index < 0) {
|
73
|
+
// load all segments
|
74
|
+
LoadAllSegments(l);
|
75
|
+
index = nodes.size() + index;
|
76
|
+
if (index < 0) {
|
77
|
+
return nullptr;
|
78
|
+
}
|
79
|
+
return nodes[index].node.get();
|
80
|
+
} else {
|
81
|
+
// lazily load segments until we reach the specific segment
|
82
|
+
while (idx_t(index) >= nodes.size() && LoadNextSegment(l)) {
|
83
|
+
}
|
84
|
+
if (idx_t(index) >= nodes.size()) {
|
85
|
+
return nullptr;
|
86
|
+
}
|
87
|
+
return nodes[index].node.get();
|
88
|
+
}
|
89
|
+
}
|
90
|
+
//! Gets the next segment
|
91
|
+
T *GetNextSegment(T *segment) {
|
92
|
+
if (!SUPPORTS_LAZY_LOADING) {
|
93
|
+
return segment->Next();
|
94
|
+
}
|
95
|
+
if (finished_loading) {
|
96
|
+
return segment->Next();
|
97
|
+
}
|
98
|
+
auto l = Lock();
|
99
|
+
return GetNextSegment(l, segment);
|
100
|
+
}
|
101
|
+
T *GetNextSegment(SegmentLock &l, T *segment) {
|
102
|
+
if (!segment) {
|
103
|
+
return nullptr;
|
104
|
+
}
|
105
|
+
#ifdef DEBUG
|
106
|
+
D_ASSERT(nodes[segment->index].node.get() == segment);
|
107
|
+
#endif
|
108
|
+
return GetSegmentByIndex(l, segment->index + 1);
|
109
|
+
}
|
43
110
|
|
44
111
|
//! Gets a pointer to the last segment. Useful for appends.
|
45
|
-
|
46
|
-
|
112
|
+
T *GetLastSegment(SegmentLock &l) {
|
113
|
+
LoadAllSegments(l);
|
114
|
+
if (nodes.empty()) {
|
115
|
+
return nullptr;
|
116
|
+
}
|
117
|
+
return nodes.back().node.get();
|
118
|
+
}
|
47
119
|
//! Gets a pointer to a specific column segment for the given row
|
48
|
-
|
49
|
-
|
120
|
+
T *GetSegment(idx_t row_number) {
|
121
|
+
auto l = Lock();
|
122
|
+
return GetSegment(l, row_number);
|
123
|
+
}
|
124
|
+
T *GetSegment(SegmentLock &l, idx_t row_number) {
|
125
|
+
return nodes[GetSegmentIndex(l, row_number)].node.get();
|
126
|
+
}
|
50
127
|
|
51
128
|
//! Append a column segment to the tree
|
52
|
-
void
|
53
|
-
|
129
|
+
void AppendSegmentInternal(SegmentLock &l, unique_ptr<T> segment) {
|
130
|
+
D_ASSERT(segment);
|
131
|
+
// add the node to the list of nodes
|
132
|
+
if (!nodes.empty()) {
|
133
|
+
nodes.back().node->next = segment.get();
|
134
|
+
}
|
135
|
+
SegmentNode<T> node;
|
136
|
+
segment->index = nodes.size();
|
137
|
+
node.row_start = segment->start;
|
138
|
+
node.node = std::move(segment);
|
139
|
+
nodes.push_back(std::move(node));
|
140
|
+
}
|
141
|
+
void AppendSegment(unique_ptr<T> segment) {
|
142
|
+
auto l = Lock();
|
143
|
+
AppendSegment(l, std::move(segment));
|
144
|
+
}
|
145
|
+
void AppendSegment(SegmentLock &l, unique_ptr<T> segment) {
|
146
|
+
LoadAllSegments(l);
|
147
|
+
AppendSegmentInternal(l, std::move(segment));
|
148
|
+
}
|
54
149
|
//! Debug method, check whether the segment is in the segment tree
|
55
|
-
bool HasSegment(
|
56
|
-
|
150
|
+
bool HasSegment(T *segment) {
|
151
|
+
auto l = Lock();
|
152
|
+
return HasSegment(l, segment);
|
153
|
+
}
|
154
|
+
bool HasSegment(SegmentLock &, T *segment) {
|
155
|
+
return segment->index < nodes.size() && nodes[segment->index].node.get() == segment;
|
156
|
+
}
|
57
157
|
|
58
158
|
//! Replace this tree with another tree, taking over its nodes in-place
|
59
|
-
void Replace(SegmentTree &other)
|
60
|
-
|
159
|
+
void Replace(SegmentTree<T> &other) {
|
160
|
+
auto l = Lock();
|
161
|
+
Replace(l, other);
|
162
|
+
}
|
163
|
+
void Replace(SegmentLock &l, SegmentTree<T> &other) {
|
164
|
+
other.LoadAllSegments(l);
|
165
|
+
nodes = std::move(other.nodes);
|
166
|
+
}
|
61
167
|
|
62
168
|
//! Erase all segments after a specific segment
|
63
|
-
void EraseSegments(SegmentLock
|
169
|
+
void EraseSegments(SegmentLock &l, idx_t segment_start) {
|
170
|
+
LoadAllSegments(l);
|
171
|
+
if (segment_start >= nodes.size() - 1) {
|
172
|
+
return;
|
173
|
+
}
|
174
|
+
nodes.erase(nodes.begin() + segment_start + 1, nodes.end());
|
175
|
+
}
|
64
176
|
|
65
177
|
//! Get the segment index of the column segment for the given row
|
66
|
-
idx_t GetSegmentIndex(idx_t row_number)
|
67
|
-
|
68
|
-
|
178
|
+
idx_t GetSegmentIndex(SegmentLock &l, idx_t row_number) {
|
179
|
+
idx_t segment_index;
|
180
|
+
if (TryGetSegmentIndex(l, row_number, segment_index)) {
|
181
|
+
return segment_index;
|
182
|
+
}
|
183
|
+
string error;
|
184
|
+
error = StringUtil::Format("Attempting to find row number \"%lld\" in %lld nodes\n", row_number, nodes.size());
|
185
|
+
for (idx_t i = 0; i < nodes.size(); i++) {
|
186
|
+
error += StringUtil::Format("Node %lld: Start %lld, Count %lld", i, nodes[i].row_start,
|
187
|
+
nodes[i].node->count.load());
|
188
|
+
}
|
189
|
+
throw InternalException("Could not find node in column segment tree!\n%s%s", error, Exception::GetStackTrace());
|
190
|
+
}
|
191
|
+
|
192
|
+
bool TryGetSegmentIndex(SegmentLock &l, idx_t row_number, idx_t &result) {
|
193
|
+
// load segments until the row number is within bounds
|
194
|
+
while (nodes.empty() || (row_number >= (nodes.back().row_start + nodes.back().node->count))) {
|
195
|
+
if (!LoadNextSegment(l)) {
|
196
|
+
break;
|
197
|
+
}
|
198
|
+
}
|
199
|
+
if (nodes.empty()) {
|
200
|
+
return false;
|
201
|
+
}
|
202
|
+
D_ASSERT(!nodes.empty());
|
203
|
+
D_ASSERT(row_number >= nodes[0].row_start);
|
204
|
+
D_ASSERT(row_number < nodes.back().row_start + nodes.back().node->count);
|
205
|
+
idx_t lower = 0;
|
206
|
+
idx_t upper = nodes.size() - 1;
|
207
|
+
// binary search to find the node
|
208
|
+
while (lower <= upper) {
|
209
|
+
idx_t index = (lower + upper) / 2;
|
210
|
+
D_ASSERT(index < nodes.size());
|
211
|
+
auto &entry = nodes[index];
|
212
|
+
D_ASSERT(entry.row_start == entry.node->start);
|
213
|
+
if (row_number < entry.row_start) {
|
214
|
+
upper = index - 1;
|
215
|
+
} else if (row_number >= entry.row_start + entry.node->count) {
|
216
|
+
lower = index + 1;
|
217
|
+
} else {
|
218
|
+
result = index;
|
219
|
+
return true;
|
220
|
+
}
|
221
|
+
}
|
222
|
+
return false;
|
223
|
+
}
|
69
224
|
|
70
|
-
void Verify(SegmentLock &)
|
71
|
-
|
225
|
+
void Verify(SegmentLock &) {
|
226
|
+
#ifdef DEBUG
|
227
|
+
idx_t base_start = nodes.empty() ? 0 : nodes[0].node->start;
|
228
|
+
for (idx_t i = 0; i < nodes.size(); i++) {
|
229
|
+
D_ASSERT(nodes[i].row_start == nodes[i].node->start);
|
230
|
+
D_ASSERT(nodes[i].node->start == base_start);
|
231
|
+
base_start += nodes[i].node->count;
|
232
|
+
}
|
233
|
+
#endif
|
234
|
+
}
|
235
|
+
void Verify() {
|
236
|
+
#ifdef DEBUG
|
237
|
+
auto l = Lock();
|
238
|
+
Verify(l);
|
239
|
+
#endif
|
240
|
+
}
|
241
|
+
|
242
|
+
SegmentIterationHelper Segments() {
|
243
|
+
return SegmentIterationHelper(*this);
|
244
|
+
}
|
245
|
+
|
246
|
+
protected:
|
247
|
+
atomic<bool> finished_loading;
|
248
|
+
|
249
|
+
//! Load the next segment - only used when lazily loading
|
250
|
+
virtual unique_ptr<T> LoadSegment() {
|
251
|
+
return nullptr;
|
252
|
+
}
|
72
253
|
|
73
254
|
private:
|
74
255
|
//! The nodes in the tree, can be binary searched
|
75
|
-
vector<SegmentNode
|
256
|
+
vector<SegmentNode<T>> nodes;
|
76
257
|
//! Lock to access or modify the nodes
|
77
258
|
mutex node_lock;
|
259
|
+
|
260
|
+
private:
|
261
|
+
class SegmentIterationHelper {
|
262
|
+
public:
|
263
|
+
explicit SegmentIterationHelper(SegmentTree &tree) : tree(tree) {
|
264
|
+
}
|
265
|
+
|
266
|
+
private:
|
267
|
+
SegmentTree &tree;
|
268
|
+
|
269
|
+
private:
|
270
|
+
class SegmentIterator {
|
271
|
+
public:
|
272
|
+
SegmentIterator(SegmentTree &tree_p, T *current_p) : tree(tree_p), current(current_p) {
|
273
|
+
}
|
274
|
+
|
275
|
+
SegmentTree &tree;
|
276
|
+
T *current;
|
277
|
+
|
278
|
+
public:
|
279
|
+
void Next() {
|
280
|
+
current = tree.GetNextSegment(current);
|
281
|
+
}
|
282
|
+
|
283
|
+
SegmentIterator &operator++() {
|
284
|
+
Next();
|
285
|
+
return *this;
|
286
|
+
}
|
287
|
+
bool operator!=(const SegmentIterator &other) const {
|
288
|
+
return current != other.current;
|
289
|
+
}
|
290
|
+
T &operator*() const {
|
291
|
+
D_ASSERT(current);
|
292
|
+
return *current;
|
293
|
+
}
|
294
|
+
};
|
295
|
+
|
296
|
+
public:
|
297
|
+
SegmentIterator begin() {
|
298
|
+
return SegmentIterator(tree, tree.GetRootSegment());
|
299
|
+
}
|
300
|
+
SegmentIterator end() {
|
301
|
+
return SegmentIterator(tree, nullptr);
|
302
|
+
}
|
303
|
+
};
|
304
|
+
|
305
|
+
//! Load the next segment, if there are any left to load
|
306
|
+
bool LoadNextSegment(SegmentLock &l) {
|
307
|
+
if (finished_loading) {
|
308
|
+
return false;
|
309
|
+
}
|
310
|
+
auto result = LoadSegment();
|
311
|
+
if (result) {
|
312
|
+
AppendSegmentInternal(l, std::move(result));
|
313
|
+
return true;
|
314
|
+
}
|
315
|
+
return false;
|
316
|
+
}
|
317
|
+
|
318
|
+
//! Load all segments, if there are any left to load
|
319
|
+
void LoadAllSegments(SegmentLock &l) {
|
320
|
+
while (LoadNextSegment(l))
|
321
|
+
;
|
322
|
+
}
|
78
323
|
};
|
79
324
|
|
80
325
|
} // namespace duckdb
|
@@ -105,13 +105,7 @@ void FilterPushdown::GenerateFilters() {
|
|
105
105
|
});
|
106
106
|
}
|
107
107
|
|
108
|
-
unique_ptr<LogicalOperator> FilterPushdown::
|
109
|
-
// unhandled type, first perform filter pushdown in its children
|
110
|
-
for (auto &child : op->children) {
|
111
|
-
FilterPushdown pushdown(optimizer);
|
112
|
-
child = pushdown.Rewrite(std::move(child));
|
113
|
-
}
|
114
|
-
// now push any existing filters
|
108
|
+
unique_ptr<LogicalOperator> FilterPushdown::PushFinalFilters(unique_ptr<LogicalOperator> op) {
|
115
109
|
if (filters.empty()) {
|
116
110
|
// no filters to push
|
117
111
|
return op;
|
@@ -124,6 +118,16 @@ unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOpe
|
|
124
118
|
return std::move(filter);
|
125
119
|
}
|
126
120
|
|
121
|
+
unique_ptr<LogicalOperator> FilterPushdown::FinishPushdown(unique_ptr<LogicalOperator> op) {
|
122
|
+
// unhandled type, first perform filter pushdown in its children
|
123
|
+
for (auto &child : op->children) {
|
124
|
+
FilterPushdown pushdown(optimizer);
|
125
|
+
child = pushdown.Rewrite(std::move(child));
|
126
|
+
}
|
127
|
+
// now push any existing filters
|
128
|
+
return PushFinalFilters(std::move(op));
|
129
|
+
}
|
130
|
+
|
127
131
|
void FilterPushdown::Filter::ExtractBindings() {
|
128
132
|
bindings.clear();
|
129
133
|
LogicalJoin::GetExpressionBindings(*filter, bindings);
|
@@ -122,16 +122,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownLeftJoin(unique_ptr<LogicalO
|
|
122
122
|
right_pushdown.GenerateFilters();
|
123
123
|
op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
|
124
124
|
op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
|
125
|
-
|
126
|
-
// no filters to push
|
127
|
-
return op;
|
128
|
-
}
|
129
|
-
auto filter = make_unique<LogicalFilter>();
|
130
|
-
for (auto &f : filters) {
|
131
|
-
filter->expressions.push_back(std::move(f->filter));
|
132
|
-
}
|
133
|
-
filter->children.push_back(std::move(op));
|
134
|
-
return std::move(filter);
|
125
|
+
return PushFinalFilters(std::move(op));
|
135
126
|
}
|
136
127
|
|
137
128
|
} // namespace duckdb
|
@@ -77,7 +77,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownMarkJoin(unique_ptr<LogicalO
|
|
77
77
|
}
|
78
78
|
op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
|
79
79
|
op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
|
80
|
-
return
|
80
|
+
return PushFinalFilters(std::move(op));
|
81
81
|
}
|
82
82
|
|
83
83
|
} // namespace duckdb
|
@@ -23,7 +23,7 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownSingleJoin(unique_ptr<Logica
|
|
23
23
|
}
|
24
24
|
op->children[0] = left_pushdown.Rewrite(std::move(op->children[0]));
|
25
25
|
op->children[1] = right_pushdown.Rewrite(std::move(op->children[1]));
|
26
|
-
return
|
26
|
+
return PushFinalFilters(std::move(op));
|
27
27
|
}
|
28
28
|
|
29
29
|
} // namespace duckdb
|
@@ -49,6 +49,17 @@ unique_ptr<ParsedExpression> Transformer::TransformArrayAccess(duckdb_libpgquery
|
|
49
49
|
result = make_unique<OperatorExpression>(ExpressionType::STRUCT_EXTRACT, std::move(children));
|
50
50
|
break;
|
51
51
|
}
|
52
|
+
case duckdb_libpgquery::T_PGFuncCall: {
|
53
|
+
auto func = (duckdb_libpgquery::PGFuncCall *)target;
|
54
|
+
auto function = TransformFuncCall(func);
|
55
|
+
if (function->type != ExpressionType::FUNCTION) {
|
56
|
+
throw ParserException("%s.%s() call must be a function", result->ToString(), function->ToString());
|
57
|
+
}
|
58
|
+
auto &f = (FunctionExpression &)*function;
|
59
|
+
f.children.insert(f.children.begin(), std::move(result));
|
60
|
+
result = std::move(function);
|
61
|
+
break;
|
62
|
+
}
|
52
63
|
default:
|
53
64
|
throw NotImplementedException("Unimplemented subscript type");
|
54
65
|
}
|
@@ -10,6 +10,7 @@
|
|
10
10
|
#include "duckdb/parser/statement/drop_statement.hpp"
|
11
11
|
#include "duckdb/parser/parsed_data/drop_info.hpp"
|
12
12
|
#include "duckdb/parser/expression/cast_expression.hpp"
|
13
|
+
#include "duckdb/parser/expression/function_expression.hpp"
|
13
14
|
#include "duckdb/parser/result_modifier.hpp"
|
14
15
|
#include "duckdb/parser/tableref/subqueryref.hpp"
|
15
16
|
|
@@ -99,15 +100,35 @@ unique_ptr<SQLStatement> Transformer::CreatePivotStatement(unique_ptr<SQLStateme
|
|
99
100
|
unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PGSelectStmt *stmt) {
|
100
101
|
auto pivot = stmt->pivot;
|
101
102
|
auto source = TransformTableRefNode(pivot->source);
|
102
|
-
auto columns = TransformPivotList(pivot->columns);
|
103
103
|
|
104
104
|
auto select_node = make_unique<SelectNode>();
|
105
105
|
// handle the CTEs
|
106
106
|
if (stmt->withClause) {
|
107
107
|
TransformCTE(reinterpret_cast<duckdb_libpgquery::PGWithClause *>(stmt->withClause), select_node->cte_map);
|
108
108
|
}
|
109
|
+
if (!pivot->columns) {
|
110
|
+
// no pivot columns - not actually a pivot
|
111
|
+
select_node->from_table = std::move(source);
|
112
|
+
if (pivot->groups) {
|
113
|
+
auto groups = TransformStringList(pivot->groups);
|
114
|
+
GroupingSet set;
|
115
|
+
for (idx_t gr = 0; gr < groups.size(); gr++) {
|
116
|
+
auto &group = groups[gr];
|
117
|
+
auto colref = make_unique<ColumnRefExpression>(group);
|
118
|
+
select_node->select_list.push_back(colref->Copy());
|
119
|
+
select_node->groups.group_expressions.push_back(std::move(colref));
|
120
|
+
set.insert(gr);
|
121
|
+
}
|
122
|
+
select_node->groups.grouping_sets.push_back(std::move(set));
|
123
|
+
}
|
124
|
+
if (pivot->aggrs) {
|
125
|
+
TransformExpressionList(*pivot->aggrs, select_node->select_list);
|
126
|
+
}
|
127
|
+
return std::move(select_node);
|
128
|
+
}
|
109
129
|
|
110
130
|
// generate CREATE TYPE statements for each of the columns that do not have an IN list
|
131
|
+
auto columns = TransformPivotList(pivot->columns);
|
111
132
|
auto pivot_idx = PivotEntryCount();
|
112
133
|
for (idx_t c = 0; c < columns.size(); c++) {
|
113
134
|
auto &col = columns[c];
|
@@ -131,13 +152,17 @@ unique_ptr<QueryNode> Transformer::TransformPivotStatement(duckdb_libpgquery::PG
|
|
131
152
|
|
132
153
|
auto pivot_ref = make_unique<PivotRef>();
|
133
154
|
pivot_ref->source = std::move(source);
|
134
|
-
if (pivot->
|
135
|
-
|
155
|
+
if (pivot->unpivots) {
|
156
|
+
pivot_ref->unpivot_names = TransformStringList(pivot->unpivots);
|
136
157
|
} else {
|
137
|
-
if (
|
138
|
-
|
158
|
+
if (pivot->aggrs) {
|
159
|
+
TransformExpressionList(*pivot->aggrs, pivot_ref->aggregates);
|
160
|
+
} else {
|
161
|
+
// pivot but no aggregates specified - push a count star
|
162
|
+
vector<unique_ptr<ParsedExpression>> children;
|
163
|
+
auto function = make_unique<FunctionExpression>("count_star", std::move(children));
|
164
|
+
pivot_ref->aggregates.push_back(std::move(function));
|
139
165
|
}
|
140
|
-
pivot_ref->unpivot_names = TransformStringList(pivot->unpivots);
|
141
166
|
}
|
142
167
|
if (pivot->groups) {
|
143
168
|
pivot_ref->groups = TransformStringList(pivot->groups);
|
@@ -38,10 +38,28 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
|
|
38
38
|
"function has to be called in a FROM clause (similar to a table).",
|
39
39
|
function.function_name)));
|
40
40
|
}
|
41
|
-
// not a table function -
|
42
|
-
|
43
|
-
|
44
|
-
|
41
|
+
// not a table function - check if the schema is set
|
42
|
+
if (!function.schema.empty()) {
|
43
|
+
// the schema is set - check if we can turn this the schema into a column ref
|
44
|
+
string error;
|
45
|
+
unique_ptr<ColumnRefExpression> colref;
|
46
|
+
if (function.catalog.empty()) {
|
47
|
+
colref = make_unique<ColumnRefExpression>(function.schema);
|
48
|
+
} else {
|
49
|
+
colref = make_unique<ColumnRefExpression>(function.schema, function.catalog);
|
50
|
+
}
|
51
|
+
auto new_colref = QualifyColumnName(*colref, error);
|
52
|
+
if (error.empty()) {
|
53
|
+
// we can! transform this into a function call on the column
|
54
|
+
// i.e. "x.lower()" becomes "lower(x)"
|
55
|
+
function.children.insert(function.children.begin(), std::move(colref));
|
56
|
+
function.catalog = INVALID_CATALOG;
|
57
|
+
function.schema = INVALID_SCHEMA;
|
58
|
+
}
|
59
|
+
}
|
60
|
+
// rebind the function
|
61
|
+
func = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
|
62
|
+
function.function_name, false, error_context);
|
45
63
|
}
|
46
64
|
|
47
65
|
if (func->type != CatalogType::AGGREGATE_FUNCTION_ENTRY &&
|
@@ -56,7 +56,7 @@ static void ConstructPivots(PivotRef &ref, idx_t pivot_idx, vector<unique_ptr<Pa
|
|
56
56
|
auto &function = (FunctionExpression &)*copy;
|
57
57
|
// add the filter and alias to the aggregate function
|
58
58
|
function.filter = expr->Copy();
|
59
|
-
if (ref.aggregates.size() > 1) {
|
59
|
+
if (ref.aggregates.size() > 1 || !function.alias.empty()) {
|
60
60
|
// if there are multiple aggregates specified we add the name of the aggregate as well
|
61
61
|
function.alias = name + "_" + function.GetName();
|
62
62
|
} else {
|
@@ -1,7 +1,5 @@
|
|
1
1
|
#include "duckdb/storage/checkpoint/table_data_reader.hpp"
|
2
2
|
#include "duckdb/storage/meta_block_reader.hpp"
|
3
|
-
|
4
|
-
#include "duckdb/common/vector_operations/vector_operations.hpp"
|
5
3
|
#include "duckdb/common/types/null_value.hpp"
|
6
4
|
|
7
5
|
#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
|
@@ -9,9 +7,6 @@
|
|
9
7
|
#include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
|
10
8
|
|
11
9
|
#include "duckdb/main/database.hpp"
|
12
|
-
#include "duckdb/main/client_context.hpp"
|
13
|
-
|
14
|
-
#include "duckdb/storage/table/row_group.hpp"
|
15
10
|
|
16
11
|
namespace duckdb {
|
17
12
|
|
@@ -27,12 +22,9 @@ void TableDataReader::ReadTableData() {
|
|
27
22
|
info.data->table_stats.Deserialize(reader, columns);
|
28
23
|
|
29
24
|
// deserialize each of the individual row groups
|
30
|
-
|
31
|
-
info.data->
|
32
|
-
|
33
|
-
auto row_group_pointer = RowGroup::Deserialize(reader, columns);
|
34
|
-
info.data->row_groups.push_back(std::move(row_group_pointer));
|
35
|
-
}
|
25
|
+
info.data->row_group_count = reader.Read<uint64_t>();
|
26
|
+
info.data->block_id = reader.block->BlockId();
|
27
|
+
info.data->offset = reader.offset;
|
36
28
|
}
|
37
29
|
|
38
30
|
} // namespace duckdb
|
@@ -50,13 +50,19 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
|
|
50
50
|
|
51
51
|
// now start writing the row group pointers to disk
|
52
52
|
table_data_writer.Write<uint64_t>(row_group_pointers.size());
|
53
|
+
idx_t total_rows = 0;
|
53
54
|
for (auto &row_group_pointer : row_group_pointers) {
|
55
|
+
auto row_group_count = row_group_pointer.row_start + row_group_pointer.tuple_count;
|
56
|
+
if (row_group_count > total_rows) {
|
57
|
+
total_rows = row_group_count;
|
58
|
+
}
|
54
59
|
RowGroup::Serialize(row_group_pointer, table_data_writer);
|
55
60
|
}
|
56
61
|
|
57
62
|
// Pointer to the table itself goes to the metadata stream.
|
58
63
|
meta_data_writer.Write<block_id_t>(pointer.block_id);
|
59
64
|
meta_data_writer.Write<uint64_t>(pointer.offset);
|
65
|
+
meta_data_writer.Write<idx_t>(total_rows);
|
60
66
|
|
61
67
|
// Now we serialize indexes in the table_metadata_writer
|
62
68
|
std::vector<BlockPointer> index_pointers = info->indexes.SerializeIndexes(table_data_writer);
|
@@ -467,6 +467,7 @@ void CheckpointReader::ReadTableData(ClientContext &context, MetaBlockReader &re
|
|
467
467
|
TableDataReader data_reader(table_data_reader, bound_info);
|
468
468
|
|
469
469
|
data_reader.ReadTableData();
|
470
|
+
bound_info.data->total_rows = reader.Read<idx_t>();
|
470
471
|
|
471
472
|
// Get any indexes block info
|
472
473
|
idx_t num_indexes = reader.Read<idx_t>();
|
@@ -33,7 +33,7 @@ void ConstantFillFunction(ColumnSegment &segment, Vector &result, idx_t start_id
|
|
33
33
|
auto &nstats = segment.stats.statistics;
|
34
34
|
|
35
35
|
auto data = FlatVector::GetData<T>(result);
|
36
|
-
auto constant_value = NumericStats::
|
36
|
+
auto constant_value = NumericStats::GetMin<T>(nstats);
|
37
37
|
for (idx_t i = 0; i < count; i++) {
|
38
38
|
data[start_idx + i] = constant_value;
|
39
39
|
}
|
@@ -71,7 +71,7 @@ void ConstantScanFunction(ColumnSegment &segment, ColumnScanState &state, idx_t
|
|
71
71
|
auto &nstats = segment.stats.statistics;
|
72
72
|
|
73
73
|
auto data = FlatVector::GetData<T>(result);
|
74
|
-
data[0] = NumericStats::
|
74
|
+
data[0] = NumericStats::GetMin<T>(nstats);
|
75
75
|
result.SetVectorType(VectorType::CONSTANT_VECTOR);
|
76
76
|
}
|
77
77
|
|
@@ -45,7 +45,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
|
|
45
45
|
auto types = GetTypes();
|
46
46
|
this->row_groups =
|
47
47
|
make_shared<RowGroupCollection>(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0);
|
48
|
-
if (data &&
|
48
|
+
if (data && data->row_group_count > 0) {
|
49
49
|
this->row_groups->Initialize(*data);
|
50
50
|
} else {
|
51
51
|
this->row_groups->InitializeEmpty();
|