duckdb 0.7.2-dev832.0 → 0.7.2-dev865.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/src/catalog/catalog.cpp +21 -5
  3. package/src/duckdb/src/common/types/value.cpp +0 -93
  4. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +3 -0
  5. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +5 -8
  6. package/src/duckdb/src/function/scalar/date/date_part.cpp +2 -2
  7. package/src/duckdb/src/function/scalar/date/date_trunc.cpp +2 -2
  8. package/src/duckdb/src/function/scalar/list/list_aggregates.cpp +1 -1
  9. package/src/duckdb/src/function/scalar/list/list_lambdas.cpp +4 -0
  10. package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +8 -8
  11. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  12. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +3 -0
  13. package/src/duckdb/src/include/duckdb/common/types/value.hpp +0 -31
  14. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +9 -52
  15. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp +62 -0
  16. package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +2 -1
  17. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -3
  18. package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp +3 -2
  19. package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +5 -3
  20. package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +4 -1
  21. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +6 -3
  22. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
  23. package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +37 -0
  24. package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +8 -1
  25. package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp +4 -3
  26. package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +271 -26
  27. package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +11 -0
  28. package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +22 -4
  29. package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -11
  30. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +6 -0
  31. package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -0
  32. package/src/duckdb/src/storage/compression/numeric_constant.cpp +2 -2
  33. package/src/duckdb/src/storage/data_table.cpp +1 -1
  34. package/src/duckdb/src/storage/statistics/numeric_stats.cpp +145 -83
  35. package/src/duckdb/src/storage/statistics/numeric_stats_union.cpp +65 -0
  36. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  37. package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +1 -6
  38. package/src/duckdb/src/storage/table/column_data.cpp +29 -35
  39. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +5 -5
  40. package/src/duckdb/src/storage/table/column_segment.cpp +8 -7
  41. package/src/duckdb/src/storage/table/list_column_data.cpp +2 -1
  42. package/src/duckdb/src/storage/table/persistent_table_data.cpp +2 -1
  43. package/src/duckdb/src/storage/table/row_group.cpp +9 -9
  44. package/src/duckdb/src/storage/table/row_group_collection.cpp +82 -66
  45. package/src/duckdb/src/storage/table/scan_state.cpp +22 -3
  46. package/src/duckdb/src/storage/table/standard_column_data.cpp +1 -0
  47. package/src/duckdb/src/storage/table/struct_column_data.cpp +1 -0
  48. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +11780 -11512
  49. package/src/duckdb/ub_src_storage_statistics.cpp +2 -0
  50. package/src/duckdb/ub_src_storage_table.cpp +0 -2
  51. package/src/duckdb/src/storage/table/segment_tree.cpp +0 -179
@@ -9,7 +9,6 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/common/vector_size.hpp"
12
- #include "duckdb/storage/table/segment_base.hpp"
13
12
  #include "duckdb/storage/table/chunk_info.hpp"
14
13
  #include "duckdb/storage/table/append_state.hpp"
15
14
  #include "duckdb/storage/table/scan_state.hpp"
@@ -17,6 +16,7 @@
17
16
  #include "duckdb/common/enums/scan_options.hpp"
18
17
  #include "duckdb/common/mutex.hpp"
19
18
  #include "duckdb/parser/column_list.hpp"
19
+ #include "duckdb/storage/table/segment_base.hpp"
20
20
 
21
21
  namespace duckdb {
22
22
  class AttachedDatabase;
@@ -42,7 +42,7 @@ struct RowGroupWriteData {
42
42
  vector<BaseStatistics> statistics;
43
43
  };
44
44
 
45
- class RowGroup : public SegmentBase {
45
+ class RowGroup : public SegmentBase<RowGroup> {
46
46
  public:
47
47
  friend class ColumnData;
48
48
  friend class VersionDeleteState;
@@ -58,6 +58,9 @@ public:
58
58
  RowGroup(RowGroup &row_group, idx_t start);
59
59
  ~RowGroup();
60
60
 
61
+ //! The index within the segment tree
62
+ idx_t index;
63
+
61
64
  private:
62
65
  //! The database instance
63
66
  AttachedDatabase &db;
@@ -135,7 +138,7 @@ public:
135
138
  RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
136
139
  RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
137
140
  static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
138
- static RowGroupPointer Deserialize(Deserializer &source, const ColumnList &columns);
141
+ static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
139
142
 
140
143
  void InitializeAppend(RowGroupAppendState &append_state);
141
144
  void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
@@ -23,6 +23,8 @@ class TableStatistics;
23
23
 
24
24
  class BoundConstraint;
25
25
 
26
+ class RowGroupSegmentTree;
27
+
26
28
  class RowGroupCollection {
27
29
  public:
28
30
  RowGroupCollection(shared_ptr<DataTableInfo> info, BlockManager &block_manager, vector<LogicalType> types,
@@ -46,8 +48,8 @@ public:
46
48
  void InitializeCreateIndexScan(CreateIndexScanState &state);
47
49
  void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
48
50
  idx_t end_row);
49
- static bool InitializeScanInRowGroup(CollectionScanState &state, RowGroup *row_group, idx_t vector_index,
50
- idx_t max_row);
51
+ static bool InitializeScanInRowGroup(CollectionScanState &state, ParallelCollectionScanState &parallel_state,
52
+ idx_t vector_index, idx_t max_row);
51
53
  void InitializeParallelScan(ParallelCollectionScanState &state);
52
54
  bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
53
55
 
@@ -109,7 +111,7 @@ private:
109
111
  vector<LogicalType> types;
110
112
  idx_t row_start;
111
113
  //! The segment trees holding the various row_groups of the table
112
- shared_ptr<SegmentTree> row_groups;
114
+ shared_ptr<RowGroupSegmentTree> row_groups;
113
115
  //! Table statistics
114
116
  TableStatistics stats;
115
117
  };
@@ -0,0 +1,37 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/storage/table/row_group_segment_tree.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/storage/table/segment_tree.hpp"
12
+ #include "duckdb/storage/table/row_group.hpp"
13
+
14
+ namespace duckdb {
15
+ struct DataTableInfo;
16
+ class PersistentTableData;
17
+ class MetaBlockReader;
18
+
19
+ class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
20
+ public:
21
+ RowGroupSegmentTree(DataTableInfo &table_info_p, BlockManager &block_manager_p, vector<LogicalType> column_types_p);
22
+ ~RowGroupSegmentTree() override;
23
+
24
+ void Initialize(PersistentTableData &data);
25
+
26
+ protected:
27
+ unique_ptr<RowGroup> LoadSegment() override;
28
+
29
+ DataTableInfo &info;
30
+ BlockManager &block_manager;
31
+ vector<LogicalType> column_types;
32
+ idx_t current_row_group;
33
+ idx_t max_row_group;
34
+ unique_ptr<MetaBlockReader> reader;
35
+ };
36
+
37
+ } // namespace duckdb
@@ -25,10 +25,12 @@ class RowGroupCollection;
25
25
  class UpdateSegment;
26
26
  class TableScanState;
27
27
  class ColumnSegment;
28
+ class ColumnSegmentTree;
28
29
  class ValiditySegment;
29
30
  class TableFilterSet;
30
31
  class ColumnData;
31
32
  class DuckTransaction;
33
+ class RowGroupSegmentTree;
32
34
 
33
35
  struct SegmentScanState {
34
36
  virtual ~SegmentScanState() {
@@ -45,6 +47,8 @@ typedef unordered_map<block_id_t, BufferHandle> buffer_handle_set_t;
45
47
  struct ColumnScanState {
46
48
  //! The column segment that is currently being scanned
47
49
  ColumnSegment *current = nullptr;
50
+ //! Column segment tree
51
+ ColumnSegmentTree *segment_tree = nullptr;
48
52
  //! The current row index of the scan
49
53
  idx_t row_index = 0;
50
54
  //! The internal row index (i.e. the position of the SegmentScanState)
@@ -114,10 +118,12 @@ private:
114
118
  class CollectionScanState {
115
119
  public:
116
120
  CollectionScanState(TableScanState &parent_p)
117
- : row_group_state(*this), max_row(0), batch_index(0), parent(parent_p) {};
121
+ : row_group_state(*this), row_groups(nullptr), max_row(0), batch_index(0), parent(parent_p) {};
118
122
 
119
123
  //! The row_group scan state
120
124
  RowGroupScanState row_group_state;
125
+ //! Row group segment tree
126
+ RowGroupSegmentTree *row_groups;
121
127
  //! The total maximum row index
122
128
  idx_t max_row;
123
129
  //! The current batch index
@@ -129,6 +135,7 @@ public:
129
135
  AdaptiveFilter *GetAdaptiveFilter();
130
136
  bool Scan(DuckTransaction &transaction, DataChunk &result);
131
137
  bool ScanCommitted(DataChunk &result, TableScanType type);
138
+ bool ScanCommitted(DataChunk &result, SegmentLock &l, TableScanType type);
132
139
 
133
140
  private:
134
141
  TableScanState &parent;
@@ -13,13 +13,14 @@
13
13
 
14
14
  namespace duckdb {
15
15
 
16
+ template <class T>
16
17
  class SegmentBase {
17
18
  public:
18
19
  SegmentBase(idx_t start, idx_t count) : start(start), count(count), next(nullptr) {
19
20
  }
20
21
  virtual ~SegmentBase() {
21
22
  }
22
- SegmentBase *Next() {
23
+ T *Next() {
23
24
  #ifndef DUCKDB_R_BUILD
24
25
  return next.load();
25
26
  #else
@@ -34,9 +35,9 @@ public:
34
35
  //! The next segment after this one
35
36
 
36
37
  #ifndef DUCKDB_R_BUILD
37
- atomic<SegmentBase *> next;
38
+ atomic<T *> next;
38
39
  #else
39
- SegmentBase *next;
40
+ T *next;
40
41
  #endif
41
42
  };
42
43
 
@@ -10,71 +10,316 @@
10
10
 
11
11
  #include "duckdb/common/constants.hpp"
12
12
  #include "duckdb/storage/storage_lock.hpp"
13
- #include "duckdb/storage/table/segment_base.hpp"
14
13
  #include "duckdb/storage/table/segment_lock.hpp"
15
14
  #include "duckdb/common/vector.hpp"
16
15
  #include "duckdb/common/mutex.hpp"
16
+ #include "duckdb/common/string_util.hpp"
17
17
 
18
18
  namespace duckdb {
19
19
 
20
+ template <class T>
20
21
  struct SegmentNode {
21
22
  idx_t row_start;
22
- unique_ptr<SegmentBase> node;
23
+ unique_ptr<T> node;
23
24
  };
24
25
 
25
26
  //! The SegmentTree maintains a list of all segments of a specific column in a table, and allows searching for a segment
26
27
  //! by row number
28
+ template <class T, bool SUPPORTS_LAZY_LOADING = false>
27
29
  class SegmentTree {
30
+ private:
31
+ class SegmentIterationHelper;
32
+
28
33
  public:
34
+ explicit SegmentTree() : finished_loading(true) {
35
+ }
36
+ virtual ~SegmentTree() {
37
+ }
38
+
29
39
  //! Locks the segment tree. All methods to the segment tree either lock the segment tree, or take an already
30
40
  //! obtained lock.
31
- SegmentLock Lock();
41
+ SegmentLock Lock() {
42
+ return SegmentLock(node_lock);
43
+ }
32
44
 
33
- bool IsEmpty(SegmentLock &);
45
+ bool IsEmpty(SegmentLock &l) {
46
+ return GetRootSegment(l) == nullptr;
47
+ }
34
48
 
35
49
  //! Gets a pointer to the first segment. Useful for scans.
36
- SegmentBase *GetRootSegment();
37
- SegmentBase *GetRootSegment(SegmentLock &);
50
+ T *GetRootSegment() {
51
+ auto l = Lock();
52
+ return GetRootSegment(l);
53
+ }
54
+
55
+ T *GetRootSegment(SegmentLock &l) {
56
+ if (nodes.empty()) {
57
+ LoadNextSegment(l);
58
+ }
59
+ return nodes.empty() ? nullptr : nodes[0].node.get();
60
+ }
38
61
  //! Obtains ownership of the data of the segment tree
39
- vector<SegmentNode> MoveSegments(SegmentLock &);
62
+ vector<SegmentNode<T>> MoveSegments(SegmentLock &l) {
63
+ LoadAllSegments(l);
64
+ return std::move(nodes);
65
+ }
40
66
  //! Gets a pointer to the nth segment. Negative numbers start from the back.
41
- SegmentBase *GetSegmentByIndex(int64_t index);
42
- SegmentBase *GetSegmentByIndex(SegmentLock &, int64_t index);
67
+ T *GetSegmentByIndex(int64_t index) {
68
+ auto l = Lock();
69
+ return GetSegmentByIndex(l, index);
70
+ }
71
+ T *GetSegmentByIndex(SegmentLock &l, int64_t index) {
72
+ if (index < 0) {
73
+ // load all segments
74
+ LoadAllSegments(l);
75
+ index = nodes.size() + index;
76
+ if (index < 0) {
77
+ return nullptr;
78
+ }
79
+ return nodes[index].node.get();
80
+ } else {
81
+ // lazily load segments until we reach the specific segment
82
+ while (idx_t(index) >= nodes.size() && LoadNextSegment(l)) {
83
+ }
84
+ if (idx_t(index) >= nodes.size()) {
85
+ return nullptr;
86
+ }
87
+ return nodes[index].node.get();
88
+ }
89
+ }
90
+ //! Gets the next segment
91
+ T *GetNextSegment(T *segment) {
92
+ if (!SUPPORTS_LAZY_LOADING) {
93
+ return segment->Next();
94
+ }
95
+ if (finished_loading) {
96
+ return segment->Next();
97
+ }
98
+ auto l = Lock();
99
+ return GetNextSegment(l, segment);
100
+ }
101
+ T *GetNextSegment(SegmentLock &l, T *segment) {
102
+ if (!segment) {
103
+ return nullptr;
104
+ }
105
+ #ifdef DEBUG
106
+ D_ASSERT(nodes[segment->index].node.get() == segment);
107
+ #endif
108
+ return GetSegmentByIndex(l, segment->index + 1);
109
+ }
43
110
 
44
111
  //! Gets a pointer to the last segment. Useful for appends.
45
- SegmentBase *GetLastSegment();
46
- SegmentBase *GetLastSegment(SegmentLock &);
112
+ T *GetLastSegment(SegmentLock &l) {
113
+ LoadAllSegments(l);
114
+ if (nodes.empty()) {
115
+ return nullptr;
116
+ }
117
+ return nodes.back().node.get();
118
+ }
47
119
  //! Gets a pointer to a specific column segment for the given row
48
- SegmentBase *GetSegment(idx_t row_number);
49
- SegmentBase *GetSegment(SegmentLock &, idx_t row_number);
120
+ T *GetSegment(idx_t row_number) {
121
+ auto l = Lock();
122
+ return GetSegment(l, row_number);
123
+ }
124
+ T *GetSegment(SegmentLock &l, idx_t row_number) {
125
+ return nodes[GetSegmentIndex(l, row_number)].node.get();
126
+ }
50
127
 
51
128
  //! Append a column segment to the tree
52
- void AppendSegment(unique_ptr<SegmentBase> segment);
53
- void AppendSegment(SegmentLock &, unique_ptr<SegmentBase> segment);
129
+ void AppendSegmentInternal(SegmentLock &l, unique_ptr<T> segment) {
130
+ D_ASSERT(segment);
131
+ // add the node to the list of nodes
132
+ if (!nodes.empty()) {
133
+ nodes.back().node->next = segment.get();
134
+ }
135
+ SegmentNode<T> node;
136
+ segment->index = nodes.size();
137
+ node.row_start = segment->start;
138
+ node.node = std::move(segment);
139
+ nodes.push_back(std::move(node));
140
+ }
141
+ void AppendSegment(unique_ptr<T> segment) {
142
+ auto l = Lock();
143
+ AppendSegment(l, std::move(segment));
144
+ }
145
+ void AppendSegment(SegmentLock &l, unique_ptr<T> segment) {
146
+ LoadAllSegments(l);
147
+ AppendSegmentInternal(l, std::move(segment));
148
+ }
54
149
  //! Debug method, check whether the segment is in the segment tree
55
- bool HasSegment(SegmentBase *segment);
56
- bool HasSegment(SegmentLock &, SegmentBase *segment);
150
+ bool HasSegment(T *segment) {
151
+ auto l = Lock();
152
+ return HasSegment(l, segment);
153
+ }
154
+ bool HasSegment(SegmentLock &, T *segment) {
155
+ return segment->index < nodes.size() && nodes[segment->index].node.get() == segment;
156
+ }
57
157
 
58
158
  //! Replace this tree with another tree, taking over its nodes in-place
59
- void Replace(SegmentTree &other);
60
- void Replace(SegmentLock &, SegmentTree &other);
159
+ void Replace(SegmentTree<T> &other) {
160
+ auto l = Lock();
161
+ Replace(l, other);
162
+ }
163
+ void Replace(SegmentLock &l, SegmentTree<T> &other) {
164
+ other.LoadAllSegments(l);
165
+ nodes = std::move(other.nodes);
166
+ }
61
167
 
62
168
  //! Erase all segments after a specific segment
63
- void EraseSegments(SegmentLock &, idx_t segment_start);
169
+ void EraseSegments(SegmentLock &l, idx_t segment_start) {
170
+ LoadAllSegments(l);
171
+ if (segment_start >= nodes.size() - 1) {
172
+ return;
173
+ }
174
+ nodes.erase(nodes.begin() + segment_start + 1, nodes.end());
175
+ }
64
176
 
65
177
  //! Get the segment index of the column segment for the given row
66
- idx_t GetSegmentIndex(idx_t row_number);
67
- idx_t GetSegmentIndex(SegmentLock &, idx_t row_number);
68
- bool TryGetSegmentIndex(SegmentLock &, idx_t row_number, idx_t &);
178
+ idx_t GetSegmentIndex(SegmentLock &l, idx_t row_number) {
179
+ idx_t segment_index;
180
+ if (TryGetSegmentIndex(l, row_number, segment_index)) {
181
+ return segment_index;
182
+ }
183
+ string error;
184
+ error = StringUtil::Format("Attempting to find row number \"%lld\" in %lld nodes\n", row_number, nodes.size());
185
+ for (idx_t i = 0; i < nodes.size(); i++) {
186
+ error += StringUtil::Format("Node %lld: Start %lld, Count %lld", i, nodes[i].row_start,
187
+ nodes[i].node->count.load());
188
+ }
189
+ throw InternalException("Could not find node in column segment tree!\n%s%s", error, Exception::GetStackTrace());
190
+ }
191
+
192
+ bool TryGetSegmentIndex(SegmentLock &l, idx_t row_number, idx_t &result) {
193
+ // load segments until the row number is within bounds
194
+ while (nodes.empty() || (row_number >= (nodes.back().row_start + nodes.back().node->count))) {
195
+ if (!LoadNextSegment(l)) {
196
+ break;
197
+ }
198
+ }
199
+ if (nodes.empty()) {
200
+ return false;
201
+ }
202
+ D_ASSERT(!nodes.empty());
203
+ D_ASSERT(row_number >= nodes[0].row_start);
204
+ D_ASSERT(row_number < nodes.back().row_start + nodes.back().node->count);
205
+ idx_t lower = 0;
206
+ idx_t upper = nodes.size() - 1;
207
+ // binary search to find the node
208
+ while (lower <= upper) {
209
+ idx_t index = (lower + upper) / 2;
210
+ D_ASSERT(index < nodes.size());
211
+ auto &entry = nodes[index];
212
+ D_ASSERT(entry.row_start == entry.node->start);
213
+ if (row_number < entry.row_start) {
214
+ upper = index - 1;
215
+ } else if (row_number >= entry.row_start + entry.node->count) {
216
+ lower = index + 1;
217
+ } else {
218
+ result = index;
219
+ return true;
220
+ }
221
+ }
222
+ return false;
223
+ }
69
224
 
70
- void Verify(SegmentLock &);
71
- void Verify();
225
+ void Verify(SegmentLock &) {
226
+ #ifdef DEBUG
227
+ idx_t base_start = nodes.empty() ? 0 : nodes[0].node->start;
228
+ for (idx_t i = 0; i < nodes.size(); i++) {
229
+ D_ASSERT(nodes[i].row_start == nodes[i].node->start);
230
+ D_ASSERT(nodes[i].node->start == base_start);
231
+ base_start += nodes[i].node->count;
232
+ }
233
+ #endif
234
+ }
235
+ void Verify() {
236
+ #ifdef DEBUG
237
+ auto l = Lock();
238
+ Verify(l);
239
+ #endif
240
+ }
241
+
242
+ SegmentIterationHelper Segments() {
243
+ return SegmentIterationHelper(*this);
244
+ }
245
+
246
+ protected:
247
+ atomic<bool> finished_loading;
248
+
249
+ //! Load the next segment - only used when lazily loading
250
+ virtual unique_ptr<T> LoadSegment() {
251
+ return nullptr;
252
+ }
72
253
 
73
254
  private:
74
255
  //! The nodes in the tree, can be binary searched
75
- vector<SegmentNode> nodes;
256
+ vector<SegmentNode<T>> nodes;
76
257
  //! Lock to access or modify the nodes
77
258
  mutex node_lock;
259
+
260
+ private:
261
+ class SegmentIterationHelper {
262
+ public:
263
+ explicit SegmentIterationHelper(SegmentTree &tree) : tree(tree) {
264
+ }
265
+
266
+ private:
267
+ SegmentTree &tree;
268
+
269
+ private:
270
+ class SegmentIterator {
271
+ public:
272
+ SegmentIterator(SegmentTree &tree_p, T *current_p) : tree(tree_p), current(current_p) {
273
+ }
274
+
275
+ SegmentTree &tree;
276
+ T *current;
277
+
278
+ public:
279
+ void Next() {
280
+ current = tree.GetNextSegment(current);
281
+ }
282
+
283
+ SegmentIterator &operator++() {
284
+ Next();
285
+ return *this;
286
+ }
287
+ bool operator!=(const SegmentIterator &other) const {
288
+ return current != other.current;
289
+ }
290
+ T &operator*() const {
291
+ D_ASSERT(current);
292
+ return *current;
293
+ }
294
+ };
295
+
296
+ public:
297
+ SegmentIterator begin() {
298
+ return SegmentIterator(tree, tree.GetRootSegment());
299
+ }
300
+ SegmentIterator end() {
301
+ return SegmentIterator(tree, nullptr);
302
+ }
303
+ };
304
+
305
+ //! Load the next segment, if there are any left to load
306
+ bool LoadNextSegment(SegmentLock &l) {
307
+ if (finished_loading) {
308
+ return false;
309
+ }
310
+ auto result = LoadSegment();
311
+ if (result) {
312
+ AppendSegmentInternal(l, std::move(result));
313
+ return true;
314
+ }
315
+ return false;
316
+ }
317
+
318
+ //! Load all segments, if there are any left to load
319
+ void LoadAllSegments(SegmentLock &l) {
320
+ while (LoadNextSegment(l))
321
+ ;
322
+ }
78
323
  };
79
324
 
80
325
  } // namespace duckdb
@@ -49,6 +49,17 @@ unique_ptr<ParsedExpression> Transformer::TransformArrayAccess(duckdb_libpgquery
49
49
  result = make_unique<OperatorExpression>(ExpressionType::STRUCT_EXTRACT, std::move(children));
50
50
  break;
51
51
  }
52
+ case duckdb_libpgquery::T_PGFuncCall: {
53
+ auto func = (duckdb_libpgquery::PGFuncCall *)target;
54
+ auto function = TransformFuncCall(func);
55
+ if (function->type != ExpressionType::FUNCTION) {
56
+ throw ParserException("%s.%s() call must be a function", result->ToString(), function->ToString());
57
+ }
58
+ auto &f = (FunctionExpression &)*function;
59
+ f.children.insert(f.children.begin(), std::move(result));
60
+ result = std::move(function);
61
+ break;
62
+ }
52
63
  default:
53
64
  throw NotImplementedException("Unimplemented subscript type");
54
65
  }
@@ -38,10 +38,28 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
38
38
  "function has to be called in a FROM clause (similar to a table).",
39
39
  function.function_name)));
40
40
  }
41
- // not a table function - search again without if_exists to throw the error
42
- Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
43
- function.function_name, false, error_context);
44
- throw InternalException("Catalog::GetEntry for scalar function did not throw a second time");
41
+ // not a table function - check if the schema is set
42
+ if (!function.schema.empty()) {
43
+ // the schema is set - check if we can turn this the schema into a column ref
44
+ string error;
45
+ unique_ptr<ColumnRefExpression> colref;
46
+ if (function.catalog.empty()) {
47
+ colref = make_unique<ColumnRefExpression>(function.schema);
48
+ } else {
49
+ colref = make_unique<ColumnRefExpression>(function.schema, function.catalog);
50
+ }
51
+ auto new_colref = QualifyColumnName(*colref, error);
52
+ if (error.empty()) {
53
+ // we can! transform this into a function call on the column
54
+ // i.e. "x.lower()" becomes "lower(x)"
55
+ function.children.insert(function.children.begin(), std::move(colref));
56
+ function.catalog = INVALID_CATALOG;
57
+ function.schema = INVALID_SCHEMA;
58
+ }
59
+ }
60
+ // rebind the function
61
+ func = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
62
+ function.function_name, false, error_context);
45
63
  }
46
64
 
47
65
  if (func->type != CatalogType::AGGREGATE_FUNCTION_ENTRY &&
@@ -1,7 +1,5 @@
1
1
  #include "duckdb/storage/checkpoint/table_data_reader.hpp"
2
2
  #include "duckdb/storage/meta_block_reader.hpp"
3
-
4
- #include "duckdb/common/vector_operations/vector_operations.hpp"
5
3
  #include "duckdb/common/types/null_value.hpp"
6
4
 
7
5
  #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
@@ -9,9 +7,6 @@
9
7
  #include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
10
8
 
11
9
  #include "duckdb/main/database.hpp"
12
- #include "duckdb/main/client_context.hpp"
13
-
14
- #include "duckdb/storage/table/row_group.hpp"
15
10
 
16
11
  namespace duckdb {
17
12
 
@@ -27,12 +22,9 @@ void TableDataReader::ReadTableData() {
27
22
  info.data->table_stats.Deserialize(reader, columns);
28
23
 
29
24
  // deserialize each of the individual row groups
30
- auto row_group_count = reader.Read<uint64_t>();
31
- info.data->row_groups.reserve(row_group_count);
32
- for (idx_t i = 0; i < row_group_count; i++) {
33
- auto row_group_pointer = RowGroup::Deserialize(reader, columns);
34
- info.data->row_groups.push_back(std::move(row_group_pointer));
35
- }
25
+ info.data->row_group_count = reader.Read<uint64_t>();
26
+ info.data->block_id = reader.block->BlockId();
27
+ info.data->offset = reader.offset;
36
28
  }
37
29
 
38
30
  } // namespace duckdb
@@ -50,13 +50,19 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
50
50
 
51
51
  // now start writing the row group pointers to disk
52
52
  table_data_writer.Write<uint64_t>(row_group_pointers.size());
53
+ idx_t total_rows = 0;
53
54
  for (auto &row_group_pointer : row_group_pointers) {
55
+ auto row_group_count = row_group_pointer.row_start + row_group_pointer.tuple_count;
56
+ if (row_group_count > total_rows) {
57
+ total_rows = row_group_count;
58
+ }
54
59
  RowGroup::Serialize(row_group_pointer, table_data_writer);
55
60
  }
56
61
 
57
62
  // Pointer to the table itself goes to the metadata stream.
58
63
  meta_data_writer.Write<block_id_t>(pointer.block_id);
59
64
  meta_data_writer.Write<uint64_t>(pointer.offset);
65
+ meta_data_writer.Write<idx_t>(total_rows);
60
66
 
61
67
  // Now we serialize indexes in the table_metadata_writer
62
68
  std::vector<BlockPointer> index_pointers = info->indexes.SerializeIndexes(table_data_writer);
@@ -467,6 +467,7 @@ void CheckpointReader::ReadTableData(ClientContext &context, MetaBlockReader &re
467
467
  TableDataReader data_reader(table_data_reader, bound_info);
468
468
 
469
469
  data_reader.ReadTableData();
470
+ bound_info.data->total_rows = reader.Read<idx_t>();
470
471
 
471
472
  // Get any indexes block info
472
473
  idx_t num_indexes = reader.Read<idx_t>();
@@ -33,7 +33,7 @@ void ConstantFillFunction(ColumnSegment &segment, Vector &result, idx_t start_id
33
33
  auto &nstats = segment.stats.statistics;
34
34
 
35
35
  auto data = FlatVector::GetData<T>(result);
36
- auto constant_value = NumericStats::GetMinUnsafe<T>(nstats);
36
+ auto constant_value = NumericStats::GetMin<T>(nstats);
37
37
  for (idx_t i = 0; i < count; i++) {
38
38
  data[start_idx + i] = constant_value;
39
39
  }
@@ -71,7 +71,7 @@ void ConstantScanFunction(ColumnSegment &segment, ColumnScanState &state, idx_t
71
71
  auto &nstats = segment.stats.statistics;
72
72
 
73
73
  auto data = FlatVector::GetData<T>(result);
74
- data[0] = NumericStats::GetMinUnsafe<T>(nstats);
74
+ data[0] = NumericStats::GetMin<T>(nstats);
75
75
  result.SetVectorType(VectorType::CONSTANT_VECTOR);
76
76
  }
77
77
 
@@ -45,7 +45,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
45
45
  auto types = GetTypes();
46
46
  this->row_groups =
47
47
  make_shared<RowGroupCollection>(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0);
48
- if (data && !data->row_groups.empty()) {
48
+ if (data && data->row_group_count > 0) {
49
49
  this->row_groups->Initialize(*data);
50
50
  } else {
51
51
  this->row_groups->InitializeEmpty();