duckdb 0.7.2-dev832.0 → 0.7.2-dev865.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/src/catalog/catalog.cpp +21 -5
- package/src/duckdb/src/common/types/value.cpp +0 -93
- package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +3 -0
- package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +5 -8
- package/src/duckdb/src/function/scalar/date/date_part.cpp +2 -2
- package/src/duckdb/src/function/scalar/date/date_trunc.cpp +2 -2
- package/src/duckdb/src/function/scalar/list/list_aggregates.cpp +1 -1
- package/src/duckdb/src/function/scalar/list/list_lambdas.cpp +4 -0
- package/src/duckdb/src/function/scalar/operators/arithmetic.cpp +8 -8
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +3 -0
- package/src/duckdb/src/include/duckdb/common/types/value.hpp +0 -31
- package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +9 -52
- package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats_union.hpp +62 -0
- package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +2 -1
- package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -3
- package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +5 -3
- package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +4 -1
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +6 -3
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
- package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +37 -0
- package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +8 -1
- package/src/duckdb/src/include/duckdb/storage/table/segment_base.hpp +4 -3
- package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +271 -26
- package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +11 -0
- package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +22 -4
- package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -11
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +6 -0
- package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -0
- package/src/duckdb/src/storage/compression/numeric_constant.cpp +2 -2
- package/src/duckdb/src/storage/data_table.cpp +1 -1
- package/src/duckdb/src/storage/statistics/numeric_stats.cpp +145 -83
- package/src/duckdb/src/storage/statistics/numeric_stats_union.cpp +65 -0
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +1 -6
- package/src/duckdb/src/storage/table/column_data.cpp +29 -35
- package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +5 -5
- package/src/duckdb/src/storage/table/column_segment.cpp +8 -7
- package/src/duckdb/src/storage/table/list_column_data.cpp +2 -1
- package/src/duckdb/src/storage/table/persistent_table_data.cpp +2 -1
- package/src/duckdb/src/storage/table/row_group.cpp +9 -9
- package/src/duckdb/src/storage/table/row_group_collection.cpp +82 -66
- package/src/duckdb/src/storage/table/scan_state.cpp +22 -3
- package/src/duckdb/src/storage/table/standard_column_data.cpp +1 -0
- package/src/duckdb/src/storage/table/struct_column_data.cpp +1 -0
- package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +11780 -11512
- package/src/duckdb/ub_src_storage_statistics.cpp +2 -0
- package/src/duckdb/ub_src_storage_table.cpp +0 -2
- package/src/duckdb/src/storage/table/segment_tree.cpp +0 -179
@@ -9,7 +9,6 @@
|
|
9
9
|
#pragma once
|
10
10
|
|
11
11
|
#include "duckdb/common/vector_size.hpp"
|
12
|
-
#include "duckdb/storage/table/segment_base.hpp"
|
13
12
|
#include "duckdb/storage/table/chunk_info.hpp"
|
14
13
|
#include "duckdb/storage/table/append_state.hpp"
|
15
14
|
#include "duckdb/storage/table/scan_state.hpp"
|
@@ -17,6 +16,7 @@
|
|
17
16
|
#include "duckdb/common/enums/scan_options.hpp"
|
18
17
|
#include "duckdb/common/mutex.hpp"
|
19
18
|
#include "duckdb/parser/column_list.hpp"
|
19
|
+
#include "duckdb/storage/table/segment_base.hpp"
|
20
20
|
|
21
21
|
namespace duckdb {
|
22
22
|
class AttachedDatabase;
|
@@ -42,7 +42,7 @@ struct RowGroupWriteData {
|
|
42
42
|
vector<BaseStatistics> statistics;
|
43
43
|
};
|
44
44
|
|
45
|
-
class RowGroup : public SegmentBase {
|
45
|
+
class RowGroup : public SegmentBase<RowGroup> {
|
46
46
|
public:
|
47
47
|
friend class ColumnData;
|
48
48
|
friend class VersionDeleteState;
|
@@ -58,6 +58,9 @@ public:
|
|
58
58
|
RowGroup(RowGroup &row_group, idx_t start);
|
59
59
|
~RowGroup();
|
60
60
|
|
61
|
+
//! The index within the segment tree
|
62
|
+
idx_t index;
|
63
|
+
|
61
64
|
private:
|
62
65
|
//! The database instance
|
63
66
|
AttachedDatabase &db;
|
@@ -135,7 +138,7 @@ public:
|
|
135
138
|
RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
|
136
139
|
RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
|
137
140
|
static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
|
138
|
-
static RowGroupPointer Deserialize(Deserializer &source, const
|
141
|
+
static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
|
139
142
|
|
140
143
|
void InitializeAppend(RowGroupAppendState &append_state);
|
141
144
|
void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
|
@@ -23,6 +23,8 @@ class TableStatistics;
|
|
23
23
|
|
24
24
|
class BoundConstraint;
|
25
25
|
|
26
|
+
class RowGroupSegmentTree;
|
27
|
+
|
26
28
|
class RowGroupCollection {
|
27
29
|
public:
|
28
30
|
RowGroupCollection(shared_ptr<DataTableInfo> info, BlockManager &block_manager, vector<LogicalType> types,
|
@@ -46,8 +48,8 @@ public:
|
|
46
48
|
void InitializeCreateIndexScan(CreateIndexScanState &state);
|
47
49
|
void InitializeScanWithOffset(CollectionScanState &state, const vector<column_t> &column_ids, idx_t start_row,
|
48
50
|
idx_t end_row);
|
49
|
-
static bool InitializeScanInRowGroup(CollectionScanState &state,
|
50
|
-
idx_t max_row);
|
51
|
+
static bool InitializeScanInRowGroup(CollectionScanState &state, ParallelCollectionScanState ¶llel_state,
|
52
|
+
idx_t vector_index, idx_t max_row);
|
51
53
|
void InitializeParallelScan(ParallelCollectionScanState &state);
|
52
54
|
bool NextParallelScan(ClientContext &context, ParallelCollectionScanState &state, CollectionScanState &scan_state);
|
53
55
|
|
@@ -109,7 +111,7 @@ private:
|
|
109
111
|
vector<LogicalType> types;
|
110
112
|
idx_t row_start;
|
111
113
|
//! The segment trees holding the various row_groups of the table
|
112
|
-
shared_ptr<
|
114
|
+
shared_ptr<RowGroupSegmentTree> row_groups;
|
113
115
|
//! Table statistics
|
114
116
|
TableStatistics stats;
|
115
117
|
};
|
@@ -0,0 +1,37 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/storage/table/row_group_segment_tree.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/storage/table/segment_tree.hpp"
|
12
|
+
#include "duckdb/storage/table/row_group.hpp"
|
13
|
+
|
14
|
+
namespace duckdb {
|
15
|
+
struct DataTableInfo;
|
16
|
+
class PersistentTableData;
|
17
|
+
class MetaBlockReader;
|
18
|
+
|
19
|
+
class RowGroupSegmentTree : public SegmentTree<RowGroup, true> {
|
20
|
+
public:
|
21
|
+
RowGroupSegmentTree(DataTableInfo &table_info_p, BlockManager &block_manager_p, vector<LogicalType> column_types_p);
|
22
|
+
~RowGroupSegmentTree() override;
|
23
|
+
|
24
|
+
void Initialize(PersistentTableData &data);
|
25
|
+
|
26
|
+
protected:
|
27
|
+
unique_ptr<RowGroup> LoadSegment() override;
|
28
|
+
|
29
|
+
DataTableInfo &info;
|
30
|
+
BlockManager &block_manager;
|
31
|
+
vector<LogicalType> column_types;
|
32
|
+
idx_t current_row_group;
|
33
|
+
idx_t max_row_group;
|
34
|
+
unique_ptr<MetaBlockReader> reader;
|
35
|
+
};
|
36
|
+
|
37
|
+
} // namespace duckdb
|
@@ -25,10 +25,12 @@ class RowGroupCollection;
|
|
25
25
|
class UpdateSegment;
|
26
26
|
class TableScanState;
|
27
27
|
class ColumnSegment;
|
28
|
+
class ColumnSegmentTree;
|
28
29
|
class ValiditySegment;
|
29
30
|
class TableFilterSet;
|
30
31
|
class ColumnData;
|
31
32
|
class DuckTransaction;
|
33
|
+
class RowGroupSegmentTree;
|
32
34
|
|
33
35
|
struct SegmentScanState {
|
34
36
|
virtual ~SegmentScanState() {
|
@@ -45,6 +47,8 @@ typedef unordered_map<block_id_t, BufferHandle> buffer_handle_set_t;
|
|
45
47
|
struct ColumnScanState {
|
46
48
|
//! The column segment that is currently being scanned
|
47
49
|
ColumnSegment *current = nullptr;
|
50
|
+
//! Column segment tree
|
51
|
+
ColumnSegmentTree *segment_tree = nullptr;
|
48
52
|
//! The current row index of the scan
|
49
53
|
idx_t row_index = 0;
|
50
54
|
//! The internal row index (i.e. the position of the SegmentScanState)
|
@@ -114,10 +118,12 @@ private:
|
|
114
118
|
class CollectionScanState {
|
115
119
|
public:
|
116
120
|
CollectionScanState(TableScanState &parent_p)
|
117
|
-
: row_group_state(*this), max_row(0), batch_index(0), parent(parent_p) {};
|
121
|
+
: row_group_state(*this), row_groups(nullptr), max_row(0), batch_index(0), parent(parent_p) {};
|
118
122
|
|
119
123
|
//! The row_group scan state
|
120
124
|
RowGroupScanState row_group_state;
|
125
|
+
//! Row group segment tree
|
126
|
+
RowGroupSegmentTree *row_groups;
|
121
127
|
//! The total maximum row index
|
122
128
|
idx_t max_row;
|
123
129
|
//! The current batch index
|
@@ -129,6 +135,7 @@ public:
|
|
129
135
|
AdaptiveFilter *GetAdaptiveFilter();
|
130
136
|
bool Scan(DuckTransaction &transaction, DataChunk &result);
|
131
137
|
bool ScanCommitted(DataChunk &result, TableScanType type);
|
138
|
+
bool ScanCommitted(DataChunk &result, SegmentLock &l, TableScanType type);
|
132
139
|
|
133
140
|
private:
|
134
141
|
TableScanState &parent;
|
@@ -13,13 +13,14 @@
|
|
13
13
|
|
14
14
|
namespace duckdb {
|
15
15
|
|
16
|
+
template <class T>
|
16
17
|
class SegmentBase {
|
17
18
|
public:
|
18
19
|
SegmentBase(idx_t start, idx_t count) : start(start), count(count), next(nullptr) {
|
19
20
|
}
|
20
21
|
virtual ~SegmentBase() {
|
21
22
|
}
|
22
|
-
|
23
|
+
T *Next() {
|
23
24
|
#ifndef DUCKDB_R_BUILD
|
24
25
|
return next.load();
|
25
26
|
#else
|
@@ -34,9 +35,9 @@ public:
|
|
34
35
|
//! The next segment after this one
|
35
36
|
|
36
37
|
#ifndef DUCKDB_R_BUILD
|
37
|
-
atomic<
|
38
|
+
atomic<T *> next;
|
38
39
|
#else
|
39
|
-
|
40
|
+
T *next;
|
40
41
|
#endif
|
41
42
|
};
|
42
43
|
|
@@ -10,71 +10,316 @@
|
|
10
10
|
|
11
11
|
#include "duckdb/common/constants.hpp"
|
12
12
|
#include "duckdb/storage/storage_lock.hpp"
|
13
|
-
#include "duckdb/storage/table/segment_base.hpp"
|
14
13
|
#include "duckdb/storage/table/segment_lock.hpp"
|
15
14
|
#include "duckdb/common/vector.hpp"
|
16
15
|
#include "duckdb/common/mutex.hpp"
|
16
|
+
#include "duckdb/common/string_util.hpp"
|
17
17
|
|
18
18
|
namespace duckdb {
|
19
19
|
|
20
|
+
template <class T>
|
20
21
|
struct SegmentNode {
|
21
22
|
idx_t row_start;
|
22
|
-
unique_ptr<
|
23
|
+
unique_ptr<T> node;
|
23
24
|
};
|
24
25
|
|
25
26
|
//! The SegmentTree maintains a list of all segments of a specific column in a table, and allows searching for a segment
|
26
27
|
//! by row number
|
28
|
+
template <class T, bool SUPPORTS_LAZY_LOADING = false>
|
27
29
|
class SegmentTree {
|
30
|
+
private:
|
31
|
+
class SegmentIterationHelper;
|
32
|
+
|
28
33
|
public:
|
34
|
+
explicit SegmentTree() : finished_loading(true) {
|
35
|
+
}
|
36
|
+
virtual ~SegmentTree() {
|
37
|
+
}
|
38
|
+
|
29
39
|
//! Locks the segment tree. All methods to the segment tree either lock the segment tree, or take an already
|
30
40
|
//! obtained lock.
|
31
|
-
SegmentLock Lock()
|
41
|
+
SegmentLock Lock() {
|
42
|
+
return SegmentLock(node_lock);
|
43
|
+
}
|
32
44
|
|
33
|
-
bool IsEmpty(SegmentLock &)
|
45
|
+
bool IsEmpty(SegmentLock &l) {
|
46
|
+
return GetRootSegment(l) == nullptr;
|
47
|
+
}
|
34
48
|
|
35
49
|
//! Gets a pointer to the first segment. Useful for scans.
|
36
|
-
|
37
|
-
|
50
|
+
T *GetRootSegment() {
|
51
|
+
auto l = Lock();
|
52
|
+
return GetRootSegment(l);
|
53
|
+
}
|
54
|
+
|
55
|
+
T *GetRootSegment(SegmentLock &l) {
|
56
|
+
if (nodes.empty()) {
|
57
|
+
LoadNextSegment(l);
|
58
|
+
}
|
59
|
+
return nodes.empty() ? nullptr : nodes[0].node.get();
|
60
|
+
}
|
38
61
|
//! Obtains ownership of the data of the segment tree
|
39
|
-
vector<SegmentNode
|
62
|
+
vector<SegmentNode<T>> MoveSegments(SegmentLock &l) {
|
63
|
+
LoadAllSegments(l);
|
64
|
+
return std::move(nodes);
|
65
|
+
}
|
40
66
|
//! Gets a pointer to the nth segment. Negative numbers start from the back.
|
41
|
-
|
42
|
-
|
67
|
+
T *GetSegmentByIndex(int64_t index) {
|
68
|
+
auto l = Lock();
|
69
|
+
return GetSegmentByIndex(l, index);
|
70
|
+
}
|
71
|
+
T *GetSegmentByIndex(SegmentLock &l, int64_t index) {
|
72
|
+
if (index < 0) {
|
73
|
+
// load all segments
|
74
|
+
LoadAllSegments(l);
|
75
|
+
index = nodes.size() + index;
|
76
|
+
if (index < 0) {
|
77
|
+
return nullptr;
|
78
|
+
}
|
79
|
+
return nodes[index].node.get();
|
80
|
+
} else {
|
81
|
+
// lazily load segments until we reach the specific segment
|
82
|
+
while (idx_t(index) >= nodes.size() && LoadNextSegment(l)) {
|
83
|
+
}
|
84
|
+
if (idx_t(index) >= nodes.size()) {
|
85
|
+
return nullptr;
|
86
|
+
}
|
87
|
+
return nodes[index].node.get();
|
88
|
+
}
|
89
|
+
}
|
90
|
+
//! Gets the next segment
|
91
|
+
T *GetNextSegment(T *segment) {
|
92
|
+
if (!SUPPORTS_LAZY_LOADING) {
|
93
|
+
return segment->Next();
|
94
|
+
}
|
95
|
+
if (finished_loading) {
|
96
|
+
return segment->Next();
|
97
|
+
}
|
98
|
+
auto l = Lock();
|
99
|
+
return GetNextSegment(l, segment);
|
100
|
+
}
|
101
|
+
T *GetNextSegment(SegmentLock &l, T *segment) {
|
102
|
+
if (!segment) {
|
103
|
+
return nullptr;
|
104
|
+
}
|
105
|
+
#ifdef DEBUG
|
106
|
+
D_ASSERT(nodes[segment->index].node.get() == segment);
|
107
|
+
#endif
|
108
|
+
return GetSegmentByIndex(l, segment->index + 1);
|
109
|
+
}
|
43
110
|
|
44
111
|
//! Gets a pointer to the last segment. Useful for appends.
|
45
|
-
|
46
|
-
|
112
|
+
T *GetLastSegment(SegmentLock &l) {
|
113
|
+
LoadAllSegments(l);
|
114
|
+
if (nodes.empty()) {
|
115
|
+
return nullptr;
|
116
|
+
}
|
117
|
+
return nodes.back().node.get();
|
118
|
+
}
|
47
119
|
//! Gets a pointer to a specific column segment for the given row
|
48
|
-
|
49
|
-
|
120
|
+
T *GetSegment(idx_t row_number) {
|
121
|
+
auto l = Lock();
|
122
|
+
return GetSegment(l, row_number);
|
123
|
+
}
|
124
|
+
T *GetSegment(SegmentLock &l, idx_t row_number) {
|
125
|
+
return nodes[GetSegmentIndex(l, row_number)].node.get();
|
126
|
+
}
|
50
127
|
|
51
128
|
//! Append a column segment to the tree
|
52
|
-
void
|
53
|
-
|
129
|
+
void AppendSegmentInternal(SegmentLock &l, unique_ptr<T> segment) {
|
130
|
+
D_ASSERT(segment);
|
131
|
+
// add the node to the list of nodes
|
132
|
+
if (!nodes.empty()) {
|
133
|
+
nodes.back().node->next = segment.get();
|
134
|
+
}
|
135
|
+
SegmentNode<T> node;
|
136
|
+
segment->index = nodes.size();
|
137
|
+
node.row_start = segment->start;
|
138
|
+
node.node = std::move(segment);
|
139
|
+
nodes.push_back(std::move(node));
|
140
|
+
}
|
141
|
+
void AppendSegment(unique_ptr<T> segment) {
|
142
|
+
auto l = Lock();
|
143
|
+
AppendSegment(l, std::move(segment));
|
144
|
+
}
|
145
|
+
void AppendSegment(SegmentLock &l, unique_ptr<T> segment) {
|
146
|
+
LoadAllSegments(l);
|
147
|
+
AppendSegmentInternal(l, std::move(segment));
|
148
|
+
}
|
54
149
|
//! Debug method, check whether the segment is in the segment tree
|
55
|
-
bool HasSegment(
|
56
|
-
|
150
|
+
bool HasSegment(T *segment) {
|
151
|
+
auto l = Lock();
|
152
|
+
return HasSegment(l, segment);
|
153
|
+
}
|
154
|
+
bool HasSegment(SegmentLock &, T *segment) {
|
155
|
+
return segment->index < nodes.size() && nodes[segment->index].node.get() == segment;
|
156
|
+
}
|
57
157
|
|
58
158
|
//! Replace this tree with another tree, taking over its nodes in-place
|
59
|
-
void Replace(SegmentTree &other)
|
60
|
-
|
159
|
+
void Replace(SegmentTree<T> &other) {
|
160
|
+
auto l = Lock();
|
161
|
+
Replace(l, other);
|
162
|
+
}
|
163
|
+
void Replace(SegmentLock &l, SegmentTree<T> &other) {
|
164
|
+
other.LoadAllSegments(l);
|
165
|
+
nodes = std::move(other.nodes);
|
166
|
+
}
|
61
167
|
|
62
168
|
//! Erase all segments after a specific segment
|
63
|
-
void EraseSegments(SegmentLock
|
169
|
+
void EraseSegments(SegmentLock &l, idx_t segment_start) {
|
170
|
+
LoadAllSegments(l);
|
171
|
+
if (segment_start >= nodes.size() - 1) {
|
172
|
+
return;
|
173
|
+
}
|
174
|
+
nodes.erase(nodes.begin() + segment_start + 1, nodes.end());
|
175
|
+
}
|
64
176
|
|
65
177
|
//! Get the segment index of the column segment for the given row
|
66
|
-
idx_t GetSegmentIndex(idx_t row_number)
|
67
|
-
|
68
|
-
|
178
|
+
idx_t GetSegmentIndex(SegmentLock &l, idx_t row_number) {
|
179
|
+
idx_t segment_index;
|
180
|
+
if (TryGetSegmentIndex(l, row_number, segment_index)) {
|
181
|
+
return segment_index;
|
182
|
+
}
|
183
|
+
string error;
|
184
|
+
error = StringUtil::Format("Attempting to find row number \"%lld\" in %lld nodes\n", row_number, nodes.size());
|
185
|
+
for (idx_t i = 0; i < nodes.size(); i++) {
|
186
|
+
error += StringUtil::Format("Node %lld: Start %lld, Count %lld", i, nodes[i].row_start,
|
187
|
+
nodes[i].node->count.load());
|
188
|
+
}
|
189
|
+
throw InternalException("Could not find node in column segment tree!\n%s%s", error, Exception::GetStackTrace());
|
190
|
+
}
|
191
|
+
|
192
|
+
bool TryGetSegmentIndex(SegmentLock &l, idx_t row_number, idx_t &result) {
|
193
|
+
// load segments until the row number is within bounds
|
194
|
+
while (nodes.empty() || (row_number >= (nodes.back().row_start + nodes.back().node->count))) {
|
195
|
+
if (!LoadNextSegment(l)) {
|
196
|
+
break;
|
197
|
+
}
|
198
|
+
}
|
199
|
+
if (nodes.empty()) {
|
200
|
+
return false;
|
201
|
+
}
|
202
|
+
D_ASSERT(!nodes.empty());
|
203
|
+
D_ASSERT(row_number >= nodes[0].row_start);
|
204
|
+
D_ASSERT(row_number < nodes.back().row_start + nodes.back().node->count);
|
205
|
+
idx_t lower = 0;
|
206
|
+
idx_t upper = nodes.size() - 1;
|
207
|
+
// binary search to find the node
|
208
|
+
while (lower <= upper) {
|
209
|
+
idx_t index = (lower + upper) / 2;
|
210
|
+
D_ASSERT(index < nodes.size());
|
211
|
+
auto &entry = nodes[index];
|
212
|
+
D_ASSERT(entry.row_start == entry.node->start);
|
213
|
+
if (row_number < entry.row_start) {
|
214
|
+
upper = index - 1;
|
215
|
+
} else if (row_number >= entry.row_start + entry.node->count) {
|
216
|
+
lower = index + 1;
|
217
|
+
} else {
|
218
|
+
result = index;
|
219
|
+
return true;
|
220
|
+
}
|
221
|
+
}
|
222
|
+
return false;
|
223
|
+
}
|
69
224
|
|
70
|
-
void Verify(SegmentLock &)
|
71
|
-
|
225
|
+
void Verify(SegmentLock &) {
|
226
|
+
#ifdef DEBUG
|
227
|
+
idx_t base_start = nodes.empty() ? 0 : nodes[0].node->start;
|
228
|
+
for (idx_t i = 0; i < nodes.size(); i++) {
|
229
|
+
D_ASSERT(nodes[i].row_start == nodes[i].node->start);
|
230
|
+
D_ASSERT(nodes[i].node->start == base_start);
|
231
|
+
base_start += nodes[i].node->count;
|
232
|
+
}
|
233
|
+
#endif
|
234
|
+
}
|
235
|
+
void Verify() {
|
236
|
+
#ifdef DEBUG
|
237
|
+
auto l = Lock();
|
238
|
+
Verify(l);
|
239
|
+
#endif
|
240
|
+
}
|
241
|
+
|
242
|
+
SegmentIterationHelper Segments() {
|
243
|
+
return SegmentIterationHelper(*this);
|
244
|
+
}
|
245
|
+
|
246
|
+
protected:
|
247
|
+
atomic<bool> finished_loading;
|
248
|
+
|
249
|
+
//! Load the next segment - only used when lazily loading
|
250
|
+
virtual unique_ptr<T> LoadSegment() {
|
251
|
+
return nullptr;
|
252
|
+
}
|
72
253
|
|
73
254
|
private:
|
74
255
|
//! The nodes in the tree, can be binary searched
|
75
|
-
vector<SegmentNode
|
256
|
+
vector<SegmentNode<T>> nodes;
|
76
257
|
//! Lock to access or modify the nodes
|
77
258
|
mutex node_lock;
|
259
|
+
|
260
|
+
private:
|
261
|
+
class SegmentIterationHelper {
|
262
|
+
public:
|
263
|
+
explicit SegmentIterationHelper(SegmentTree &tree) : tree(tree) {
|
264
|
+
}
|
265
|
+
|
266
|
+
private:
|
267
|
+
SegmentTree &tree;
|
268
|
+
|
269
|
+
private:
|
270
|
+
class SegmentIterator {
|
271
|
+
public:
|
272
|
+
SegmentIterator(SegmentTree &tree_p, T *current_p) : tree(tree_p), current(current_p) {
|
273
|
+
}
|
274
|
+
|
275
|
+
SegmentTree &tree;
|
276
|
+
T *current;
|
277
|
+
|
278
|
+
public:
|
279
|
+
void Next() {
|
280
|
+
current = tree.GetNextSegment(current);
|
281
|
+
}
|
282
|
+
|
283
|
+
SegmentIterator &operator++() {
|
284
|
+
Next();
|
285
|
+
return *this;
|
286
|
+
}
|
287
|
+
bool operator!=(const SegmentIterator &other) const {
|
288
|
+
return current != other.current;
|
289
|
+
}
|
290
|
+
T &operator*() const {
|
291
|
+
D_ASSERT(current);
|
292
|
+
return *current;
|
293
|
+
}
|
294
|
+
};
|
295
|
+
|
296
|
+
public:
|
297
|
+
SegmentIterator begin() {
|
298
|
+
return SegmentIterator(tree, tree.GetRootSegment());
|
299
|
+
}
|
300
|
+
SegmentIterator end() {
|
301
|
+
return SegmentIterator(tree, nullptr);
|
302
|
+
}
|
303
|
+
};
|
304
|
+
|
305
|
+
//! Load the next segment, if there are any left to load
|
306
|
+
bool LoadNextSegment(SegmentLock &l) {
|
307
|
+
if (finished_loading) {
|
308
|
+
return false;
|
309
|
+
}
|
310
|
+
auto result = LoadSegment();
|
311
|
+
if (result) {
|
312
|
+
AppendSegmentInternal(l, std::move(result));
|
313
|
+
return true;
|
314
|
+
}
|
315
|
+
return false;
|
316
|
+
}
|
317
|
+
|
318
|
+
//! Load all segments, if there are any left to load
|
319
|
+
void LoadAllSegments(SegmentLock &l) {
|
320
|
+
while (LoadNextSegment(l))
|
321
|
+
;
|
322
|
+
}
|
78
323
|
};
|
79
324
|
|
80
325
|
} // namespace duckdb
|
@@ -49,6 +49,17 @@ unique_ptr<ParsedExpression> Transformer::TransformArrayAccess(duckdb_libpgquery
|
|
49
49
|
result = make_unique<OperatorExpression>(ExpressionType::STRUCT_EXTRACT, std::move(children));
|
50
50
|
break;
|
51
51
|
}
|
52
|
+
case duckdb_libpgquery::T_PGFuncCall: {
|
53
|
+
auto func = (duckdb_libpgquery::PGFuncCall *)target;
|
54
|
+
auto function = TransformFuncCall(func);
|
55
|
+
if (function->type != ExpressionType::FUNCTION) {
|
56
|
+
throw ParserException("%s.%s() call must be a function", result->ToString(), function->ToString());
|
57
|
+
}
|
58
|
+
auto &f = (FunctionExpression &)*function;
|
59
|
+
f.children.insert(f.children.begin(), std::move(result));
|
60
|
+
result = std::move(function);
|
61
|
+
break;
|
62
|
+
}
|
52
63
|
default:
|
53
64
|
throw NotImplementedException("Unimplemented subscript type");
|
54
65
|
}
|
@@ -38,10 +38,28 @@ BindResult ExpressionBinder::BindExpression(FunctionExpression &function, idx_t
|
|
38
38
|
"function has to be called in a FROM clause (similar to a table).",
|
39
39
|
function.function_name)));
|
40
40
|
}
|
41
|
-
// not a table function -
|
42
|
-
|
43
|
-
|
44
|
-
|
41
|
+
// not a table function - check if the schema is set
|
42
|
+
if (!function.schema.empty()) {
|
43
|
+
// the schema is set - check if we can turn this the schema into a column ref
|
44
|
+
string error;
|
45
|
+
unique_ptr<ColumnRefExpression> colref;
|
46
|
+
if (function.catalog.empty()) {
|
47
|
+
colref = make_unique<ColumnRefExpression>(function.schema);
|
48
|
+
} else {
|
49
|
+
colref = make_unique<ColumnRefExpression>(function.schema, function.catalog);
|
50
|
+
}
|
51
|
+
auto new_colref = QualifyColumnName(*colref, error);
|
52
|
+
if (error.empty()) {
|
53
|
+
// we can! transform this into a function call on the column
|
54
|
+
// i.e. "x.lower()" becomes "lower(x)"
|
55
|
+
function.children.insert(function.children.begin(), std::move(colref));
|
56
|
+
function.catalog = INVALID_CATALOG;
|
57
|
+
function.schema = INVALID_SCHEMA;
|
58
|
+
}
|
59
|
+
}
|
60
|
+
// rebind the function
|
61
|
+
func = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, function.catalog, function.schema,
|
62
|
+
function.function_name, false, error_context);
|
45
63
|
}
|
46
64
|
|
47
65
|
if (func->type != CatalogType::AGGREGATE_FUNCTION_ENTRY &&
|
@@ -1,7 +1,5 @@
|
|
1
1
|
#include "duckdb/storage/checkpoint/table_data_reader.hpp"
|
2
2
|
#include "duckdb/storage/meta_block_reader.hpp"
|
3
|
-
|
4
|
-
#include "duckdb/common/vector_operations/vector_operations.hpp"
|
5
3
|
#include "duckdb/common/types/null_value.hpp"
|
6
4
|
|
7
5
|
#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
|
@@ -9,9 +7,6 @@
|
|
9
7
|
#include "duckdb/planner/parsed_data/bound_create_table_info.hpp"
|
10
8
|
|
11
9
|
#include "duckdb/main/database.hpp"
|
12
|
-
#include "duckdb/main/client_context.hpp"
|
13
|
-
|
14
|
-
#include "duckdb/storage/table/row_group.hpp"
|
15
10
|
|
16
11
|
namespace duckdb {
|
17
12
|
|
@@ -27,12 +22,9 @@ void TableDataReader::ReadTableData() {
|
|
27
22
|
info.data->table_stats.Deserialize(reader, columns);
|
28
23
|
|
29
24
|
// deserialize each of the individual row groups
|
30
|
-
|
31
|
-
info.data->
|
32
|
-
|
33
|
-
auto row_group_pointer = RowGroup::Deserialize(reader, columns);
|
34
|
-
info.data->row_groups.push_back(std::move(row_group_pointer));
|
35
|
-
}
|
25
|
+
info.data->row_group_count = reader.Read<uint64_t>();
|
26
|
+
info.data->block_id = reader.block->BlockId();
|
27
|
+
info.data->offset = reader.offset;
|
36
28
|
}
|
37
29
|
|
38
30
|
} // namespace duckdb
|
@@ -50,13 +50,19 @@ void SingleFileTableDataWriter::FinalizeTable(TableStatistics &&global_stats, Da
|
|
50
50
|
|
51
51
|
// now start writing the row group pointers to disk
|
52
52
|
table_data_writer.Write<uint64_t>(row_group_pointers.size());
|
53
|
+
idx_t total_rows = 0;
|
53
54
|
for (auto &row_group_pointer : row_group_pointers) {
|
55
|
+
auto row_group_count = row_group_pointer.row_start + row_group_pointer.tuple_count;
|
56
|
+
if (row_group_count > total_rows) {
|
57
|
+
total_rows = row_group_count;
|
58
|
+
}
|
54
59
|
RowGroup::Serialize(row_group_pointer, table_data_writer);
|
55
60
|
}
|
56
61
|
|
57
62
|
// Pointer to the table itself goes to the metadata stream.
|
58
63
|
meta_data_writer.Write<block_id_t>(pointer.block_id);
|
59
64
|
meta_data_writer.Write<uint64_t>(pointer.offset);
|
65
|
+
meta_data_writer.Write<idx_t>(total_rows);
|
60
66
|
|
61
67
|
// Now we serialize indexes in the table_metadata_writer
|
62
68
|
std::vector<BlockPointer> index_pointers = info->indexes.SerializeIndexes(table_data_writer);
|
@@ -467,6 +467,7 @@ void CheckpointReader::ReadTableData(ClientContext &context, MetaBlockReader &re
|
|
467
467
|
TableDataReader data_reader(table_data_reader, bound_info);
|
468
468
|
|
469
469
|
data_reader.ReadTableData();
|
470
|
+
bound_info.data->total_rows = reader.Read<idx_t>();
|
470
471
|
|
471
472
|
// Get any indexes block info
|
472
473
|
idx_t num_indexes = reader.Read<idx_t>();
|
@@ -33,7 +33,7 @@ void ConstantFillFunction(ColumnSegment &segment, Vector &result, idx_t start_id
|
|
33
33
|
auto &nstats = segment.stats.statistics;
|
34
34
|
|
35
35
|
auto data = FlatVector::GetData<T>(result);
|
36
|
-
auto constant_value = NumericStats::
|
36
|
+
auto constant_value = NumericStats::GetMin<T>(nstats);
|
37
37
|
for (idx_t i = 0; i < count; i++) {
|
38
38
|
data[start_idx + i] = constant_value;
|
39
39
|
}
|
@@ -71,7 +71,7 @@ void ConstantScanFunction(ColumnSegment &segment, ColumnScanState &state, idx_t
|
|
71
71
|
auto &nstats = segment.stats.statistics;
|
72
72
|
|
73
73
|
auto data = FlatVector::GetData<T>(result);
|
74
|
-
data[0] = NumericStats::
|
74
|
+
data[0] = NumericStats::GetMin<T>(nstats);
|
75
75
|
result.SetVectorType(VectorType::CONSTANT_VECTOR);
|
76
76
|
}
|
77
77
|
|
@@ -45,7 +45,7 @@ DataTable::DataTable(AttachedDatabase &db, shared_ptr<TableIOManager> table_io_m
|
|
45
45
|
auto types = GetTypes();
|
46
46
|
this->row_groups =
|
47
47
|
make_shared<RowGroupCollection>(info, TableIOManager::Get(*this).GetBlockManagerForRowData(), types, 0);
|
48
|
-
if (data &&
|
48
|
+
if (data && data->row_group_count > 0) {
|
49
49
|
this->row_groups->Initialize(*data);
|
50
50
|
} else {
|
51
51
|
this->row_groups->InitializeEmpty();
|