duckdb 0.8.2-dev2356.0 → 0.8.2-dev2399.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +7 -7
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-datefunc.cpp +9 -0
- package/src/duckdb/extension/icu/icu-datepart.cpp +7 -5
- package/src/duckdb/extension/icu/icu-strptime.cpp +1 -20
- package/src/duckdb/src/common/types/list_segment.cpp +42 -134
- package/src/duckdb/src/common/types/vector.cpp +21 -0
- package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +5 -7
- package/src/duckdb/src/core_functions/aggregate/holistic/quantile.cpp +17 -19
- package/src/duckdb/src/core_functions/aggregate/nested/list.cpp +80 -61
- package/src/duckdb/src/execution/aggregate_hashtable.cpp +6 -0
- package/src/duckdb/src/execution/perfect_aggregate_hashtable.cpp +11 -5
- package/src/duckdb/src/execution/window_executor.cpp +18 -20
- package/src/duckdb/src/function/aggregate/distributive/count.cpp +2 -2
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/types/list_segment.hpp +9 -11
- package/src/duckdb/src/include/duckdb/common/types/vector.hpp +7 -0
- package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp +7 -2
- package/src/duckdb/src/include/duckdb/execution/perfect_aggregate_hashtable.hpp +4 -2
- package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +0 -2
- package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +0 -1
- package/src/duckdb/src/parser/transform/expression/transform_function.cpp +3 -0
- package/src/duckdb/src/storage/serialization/serialize_constraint.cpp +2 -2
- package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +2 -2
- package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +2 -2
- package/src/duckdb/src/storage/serialization/serialize_tableref.cpp +2 -4
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
@@ -5,24 +5,6 @@
|
|
5
5
|
|
6
6
|
namespace duckdb {
|
7
7
|
|
8
|
-
static void RecursiveFlatten(Vector &vector, idx_t &count) {
|
9
|
-
if (vector.GetVectorType() != VectorType::FLAT_VECTOR) {
|
10
|
-
vector.Flatten(count);
|
11
|
-
}
|
12
|
-
|
13
|
-
auto internal_type = vector.GetType().InternalType();
|
14
|
-
if (internal_type == PhysicalType::LIST) {
|
15
|
-
auto &child_vector = ListVector::GetEntry(vector);
|
16
|
-
auto child_vector_count = ListVector::GetListSize(vector);
|
17
|
-
RecursiveFlatten(child_vector, child_vector_count);
|
18
|
-
} else if (internal_type == PhysicalType::STRUCT) {
|
19
|
-
auto &children = StructVector::GetEntries(vector);
|
20
|
-
for (auto &child : children) {
|
21
|
-
RecursiveFlatten(*child, count);
|
22
|
-
}
|
23
|
-
}
|
24
|
-
}
|
25
|
-
|
26
8
|
struct ListBindData : public FunctionData {
|
27
9
|
explicit ListBindData(const LogicalType &stype_p);
|
28
10
|
~ListBindData() override;
|
@@ -60,12 +42,6 @@ struct ListFunction {
|
|
60
42
|
state.linked_list.first_segment = nullptr;
|
61
43
|
state.linked_list.last_segment = nullptr;
|
62
44
|
}
|
63
|
-
|
64
|
-
template <class STATE>
|
65
|
-
static void Destroy(STATE &state, AggregateInputData &aggr_input_data) {
|
66
|
-
// nop
|
67
|
-
}
|
68
|
-
|
69
45
|
static bool IgnoreNull() {
|
70
46
|
return false;
|
71
47
|
}
|
@@ -73,58 +49,54 @@ struct ListFunction {
|
|
73
49
|
|
74
50
|
static void ListUpdateFunction(Vector inputs[], AggregateInputData &aggr_input_data, idx_t input_count,
|
75
51
|
Vector &state_vector, idx_t count) {
|
76
|
-
D_ASSERT(input_count == 1);
|
77
52
|
|
53
|
+
D_ASSERT(input_count == 1);
|
78
54
|
auto &input = inputs[0];
|
79
|
-
|
80
|
-
|
55
|
+
RecursiveUnifiedVectorFormat input_data;
|
56
|
+
Vector::RecursiveToUnifiedFormat(input, count, input_data);
|
81
57
|
|
82
|
-
|
83
|
-
|
58
|
+
UnifiedVectorFormat states_data;
|
59
|
+
state_vector.ToUnifiedFormat(count, states_data);
|
60
|
+
auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
|
84
61
|
|
85
62
|
auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
|
86
63
|
|
87
64
|
for (idx_t i = 0; i < count; i++) {
|
88
|
-
auto &state = *states[
|
89
|
-
list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list,
|
65
|
+
auto &state = *states[states_data.sel->get_index(i)];
|
66
|
+
list_bind_data.functions.AppendRow(aggr_input_data.allocator, state.linked_list, input_data, i);
|
90
67
|
}
|
91
68
|
}
|
92
69
|
|
93
|
-
static void ListCombineFunction(Vector &
|
94
|
-
UnifiedVectorFormat sdata;
|
95
|
-
state.ToUnifiedFormat(count, sdata);
|
96
|
-
auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(sdata);
|
70
|
+
static void ListCombineFunction(Vector &states_vector, Vector &combined, AggregateInputData &, idx_t count) {
|
97
71
|
|
98
|
-
|
72
|
+
UnifiedVectorFormat states_data;
|
73
|
+
states_vector.ToUnifiedFormat(count, states_data);
|
74
|
+
auto states_ptr = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
|
99
75
|
|
100
76
|
auto combined_ptr = FlatVector::GetData<ListAggState *>(combined);
|
101
77
|
for (idx_t i = 0; i < count; i++) {
|
102
|
-
auto &state = *states_ptr[sdata.sel->get_index(i)];
|
103
|
-
if (state.linked_list.total_capacity == 0) {
|
104
|
-
// NULL, no need to append.
|
105
|
-
continue;
|
106
|
-
}
|
107
78
|
|
108
|
-
|
109
|
-
|
110
|
-
list_bind_data.functions.CopyLinkedList(state.linked_list, copied_linked_list, aggr_input_data.allocator);
|
79
|
+
auto &state = *states_ptr[states_data.sel->get_index(i)];
|
80
|
+
D_ASSERT(state.linked_list.total_capacity != 0);
|
111
81
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
} else {
|
116
|
-
combined_ptr[i]->linked_list.first_segment = copied_linked_list.first_segment;
|
82
|
+
if (combined_ptr[i]->linked_list.total_capacity == 0) {
|
83
|
+
combined_ptr[i]->linked_list = state.linked_list;
|
84
|
+
continue;
|
117
85
|
}
|
118
|
-
|
119
|
-
|
86
|
+
|
87
|
+
// append the linked list
|
88
|
+
combined_ptr[i]->linked_list.last_segment->next = state.linked_list.first_segment;
|
89
|
+
combined_ptr[i]->linked_list.last_segment = state.linked_list.last_segment;
|
90
|
+
combined_ptr[i]->linked_list.total_capacity += state.linked_list.total_capacity;
|
120
91
|
}
|
121
92
|
}
|
122
93
|
|
123
|
-
static void ListFinalize(Vector &
|
94
|
+
static void ListFinalize(Vector &states_vector, AggregateInputData &aggr_input_data, Vector &result, idx_t count,
|
124
95
|
idx_t offset) {
|
125
|
-
|
126
|
-
|
127
|
-
|
96
|
+
|
97
|
+
UnifiedVectorFormat states_data;
|
98
|
+
states_vector.ToUnifiedFormat(count, states_data);
|
99
|
+
auto states = UnifiedVectorFormat::GetData<ListAggState *>(states_data);
|
128
100
|
|
129
101
|
D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
|
130
102
|
|
@@ -133,9 +105,11 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
|
|
133
105
|
size_t total_len = ListVector::GetListSize(result);
|
134
106
|
|
135
107
|
auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
|
136
|
-
|
108
|
+
|
109
|
+
// first iterate over all entries and set up the list entries, and get the newly required total length
|
137
110
|
for (idx_t i = 0; i < count; i++) {
|
138
|
-
|
111
|
+
|
112
|
+
auto &state = *states[states_data.sel->get_index(i)];
|
139
113
|
const auto rid = i + offset;
|
140
114
|
result_data[rid].offset = total_len;
|
141
115
|
if (state.linked_list.total_capacity == 0) {
|
@@ -143,16 +117,19 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
|
|
143
117
|
result_data[rid].length = 0;
|
144
118
|
continue;
|
145
119
|
}
|
120
|
+
|
146
121
|
// set the length and offset of this list in the result vector
|
147
122
|
auto total_capacity = state.linked_list.total_capacity;
|
148
123
|
result_data[rid].length = total_capacity;
|
149
124
|
total_len += total_capacity;
|
150
125
|
}
|
151
|
-
|
126
|
+
|
127
|
+
// reserve capacity, then iterate over all entries again and copy over the data to the child vector
|
152
128
|
ListVector::Reserve(result, total_len);
|
153
129
|
auto &result_child = ListVector::GetEntry(result);
|
154
130
|
for (idx_t i = 0; i < count; i++) {
|
155
|
-
|
131
|
+
|
132
|
+
auto &state = *states[states_data.sel->get_index(i)];
|
156
133
|
const auto rid = i + offset;
|
157
134
|
if (state.linked_list.total_capacity == 0) {
|
158
135
|
continue;
|
@@ -161,6 +138,48 @@ static void ListFinalize(Vector &state_vector, AggregateInputData &aggr_input_da
|
|
161
138
|
idx_t current_offset = result_data[rid].offset;
|
162
139
|
list_bind_data.functions.BuildListVector(state.linked_list, result_child, current_offset);
|
163
140
|
}
|
141
|
+
|
142
|
+
ListVector::SetListSize(result, total_len);
|
143
|
+
}
|
144
|
+
|
145
|
+
static void ListWindow(Vector inputs[], const ValidityMask &filter_mask, AggregateInputData &aggr_input_data,
|
146
|
+
idx_t input_count, data_ptr_t state, const FrameBounds &frame, const FrameBounds &prev,
|
147
|
+
Vector &result, idx_t rid, idx_t bias) {
|
148
|
+
|
149
|
+
auto &list_bind_data = aggr_input_data.bind_data->Cast<ListBindData>();
|
150
|
+
LinkedList linked_list;
|
151
|
+
|
152
|
+
// UPDATE step
|
153
|
+
|
154
|
+
D_ASSERT(input_count == 1);
|
155
|
+
auto &input = inputs[0];
|
156
|
+
|
157
|
+
// FIXME: we unify more values than necessary (count is frame.end)
|
158
|
+
RecursiveUnifiedVectorFormat input_data;
|
159
|
+
Vector::RecursiveToUnifiedFormat(input, frame.end, input_data);
|
160
|
+
|
161
|
+
for (idx_t i = frame.start; i < frame.end; i++) {
|
162
|
+
list_bind_data.functions.AppendRow(aggr_input_data.allocator, linked_list, input_data, i);
|
163
|
+
}
|
164
|
+
|
165
|
+
// FINALIZE step
|
166
|
+
|
167
|
+
D_ASSERT(result.GetType().id() == LogicalTypeId::LIST);
|
168
|
+
auto result_data = FlatVector::GetData<list_entry_t>(result);
|
169
|
+
size_t total_len = ListVector::GetListSize(result);
|
170
|
+
|
171
|
+
// set the length and offset of this list in the result vector
|
172
|
+
result_data[rid].offset = total_len;
|
173
|
+
result_data[rid].length = linked_list.total_capacity;
|
174
|
+
D_ASSERT(linked_list.total_capacity != 0);
|
175
|
+
total_len += linked_list.total_capacity;
|
176
|
+
|
177
|
+
// reserve capacity, then copy over the data to the child vector
|
178
|
+
ListVector::Reserve(result, total_len);
|
179
|
+
auto &result_child = ListVector::GetEntry(result);
|
180
|
+
idx_t offset = result_data[rid].offset;
|
181
|
+
list_bind_data.functions.BuildListVector(linked_list, result_child, offset);
|
182
|
+
|
164
183
|
ListVector::SetListSize(result, total_len);
|
165
184
|
}
|
166
185
|
|
@@ -182,8 +201,8 @@ unique_ptr<FunctionData> ListBindFunction(ClientContext &context, AggregateFunct
|
|
182
201
|
AggregateFunction ListFun::GetFunction() {
|
183
202
|
return AggregateFunction({LogicalType::ANY}, LogicalTypeId::LIST, AggregateFunction::StateSize<ListAggState>,
|
184
203
|
AggregateFunction::StateInitialize<ListAggState, ListFunction>, ListUpdateFunction,
|
185
|
-
ListCombineFunction, ListFinalize, nullptr, ListBindFunction,
|
186
|
-
|
204
|
+
ListCombineFunction, ListFinalize, nullptr, ListBindFunction, nullptr, nullptr,
|
205
|
+
ListWindow);
|
187
206
|
}
|
188
207
|
|
189
208
|
} // namespace duckdb
|
@@ -584,6 +584,12 @@ void GroupedAggregateHashTable::Combine(GroupedAggregateHashTable &other) {
|
|
584
584
|
}
|
585
585
|
|
586
586
|
Verify();
|
587
|
+
|
588
|
+
// if we combine states, then we also need to combine the arena allocators
|
589
|
+
for (auto &stored_allocator : other.stored_allocators) {
|
590
|
+
stored_allocators.push_back(stored_allocator);
|
591
|
+
}
|
592
|
+
stored_allocators.push_back(other.aggregate_allocator);
|
587
593
|
}
|
588
594
|
|
589
595
|
void GroupedAggregateHashTable::Append(GroupedAggregateHashTable &other) {
|
@@ -12,7 +12,8 @@ PerfectAggregateHashTable::PerfectAggregateHashTable(ClientContext &context, All
|
|
12
12
|
vector<Value> group_minima_p, vector<idx_t> required_bits_p)
|
13
13
|
: BaseAggregateHashTable(context, allocator, aggregate_objects_p, std::move(payload_types_p)),
|
14
14
|
addresses(LogicalType::POINTER), required_bits(std::move(required_bits_p)), total_required_bits(0),
|
15
|
-
group_minima(std::move(group_minima_p)), sel(STANDARD_VECTOR_SIZE),
|
15
|
+
group_minima(std::move(group_minima_p)), sel(STANDARD_VECTOR_SIZE),
|
16
|
+
aggregate_allocator(make_uniq<ArenaAllocator>(allocator)) {
|
16
17
|
for (auto &group_bits : required_bits) {
|
17
18
|
total_required_bits += group_bits;
|
18
19
|
}
|
@@ -136,7 +137,7 @@ void PerfectAggregateHashTable::AddChunk(DataChunk &groups, DataChunk &payload)
|
|
136
137
|
// after finding the group location we update the aggregates
|
137
138
|
idx_t payload_idx = 0;
|
138
139
|
auto &aggregates = layout.GetAggregates();
|
139
|
-
RowOperationsState row_state(aggregate_allocator);
|
140
|
+
RowOperationsState row_state(*aggregate_allocator);
|
140
141
|
for (idx_t aggr_idx = 0; aggr_idx < aggregates.size(); aggr_idx++) {
|
141
142
|
auto &aggregate = aggregates[aggr_idx];
|
142
143
|
auto input_count = (idx_t)aggregate.child_count;
|
@@ -165,7 +166,7 @@ void PerfectAggregateHashTable::Combine(PerfectAggregateHashTable &other) {
|
|
165
166
|
data_ptr_t source_ptr = other.data;
|
166
167
|
data_ptr_t target_ptr = data;
|
167
168
|
idx_t combine_count = 0;
|
168
|
-
RowOperationsState row_state(aggregate_allocator);
|
169
|
+
RowOperationsState row_state(*aggregate_allocator);
|
169
170
|
for (idx_t i = 0; i < total_groups; i++) {
|
170
171
|
auto has_entry_source = other.group_is_set[i];
|
171
172
|
// we only have any work to do if the source has an entry for this group
|
@@ -183,6 +184,11 @@ void PerfectAggregateHashTable::Combine(PerfectAggregateHashTable &other) {
|
|
183
184
|
target_ptr += tuple_size;
|
184
185
|
}
|
185
186
|
RowOperations::CombineStates(row_state, layout, source_addresses, target_addresses, combine_count);
|
187
|
+
|
188
|
+
// FIXME: after moving the arena allocator, we currently have to ensure that the pointer is not nullptr, because the
|
189
|
+
// FIXME: Destroy()-function of the hash table expects an allocator in some cases (e.g., for sorted aggregates)
|
190
|
+
stored_allocators.push_back(std::move(other.aggregate_allocator));
|
191
|
+
other.aggregate_allocator = make_uniq<ArenaAllocator>(allocator);
|
186
192
|
}
|
187
193
|
|
188
194
|
template <class T>
|
@@ -268,7 +274,7 @@ void PerfectAggregateHashTable::Scan(idx_t &scan_position, DataChunk &result) {
|
|
268
274
|
}
|
269
275
|
// then construct the payloads
|
270
276
|
result.SetCardinality(entry_count);
|
271
|
-
RowOperationsState row_state(aggregate_allocator);
|
277
|
+
RowOperationsState row_state(*aggregate_allocator);
|
272
278
|
RowOperations::FinalizeStates(row_state, layout, addresses, result, grouping_columns);
|
273
279
|
}
|
274
280
|
|
@@ -289,7 +295,7 @@ void PerfectAggregateHashTable::Destroy() {
|
|
289
295
|
idx_t count = 0;
|
290
296
|
|
291
297
|
// iterate over all initialised slots of the hash table
|
292
|
-
RowOperationsState row_state(aggregate_allocator);
|
298
|
+
RowOperationsState row_state(*aggregate_allocator);
|
293
299
|
data_ptr_t payload_ptr = data;
|
294
300
|
for (idx_t i = 0; i < total_groups; i++) {
|
295
301
|
if (group_is_set[i]) {
|
@@ -204,19 +204,19 @@ static idx_t FindTypedRangeBound(const WindowInputColumn &over, const idx_t orde
|
|
204
204
|
WindowColumnIterator<T> begin(over, order_begin);
|
205
205
|
WindowColumnIterator<T> end(over, order_end);
|
206
206
|
|
207
|
-
if (order_begin < prev.
|
208
|
-
const auto first = over.GetCell<T>(prev.
|
207
|
+
if (order_begin < prev.start && prev.start < order_end) {
|
208
|
+
const auto first = over.GetCell<T>(prev.start);
|
209
209
|
if (!comp(val, first)) {
|
210
210
|
// prev.first <= val, so we can start further forward
|
211
|
-
begin += (prev.
|
211
|
+
begin += (prev.start - order_begin);
|
212
212
|
}
|
213
213
|
}
|
214
|
-
if (order_begin <= prev.
|
215
|
-
const auto second = over.GetCell<T>(prev.
|
214
|
+
if (order_begin <= prev.end && prev.end < order_end) {
|
215
|
+
const auto second = over.GetCell<T>(prev.end);
|
216
216
|
if (!comp(second, val)) {
|
217
217
|
// val <= prev.second, so we can end further back
|
218
218
|
// (prev.second is the largest peer)
|
219
|
-
end -= (order_end - prev.
|
219
|
+
end -= (order_end - prev.end - 1);
|
220
220
|
}
|
221
221
|
}
|
222
222
|
|
@@ -278,8 +278,6 @@ static idx_t FindOrderedRangeBound(const WindowInputColumn &over, const OrderTyp
|
|
278
278
|
}
|
279
279
|
|
280
280
|
struct WindowBoundariesState {
|
281
|
-
using FrameBounds = std::pair<idx_t, idx_t>;
|
282
|
-
|
283
281
|
static inline bool IsScalar(const unique_ptr<Expression> &expr) {
|
284
282
|
return expr ? expr->IsScalar() : true;
|
285
283
|
}
|
@@ -375,8 +373,8 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
|
|
375
373
|
}
|
376
374
|
|
377
375
|
// Reset range hints
|
378
|
-
prev.
|
379
|
-
prev.
|
376
|
+
prev.start = valid_start;
|
377
|
+
prev.end = valid_end;
|
380
378
|
}
|
381
379
|
} else if (!is_peer) {
|
382
380
|
peer_start = row_idx;
|
@@ -427,9 +425,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
|
|
427
425
|
if (boundary_start.CellIsNull(chunk_idx)) {
|
428
426
|
window_start = peer_start;
|
429
427
|
} else {
|
430
|
-
prev.
|
428
|
+
prev.start = FindOrderedRangeBound<true>(range_collection, range_sense, valid_start, row_idx,
|
431
429
|
boundary_start, chunk_idx, prev);
|
432
|
-
window_start = prev.
|
430
|
+
window_start = prev.start;
|
433
431
|
}
|
434
432
|
break;
|
435
433
|
}
|
@@ -437,9 +435,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
|
|
437
435
|
if (boundary_start.CellIsNull(chunk_idx)) {
|
438
436
|
window_start = peer_start;
|
439
437
|
} else {
|
440
|
-
prev.
|
438
|
+
prev.start = FindOrderedRangeBound<true>(range_collection, range_sense, row_idx, valid_end, boundary_start,
|
441
439
|
chunk_idx, prev);
|
442
|
-
window_start = prev.
|
440
|
+
window_start = prev.start;
|
443
441
|
}
|
444
442
|
break;
|
445
443
|
}
|
@@ -472,9 +470,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
|
|
472
470
|
if (boundary_end.CellIsNull(chunk_idx)) {
|
473
471
|
window_end = peer_end;
|
474
472
|
} else {
|
475
|
-
prev.
|
476
|
-
|
477
|
-
window_end = prev.
|
473
|
+
prev.end = FindOrderedRangeBound<false>(range_collection, range_sense, valid_start, row_idx, boundary_end,
|
474
|
+
chunk_idx, prev);
|
475
|
+
window_end = prev.end;
|
478
476
|
}
|
479
477
|
break;
|
480
478
|
}
|
@@ -482,9 +480,9 @@ void WindowBoundariesState::Update(const idx_t row_idx, const WindowInputColumn
|
|
482
480
|
if (boundary_end.CellIsNull(chunk_idx)) {
|
483
481
|
window_end = peer_end;
|
484
482
|
} else {
|
485
|
-
prev.
|
486
|
-
|
487
|
-
window_end = prev.
|
483
|
+
prev.end = FindOrderedRangeBound<false>(range_collection, range_sense, row_idx, valid_end, boundary_end,
|
484
|
+
chunk_idx, prev);
|
485
|
+
window_end = prev.end;
|
488
486
|
}
|
489
487
|
break;
|
490
488
|
}
|
@@ -39,8 +39,8 @@ struct CountStarFunction : public BaseCountFunction {
|
|
39
39
|
Vector &result, idx_t rid, idx_t bias) {
|
40
40
|
D_ASSERT(input_count == 0);
|
41
41
|
auto data = FlatVector::GetData<RESULT_TYPE>(result);
|
42
|
-
const auto begin = frame.
|
43
|
-
const auto end = frame.
|
42
|
+
const auto begin = frame.start;
|
43
|
+
const auto end = frame.end;
|
44
44
|
// Slice to any filtered rows
|
45
45
|
if (!filter_mask.AllValid()) {
|
46
46
|
RESULT_TYPE filtered = 0;
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.8.2-
|
2
|
+
#define DUCKDB_VERSION "0.8.2-dev2399"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "20ad35b3fa"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|
@@ -22,14 +22,14 @@ struct ListSegment {
|
|
22
22
|
ListSegment *next;
|
23
23
|
};
|
24
24
|
struct LinkedList {
|
25
|
-
LinkedList() {};
|
25
|
+
LinkedList() : total_capacity(0), first_segment(nullptr), last_segment(nullptr) {};
|
26
26
|
LinkedList(idx_t total_capacity_p, ListSegment *first_segment_p, ListSegment *last_segment_p)
|
27
27
|
: total_capacity(total_capacity_p), first_segment(first_segment_p), last_segment(last_segment_p) {
|
28
28
|
}
|
29
29
|
|
30
|
-
idx_t total_capacity
|
31
|
-
ListSegment *first_segment
|
32
|
-
ListSegment *last_segment
|
30
|
+
idx_t total_capacity;
|
31
|
+
ListSegment *first_segment;
|
32
|
+
ListSegment *last_segment;
|
33
33
|
};
|
34
34
|
|
35
35
|
// forward declarations
|
@@ -37,23 +37,21 @@ struct ListSegmentFunctions;
|
|
37
37
|
typedef ListSegment *(*create_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
38
38
|
uint16_t capacity);
|
39
39
|
typedef void (*write_data_to_segment_t)(const ListSegmentFunctions &functions, ArenaAllocator &allocator,
|
40
|
-
ListSegment *segment,
|
40
|
+
ListSegment *segment, RecursiveUnifiedVectorFormat &input_data,
|
41
|
+
idx_t &entry_idx);
|
41
42
|
typedef void (*read_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *segment,
|
42
43
|
Vector &result, idx_t &total_count);
|
43
|
-
typedef ListSegment *(*copy_data_from_segment_t)(const ListSegmentFunctions &functions, const ListSegment *source,
|
44
|
-
ArenaAllocator &allocator);
|
45
44
|
|
46
45
|
struct ListSegmentFunctions {
|
47
46
|
create_segment_t create_segment;
|
48
47
|
write_data_to_segment_t write_data;
|
49
48
|
read_data_from_segment_t read_data;
|
50
|
-
|
49
|
+
|
51
50
|
vector<ListSegmentFunctions> child_functions;
|
52
51
|
|
53
|
-
void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list,
|
54
|
-
idx_t &
|
52
|
+
void AppendRow(ArenaAllocator &allocator, LinkedList &linked_list, RecursiveUnifiedVectorFormat &input_data,
|
53
|
+
idx_t &entry_idx) const;
|
55
54
|
void BuildListVector(const LinkedList &linked_list, Vector &result, idx_t &initial_total_count) const;
|
56
|
-
void CopyLinkedList(const LinkedList &source_list, LinkedList &target_list, ArenaAllocator &allocator) const;
|
57
55
|
};
|
58
56
|
|
59
57
|
void GetSegmentDataFunctions(ListSegmentFunctions &functions, const LogicalType &type);
|
@@ -35,6 +35,11 @@ struct UnifiedVectorFormat {
|
|
35
35
|
}
|
36
36
|
};
|
37
37
|
|
38
|
+
struct RecursiveUnifiedVectorFormat {
|
39
|
+
UnifiedVectorFormat unified;
|
40
|
+
vector<RecursiveUnifiedVectorFormat> children;
|
41
|
+
};
|
42
|
+
|
38
43
|
class VectorCache;
|
39
44
|
class VectorStructBuffer;
|
40
45
|
class VectorListBuffer;
|
@@ -140,6 +145,8 @@ public:
|
|
140
145
|
//! The most common vector types (flat, constant & dictionary) can be converted to the canonical format "for free"
|
141
146
|
//! ToUnifiedFormat was originally called Orrify, as a tribute to Orri Erling who came up with it
|
142
147
|
DUCKDB_API void ToUnifiedFormat(idx_t count, UnifiedVectorFormat &data);
|
148
|
+
//! Recursively calls UnifiedVectorFormat on a vector and its child vectors (for nested types)
|
149
|
+
static void RecursiveToUnifiedFormat(Vector &input, idx_t count, RecursiveUnifiedVectorFormat &data);
|
143
150
|
|
144
151
|
//! Turn the vector into a sequence vector
|
145
152
|
DUCKDB_API void Sequence(int64_t start, int64_t increment, idx_t count);
|
@@ -15,9 +15,14 @@
|
|
15
15
|
|
16
16
|
namespace duckdb {
|
17
17
|
|
18
|
+
// structs
|
18
19
|
struct AggregateInputData;
|
19
|
-
|
20
|
-
|
20
|
+
struct FrameBounds {
|
21
|
+
FrameBounds() : start(0), end(0) {};
|
22
|
+
FrameBounds(idx_t start, idx_t end) : start(start), end(end) {};
|
23
|
+
idx_t start = 0;
|
24
|
+
idx_t end = 0;
|
25
|
+
};
|
21
26
|
|
22
27
|
class AggregateExecutor {
|
23
28
|
private:
|
@@ -56,8 +56,10 @@ protected:
|
|
56
56
|
//! Reused selection vector
|
57
57
|
SelectionVector sel;
|
58
58
|
|
59
|
-
//! The arena allocator used by the aggregates for their internal state
|
60
|
-
ArenaAllocator aggregate_allocator;
|
59
|
+
//! The active arena allocator used by the aggregates for their internal state
|
60
|
+
unique_ptr<ArenaAllocator> aggregate_allocator;
|
61
|
+
//! Owning arena allocators that this HT has data from
|
62
|
+
vector<unique_ptr<ArenaAllocator>> stored_allocators;
|
61
63
|
|
62
64
|
private:
|
63
65
|
//! Destroy the perfect aggregate HT (called automatically by the destructor)
|
@@ -117,8 +117,6 @@ public:
|
|
117
117
|
|
118
118
|
class WindowSegmentTree : public WindowAggregator {
|
119
119
|
public:
|
120
|
-
using FrameBounds = std::pair<idx_t, idx_t>;
|
121
|
-
|
122
120
|
WindowSegmentTree(AggregateObject aggr, const LogicalType &result_type, idx_t count, WindowAggregationMode mode_p);
|
123
121
|
~WindowSegmentTree() override;
|
124
122
|
|
@@ -41,7 +41,6 @@ typedef void (*aggregate_simple_update_t)(Vector inputs[], AggregateInputData &a
|
|
41
41
|
data_ptr_t state, idx_t count);
|
42
42
|
|
43
43
|
//! The type used for updating complex windowed aggregate functions (optional)
|
44
|
-
typedef std::pair<idx_t, idx_t> FrameBounds;
|
45
44
|
typedef void (*aggregate_window_t)(Vector inputs[], const ValidityMask &filter_mask,
|
46
45
|
AggregateInputData &aggr_input_data, idx_t input_count, data_ptr_t state,
|
47
46
|
const FrameBounds &frame, const FrameBounds &prev, Vector &result, idx_t rid,
|
@@ -209,6 +209,9 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
|
|
209
209
|
}
|
210
210
|
window_ref = it->second;
|
211
211
|
D_ASSERT(window_ref);
|
212
|
+
if (window_ref->startOffset || window_ref->endOffset || window_ref->frameOptions != FRAMEOPTION_DEFAULTS) {
|
213
|
+
throw ParserException("cannot copy window \"%s\" because it has a frame clause", window_spec->refname);
|
214
|
+
}
|
212
215
|
}
|
213
216
|
in_window_definition = true;
|
214
217
|
TransformWindowDef(*window_ref, *expr);
|
@@ -50,7 +50,7 @@ void ForeignKeyConstraint::FormatSerialize(FormatSerializer &serializer) const {
|
|
50
50
|
Constraint::FormatSerialize(serializer);
|
51
51
|
serializer.WriteProperty("pk_columns", pk_columns);
|
52
52
|
serializer.WriteProperty("fk_columns", fk_columns);
|
53
|
-
serializer.WriteProperty("
|
53
|
+
serializer.WriteProperty("fk_type", info.type);
|
54
54
|
serializer.WriteProperty("schema", info.schema);
|
55
55
|
serializer.WriteProperty("table", info.table);
|
56
56
|
serializer.WriteProperty("pk_keys", info.pk_keys);
|
@@ -61,7 +61,7 @@ unique_ptr<Constraint> ForeignKeyConstraint::FormatDeserialize(FormatDeserialize
|
|
61
61
|
auto result = duckdb::unique_ptr<ForeignKeyConstraint>(new ForeignKeyConstraint());
|
62
62
|
deserializer.ReadProperty("pk_columns", result->pk_columns);
|
63
63
|
deserializer.ReadProperty("fk_columns", result->fk_columns);
|
64
|
-
deserializer.ReadProperty("
|
64
|
+
deserializer.ReadProperty("fk_type", result->info.type);
|
65
65
|
deserializer.ReadProperty("schema", result->info.schema);
|
66
66
|
deserializer.ReadProperty("table", result->info.table);
|
67
67
|
deserializer.ReadProperty("pk_keys", result->info.pk_keys);
|
@@ -160,13 +160,13 @@ unique_ptr<CreateInfo> CreateTableInfo::FormatDeserialize(FormatDeserializer &de
|
|
160
160
|
void CreateTypeInfo::FormatSerialize(FormatSerializer &serializer) const {
|
161
161
|
CreateInfo::FormatSerialize(serializer);
|
162
162
|
serializer.WriteProperty("name", name);
|
163
|
-
serializer.WriteProperty("
|
163
|
+
serializer.WriteProperty("logical_type", type);
|
164
164
|
}
|
165
165
|
|
166
166
|
unique_ptr<CreateInfo> CreateTypeInfo::FormatDeserialize(FormatDeserializer &deserializer) {
|
167
167
|
auto result = duckdb::unique_ptr<CreateTypeInfo>(new CreateTypeInfo());
|
168
168
|
deserializer.ReadProperty("name", result->name);
|
169
|
-
deserializer.ReadProperty("
|
169
|
+
deserializer.ReadProperty("logical_type", result->type);
|
170
170
|
return std::move(result);
|
171
171
|
}
|
172
172
|
|
@@ -176,7 +176,7 @@ void AlterForeignKeyInfo::FormatSerialize(FormatSerializer &serializer) const {
|
|
176
176
|
serializer.WriteProperty("fk_columns", fk_columns);
|
177
177
|
serializer.WriteProperty("pk_keys", pk_keys);
|
178
178
|
serializer.WriteProperty("fk_keys", fk_keys);
|
179
|
-
serializer.WriteProperty("
|
179
|
+
serializer.WriteProperty("alter_fk_type", type);
|
180
180
|
}
|
181
181
|
|
182
182
|
unique_ptr<AlterTableInfo> AlterForeignKeyInfo::FormatDeserialize(FormatDeserializer &deserializer) {
|
@@ -186,7 +186,7 @@ unique_ptr<AlterTableInfo> AlterForeignKeyInfo::FormatDeserialize(FormatDeserial
|
|
186
186
|
deserializer.ReadProperty("fk_columns", result->fk_columns);
|
187
187
|
deserializer.ReadProperty("pk_keys", result->pk_keys);
|
188
188
|
deserializer.ReadProperty("fk_keys", result->fk_keys);
|
189
|
-
deserializer.ReadProperty("
|
189
|
+
deserializer.ReadProperty("alter_fk_type", result->type);
|
190
190
|
return std::move(result);
|
191
191
|
}
|
192
192
|
|
@@ -96,7 +96,7 @@ void JoinRef::FormatSerialize(FormatSerializer &serializer) const {
|
|
96
96
|
serializer.WriteProperty("left", *left);
|
97
97
|
serializer.WriteProperty("right", *right);
|
98
98
|
serializer.WriteOptionalProperty("condition", condition);
|
99
|
-
serializer.WriteProperty("
|
99
|
+
serializer.WriteProperty("join_type", type);
|
100
100
|
serializer.WriteProperty("ref_type", ref_type);
|
101
101
|
serializer.WriteProperty("using_columns", using_columns);
|
102
102
|
}
|
@@ -106,7 +106,7 @@ unique_ptr<TableRef> JoinRef::FormatDeserialize(FormatDeserializer &deserializer
|
|
106
106
|
deserializer.ReadProperty("left", result->left);
|
107
107
|
deserializer.ReadProperty("right", result->right);
|
108
108
|
deserializer.ReadOptionalProperty("condition", result->condition);
|
109
|
-
deserializer.ReadProperty("
|
109
|
+
deserializer.ReadProperty("join_type", result->type);
|
110
110
|
deserializer.ReadProperty("ref_type", result->ref_type);
|
111
111
|
deserializer.ReadProperty("using_columns", result->using_columns);
|
112
112
|
return std::move(result);
|
@@ -151,14 +151,12 @@ unique_ptr<TableRef> SubqueryRef::FormatDeserialize(FormatDeserializer &deserial
|
|
151
151
|
void TableFunctionRef::FormatSerialize(FormatSerializer &serializer) const {
|
152
152
|
TableRef::FormatSerialize(serializer);
|
153
153
|
serializer.WriteProperty("function", *function);
|
154
|
-
serializer.WriteProperty("alias", alias);
|
155
154
|
serializer.WriteProperty("column_name_alias", column_name_alias);
|
156
155
|
}
|
157
156
|
|
158
157
|
unique_ptr<TableRef> TableFunctionRef::FormatDeserialize(FormatDeserializer &deserializer) {
|
159
158
|
auto result = duckdb::unique_ptr<TableFunctionRef>(new TableFunctionRef());
|
160
159
|
deserializer.ReadProperty("function", result->function);
|
161
|
-
deserializer.ReadProperty("alias", result->alias);
|
162
160
|
deserializer.ReadProperty("column_name_alias", result->column_name_alias);
|
163
161
|
return std::move(result);
|
164
162
|
}
|
@@ -348,17 +348,17 @@
|
|
348
348
|
|
349
349
|
#include "extension/icu/third_party/icu/i18n/wintzimpl.cpp"
|
350
350
|
|
351
|
-
#include "extension/icu/third_party/icu/i18n/double-conversion-
|
351
|
+
#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
|
352
352
|
|
353
353
|
#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
|
354
354
|
|
355
|
-
#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
|
356
|
-
|
357
|
-
#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
|
358
|
-
|
359
355
|
#include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
|
360
356
|
|
361
357
|
#include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
|
362
358
|
|
363
|
-
#include "extension/icu/third_party/icu/i18n/double-conversion-
|
359
|
+
#include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
|
360
|
+
|
361
|
+
#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
|
362
|
+
|
363
|
+
#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
|
364
364
|
|