duckdb 0.8.2-dev4025.0 → 0.8.2-dev4126.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/binding.gyp +1 -0
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/json/buffered_json_reader.cpp +76 -74
  4. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +35 -32
  5. package/src/duckdb/extension/json/include/json_scan.hpp +9 -6
  6. package/src/duckdb/extension/json/json_scan.cpp +124 -121
  7. package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp +5 -0
  8. package/src/duckdb/src/common/radix_partitioning.cpp +1 -1
  9. package/src/duckdb/src/common/sort/partition_state.cpp +5 -1
  10. package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +1 -1
  11. package/src/duckdb/src/core_functions/function_list.cpp +7 -0
  12. package/src/duckdb/src/core_functions/scalar/list/list_cosine_similarity.cpp +78 -0
  13. package/src/duckdb/src/core_functions/scalar/list/list_distance.cpp +72 -0
  14. package/src/duckdb/src/core_functions/scalar/list/list_inner_product.cpp +70 -0
  15. package/src/duckdb/src/execution/index/art/art.cpp +111 -92
  16. package/src/duckdb/src/execution/index/art/iterator.cpp +21 -27
  17. package/src/duckdb/src/execution/index/art/leaf.cpp +72 -153
  18. package/src/duckdb/src/execution/index/art/node.cpp +109 -203
  19. package/src/duckdb/src/execution/index/art/node16.cpp +32 -64
  20. package/src/duckdb/src/execution/index/art/node256.cpp +38 -53
  21. package/src/duckdb/src/execution/index/art/node4.cpp +31 -62
  22. package/src/duckdb/src/execution/index/art/node48.cpp +43 -65
  23. package/src/duckdb/src/execution/index/art/prefix.cpp +70 -141
  24. package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +345 -0
  25. package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +74 -0
  26. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +1 -1
  27. package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +1 -1
  28. package/src/duckdb/src/function/table/system/duckdb_columns.cpp +3 -1
  29. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  30. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp +2 -0
  31. package/src/duckdb/src/include/duckdb/common/optional_idx.hpp +1 -1
  32. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +51 -0
  33. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +17 -7
  34. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +5 -5
  35. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +10 -16
  36. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +38 -116
  37. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +17 -18
  38. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +17 -23
  39. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +17 -18
  40. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +17 -24
  41. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +16 -22
  42. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +126 -0
  43. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +79 -0
  44. package/src/duckdb/src/include/duckdb/execution/index/index_pointer.hpp +96 -0
  45. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +1 -1
  46. package/src/duckdb/src/include/duckdb/storage/block.hpp +1 -1
  47. package/src/duckdb/src/include/duckdb/storage/index.hpp +10 -8
  48. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +3 -0
  49. package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +14 -5
  50. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +2 -3
  51. package/src/duckdb/src/storage/checkpoint_manager.cpp +16 -21
  52. package/src/duckdb/src/storage/data_table.cpp +3 -3
  53. package/src/duckdb/src/storage/index.cpp +7 -1
  54. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +21 -21
  55. package/src/duckdb/src/storage/standard_buffer_manager.cpp +0 -8
  56. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  57. package/src/duckdb/src/storage/table_index_list.cpp +1 -1
  58. package/src/duckdb/src/transaction/commit_state.cpp +5 -1
  59. package/src/duckdb/ub_src_core_functions_scalar_list.cpp +6 -0
  60. package/src/duckdb/ub_src_execution_index.cpp +4 -0
  61. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  62. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +0 -238
  63. package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp +0 -115
@@ -2,16 +2,14 @@
2
2
 
3
3
  #include "duckdb/execution/index/art/node16.hpp"
4
4
  #include "duckdb/execution/index/art/node256.hpp"
5
- #include "duckdb/storage/metadata/metadata_reader.hpp"
6
- #include "duckdb/storage/metadata/metadata_writer.hpp"
7
5
 
8
6
  namespace duckdb {
9
7
 
10
8
  Node48 &Node48::New(ART &art, Node &node) {
11
9
 
12
10
  node = Node::GetAllocator(art, NType::NODE_48).New();
13
- node.SetType((uint8_t)NType::NODE_48);
14
- auto &n48 = Node48::Get(art, node);
11
+ node.SetMetadata(static_cast<uint8_t>(NType::NODE_48));
12
+ auto &n48 = Node::RefMutable<Node48>(art, node, NType::NODE_48);
15
13
 
16
14
  n48.count = 0;
17
15
  for (idx_t i = 0; i < Node::NODE_256_CAPACITY; i++) {
@@ -20,7 +18,7 @@ Node48 &Node48::New(ART &art, Node &node) {
20
18
 
21
19
  // necessary for faster child insertion/deletion
22
20
  for (idx_t i = 0; i < Node::NODE_48_CAPACITY; i++) {
23
- n48.children[i].Reset();
21
+ n48.children[i].Clear();
24
22
  }
25
23
 
26
24
  return n48;
@@ -28,10 +26,8 @@ Node48 &Node48::New(ART &art, Node &node) {
28
26
 
29
27
  void Node48::Free(ART &art, Node &node) {
30
28
 
31
- D_ASSERT(node.IsSet());
32
- D_ASSERT(!node.IsSerialized());
33
-
34
- auto &n48 = Node48::Get(art, node);
29
+ D_ASSERT(node.HasMetadata());
30
+ auto &n48 = Node::RefMutable<Node48>(art, node, NType::NODE_48);
35
31
 
36
32
  if (!n48.count) {
37
33
  return;
@@ -47,8 +43,8 @@ void Node48::Free(ART &art, Node &node) {
47
43
 
48
44
  Node48 &Node48::GrowNode16(ART &art, Node &node48, Node &node16) {
49
45
 
50
- auto &n16 = Node16::Get(art, node16);
51
- auto &n48 = Node48::New(art, node48);
46
+ auto &n16 = Node::RefMutable<Node16>(art, node16, NType::NODE_16);
47
+ auto &n48 = New(art, node48);
52
48
 
53
49
  n48.count = n16.count;
54
50
  for (idx_t i = 0; i < Node::NODE_256_CAPACITY; i++) {
@@ -62,7 +58,7 @@ Node48 &Node48::GrowNode16(ART &art, Node &node48, Node &node16) {
62
58
 
63
59
  // necessary for faster child insertion/deletion
64
60
  for (idx_t i = n16.count; i < Node::NODE_48_CAPACITY; i++) {
65
- n48.children[i].Reset();
61
+ n48.children[i].Clear();
66
62
  }
67
63
 
68
64
  n16.count = 0;
@@ -72,13 +68,13 @@ Node48 &Node48::GrowNode16(ART &art, Node &node48, Node &node16) {
72
68
 
73
69
  Node48 &Node48::ShrinkNode256(ART &art, Node &node48, Node &node256) {
74
70
 
75
- auto &n48 = Node48::New(art, node48);
76
- auto &n256 = Node256::Get(art, node256);
71
+ auto &n48 = New(art, node48);
72
+ auto &n256 = Node::RefMutable<Node256>(art, node256, NType::NODE_256);
77
73
 
78
74
  n48.count = 0;
79
75
  for (idx_t i = 0; i < Node::NODE_256_CAPACITY; i++) {
80
76
  D_ASSERT(n48.count <= Node::NODE_48_CAPACITY);
81
- if (n256.children[i].IsSet()) {
77
+ if (n256.children[i].HasMetadata()) {
82
78
  n48.child_index[i] = n48.count;
83
79
  n48.children[n48.count] = n256.children[i];
84
80
  n48.count++;
@@ -89,7 +85,7 @@ Node48 &Node48::ShrinkNode256(ART &art, Node &node48, Node &node256) {
89
85
 
90
86
  // necessary for faster child insertion/deletion
91
87
  for (idx_t i = n48.count; i < Node::NODE_48_CAPACITY; i++) {
92
- n48.children[i].Reset();
88
+ n48.children[i].Clear();
93
89
  }
94
90
 
95
91
  n256.count = 0;
@@ -108,9 +104,8 @@ void Node48::InitializeMerge(ART &art, const ARTFlags &flags) {
108
104
 
109
105
  void Node48::InsertChild(ART &art, Node &node, const uint8_t byte, const Node child) {
110
106
 
111
- D_ASSERT(node.IsSet());
112
- D_ASSERT(!node.IsSerialized());
113
- auto &n48 = Node48::Get(art, node);
107
+ D_ASSERT(node.HasMetadata());
108
+ auto &n48 = Node::RefMutable<Node48>(art, node, NType::NODE_48);
114
109
 
115
110
  // ensure that there is no other child at the same byte
116
111
  D_ASSERT(n48.child_index[byte] == Node::EMPTY_MARKER);
@@ -119,10 +114,10 @@ void Node48::InsertChild(ART &art, Node &node, const uint8_t byte, const Node ch
119
114
  if (n48.count < Node::NODE_48_CAPACITY) {
120
115
  // still space, just insert the child
121
116
  idx_t child_pos = n48.count;
122
- if (n48.children[child_pos].IsSet()) {
117
+ if (n48.children[child_pos].HasMetadata()) {
123
118
  // find an empty position in the node list if the current position is occupied
124
119
  child_pos = 0;
125
- while (n48.children[child_pos].IsSet()) {
120
+ while (n48.children[child_pos].HasMetadata()) {
126
121
  child_pos++;
127
122
  }
128
123
  }
@@ -140,9 +135,8 @@ void Node48::InsertChild(ART &art, Node &node, const uint8_t byte, const Node ch
140
135
 
141
136
  void Node48::DeleteChild(ART &art, Node &node, const uint8_t byte) {
142
137
 
143
- D_ASSERT(node.IsSet());
144
- D_ASSERT(!node.IsSerialized());
145
- auto &n48 = Node48::Get(art, node);
138
+ D_ASSERT(node.HasMetadata());
139
+ auto &n48 = Node::RefMutable<Node48>(art, node, NType::NODE_48);
146
140
 
147
141
  // free the child and decrease the count
148
142
  Node::Free(art, n48.children[n48.child_index[byte]]);
@@ -156,58 +150,42 @@ void Node48::DeleteChild(ART &art, Node &node, const uint8_t byte) {
156
150
  }
157
151
  }
158
152
 
159
- optional_ptr<Node> Node48::GetNextChild(uint8_t &byte) {
153
+ optional_ptr<const Node> Node48::GetChild(const uint8_t byte) const {
154
+ if (child_index[byte] != Node::EMPTY_MARKER) {
155
+ D_ASSERT(children[child_index[byte]].HasMetadata());
156
+ return &children[child_index[byte]];
157
+ }
158
+ return nullptr;
159
+ }
160
+
161
+ optional_ptr<Node> Node48::GetChildMutable(const uint8_t byte) {
162
+ if (child_index[byte] != Node::EMPTY_MARKER) {
163
+ D_ASSERT(children[child_index[byte]].HasMetadata());
164
+ return &children[child_index[byte]];
165
+ }
166
+ return nullptr;
167
+ }
160
168
 
169
+ optional_ptr<const Node> Node48::GetNextChild(uint8_t &byte) const {
161
170
  for (idx_t i = byte; i < Node::NODE_256_CAPACITY; i++) {
162
171
  if (child_index[i] != Node::EMPTY_MARKER) {
163
172
  byte = i;
164
- D_ASSERT(children[child_index[i]].IsSet());
173
+ D_ASSERT(children[child_index[i]].HasMetadata());
165
174
  return &children[child_index[i]];
166
175
  }
167
176
  }
168
177
  return nullptr;
169
178
  }
170
179
 
171
- BlockPointer Node48::Serialize(ART &art, MetadataWriter &writer) {
172
-
173
- // recurse into children and retrieve child block pointers
174
- vector<BlockPointer> child_block_pointers;
175
- for (idx_t i = 0; i < Node::NODE_48_CAPACITY; i++) {
176
- child_block_pointers.push_back(children[i].Serialize(art, writer));
177
- }
178
-
179
- // get pointer and write fields
180
- auto block_pointer = writer.GetBlockPointer();
181
- writer.Write(NType::NODE_48);
182
- writer.Write<uint8_t>(count);
183
-
184
- // write key values
185
- for (idx_t i = 0; i < Node::NODE_256_CAPACITY; i++) {
186
- writer.Write(child_index[i]);
187
- }
188
-
189
- // write child block pointers
190
- for (auto &child_block_pointer : child_block_pointers) {
191
- writer.Write(child_block_pointer.block_id);
192
- writer.Write(child_block_pointer.offset);
193
- }
194
-
195
- return block_pointer;
196
- }
197
-
198
- void Node48::Deserialize(MetadataReader &reader) {
199
-
200
- count = reader.Read<uint8_t>();
201
-
202
- // read key values
203
- for (idx_t i = 0; i < Node::NODE_256_CAPACITY; i++) {
204
- child_index[i] = reader.Read<uint8_t>();
205
- }
206
-
207
- // read child block pointers
208
- for (idx_t i = 0; i < Node::NODE_48_CAPACITY; i++) {
209
- children[i] = Node(reader);
180
+ optional_ptr<Node> Node48::GetNextChildMutable(uint8_t &byte) {
181
+ for (idx_t i = byte; i < Node::NODE_256_CAPACITY; i++) {
182
+ if (child_index[i] != Node::EMPTY_MARKER) {
183
+ byte = i;
184
+ D_ASSERT(children[child_index[i]].HasMetadata());
185
+ return &children[child_index[i]];
186
+ }
210
187
  }
188
+ return nullptr;
211
189
  }
212
190
 
213
191
  void Node48::Vacuum(ART &art, const ARTFlags &flags) {
@@ -3,8 +3,6 @@
3
3
  #include "duckdb/execution/index/art/art.hpp"
4
4
  #include "duckdb/execution/index/art/art_key.hpp"
5
5
  #include "duckdb/execution/index/art/node.hpp"
6
- #include "duckdb/storage/metadata/metadata_reader.hpp"
7
- #include "duckdb/storage/metadata/metadata_writer.hpp"
8
6
  #include "duckdb/common/swap.hpp"
9
7
 
10
8
  namespace duckdb {
@@ -12,19 +10,19 @@ namespace duckdb {
12
10
  Prefix &Prefix::New(ART &art, Node &node) {
13
11
 
14
12
  node = Node::GetAllocator(art, NType::PREFIX).New();
15
- node.SetType((uint8_t)NType::PREFIX);
13
+ node.SetMetadata(static_cast<uint8_t>(NType::PREFIX));
16
14
 
17
- auto &prefix = Prefix::Get(art, node);
15
+ auto &prefix = Node::RefMutable<Prefix>(art, node, NType::PREFIX);
18
16
  prefix.data[Node::PREFIX_SIZE] = 0;
19
17
  return prefix;
20
18
  }
21
19
 
22
- Prefix &Prefix::New(ART &art, Node &node, uint8_t byte, Node next) {
20
+ Prefix &Prefix::New(ART &art, Node &node, uint8_t byte, const Node &next) {
23
21
 
24
22
  node = Node::GetAllocator(art, NType::PREFIX).New();
25
- node.SetType((uint8_t)NType::PREFIX);
23
+ node.SetMetadata(static_cast<uint8_t>(NType::PREFIX));
26
24
 
27
- auto &prefix = Prefix::Get(art, node);
25
+ auto &prefix = Node::RefMutable<Prefix>(art, node, NType::PREFIX);
28
26
  prefix.data[Node::PREFIX_SIZE] = 1;
29
27
  prefix.data[0] = byte;
30
28
  prefix.ptr = next;
@@ -40,8 +38,8 @@ void Prefix::New(ART &art, reference<Node> &node, const ARTKey &key, const uint3
40
38
 
41
39
  while (count) {
42
40
  node.get() = Node::GetAllocator(art, NType::PREFIX).New();
43
- node.get().SetType((uint8_t)NType::PREFIX);
44
- auto &prefix = Prefix::Get(art, node);
41
+ node.get().SetMetadata(static_cast<uint8_t>(NType::PREFIX));
42
+ auto &prefix = Node::RefMutable<Prefix>(art, node, NType::PREFIX);
45
43
 
46
44
  auto this_count = MinValue((uint32_t)Node::PREFIX_SIZE, count);
47
45
  prefix.data[Node::PREFIX_SIZE] = (uint8_t)this_count;
@@ -57,54 +55,49 @@ void Prefix::Free(ART &art, Node &node) {
57
55
 
58
56
  Node current_node = node;
59
57
  Node next_node;
60
- while (current_node.IsSet() && !current_node.IsSerialized() && current_node.GetType() == NType::PREFIX) {
61
- next_node = Prefix::Get(art, current_node).ptr;
58
+ while (current_node.HasMetadata() && current_node.GetType() == NType::PREFIX) {
59
+ next_node = Node::RefMutable<Prefix>(art, current_node, NType::PREFIX).ptr;
62
60
  Node::GetAllocator(art, NType::PREFIX).Free(current_node);
63
61
  current_node = next_node;
64
62
  }
65
63
 
66
64
  Node::Free(art, current_node);
67
- node.Reset();
65
+ node.Clear();
68
66
  }
69
67
 
70
68
  void Prefix::InitializeMerge(ART &art, Node &node, const ARTFlags &flags) {
71
69
 
72
- auto merge_buffer_count = flags.merge_buffer_counts[(uint8_t)NType::PREFIX - 1];
70
+ auto merge_buffer_count = flags.merge_buffer_counts[static_cast<uint8_t>(NType::PREFIX) - 1];
73
71
 
74
72
  Node next_node = node;
75
- reference<Prefix> prefix = Prefix::Get(art, next_node);
73
+ reference<Prefix> prefix = Node::RefMutable<Prefix>(art, next_node, NType::PREFIX);
76
74
 
77
75
  while (next_node.GetType() == NType::PREFIX) {
78
76
  next_node = prefix.get().ptr;
79
77
  if (prefix.get().ptr.GetType() == NType::PREFIX) {
80
- prefix.get().ptr.AddToBufferID(merge_buffer_count);
81
- prefix = Prefix::Get(art, next_node);
78
+ prefix.get().ptr.IncreaseBufferId(merge_buffer_count);
79
+ prefix = Node::RefMutable<Prefix>(art, next_node, NType::PREFIX);
82
80
  }
83
81
  }
84
82
 
85
- node.AddToBufferID(merge_buffer_count);
83
+ node.IncreaseBufferId(merge_buffer_count);
86
84
  prefix.get().ptr.InitializeMerge(art, flags);
87
85
  }
88
86
 
89
87
  void Prefix::Concatenate(ART &art, Node &prefix_node, const uint8_t byte, Node &child_prefix_node) {
90
88
 
91
- D_ASSERT(prefix_node.IsSet() && !prefix_node.IsSerialized());
92
- D_ASSERT(child_prefix_node.IsSet());
93
-
94
- if (child_prefix_node.IsSerialized()) {
95
- child_prefix_node.Deserialize(art);
96
- }
89
+ D_ASSERT(prefix_node.HasMetadata() && child_prefix_node.HasMetadata());
97
90
 
98
91
  // append a byte and a child_prefix to prefix
99
92
  if (prefix_node.GetType() == NType::PREFIX) {
100
93
 
101
94
  // get the tail
102
- reference<Prefix> prefix = Prefix::Get(art, prefix_node);
103
- D_ASSERT(prefix.get().ptr.IsSet() && !prefix.get().ptr.IsSerialized());
95
+ reference<Prefix> prefix = Node::RefMutable<Prefix>(art, prefix_node, NType::PREFIX);
96
+ D_ASSERT(prefix.get().ptr.HasMetadata());
104
97
 
105
98
  while (prefix.get().ptr.GetType() == NType::PREFIX) {
106
- prefix = Prefix::Get(art, prefix.get().ptr);
107
- D_ASSERT(prefix.get().ptr.IsSet() && !prefix.get().ptr.IsSerialized());
99
+ prefix = Node::RefMutable<Prefix>(art, prefix.get().ptr, NType::PREFIX);
100
+ D_ASSERT(prefix.get().ptr.HasMetadata());
108
101
  }
109
102
 
110
103
  // append the byte
@@ -124,24 +117,24 @@ void Prefix::Concatenate(ART &art, Node &prefix_node, const uint8_t byte, Node &
124
117
  if (prefix_node.GetType() != NType::PREFIX && child_prefix_node.GetType() == NType::PREFIX) {
125
118
 
126
119
  auto child_prefix = child_prefix_node;
127
- auto &prefix = Prefix::New(art, prefix_node, byte, Node());
120
+ auto &prefix = New(art, prefix_node, byte);
128
121
  prefix.Append(art, child_prefix);
129
122
  return;
130
123
  }
131
124
 
132
125
  // neither prefix nor child_prefix are prefix nodes
133
126
  // create a new prefix containing the byte
134
- Prefix::New(art, prefix_node, byte, child_prefix_node);
127
+ New(art, prefix_node, byte, child_prefix_node);
135
128
  }
136
129
 
137
- idx_t Prefix::Traverse(ART &art, reference<Node> &prefix_node, const ARTKey &key, idx_t &depth) {
130
+ idx_t Prefix::Traverse(ART &art, reference<const Node> &prefix_node, const ARTKey &key, idx_t &depth) {
138
131
 
139
- D_ASSERT(prefix_node.get().IsSet() && !prefix_node.get().IsSerialized());
132
+ D_ASSERT(prefix_node.get().HasMetadata());
140
133
  D_ASSERT(prefix_node.get().GetType() == NType::PREFIX);
141
134
 
142
135
  // compare prefix nodes to key bytes
143
136
  while (prefix_node.get().GetType() == NType::PREFIX) {
144
- auto &prefix = Prefix::Get(art, prefix_node);
137
+ auto &prefix = Node::Ref<const Prefix>(art, prefix_node, NType::PREFIX);
145
138
  for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
146
139
  if (prefix.data[i] != key[depth]) {
147
140
  return i;
@@ -149,10 +142,28 @@ idx_t Prefix::Traverse(ART &art, reference<Node> &prefix_node, const ARTKey &key
149
142
  depth++;
150
143
  }
151
144
  prefix_node = prefix.ptr;
152
- D_ASSERT(prefix_node.get().IsSet());
153
- if (prefix_node.get().IsSerialized()) {
154
- prefix_node.get().Deserialize(art);
145
+ D_ASSERT(prefix_node.get().HasMetadata());
146
+ }
147
+
148
+ return DConstants::INVALID_INDEX;
149
+ }
150
+
151
+ idx_t Prefix::TraverseMutable(ART &art, reference<Node> &prefix_node, const ARTKey &key, idx_t &depth) {
152
+
153
+ D_ASSERT(prefix_node.get().HasMetadata());
154
+ D_ASSERT(prefix_node.get().GetType() == NType::PREFIX);
155
+
156
+ // compare prefix nodes to key bytes
157
+ while (prefix_node.get().GetType() == NType::PREFIX) {
158
+ auto &prefix = Node::RefMutable<Prefix>(art, prefix_node, NType::PREFIX);
159
+ for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
160
+ if (prefix.data[i] != key[depth]) {
161
+ return i;
162
+ }
163
+ depth++;
155
164
  }
165
+ prefix_node = prefix.ptr;
166
+ D_ASSERT(prefix_node.get().HasMetadata());
156
167
  }
157
168
 
158
169
  return DConstants::INVALID_INDEX;
@@ -160,8 +171,8 @@ idx_t Prefix::Traverse(ART &art, reference<Node> &prefix_node, const ARTKey &key
160
171
 
161
172
  bool Prefix::Traverse(ART &art, reference<Node> &l_node, reference<Node> &r_node, idx_t &mismatch_position) {
162
173
 
163
- auto &l_prefix = Prefix::Get(art, l_node.get());
164
- auto &r_prefix = Prefix::Get(art, r_node.get());
174
+ auto &l_prefix = Node::RefMutable<Prefix>(art, l_node.get(), NType::PREFIX);
175
+ auto &r_prefix = Node::RefMutable<Prefix>(art, r_node.get(), NType::PREFIX);
165
176
 
166
177
  // compare prefix bytes
167
178
  idx_t max_count = MinValue(l_prefix.data[Node::PREFIX_SIZE], r_prefix.data[Node::PREFIX_SIZE]);
@@ -197,16 +208,16 @@ bool Prefix::Traverse(ART &art, reference<Node> &l_node, reference<Node> &r_node
197
208
 
198
209
  void Prefix::Reduce(ART &art, Node &prefix_node, const idx_t n) {
199
210
 
200
- D_ASSERT(prefix_node.IsSet() && !prefix_node.IsSerialized());
211
+ D_ASSERT(prefix_node.HasMetadata());
201
212
  D_ASSERT(n < Node::PREFIX_SIZE);
202
213
 
203
- reference<Prefix> prefix = Prefix::Get(art, prefix_node);
214
+ reference<Prefix> prefix = Node::RefMutable<Prefix>(art, prefix_node, NType::PREFIX);
204
215
 
205
216
  // free this prefix node
206
217
  if (n == (idx_t)(prefix.get().data[Node::PREFIX_SIZE] - 1)) {
207
218
  auto next_ptr = prefix.get().ptr;
208
- D_ASSERT(next_ptr.IsSet());
209
- prefix.get().ptr.Reset();
219
+ D_ASSERT(next_ptr.HasMetadata());
220
+ prefix.get().ptr.Clear();
210
221
  Node::Free(art, prefix_node);
211
222
  prefix_node = next_ptr;
212
223
  return;
@@ -225,9 +236,9 @@ void Prefix::Reduce(ART &art, Node &prefix_node, const idx_t n) {
225
236
 
226
237
  void Prefix::Split(ART &art, reference<Node> &prefix_node, Node &child_node, idx_t position) {
227
238
 
228
- D_ASSERT(prefix_node.get().IsSet() && !prefix_node.get().IsSerialized());
239
+ D_ASSERT(prefix_node.get().HasMetadata());
229
240
 
230
- auto &prefix = Prefix::Get(art, prefix_node);
241
+ auto &prefix = Node::RefMutable<Prefix>(art, prefix_node, NType::PREFIX);
231
242
 
232
243
  // the split is at the last byte of this prefix, so the child_node contains all subsequent
233
244
  // prefix nodes (prefix.ptr) (if any), and the count of this prefix decreases by one,
@@ -241,15 +252,12 @@ void Prefix::Split(ART &art, reference<Node> &prefix_node, Node &child_node, idx
241
252
 
242
253
  // append the remaining bytes after the split
243
254
  if (position + 1 < prefix.data[Node::PREFIX_SIZE]) {
244
- reference<Prefix> child_prefix = Prefix::New(art, child_node);
255
+ reference<Prefix> child_prefix = New(art, child_node);
245
256
  for (idx_t i = position + 1; i < prefix.data[Node::PREFIX_SIZE]; i++) {
246
257
  child_prefix = child_prefix.get().Append(art, prefix.data[i]);
247
258
  }
248
259
 
249
- D_ASSERT(prefix.ptr.IsSet());
250
- if (prefix.ptr.IsSerialized()) {
251
- prefix.ptr.Deserialize(art);
252
- }
260
+ D_ASSERT(prefix.ptr.HasMetadata());
253
261
 
254
262
  if (prefix.ptr.GetType() == NType::PREFIX) {
255
263
  child_prefix.get().Append(art, prefix.ptr);
@@ -269,7 +277,7 @@ void Prefix::Split(ART &art, reference<Node> &prefix_node, Node &child_node, idx
269
277
 
270
278
  // no bytes left before the split, free this node
271
279
  if (position == 0) {
272
- prefix.ptr.Reset();
280
+ prefix.ptr.Clear();
273
281
  Node::Free(art, prefix_node.get());
274
282
  return;
275
283
  }
@@ -279,15 +287,15 @@ void Prefix::Split(ART &art, reference<Node> &prefix_node, Node &child_node, idx
279
287
  return;
280
288
  }
281
289
 
282
- string Prefix::VerifyAndToString(ART &art, Node &node, const bool only_verify) {
290
+ string Prefix::VerifyAndToString(ART &art, const Node &node, const bool only_verify) {
283
291
 
284
292
  // NOTE: we could do this recursively, but the function-call overhead can become kinda crazy
285
293
  string str = "";
286
294
 
287
- reference<Node> node_ref(node);
295
+ reference<const Node> node_ref(node);
288
296
  while (node_ref.get().GetType() == NType::PREFIX) {
289
297
 
290
- auto &prefix = Prefix::Get(art, node_ref);
298
+ auto &prefix = Node::Ref<const Prefix>(art, node_ref, NType::PREFIX);
291
299
  D_ASSERT(prefix.data[Node::PREFIX_SIZE] != 0);
292
300
  D_ASSERT(prefix.data[Node::PREFIX_SIZE] <= Node::PREFIX_SIZE);
293
301
 
@@ -297,82 +305,25 @@ string Prefix::VerifyAndToString(ART &art, Node &node, const bool only_verify) {
297
305
  }
298
306
  str += "] ";
299
307
 
300
- if (prefix.ptr.IsSerialized()) {
301
- return str + " serialized";
302
- }
303
308
  node_ref = prefix.ptr;
304
309
  }
305
310
 
306
- return str + node_ref.get().VerifyAndToString(art, only_verify);
307
- }
308
-
309
- BlockPointer Prefix::Serialize(ART &art, Node &node, MetadataWriter &writer) {
310
- reference<Node> first_non_prefix(node);
311
- idx_t total_count = Prefix::TotalCount(art, first_non_prefix);
312
- auto child_block_pointer = first_non_prefix.get().Serialize(art, writer);
313
-
314
- // get pointer and write fields
315
- auto block_pointer = writer.GetBlockPointer();
316
- writer.Write(NType::PREFIX);
317
- writer.Write<idx_t>(total_count);
318
-
319
- reference<Node> current_node(node);
320
- while (current_node.get().GetType() == NType::PREFIX) {
321
-
322
- // write prefix bytes
323
- D_ASSERT(!current_node.get().IsSerialized());
324
- auto &prefix = Prefix::Get(art, current_node);
325
- for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
326
- writer.Write(prefix.data[i]);
327
- }
328
-
329
- current_node = prefix.ptr;
330
- }
331
-
332
- // write child block pointer
333
- writer.Write(child_block_pointer.block_id);
334
- writer.Write(child_block_pointer.offset);
335
-
336
- return block_pointer;
337
- }
338
-
339
- void Prefix::Deserialize(ART &art, Node &node, MetadataReader &reader) {
340
- auto total_count = reader.Read<idx_t>();
341
- reference<Node> current_node(node);
342
-
343
- while (total_count) {
344
- current_node.get() = Node::GetAllocator(art, NType::PREFIX).New();
345
- current_node.get().SetType((uint8_t)NType::PREFIX);
346
-
347
- auto &prefix = Prefix::Get(art, current_node);
348
- prefix.data[Node::PREFIX_SIZE] = MinValue((idx_t)Node::PREFIX_SIZE, total_count);
349
-
350
- // read bytes
351
- for (idx_t i = 0; i < prefix.data[Node::PREFIX_SIZE]; i++) {
352
- prefix.data[i] = reader.Read<uint8_t>();
353
- }
354
-
355
- total_count -= prefix.data[Node::PREFIX_SIZE];
356
- current_node = prefix.ptr;
357
- prefix.ptr.Reset();
358
- }
359
-
360
- // read child block pointer
361
- current_node.get() = Node(reader);
311
+ auto subtree = node_ref.get().VerifyAndToString(art, only_verify);
312
+ return only_verify ? "" : str + subtree;
362
313
  }
363
314
 
364
315
  void Prefix::Vacuum(ART &art, Node &node, const ARTFlags &flags) {
365
316
 
366
- bool flag_set = flags.vacuum_flags[(uint8_t)NType::PREFIX - 1];
317
+ bool flag_set = flags.vacuum_flags[static_cast<uint8_t>(NType::PREFIX) - 1];
367
318
  auto &allocator = Node::GetAllocator(art, NType::PREFIX);
368
319
 
369
320
  reference<Node> node_ref(node);
370
- while (!node_ref.get().IsSerialized() && node_ref.get().GetType() == NType::PREFIX) {
321
+ while (node_ref.get().GetType() == NType::PREFIX) {
371
322
  if (flag_set && allocator.NeedsVacuum(node_ref)) {
372
323
  node_ref.get() = allocator.VacuumPointer(node_ref);
373
- node_ref.get().SetType((uint8_t)NType::PREFIX);
324
+ node_ref.get().SetMetadata(static_cast<uint8_t>(NType::PREFIX));
374
325
  }
375
- auto &prefix = Prefix::Get(art, node_ref);
326
+ auto &prefix = Node::RefMutable<Prefix>(art, node_ref, NType::PREFIX);
376
327
  node_ref = prefix.ptr;
377
328
  }
378
329
 
@@ -385,7 +336,7 @@ Prefix &Prefix::Append(ART &art, const uint8_t byte) {
385
336
 
386
337
  // we need a new prefix node
387
338
  if (prefix.get().data[Node::PREFIX_SIZE] == Node::PREFIX_SIZE) {
388
- prefix = Prefix::New(art, prefix.get().ptr);
339
+ prefix = New(art, prefix.get().ptr);
389
340
  }
390
341
 
391
342
  prefix.get().data[prefix.get().data[Node::PREFIX_SIZE]] = byte;
@@ -395,22 +346,18 @@ Prefix &Prefix::Append(ART &art, const uint8_t byte) {
395
346
 
396
347
  void Prefix::Append(ART &art, Node other_prefix) {
397
348
 
398
- // NOTE: all usages of this function already deserialize the other prefix
399
- D_ASSERT(other_prefix.IsSet() && !other_prefix.IsSerialized());
349
+ D_ASSERT(other_prefix.HasMetadata());
400
350
 
401
351
  reference<Prefix> prefix(*this);
402
352
  while (other_prefix.GetType() == NType::PREFIX) {
403
353
 
404
354
  // copy prefix bytes
405
- auto &other = Prefix::Get(art, other_prefix);
355
+ auto &other = Node::RefMutable<Prefix>(art, other_prefix, NType::PREFIX);
406
356
  for (idx_t i = 0; i < other.data[Node::PREFIX_SIZE]; i++) {
407
357
  prefix = prefix.get().Append(art, other.data[i]);
408
358
  }
409
359
 
410
- D_ASSERT(other.ptr.IsSet());
411
- if (other.ptr.IsSerialized()) {
412
- other.ptr.Deserialize(art);
413
- }
360
+ D_ASSERT(other.ptr.HasMetadata());
414
361
 
415
362
  prefix.get().ptr = other.ptr;
416
363
  Node::GetAllocator(art, NType::PREFIX).Free(other_prefix);
@@ -420,22 +367,4 @@ void Prefix::Append(ART &art, Node other_prefix) {
420
367
  D_ASSERT(prefix.get().ptr.GetType() != NType::PREFIX);
421
368
  }
422
369
 
423
- idx_t Prefix::TotalCount(ART &art, reference<Node> &node) {
424
-
425
- // NOTE: first prefix in the prefix chain is already deserialized
426
- D_ASSERT(node.get().IsSet() && !node.get().IsSerialized());
427
-
428
- idx_t count = 0;
429
- while (node.get().GetType() == NType::PREFIX) {
430
- auto &prefix = Prefix::Get(art, node);
431
- count += prefix.data[Node::PREFIX_SIZE];
432
-
433
- if (prefix.ptr.IsSerialized()) {
434
- prefix.ptr.Deserialize(art);
435
- }
436
- node = prefix.ptr;
437
- }
438
- return count;
439
- }
440
-
441
370
  } // namespace duckdb