duckdb 0.8.2-dev2673.0 → 0.8.2-dev2700.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/binding.gyp +1 -0
  2. package/package.json +1 -1
  3. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +6 -6
  4. package/src/duckdb/src/common/field_writer.cpp +1 -0
  5. package/src/duckdb/src/common/local_file_system.cpp +5 -0
  6. package/src/duckdb/src/common/types/interval.cpp +3 -0
  7. package/src/duckdb/src/execution/index/art/art.cpp +6 -9
  8. package/src/duckdb/src/execution/index/art/leaf.cpp +4 -4
  9. package/src/duckdb/src/execution/index/art/node.cpp +9 -12
  10. package/src/duckdb/src/execution/index/art/node16.cpp +4 -4
  11. package/src/duckdb/src/execution/index/art/node256.cpp +4 -4
  12. package/src/duckdb/src/execution/index/art/node4.cpp +4 -5
  13. package/src/duckdb/src/execution/index/art/node48.cpp +4 -4
  14. package/src/duckdb/src/execution/index/art/prefix.cpp +4 -6
  15. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  16. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp +1 -1
  17. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +2 -2
  18. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +4 -4
  19. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +6 -4
  20. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +2 -2
  21. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +2 -2
  22. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +2 -2
  23. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +2 -2
  24. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +2 -2
  25. package/src/duckdb/src/include/duckdb/storage/block.hpp +27 -4
  26. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +9 -9
  27. package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp +5 -5
  28. package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp +2 -2
  29. package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +3 -3
  30. package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +19 -16
  31. package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp +1 -1
  32. package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp +2 -2
  33. package/src/duckdb/src/include/duckdb/storage/index.hpp +2 -2
  34. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +88 -0
  35. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp +54 -0
  36. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +45 -0
  37. package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp +2 -2
  38. package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +6 -5
  39. package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +2 -2
  40. package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +2 -2
  41. package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +2 -2
  42. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +1 -1
  43. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +2 -0
  44. package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +2 -2
  45. package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +1 -1
  46. package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp +3 -0
  47. package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +3 -4
  48. package/src/duckdb/src/storage/buffer/block_manager.cpp +10 -9
  49. package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp +1 -1
  50. package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -4
  51. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +7 -7
  52. package/src/duckdb/src/storage/checkpoint_manager.cpp +49 -43
  53. package/src/duckdb/src/storage/index.cpp +1 -1
  54. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +267 -0
  55. package/src/duckdb/src/storage/metadata/metadata_reader.cpp +80 -0
  56. package/src/duckdb/src/storage/metadata/metadata_writer.cpp +86 -0
  57. package/src/duckdb/src/storage/single_file_block_manager.cpp +47 -52
  58. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  59. package/src/duckdb/src/storage/storage_manager.cpp +4 -3
  60. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -3
  61. package/src/duckdb/src/storage/table/persistent_table_data.cpp +1 -2
  62. package/src/duckdb/src/storage/table/row_group.cpp +9 -10
  63. package/src/duckdb/src/storage/table/row_group_collection.cpp +6 -3
  64. package/src/duckdb/src/storage/table_index_list.cpp +1 -1
  65. package/src/duckdb/src/storage/wal_replay.cpp +3 -2
  66. package/src/duckdb/src/storage/write_ahead_log.cpp +3 -2
  67. package/src/duckdb/ub_src_storage.cpp +0 -4
  68. package/src/duckdb/ub_src_storage_metadata.cpp +6 -0
  69. package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -46
  70. package/src/duckdb/src/include/duckdb/storage/meta_block_writer.hpp +0 -50
  71. package/src/duckdb/src/storage/meta_block_reader.cpp +0 -69
  72. package/src/duckdb/src/storage/meta_block_writer.cpp +0 -80
@@ -0,0 +1,80 @@
1
+ #include "duckdb/storage/metadata/metadata_reader.hpp"
2
+
3
+ namespace duckdb {
4
+
5
+ MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer, BlockReaderType type)
6
+ : manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true), index(0), offset(0),
7
+ next_offset(pointer.offset), capacity(0) {
8
+ }
9
+
10
+ MetadataReader::MetadataReader(MetadataManager &manager, BlockPointer pointer)
11
+ : MetadataReader(manager, MetadataManager::FromBlockPointer(pointer)) {
12
+ }
13
+
14
+ MetadataPointer MetadataReader::FromDiskPointer(MetaBlockPointer pointer) {
15
+ if (type == BlockReaderType::EXISTING_BLOCKS) {
16
+ return manager.FromDiskPointer(pointer);
17
+ } else {
18
+ return manager.RegisterDiskPointer(pointer);
19
+ }
20
+ }
21
+
22
+ MetadataReader::~MetadataReader() {
23
+ }
24
+
25
+ void MetadataReader::ReadData(data_ptr_t buffer, idx_t read_size) {
26
+ while (offset + read_size > capacity) {
27
+ // cannot read entire entry from block
28
+ // first read what we can from this block
29
+ idx_t to_read = capacity - offset;
30
+ if (to_read > 0) {
31
+ memcpy(buffer, Ptr(), to_read);
32
+ read_size -= to_read;
33
+ buffer += to_read;
34
+ offset += read_size;
35
+ }
36
+ // then move to the next block
37
+ ReadNextBlock();
38
+ }
39
+ // we have enough left in this block to read from the buffer
40
+ memcpy(buffer, Ptr(), read_size);
41
+ offset += read_size;
42
+ }
43
+
44
+ MetaBlockPointer MetadataReader::GetMetaBlockPointer() {
45
+ return manager.GetDiskPointer(block.pointer, offset);
46
+ }
47
+
48
+ void MetadataReader::ReadNextBlock() {
49
+ if (!has_next_block) {
50
+ throw IOException("No more data remaining in MetadataReader");
51
+ }
52
+ block = manager.Pin(next_pointer);
53
+ index = next_pointer.index;
54
+
55
+ idx_t next_block = Load<idx_t>(BasePtr());
56
+ if (next_block == idx_t(-1)) {
57
+ has_next_block = false;
58
+ } else {
59
+ next_pointer = FromDiskPointer(MetaBlockPointer(next_block, 0));
60
+ }
61
+ if (next_offset < sizeof(block_id_t)) {
62
+ next_offset = sizeof(block_id_t);
63
+ }
64
+ if (next_offset > MetadataManager::METADATA_BLOCK_SIZE) {
65
+ throw InternalException("next_offset cannot be bigger than block size");
66
+ }
67
+ offset = next_offset;
68
+ next_offset = sizeof(block_id_t);
69
+ capacity = MetadataManager::METADATA_BLOCK_SIZE;
70
+ }
71
+
72
+ data_ptr_t MetadataReader::BasePtr() {
73
+ return block.handle.Ptr() + index * MetadataManager::METADATA_BLOCK_SIZE;
74
+ }
75
+
76
+ data_ptr_t MetadataReader::Ptr() {
77
+ return BasePtr() + offset;
78
+ }
79
+
80
+ } // namespace duckdb
@@ -0,0 +1,86 @@
1
+ #include "duckdb/storage/metadata/metadata_writer.hpp"
2
+ #include "duckdb/storage/block_manager.hpp"
3
+
4
+ namespace duckdb {
5
+
6
+ MetadataWriter::MetadataWriter(MetadataManager &manager) : manager(manager), capacity(0), offset(0) {
7
+ }
8
+
9
+ MetadataWriter::~MetadataWriter() {
10
+ // If there's an exception during checkpoint, this can get destroyed without
11
+ // flushing the data...which is fine, because none of the unwritten data
12
+ // will be referenced.
13
+ //
14
+ // Otherwise, we should have explicitly flushed (and thereby nulled the block).
15
+ D_ASSERT(!block.handle.IsValid() || Exception::UncaughtException());
16
+ }
17
+
18
+ BlockPointer MetadataWriter::GetBlockPointer() {
19
+ return MetadataManager::ToBlockPointer(GetMetaBlockPointer());
20
+ }
21
+
22
+ MetaBlockPointer MetadataWriter::GetMetaBlockPointer() {
23
+ if (offset >= capacity) {
24
+ // at the end of the block - fetch the next block
25
+ NextBlock();
26
+ D_ASSERT(capacity > 0);
27
+ }
28
+ return manager.GetDiskPointer(block.pointer, offset);
29
+ }
30
+
31
+ MetadataHandle MetadataWriter::NextHandle() {
32
+ return manager.AllocateHandle();
33
+ }
34
+
35
+ void MetadataWriter::NextBlock() {
36
+ // now we need to get a new block id
37
+ auto new_handle = NextHandle();
38
+
39
+ // write the block id of the new block to the start of the current block
40
+ if (capacity > 0) {
41
+ Store<idx_t>(manager.GetDiskPointer(new_handle.pointer).block_pointer, BasePtr());
42
+ }
43
+ // now update the block id of the block
44
+ block = std::move(new_handle);
45
+ current_pointer = block.pointer;
46
+ offset = sizeof(idx_t);
47
+ capacity = MetadataManager::METADATA_BLOCK_SIZE;
48
+ Store<idx_t>(-1, BasePtr());
49
+ }
50
+
51
+ void MetadataWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
52
+ while (offset + write_size > capacity) {
53
+ // we need to make a new block
54
+ // first copy what we can
55
+ D_ASSERT(offset <= capacity);
56
+ idx_t copy_amount = capacity - offset;
57
+ if (copy_amount > 0) {
58
+ memcpy(Ptr(), buffer, copy_amount);
59
+ buffer += copy_amount;
60
+ offset += copy_amount;
61
+ write_size -= copy_amount;
62
+ }
63
+ // move forward to the next block
64
+ NextBlock();
65
+ }
66
+ memcpy(Ptr(), buffer, write_size);
67
+ offset += write_size;
68
+ }
69
+
70
+ void MetadataWriter::Flush() {
71
+ if (offset < capacity) {
72
+ // clear remaining bytes of block (if any)
73
+ memset(Ptr(), 0, capacity - offset);
74
+ }
75
+ block.handle.Destroy();
76
+ }
77
+
78
+ data_ptr_t MetadataWriter::BasePtr() {
79
+ return block.handle.Ptr() + current_pointer.index * MetadataManager::METADATA_BLOCK_SIZE;
80
+ }
81
+
82
+ data_ptr_t MetadataWriter::Ptr() {
83
+ return BasePtr() + offset;
84
+ }
85
+
86
+ } // namespace duckdb
@@ -6,8 +6,8 @@
6
6
  #include "duckdb/common/serializer/buffered_deserializer.hpp"
7
7
  #include "duckdb/common/serializer/buffered_serializer.hpp"
8
8
  #include "duckdb/common/field_writer.hpp"
9
- #include "duckdb/storage/meta_block_reader.hpp"
10
- #include "duckdb/storage/meta_block_writer.hpp"
9
+ #include "duckdb/storage/metadata/metadata_reader.hpp"
10
+ #include "duckdb/storage/metadata/metadata_writer.hpp"
11
11
  #include "duckdb/storage/buffer_manager.hpp"
12
12
  #include "duckdb/main/config.hpp"
13
13
 
@@ -81,16 +81,16 @@ MainHeader MainHeader::Deserialize(Deserializer &source) {
81
81
 
82
82
  void DatabaseHeader::Serialize(Serializer &ser) {
83
83
  ser.Write<uint64_t>(iteration);
84
- ser.Write<block_id_t>(meta_block);
85
- ser.Write<block_id_t>(free_list);
84
+ ser.Write<idx_t>(meta_block);
85
+ ser.Write<idx_t>(free_list);
86
86
  ser.Write<uint64_t>(block_count);
87
87
  }
88
88
 
89
89
  DatabaseHeader DatabaseHeader::Deserialize(Deserializer &source) {
90
90
  DatabaseHeader header;
91
91
  header.iteration = source.Read<uint64_t>();
92
- header.meta_block = source.Read<block_id_t>();
93
- header.free_list = source.Read<block_id_t>();
92
+ header.meta_block = source.Read<idx_t>();
93
+ header.free_list = source.Read<idx_t>();
94
94
  header.block_count = source.Read<uint64_t>();
95
95
  return header;
96
96
  }
@@ -241,11 +241,12 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
241
241
  }
242
242
 
243
243
  void SingleFileBlockManager::LoadFreeList() {
244
- if (free_list_id == INVALID_BLOCK) {
244
+ MetaBlockPointer free_pointer(free_list_id, 0);
245
+ if (!free_pointer.IsValid()) {
245
246
  // no free list
246
247
  return;
247
248
  }
248
- MetaBlockReader reader(*this, free_list_id);
249
+ MetadataReader reader(GetMetadataManager(), free_pointer, BlockReaderType::REGISTER_BLOCKS);
249
250
  auto free_list_count = reader.Read<uint64_t>();
250
251
  free_list.clear();
251
252
  for (idx_t i = 0; i < free_list_count; i++) {
@@ -258,10 +259,12 @@ void SingleFileBlockManager::LoadFreeList() {
258
259
  auto usage_count = reader.Read<uint32_t>();
259
260
  multi_use_blocks[block_id] = usage_count;
260
261
  }
262
+ GetMetadataManager().Deserialize(reader);
263
+ GetMetadataManager().MarkBlocksAsModified();
261
264
  }
262
265
 
263
- bool SingleFileBlockManager::IsRootBlock(block_id_t root) {
264
- return root == meta_block;
266
+ bool SingleFileBlockManager::IsRootBlock(MetaBlockPointer root) {
267
+ return root.block_pointer == meta_block;
265
268
  }
266
269
 
267
270
  block_id_t SingleFileBlockManager::GetFreeBlockId() {
@@ -327,7 +330,7 @@ void SingleFileBlockManager::IncreaseBlockReferenceCount(block_id_t block_id) {
327
330
  }
328
331
  }
329
332
 
330
- block_id_t SingleFileBlockManager::GetMetaBlock() {
333
+ idx_t SingleFileBlockManager::GetMetaBlock() {
331
334
  return meta_block;
332
335
  }
333
336
 
@@ -391,50 +394,42 @@ void SingleFileBlockManager::Truncate() {
391
394
  handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
392
395
  }
393
396
 
394
- vector<block_id_t> SingleFileBlockManager::GetFreeListBlocks() {
395
- vector<block_id_t> free_list_blocks;
396
-
397
- if (!free_list.empty() || !multi_use_blocks.empty() || !modified_blocks.empty()) {
398
- // there are blocks in the free list or multi_use_blocks
399
- // figure out how many blocks we need to write these to the file
400
- auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
401
- auto multi_use_blocks_size =
402
- sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
403
- auto total_size = free_list_size + multi_use_blocks_size;
404
- // because of potential alignment issues and needing to store a next pointer in a block we subtract
405
- // a bit from the max block size
406
- auto space_in_block = Storage::BLOCK_SIZE - 4 * sizeof(block_id_t);
407
- auto total_blocks = (total_size + space_in_block - 1) / space_in_block;
408
- D_ASSERT(total_size > 0);
409
- D_ASSERT(total_blocks > 0);
410
-
411
- // reserve the blocks that we are going to write
412
- // since these blocks are no longer free we cannot just include them in the free list!
413
- for (idx_t i = 0; i < total_blocks; i++) {
414
- auto block_id = GetFreeBlockId();
415
- free_list_blocks.push_back(block_id);
416
- }
397
+ vector<MetadataHandle> SingleFileBlockManager::GetFreeListBlocks() {
398
+ vector<MetadataHandle> free_list_blocks;
399
+
400
+ auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
401
+ auto multi_use_blocks_size = sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
402
+ auto metadata_blocks = sizeof(uint64_t) + (sizeof(idx_t) * 2) * GetMetadataManager().BlockCount();
403
+ auto total_size = free_list_size + multi_use_blocks_size + metadata_blocks;
404
+
405
+ // reserve the blocks that we are going to write
406
+ // since these blocks are no longer free we cannot just include them in the free list!
407
+ auto block_size = MetadataManager::METADATA_BLOCK_SIZE - sizeof(idx_t);
408
+ while (total_size > 0) {
409
+ auto free_list_handle = GetMetadataManager().AllocateHandle();
410
+ free_list_blocks.push_back(std::move(free_list_handle));
411
+ total_size -= MinValue<idx_t>(total_size, block_size);
417
412
  }
418
413
 
419
414
  return free_list_blocks;
420
415
  }
421
416
 
422
- class FreeListBlockWriter : public MetaBlockWriter {
417
+ class FreeListBlockWriter : public MetadataWriter {
423
418
  public:
424
- FreeListBlockWriter(BlockManager &block_manager, vector<block_id_t> &free_list_blocks_p)
425
- : MetaBlockWriter(block_manager, free_list_blocks_p[0]), free_list_blocks(free_list_blocks_p), index(1) {
419
+ FreeListBlockWriter(MetadataManager &manager, vector<MetadataHandle> free_list_blocks_p)
420
+ : MetadataWriter(manager), free_list_blocks(std::move(free_list_blocks_p)), index(0) {
426
421
  }
427
422
 
428
- vector<block_id_t> &free_list_blocks;
423
+ vector<MetadataHandle> free_list_blocks;
429
424
  idx_t index;
430
425
 
431
426
  protected:
432
- block_id_t GetNextBlockId() override {
427
+ MetadataHandle NextHandle() override {
433
428
  if (index >= free_list_blocks.size()) {
434
429
  throw InternalException(
435
430
  "Free List Block Writer ran out of blocks, this means not enough blocks were allocated up front");
436
431
  }
437
- return free_list_blocks[index++];
432
+ return std::move(free_list_blocks[index++]);
438
433
  }
439
434
  };
440
435
 
@@ -442,7 +437,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
442
437
  // set the iteration count
443
438
  header.iteration = ++iteration_count;
444
439
 
445
- vector<block_id_t> free_list_blocks = GetFreeListBlocks();
440
+ auto free_list_blocks = GetFreeListBlocks();
446
441
 
447
442
  // now handle the free list
448
443
  // add all modified blocks to the free list: they can now be written to again
@@ -451,20 +446,16 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
451
446
  }
452
447
  modified_blocks.clear();
453
448
 
449
+ auto &metadata_manager = GetMetadataManager();
454
450
  if (!free_list_blocks.empty()) {
455
451
  // there are blocks to write, either in the free_list or in the modified_blocks
456
452
  // we write these blocks specifically to the free_list_blocks
457
- // a normal MetaBlockWriter will fetch blocks to use from the free_list
453
+ // a normal MetadataWriter will fetch blocks to use from the free_list
458
454
  // but since we are WRITING the free_list, this behavior is sub-optimal
455
+ FreeListBlockWriter writer(metadata_manager, std::move(free_list_blocks));
459
456
 
460
- FreeListBlockWriter writer(*this, free_list_blocks);
461
-
462
- auto ptr = writer.GetBlockPointer();
463
- D_ASSERT(ptr.block_id == free_list_blocks[0]);
464
- header.free_list = ptr.block_id;
465
- for (auto &block_id : free_list_blocks) {
466
- modified_blocks.insert(block_id);
467
- }
457
+ auto ptr = writer.GetMetaBlockPointer();
458
+ header.free_list = ptr.block_pointer;
468
459
 
469
460
  writer.Write<uint64_t>(free_list.size());
470
461
  for (auto &block_id : free_list) {
@@ -475,11 +466,13 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
475
466
  writer.Write<block_id_t>(entry.first);
476
467
  writer.Write<uint32_t>(entry.second);
477
468
  }
469
+ GetMetadataManager().Serialize(writer);
478
470
  writer.Flush();
479
471
  } else {
480
472
  // no blocks in the free list
481
- header.free_list = INVALID_BLOCK;
473
+ header.free_list = DConstants::INVALID_INDEX;
482
474
  }
475
+ metadata_manager.Flush();
483
476
  header.block_count = max_block;
484
477
 
485
478
  auto &config = DBConfig::Get(db);
@@ -494,7 +487,9 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
494
487
  }
495
488
  // set the header inside the buffer
496
489
  header_buffer.Clear();
497
- Store<DatabaseHeader>(header, header_buffer.buffer);
490
+ BufferedSerializer serializer;
491
+ header.Serialize(serializer);
492
+ memcpy(header_buffer.buffer, serializer.blob.data.get(), serializer.blob.size);
498
493
  // now write the header to the file, active_header determines whether we write to h1 or h2
499
494
  // note that if active_header is h1 we write to h2, and vice versa
500
495
  ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
@@ -2,7 +2,7 @@
2
2
 
3
3
  namespace duckdb {
4
4
 
5
- const uint64_t VERSION_NUMBER = 54;
5
+ const uint64_t VERSION_NUMBER = 55;
6
6
 
7
7
  struct StorageVersionInfo {
8
8
  const char *version_name;
@@ -82,6 +82,9 @@ public:
82
82
  BlockManager &GetBlockManagerForRowData() override {
83
83
  return block_manager;
84
84
  }
85
+ MetadataManager &GetMetadataManager() override {
86
+ return block_manager.GetMetadataManager();
87
+ }
85
88
  };
86
89
 
87
90
  SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only)
@@ -140,8 +143,6 @@ void SingleFileStorageManager::LoadDatabase() {
140
143
  //! Load from storage
141
144
  auto checkpointer = SingleFileCheckpointReader(*this);
142
145
  checkpointer.LoadFromStorage();
143
- // finish load checkpoint, clear the cached handles of meta blocks
144
- block_manager->ClearMetaBlockHandles();
145
146
  // check if the WAL file exists
146
147
  if (fs.FileExists(wal_path)) {
147
148
  // replay the WAL
@@ -223,7 +224,7 @@ unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(T
223
224
  return make_uniq<SingleFileStorageCommitState>(*this, checkpoint);
224
225
  }
225
226
 
226
- bool SingleFileStorageManager::IsCheckpointClean(block_id_t checkpoint_id) {
227
+ bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id) {
227
228
  return block_manager->IsRootBlock(checkpoint_id);
228
229
  }
229
230
 
@@ -242,9 +242,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
242
242
  }
243
243
  }
244
244
 
245
- void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes) {
246
- D_ASSERT(!nodes.empty());
247
- this->nodes = std::move(nodes);
245
+ void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes_p) {
246
+ D_ASSERT(!nodes_p.empty());
247
+ this->nodes = std::move(nodes_p);
248
248
  // first check if any of the segments have changes
249
249
  if (!HasChanges()) {
250
250
  // no changes: only need to write the metadata for this column
@@ -3,8 +3,7 @@
3
3
 
4
4
  namespace duckdb {
5
5
 
6
- PersistentTableData::PersistentTableData(idx_t column_count)
7
- : total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
6
+ PersistentTableData::PersistentTableData(idx_t column_count) : total_rows(0), row_group_count(0) {
8
7
  }
9
8
 
10
9
  PersistentTableData::~PersistentTableData() {
@@ -11,7 +11,7 @@
11
11
  #include "duckdb/planner/table_filter.hpp"
12
12
  #include "duckdb/execution/expression_executor.hpp"
13
13
  #include "duckdb/storage/checkpoint/table_data_writer.hpp"
14
- #include "duckdb/storage/meta_block_reader.hpp"
14
+ #include "duckdb/storage/metadata/metadata_reader.hpp"
15
15
  #include "duckdb/transaction/duck_transaction_manager.hpp"
16
16
  #include "duckdb/main/database.hpp"
17
17
  #include "duckdb/main/attached_database.hpp"
@@ -116,11 +116,10 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
116
116
  if (column_pointers.size() != columns.size()) {
117
117
  throw InternalException("Lazy loading a column but the pointer was not set");
118
118
  }
119
- auto &block_manager = GetCollection().GetBlockManager();
119
+ auto &metadata_manager = GetCollection().GetMetadataManager();
120
120
  auto &types = GetCollection().GetTypes();
121
121
  auto &block_pointer = column_pointers[c];
122
- MetaBlockReader column_data_reader(block_manager, block_pointer.block_id);
123
- column_data_reader.offset = block_pointer.offset;
122
+ MetadataReader column_data_reader(metadata_manager, block_pointer);
124
123
  this->columns[c] =
125
124
  ColumnData::Deserialize(GetBlockManager(), GetTableInfo(), c, start, column_data_reader, types[c], nullptr);
126
125
  is_loaded[c] = true;
@@ -832,7 +831,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
832
831
  for (auto &state : result.states) {
833
832
  // get the current position of the table data writer
834
833
  auto &data_writer = writer.GetPayloadWriter();
835
- auto pointer = data_writer.GetBlockPointer();
834
+ auto pointer = data_writer.GetMetaBlockPointer();
836
835
 
837
836
  // store the stats and the data pointers in the row group pointers
838
837
  row_group_pointer.data_pointers.push_back(pointer);
@@ -898,8 +897,8 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
898
897
  writer.WriteField<uint64_t>(pointer.tuple_count);
899
898
  auto &serializer = writer.GetSerializer();
900
899
  for (auto &data_pointer : pointer.data_pointers) {
901
- serializer.Write<block_id_t>(data_pointer.block_id);
902
- serializer.Write<uint64_t>(data_pointer.offset);
900
+ serializer.Write<idx_t>(data_pointer.block_pointer);
901
+ serializer.Write<uint32_t>(data_pointer.offset);
903
902
  }
904
903
  CheckpointDeletes(pointer.versions.get(), serializer);
905
904
  writer.Finalize();
@@ -917,9 +916,9 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<Lo
917
916
 
918
917
  auto &source = reader.GetSource();
919
918
  for (idx_t i = 0; i < physical_columns; i++) {
920
- BlockPointer pointer;
921
- pointer.block_id = source.Read<block_id_t>();
922
- pointer.offset = source.Read<uint64_t>();
919
+ MetaBlockPointer pointer;
920
+ pointer.block_pointer = source.Read<idx_t>();
921
+ pointer.offset = source.Read<uint32_t>();
923
922
  result.data_pointers.push_back(pointer);
924
923
  }
925
924
  result.versions = DeserializeDeletes(source);
@@ -6,7 +6,7 @@
6
6
  #include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
7
7
  #include "duckdb/storage/checkpoint/table_data_writer.hpp"
8
8
  #include "duckdb/storage/table/row_group_segment_tree.hpp"
9
- #include "duckdb/storage/meta_block_reader.hpp"
9
+ #include "duckdb/storage/metadata/metadata_reader.hpp"
10
10
  #include "duckdb/storage/table/append_state.hpp"
11
11
  #include "duckdb/storage/table/scan_state.hpp"
12
12
  #include "duckdb/storage/table_storage_info.hpp"
@@ -27,8 +27,7 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
27
27
  current_row_group = 0;
28
28
  max_row_group = data.row_group_count;
29
29
  finished_loading = false;
30
- reader = make_uniq<MetaBlockReader>(collection.GetBlockManager(), data.block_id);
31
- reader->offset = data.offset;
30
+ reader = make_uniq<MetadataReader>(collection.GetMetadataManager(), data.block_pointer);
32
31
  }
33
32
 
34
33
  unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
@@ -67,6 +66,10 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
67
66
  return GetTableInfo().db;
68
67
  }
69
68
 
69
+ MetadataManager &RowGroupCollection::GetMetadataManager() {
70
+ return GetBlockManager().GetMetadataManager();
71
+ }
72
+
70
73
  //===--------------------------------------------------------------------===//
71
74
  // Initialize
72
75
  //===--------------------------------------------------------------------===//
@@ -79,7 +79,7 @@ vector<column_t> TableIndexList::GetRequiredColumns() {
79
79
  return result;
80
80
  }
81
81
 
82
- vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetaBlockWriter &writer) {
82
+ vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetadataWriter &writer) {
83
83
  vector<BlockPointer> blocks_info;
84
84
  for (auto &index : indexes) {
85
85
  blocks_info.emplace_back(index->Serialize(writer));
@@ -60,7 +60,7 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
60
60
  return false;
61
61
  } // LCOV_EXCL_STOP
62
62
  initial_reader.reset();
63
- if (checkpoint_state.checkpoint_id != INVALID_BLOCK) {
63
+ if (checkpoint_state.checkpoint_id.IsValid()) {
64
64
  // there is a checkpoint flag: check if we need to deserialize the WAL
65
65
  auto &manager = database.GetStorageManager();
66
66
  if (manager.IsCheckpointClean(checkpoint_state.checkpoint_id)) {
@@ -523,7 +523,8 @@ void ReplayState::ReplayUpdate() {
523
523
  }
524
524
 
525
525
  void ReplayState::ReplayCheckpoint() {
526
- checkpoint_id = source.Read<block_id_t>();
526
+ checkpoint_id.block_pointer = source.Read<idx_t>();
527
+ checkpoint_id.offset = source.Read<uint32_t>();
527
528
  }
528
529
 
529
530
  } // namespace duckdb
@@ -48,9 +48,10 @@ void WriteAheadLog::Delete() {
48
48
  //===--------------------------------------------------------------------===//
49
49
  // Write Entries
50
50
  //===--------------------------------------------------------------------===//
51
- void WriteAheadLog::WriteCheckpoint(block_id_t meta_block) {
51
+ void WriteAheadLog::WriteCheckpoint(MetaBlockPointer meta_block) {
52
52
  writer->Write<WALType>(WALType::CHECKPOINT);
53
- writer->Write<block_id_t>(meta_block);
53
+ writer->Write<idx_t>(meta_block.block_pointer);
54
+ writer->Write<uint32_t>(meta_block.offset);
54
55
  }
55
56
 
56
57
  //===--------------------------------------------------------------------===//
@@ -14,10 +14,6 @@
14
14
 
15
15
  #include "src/storage/magic_bytes.cpp"
16
16
 
17
- #include "src/storage/meta_block_reader.cpp"
18
-
19
- #include "src/storage/meta_block_writer.cpp"
20
-
21
17
  #include "src/storage/storage_manager.cpp"
22
18
 
23
19
  #include "src/storage/standard_buffer_manager.cpp"
@@ -0,0 +1,6 @@
1
+ #include "src/storage/metadata/metadata_manager.cpp"
2
+
3
+ #include "src/storage/metadata/metadata_reader.cpp"
4
+
5
+ #include "src/storage/metadata/metadata_writer.cpp"
6
+
@@ -1,46 +0,0 @@
1
- //===----------------------------------------------------------------------===//
2
- // DuckDB
3
- //
4
- // duckdb/storage/meta_block_reader.hpp
5
- //
6
- //
7
- //===----------------------------------------------------------------------===//
8
-
9
- #pragma once
10
-
11
- #include "duckdb/common/common.hpp"
12
- #include "duckdb/common/serializer.hpp"
13
- #include "duckdb/storage/block.hpp"
14
- #include "duckdb/storage/buffer/buffer_handle.hpp"
15
-
16
- namespace duckdb {
17
- class BlockHandle;
18
- class BlockManager;
19
- class BufferHandle;
20
- class DatabaseInstance;
21
-
22
- //! This struct is responsible for reading meta data from disk
23
- class MetaBlockReader : public Deserializer {
24
- public:
25
- MetaBlockReader(BlockManager &block_manager, block_id_t block, bool free_blocks_on_read = true);
26
- ~MetaBlockReader() override;
27
-
28
- BlockManager &block_manager;
29
- shared_ptr<BlockHandle> block;
30
- BufferHandle handle;
31
- idx_t offset;
32
- block_id_t next_block;
33
- bool free_blocks_on_read;
34
-
35
- public:
36
- //! Read content of size read_size into the buffer
37
- void ReadData(data_ptr_t buffer, idx_t read_size) override;
38
-
39
- ClientContext &GetContext() override;
40
- void SetContext(ClientContext &context_p);
41
-
42
- private:
43
- void ReadNewBlock(block_id_t id);
44
- optional_ptr<ClientContext> context;
45
- };
46
- } // namespace duckdb