duckdb 0.8.2-dev2673.0 → 0.8.2-dev2700.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +1 -0
- package/package.json +1 -1
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +6 -6
- package/src/duckdb/src/common/field_writer.cpp +1 -0
- package/src/duckdb/src/common/local_file_system.cpp +5 -0
- package/src/duckdb/src/common/types/interval.cpp +3 -0
- package/src/duckdb/src/execution/index/art/art.cpp +6 -9
- package/src/duckdb/src/execution/index/art/leaf.cpp +4 -4
- package/src/duckdb/src/execution/index/art/node.cpp +9 -12
- package/src/duckdb/src/execution/index/art/node16.cpp +4 -4
- package/src/duckdb/src/execution/index/art/node256.cpp +4 -4
- package/src/duckdb/src/execution/index/art/node4.cpp +4 -5
- package/src/duckdb/src/execution/index/art/node48.cpp +4 -4
- package/src/duckdb/src/execution/index/art/prefix.cpp +4 -6
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +4 -4
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +6 -4
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/block.hpp +27 -4
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +9 -9
- package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp +5 -5
- package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +19 -16
- package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/index.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +88 -0
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp +54 -0
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +45 -0
- package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +6 -5
- package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +3 -4
- package/src/duckdb/src/storage/buffer/block_manager.cpp +10 -9
- package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp +1 -1
- package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -4
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +7 -7
- package/src/duckdb/src/storage/checkpoint_manager.cpp +49 -43
- package/src/duckdb/src/storage/index.cpp +1 -1
- package/src/duckdb/src/storage/metadata/metadata_manager.cpp +267 -0
- package/src/duckdb/src/storage/metadata/metadata_reader.cpp +80 -0
- package/src/duckdb/src/storage/metadata/metadata_writer.cpp +86 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +47 -52
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/storage_manager.cpp +4 -3
- package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -3
- package/src/duckdb/src/storage/table/persistent_table_data.cpp +1 -2
- package/src/duckdb/src/storage/table/row_group.cpp +9 -10
- package/src/duckdb/src/storage/table/row_group_collection.cpp +6 -3
- package/src/duckdb/src/storage/table_index_list.cpp +1 -1
- package/src/duckdb/src/storage/wal_replay.cpp +3 -2
- package/src/duckdb/src/storage/write_ahead_log.cpp +3 -2
- package/src/duckdb/ub_src_storage.cpp +0 -4
- package/src/duckdb/ub_src_storage_metadata.cpp +6 -0
- package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -46
- package/src/duckdb/src/include/duckdb/storage/meta_block_writer.hpp +0 -50
- package/src/duckdb/src/storage/meta_block_reader.cpp +0 -69
- package/src/duckdb/src/storage/meta_block_writer.cpp +0 -80
@@ -0,0 +1,80 @@
|
|
1
|
+
#include "duckdb/storage/metadata/metadata_reader.hpp"
|
2
|
+
|
3
|
+
namespace duckdb {
|
4
|
+
|
5
|
+
MetadataReader::MetadataReader(MetadataManager &manager, MetaBlockPointer pointer, BlockReaderType type)
|
6
|
+
: manager(manager), type(type), next_pointer(FromDiskPointer(pointer)), has_next_block(true), index(0), offset(0),
|
7
|
+
next_offset(pointer.offset), capacity(0) {
|
8
|
+
}
|
9
|
+
|
10
|
+
MetadataReader::MetadataReader(MetadataManager &manager, BlockPointer pointer)
|
11
|
+
: MetadataReader(manager, MetadataManager::FromBlockPointer(pointer)) {
|
12
|
+
}
|
13
|
+
|
14
|
+
MetadataPointer MetadataReader::FromDiskPointer(MetaBlockPointer pointer) {
|
15
|
+
if (type == BlockReaderType::EXISTING_BLOCKS) {
|
16
|
+
return manager.FromDiskPointer(pointer);
|
17
|
+
} else {
|
18
|
+
return manager.RegisterDiskPointer(pointer);
|
19
|
+
}
|
20
|
+
}
|
21
|
+
|
22
|
+
MetadataReader::~MetadataReader() {
|
23
|
+
}
|
24
|
+
|
25
|
+
void MetadataReader::ReadData(data_ptr_t buffer, idx_t read_size) {
|
26
|
+
while (offset + read_size > capacity) {
|
27
|
+
// cannot read entire entry from block
|
28
|
+
// first read what we can from this block
|
29
|
+
idx_t to_read = capacity - offset;
|
30
|
+
if (to_read > 0) {
|
31
|
+
memcpy(buffer, Ptr(), to_read);
|
32
|
+
read_size -= to_read;
|
33
|
+
buffer += to_read;
|
34
|
+
offset += read_size;
|
35
|
+
}
|
36
|
+
// then move to the next block
|
37
|
+
ReadNextBlock();
|
38
|
+
}
|
39
|
+
// we have enough left in this block to read from the buffer
|
40
|
+
memcpy(buffer, Ptr(), read_size);
|
41
|
+
offset += read_size;
|
42
|
+
}
|
43
|
+
|
44
|
+
MetaBlockPointer MetadataReader::GetMetaBlockPointer() {
|
45
|
+
return manager.GetDiskPointer(block.pointer, offset);
|
46
|
+
}
|
47
|
+
|
48
|
+
void MetadataReader::ReadNextBlock() {
|
49
|
+
if (!has_next_block) {
|
50
|
+
throw IOException("No more data remaining in MetadataReader");
|
51
|
+
}
|
52
|
+
block = manager.Pin(next_pointer);
|
53
|
+
index = next_pointer.index;
|
54
|
+
|
55
|
+
idx_t next_block = Load<idx_t>(BasePtr());
|
56
|
+
if (next_block == idx_t(-1)) {
|
57
|
+
has_next_block = false;
|
58
|
+
} else {
|
59
|
+
next_pointer = FromDiskPointer(MetaBlockPointer(next_block, 0));
|
60
|
+
}
|
61
|
+
if (next_offset < sizeof(block_id_t)) {
|
62
|
+
next_offset = sizeof(block_id_t);
|
63
|
+
}
|
64
|
+
if (next_offset > MetadataManager::METADATA_BLOCK_SIZE) {
|
65
|
+
throw InternalException("next_offset cannot be bigger than block size");
|
66
|
+
}
|
67
|
+
offset = next_offset;
|
68
|
+
next_offset = sizeof(block_id_t);
|
69
|
+
capacity = MetadataManager::METADATA_BLOCK_SIZE;
|
70
|
+
}
|
71
|
+
|
72
|
+
data_ptr_t MetadataReader::BasePtr() {
|
73
|
+
return block.handle.Ptr() + index * MetadataManager::METADATA_BLOCK_SIZE;
|
74
|
+
}
|
75
|
+
|
76
|
+
data_ptr_t MetadataReader::Ptr() {
|
77
|
+
return BasePtr() + offset;
|
78
|
+
}
|
79
|
+
|
80
|
+
} // namespace duckdb
|
@@ -0,0 +1,86 @@
|
|
1
|
+
#include "duckdb/storage/metadata/metadata_writer.hpp"
|
2
|
+
#include "duckdb/storage/block_manager.hpp"
|
3
|
+
|
4
|
+
namespace duckdb {
|
5
|
+
|
6
|
+
MetadataWriter::MetadataWriter(MetadataManager &manager) : manager(manager), capacity(0), offset(0) {
|
7
|
+
}
|
8
|
+
|
9
|
+
MetadataWriter::~MetadataWriter() {
|
10
|
+
// If there's an exception during checkpoint, this can get destroyed without
|
11
|
+
// flushing the data...which is fine, because none of the unwritten data
|
12
|
+
// will be referenced.
|
13
|
+
//
|
14
|
+
// Otherwise, we should have explicitly flushed (and thereby nulled the block).
|
15
|
+
D_ASSERT(!block.handle.IsValid() || Exception::UncaughtException());
|
16
|
+
}
|
17
|
+
|
18
|
+
BlockPointer MetadataWriter::GetBlockPointer() {
|
19
|
+
return MetadataManager::ToBlockPointer(GetMetaBlockPointer());
|
20
|
+
}
|
21
|
+
|
22
|
+
MetaBlockPointer MetadataWriter::GetMetaBlockPointer() {
|
23
|
+
if (offset >= capacity) {
|
24
|
+
// at the end of the block - fetch the next block
|
25
|
+
NextBlock();
|
26
|
+
D_ASSERT(capacity > 0);
|
27
|
+
}
|
28
|
+
return manager.GetDiskPointer(block.pointer, offset);
|
29
|
+
}
|
30
|
+
|
31
|
+
MetadataHandle MetadataWriter::NextHandle() {
|
32
|
+
return manager.AllocateHandle();
|
33
|
+
}
|
34
|
+
|
35
|
+
void MetadataWriter::NextBlock() {
|
36
|
+
// now we need to get a new block id
|
37
|
+
auto new_handle = NextHandle();
|
38
|
+
|
39
|
+
// write the block id of the new block to the start of the current block
|
40
|
+
if (capacity > 0) {
|
41
|
+
Store<idx_t>(manager.GetDiskPointer(new_handle.pointer).block_pointer, BasePtr());
|
42
|
+
}
|
43
|
+
// now update the block id of the block
|
44
|
+
block = std::move(new_handle);
|
45
|
+
current_pointer = block.pointer;
|
46
|
+
offset = sizeof(idx_t);
|
47
|
+
capacity = MetadataManager::METADATA_BLOCK_SIZE;
|
48
|
+
Store<idx_t>(-1, BasePtr());
|
49
|
+
}
|
50
|
+
|
51
|
+
void MetadataWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
|
52
|
+
while (offset + write_size > capacity) {
|
53
|
+
// we need to make a new block
|
54
|
+
// first copy what we can
|
55
|
+
D_ASSERT(offset <= capacity);
|
56
|
+
idx_t copy_amount = capacity - offset;
|
57
|
+
if (copy_amount > 0) {
|
58
|
+
memcpy(Ptr(), buffer, copy_amount);
|
59
|
+
buffer += copy_amount;
|
60
|
+
offset += copy_amount;
|
61
|
+
write_size -= copy_amount;
|
62
|
+
}
|
63
|
+
// move forward to the next block
|
64
|
+
NextBlock();
|
65
|
+
}
|
66
|
+
memcpy(Ptr(), buffer, write_size);
|
67
|
+
offset += write_size;
|
68
|
+
}
|
69
|
+
|
70
|
+
void MetadataWriter::Flush() {
|
71
|
+
if (offset < capacity) {
|
72
|
+
// clear remaining bytes of block (if any)
|
73
|
+
memset(Ptr(), 0, capacity - offset);
|
74
|
+
}
|
75
|
+
block.handle.Destroy();
|
76
|
+
}
|
77
|
+
|
78
|
+
data_ptr_t MetadataWriter::BasePtr() {
|
79
|
+
return block.handle.Ptr() + current_pointer.index * MetadataManager::METADATA_BLOCK_SIZE;
|
80
|
+
}
|
81
|
+
|
82
|
+
data_ptr_t MetadataWriter::Ptr() {
|
83
|
+
return BasePtr() + offset;
|
84
|
+
}
|
85
|
+
|
86
|
+
} // namespace duckdb
|
@@ -6,8 +6,8 @@
|
|
6
6
|
#include "duckdb/common/serializer/buffered_deserializer.hpp"
|
7
7
|
#include "duckdb/common/serializer/buffered_serializer.hpp"
|
8
8
|
#include "duckdb/common/field_writer.hpp"
|
9
|
-
#include "duckdb/storage/
|
10
|
-
#include "duckdb/storage/
|
9
|
+
#include "duckdb/storage/metadata/metadata_reader.hpp"
|
10
|
+
#include "duckdb/storage/metadata/metadata_writer.hpp"
|
11
11
|
#include "duckdb/storage/buffer_manager.hpp"
|
12
12
|
#include "duckdb/main/config.hpp"
|
13
13
|
|
@@ -81,16 +81,16 @@ MainHeader MainHeader::Deserialize(Deserializer &source) {
|
|
81
81
|
|
82
82
|
void DatabaseHeader::Serialize(Serializer &ser) {
|
83
83
|
ser.Write<uint64_t>(iteration);
|
84
|
-
ser.Write<
|
85
|
-
ser.Write<
|
84
|
+
ser.Write<idx_t>(meta_block);
|
85
|
+
ser.Write<idx_t>(free_list);
|
86
86
|
ser.Write<uint64_t>(block_count);
|
87
87
|
}
|
88
88
|
|
89
89
|
DatabaseHeader DatabaseHeader::Deserialize(Deserializer &source) {
|
90
90
|
DatabaseHeader header;
|
91
91
|
header.iteration = source.Read<uint64_t>();
|
92
|
-
header.meta_block = source.Read<
|
93
|
-
header.free_list = source.Read<
|
92
|
+
header.meta_block = source.Read<idx_t>();
|
93
|
+
header.free_list = source.Read<idx_t>();
|
94
94
|
header.block_count = source.Read<uint64_t>();
|
95
95
|
return header;
|
96
96
|
}
|
@@ -241,11 +241,12 @@ void SingleFileBlockManager::Initialize(DatabaseHeader &header) {
|
|
241
241
|
}
|
242
242
|
|
243
243
|
void SingleFileBlockManager::LoadFreeList() {
|
244
|
-
|
244
|
+
MetaBlockPointer free_pointer(free_list_id, 0);
|
245
|
+
if (!free_pointer.IsValid()) {
|
245
246
|
// no free list
|
246
247
|
return;
|
247
248
|
}
|
248
|
-
|
249
|
+
MetadataReader reader(GetMetadataManager(), free_pointer, BlockReaderType::REGISTER_BLOCKS);
|
249
250
|
auto free_list_count = reader.Read<uint64_t>();
|
250
251
|
free_list.clear();
|
251
252
|
for (idx_t i = 0; i < free_list_count; i++) {
|
@@ -258,10 +259,12 @@ void SingleFileBlockManager::LoadFreeList() {
|
|
258
259
|
auto usage_count = reader.Read<uint32_t>();
|
259
260
|
multi_use_blocks[block_id] = usage_count;
|
260
261
|
}
|
262
|
+
GetMetadataManager().Deserialize(reader);
|
263
|
+
GetMetadataManager().MarkBlocksAsModified();
|
261
264
|
}
|
262
265
|
|
263
|
-
bool SingleFileBlockManager::IsRootBlock(
|
264
|
-
return root == meta_block;
|
266
|
+
bool SingleFileBlockManager::IsRootBlock(MetaBlockPointer root) {
|
267
|
+
return root.block_pointer == meta_block;
|
265
268
|
}
|
266
269
|
|
267
270
|
block_id_t SingleFileBlockManager::GetFreeBlockId() {
|
@@ -327,7 +330,7 @@ void SingleFileBlockManager::IncreaseBlockReferenceCount(block_id_t block_id) {
|
|
327
330
|
}
|
328
331
|
}
|
329
332
|
|
330
|
-
|
333
|
+
idx_t SingleFileBlockManager::GetMetaBlock() {
|
331
334
|
return meta_block;
|
332
335
|
}
|
333
336
|
|
@@ -391,50 +394,42 @@ void SingleFileBlockManager::Truncate() {
|
|
391
394
|
handle->Truncate(BLOCK_START + max_block * Storage::BLOCK_ALLOC_SIZE);
|
392
395
|
}
|
393
396
|
|
394
|
-
vector<
|
395
|
-
vector<
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
auto
|
407
|
-
|
408
|
-
|
409
|
-
D_ASSERT(total_blocks > 0);
|
410
|
-
|
411
|
-
// reserve the blocks that we are going to write
|
412
|
-
// since these blocks are no longer free we cannot just include them in the free list!
|
413
|
-
for (idx_t i = 0; i < total_blocks; i++) {
|
414
|
-
auto block_id = GetFreeBlockId();
|
415
|
-
free_list_blocks.push_back(block_id);
|
416
|
-
}
|
397
|
+
vector<MetadataHandle> SingleFileBlockManager::GetFreeListBlocks() {
|
398
|
+
vector<MetadataHandle> free_list_blocks;
|
399
|
+
|
400
|
+
auto free_list_size = sizeof(uint64_t) + sizeof(block_id_t) * (free_list.size() + modified_blocks.size());
|
401
|
+
auto multi_use_blocks_size = sizeof(uint64_t) + (sizeof(block_id_t) + sizeof(uint32_t)) * multi_use_blocks.size();
|
402
|
+
auto metadata_blocks = sizeof(uint64_t) + (sizeof(idx_t) * 2) * GetMetadataManager().BlockCount();
|
403
|
+
auto total_size = free_list_size + multi_use_blocks_size + metadata_blocks;
|
404
|
+
|
405
|
+
// reserve the blocks that we are going to write
|
406
|
+
// since these blocks are no longer free we cannot just include them in the free list!
|
407
|
+
auto block_size = MetadataManager::METADATA_BLOCK_SIZE - sizeof(idx_t);
|
408
|
+
while (total_size > 0) {
|
409
|
+
auto free_list_handle = GetMetadataManager().AllocateHandle();
|
410
|
+
free_list_blocks.push_back(std::move(free_list_handle));
|
411
|
+
total_size -= MinValue<idx_t>(total_size, block_size);
|
417
412
|
}
|
418
413
|
|
419
414
|
return free_list_blocks;
|
420
415
|
}
|
421
416
|
|
422
|
-
class FreeListBlockWriter : public
|
417
|
+
class FreeListBlockWriter : public MetadataWriter {
|
423
418
|
public:
|
424
|
-
FreeListBlockWriter(
|
425
|
-
:
|
419
|
+
FreeListBlockWriter(MetadataManager &manager, vector<MetadataHandle> free_list_blocks_p)
|
420
|
+
: MetadataWriter(manager), free_list_blocks(std::move(free_list_blocks_p)), index(0) {
|
426
421
|
}
|
427
422
|
|
428
|
-
vector<
|
423
|
+
vector<MetadataHandle> free_list_blocks;
|
429
424
|
idx_t index;
|
430
425
|
|
431
426
|
protected:
|
432
|
-
|
427
|
+
MetadataHandle NextHandle() override {
|
433
428
|
if (index >= free_list_blocks.size()) {
|
434
429
|
throw InternalException(
|
435
430
|
"Free List Block Writer ran out of blocks, this means not enough blocks were allocated up front");
|
436
431
|
}
|
437
|
-
return free_list_blocks[index++];
|
432
|
+
return std::move(free_list_blocks[index++]);
|
438
433
|
}
|
439
434
|
};
|
440
435
|
|
@@ -442,7 +437,7 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
|
|
442
437
|
// set the iteration count
|
443
438
|
header.iteration = ++iteration_count;
|
444
439
|
|
445
|
-
|
440
|
+
auto free_list_blocks = GetFreeListBlocks();
|
446
441
|
|
447
442
|
// now handle the free list
|
448
443
|
// add all modified blocks to the free list: they can now be written to again
|
@@ -451,20 +446,16 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
|
|
451
446
|
}
|
452
447
|
modified_blocks.clear();
|
453
448
|
|
449
|
+
auto &metadata_manager = GetMetadataManager();
|
454
450
|
if (!free_list_blocks.empty()) {
|
455
451
|
// there are blocks to write, either in the free_list or in the modified_blocks
|
456
452
|
// we write these blocks specifically to the free_list_blocks
|
457
|
-
// a normal
|
453
|
+
// a normal MetadataWriter will fetch blocks to use from the free_list
|
458
454
|
// but since we are WRITING the free_list, this behavior is sub-optimal
|
455
|
+
FreeListBlockWriter writer(metadata_manager, std::move(free_list_blocks));
|
459
456
|
|
460
|
-
|
461
|
-
|
462
|
-
auto ptr = writer.GetBlockPointer();
|
463
|
-
D_ASSERT(ptr.block_id == free_list_blocks[0]);
|
464
|
-
header.free_list = ptr.block_id;
|
465
|
-
for (auto &block_id : free_list_blocks) {
|
466
|
-
modified_blocks.insert(block_id);
|
467
|
-
}
|
457
|
+
auto ptr = writer.GetMetaBlockPointer();
|
458
|
+
header.free_list = ptr.block_pointer;
|
468
459
|
|
469
460
|
writer.Write<uint64_t>(free_list.size());
|
470
461
|
for (auto &block_id : free_list) {
|
@@ -475,11 +466,13 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
|
|
475
466
|
writer.Write<block_id_t>(entry.first);
|
476
467
|
writer.Write<uint32_t>(entry.second);
|
477
468
|
}
|
469
|
+
GetMetadataManager().Serialize(writer);
|
478
470
|
writer.Flush();
|
479
471
|
} else {
|
480
472
|
// no blocks in the free list
|
481
|
-
header.free_list =
|
473
|
+
header.free_list = DConstants::INVALID_INDEX;
|
482
474
|
}
|
475
|
+
metadata_manager.Flush();
|
483
476
|
header.block_count = max_block;
|
484
477
|
|
485
478
|
auto &config = DBConfig::Get(db);
|
@@ -494,7 +487,9 @@ void SingleFileBlockManager::WriteHeader(DatabaseHeader header) {
|
|
494
487
|
}
|
495
488
|
// set the header inside the buffer
|
496
489
|
header_buffer.Clear();
|
497
|
-
|
490
|
+
BufferedSerializer serializer;
|
491
|
+
header.Serialize(serializer);
|
492
|
+
memcpy(header_buffer.buffer, serializer.blob.data.get(), serializer.blob.size);
|
498
493
|
// now write the header to the file, active_header determines whether we write to h1 or h2
|
499
494
|
// note that if active_header is h1 we write to h2, and vice versa
|
500
495
|
ChecksumAndWrite(header_buffer, active_header == 1 ? Storage::FILE_HEADER_SIZE : Storage::FILE_HEADER_SIZE * 2);
|
@@ -82,6 +82,9 @@ public:
|
|
82
82
|
BlockManager &GetBlockManagerForRowData() override {
|
83
83
|
return block_manager;
|
84
84
|
}
|
85
|
+
MetadataManager &GetMetadataManager() override {
|
86
|
+
return block_manager.GetMetadataManager();
|
87
|
+
}
|
85
88
|
};
|
86
89
|
|
87
90
|
SingleFileStorageManager::SingleFileStorageManager(AttachedDatabase &db, string path, bool read_only)
|
@@ -140,8 +143,6 @@ void SingleFileStorageManager::LoadDatabase() {
|
|
140
143
|
//! Load from storage
|
141
144
|
auto checkpointer = SingleFileCheckpointReader(*this);
|
142
145
|
checkpointer.LoadFromStorage();
|
143
|
-
// finish load checkpoint, clear the cached handles of meta blocks
|
144
|
-
block_manager->ClearMetaBlockHandles();
|
145
146
|
// check if the WAL file exists
|
146
147
|
if (fs.FileExists(wal_path)) {
|
147
148
|
// replay the WAL
|
@@ -223,7 +224,7 @@ unique_ptr<StorageCommitState> SingleFileStorageManager::GenStorageCommitState(T
|
|
223
224
|
return make_uniq<SingleFileStorageCommitState>(*this, checkpoint);
|
224
225
|
}
|
225
226
|
|
226
|
-
bool SingleFileStorageManager::IsCheckpointClean(
|
227
|
+
bool SingleFileStorageManager::IsCheckpointClean(MetaBlockPointer checkpoint_id) {
|
227
228
|
return block_manager->IsRootBlock(checkpoint_id);
|
228
229
|
}
|
229
230
|
|
@@ -242,9 +242,9 @@ void ColumnDataCheckpointer::WritePersistentSegments() {
|
|
242
242
|
}
|
243
243
|
}
|
244
244
|
|
245
|
-
void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>>
|
246
|
-
D_ASSERT(!
|
247
|
-
this->nodes = std::move(
|
245
|
+
void ColumnDataCheckpointer::Checkpoint(vector<SegmentNode<ColumnSegment>> nodes_p) {
|
246
|
+
D_ASSERT(!nodes_p.empty());
|
247
|
+
this->nodes = std::move(nodes_p);
|
248
248
|
// first check if any of the segments have changes
|
249
249
|
if (!HasChanges()) {
|
250
250
|
// no changes: only need to write the metadata for this column
|
@@ -3,8 +3,7 @@
|
|
3
3
|
|
4
4
|
namespace duckdb {
|
5
5
|
|
6
|
-
PersistentTableData::PersistentTableData(idx_t column_count)
|
7
|
-
: total_rows(0), row_group_count(0), block_id(INVALID_BLOCK), offset(0) {
|
6
|
+
PersistentTableData::PersistentTableData(idx_t column_count) : total_rows(0), row_group_count(0) {
|
8
7
|
}
|
9
8
|
|
10
9
|
PersistentTableData::~PersistentTableData() {
|
@@ -11,7 +11,7 @@
|
|
11
11
|
#include "duckdb/planner/table_filter.hpp"
|
12
12
|
#include "duckdb/execution/expression_executor.hpp"
|
13
13
|
#include "duckdb/storage/checkpoint/table_data_writer.hpp"
|
14
|
-
#include "duckdb/storage/
|
14
|
+
#include "duckdb/storage/metadata/metadata_reader.hpp"
|
15
15
|
#include "duckdb/transaction/duck_transaction_manager.hpp"
|
16
16
|
#include "duckdb/main/database.hpp"
|
17
17
|
#include "duckdb/main/attached_database.hpp"
|
@@ -116,11 +116,10 @@ ColumnData &RowGroup::GetColumn(storage_t c) {
|
|
116
116
|
if (column_pointers.size() != columns.size()) {
|
117
117
|
throw InternalException("Lazy loading a column but the pointer was not set");
|
118
118
|
}
|
119
|
-
auto &
|
119
|
+
auto &metadata_manager = GetCollection().GetMetadataManager();
|
120
120
|
auto &types = GetCollection().GetTypes();
|
121
121
|
auto &block_pointer = column_pointers[c];
|
122
|
-
|
123
|
-
column_data_reader.offset = block_pointer.offset;
|
122
|
+
MetadataReader column_data_reader(metadata_manager, block_pointer);
|
124
123
|
this->columns[c] =
|
125
124
|
ColumnData::Deserialize(GetBlockManager(), GetTableInfo(), c, start, column_data_reader, types[c], nullptr);
|
126
125
|
is_loaded[c] = true;
|
@@ -832,7 +831,7 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
|
|
832
831
|
for (auto &state : result.states) {
|
833
832
|
// get the current position of the table data writer
|
834
833
|
auto &data_writer = writer.GetPayloadWriter();
|
835
|
-
auto pointer = data_writer.
|
834
|
+
auto pointer = data_writer.GetMetaBlockPointer();
|
836
835
|
|
837
836
|
// store the stats and the data pointers in the row group pointers
|
838
837
|
row_group_pointer.data_pointers.push_back(pointer);
|
@@ -898,8 +897,8 @@ void RowGroup::Serialize(RowGroupPointer &pointer, Serializer &main_serializer)
|
|
898
897
|
writer.WriteField<uint64_t>(pointer.tuple_count);
|
899
898
|
auto &serializer = writer.GetSerializer();
|
900
899
|
for (auto &data_pointer : pointer.data_pointers) {
|
901
|
-
serializer.Write<
|
902
|
-
serializer.Write<
|
900
|
+
serializer.Write<idx_t>(data_pointer.block_pointer);
|
901
|
+
serializer.Write<uint32_t>(data_pointer.offset);
|
903
902
|
}
|
904
903
|
CheckpointDeletes(pointer.versions.get(), serializer);
|
905
904
|
writer.Finalize();
|
@@ -917,9 +916,9 @@ RowGroupPointer RowGroup::Deserialize(Deserializer &main_source, const vector<Lo
|
|
917
916
|
|
918
917
|
auto &source = reader.GetSource();
|
919
918
|
for (idx_t i = 0; i < physical_columns; i++) {
|
920
|
-
|
921
|
-
pointer.
|
922
|
-
pointer.offset = source.Read<
|
919
|
+
MetaBlockPointer pointer;
|
920
|
+
pointer.block_pointer = source.Read<idx_t>();
|
921
|
+
pointer.offset = source.Read<uint32_t>();
|
923
922
|
result.data_pointers.push_back(pointer);
|
924
923
|
}
|
925
924
|
result.versions = DeserializeDeletes(source);
|
@@ -6,7 +6,7 @@
|
|
6
6
|
#include "duckdb/planner/constraints/bound_not_null_constraint.hpp"
|
7
7
|
#include "duckdb/storage/checkpoint/table_data_writer.hpp"
|
8
8
|
#include "duckdb/storage/table/row_group_segment_tree.hpp"
|
9
|
-
#include "duckdb/storage/
|
9
|
+
#include "duckdb/storage/metadata/metadata_reader.hpp"
|
10
10
|
#include "duckdb/storage/table/append_state.hpp"
|
11
11
|
#include "duckdb/storage/table/scan_state.hpp"
|
12
12
|
#include "duckdb/storage/table_storage_info.hpp"
|
@@ -27,8 +27,7 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
|
|
27
27
|
current_row_group = 0;
|
28
28
|
max_row_group = data.row_group_count;
|
29
29
|
finished_loading = false;
|
30
|
-
reader = make_uniq<
|
31
|
-
reader->offset = data.offset;
|
30
|
+
reader = make_uniq<MetadataReader>(collection.GetMetadataManager(), data.block_pointer);
|
32
31
|
}
|
33
32
|
|
34
33
|
unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
|
@@ -67,6 +66,10 @@ AttachedDatabase &RowGroupCollection::GetAttached() {
|
|
67
66
|
return GetTableInfo().db;
|
68
67
|
}
|
69
68
|
|
69
|
+
MetadataManager &RowGroupCollection::GetMetadataManager() {
|
70
|
+
return GetBlockManager().GetMetadataManager();
|
71
|
+
}
|
72
|
+
|
70
73
|
//===--------------------------------------------------------------------===//
|
71
74
|
// Initialize
|
72
75
|
//===--------------------------------------------------------------------===//
|
@@ -79,7 +79,7 @@ vector<column_t> TableIndexList::GetRequiredColumns() {
|
|
79
79
|
return result;
|
80
80
|
}
|
81
81
|
|
82
|
-
vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::
|
82
|
+
vector<BlockPointer> TableIndexList::SerializeIndexes(duckdb::MetadataWriter &writer) {
|
83
83
|
vector<BlockPointer> blocks_info;
|
84
84
|
for (auto &index : indexes) {
|
85
85
|
blocks_info.emplace_back(index->Serialize(writer));
|
@@ -60,7 +60,7 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
|
|
60
60
|
return false;
|
61
61
|
} // LCOV_EXCL_STOP
|
62
62
|
initial_reader.reset();
|
63
|
-
if (checkpoint_state.checkpoint_id
|
63
|
+
if (checkpoint_state.checkpoint_id.IsValid()) {
|
64
64
|
// there is a checkpoint flag: check if we need to deserialize the WAL
|
65
65
|
auto &manager = database.GetStorageManager();
|
66
66
|
if (manager.IsCheckpointClean(checkpoint_state.checkpoint_id)) {
|
@@ -523,7 +523,8 @@ void ReplayState::ReplayUpdate() {
|
|
523
523
|
}
|
524
524
|
|
525
525
|
void ReplayState::ReplayCheckpoint() {
|
526
|
-
checkpoint_id = source.Read<
|
526
|
+
checkpoint_id.block_pointer = source.Read<idx_t>();
|
527
|
+
checkpoint_id.offset = source.Read<uint32_t>();
|
527
528
|
}
|
528
529
|
|
529
530
|
} // namespace duckdb
|
@@ -48,9 +48,10 @@ void WriteAheadLog::Delete() {
|
|
48
48
|
//===--------------------------------------------------------------------===//
|
49
49
|
// Write Entries
|
50
50
|
//===--------------------------------------------------------------------===//
|
51
|
-
void WriteAheadLog::WriteCheckpoint(
|
51
|
+
void WriteAheadLog::WriteCheckpoint(MetaBlockPointer meta_block) {
|
52
52
|
writer->Write<WALType>(WALType::CHECKPOINT);
|
53
|
-
writer->Write<
|
53
|
+
writer->Write<idx_t>(meta_block.block_pointer);
|
54
|
+
writer->Write<uint32_t>(meta_block.offset);
|
54
55
|
}
|
55
56
|
|
56
57
|
//===--------------------------------------------------------------------===//
|
@@ -1,46 +0,0 @@
|
|
1
|
-
//===----------------------------------------------------------------------===//
|
2
|
-
// DuckDB
|
3
|
-
//
|
4
|
-
// duckdb/storage/meta_block_reader.hpp
|
5
|
-
//
|
6
|
-
//
|
7
|
-
//===----------------------------------------------------------------------===//
|
8
|
-
|
9
|
-
#pragma once
|
10
|
-
|
11
|
-
#include "duckdb/common/common.hpp"
|
12
|
-
#include "duckdb/common/serializer.hpp"
|
13
|
-
#include "duckdb/storage/block.hpp"
|
14
|
-
#include "duckdb/storage/buffer/buffer_handle.hpp"
|
15
|
-
|
16
|
-
namespace duckdb {
|
17
|
-
class BlockHandle;
|
18
|
-
class BlockManager;
|
19
|
-
class BufferHandle;
|
20
|
-
class DatabaseInstance;
|
21
|
-
|
22
|
-
//! This struct is responsible for reading meta data from disk
|
23
|
-
class MetaBlockReader : public Deserializer {
|
24
|
-
public:
|
25
|
-
MetaBlockReader(BlockManager &block_manager, block_id_t block, bool free_blocks_on_read = true);
|
26
|
-
~MetaBlockReader() override;
|
27
|
-
|
28
|
-
BlockManager &block_manager;
|
29
|
-
shared_ptr<BlockHandle> block;
|
30
|
-
BufferHandle handle;
|
31
|
-
idx_t offset;
|
32
|
-
block_id_t next_block;
|
33
|
-
bool free_blocks_on_read;
|
34
|
-
|
35
|
-
public:
|
36
|
-
//! Read content of size read_size into the buffer
|
37
|
-
void ReadData(data_ptr_t buffer, idx_t read_size) override;
|
38
|
-
|
39
|
-
ClientContext &GetContext() override;
|
40
|
-
void SetContext(ClientContext &context_p);
|
41
|
-
|
42
|
-
private:
|
43
|
-
void ReadNewBlock(block_id_t id);
|
44
|
-
optional_ptr<ClientContext> context;
|
45
|
-
};
|
46
|
-
} // namespace duckdb
|