duckdb 0.8.2-dev2673.0 → 0.8.2-dev2700.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +1 -0
- package/package.json +1 -1
- package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +6 -6
- package/src/duckdb/src/common/field_writer.cpp +1 -0
- package/src/duckdb/src/common/local_file_system.cpp +5 -0
- package/src/duckdb/src/common/types/interval.cpp +3 -0
- package/src/duckdb/src/execution/index/art/art.cpp +6 -9
- package/src/duckdb/src/execution/index/art/leaf.cpp +4 -4
- package/src/duckdb/src/execution/index/art/node.cpp +9 -12
- package/src/duckdb/src/execution/index/art/node16.cpp +4 -4
- package/src/duckdb/src/execution/index/art/node256.cpp +4 -4
- package/src/duckdb/src/execution/index/art/node4.cpp +4 -5
- package/src/duckdb/src/execution/index/art/node48.cpp +4 -4
- package/src/duckdb/src/execution/index/art/prefix.cpp +4 -6
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +4 -4
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +6 -4
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/block.hpp +27 -4
- package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +9 -9
- package/src/duckdb/src/include/duckdb/storage/checkpoint/row_group_writer.hpp +5 -5
- package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_reader.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +19 -16
- package/src/duckdb/src/include/duckdb/storage/data_pointer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/in_memory_block_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/index.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +88 -0
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp +54 -0
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +45 -0
- package/src/duckdb/src/include/duckdb/storage/partial_block_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/single_file_block_manager.hpp +6 -5
- package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/persistent_table_data.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +2 -0
- package/src/duckdb/src/include/duckdb/storage/table/row_group_segment_tree.hpp +2 -2
- package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +3 -4
- package/src/duckdb/src/storage/buffer/block_manager.cpp +10 -9
- package/src/duckdb/src/storage/checkpoint/row_group_writer.cpp +1 -1
- package/src/duckdb/src/storage/checkpoint/table_data_reader.cpp +3 -4
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +7 -7
- package/src/duckdb/src/storage/checkpoint_manager.cpp +49 -43
- package/src/duckdb/src/storage/index.cpp +1 -1
- package/src/duckdb/src/storage/metadata/metadata_manager.cpp +267 -0
- package/src/duckdb/src/storage/metadata/metadata_reader.cpp +80 -0
- package/src/duckdb/src/storage/metadata/metadata_writer.cpp +86 -0
- package/src/duckdb/src/storage/single_file_block_manager.cpp +47 -52
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/storage_manager.cpp +4 -3
- package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -3
- package/src/duckdb/src/storage/table/persistent_table_data.cpp +1 -2
- package/src/duckdb/src/storage/table/row_group.cpp +9 -10
- package/src/duckdb/src/storage/table/row_group_collection.cpp +6 -3
- package/src/duckdb/src/storage/table_index_list.cpp +1 -1
- package/src/duckdb/src/storage/wal_replay.cpp +3 -2
- package/src/duckdb/src/storage/write_ahead_log.cpp +3 -2
- package/src/duckdb/ub_src_storage.cpp +0 -4
- package/src/duckdb/ub_src_storage_metadata.cpp +6 -0
- package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -46
- package/src/duckdb/src/include/duckdb/storage/meta_block_writer.hpp +0 -50
- package/src/duckdb/src/storage/meta_block_reader.cpp +0 -69
- package/src/duckdb/src/storage/meta_block_writer.cpp +0 -80
@@ -1,50 +0,0 @@
|
|
1
|
-
//===----------------------------------------------------------------------===//
|
2
|
-
// DuckDB
|
3
|
-
//
|
4
|
-
// duckdb/storage/meta_block_writer.hpp
|
5
|
-
//
|
6
|
-
//
|
7
|
-
//===----------------------------------------------------------------------===//
|
8
|
-
|
9
|
-
#pragma once
|
10
|
-
|
11
|
-
#include "duckdb/common/common.hpp"
|
12
|
-
#include "duckdb/common/serializer.hpp"
|
13
|
-
#include "duckdb/storage/block.hpp"
|
14
|
-
#include "duckdb/storage/block_manager.hpp"
|
15
|
-
#include "duckdb/common/set.hpp"
|
16
|
-
|
17
|
-
namespace duckdb {
|
18
|
-
class DatabaseInstance;
|
19
|
-
|
20
|
-
//! This struct is responsible for writing data to disk in a stream of blocks.
|
21
|
-
class MetaBlockWriter : public Serializer {
|
22
|
-
public:
|
23
|
-
MetaBlockWriter(BlockManager &block_manager, block_id_t initial_block_id = INVALID_BLOCK);
|
24
|
-
~MetaBlockWriter() override;
|
25
|
-
|
26
|
-
BlockManager &block_manager;
|
27
|
-
|
28
|
-
protected:
|
29
|
-
unique_ptr<Block> block;
|
30
|
-
set<block_id_t> written_blocks;
|
31
|
-
idx_t offset;
|
32
|
-
|
33
|
-
public:
|
34
|
-
BlockPointer GetBlockPointer();
|
35
|
-
virtual void Flush();
|
36
|
-
|
37
|
-
void WriteData(const_data_ptr_t buffer, idx_t write_size) override;
|
38
|
-
|
39
|
-
void MarkWrittenBlocks() {
|
40
|
-
for (auto &block_id : written_blocks) {
|
41
|
-
block_manager.MarkBlockAsModified(block_id);
|
42
|
-
}
|
43
|
-
}
|
44
|
-
|
45
|
-
protected:
|
46
|
-
virtual block_id_t GetNextBlockId();
|
47
|
-
void AdvanceBlock();
|
48
|
-
};
|
49
|
-
|
50
|
-
} // namespace duckdb
|
@@ -1,69 +0,0 @@
|
|
1
|
-
#include "duckdb/storage/meta_block_reader.hpp"
|
2
|
-
#include "duckdb/storage/buffer_manager.hpp"
|
3
|
-
#include "duckdb/main/connection_manager.hpp"
|
4
|
-
#include "duckdb/main/database.hpp"
|
5
|
-
|
6
|
-
#include <cstring>
|
7
|
-
|
8
|
-
namespace duckdb {
|
9
|
-
|
10
|
-
MetaBlockReader::MetaBlockReader(BlockManager &block_manager, block_id_t block_id, bool free_blocks_on_read)
|
11
|
-
: block_manager(block_manager), offset(0), next_block(-1), free_blocks_on_read(free_blocks_on_read) {
|
12
|
-
ReadNewBlock(block_id);
|
13
|
-
}
|
14
|
-
|
15
|
-
MetaBlockReader::~MetaBlockReader() {
|
16
|
-
}
|
17
|
-
|
18
|
-
void MetaBlockReader::ReadData(data_ptr_t buffer, idx_t read_size) {
|
19
|
-
while (offset + read_size > handle.GetFileBuffer().size) {
|
20
|
-
// cannot read entire entry from block
|
21
|
-
// first read what we can from this block
|
22
|
-
idx_t to_read = handle.GetFileBuffer().size - offset;
|
23
|
-
if (to_read > 0) {
|
24
|
-
memcpy(buffer, handle.Ptr() + offset, to_read);
|
25
|
-
read_size -= to_read;
|
26
|
-
buffer += to_read;
|
27
|
-
}
|
28
|
-
// then move to the next block
|
29
|
-
if (next_block == INVALID_BLOCK) {
|
30
|
-
throw IOException("Cannot read from INVALID_BLOCK.");
|
31
|
-
}
|
32
|
-
ReadNewBlock(next_block);
|
33
|
-
}
|
34
|
-
// we have enough left in this block to read from the buffer
|
35
|
-
memcpy(buffer, handle.Ptr() + offset, read_size);
|
36
|
-
offset += read_size;
|
37
|
-
}
|
38
|
-
|
39
|
-
ClientContext &MetaBlockReader::GetContext() {
|
40
|
-
if (!context) {
|
41
|
-
throw InternalException("Meta Block Reader is missing context");
|
42
|
-
}
|
43
|
-
return *context;
|
44
|
-
}
|
45
|
-
|
46
|
-
void MetaBlockReader::ReadNewBlock(block_id_t id) {
|
47
|
-
auto &buffer_manager = block_manager.buffer_manager;
|
48
|
-
|
49
|
-
// Marking these blocks as modified will cause them to be moved to the free
|
50
|
-
// list upon the next successful checkpoint. Marking them modified here
|
51
|
-
// assumes MetaBlockReader is exclusively used for reading checkpoint data,
|
52
|
-
// and thus any blocks we're reading will be obviated by the next checkpoint.
|
53
|
-
if (free_blocks_on_read) {
|
54
|
-
block_manager.MarkBlockAsModified(id);
|
55
|
-
}
|
56
|
-
block = block_manager.RegisterBlock(id, true);
|
57
|
-
handle = buffer_manager.Pin(block);
|
58
|
-
|
59
|
-
next_block = Load<block_id_t>(handle.Ptr());
|
60
|
-
D_ASSERT(next_block >= -1);
|
61
|
-
offset = sizeof(block_id_t);
|
62
|
-
}
|
63
|
-
|
64
|
-
void MetaBlockReader::SetContext(ClientContext &context_p) {
|
65
|
-
D_ASSERT(!context);
|
66
|
-
context = &context_p;
|
67
|
-
}
|
68
|
-
|
69
|
-
} // namespace duckdb
|
@@ -1,80 +0,0 @@
|
|
1
|
-
#include "duckdb/storage/meta_block_writer.hpp"
|
2
|
-
|
3
|
-
#include <cstring>
|
4
|
-
|
5
|
-
namespace duckdb {
|
6
|
-
|
7
|
-
MetaBlockWriter::MetaBlockWriter(BlockManager &block_manager, block_id_t initial_block_id)
|
8
|
-
: block_manager(block_manager) {
|
9
|
-
if (initial_block_id == INVALID_BLOCK) {
|
10
|
-
initial_block_id = MetaBlockWriter::GetNextBlockId();
|
11
|
-
}
|
12
|
-
block = block_manager.CreateBlock(initial_block_id, nullptr);
|
13
|
-
Store<block_id_t>(-1, block->buffer);
|
14
|
-
offset = sizeof(block_id_t);
|
15
|
-
}
|
16
|
-
|
17
|
-
MetaBlockWriter::~MetaBlockWriter() {
|
18
|
-
// If there's an exception during checkpoint, this can get destroyed without
|
19
|
-
// flushing the data...which is fine, because none of the unwritten data
|
20
|
-
// will be referenced.
|
21
|
-
//
|
22
|
-
// Otherwise, we should have explicitly flushed (and thereby nulled the block).
|
23
|
-
D_ASSERT(!block || Exception::UncaughtException());
|
24
|
-
}
|
25
|
-
|
26
|
-
block_id_t MetaBlockWriter::GetNextBlockId() {
|
27
|
-
return block_manager.GetFreeBlockId();
|
28
|
-
}
|
29
|
-
|
30
|
-
BlockPointer MetaBlockWriter::GetBlockPointer() {
|
31
|
-
BlockPointer pointer;
|
32
|
-
pointer.block_id = block->id;
|
33
|
-
pointer.offset = offset;
|
34
|
-
return pointer;
|
35
|
-
}
|
36
|
-
|
37
|
-
void MetaBlockWriter::Flush() {
|
38
|
-
if (offset < block->size) {
|
39
|
-
// clear remaining bytes of block (if any)
|
40
|
-
memset(block->buffer + offset, 0, block->size - offset);
|
41
|
-
}
|
42
|
-
AdvanceBlock();
|
43
|
-
block = nullptr;
|
44
|
-
}
|
45
|
-
|
46
|
-
void MetaBlockWriter::AdvanceBlock() {
|
47
|
-
written_blocks.insert(block->id);
|
48
|
-
if (offset > sizeof(block_id_t)) {
|
49
|
-
block_manager.Write(*block);
|
50
|
-
offset = sizeof(block_id_t);
|
51
|
-
}
|
52
|
-
}
|
53
|
-
|
54
|
-
void MetaBlockWriter::WriteData(const_data_ptr_t buffer, idx_t write_size) {
|
55
|
-
while (offset + write_size > block->size) {
|
56
|
-
// we need to make a new block
|
57
|
-
// first copy what we can
|
58
|
-
D_ASSERT(offset <= block->size);
|
59
|
-
idx_t copy_amount = block->size - offset;
|
60
|
-
if (copy_amount > 0) {
|
61
|
-
memcpy(block->buffer + offset, buffer, copy_amount);
|
62
|
-
buffer += copy_amount;
|
63
|
-
offset += copy_amount;
|
64
|
-
write_size -= copy_amount;
|
65
|
-
}
|
66
|
-
// now we need to get a new block id
|
67
|
-
block_id_t new_block_id = GetNextBlockId();
|
68
|
-
// write the block id of the new block to the start of the current block
|
69
|
-
Store<block_id_t>(new_block_id, block->buffer);
|
70
|
-
// first flush the old block
|
71
|
-
AdvanceBlock();
|
72
|
-
// now update the block id of the block
|
73
|
-
block->id = new_block_id;
|
74
|
-
Store<block_id_t>(-1, block->buffer);
|
75
|
-
}
|
76
|
-
memcpy(block->buffer + offset, buffer, write_size);
|
77
|
-
offset += write_size;
|
78
|
-
}
|
79
|
-
|
80
|
-
} // namespace duckdb
|