duckdb 0.8.2-dev3989.0 → 0.8.2-dev4126.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +8 -7
- package/package.json +1 -1
- package/src/duckdb/extension/json/buffered_json_reader.cpp +76 -74
- package/src/duckdb/extension/json/include/buffered_json_reader.hpp +35 -32
- package/src/duckdb/extension/json/include/json_scan.hpp +9 -6
- package/src/duckdb/extension/json/json_scan.cpp +124 -121
- package/src/duckdb/extension/parquet/parquet_extension.cpp +23 -13
- package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp +5 -0
- package/src/duckdb/src/common/crypto/md5.cpp +2 -12
- package/src/duckdb/src/common/radix_partitioning.cpp +1 -1
- package/src/duckdb/src/common/sort/partition_state.cpp +5 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +1 -1
- package/src/duckdb/src/core_functions/function_list.cpp +8 -0
- package/src/duckdb/src/core_functions/scalar/list/list_cosine_similarity.cpp +78 -0
- package/src/duckdb/src/core_functions/scalar/list/list_distance.cpp +72 -0
- package/src/duckdb/src/core_functions/scalar/list/list_inner_product.cpp +70 -0
- package/src/duckdb/src/core_functions/scalar/string/sha256.cpp +32 -0
- package/src/duckdb/src/execution/index/art/art.cpp +111 -92
- package/src/duckdb/src/execution/index/art/iterator.cpp +21 -27
- package/src/duckdb/src/execution/index/art/leaf.cpp +72 -153
- package/src/duckdb/src/execution/index/art/node.cpp +109 -203
- package/src/duckdb/src/execution/index/art/node16.cpp +32 -64
- package/src/duckdb/src/execution/index/art/node256.cpp +38 -53
- package/src/duckdb/src/execution/index/art/node4.cpp +31 -62
- package/src/duckdb/src/execution/index/art/node48.cpp +43 -65
- package/src/duckdb/src/execution/index/art/prefix.cpp +70 -141
- package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +345 -0
- package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +74 -0
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +1 -1
- package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +1 -1
- package/src/duckdb/src/function/scalar/string/suffix.cpp +1 -1
- package/src/duckdb/src/function/table/system/duckdb_columns.cpp +3 -1
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/optional_idx.hpp +1 -1
- package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +51 -0
- package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +9 -0
- package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +17 -7
- package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +5 -5
- package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +10 -16
- package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +38 -116
- package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +17 -18
- package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +17 -23
- package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +17 -18
- package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +17 -24
- package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +16 -22
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +126 -0
- package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +79 -0
- package/src/duckdb/src/include/duckdb/execution/index/index_pointer.hpp +96 -0
- package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/operator/logical_join.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/block.hpp +1 -1
- package/src/duckdb/src/include/duckdb/storage/index.hpp +10 -8
- package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +3 -0
- package/src/duckdb/src/main/extension/extension_helper.cpp +15 -1
- package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +14 -5
- package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +2 -3
- package/src/duckdb/src/storage/checkpoint_manager.cpp +16 -21
- package/src/duckdb/src/storage/data_table.cpp +3 -3
- package/src/duckdb/src/storage/index.cpp +7 -1
- package/src/duckdb/src/storage/metadata/metadata_manager.cpp +21 -21
- package/src/duckdb/src/storage/standard_buffer_manager.cpp +10 -16
- package/src/duckdb/src/storage/storage_info.cpp +1 -1
- package/src/duckdb/src/storage/table_index_list.cpp +1 -1
- package/src/duckdb/src/transaction/commit_state.cpp +5 -1
- package/src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp +4 -1
- package/src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp +24 -2
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +5 -5
- package/src/duckdb/ub_src_core_functions_scalar_list.cpp +6 -0
- package/src/duckdb/ub_src_core_functions_scalar_string.cpp +2 -0
- package/src/duckdb/ub_src_execution_index.cpp +4 -0
- package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
- package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +0 -238
- package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp +0 -115
@@ -1,115 +0,0 @@
|
|
1
|
-
//===----------------------------------------------------------------------===//
|
2
|
-
// DuckDB
|
3
|
-
//
|
4
|
-
// duckdb/execution/index/art/fixed_size_allocator.hpp
|
5
|
-
//
|
6
|
-
//
|
7
|
-
//===----------------------------------------------------------------------===//
|
8
|
-
|
9
|
-
#pragma once
|
10
|
-
|
11
|
-
#include "duckdb/common/types/validity_mask.hpp"
|
12
|
-
#include "duckdb/common/unordered_set.hpp"
|
13
|
-
#include "duckdb/storage/buffer_manager.hpp"
|
14
|
-
#include "duckdb/execution/index/art/node.hpp"
|
15
|
-
|
16
|
-
namespace duckdb {
|
17
|
-
|
18
|
-
// structs
|
19
|
-
struct BufferEntry {
|
20
|
-
BufferEntry(const data_ptr_t &ptr, const idx_t &allocation_count) : ptr(ptr), allocation_count(allocation_count) {
|
21
|
-
}
|
22
|
-
data_ptr_t ptr;
|
23
|
-
idx_t allocation_count;
|
24
|
-
};
|
25
|
-
|
26
|
-
//! The FixedSizeAllocator provides pointers to fixed-size sections of pre-allocated memory buffers.
|
27
|
-
//! The pointers are Node pointers, and the leftmost byte (serialize flag and type) must always be zero.
|
28
|
-
class FixedSizeAllocator {
|
29
|
-
public:
|
30
|
-
//! Fixed size of the buffers
|
31
|
-
static constexpr idx_t BUFFER_ALLOC_SIZE = Storage::BLOCK_ALLOC_SIZE;
|
32
|
-
//! We can vacuum 10% or more of the total memory usage of the allocator
|
33
|
-
static constexpr uint8_t VACUUM_THRESHOLD = 10;
|
34
|
-
|
35
|
-
//! Constants for fast offset calculations in the bitmask
|
36
|
-
static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
|
37
|
-
static constexpr uint8_t SHIFT[] = {32, 16, 8, 4, 2, 1};
|
38
|
-
|
39
|
-
public:
|
40
|
-
explicit FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator);
|
41
|
-
~FixedSizeAllocator();
|
42
|
-
|
43
|
-
//! Allocation size of one element in a buffer
|
44
|
-
idx_t allocation_size;
|
45
|
-
//! Total number of allocations
|
46
|
-
idx_t total_allocations;
|
47
|
-
//! Number of validity_t values in the bitmask
|
48
|
-
idx_t bitmask_count;
|
49
|
-
//! First starting byte of the payload
|
50
|
-
idx_t allocation_offset;
|
51
|
-
//! Number of possible allocations per buffer
|
52
|
-
idx_t allocations_per_buffer;
|
53
|
-
|
54
|
-
//! Buffers containing the data
|
55
|
-
vector<BufferEntry> buffers;
|
56
|
-
//! Buffers with free space
|
57
|
-
unordered_set<idx_t> buffers_with_free_space;
|
58
|
-
|
59
|
-
//! Minimum buffer ID of buffers that can be vacuumed
|
60
|
-
idx_t min_vacuum_buffer_id;
|
61
|
-
|
62
|
-
//! Buffer manager of the database instance
|
63
|
-
Allocator &allocator;
|
64
|
-
|
65
|
-
public:
|
66
|
-
//! Get a new Node pointer to data, might cause a new buffer allocation
|
67
|
-
Node New();
|
68
|
-
//! Free the data of the Node pointer
|
69
|
-
void Free(const Node ptr);
|
70
|
-
//! Get the data of the Node pointer
|
71
|
-
template <class T>
|
72
|
-
inline T *Get(const Node ptr) const {
|
73
|
-
return (T *)Get(ptr);
|
74
|
-
}
|
75
|
-
|
76
|
-
//! Resets the allocator, e.g., becomes necessary during DELETE FROM table
|
77
|
-
void Reset();
|
78
|
-
|
79
|
-
//! Returns the allocated memory size in bytes
|
80
|
-
inline idx_t GetMemoryUsage() const {
|
81
|
-
return buffers.size() * BUFFER_ALLOC_SIZE;
|
82
|
-
}
|
83
|
-
|
84
|
-
//! Merge another FixedSizeAllocator into this allocator. Both must have the same allocation size
|
85
|
-
void Merge(FixedSizeAllocator &other);
|
86
|
-
|
87
|
-
//! Initialize a vacuum operation, and return true, if the allocator needs a vacuum
|
88
|
-
bool InitializeVacuum();
|
89
|
-
//! Finalize a vacuum operation by freeing all buffers exceeding the min_vacuum_buffer_id
|
90
|
-
void FinalizeVacuum();
|
91
|
-
//! Returns true, if a Node pointer qualifies for a vacuum operation, and false otherwise
|
92
|
-
inline bool NeedsVacuum(const Node ptr) const {
|
93
|
-
if (ptr.GetBufferId() >= min_vacuum_buffer_id) {
|
94
|
-
return true;
|
95
|
-
}
|
96
|
-
return false;
|
97
|
-
}
|
98
|
-
//! Vacuums a Node pointer
|
99
|
-
Node VacuumPointer(const Node ptr);
|
100
|
-
|
101
|
-
//! Verify that the allocation counts match the existing positions on the buffers
|
102
|
-
void Verify() const;
|
103
|
-
|
104
|
-
private:
|
105
|
-
//! Returns the data_ptr_t of a Node pointer
|
106
|
-
inline data_ptr_t Get(const Node ptr) const {
|
107
|
-
D_ASSERT(ptr.GetBufferId() < buffers.size());
|
108
|
-
D_ASSERT(ptr.GetOffset() < allocations_per_buffer);
|
109
|
-
return buffers[ptr.GetBufferId()].ptr + ptr.GetOffset() * allocation_size + allocation_offset;
|
110
|
-
}
|
111
|
-
//! Returns the first free offset in a bitmask
|
112
|
-
uint32_t GetOffset(ValidityMask &mask, const idx_t allocation_count);
|
113
|
-
};
|
114
|
-
|
115
|
-
} // namespace duckdb
|