duckdb 0.7.2-dev1734.0 → 0.7.2-dev1867.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/src/catalog/catalog.cpp +27 -27
- package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp +6 -6
- package/src/duckdb/src/catalog/catalog_set.cpp +27 -25
- package/src/duckdb/src/catalog/default/default_functions.cpp +4 -4
- package/src/duckdb/src/catalog/default/default_types.cpp +4 -4
- package/src/duckdb/src/catalog/default/default_views.cpp +4 -4
- package/src/duckdb/src/catalog/dependency_list.cpp +7 -6
- package/src/duckdb/src/catalog/dependency_manager.cpp +44 -38
- package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +11 -6
- package/src/duckdb/src/common/types/batched_data_collection.cpp +2 -1
- package/src/duckdb/src/common/types/column_data_allocator.cpp +1 -0
- package/src/duckdb/src/common/types/vector.cpp +2 -2
- package/src/duckdb/src/common/types.cpp +2 -2
- package/src/duckdb/src/common/vector_operations/vector_copy.cpp +14 -11
- package/src/duckdb/src/execution/operator/aggregate/distinct_aggregate_data.cpp +1 -1
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +51 -50
- package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +14 -13
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +20 -20
- package/src/duckdb/src/execution/operator/schema/physical_create_table.cpp +2 -2
- package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan/plan_create_table.cpp +3 -3
- package/src/duckdb/src/execution/physical_plan/plan_delete.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan/plan_insert.cpp +1 -1
- package/src/duckdb/src/execution/physical_plan/plan_update.cpp +1 -1
- package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +3 -3
- package/src/duckdb/src/function/cast/cast_function_set.cpp +2 -1
- package/src/duckdb/src/function/scalar/math/numeric.cpp +57 -0
- package/src/duckdb/src/function/scalar/math_functions.cpp +1 -0
- package/src/duckdb/src/function/scalar/sequence/nextval.cpp +29 -29
- package/src/duckdb/src/function/scalar/string/damerau_levenshtein.cpp +106 -0
- package/src/duckdb/src/function/scalar/string/hex.cpp +261 -78
- package/src/duckdb/src/function/scalar/string/regexp.cpp +145 -28
- package/src/duckdb/src/function/scalar/string_functions.cpp +1 -0
- package/src/duckdb/src/function/table/checkpoint.cpp +4 -4
- package/src/duckdb/src/function/table/system/duckdb_columns.cpp +24 -24
- package/src/duckdb/src/function/table/system/duckdb_constraints.cpp +7 -6
- package/src/duckdb/src/function/table/system/duckdb_databases.cpp +1 -1
- package/src/duckdb/src/function/table/system/duckdb_dependencies.cpp +11 -11
- package/src/duckdb/src/function/table/system/pragma_database_size.cpp +1 -1
- package/src/duckdb/src/function/table/system/pragma_table_info.cpp +17 -18
- package/src/duckdb/src/function/table/table_scan.cpp +8 -11
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +9 -9
- package/src/duckdb/src/include/duckdb/catalog/catalog_entry_map.hpp +38 -0
- package/src/duckdb/src/include/duckdb/catalog/catalog_transaction.hpp +4 -3
- package/src/duckdb/src/include/duckdb/catalog/default/default_functions.hpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/default/default_types.hpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/default/default_views.hpp +2 -2
- package/src/duckdb/src/include/duckdb/catalog/dependency.hpp +4 -5
- package/src/duckdb/src/include/duckdb/catalog/dependency_list.hpp +4 -5
- package/src/duckdb/src/include/duckdb/catalog/dependency_manager.hpp +10 -9
- package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -1
- package/src/duckdb/src/include/duckdb/common/bit_utils.hpp +147 -0
- package/src/duckdb/src/include/duckdb/common/field_writer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/helper.hpp +9 -0
- package/src/duckdb/src/include/duckdb/common/hugeint.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/optional_ptr.hpp +29 -6
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +6 -5
- package/src/duckdb/src/include/duckdb/common/serializer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/string_util.hpp +7 -0
- package/src/duckdb/src/include/duckdb/common/types/row_data_collection.hpp +1 -0
- package/src/duckdb/src/include/duckdb/common/types.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +5 -5
- package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_table.hpp +2 -2
- package/src/duckdb/src/include/duckdb/function/cast/default_casts.hpp +3 -2
- package/src/duckdb/src/include/duckdb/function/scalar/math_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/database_manager.hpp +4 -3
- package/src/duckdb/src/include/duckdb/main/query_result.hpp +3 -2
- package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp +7 -7
- package/src/duckdb/src/include/duckdb/optimizer/matcher/expression_matcher.hpp +11 -11
- package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +8 -8
- package/src/duckdb/src/include/duckdb/optimizer/rule/arithmetic_simplification.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/case_simplification.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/comparison_simplification.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/conjunction_simplification.hpp +2 -2
- package/src/duckdb/src/include/duckdb/optimizer/rule/constant_folding.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/date_part_simplification.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/distributivity.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/empty_needle_removal.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/enum_comparison.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/equal_or_null_simplification.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/in_clause_simplification.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/like_optimizations.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/move_constants.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule/regex_optimizations.hpp +1 -1
- package/src/duckdb/src/include/duckdb/optimizer/rule.hpp +2 -2
- package/src/duckdb/src/include/duckdb/parser/base_expression.hpp +1 -1
- package/src/duckdb/src/include/duckdb/parser/expression_map.hpp +19 -6
- package/src/duckdb/src/include/duckdb/parser/expression_util.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression.hpp +5 -2
- package/src/duckdb/src/include/duckdb/planner/expression_binder/base_select_binder.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/expression_binder/order_binder.hpp +3 -3
- package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp +15 -7
- package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp +3 -0
- package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp +49 -126
- package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp128.hpp +1 -0
- package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp +0 -97
- package/src/duckdb/src/include/duckdb/storage/compression/patas/algorithm/patas.hpp +1 -0
- package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +5 -5
- package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp +159 -0
- package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +1 -0
- package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp +6 -5
- package/src/duckdb/src/main/client_context.cpp +1 -1
- package/src/duckdb/src/main/database.cpp +2 -1
- package/src/duckdb/src/main/database_manager.cpp +4 -4
- package/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp +2 -2
- package/src/duckdb/src/optimizer/cse_optimizer.cpp +4 -4
- package/src/duckdb/src/optimizer/deliminator.cpp +13 -11
- package/src/duckdb/src/optimizer/expression_rewriter.cpp +2 -2
- package/src/duckdb/src/optimizer/filter_combiner.cpp +67 -65
- package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +1 -0
- package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +26 -25
- package/src/duckdb/src/optimizer/matcher/expression_matcher.cpp +23 -21
- package/src/duckdb/src/optimizer/rule/arithmetic_simplification.cpp +3 -3
- package/src/duckdb/src/optimizer/rule/case_simplification.cpp +2 -2
- package/src/duckdb/src/optimizer/rule/comparison_simplification.cpp +6 -7
- package/src/duckdb/src/optimizer/rule/conjunction_simplification.cpp +9 -8
- package/src/duckdb/src/optimizer/rule/constant_folding.cpp +7 -7
- package/src/duckdb/src/optimizer/rule/date_part_simplification.cpp +3 -3
- package/src/duckdb/src/optimizer/rule/distributivity.cpp +5 -5
- package/src/duckdb/src/optimizer/rule/empty_needle_removal.cpp +6 -6
- package/src/duckdb/src/optimizer/rule/enum_comparison.cpp +4 -4
- package/src/duckdb/src/optimizer/rule/equal_or_null_simplification.cpp +23 -26
- package/src/duckdb/src/optimizer/rule/in_clause_simplification_rule.cpp +2 -3
- package/src/duckdb/src/optimizer/rule/like_optimizations.cpp +3 -3
- package/src/duckdb/src/optimizer/rule/move_constants.cpp +6 -6
- package/src/duckdb/src/optimizer/rule/ordered_aggregate_optimizer.cpp +2 -2
- package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +11 -10
- package/src/duckdb/src/parser/expression_util.cpp +6 -6
- package/src/duckdb/src/parser/transform/helpers/transform_groupby.cpp +3 -3
- package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp +2 -2
- package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +3 -3
- package/src/duckdb/src/planner/binder/query_node/bind_setop_node.cpp +5 -5
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +2 -2
- package/src/duckdb/src/planner/expression_binder/base_select_binder.cpp +4 -4
- package/src/duckdb/src/planner/expression_binder/order_binder.cpp +3 -3
- package/src/duckdb/src/storage/buffer/block_handle.cpp +7 -6
- package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -1
- package/src/duckdb/src/storage/buffer/buffer_handle.cpp +1 -0
- package/src/duckdb/src/storage/buffer/buffer_pool.cpp +6 -2
- package/src/duckdb/src/storage/buffer/buffer_pool_reservation.cpp +7 -4
- package/src/duckdb/src/storage/buffer_manager.cpp +35 -726
- package/src/duckdb/src/storage/checkpoint_manager.cpp +2 -2
- package/src/duckdb/src/storage/meta_block_reader.cpp +6 -5
- package/src/duckdb/src/storage/standard_buffer_manager.cpp +801 -0
- package/src/duckdb/src/storage/wal_replay.cpp +2 -2
- package/src/duckdb/src/transaction/meta_transaction.cpp +13 -13
- package/src/duckdb/src/transaction/transaction.cpp +1 -1
- package/src/duckdb/src/transaction/transaction_context.cpp +1 -1
- package/src/duckdb/ub_src_function_scalar_string.cpp +2 -0
- package/src/duckdb/ub_src_storage.cpp +2 -0
@@ -10,68 +10,6 @@
|
|
10
10
|
|
11
11
|
#include "duckdb.h"
|
12
12
|
|
13
|
-
#ifdef _MSC_VER
|
14
|
-
#define __restrict__
|
15
|
-
#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
|
16
|
-
#define __ORDER_LITTLE_ENDIAN__ 2
|
17
|
-
#include <intrin.h>
|
18
|
-
static inline int __builtin_ctzll(unsigned long long x) {
|
19
|
-
#ifdef _WIN64
|
20
|
-
unsigned long ret;
|
21
|
-
_BitScanForward64(&ret, x);
|
22
|
-
return (int)ret;
|
23
|
-
#else
|
24
|
-
unsigned long low, high;
|
25
|
-
bool low_set = _BitScanForward(&low, (unsigned __int32)(x)) != 0;
|
26
|
-
_BitScanForward(&high, (unsigned __int32)(x >> 32));
|
27
|
-
high += 32;
|
28
|
-
return low_set ? low : high;
|
29
|
-
#endif
|
30
|
-
}
|
31
|
-
static inline int __builtin_clzll(unsigned long long mask) {
|
32
|
-
unsigned long where;
|
33
|
-
// BitScanReverse scans from MSB to LSB for first set bit.
|
34
|
-
// Returns 0 if no set bit is found.
|
35
|
-
#if defined(_WIN64)
|
36
|
-
if (_BitScanReverse64(&where, mask))
|
37
|
-
return static_cast<int>(63 - where);
|
38
|
-
#elif defined(_WIN32)
|
39
|
-
// Scan the high 32 bits.
|
40
|
-
if (_BitScanReverse(&where, static_cast<unsigned long>(mask >> 32)))
|
41
|
-
return static_cast<int>(63 - (where + 32)); // Create a bit offset from the MSB.
|
42
|
-
// Scan the low 32 bits.
|
43
|
-
if (_BitScanReverse(&where, static_cast<unsigned long>(mask)))
|
44
|
-
return static_cast<int>(63 - where);
|
45
|
-
#else
|
46
|
-
#error "Implementation of __builtin_clzll required"
|
47
|
-
#endif
|
48
|
-
return 64; // Undefined Behavior.
|
49
|
-
}
|
50
|
-
|
51
|
-
static inline int __builtin_ctz(unsigned int value) {
|
52
|
-
unsigned long trailing_zero = 0;
|
53
|
-
|
54
|
-
if (_BitScanForward(&trailing_zero, value)) {
|
55
|
-
return trailing_zero;
|
56
|
-
} else {
|
57
|
-
// This is undefined, I better choose 32 than 0
|
58
|
-
return 32;
|
59
|
-
}
|
60
|
-
}
|
61
|
-
|
62
|
-
static inline int __builtin_clz(unsigned int value) {
|
63
|
-
unsigned long leading_zero = 0;
|
64
|
-
|
65
|
-
if (_BitScanReverse(&leading_zero, value)) {
|
66
|
-
return 31 - leading_zero;
|
67
|
-
} else {
|
68
|
-
// Same remarks as above
|
69
|
-
return 32;
|
70
|
-
}
|
71
|
-
}
|
72
|
-
|
73
|
-
#endif
|
74
|
-
|
75
13
|
namespace duckdb {
|
76
14
|
|
77
15
|
template <class T>
|
@@ -89,41 +27,6 @@ struct SignificantBits<uint32_t> {
|
|
89
27
|
static constexpr uint8_t mask = ((uint8_t)1 << size) - 1;
|
90
28
|
};
|
91
29
|
|
92
|
-
template <class T>
|
93
|
-
struct CountZeros {};
|
94
|
-
|
95
|
-
template <>
|
96
|
-
struct CountZeros<uint32_t> {
|
97
|
-
inline static int Leading(uint32_t value) {
|
98
|
-
if (!value) {
|
99
|
-
return 32;
|
100
|
-
}
|
101
|
-
return __builtin_clz(value);
|
102
|
-
}
|
103
|
-
inline static int Trailing(uint32_t value) {
|
104
|
-
if (!value) {
|
105
|
-
return 32;
|
106
|
-
}
|
107
|
-
return __builtin_ctz(value);
|
108
|
-
}
|
109
|
-
};
|
110
|
-
|
111
|
-
template <>
|
112
|
-
struct CountZeros<uint64_t> {
|
113
|
-
inline static int Leading(uint64_t value) {
|
114
|
-
if (!value) {
|
115
|
-
return 64;
|
116
|
-
}
|
117
|
-
return __builtin_clzll(value);
|
118
|
-
}
|
119
|
-
inline static int Trailing(uint64_t value) {
|
120
|
-
if (!value) {
|
121
|
-
return 64;
|
122
|
-
}
|
123
|
-
return __builtin_ctzll(value);
|
124
|
-
}
|
125
|
-
};
|
126
|
-
|
127
30
|
struct ChimpConstants {
|
128
31
|
struct Compression {
|
129
32
|
static constexpr uint8_t LEADING_ROUND[] = {0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 12, 12, 12, 12,
|
@@ -14,6 +14,7 @@
|
|
14
14
|
#include "duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp"
|
15
15
|
#include "duckdb/storage/compression/chimp/algorithm/packed_data.hpp"
|
16
16
|
#include "duckdb/storage/compression/patas/shared.hpp"
|
17
|
+
#include "duckdb/common/bit_utils.hpp"
|
17
18
|
|
18
19
|
namespace duckdb {
|
19
20
|
|
@@ -37,13 +37,13 @@ public:
|
|
37
37
|
void ReadData(data_ptr_t buffer, idx_t read_size) override;
|
38
38
|
|
39
39
|
ClientContext &GetContext() override;
|
40
|
-
Catalog
|
41
|
-
void SetCatalog(Catalog
|
42
|
-
void SetContext(ClientContext
|
40
|
+
optional_ptr<Catalog> GetCatalog() override;
|
41
|
+
void SetCatalog(Catalog &catalog_p);
|
42
|
+
void SetContext(ClientContext &context_p);
|
43
43
|
|
44
44
|
private:
|
45
45
|
void ReadNewBlock(block_id_t id);
|
46
|
-
|
47
|
-
|
46
|
+
optional_ptr<ClientContext> context;
|
47
|
+
optional_ptr<Catalog> catalog;
|
48
48
|
};
|
49
49
|
} // namespace duckdb
|
@@ -0,0 +1,159 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/storage/standard_buffer_manager.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/common/allocator.hpp"
|
12
|
+
#include "duckdb/common/atomic.hpp"
|
13
|
+
#include "duckdb/common/file_system.hpp"
|
14
|
+
#include "duckdb/common/mutex.hpp"
|
15
|
+
#include "duckdb/storage/block_manager.hpp"
|
16
|
+
|
17
|
+
#include "duckdb/storage/buffer/block_handle.hpp"
|
18
|
+
#include "duckdb/storage/buffer_manager.hpp"
|
19
|
+
#include "duckdb/storage/buffer/buffer_pool.hpp"
|
20
|
+
|
21
|
+
namespace duckdb {
|
22
|
+
class BlockManager;
|
23
|
+
class DatabaseInstance;
|
24
|
+
class TemporaryDirectoryHandle;
|
25
|
+
struct EvictionQueue;
|
26
|
+
|
27
|
+
//! The BufferManager is in charge of handling memory management for a single database. It cooperatively shares a
|
28
|
+
//! BufferPool with other BufferManagers, belonging to different databases. It hands out memory buffers that can
|
29
|
+
//! be used by the database internally, and offers configuration options specific to a database, which need not be
|
30
|
+
//! shared by the BufferPool, including whether to support swapping temp buffers to disk, and where to swap them to.
|
31
|
+
class StandardBufferManager : public BufferManager {
|
32
|
+
friend class BufferHandle;
|
33
|
+
friend class BlockHandle;
|
34
|
+
friend class BlockManager;
|
35
|
+
|
36
|
+
public:
|
37
|
+
StandardBufferManager(DatabaseInstance &db, string temp_directory);
|
38
|
+
virtual ~StandardBufferManager();
|
39
|
+
|
40
|
+
public:
|
41
|
+
static unique_ptr<StandardBufferManager> CreateBufferManager(DatabaseInstance &db, string temp_directory);
|
42
|
+
//! Registers an in-memory buffer that cannot be unloaded until it is destroyed
|
43
|
+
//! This buffer can be small (smaller than BLOCK_SIZE)
|
44
|
+
//! Unpin and pin are nops on this block of memory
|
45
|
+
shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size) final override;
|
46
|
+
|
47
|
+
idx_t GetUsedMemory() const final override;
|
48
|
+
void IncreaseUsedMemory(idx_t amount, bool unsafe = false) final override;
|
49
|
+
void DecreaseUsedMemory(idx_t amount) final override;
|
50
|
+
idx_t GetMaxMemory() const final override;
|
51
|
+
|
52
|
+
//! Allocate an in-memory buffer with a single pin.
|
53
|
+
//! The allocated memory is released when the buffer handle is destroyed.
|
54
|
+
DUCKDB_API BufferHandle Allocate(idx_t block_size, bool can_destroy = true,
|
55
|
+
shared_ptr<BlockHandle> *block = nullptr) final override;
|
56
|
+
|
57
|
+
//! Reallocate an in-memory buffer that is pinned.
|
58
|
+
void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) final override;
|
59
|
+
|
60
|
+
BufferHandle Pin(shared_ptr<BlockHandle> &handle) final override;
|
61
|
+
void Unpin(shared_ptr<BlockHandle> &handle) final override;
|
62
|
+
|
63
|
+
//! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
|
64
|
+
//! blocks can be evicted
|
65
|
+
void SetLimit(idx_t limit = (idx_t)-1) final override;
|
66
|
+
|
67
|
+
//! Returns a list of all temporary files
|
68
|
+
vector<TemporaryFileInformation> GetTemporaryFiles() final override;
|
69
|
+
|
70
|
+
const string &GetTemporaryDirectory() final override {
|
71
|
+
return temp_directory;
|
72
|
+
}
|
73
|
+
|
74
|
+
void SetTemporaryDirectory(const string &new_dir) final override;
|
75
|
+
|
76
|
+
DUCKDB_API Allocator &GetBufferAllocator() final override;
|
77
|
+
|
78
|
+
DatabaseInstance &GetDatabase() final override {
|
79
|
+
return db;
|
80
|
+
}
|
81
|
+
|
82
|
+
//! Construct a managed buffer.
|
83
|
+
unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
|
84
|
+
FileBufferType type = FileBufferType::MANAGED_BUFFER) override;
|
85
|
+
|
86
|
+
DUCKDB_API void ReserveMemory(idx_t size) final override;
|
87
|
+
DUCKDB_API void FreeReservedMemory(idx_t size) final override;
|
88
|
+
bool HasTemporaryDirectory() const final override;
|
89
|
+
|
90
|
+
protected:
|
91
|
+
//! Helper
|
92
|
+
template <typename... ARGS>
|
93
|
+
TempBufferPoolReservation EvictBlocksOrThrow(idx_t memory_delta, unique_ptr<FileBuffer> *buffer, ARGS...);
|
94
|
+
|
95
|
+
//! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
|
96
|
+
//! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
|
97
|
+
//! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
|
98
|
+
//! This needs to be private to prevent creating blocks without ever pinning them:
|
99
|
+
//! blocks that are never pinned are never added to the eviction queue
|
100
|
+
shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
|
101
|
+
|
102
|
+
//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
|
103
|
+
//! (i.e. not enough blocks could be evicted)
|
104
|
+
//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
|
105
|
+
//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
|
106
|
+
//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
|
107
|
+
//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
|
108
|
+
BufferPool::EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
|
109
|
+
unique_ptr<FileBuffer> *buffer = nullptr);
|
110
|
+
|
111
|
+
//! Garbage collect eviction queue
|
112
|
+
void PurgeQueue() final override;
|
113
|
+
|
114
|
+
BufferPool &GetBufferPool() final override;
|
115
|
+
|
116
|
+
//! Write a temporary buffer to disk
|
117
|
+
void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer) final override;
|
118
|
+
//! Read a temporary buffer from disk
|
119
|
+
unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> buffer = nullptr) final override;
|
120
|
+
//! Get the path of the temporary buffer
|
121
|
+
string GetTemporaryPath(block_id_t id);
|
122
|
+
|
123
|
+
void DeleteTemporaryFile(block_id_t id) final override;
|
124
|
+
|
125
|
+
void RequireTemporaryDirectory();
|
126
|
+
|
127
|
+
void AddToEvictionQueue(shared_ptr<BlockHandle> &handle) final override;
|
128
|
+
|
129
|
+
const char *InMemoryWarning();
|
130
|
+
|
131
|
+
static data_ptr_t BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size);
|
132
|
+
static void BufferAllocatorFree(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t size);
|
133
|
+
static data_ptr_t BufferAllocatorRealloc(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t old_size,
|
134
|
+
idx_t size);
|
135
|
+
|
136
|
+
//! When the BlockHandle reaches 0 readers, this creates a new FileBuffer for this BlockHandle and
|
137
|
+
//! overwrites the data within with garbage. Any readers that do not hold the pin will notice
|
138
|
+
void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
|
139
|
+
|
140
|
+
protected:
|
141
|
+
//! The database instance
|
142
|
+
DatabaseInstance &db;
|
143
|
+
//! The buffer pool
|
144
|
+
BufferPool &buffer_pool;
|
145
|
+
//! The directory name where temporary files are stored
|
146
|
+
string temp_directory;
|
147
|
+
//! Lock for creating the temp handle
|
148
|
+
mutex temp_handle_lock;
|
149
|
+
//! Handle for the temporary directory
|
150
|
+
unique_ptr<TemporaryDirectoryHandle> temp_directory_handle;
|
151
|
+
//! The temporary id used for managed buffers
|
152
|
+
atomic<block_id_t> temporary_id;
|
153
|
+
//! Allocator associated with the buffer manager, that passes all allocations through this buffer manager
|
154
|
+
Allocator buffer_allocator;
|
155
|
+
//! Block manager for temp data
|
156
|
+
unique_ptr<BlockManager> temp_block_manager;
|
157
|
+
};
|
158
|
+
|
159
|
+
} // namespace duckdb
|
@@ -13,6 +13,7 @@
|
|
13
13
|
#include "duckdb/main/valid_checker.hpp"
|
14
14
|
#include "duckdb/common/types/timestamp.hpp"
|
15
15
|
#include "duckdb/common/unordered_map.hpp"
|
16
|
+
#include "duckdb/common/optional_ptr.hpp"
|
16
17
|
|
17
18
|
namespace duckdb {
|
18
19
|
class AttachedDatabase;
|
@@ -42,7 +43,7 @@ public:
|
|
42
43
|
return start_timestamp;
|
43
44
|
}
|
44
45
|
|
45
|
-
Transaction &GetTransaction(AttachedDatabase
|
46
|
+
Transaction &GetTransaction(AttachedDatabase &db);
|
46
47
|
|
47
48
|
string Commit();
|
48
49
|
void Rollback();
|
@@ -50,8 +51,8 @@ public:
|
|
50
51
|
idx_t GetActiveQuery();
|
51
52
|
void SetActiveQuery(transaction_t query_number);
|
52
53
|
|
53
|
-
void ModifyDatabase(AttachedDatabase
|
54
|
-
AttachedDatabase
|
54
|
+
void ModifyDatabase(AttachedDatabase &db);
|
55
|
+
optional_ptr<AttachedDatabase> ModifiedDatabase() {
|
55
56
|
return modified_database;
|
56
57
|
}
|
57
58
|
|
@@ -59,9 +60,9 @@ private:
|
|
59
60
|
//! The set of active transactions for each database
|
60
61
|
unordered_map<AttachedDatabase *, Transaction *> transactions;
|
61
62
|
//! The set of transactions in order of when they were started
|
62
|
-
vector<AttachedDatabase
|
63
|
+
vector<optional_ptr<AttachedDatabase>> all_transactions;
|
63
64
|
//! The database we are modifying - we can only modify one database per transaction
|
64
|
-
AttachedDatabase
|
65
|
+
optional_ptr<AttachedDatabase> modified_database;
|
65
66
|
};
|
66
67
|
|
67
68
|
} // namespace duckdb
|
@@ -390,7 +390,7 @@ unique_ptr<PendingQueryResult> ClientContext::PendingPreparedStatement(ClientCon
|
|
390
390
|
"Cannot execute statement of type \"%s\" on database \"%s\" which is attached in read-only mode!",
|
391
391
|
StatementTypeToString(statement.statement_type), modified_database));
|
392
392
|
}
|
393
|
-
transaction.ModifyDatabase(entry);
|
393
|
+
transaction.ModifyDatabase(*entry);
|
394
394
|
}
|
395
395
|
|
396
396
|
// bind the bound values before execution
|
@@ -18,6 +18,7 @@
|
|
18
18
|
#include "duckdb/storage/magic_bytes.hpp"
|
19
19
|
#include "duckdb/storage/storage_extension.hpp"
|
20
20
|
#include "duckdb/execution/operator/helper/physical_set.hpp"
|
21
|
+
#include "duckdb/storage/standard_buffer_manager.hpp"
|
21
22
|
|
22
23
|
#ifndef DUCKDB_NO_THREADS
|
23
24
|
#include "duckdb/common/thread.hpp"
|
@@ -200,7 +201,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
|
|
200
201
|
}
|
201
202
|
|
202
203
|
db_manager = make_uniq<DatabaseManager>(*this);
|
203
|
-
buffer_manager = make_uniq<
|
204
|
+
buffer_manager = make_uniq<StandardBufferManager>(*this, config.options.temporary_directory);
|
204
205
|
scheduler = make_uniq<TaskScheduler>(*this);
|
205
206
|
object_cache = make_uniq<ObjectCache>();
|
206
207
|
connection_manager = make_uniq<ConnectionManager>();
|
@@ -23,7 +23,7 @@ void DatabaseManager::InitializeSystemCatalog() {
|
|
23
23
|
system->Initialize();
|
24
24
|
}
|
25
25
|
|
26
|
-
AttachedDatabase
|
26
|
+
optional_ptr<AttachedDatabase> DatabaseManager::GetDatabase(ClientContext &context, const string &name) {
|
27
27
|
if (StringUtil::Lower(name) == TEMP_CATALOG) {
|
28
28
|
return context.client_data->temporary_objects.get();
|
29
29
|
}
|
@@ -50,7 +50,7 @@ void DatabaseManager::DetachDatabase(ClientContext &context, const string &name,
|
|
50
50
|
}
|
51
51
|
}
|
52
52
|
|
53
|
-
AttachedDatabase
|
53
|
+
optional_ptr<AttachedDatabase> DatabaseManager::GetDatabaseFromPath(ClientContext &context, const string &path) {
|
54
54
|
auto databases = GetDatabases(context);
|
55
55
|
for (auto db : databases) {
|
56
56
|
if (db->IsSystem()) {
|
@@ -81,8 +81,8 @@ const string &DatabaseManager::GetDefaultDatabase(ClientContext &context) {
|
|
81
81
|
return default_entry.catalog;
|
82
82
|
}
|
83
83
|
|
84
|
-
vector<AttachedDatabase
|
85
|
-
vector<AttachedDatabase
|
84
|
+
vector<optional_ptr<AttachedDatabase>> DatabaseManager::GetDatabases(ClientContext &context) {
|
85
|
+
vector<optional_ptr<AttachedDatabase>> result;
|
86
86
|
databases->Scan(context, [&](CatalogEntry *entry) { result.push_back((AttachedDatabase *)entry); });
|
87
87
|
result.push_back(system.get());
|
88
88
|
result.push_back(context.client_data->temporary_objects.get());
|
@@ -33,10 +33,10 @@ void CommonAggregateOptimizer::ExtractCommonAggregates(LogicalAggregate &aggr) {
|
|
33
33
|
idx_t total_erased = 0;
|
34
34
|
for (idx_t i = 0; i < aggr.expressions.size(); i++) {
|
35
35
|
idx_t original_index = i + total_erased;
|
36
|
-
auto entry = aggregate_remap.find(aggr.expressions[i]
|
36
|
+
auto entry = aggregate_remap.find(*aggr.expressions[i]);
|
37
37
|
if (entry == aggregate_remap.end()) {
|
38
38
|
// aggregate does not exist yet: add it to the map
|
39
|
-
aggregate_remap[aggr.expressions[i]
|
39
|
+
aggregate_remap[*aggr.expressions[i]] = i;
|
40
40
|
if (i != original_index) {
|
41
41
|
// this aggregate is not erased, however an agregate BEFORE it has been erased
|
42
42
|
// so we need to remap this aggregaet
|
@@ -60,10 +60,10 @@ void CommonSubExpressionOptimizer::CountExpressions(Expression &expr, CSEReplace
|
|
60
60
|
}
|
61
61
|
if (expr.expression_class != ExpressionClass::BOUND_AGGREGATE && !expr.HasSideEffects()) {
|
62
62
|
// we can't move aggregates to a projection, so we only consider the children of the aggregate
|
63
|
-
auto node = state.expression_count.find(
|
63
|
+
auto node = state.expression_count.find(expr);
|
64
64
|
if (node == state.expression_count.end()) {
|
65
65
|
// first time we encounter this expression, insert this node with [count = 1]
|
66
|
-
state.expression_count[
|
66
|
+
state.expression_count[expr] = CSENode();
|
67
67
|
} else {
|
68
68
|
// we encountered this expression before, increment the occurrence count
|
69
69
|
node->second.count++;
|
@@ -95,8 +95,8 @@ void CommonSubExpressionOptimizer::PerformCSEReplacement(unique_ptr<Expression>
|
|
95
95
|
// check if this child is eligible for CSE elimination
|
96
96
|
bool can_cse = expr.expression_class != ExpressionClass::BOUND_CONJUNCTION &&
|
97
97
|
expr.expression_class != ExpressionClass::BOUND_CASE;
|
98
|
-
if (can_cse && state.expression_count.find(
|
99
|
-
auto &node = state.expression_count[
|
98
|
+
if (can_cse && state.expression_count.find(expr) != state.expression_count.end()) {
|
99
|
+
auto &node = state.expression_count[expr];
|
100
100
|
if (node.count > 1) {
|
101
101
|
// this expression occurs more than once! push it into the projection
|
102
102
|
// check if it has already been pushed into the projection
|
@@ -106,8 +106,10 @@ void DeliminatorPlanUpdater::VisitOperator(LogicalOperator &op) {
|
|
106
106
|
}
|
107
107
|
|
108
108
|
void DeliminatorPlanUpdater::VisitExpression(unique_ptr<Expression> *expression) {
|
109
|
-
|
110
|
-
|
109
|
+
auto &expr = **expression;
|
110
|
+
auto entry = expr_map.find(expr);
|
111
|
+
if (entry != expr_map.end()) {
|
112
|
+
*expression = entry->second->Copy();
|
111
113
|
} else {
|
112
114
|
VisitExpressionChildren(**expression);
|
113
115
|
}
|
@@ -207,17 +209,17 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
|
|
207
209
|
}
|
208
210
|
// check if joining with the DelimGet is redundant, and collect relevant column information
|
209
211
|
bool all_equality_conditions = true;
|
210
|
-
vector<Expression
|
212
|
+
vector<reference<Expression>> nulls_are_not_equal_exprs;
|
211
213
|
for (auto &cond : join.conditions) {
|
212
214
|
all_equality_conditions = all_equality_conditions && IsEqualityJoinCondition(cond);
|
213
|
-
auto delim_side = delim_idx == 0 ? cond.left
|
214
|
-
auto other_side = delim_idx == 0 ? cond.right
|
215
|
-
if (delim_side
|
215
|
+
auto &delim_side = delim_idx == 0 ? *cond.left : *cond.right;
|
216
|
+
auto &other_side = delim_idx == 0 ? *cond.right : *cond.left;
|
217
|
+
if (delim_side.type != ExpressionType::BOUND_COLUMN_REF) {
|
216
218
|
// non-colref e.g. expression -(4, 1) in 4-i=j where i is from DelimGet
|
217
219
|
// FIXME: might be possible to also eliminate these
|
218
220
|
return false;
|
219
221
|
}
|
220
|
-
updater.expr_map[delim_side] = other_side;
|
222
|
+
updater.expr_map[delim_side] = &other_side;
|
221
223
|
if (cond.comparison != ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
|
222
224
|
nulls_are_not_equal_exprs.push_back(other_side);
|
223
225
|
}
|
@@ -229,7 +231,7 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
|
|
229
231
|
updater.projection_map[cb] = true;
|
230
232
|
updater.reverse_proj_or_agg_map[cb] = proj_or_agg.expressions[cb.column_index].get();
|
231
233
|
for (auto &expr : nulls_are_not_equal_exprs) {
|
232
|
-
if (proj_or_agg.expressions[cb.column_index]->Equals(expr)) {
|
234
|
+
if (proj_or_agg.expressions[cb.column_index]->Equals(&expr.get())) {
|
233
235
|
updater.projection_map[cb] = false;
|
234
236
|
break;
|
235
237
|
}
|
@@ -252,8 +254,8 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
|
|
252
254
|
updater.projection_map[cb] = true;
|
253
255
|
updater.reverse_proj_or_agg_map[cb] = all_agg_exprs[cb.column_index];
|
254
256
|
for (auto &expr : nulls_are_not_equal_exprs) {
|
255
|
-
if ((cb.table_index == agg.group_index && agg.groups[cb.column_index]->Equals(expr)) ||
|
256
|
-
(cb.table_index == agg.aggregate_index && agg.expressions[cb.column_index]->Equals(expr))) {
|
257
|
+
if ((cb.table_index == agg.group_index && agg.groups[cb.column_index]->Equals(&expr.get())) ||
|
258
|
+
(cb.table_index == agg.aggregate_index && agg.expressions[cb.column_index]->Equals(&expr.get()))) {
|
257
259
|
updater.projection_map[cb] = false;
|
258
260
|
break;
|
259
261
|
}
|
@@ -276,7 +278,7 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
|
|
276
278
|
for (auto &expr : nulls_are_not_equal_exprs) {
|
277
279
|
auto is_not_null_expr =
|
278
280
|
make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
|
279
|
-
is_not_null_expr->children.push_back(expr
|
281
|
+
is_not_null_expr->children.push_back(expr.get().Copy());
|
280
282
|
filter_op->expressions.push_back(std::move(is_not_null_expr));
|
281
283
|
}
|
282
284
|
}
|
@@ -12,8 +12,8 @@ namespace duckdb {
|
|
12
12
|
unique_ptr<Expression> ExpressionRewriter::ApplyRules(LogicalOperator &op, const vector<Rule *> &rules,
|
13
13
|
unique_ptr<Expression> expr, bool &changes_made, bool is_root) {
|
14
14
|
for (auto &rule : rules) {
|
15
|
-
vector<Expression
|
16
|
-
if (rule->root->Match(expr
|
15
|
+
vector<reference<Expression>> bindings;
|
16
|
+
if (rule->root->Match(*expr, bindings)) {
|
17
17
|
// the rule matches! try to apply it
|
18
18
|
bool rule_made_change = false;
|
19
19
|
auto result = rule->Apply(op, bindings, rule_made_change, is_root);
|