duckdb 0.7.2-dev1734.0 → 0.7.2-dev1867.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/src/catalog/catalog.cpp +27 -27
  3. package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp +6 -6
  4. package/src/duckdb/src/catalog/catalog_set.cpp +27 -25
  5. package/src/duckdb/src/catalog/default/default_functions.cpp +4 -4
  6. package/src/duckdb/src/catalog/default/default_types.cpp +4 -4
  7. package/src/duckdb/src/catalog/default/default_views.cpp +4 -4
  8. package/src/duckdb/src/catalog/dependency_list.cpp +7 -6
  9. package/src/duckdb/src/catalog/dependency_manager.cpp +44 -38
  10. package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +11 -6
  11. package/src/duckdb/src/common/types/batched_data_collection.cpp +2 -1
  12. package/src/duckdb/src/common/types/column_data_allocator.cpp +1 -0
  13. package/src/duckdb/src/common/types/vector.cpp +2 -2
  14. package/src/duckdb/src/common/types.cpp +2 -2
  15. package/src/duckdb/src/common/vector_operations/vector_copy.cpp +14 -11
  16. package/src/duckdb/src/execution/operator/aggregate/distinct_aggregate_data.cpp +1 -1
  17. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +51 -50
  18. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +14 -13
  19. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +20 -20
  20. package/src/duckdb/src/execution/operator/schema/physical_create_table.cpp +2 -2
  21. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +1 -1
  22. package/src/duckdb/src/execution/physical_plan/plan_create_table.cpp +3 -3
  23. package/src/duckdb/src/execution/physical_plan/plan_delete.cpp +1 -1
  24. package/src/duckdb/src/execution/physical_plan/plan_insert.cpp +1 -1
  25. package/src/duckdb/src/execution/physical_plan/plan_update.cpp +1 -1
  26. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +3 -3
  27. package/src/duckdb/src/function/cast/cast_function_set.cpp +2 -1
  28. package/src/duckdb/src/function/scalar/math/numeric.cpp +57 -0
  29. package/src/duckdb/src/function/scalar/math_functions.cpp +1 -0
  30. package/src/duckdb/src/function/scalar/sequence/nextval.cpp +29 -29
  31. package/src/duckdb/src/function/scalar/string/damerau_levenshtein.cpp +106 -0
  32. package/src/duckdb/src/function/scalar/string/hex.cpp +261 -78
  33. package/src/duckdb/src/function/scalar/string/regexp.cpp +145 -28
  34. package/src/duckdb/src/function/scalar/string_functions.cpp +1 -0
  35. package/src/duckdb/src/function/table/checkpoint.cpp +4 -4
  36. package/src/duckdb/src/function/table/system/duckdb_columns.cpp +24 -24
  37. package/src/duckdb/src/function/table/system/duckdb_constraints.cpp +7 -6
  38. package/src/duckdb/src/function/table/system/duckdb_databases.cpp +1 -1
  39. package/src/duckdb/src/function/table/system/duckdb_dependencies.cpp +11 -11
  40. package/src/duckdb/src/function/table/system/pragma_database_size.cpp +1 -1
  41. package/src/duckdb/src/function/table/system/pragma_table_info.cpp +17 -18
  42. package/src/duckdb/src/function/table/table_scan.cpp +8 -11
  43. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  44. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +9 -9
  45. package/src/duckdb/src/include/duckdb/catalog/catalog_entry_map.hpp +38 -0
  46. package/src/duckdb/src/include/duckdb/catalog/catalog_transaction.hpp +4 -3
  47. package/src/duckdb/src/include/duckdb/catalog/default/default_functions.hpp +2 -2
  48. package/src/duckdb/src/include/duckdb/catalog/default/default_types.hpp +2 -2
  49. package/src/duckdb/src/include/duckdb/catalog/default/default_views.hpp +2 -2
  50. package/src/duckdb/src/include/duckdb/catalog/dependency.hpp +4 -5
  51. package/src/duckdb/src/include/duckdb/catalog/dependency_list.hpp +4 -5
  52. package/src/duckdb/src/include/duckdb/catalog/dependency_manager.hpp +10 -9
  53. package/src/duckdb/src/include/duckdb/common/allocator.hpp +2 -1
  54. package/src/duckdb/src/include/duckdb/common/bit_utils.hpp +147 -0
  55. package/src/duckdb/src/include/duckdb/common/field_writer.hpp +1 -1
  56. package/src/duckdb/src/include/duckdb/common/helper.hpp +9 -0
  57. package/src/duckdb/src/include/duckdb/common/hugeint.hpp +1 -0
  58. package/src/duckdb/src/include/duckdb/common/optional_ptr.hpp +29 -6
  59. package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +6 -5
  60. package/src/duckdb/src/include/duckdb/common/serializer.hpp +1 -1
  61. package/src/duckdb/src/include/duckdb/common/string_util.hpp +7 -0
  62. package/src/duckdb/src/include/duckdb/common/types/row_data_collection.hpp +1 -0
  63. package/src/duckdb/src/include/duckdb/common/types.hpp +1 -1
  64. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
  65. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +5 -5
  66. package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_table.hpp +2 -2
  67. package/src/duckdb/src/include/duckdb/function/cast/default_casts.hpp +3 -2
  68. package/src/duckdb/src/include/duckdb/function/scalar/math_functions.hpp +4 -0
  69. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +4 -0
  70. package/src/duckdb/src/include/duckdb/main/database_manager.hpp +4 -3
  71. package/src/duckdb/src/include/duckdb/main/query_result.hpp +3 -2
  72. package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp +7 -7
  73. package/src/duckdb/src/include/duckdb/optimizer/matcher/expression_matcher.hpp +11 -11
  74. package/src/duckdb/src/include/duckdb/optimizer/matcher/set_matcher.hpp +8 -8
  75. package/src/duckdb/src/include/duckdb/optimizer/rule/arithmetic_simplification.hpp +1 -1
  76. package/src/duckdb/src/include/duckdb/optimizer/rule/case_simplification.hpp +1 -1
  77. package/src/duckdb/src/include/duckdb/optimizer/rule/comparison_simplification.hpp +1 -1
  78. package/src/duckdb/src/include/duckdb/optimizer/rule/conjunction_simplification.hpp +2 -2
  79. package/src/duckdb/src/include/duckdb/optimizer/rule/constant_folding.hpp +1 -1
  80. package/src/duckdb/src/include/duckdb/optimizer/rule/date_part_simplification.hpp +1 -1
  81. package/src/duckdb/src/include/duckdb/optimizer/rule/distributivity.hpp +1 -1
  82. package/src/duckdb/src/include/duckdb/optimizer/rule/empty_needle_removal.hpp +1 -1
  83. package/src/duckdb/src/include/duckdb/optimizer/rule/enum_comparison.hpp +1 -1
  84. package/src/duckdb/src/include/duckdb/optimizer/rule/equal_or_null_simplification.hpp +1 -1
  85. package/src/duckdb/src/include/duckdb/optimizer/rule/in_clause_simplification.hpp +1 -1
  86. package/src/duckdb/src/include/duckdb/optimizer/rule/like_optimizations.hpp +1 -1
  87. package/src/duckdb/src/include/duckdb/optimizer/rule/move_constants.hpp +1 -1
  88. package/src/duckdb/src/include/duckdb/optimizer/rule/ordered_aggregate_optimizer.hpp +1 -1
  89. package/src/duckdb/src/include/duckdb/optimizer/rule/regex_optimizations.hpp +1 -1
  90. package/src/duckdb/src/include/duckdb/optimizer/rule.hpp +2 -2
  91. package/src/duckdb/src/include/duckdb/parser/base_expression.hpp +1 -1
  92. package/src/duckdb/src/include/duckdb/parser/expression_map.hpp +19 -6
  93. package/src/duckdb/src/include/duckdb/parser/expression_util.hpp +1 -1
  94. package/src/duckdb/src/include/duckdb/planner/expression.hpp +5 -2
  95. package/src/duckdb/src/include/duckdb/planner/expression_binder/base_select_binder.hpp +1 -1
  96. package/src/duckdb/src/include/duckdb/planner/expression_binder/order_binder.hpp +3 -3
  97. package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp +15 -7
  98. package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp +3 -0
  99. package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp +49 -126
  100. package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp128.hpp +1 -0
  101. package/src/duckdb/src/include/duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp +0 -97
  102. package/src/duckdb/src/include/duckdb/storage/compression/patas/algorithm/patas.hpp +1 -0
  103. package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +5 -5
  104. package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp +159 -0
  105. package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +1 -0
  106. package/src/duckdb/src/include/duckdb/transaction/meta_transaction.hpp +6 -5
  107. package/src/duckdb/src/main/client_context.cpp +1 -1
  108. package/src/duckdb/src/main/database.cpp +2 -1
  109. package/src/duckdb/src/main/database_manager.cpp +4 -4
  110. package/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp +2 -2
  111. package/src/duckdb/src/optimizer/cse_optimizer.cpp +4 -4
  112. package/src/duckdb/src/optimizer/deliminator.cpp +13 -11
  113. package/src/duckdb/src/optimizer/expression_rewriter.cpp +2 -2
  114. package/src/duckdb/src/optimizer/filter_combiner.cpp +67 -65
  115. package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +1 -0
  116. package/src/duckdb/src/optimizer/join_order/join_order_optimizer.cpp +26 -25
  117. package/src/duckdb/src/optimizer/matcher/expression_matcher.cpp +23 -21
  118. package/src/duckdb/src/optimizer/rule/arithmetic_simplification.cpp +3 -3
  119. package/src/duckdb/src/optimizer/rule/case_simplification.cpp +2 -2
  120. package/src/duckdb/src/optimizer/rule/comparison_simplification.cpp +6 -7
  121. package/src/duckdb/src/optimizer/rule/conjunction_simplification.cpp +9 -8
  122. package/src/duckdb/src/optimizer/rule/constant_folding.cpp +7 -7
  123. package/src/duckdb/src/optimizer/rule/date_part_simplification.cpp +3 -3
  124. package/src/duckdb/src/optimizer/rule/distributivity.cpp +5 -5
  125. package/src/duckdb/src/optimizer/rule/empty_needle_removal.cpp +6 -6
  126. package/src/duckdb/src/optimizer/rule/enum_comparison.cpp +4 -4
  127. package/src/duckdb/src/optimizer/rule/equal_or_null_simplification.cpp +23 -26
  128. package/src/duckdb/src/optimizer/rule/in_clause_simplification_rule.cpp +2 -3
  129. package/src/duckdb/src/optimizer/rule/like_optimizations.cpp +3 -3
  130. package/src/duckdb/src/optimizer/rule/move_constants.cpp +6 -6
  131. package/src/duckdb/src/optimizer/rule/ordered_aggregate_optimizer.cpp +2 -2
  132. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +11 -10
  133. package/src/duckdb/src/parser/expression_util.cpp +6 -6
  134. package/src/duckdb/src/parser/transform/helpers/transform_groupby.cpp +3 -3
  135. package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp +2 -2
  136. package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +3 -3
  137. package/src/duckdb/src/planner/binder/query_node/bind_setop_node.cpp +5 -5
  138. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +2 -2
  139. package/src/duckdb/src/planner/expression_binder/base_select_binder.cpp +4 -4
  140. package/src/duckdb/src/planner/expression_binder/order_binder.cpp +3 -3
  141. package/src/duckdb/src/storage/buffer/block_handle.cpp +7 -6
  142. package/src/duckdb/src/storage/buffer/block_manager.cpp +3 -1
  143. package/src/duckdb/src/storage/buffer/buffer_handle.cpp +1 -0
  144. package/src/duckdb/src/storage/buffer/buffer_pool.cpp +6 -2
  145. package/src/duckdb/src/storage/buffer/buffer_pool_reservation.cpp +7 -4
  146. package/src/duckdb/src/storage/buffer_manager.cpp +35 -726
  147. package/src/duckdb/src/storage/checkpoint_manager.cpp +2 -2
  148. package/src/duckdb/src/storage/meta_block_reader.cpp +6 -5
  149. package/src/duckdb/src/storage/standard_buffer_manager.cpp +801 -0
  150. package/src/duckdb/src/storage/wal_replay.cpp +2 -2
  151. package/src/duckdb/src/transaction/meta_transaction.cpp +13 -13
  152. package/src/duckdb/src/transaction/transaction.cpp +1 -1
  153. package/src/duckdb/src/transaction/transaction_context.cpp +1 -1
  154. package/src/duckdb/ub_src_function_scalar_string.cpp +2 -0
  155. package/src/duckdb/ub_src_storage.cpp +2 -0
@@ -10,68 +10,6 @@
10
10
 
11
11
  #include "duckdb.h"
12
12
 
13
- #ifdef _MSC_VER
14
- #define __restrict__
15
- #define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
16
- #define __ORDER_LITTLE_ENDIAN__ 2
17
- #include <intrin.h>
18
- static inline int __builtin_ctzll(unsigned long long x) {
19
- #ifdef _WIN64
20
- unsigned long ret;
21
- _BitScanForward64(&ret, x);
22
- return (int)ret;
23
- #else
24
- unsigned long low, high;
25
- bool low_set = _BitScanForward(&low, (unsigned __int32)(x)) != 0;
26
- _BitScanForward(&high, (unsigned __int32)(x >> 32));
27
- high += 32;
28
- return low_set ? low : high;
29
- #endif
30
- }
31
- static inline int __builtin_clzll(unsigned long long mask) {
32
- unsigned long where;
33
- // BitScanReverse scans from MSB to LSB for first set bit.
34
- // Returns 0 if no set bit is found.
35
- #if defined(_WIN64)
36
- if (_BitScanReverse64(&where, mask))
37
- return static_cast<int>(63 - where);
38
- #elif defined(_WIN32)
39
- // Scan the high 32 bits.
40
- if (_BitScanReverse(&where, static_cast<unsigned long>(mask >> 32)))
41
- return static_cast<int>(63 - (where + 32)); // Create a bit offset from the MSB.
42
- // Scan the low 32 bits.
43
- if (_BitScanReverse(&where, static_cast<unsigned long>(mask)))
44
- return static_cast<int>(63 - where);
45
- #else
46
- #error "Implementation of __builtin_clzll required"
47
- #endif
48
- return 64; // Undefined Behavior.
49
- }
50
-
51
- static inline int __builtin_ctz(unsigned int value) {
52
- unsigned long trailing_zero = 0;
53
-
54
- if (_BitScanForward(&trailing_zero, value)) {
55
- return trailing_zero;
56
- } else {
57
- // This is undefined, I better choose 32 than 0
58
- return 32;
59
- }
60
- }
61
-
62
- static inline int __builtin_clz(unsigned int value) {
63
- unsigned long leading_zero = 0;
64
-
65
- if (_BitScanReverse(&leading_zero, value)) {
66
- return 31 - leading_zero;
67
- } else {
68
- // Same remarks as above
69
- return 32;
70
- }
71
- }
72
-
73
- #endif
74
-
75
13
  namespace duckdb {
76
14
 
77
15
  template <class T>
@@ -89,41 +27,6 @@ struct SignificantBits<uint32_t> {
89
27
  static constexpr uint8_t mask = ((uint8_t)1 << size) - 1;
90
28
  };
91
29
 
92
- template <class T>
93
- struct CountZeros {};
94
-
95
- template <>
96
- struct CountZeros<uint32_t> {
97
- inline static int Leading(uint32_t value) {
98
- if (!value) {
99
- return 32;
100
- }
101
- return __builtin_clz(value);
102
- }
103
- inline static int Trailing(uint32_t value) {
104
- if (!value) {
105
- return 32;
106
- }
107
- return __builtin_ctz(value);
108
- }
109
- };
110
-
111
- template <>
112
- struct CountZeros<uint64_t> {
113
- inline static int Leading(uint64_t value) {
114
- if (!value) {
115
- return 64;
116
- }
117
- return __builtin_clzll(value);
118
- }
119
- inline static int Trailing(uint64_t value) {
120
- if (!value) {
121
- return 64;
122
- }
123
- return __builtin_ctzll(value);
124
- }
125
- };
126
-
127
30
  struct ChimpConstants {
128
31
  struct Compression {
129
32
  static constexpr uint8_t LEADING_ROUND[] = {0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 12, 12, 12, 12,
@@ -14,6 +14,7 @@
14
14
  #include "duckdb/storage/compression/chimp/algorithm/chimp_utils.hpp"
15
15
  #include "duckdb/storage/compression/chimp/algorithm/packed_data.hpp"
16
16
  #include "duckdb/storage/compression/patas/shared.hpp"
17
+ #include "duckdb/common/bit_utils.hpp"
17
18
 
18
19
  namespace duckdb {
19
20
 
@@ -37,13 +37,13 @@ public:
37
37
  void ReadData(data_ptr_t buffer, idx_t read_size) override;
38
38
 
39
39
  ClientContext &GetContext() override;
40
- Catalog *GetCatalog() override;
41
- void SetCatalog(Catalog *catalog_p);
42
- void SetContext(ClientContext *context_p);
40
+ optional_ptr<Catalog> GetCatalog() override;
41
+ void SetCatalog(Catalog &catalog_p);
42
+ void SetContext(ClientContext &context_p);
43
43
 
44
44
  private:
45
45
  void ReadNewBlock(block_id_t id);
46
- Catalog *catalog = nullptr;
47
- ClientContext *context = nullptr;
46
+ optional_ptr<ClientContext> context;
47
+ optional_ptr<Catalog> catalog;
48
48
  };
49
49
  } // namespace duckdb
@@ -0,0 +1,159 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/storage/standard_buffer_manager.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/common/allocator.hpp"
12
+ #include "duckdb/common/atomic.hpp"
13
+ #include "duckdb/common/file_system.hpp"
14
+ #include "duckdb/common/mutex.hpp"
15
+ #include "duckdb/storage/block_manager.hpp"
16
+
17
+ #include "duckdb/storage/buffer/block_handle.hpp"
18
+ #include "duckdb/storage/buffer_manager.hpp"
19
+ #include "duckdb/storage/buffer/buffer_pool.hpp"
20
+
21
+ namespace duckdb {
22
+ class BlockManager;
23
+ class DatabaseInstance;
24
+ class TemporaryDirectoryHandle;
25
+ struct EvictionQueue;
26
+
27
+ //! The BufferManager is in charge of handling memory management for a single database. It cooperatively shares a
28
+ //! BufferPool with other BufferManagers, belonging to different databases. It hands out memory buffers that can
29
+ //! be used by the database internally, and offers configuration options specific to a database, which need not be
30
+ //! shared by the BufferPool, including whether to support swapping temp buffers to disk, and where to swap them to.
31
+ class StandardBufferManager : public BufferManager {
32
+ friend class BufferHandle;
33
+ friend class BlockHandle;
34
+ friend class BlockManager;
35
+
36
+ public:
37
+ StandardBufferManager(DatabaseInstance &db, string temp_directory);
38
+ virtual ~StandardBufferManager();
39
+
40
+ public:
41
+ static unique_ptr<StandardBufferManager> CreateBufferManager(DatabaseInstance &db, string temp_directory);
42
+ //! Registers an in-memory buffer that cannot be unloaded until it is destroyed
43
+ //! This buffer can be small (smaller than BLOCK_SIZE)
44
+ //! Unpin and pin are nops on this block of memory
45
+ shared_ptr<BlockHandle> RegisterSmallMemory(idx_t block_size) final override;
46
+
47
+ idx_t GetUsedMemory() const final override;
48
+ void IncreaseUsedMemory(idx_t amount, bool unsafe = false) final override;
49
+ void DecreaseUsedMemory(idx_t amount) final override;
50
+ idx_t GetMaxMemory() const final override;
51
+
52
+ //! Allocate an in-memory buffer with a single pin.
53
+ //! The allocated memory is released when the buffer handle is destroyed.
54
+ DUCKDB_API BufferHandle Allocate(idx_t block_size, bool can_destroy = true,
55
+ shared_ptr<BlockHandle> *block = nullptr) final override;
56
+
57
+ //! Reallocate an in-memory buffer that is pinned.
58
+ void ReAllocate(shared_ptr<BlockHandle> &handle, idx_t block_size) final override;
59
+
60
+ BufferHandle Pin(shared_ptr<BlockHandle> &handle) final override;
61
+ void Unpin(shared_ptr<BlockHandle> &handle) final override;
62
+
63
+ //! Set a new memory limit to the buffer manager, throws an exception if the new limit is too low and not enough
64
+ //! blocks can be evicted
65
+ void SetLimit(idx_t limit = (idx_t)-1) final override;
66
+
67
+ //! Returns a list of all temporary files
68
+ vector<TemporaryFileInformation> GetTemporaryFiles() final override;
69
+
70
+ const string &GetTemporaryDirectory() final override {
71
+ return temp_directory;
72
+ }
73
+
74
+ void SetTemporaryDirectory(const string &new_dir) final override;
75
+
76
+ DUCKDB_API Allocator &GetBufferAllocator() final override;
77
+
78
+ DatabaseInstance &GetDatabase() final override {
79
+ return db;
80
+ }
81
+
82
+ //! Construct a managed buffer.
83
+ unique_ptr<FileBuffer> ConstructManagedBuffer(idx_t size, unique_ptr<FileBuffer> &&source,
84
+ FileBufferType type = FileBufferType::MANAGED_BUFFER) override;
85
+
86
+ DUCKDB_API void ReserveMemory(idx_t size) final override;
87
+ DUCKDB_API void FreeReservedMemory(idx_t size) final override;
88
+ bool HasTemporaryDirectory() const final override;
89
+
90
+ protected:
91
+ //! Helper
92
+ template <typename... ARGS>
93
+ TempBufferPoolReservation EvictBlocksOrThrow(idx_t memory_delta, unique_ptr<FileBuffer> *buffer, ARGS...);
94
+
95
+ //! Register an in-memory buffer of arbitrary size, as long as it is >= BLOCK_SIZE. can_destroy signifies whether or
96
+ //! not the buffer can be destroyed when unpinned, or whether or not it needs to be written to a temporary file so
97
+ //! it can be reloaded. The resulting buffer will already be allocated, but needs to be pinned in order to be used.
98
+ //! This needs to be private to prevent creating blocks without ever pinning them:
99
+ //! blocks that are never pinned are never added to the eviction queue
100
+ shared_ptr<BlockHandle> RegisterMemory(idx_t block_size, bool can_destroy);
101
+
102
+ //! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
103
+ //! (i.e. not enough blocks could be evicted)
104
+ //! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
105
+ //! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
106
+ //! Returns a pair. result.first indicates if eviction was successful. result.second contains the
107
+ //! reservation handle, which can be moved to the BlockHandle that will own the reservation.
108
+ BufferPool::EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
109
+ unique_ptr<FileBuffer> *buffer = nullptr);
110
+
111
+ //! Garbage collect eviction queue
112
+ void PurgeQueue() final override;
113
+
114
+ BufferPool &GetBufferPool() final override;
115
+
116
+ //! Write a temporary buffer to disk
117
+ void WriteTemporaryBuffer(block_id_t block_id, FileBuffer &buffer) final override;
118
+ //! Read a temporary buffer from disk
119
+ unique_ptr<FileBuffer> ReadTemporaryBuffer(block_id_t id, unique_ptr<FileBuffer> buffer = nullptr) final override;
120
+ //! Get the path of the temporary buffer
121
+ string GetTemporaryPath(block_id_t id);
122
+
123
+ void DeleteTemporaryFile(block_id_t id) final override;
124
+
125
+ void RequireTemporaryDirectory();
126
+
127
+ void AddToEvictionQueue(shared_ptr<BlockHandle> &handle) final override;
128
+
129
+ const char *InMemoryWarning();
130
+
131
+ static data_ptr_t BufferAllocatorAllocate(PrivateAllocatorData *private_data, idx_t size);
132
+ static void BufferAllocatorFree(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t size);
133
+ static data_ptr_t BufferAllocatorRealloc(PrivateAllocatorData *private_data, data_ptr_t pointer, idx_t old_size,
134
+ idx_t size);
135
+
136
+ //! When the BlockHandle reaches 0 readers, this creates a new FileBuffer for this BlockHandle and
137
+ //! overwrites the data within with garbage. Any readers that do not hold the pin will notice
138
+ void VerifyZeroReaders(shared_ptr<BlockHandle> &handle);
139
+
140
+ protected:
141
+ //! The database instance
142
+ DatabaseInstance &db;
143
+ //! The buffer pool
144
+ BufferPool &buffer_pool;
145
+ //! The directory name where temporary files are stored
146
+ string temp_directory;
147
+ //! Lock for creating the temp handle
148
+ mutex temp_handle_lock;
149
+ //! Handle for the temporary directory
150
+ unique_ptr<TemporaryDirectoryHandle> temp_directory_handle;
151
+ //! The temporary id used for managed buffers
152
+ atomic<block_id_t> temporary_id;
153
+ //! Allocator associated with the buffer manager, that passes all allocations through this buffer manager
154
+ Allocator buffer_allocator;
155
+ //! Block manager for temp data
156
+ unique_ptr<BlockManager> temp_block_manager;
157
+ };
158
+
159
+ } // namespace duckdb
@@ -16,6 +16,7 @@
16
16
  #include "duckdb/storage/storage_lock.hpp"
17
17
  #include "duckdb/function/compression_function.hpp"
18
18
  #include "duckdb/storage/table/segment_base.hpp"
19
+ #include "duckdb/storage/buffer/block_handle.hpp"
19
20
 
20
21
  namespace duckdb {
21
22
  class ColumnSegment;
@@ -13,6 +13,7 @@
13
13
  #include "duckdb/main/valid_checker.hpp"
14
14
  #include "duckdb/common/types/timestamp.hpp"
15
15
  #include "duckdb/common/unordered_map.hpp"
16
+ #include "duckdb/common/optional_ptr.hpp"
16
17
 
17
18
  namespace duckdb {
18
19
  class AttachedDatabase;
@@ -42,7 +43,7 @@ public:
42
43
  return start_timestamp;
43
44
  }
44
45
 
45
- Transaction &GetTransaction(AttachedDatabase *db);
46
+ Transaction &GetTransaction(AttachedDatabase &db);
46
47
 
47
48
  string Commit();
48
49
  void Rollback();
@@ -50,8 +51,8 @@ public:
50
51
  idx_t GetActiveQuery();
51
52
  void SetActiveQuery(transaction_t query_number);
52
53
 
53
- void ModifyDatabase(AttachedDatabase *db);
54
- AttachedDatabase *ModifiedDatabase() {
54
+ void ModifyDatabase(AttachedDatabase &db);
55
+ optional_ptr<AttachedDatabase> ModifiedDatabase() {
55
56
  return modified_database;
56
57
  }
57
58
 
@@ -59,9 +60,9 @@ private:
59
60
  //! The set of active transactions for each database
60
61
  unordered_map<AttachedDatabase *, Transaction *> transactions;
61
62
  //! The set of transactions in order of when they were started
62
- vector<AttachedDatabase *> all_transactions;
63
+ vector<optional_ptr<AttachedDatabase>> all_transactions;
63
64
  //! The database we are modifying - we can only modify one database per transaction
64
- AttachedDatabase *modified_database;
65
+ optional_ptr<AttachedDatabase> modified_database;
65
66
  };
66
67
 
67
68
  } // namespace duckdb
@@ -390,7 +390,7 @@ unique_ptr<PendingQueryResult> ClientContext::PendingPreparedStatement(ClientCon
390
390
  "Cannot execute statement of type \"%s\" on database \"%s\" which is attached in read-only mode!",
391
391
  StatementTypeToString(statement.statement_type), modified_database));
392
392
  }
393
- transaction.ModifyDatabase(entry);
393
+ transaction.ModifyDatabase(*entry);
394
394
  }
395
395
 
396
396
  // bind the bound values before execution
@@ -18,6 +18,7 @@
18
18
  #include "duckdb/storage/magic_bytes.hpp"
19
19
  #include "duckdb/storage/storage_extension.hpp"
20
20
  #include "duckdb/execution/operator/helper/physical_set.hpp"
21
+ #include "duckdb/storage/standard_buffer_manager.hpp"
21
22
 
22
23
  #ifndef DUCKDB_NO_THREADS
23
24
  #include "duckdb/common/thread.hpp"
@@ -200,7 +201,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
200
201
  }
201
202
 
202
203
  db_manager = make_uniq<DatabaseManager>(*this);
203
- buffer_manager = make_uniq<BufferManager>(*this, config.options.temporary_directory);
204
+ buffer_manager = make_uniq<StandardBufferManager>(*this, config.options.temporary_directory);
204
205
  scheduler = make_uniq<TaskScheduler>(*this);
205
206
  object_cache = make_uniq<ObjectCache>();
206
207
  connection_manager = make_uniq<ConnectionManager>();
@@ -23,7 +23,7 @@ void DatabaseManager::InitializeSystemCatalog() {
23
23
  system->Initialize();
24
24
  }
25
25
 
26
- AttachedDatabase *DatabaseManager::GetDatabase(ClientContext &context, const string &name) {
26
+ optional_ptr<AttachedDatabase> DatabaseManager::GetDatabase(ClientContext &context, const string &name) {
27
27
  if (StringUtil::Lower(name) == TEMP_CATALOG) {
28
28
  return context.client_data->temporary_objects.get();
29
29
  }
@@ -50,7 +50,7 @@ void DatabaseManager::DetachDatabase(ClientContext &context, const string &name,
50
50
  }
51
51
  }
52
52
 
53
- AttachedDatabase *DatabaseManager::GetDatabaseFromPath(ClientContext &context, const string &path) {
53
+ optional_ptr<AttachedDatabase> DatabaseManager::GetDatabaseFromPath(ClientContext &context, const string &path) {
54
54
  auto databases = GetDatabases(context);
55
55
  for (auto db : databases) {
56
56
  if (db->IsSystem()) {
@@ -81,8 +81,8 @@ const string &DatabaseManager::GetDefaultDatabase(ClientContext &context) {
81
81
  return default_entry.catalog;
82
82
  }
83
83
 
84
- vector<AttachedDatabase *> DatabaseManager::GetDatabases(ClientContext &context) {
85
- vector<AttachedDatabase *> result;
84
+ vector<optional_ptr<AttachedDatabase>> DatabaseManager::GetDatabases(ClientContext &context) {
85
+ vector<optional_ptr<AttachedDatabase>> result;
86
86
  databases->Scan(context, [&](CatalogEntry *entry) { result.push_back((AttachedDatabase *)entry); });
87
87
  result.push_back(system.get());
88
88
  result.push_back(context.client_data->temporary_objects.get());
@@ -33,10 +33,10 @@ void CommonAggregateOptimizer::ExtractCommonAggregates(LogicalAggregate &aggr) {
33
33
  idx_t total_erased = 0;
34
34
  for (idx_t i = 0; i < aggr.expressions.size(); i++) {
35
35
  idx_t original_index = i + total_erased;
36
- auto entry = aggregate_remap.find(aggr.expressions[i].get());
36
+ auto entry = aggregate_remap.find(*aggr.expressions[i]);
37
37
  if (entry == aggregate_remap.end()) {
38
38
  // aggregate does not exist yet: add it to the map
39
- aggregate_remap[aggr.expressions[i].get()] = i;
39
+ aggregate_remap[*aggr.expressions[i]] = i;
40
40
  if (i != original_index) {
41
41
  // this aggregate is not erased, however an agregate BEFORE it has been erased
42
42
  // so we need to remap this aggregaet
@@ -60,10 +60,10 @@ void CommonSubExpressionOptimizer::CountExpressions(Expression &expr, CSEReplace
60
60
  }
61
61
  if (expr.expression_class != ExpressionClass::BOUND_AGGREGATE && !expr.HasSideEffects()) {
62
62
  // we can't move aggregates to a projection, so we only consider the children of the aggregate
63
- auto node = state.expression_count.find(&expr);
63
+ auto node = state.expression_count.find(expr);
64
64
  if (node == state.expression_count.end()) {
65
65
  // first time we encounter this expression, insert this node with [count = 1]
66
- state.expression_count[&expr] = CSENode();
66
+ state.expression_count[expr] = CSENode();
67
67
  } else {
68
68
  // we encountered this expression before, increment the occurrence count
69
69
  node->second.count++;
@@ -95,8 +95,8 @@ void CommonSubExpressionOptimizer::PerformCSEReplacement(unique_ptr<Expression>
95
95
  // check if this child is eligible for CSE elimination
96
96
  bool can_cse = expr.expression_class != ExpressionClass::BOUND_CONJUNCTION &&
97
97
  expr.expression_class != ExpressionClass::BOUND_CASE;
98
- if (can_cse && state.expression_count.find(&expr) != state.expression_count.end()) {
99
- auto &node = state.expression_count[&expr];
98
+ if (can_cse && state.expression_count.find(expr) != state.expression_count.end()) {
99
+ auto &node = state.expression_count[expr];
100
100
  if (node.count > 1) {
101
101
  // this expression occurs more than once! push it into the projection
102
102
  // check if it has already been pushed into the projection
@@ -106,8 +106,10 @@ void DeliminatorPlanUpdater::VisitOperator(LogicalOperator &op) {
106
106
  }
107
107
 
108
108
  void DeliminatorPlanUpdater::VisitExpression(unique_ptr<Expression> *expression) {
109
- if (expr_map.find(expression->get()) != expr_map.end()) {
110
- *expression = expr_map[expression->get()]->Copy();
109
+ auto &expr = **expression;
110
+ auto entry = expr_map.find(expr);
111
+ if (entry != expr_map.end()) {
112
+ *expression = entry->second->Copy();
111
113
  } else {
112
114
  VisitExpressionChildren(**expression);
113
115
  }
@@ -207,17 +209,17 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
207
209
  }
208
210
  // check if joining with the DelimGet is redundant, and collect relevant column information
209
211
  bool all_equality_conditions = true;
210
- vector<Expression *> nulls_are_not_equal_exprs;
212
+ vector<reference<Expression>> nulls_are_not_equal_exprs;
211
213
  for (auto &cond : join.conditions) {
212
214
  all_equality_conditions = all_equality_conditions && IsEqualityJoinCondition(cond);
213
- auto delim_side = delim_idx == 0 ? cond.left.get() : cond.right.get();
214
- auto other_side = delim_idx == 0 ? cond.right.get() : cond.left.get();
215
- if (delim_side->type != ExpressionType::BOUND_COLUMN_REF) {
215
+ auto &delim_side = delim_idx == 0 ? *cond.left : *cond.right;
216
+ auto &other_side = delim_idx == 0 ? *cond.right : *cond.left;
217
+ if (delim_side.type != ExpressionType::BOUND_COLUMN_REF) {
216
218
  // non-colref e.g. expression -(4, 1) in 4-i=j where i is from DelimGet
217
219
  // FIXME: might be possible to also eliminate these
218
220
  return false;
219
221
  }
220
- updater.expr_map[delim_side] = other_side;
222
+ updater.expr_map[delim_side] = &other_side;
221
223
  if (cond.comparison != ExpressionType::COMPARE_NOT_DISTINCT_FROM) {
222
224
  nulls_are_not_equal_exprs.push_back(other_side);
223
225
  }
@@ -229,7 +231,7 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
229
231
  updater.projection_map[cb] = true;
230
232
  updater.reverse_proj_or_agg_map[cb] = proj_or_agg.expressions[cb.column_index].get();
231
233
  for (auto &expr : nulls_are_not_equal_exprs) {
232
- if (proj_or_agg.expressions[cb.column_index]->Equals(expr)) {
234
+ if (proj_or_agg.expressions[cb.column_index]->Equals(&expr.get())) {
233
235
  updater.projection_map[cb] = false;
234
236
  break;
235
237
  }
@@ -252,8 +254,8 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
252
254
  updater.projection_map[cb] = true;
253
255
  updater.reverse_proj_or_agg_map[cb] = all_agg_exprs[cb.column_index];
254
256
  for (auto &expr : nulls_are_not_equal_exprs) {
255
- if ((cb.table_index == agg.group_index && agg.groups[cb.column_index]->Equals(expr)) ||
256
- (cb.table_index == agg.aggregate_index && agg.expressions[cb.column_index]->Equals(expr))) {
257
+ if ((cb.table_index == agg.group_index && agg.groups[cb.column_index]->Equals(&expr.get())) ||
258
+ (cb.table_index == agg.aggregate_index && agg.expressions[cb.column_index]->Equals(&expr.get()))) {
257
259
  updater.projection_map[cb] = false;
258
260
  break;
259
261
  }
@@ -276,7 +278,7 @@ bool Deliminator::RemoveCandidate(unique_ptr<LogicalOperator> *plan, unique_ptr<
276
278
  for (auto &expr : nulls_are_not_equal_exprs) {
277
279
  auto is_not_null_expr =
278
280
  make_uniq<BoundOperatorExpression>(ExpressionType::OPERATOR_IS_NOT_NULL, LogicalType::BOOLEAN);
279
- is_not_null_expr->children.push_back(expr->Copy());
281
+ is_not_null_expr->children.push_back(expr.get().Copy());
280
282
  filter_op->expressions.push_back(std::move(is_not_null_expr));
281
283
  }
282
284
  }
@@ -12,8 +12,8 @@ namespace duckdb {
12
12
  unique_ptr<Expression> ExpressionRewriter::ApplyRules(LogicalOperator &op, const vector<Rule *> &rules,
13
13
  unique_ptr<Expression> expr, bool &changes_made, bool is_root) {
14
14
  for (auto &rule : rules) {
15
- vector<Expression *> bindings;
16
- if (rule->root->Match(expr.get(), bindings)) {
15
+ vector<reference<Expression>> bindings;
16
+ if (rule->root->Match(*expr, bindings)) {
17
17
  // the rule matches! try to apply it
18
18
  bool rule_made_change = false;
19
19
  auto result = rule->Apply(op, bindings, rule_made_change, is_root);