duckdb 0.8.2-dev3989.0 → 0.8.2-dev4126.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/binding.gyp +8 -7
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/json/buffered_json_reader.cpp +76 -74
  4. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +35 -32
  5. package/src/duckdb/extension/json/include/json_scan.hpp +9 -6
  6. package/src/duckdb/extension/json/json_scan.cpp +124 -121
  7. package/src/duckdb/extension/parquet/parquet_extension.cpp +23 -13
  8. package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp +5 -0
  9. package/src/duckdb/src/common/crypto/md5.cpp +2 -12
  10. package/src/duckdb/src/common/radix_partitioning.cpp +1 -1
  11. package/src/duckdb/src/common/sort/partition_state.cpp +5 -1
  12. package/src/duckdb/src/core_functions/aggregate/holistic/mode.cpp +1 -1
  13. package/src/duckdb/src/core_functions/function_list.cpp +8 -0
  14. package/src/duckdb/src/core_functions/scalar/list/list_cosine_similarity.cpp +78 -0
  15. package/src/duckdb/src/core_functions/scalar/list/list_distance.cpp +72 -0
  16. package/src/duckdb/src/core_functions/scalar/list/list_inner_product.cpp +70 -0
  17. package/src/duckdb/src/core_functions/scalar/string/sha256.cpp +32 -0
  18. package/src/duckdb/src/execution/index/art/art.cpp +111 -92
  19. package/src/duckdb/src/execution/index/art/iterator.cpp +21 -27
  20. package/src/duckdb/src/execution/index/art/leaf.cpp +72 -153
  21. package/src/duckdb/src/execution/index/art/node.cpp +109 -203
  22. package/src/duckdb/src/execution/index/art/node16.cpp +32 -64
  23. package/src/duckdb/src/execution/index/art/node256.cpp +38 -53
  24. package/src/duckdb/src/execution/index/art/node4.cpp +31 -62
  25. package/src/duckdb/src/execution/index/art/node48.cpp +43 -65
  26. package/src/duckdb/src/execution/index/art/prefix.cpp +70 -141
  27. package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +345 -0
  28. package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +74 -0
  29. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +1 -1
  30. package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +1 -1
  31. package/src/duckdb/src/function/scalar/string/suffix.cpp +1 -1
  32. package/src/duckdb/src/function/table/system/duckdb_columns.cpp +3 -1
  33. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  34. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp +2 -0
  35. package/src/duckdb/src/include/duckdb/common/optional_idx.hpp +1 -1
  36. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +51 -0
  37. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +9 -0
  38. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +17 -7
  39. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +5 -5
  40. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +10 -16
  41. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +38 -116
  42. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +17 -18
  43. package/src/duckdb/src/include/duckdb/execution/index/art/node256.hpp +17 -23
  44. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +17 -18
  45. package/src/duckdb/src/include/duckdb/execution/index/art/node48.hpp +17 -24
  46. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +16 -22
  47. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +126 -0
  48. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +79 -0
  49. package/src/duckdb/src/include/duckdb/execution/index/index_pointer.hpp +96 -0
  50. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +1 -1
  51. package/src/duckdb/src/include/duckdb/planner/operator/logical_join.hpp +1 -1
  52. package/src/duckdb/src/include/duckdb/storage/block.hpp +1 -1
  53. package/src/duckdb/src/include/duckdb/storage/index.hpp +10 -8
  54. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_writer.hpp +3 -0
  55. package/src/duckdb/src/main/extension/extension_helper.cpp +15 -1
  56. package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +14 -5
  57. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +2 -3
  58. package/src/duckdb/src/storage/checkpoint_manager.cpp +16 -21
  59. package/src/duckdb/src/storage/data_table.cpp +3 -3
  60. package/src/duckdb/src/storage/index.cpp +7 -1
  61. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +21 -21
  62. package/src/duckdb/src/storage/standard_buffer_manager.cpp +10 -16
  63. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  64. package/src/duckdb/src/storage/table_index_list.cpp +1 -1
  65. package/src/duckdb/src/transaction/commit_state.cpp +5 -1
  66. package/src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp +4 -1
  67. package/src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp +24 -2
  68. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +5 -5
  69. package/src/duckdb/ub_src_core_functions_scalar_list.cpp +6 -0
  70. package/src/duckdb/ub_src_core_functions_scalar_string.cpp +2 -0
  71. package/src/duckdb/ub_src_execution_index.cpp +4 -0
  72. package/src/duckdb/ub_src_execution_index_art.cpp +0 -2
  73. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +0 -238
  74. package/src/duckdb/src/include/duckdb/execution/index/art/fixed_size_allocator.hpp +0 -115
@@ -1,115 +0,0 @@
1
- //===----------------------------------------------------------------------===//
2
- // DuckDB
3
- //
4
- // duckdb/execution/index/art/fixed_size_allocator.hpp
5
- //
6
- //
7
- //===----------------------------------------------------------------------===//
8
-
9
- #pragma once
10
-
11
- #include "duckdb/common/types/validity_mask.hpp"
12
- #include "duckdb/common/unordered_set.hpp"
13
- #include "duckdb/storage/buffer_manager.hpp"
14
- #include "duckdb/execution/index/art/node.hpp"
15
-
16
- namespace duckdb {
17
-
18
- // structs
19
- struct BufferEntry {
20
- BufferEntry(const data_ptr_t &ptr, const idx_t &allocation_count) : ptr(ptr), allocation_count(allocation_count) {
21
- }
22
- data_ptr_t ptr;
23
- idx_t allocation_count;
24
- };
25
-
26
- //! The FixedSizeAllocator provides pointers to fixed-size sections of pre-allocated memory buffers.
27
- //! The pointers are Node pointers, and the leftmost byte (serialize flag and type) must always be zero.
28
- class FixedSizeAllocator {
29
- public:
30
- //! Fixed size of the buffers
31
- static constexpr idx_t BUFFER_ALLOC_SIZE = Storage::BLOCK_ALLOC_SIZE;
32
- //! We can vacuum 10% or more of the total memory usage of the allocator
33
- static constexpr uint8_t VACUUM_THRESHOLD = 10;
34
-
35
- //! Constants for fast offset calculations in the bitmask
36
- static constexpr idx_t BASE[] = {0x00000000FFFFFFFF, 0x0000FFFF, 0x00FF, 0x0F, 0x3, 0x1};
37
- static constexpr uint8_t SHIFT[] = {32, 16, 8, 4, 2, 1};
38
-
39
- public:
40
- explicit FixedSizeAllocator(const idx_t allocation_size, Allocator &allocator);
41
- ~FixedSizeAllocator();
42
-
43
- //! Allocation size of one element in a buffer
44
- idx_t allocation_size;
45
- //! Total number of allocations
46
- idx_t total_allocations;
47
- //! Number of validity_t values in the bitmask
48
- idx_t bitmask_count;
49
- //! First starting byte of the payload
50
- idx_t allocation_offset;
51
- //! Number of possible allocations per buffer
52
- idx_t allocations_per_buffer;
53
-
54
- //! Buffers containing the data
55
- vector<BufferEntry> buffers;
56
- //! Buffers with free space
57
- unordered_set<idx_t> buffers_with_free_space;
58
-
59
- //! Minimum buffer ID of buffers that can be vacuumed
60
- idx_t min_vacuum_buffer_id;
61
-
62
- //! Buffer manager of the database instance
63
- Allocator &allocator;
64
-
65
- public:
66
- //! Get a new Node pointer to data, might cause a new buffer allocation
67
- Node New();
68
- //! Free the data of the Node pointer
69
- void Free(const Node ptr);
70
- //! Get the data of the Node pointer
71
- template <class T>
72
- inline T *Get(const Node ptr) const {
73
- return (T *)Get(ptr);
74
- }
75
-
76
- //! Resets the allocator, e.g., becomes necessary during DELETE FROM table
77
- void Reset();
78
-
79
- //! Returns the allocated memory size in bytes
80
- inline idx_t GetMemoryUsage() const {
81
- return buffers.size() * BUFFER_ALLOC_SIZE;
82
- }
83
-
84
- //! Merge another FixedSizeAllocator into this allocator. Both must have the same allocation size
85
- void Merge(FixedSizeAllocator &other);
86
-
87
- //! Initialize a vacuum operation, and return true, if the allocator needs a vacuum
88
- bool InitializeVacuum();
89
- //! Finalize a vacuum operation by freeing all buffers exceeding the min_vacuum_buffer_id
90
- void FinalizeVacuum();
91
- //! Returns true, if a Node pointer qualifies for a vacuum operation, and false otherwise
92
- inline bool NeedsVacuum(const Node ptr) const {
93
- if (ptr.GetBufferId() >= min_vacuum_buffer_id) {
94
- return true;
95
- }
96
- return false;
97
- }
98
- //! Vacuums a Node pointer
99
- Node VacuumPointer(const Node ptr);
100
-
101
- //! Verify that the allocation counts match the existing positions on the buffers
102
- void Verify() const;
103
-
104
- private:
105
- //! Returns the data_ptr_t of a Node pointer
106
- inline data_ptr_t Get(const Node ptr) const {
107
- D_ASSERT(ptr.GetBufferId() < buffers.size());
108
- D_ASSERT(ptr.GetOffset() < allocations_per_buffer);
109
- return buffers[ptr.GetBufferId()].ptr + ptr.GetOffset() * allocation_size + allocation_offset;
110
- }
111
- //! Returns the first free offset in a bitmask
112
- uint32_t GetOffset(ValidityMask &mask, const idx_t allocation_count);
113
- };
114
-
115
- } // namespace duckdb