duckdb 0.7.2-dev1671.0 → 0.7.2-dev1734.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/extension/icu/icu-datefunc.cpp +20 -8
- package/src/duckdb/extension/icu/icu-strptime.cpp +117 -29
- package/src/duckdb/extension/icu/include/icu-datefunc.hpp +2 -0
- package/src/duckdb/src/common/local_file_system.cpp +13 -2
- package/src/duckdb/src/common/sort/partition_state.cpp +644 -0
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/expression_executor_state.cpp +2 -3
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +77 -849
- package/src/duckdb/src/function/table/system/duckdb_extensions.cpp +2 -2
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/sort/partition_state.hpp +247 -0
- package/src/duckdb/src/include/duckdb/execution/expression_executor_state.hpp +1 -3
- package/src/duckdb/src/include/duckdb/planner/pragma_handler.hpp +3 -2
- package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp +1 -2
- package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp +77 -0
- package/src/duckdb/src/include/duckdb/storage/buffer/temporary_file_information.hpp +12 -0
- package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp +3 -59
- package/src/duckdb/src/main/extension/extension_install.cpp +11 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +29 -3
- package/src/duckdb/src/main/query_profiler.cpp +1 -1
- package/src/duckdb/src/planner/pragma_handler.cpp +7 -5
- package/src/duckdb/src/storage/buffer/block_handle.cpp +128 -0
- package/src/duckdb/src/storage/buffer/block_manager.cpp +81 -0
- package/src/duckdb/src/storage/buffer/buffer_pool.cpp +132 -0
- package/src/duckdb/src/storage/buffer/buffer_pool_reservation.cpp +32 -0
- package/src/duckdb/src/storage/buffer_manager.cpp +0 -351
- package/src/duckdb/third_party/libpg_query/postgres_parser.cpp +3 -5
- package/src/duckdb/ub_src_common_sort.cpp +2 -0
- package/src/duckdb/ub_src_storage_buffer.cpp +8 -0
@@ -75,7 +75,7 @@ unique_ptr<GlobalTableFunctionState> DuckDBExtensionsInit(ClientContext &context
|
|
75
75
|
}
|
76
76
|
installed_extensions[info.name] = std::move(info);
|
77
77
|
}
|
78
|
-
|
78
|
+
#ifndef WASM_LOADABLE_EXTENSIONS
|
79
79
|
// scan the install directory for installed extensions
|
80
80
|
auto ext_directory = ExtensionHelper::ExtensionDirectory(context);
|
81
81
|
fs.ListFiles(ext_directory, [&](const string &path, bool is_directory) {
|
@@ -96,7 +96,7 @@ unique_ptr<GlobalTableFunctionState> DuckDBExtensionsInit(ClientContext &context
|
|
96
96
|
entry->second.installed = true;
|
97
97
|
}
|
98
98
|
});
|
99
|
-
|
99
|
+
#endif
|
100
100
|
// now check the list of currently loaded extensions
|
101
101
|
auto &loaded_extensions = db.LoadedExtensions();
|
102
102
|
for (auto &ext_name : loaded_extensions) {
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.7.2-
|
2
|
+
#define DUCKDB_VERSION "0.7.2-dev1734"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "e8610c85fb"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|
@@ -0,0 +1,247 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/common/sort/partition_state.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/common/sort/sort.hpp"
|
12
|
+
#include "duckdb/common/types/partitioned_column_data.hpp"
|
13
|
+
#include "duckdb/common/radix_partitioning.hpp"
|
14
|
+
#include "duckdb/parallel/base_pipeline_event.hpp"
|
15
|
+
|
16
|
+
namespace duckdb {
|
17
|
+
|
18
|
+
class PartitionGlobalHashGroup {
|
19
|
+
public:
|
20
|
+
using GlobalSortStatePtr = unique_ptr<GlobalSortState>;
|
21
|
+
using LocalSortStatePtr = unique_ptr<LocalSortState>;
|
22
|
+
using Orders = vector<BoundOrderByNode>;
|
23
|
+
using Types = vector<LogicalType>;
|
24
|
+
|
25
|
+
PartitionGlobalHashGroup(BufferManager &buffer_manager, const Orders &partitions, const Orders &orders,
|
26
|
+
const Types &payload_types, bool external);
|
27
|
+
|
28
|
+
void ComputeMasks(ValidityMask &partition_mask, ValidityMask &order_mask);
|
29
|
+
|
30
|
+
GlobalSortStatePtr global_sort;
|
31
|
+
atomic<idx_t> count;
|
32
|
+
|
33
|
+
// Mask computation
|
34
|
+
SortLayout partition_layout;
|
35
|
+
};
|
36
|
+
|
37
|
+
class PartitionGlobalSinkState {
|
38
|
+
public:
|
39
|
+
using HashGroupPtr = unique_ptr<PartitionGlobalHashGroup>;
|
40
|
+
using Orders = vector<BoundOrderByNode>;
|
41
|
+
using Types = vector<LogicalType>;
|
42
|
+
|
43
|
+
using GroupingPartition = unique_ptr<PartitionedColumnData>;
|
44
|
+
using GroupingAppend = unique_ptr<PartitionedColumnDataAppendState>;
|
45
|
+
|
46
|
+
PartitionGlobalSinkState(ClientContext &context, const vector<unique_ptr<Expression>> &partitions_p,
|
47
|
+
const vector<BoundOrderByNode> &orders_p, const Types &payload_types,
|
48
|
+
const vector<unique_ptr<BaseStatistics>> &partitions_stats, idx_t estimated_cardinality);
|
49
|
+
|
50
|
+
void UpdateLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
|
51
|
+
void CombineLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
|
52
|
+
|
53
|
+
void BuildSortState(ColumnDataCollection &group_data, PartitionGlobalHashGroup &global_sort);
|
54
|
+
|
55
|
+
ClientContext &context;
|
56
|
+
BufferManager &buffer_manager;
|
57
|
+
Allocator &allocator;
|
58
|
+
mutex lock;
|
59
|
+
|
60
|
+
// OVER(PARTITION BY...) (hash grouping)
|
61
|
+
unique_ptr<RadixPartitionedColumnData> grouping_data;
|
62
|
+
//! Payload plus hash column
|
63
|
+
Types grouping_types;
|
64
|
+
|
65
|
+
// OVER(...) (sorting)
|
66
|
+
Orders partitions;
|
67
|
+
Orders orders;
|
68
|
+
const Types payload_types;
|
69
|
+
vector<HashGroupPtr> hash_groups;
|
70
|
+
bool external;
|
71
|
+
|
72
|
+
// OVER() (no sorting)
|
73
|
+
unique_ptr<RowDataCollection> rows;
|
74
|
+
unique_ptr<RowDataCollection> strings;
|
75
|
+
|
76
|
+
// Threading
|
77
|
+
idx_t memory_per_thread;
|
78
|
+
atomic<idx_t> count;
|
79
|
+
|
80
|
+
private:
|
81
|
+
void ResizeGroupingData(idx_t cardinality);
|
82
|
+
void SyncLocalPartition(GroupingPartition &local_partition, GroupingAppend &local_append);
|
83
|
+
};
|
84
|
+
|
85
|
+
class PartitionLocalSinkState {
|
86
|
+
public:
|
87
|
+
PartitionLocalSinkState(ClientContext &context, PartitionGlobalSinkState &gstate_p);
|
88
|
+
|
89
|
+
// Global state
|
90
|
+
PartitionGlobalSinkState &gstate;
|
91
|
+
Allocator &allocator;
|
92
|
+
|
93
|
+
// OVER(PARTITION BY...) (hash grouping)
|
94
|
+
ExpressionExecutor executor;
|
95
|
+
DataChunk group_chunk;
|
96
|
+
DataChunk payload_chunk;
|
97
|
+
unique_ptr<PartitionedColumnData> local_partition;
|
98
|
+
unique_ptr<PartitionedColumnDataAppendState> local_append;
|
99
|
+
|
100
|
+
// OVER(...) (sorting)
|
101
|
+
size_t sort_cols;
|
102
|
+
|
103
|
+
// OVER() (no sorting)
|
104
|
+
RowLayout payload_layout;
|
105
|
+
unique_ptr<RowDataCollection> rows;
|
106
|
+
unique_ptr<RowDataCollection> strings;
|
107
|
+
|
108
|
+
//! Compute the hash values
|
109
|
+
void Hash(DataChunk &input_chunk, Vector &hash_vector);
|
110
|
+
//! Sink an input chunk
|
111
|
+
void Sink(DataChunk &input_chunk);
|
112
|
+
//! Merge the state into the global state.
|
113
|
+
void Combine();
|
114
|
+
};
|
115
|
+
|
116
|
+
enum class PartitionSortStage : uint8_t { INIT, PREPARE, MERGE, SORTED };
|
117
|
+
|
118
|
+
class PartitionLocalMergeState;
|
119
|
+
|
120
|
+
class PartitionGlobalMergeState {
|
121
|
+
public:
|
122
|
+
using GroupDataPtr = unique_ptr<ColumnDataCollection>;
|
123
|
+
|
124
|
+
explicit PartitionGlobalMergeState(PartitionGlobalSinkState &sink, GroupDataPtr group_data);
|
125
|
+
|
126
|
+
bool IsSorted() const {
|
127
|
+
lock_guard<mutex> guard(lock);
|
128
|
+
return stage == PartitionSortStage::SORTED;
|
129
|
+
}
|
130
|
+
|
131
|
+
bool AssignTask(PartitionLocalMergeState &local_state);
|
132
|
+
bool TryPrepareNextStage();
|
133
|
+
void CompleteTask();
|
134
|
+
|
135
|
+
PartitionGlobalSinkState &sink;
|
136
|
+
GroupDataPtr group_data;
|
137
|
+
PartitionGlobalHashGroup *hash_group;
|
138
|
+
GlobalSortState *global_sort;
|
139
|
+
|
140
|
+
private:
|
141
|
+
mutable mutex lock;
|
142
|
+
PartitionSortStage stage;
|
143
|
+
idx_t total_tasks;
|
144
|
+
idx_t tasks_assigned;
|
145
|
+
idx_t tasks_completed;
|
146
|
+
};
|
147
|
+
|
148
|
+
class PartitionLocalMergeState {
|
149
|
+
public:
|
150
|
+
PartitionLocalMergeState() : merge_state(nullptr), stage(PartitionSortStage::INIT) {
|
151
|
+
finished = true;
|
152
|
+
}
|
153
|
+
|
154
|
+
bool TaskFinished() {
|
155
|
+
return finished;
|
156
|
+
}
|
157
|
+
|
158
|
+
void Prepare();
|
159
|
+
void Merge();
|
160
|
+
|
161
|
+
void ExecuteTask();
|
162
|
+
|
163
|
+
PartitionGlobalMergeState *merge_state;
|
164
|
+
PartitionSortStage stage;
|
165
|
+
atomic<bool> finished;
|
166
|
+
};
|
167
|
+
|
168
|
+
class PartitionGlobalMergeStates {
|
169
|
+
public:
|
170
|
+
using PartitionGlobalMergeStatePtr = unique_ptr<PartitionGlobalMergeState>;
|
171
|
+
|
172
|
+
explicit PartitionGlobalMergeStates(PartitionGlobalSinkState &sink);
|
173
|
+
|
174
|
+
vector<PartitionGlobalMergeStatePtr> states;
|
175
|
+
};
|
176
|
+
|
177
|
+
class PartitionMergeEvent : public BasePipelineEvent {
|
178
|
+
public:
|
179
|
+
PartitionMergeEvent(PartitionGlobalSinkState &gstate_p, Pipeline &pipeline_p)
|
180
|
+
: BasePipelineEvent(pipeline_p), gstate(gstate_p), merge_states(gstate_p) {
|
181
|
+
}
|
182
|
+
|
183
|
+
PartitionGlobalSinkState &gstate;
|
184
|
+
PartitionGlobalMergeStates merge_states;
|
185
|
+
|
186
|
+
public:
|
187
|
+
void Schedule() override;
|
188
|
+
};
|
189
|
+
|
190
|
+
class PartitionGlobalSourceState {
|
191
|
+
public:
|
192
|
+
explicit PartitionGlobalSourceState(PartitionGlobalSinkState &gsink_p) : gsink(gsink_p), next_bin(0) {
|
193
|
+
}
|
194
|
+
|
195
|
+
PartitionGlobalSinkState &gsink;
|
196
|
+
//! The output read position.
|
197
|
+
atomic<idx_t> next_bin;
|
198
|
+
|
199
|
+
public:
|
200
|
+
idx_t MaxThreads() {
|
201
|
+
// If there is only one partition, we have to process it on one thread.
|
202
|
+
if (!gsink.grouping_data) {
|
203
|
+
return 1;
|
204
|
+
}
|
205
|
+
|
206
|
+
// If there is not a lot of data, process serially.
|
207
|
+
if (gsink.count < STANDARD_ROW_GROUPS_SIZE) {
|
208
|
+
return 1;
|
209
|
+
}
|
210
|
+
|
211
|
+
return gsink.hash_groups.size();
|
212
|
+
}
|
213
|
+
};
|
214
|
+
|
215
|
+
// Per-thread read state
|
216
|
+
class PartitionLocalSourceState {
|
217
|
+
public:
|
218
|
+
using HashGroupPtr = unique_ptr<PartitionGlobalHashGroup>;
|
219
|
+
|
220
|
+
explicit PartitionLocalSourceState(PartitionGlobalSinkState &gstate_p);
|
221
|
+
|
222
|
+
void MaterializeSortedData();
|
223
|
+
idx_t GeneratePartition(const idx_t hash_bin);
|
224
|
+
|
225
|
+
PartitionGlobalSinkState &gstate;
|
226
|
+
|
227
|
+
//! The read partition
|
228
|
+
idx_t hash_bin;
|
229
|
+
HashGroupPtr hash_group;
|
230
|
+
|
231
|
+
//! The generated input chunks
|
232
|
+
unique_ptr<RowDataCollection> rows;
|
233
|
+
unique_ptr<RowDataCollection> heap;
|
234
|
+
RowLayout layout;
|
235
|
+
//! The partition boundary mask
|
236
|
+
vector<validity_t> partition_bits;
|
237
|
+
ValidityMask partition_mask;
|
238
|
+
//! The order boundary mask
|
239
|
+
vector<validity_t> order_bits;
|
240
|
+
ValidityMask order_mask;
|
241
|
+
//! The read cursor
|
242
|
+
unique_ptr<RowDataCollectionScanner> scanner;
|
243
|
+
//! Buffer for the inputs
|
244
|
+
DataChunk input_chunk;
|
245
|
+
};
|
246
|
+
|
247
|
+
} // namespace duckdb
|
@@ -29,7 +29,6 @@ struct ExpressionState {
|
|
29
29
|
vector<unique_ptr<ExpressionState>> child_states;
|
30
30
|
vector<LogicalType> types;
|
31
31
|
DataChunk intermediate_chunk;
|
32
|
-
string name;
|
33
32
|
CycleCounter profiler;
|
34
33
|
|
35
34
|
public:
|
@@ -55,12 +54,11 @@ public:
|
|
55
54
|
};
|
56
55
|
|
57
56
|
struct ExpressionExecutorState {
|
58
|
-
|
57
|
+
ExpressionExecutorState();
|
59
58
|
|
60
59
|
unique_ptr<ExpressionState> root_state;
|
61
60
|
ExpressionExecutor *executor = nullptr;
|
62
61
|
CycleCounter profiler;
|
63
|
-
string name;
|
64
62
|
|
65
63
|
void Verify();
|
66
64
|
};
|
@@ -29,8 +29,9 @@ private:
|
|
29
29
|
ClientContext &context;
|
30
30
|
|
31
31
|
private:
|
32
|
-
//! Handles a pragma statement,
|
33
|
-
|
32
|
+
//! Handles a pragma statement, returns whether the statement was expanded, if it was expanded the 'resulting_query'
|
33
|
+
//! contains the statement(s) to replace the current one
|
34
|
+
bool HandlePragma(SQLStatement *statement, string &resulting_query);
|
34
35
|
|
35
36
|
void HandlePragmaStatementsInternal(vector<unique_ptr<SQLStatement>> &statements);
|
36
37
|
};
|
@@ -12,14 +12,13 @@
|
|
12
12
|
#include "duckdb/common/common.hpp"
|
13
13
|
#include "duckdb/common/mutex.hpp"
|
14
14
|
#include "duckdb/storage/storage_info.hpp"
|
15
|
+
#include "duckdb/common/file_buffer.hpp"
|
15
16
|
|
16
17
|
namespace duckdb {
|
17
18
|
class BlockManager;
|
18
19
|
class BufferHandle;
|
19
|
-
class BufferManager;
|
20
20
|
class BufferPool;
|
21
21
|
class DatabaseInstance;
|
22
|
-
class FileBuffer;
|
23
22
|
|
24
23
|
enum class BlockState : uint8_t { BLOCK_UNLOADED = 0, BLOCK_LOADED = 1 };
|
25
24
|
|
@@ -0,0 +1,77 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "duckdb/common/mutex.hpp"
|
4
|
+
#include "duckdb/common/file_buffer.hpp"
|
5
|
+
#include "duckdb/storage/buffer/block_handle.hpp"
|
6
|
+
|
7
|
+
namespace duckdb {
|
8
|
+
|
9
|
+
struct EvictionQueue;
|
10
|
+
|
11
|
+
struct BufferEvictionNode {
|
12
|
+
BufferEvictionNode() {
|
13
|
+
}
|
14
|
+
BufferEvictionNode(weak_ptr<BlockHandle> handle_p, idx_t timestamp_p)
|
15
|
+
: handle(std::move(handle_p)), timestamp(timestamp_p) {
|
16
|
+
D_ASSERT(!handle.expired());
|
17
|
+
}
|
18
|
+
|
19
|
+
weak_ptr<BlockHandle> handle;
|
20
|
+
idx_t timestamp;
|
21
|
+
|
22
|
+
bool CanUnload(BlockHandle &handle_p);
|
23
|
+
|
24
|
+
shared_ptr<BlockHandle> TryGetBlockHandle();
|
25
|
+
};
|
26
|
+
|
27
|
+
//! The BufferPool is in charge of handling memory management for one or more databases. It defines memory limits
|
28
|
+
//! and implements priority eviction among all users of the pool.
|
29
|
+
class BufferPool {
|
30
|
+
friend class BlockHandle;
|
31
|
+
friend class BlockManager;
|
32
|
+
friend class BufferManager;
|
33
|
+
|
34
|
+
public:
|
35
|
+
explicit BufferPool(idx_t maximum_memory);
|
36
|
+
virtual ~BufferPool();
|
37
|
+
|
38
|
+
//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
|
39
|
+
//! blocks can be evicted
|
40
|
+
void SetLimit(idx_t limit, const char *exception_postscript);
|
41
|
+
|
42
|
+
idx_t GetUsedMemory();
|
43
|
+
|
44
|
+
idx_t GetMaxMemory();
|
45
|
+
|
46
|
+
protected:
|
47
|
+
//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
|
48
|
+
//! (i.e. not enough blocks could be evicted)
|
49
|
+
//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
|
50
|
+
//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
|
51
|
+
//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
|
52
|
+
//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
|
53
|
+
struct EvictionResult {
|
54
|
+
bool success;
|
55
|
+
TempBufferPoolReservation reservation;
|
56
|
+
};
|
57
|
+
virtual EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
|
58
|
+
unique_ptr<FileBuffer> *buffer = nullptr);
|
59
|
+
|
60
|
+
//! Garbage collect eviction queue
|
61
|
+
void PurgeQueue();
|
62
|
+
void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
|
63
|
+
|
64
|
+
private:
|
65
|
+
//! The lock for changing the memory limit
|
66
|
+
mutex limit_lock;
|
67
|
+
//! The current amount of memory that is occupied by the buffer manager (in bytes)
|
68
|
+
atomic<idx_t> current_memory;
|
69
|
+
//! The maximum amount of memory that the buffer manager can keep (in bytes)
|
70
|
+
atomic<idx_t> maximum_memory;
|
71
|
+
//! Eviction queue
|
72
|
+
unique_ptr<EvictionQueue> queue;
|
73
|
+
//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
|
74
|
+
atomic<uint32_t> queue_insertions;
|
75
|
+
};
|
76
|
+
|
77
|
+
} // namespace duckdb
|
@@ -16,6 +16,8 @@
|
|
16
16
|
#include "duckdb/storage/block_manager.hpp"
|
17
17
|
#include "duckdb/storage/buffer/block_handle.hpp"
|
18
18
|
#include "duckdb/storage/buffer/buffer_handle.hpp"
|
19
|
+
#include "duckdb/storage/buffer/buffer_pool.hpp"
|
20
|
+
#include "duckdb/storage/buffer/temporary_file_information.hpp"
|
19
21
|
|
20
22
|
namespace duckdb {
|
21
23
|
class BlockManager;
|
@@ -23,65 +25,7 @@ class DatabaseInstance;
|
|
23
25
|
class TemporaryDirectoryHandle;
|
24
26
|
struct EvictionQueue;
|
25
27
|
|
26
|
-
//! The
|
27
|
-
//! and implements priority eviction among all users of the pool.
|
28
|
-
class BufferPool {
|
29
|
-
friend class BlockHandle;
|
30
|
-
friend class BlockManager;
|
31
|
-
friend class BufferManager;
|
32
|
-
|
33
|
-
public:
|
34
|
-
explicit BufferPool(idx_t maximum_memory);
|
35
|
-
virtual ~BufferPool();
|
36
|
-
|
37
|
-
//! Set a new memory limit to the buffer pool, throws an exception if the new limit is too low and not enough
|
38
|
-
//! blocks can be evicted
|
39
|
-
void SetLimit(idx_t limit, const char *exception_postscript);
|
40
|
-
|
41
|
-
idx_t GetUsedMemory() {
|
42
|
-
return current_memory;
|
43
|
-
}
|
44
|
-
idx_t GetMaxMemory() {
|
45
|
-
return maximum_memory;
|
46
|
-
}
|
47
|
-
|
48
|
-
protected:
|
49
|
-
//! Evict blocks until the currently used memory + extra_memory fit, returns false if this was not possible
|
50
|
-
//! (i.e. not enough blocks could be evicted)
|
51
|
-
//! If the "buffer" argument is specified AND the system can find a buffer to re-use for the given allocation size
|
52
|
-
//! "buffer" will be made to point to the re-usable memory. Note that this is not guaranteed.
|
53
|
-
//! Returns a pair. result.first indicates if eviction was successful. result.second contains the
|
54
|
-
//! reservation handle, which can be moved to the BlockHandle that will own the reservation.
|
55
|
-
struct EvictionResult {
|
56
|
-
bool success;
|
57
|
-
TempBufferPoolReservation reservation;
|
58
|
-
};
|
59
|
-
virtual EvictionResult EvictBlocks(idx_t extra_memory, idx_t memory_limit,
|
60
|
-
unique_ptr<FileBuffer> *buffer = nullptr);
|
61
|
-
|
62
|
-
//! Garbage collect eviction queue
|
63
|
-
void PurgeQueue();
|
64
|
-
void AddToEvictionQueue(shared_ptr<BlockHandle> &handle);
|
65
|
-
|
66
|
-
private:
|
67
|
-
//! The lock for changing the memory limit
|
68
|
-
mutex limit_lock;
|
69
|
-
//! The current amount of memory that is occupied by the buffer manager (in bytes)
|
70
|
-
atomic<idx_t> current_memory;
|
71
|
-
//! The maximum amount of memory that the buffer manager can keep (in bytes)
|
72
|
-
atomic<idx_t> maximum_memory;
|
73
|
-
//! Eviction queue
|
74
|
-
unique_ptr<EvictionQueue> queue;
|
75
|
-
//! Total number of insertions into the eviction queue. This guides the schedule for calling PurgeQueue.
|
76
|
-
atomic<uint32_t> queue_insertions;
|
77
|
-
};
|
78
|
-
|
79
|
-
struct TemporaryFileInformation {
|
80
|
-
string path;
|
81
|
-
idx_t size;
|
82
|
-
};
|
83
|
-
|
84
|
-
//! The BufferManager is in charge of handling memory management for a singke database. It cooperatively shares a
|
28
|
+
//! The BufferManager is in charge of handling memory management for a single database. It cooperatively shares a
|
85
29
|
//! BufferPool with other BufferManagers, belonging to different databases. It hands out memory buffers that can
|
86
30
|
//! be used by the database internally, and offers configuration options specific to a database, which need not be
|
87
31
|
//! shared by the BufferPool, including whether to support swapping temp buffers to disk, and where to swap them to.
|
@@ -39,6 +39,9 @@ const vector<string> ExtensionHelper::PathComponents() {
|
|
39
39
|
}
|
40
40
|
|
41
41
|
string ExtensionHelper::ExtensionDirectory(DBConfig &config, FileSystem &fs, FileOpener *opener) {
|
42
|
+
#ifdef WASM_LOADABLE_EXTENSIONS
|
43
|
+
static_assertion(0, "ExtensionDirectory functionality is not supported in duckdb-wasm");
|
44
|
+
#endif
|
42
45
|
string extension_directory;
|
43
46
|
if (!config.options.extension_directory.empty()) { // create the extension directory if not present
|
44
47
|
extension_directory = config.options.extension_directory;
|
@@ -111,11 +114,19 @@ bool ExtensionHelper::CreateSuggestions(const string &extension_name, string &me
|
|
111
114
|
}
|
112
115
|
|
113
116
|
void ExtensionHelper::InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install) {
|
117
|
+
#ifdef WASM_LOADABLE_EXTENSIONS
|
118
|
+
// Install is currently a no-op
|
119
|
+
return;
|
120
|
+
#endif
|
114
121
|
string local_path = ExtensionDirectory(config, fs, nullptr);
|
115
122
|
InstallExtensionInternal(config, nullptr, fs, local_path, extension, force_install);
|
116
123
|
}
|
117
124
|
|
118
125
|
void ExtensionHelper::InstallExtension(ClientContext &context, const string &extension, bool force_install) {
|
126
|
+
#ifdef WASM_LOADABLE_EXTENSIONS
|
127
|
+
// Install is currently a no-op
|
128
|
+
return;
|
129
|
+
#endif
|
119
130
|
auto &config = DBConfig::GetConfig(context);
|
120
131
|
auto &fs = FileSystem::GetFileSystem(context);
|
121
132
|
string local_path = ExtensionDirectory(context);
|
@@ -4,6 +4,10 @@
|
|
4
4
|
#include "duckdb/main/error_manager.hpp"
|
5
5
|
#include "mbedtls_wrapper.hpp"
|
6
6
|
|
7
|
+
#ifdef WASM_LOADABLE_EXTENSIONS
|
8
|
+
#include <emscripten.h>
|
9
|
+
#endif
|
10
|
+
|
7
11
|
namespace duckdb {
|
8
12
|
|
9
13
|
//===--------------------------------------------------------------------===//
|
@@ -86,13 +90,35 @@ bool ExtensionHelper::TryInitialLoad(DBConfig &config, FileOpener *opener, const
|
|
86
90
|
throw IOException(config.error_manager->FormatException(ErrorType::UNSIGNED_EXTENSION, filename));
|
87
91
|
}
|
88
92
|
}
|
89
|
-
auto
|
93
|
+
auto basename = fs.ExtractBaseName(filename);
|
94
|
+
|
95
|
+
#ifdef WASM_LOADABLE_EXTENSIONS
|
96
|
+
EM_ASM(
|
97
|
+
{
|
98
|
+
// Next few lines should argubly in separate JavaScript-land function call
|
99
|
+
// TODO: move them out / have them configurable
|
100
|
+
const xhr = new XMLHttpRequest();
|
101
|
+
xhr.open("GET", UTF8ToString($0), false);
|
102
|
+
xhr.responseType = "arraybuffer";
|
103
|
+
xhr.send(null);
|
104
|
+
var uInt8Array = xhr.response;
|
105
|
+
WebAssembly.validate(uInt8Array);
|
106
|
+
console.log('Loading extension ', UTF8ToString($1));
|
107
|
+
|
108
|
+
// Here we add the uInt8Array to Emscripten's filesystem, for it to be found by dlopen
|
109
|
+
FS.writeFile(UTF8ToString($1), new Uint8Array(uInt8Array));
|
110
|
+
},
|
111
|
+
filename.c_str(), basename.c_str());
|
112
|
+
auto dopen_from = basename;
|
113
|
+
#else
|
114
|
+
auto dopen_from = filename;
|
115
|
+
#endif
|
116
|
+
|
117
|
+
auto lib_hdl = dlopen(dopen_from.c_str(), RTLD_NOW | RTLD_LOCAL);
|
90
118
|
if (!lib_hdl) {
|
91
119
|
throw IOException("Extension \"%s\" could not be loaded: %s", filename, GetDLError());
|
92
120
|
}
|
93
121
|
|
94
|
-
auto basename = fs.ExtractBaseName(filename);
|
95
|
-
|
96
122
|
ext_version_fun_t version_fun;
|
97
123
|
auto version_fun_name = basename + "_version";
|
98
124
|
|
@@ -690,7 +690,7 @@ ExpressionExecutorInfo::ExpressionExecutorInfo(ExpressionExecutor &executor, con
|
|
690
690
|
ExpressionRootInfo::ExpressionRootInfo(ExpressionExecutorState &state, string name)
|
691
691
|
: current_count(state.profiler.current_count), sample_count(state.profiler.sample_count),
|
692
692
|
sample_tuples_count(state.profiler.sample_tuples_count), tuples_count(state.profiler.tuples_count),
|
693
|
-
name(
|
693
|
+
name("expression"), time(state.profiler.time) {
|
694
694
|
// Use the name of expression-tree as extra-info
|
695
695
|
extra_info = std::move(name);
|
696
696
|
auto expression_info_p = make_uniq<ExpressionInfo>();
|
@@ -32,8 +32,9 @@ void PragmaHandler::HandlePragmaStatementsInternal(vector<unique_ptr<SQLStatemen
|
|
32
32
|
if (statements[i]->type == StatementType::PRAGMA_STATEMENT) {
|
33
33
|
// PRAGMA statement: check if we need to replace it by a new set of statements
|
34
34
|
PragmaHandler handler(context);
|
35
|
-
|
36
|
-
|
35
|
+
string new_query;
|
36
|
+
bool expanded = handler.HandlePragma(statements[i].get(), new_query);
|
37
|
+
if (expanded) {
|
37
38
|
// this PRAGMA statement gets replaced by a new query string
|
38
39
|
// push the new query string through the parser again and add it to the transformer
|
39
40
|
Parser parser(context.GetParserOptions());
|
@@ -67,7 +68,7 @@ void PragmaHandler::HandlePragmaStatements(ClientContextLock &lock, vector<uniqu
|
|
67
68
|
context.RunFunctionInTransactionInternal(lock, [&]() { HandlePragmaStatementsInternal(statements); });
|
68
69
|
}
|
69
70
|
|
70
|
-
|
71
|
+
bool PragmaHandler::HandlePragma(SQLStatement *statement, string &resulting_query) { // PragmaInfo &info
|
71
72
|
auto info = *(statement->Cast<PragmaStatement>()).info;
|
72
73
|
auto entry =
|
73
74
|
Catalog::GetEntry<PragmaFunctionCatalogEntry>(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name, false);
|
@@ -84,9 +85,10 @@ string PragmaHandler::HandlePragma(SQLStatement *statement) { // PragmaInfo &inf
|
|
84
85
|
Binder::BindNamedParameters(bound_function.named_parameters, info.named_parameters, error_context,
|
85
86
|
bound_function.name);
|
86
87
|
FunctionParameters parameters {info.parameters, info.named_parameters};
|
87
|
-
|
88
|
+
resulting_query = bound_function.query(context, parameters);
|
89
|
+
return true;
|
88
90
|
}
|
89
|
-
return
|
91
|
+
return false;
|
90
92
|
}
|
91
93
|
|
92
94
|
} // namespace duckdb
|