@feltdb/core 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands.js +4 -1
- package/dist/cli/provisioning-neutrality.js +79 -0
- package/dist/collection.d.ts +43 -1
- package/dist/collection.d.ts.map +1 -1
- package/dist/collection.js +192 -22
- package/dist/create/create.js +25 -21
- package/dist/create/managed-account.js +11 -0
- package/dist/create/package-versions.js +1 -1
- package/dist/create/server-source/crates/feltdb/src/equality_index.rs +595 -0
- package/dist/create/server-source/crates/feltdb/src/lib.rs +547 -115
- package/dist/create/server-source/crates/feltdb/src/phase1c3_acceptance.rs +11 -2
- package/dist/create/server-source/crates/feltdb/src/query_execution_diagnostics.rs +126 -0
- package/dist/create/server-source/crates/feltdb/src/state_contract.rs +292 -2
- package/dist/create/server-source/crates/feltdb/src/sync.rs +12 -0
- package/dist/create/server-source/crates/feltdb/src/workload_diagnostics.rs +443 -0
- package/dist/create/server-source/crates/feltdb/tests/pr34_query_collection.rs +233 -0
- package/dist/create/server-source/crates/feltdb/tests/pr35_equality_index.rs +892 -0
- package/dist/create/server-source/crates/feltdb-server/src/audit.rs +1137 -29
- package/dist/create/server-source/crates/feltdb-server/src/main.rs +474 -28
- package/dist/db.d.ts +33 -34
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +74 -20
- package/dist/deployment.d.ts +30 -0
- package/dist/deployment.d.ts.map +1 -0
- package/dist/deployment.js +130 -0
- package/dist/embedded-transaction.d.ts +22 -4
- package/dist/embedded-transaction.d.ts.map +1 -1
- package/dist/embedded-transaction.js +51 -5
- package/dist/feltdb.d.ts +14 -2
- package/dist/feltdb.d.ts.map +1 -1
- package/dist/file-db.d.ts +8 -15
- package/dist/file-db.d.ts.map +1 -1
- package/dist/file-db.js +234 -130
- package/dist/http-client.d.ts +14 -0
- package/dist/http-client.d.ts.map +1 -1
- package/dist/http-client.js +23 -5
- package/dist/http-db.d.ts +119 -1
- package/dist/http-db.d.ts.map +1 -1
- package/dist/http-db.js +346 -31
- package/dist/index-core.d.ts +2 -0
- package/dist/index-core.d.ts.map +1 -1
- package/dist/index-core.js +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -0
- package/dist/indexeddb-db.d.ts.map +1 -1
- package/dist/indexeddb-db.js +35 -21
- package/dist/managed-recovery.d.ts +192 -0
- package/dist/managed-recovery.d.ts.map +1 -0
- package/dist/managed-recovery.js +242 -0
- package/dist/memory-db.js +1 -1
- package/dist/studio-app/assets/{feltdb_wasm-DB8cX151.js → feltdb_wasm-CVQWgXO-.js} +1 -1
- package/dist/studio-app/assets/feltdb_wasm_bg-CNVpvaZV.wasm +0 -0
- package/dist/studio-app/assets/index-DwgNAIIX.js +29 -0
- package/dist/studio-app/index.html +1 -1
- package/dist/transaction.d.ts +30 -0
- package/dist/transaction.d.ts.map +1 -1
- package/dist/transaction.js +41 -0
- package/dist/wasm/feltdb_wasm_bg.wasm +0 -0
- package/package.json +1 -1
- package/dist/studio-app/assets/feltdb_wasm_bg-ClhDHp0S.wasm +0 -0
- package/dist/studio-app/assets/index-B0k4UAlI.js +0 -29
|
@@ -0,0 +1,443 @@
|
|
|
1
|
+
//! Phase-level execution attribution for the sustained workload.
|
|
2
|
+
//!
|
|
3
|
+
//! PR33 named query execution as the sustained-workload bottleneck. PR34
|
|
4
|
+
//! removed whole-collection materialization and PR35 removed the full predicate
|
|
5
|
+
//! scan, taking the workload from 274.1 to 1,665.9 ops/sec on one runner. PR36
|
|
6
|
+
//! asks what is left, and that question cannot be answered from throughput:
|
|
7
|
+
//! an aggregate rate says how fast the system is, never where the time went.
|
|
8
|
+
//!
|
|
9
|
+
//! So this module records **where the time went**, as elapsed nanoseconds and a
|
|
10
|
+
//! call count per execution phase. Three properties keep it honest:
|
|
11
|
+
//!
|
|
12
|
+
//! 1. **The boundaries are real.** Every phase brackets an actual span of
|
|
13
|
+
//! execution — the wait for the state lock, the index lookup, the record
|
|
14
|
+
//! fetch, the predicate, the sort, the log append. Nothing here is a
|
|
15
|
+
//! proportion inferred from wall-clock time, and a phase that cannot be
|
|
16
|
+
//! bracketed cleanly is simply absent rather than estimated.
|
|
17
|
+
//! 2. **Phases nest, and the report says so.** `STATE_LOCK_HOLD` contains
|
|
18
|
+
//! `RECORD_MUTATION`, `INDEX_MAINTENANCE` and `PERSISTENCE`; `INDEXED_QUERY`
|
|
19
|
+
//! contains `INDEX_LOOKUP`, `CANDIDATE_RETRIEVAL` and `PREDICATE_EVALUATION`.
|
|
20
|
+
//! Summing every phase would double-count, which is why [`Phase::parent`]
|
|
21
|
+
//! exists and why the analysis distinguishes exclusive time from nested time.
|
|
22
|
+
//! 3. **It is off unless asked for.** Disabled, each instrumented site costs one
|
|
23
|
+
//! relaxed atomic load and a predictable branch; no clock is read and no
|
|
24
|
+
//! counter is touched. It is enabled by `FELTDB_WORKLOAD_DIAGNOSTICS=1`, is
|
|
25
|
+
//! test instrumentation rather than a product surface, and reports counts and
|
|
26
|
+
//! durations only — never a record, a field, a value or an id.
|
|
27
|
+
//!
|
|
28
|
+
//! Instrumentation that changes what it measures is not a measurement, so the
|
|
29
|
+
//! overhead of *enabled* diagnostics is itself measured, and the headline
|
|
30
|
+
//! production figures are taken with diagnostics off.
|
|
31
|
+
|
|
32
|
+
use std::sync::atomic::{AtomicU64, Ordering};
|
|
33
|
+
use std::time::Instant;
|
|
34
|
+
|
|
35
|
+
/// One bracketed span of execution.
|
|
36
|
+
///
|
|
37
|
+
/// Ordered so that a report reads top-down: request, then query phases, then
|
|
38
|
+
/// mutation phases. The discriminant is the counter index, so adding a phase
|
|
39
|
+
/// means adding it here and to [`Phase::ALL`] and nothing else.
|
|
40
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
|
41
|
+
#[repr(usize)]
|
|
42
|
+
pub enum Phase {
|
|
43
|
+
/// Waiting to acquire the single state lock. Pure contention: no work is
|
|
44
|
+
/// done here, which is what makes it the one phase whose time is always
|
|
45
|
+
/// wasted rather than merely spent.
|
|
46
|
+
StateLockWait = 0,
|
|
47
|
+
/// Holding the state lock. Contains every phase below that touches state,
|
|
48
|
+
/// and bounds how long other operations can be made to wait.
|
|
49
|
+
StateLockHold,
|
|
50
|
+
/// A bounded query answered from the equality index, end to end.
|
|
51
|
+
IndexedQuery,
|
|
52
|
+
/// Turning `(field, value)` into candidate record ids inside the index.
|
|
53
|
+
IndexLookup,
|
|
54
|
+
/// Fetching authoritative records for candidate ids.
|
|
55
|
+
CandidateRetrieval,
|
|
56
|
+
/// Evaluating the query's conjunction against a record, on either execution.
|
|
57
|
+
PredicateEvaluation,
|
|
58
|
+
/// A bounded query answered by scanning the collection, end to end.
|
|
59
|
+
ScanQuery,
|
|
60
|
+
/// Sorting the matching set into the query's requested order.
|
|
61
|
+
Ordering,
|
|
62
|
+
/// Cloning matches and injecting `recordId` to build the visible result.
|
|
63
|
+
ResultMaterialization,
|
|
64
|
+
/// Writing or removing a record in the authoritative map.
|
|
65
|
+
RecordMutation,
|
|
66
|
+
/// Maintaining the derived equality index alongside that record change.
|
|
67
|
+
IndexMaintenance,
|
|
68
|
+
/// Appending to the durable log, including any fsync it performs.
|
|
69
|
+
Persistence,
|
|
70
|
+
/// Transaction bookkeeping around a commit: preconditions, operation
|
|
71
|
+
/// construction, revision and dedup accounting. Excludes the log append.
|
|
72
|
+
TransactionCommit,
|
|
73
|
+
/// A whole HTTP handler, measured inside the server between extraction and
|
|
74
|
+
/// response. The gap between this and the client's own timing is transport,
|
|
75
|
+
/// decode and encode.
|
|
76
|
+
HttpHandler,
|
|
77
|
+
/// Authenticating and authorizing a request: key lookup, signature and
|
|
78
|
+
/// capability checks. Nested inside `HttpHandler`, and separated from it
|
|
79
|
+
/// because "the request costs something outside state" is not an answer —
|
|
80
|
+
/// which part of the request is.
|
|
81
|
+
Authorization,
|
|
82
|
+
/// The request's whole dealing with the security audit subsystem: building
|
|
83
|
+
/// the event, submitting it to the audit writer, and waiting for the
|
|
84
|
+
/// writer to accept it. Nested inside `HttpHandler` and deliberately not
|
|
85
|
+
/// folded into `Persistence`, which is the *state* log — these are two
|
|
86
|
+
/// different durable writes with two different reasons to exist, and
|
|
87
|
+
/// attributing them together would hide which one costs what.
|
|
88
|
+
///
|
|
89
|
+
/// Since PR38 this span no longer contains a durability barrier. The
|
|
90
|
+
/// barrier happens on the audit writer's own threads, in
|
|
91
|
+
/// `AuditGroupSync`, after the request has already been released.
|
|
92
|
+
AuditWrite,
|
|
93
|
+
/// Handing the event to the audit writer's bounded queue. Fails fast when
|
|
94
|
+
/// the queue is at capacity rather than blocking the request.
|
|
95
|
+
AuditSubmit,
|
|
96
|
+
/// Waiting for the audit writer to append the submitted event to the
|
|
97
|
+
/// stream — the acceptance boundary, and the only part of the audit
|
|
98
|
+
/// pipeline a request waits for.
|
|
99
|
+
///
|
|
100
|
+
/// This is the audit path's analogue of `StateLockWait`, and it is where
|
|
101
|
+
/// the single-writer discipline shows up as time: a request waits behind
|
|
102
|
+
/// the appends in front of it, and behind whatever the writer is doing
|
|
103
|
+
/// when it arrives.
|
|
104
|
+
AuditAcceptWait,
|
|
105
|
+
/// Opening the audit file, on the writer thread. Since PR38 the writer
|
|
106
|
+
/// keeps the handle, so this happens once per stream rather than once per
|
|
107
|
+
/// event; it is still measured, because "once" is a claim.
|
|
108
|
+
AuditOpen,
|
|
109
|
+
/// Serializing one event into the writer's reusable buffer and issuing the
|
|
110
|
+
/// single `write(2)` that appends it. On the writer thread.
|
|
111
|
+
AuditAppend,
|
|
112
|
+
/// `File::flush` on the audit handle, inside a durability group. A
|
|
113
|
+
/// `std::fs::File` holds no userspace buffer, so this is measured rather
|
|
114
|
+
/// than assumed to be free.
|
|
115
|
+
AuditFlush,
|
|
116
|
+
/// `File::sync_data` on the audit handle: the durability barrier itself,
|
|
117
|
+
/// and the one phase that asks the storage stack for a guarantee rather
|
|
118
|
+
/// than for a write. On the syncer thread, off the request path.
|
|
119
|
+
AuditFsync,
|
|
120
|
+
/// One whole durability group: the flush and the barrier that make a
|
|
121
|
+
/// bounded set of already-accepted events durable. On the syncer thread.
|
|
122
|
+
AuditGroupSync,
|
|
123
|
+
/// Closing the audit file handle at shutdown. Measured because it is a
|
|
124
|
+
/// syscall, and because the point of the writer owning the handle is that
|
|
125
|
+
/// this no longer happens per event.
|
|
126
|
+
AuditClose,
|
|
127
|
+
/// The body of a request handler, from the first line of the handler
|
|
128
|
+
/// function to its return.
|
|
129
|
+
///
|
|
130
|
+
/// It exists to bracket what a middleware cannot. `HttpHandler` wraps the
|
|
131
|
+
/// whole route, so it includes axum's extraction — where the request JSON is
|
|
132
|
+
/// decoded — and the response conversion, where the response JSON is
|
|
133
|
+
/// encoded. Neither can be timed from inside a handler or from a middleware
|
|
134
|
+
/// alone. The difference `HttpHandler - HandlerBody - Authorization -
|
|
135
|
+
/// AuditWrite` is therefore routing plus decode plus encode, measured as a
|
|
136
|
+
/// difference of two spans rather than asserted.
|
|
137
|
+
HandlerBody,
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
impl Phase {
|
|
141
|
+
pub const ALL: [Phase; 25] = [
|
|
142
|
+
Phase::StateLockWait,
|
|
143
|
+
Phase::StateLockHold,
|
|
144
|
+
Phase::IndexedQuery,
|
|
145
|
+
Phase::IndexLookup,
|
|
146
|
+
Phase::CandidateRetrieval,
|
|
147
|
+
Phase::PredicateEvaluation,
|
|
148
|
+
Phase::ScanQuery,
|
|
149
|
+
Phase::Ordering,
|
|
150
|
+
Phase::ResultMaterialization,
|
|
151
|
+
Phase::RecordMutation,
|
|
152
|
+
Phase::IndexMaintenance,
|
|
153
|
+
Phase::Persistence,
|
|
154
|
+
Phase::TransactionCommit,
|
|
155
|
+
Phase::HttpHandler,
|
|
156
|
+
Phase::Authorization,
|
|
157
|
+
Phase::AuditWrite,
|
|
158
|
+
Phase::AuditSubmit,
|
|
159
|
+
Phase::AuditAcceptWait,
|
|
160
|
+
Phase::AuditOpen,
|
|
161
|
+
Phase::AuditAppend,
|
|
162
|
+
Phase::AuditFlush,
|
|
163
|
+
Phase::AuditFsync,
|
|
164
|
+
Phase::AuditGroupSync,
|
|
165
|
+
Phase::AuditClose,
|
|
166
|
+
Phase::HandlerBody,
|
|
167
|
+
];
|
|
168
|
+
|
|
169
|
+
/// The stable name this phase is reported under.
|
|
170
|
+
pub fn name(self) -> &'static str {
|
|
171
|
+
match self {
|
|
172
|
+
Phase::StateLockWait => "state_lock_wait",
|
|
173
|
+
Phase::StateLockHold => "state_lock_hold",
|
|
174
|
+
Phase::IndexedQuery => "indexed_query",
|
|
175
|
+
Phase::IndexLookup => "index_lookup",
|
|
176
|
+
Phase::CandidateRetrieval => "candidate_retrieval",
|
|
177
|
+
Phase::PredicateEvaluation => "predicate_evaluation",
|
|
178
|
+
Phase::ScanQuery => "scan_query",
|
|
179
|
+
Phase::Ordering => "ordering",
|
|
180
|
+
Phase::ResultMaterialization => "result_materialization",
|
|
181
|
+
Phase::RecordMutation => "record_mutation",
|
|
182
|
+
Phase::IndexMaintenance => "index_maintenance",
|
|
183
|
+
Phase::Persistence => "persistence",
|
|
184
|
+
Phase::TransactionCommit => "transaction_commit",
|
|
185
|
+
Phase::HttpHandler => "http_handler",
|
|
186
|
+
Phase::Authorization => "authorization",
|
|
187
|
+
Phase::AuditWrite => "audit_write",
|
|
188
|
+
Phase::AuditSubmit => "audit_submit",
|
|
189
|
+
Phase::AuditAcceptWait => "audit_accept_wait",
|
|
190
|
+
Phase::AuditOpen => "audit_open",
|
|
191
|
+
Phase::AuditAppend => "audit_append",
|
|
192
|
+
Phase::AuditFlush => "audit_flush",
|
|
193
|
+
Phase::AuditFsync => "audit_fsync",
|
|
194
|
+
Phase::AuditGroupSync => "audit_group_sync",
|
|
195
|
+
Phase::AuditClose => "audit_close",
|
|
196
|
+
Phase::HandlerBody => "handler_body",
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/// The phase whose span contains this one, when there is one.
|
|
201
|
+
///
|
|
202
|
+
/// This is what stops a reader from adding the columns up. A phase's time is
|
|
203
|
+
/// already counted inside its parent, so only sibling phases at one level
|
|
204
|
+
/// can be compared, and only exclusive time can be summed.
|
|
205
|
+
pub fn parent(self) -> Option<Phase> {
|
|
206
|
+
match self {
|
|
207
|
+
Phase::StateLockWait | Phase::StateLockHold | Phase::HttpHandler => None,
|
|
208
|
+
Phase::IndexedQuery | Phase::ScanQuery => Some(Phase::StateLockHold),
|
|
209
|
+
Phase::IndexLookup | Phase::CandidateRetrieval => Some(Phase::IndexedQuery),
|
|
210
|
+
// The predicate runs on both executions, so its parent is the lock
|
|
211
|
+
// rather than either query phase.
|
|
212
|
+
Phase::PredicateEvaluation => Some(Phase::StateLockHold),
|
|
213
|
+
Phase::Authorization | Phase::AuditWrite | Phase::HandlerBody => {
|
|
214
|
+
Some(Phase::HttpHandler)
|
|
215
|
+
}
|
|
216
|
+
Phase::Ordering | Phase::ResultMaterialization => Some(Phase::HandlerBody),
|
|
217
|
+
Phase::RecordMutation | Phase::IndexMaintenance | Phase::Persistence => {
|
|
218
|
+
Some(Phase::StateLockHold)
|
|
219
|
+
}
|
|
220
|
+
Phase::TransactionCommit => Some(Phase::StateLockHold),
|
|
221
|
+
// What the request pays decomposes into the submit and the wait for
|
|
222
|
+
// acceptance, and nothing else: since PR38 no other audit phase
|
|
223
|
+
// happens on the request's thread.
|
|
224
|
+
Phase::AuditSubmit | Phase::AuditAcceptWait => Some(Phase::AuditWrite),
|
|
225
|
+
// The flush and the barrier are the two halves of one durability
|
|
226
|
+
// group.
|
|
227
|
+
Phase::AuditFlush | Phase::AuditFsync => Some(Phase::AuditGroupSync),
|
|
228
|
+
// Deliberately parentless. `AuditOpen`, `AuditAppend`,
|
|
229
|
+
// `AuditGroupSync` and `AuditClose` run on the audit writer's own
|
|
230
|
+
// threads, concurrently with the requests that produced them.
|
|
231
|
+
// Nesting them under `AuditWrite` or `HttpHandler` would let a
|
|
232
|
+
// reader compute a share of a span they do not run inside, and
|
|
233
|
+
// that share could exceed one hundred per cent.
|
|
234
|
+
Phase::AuditOpen
|
|
235
|
+
| Phase::AuditAppend
|
|
236
|
+
| Phase::AuditGroupSync
|
|
237
|
+
| Phase::AuditClose => None,
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const PHASES: usize = Phase::ALL.len();
|
|
243
|
+
|
|
244
|
+
#[allow(clippy::declare_interior_mutable_const)]
|
|
245
|
+
const ZERO: AtomicU64 = AtomicU64::new(0);
|
|
246
|
+
static CALLS: [AtomicU64; PHASES] = [ZERO; PHASES];
|
|
247
|
+
static NANOS: [AtomicU64; PHASES] = [ZERO; PHASES];
|
|
248
|
+
static ENABLED: AtomicU64 = AtomicU64::new(UNRESOLVED);
|
|
249
|
+
|
|
250
|
+
const UNRESOLVED: u64 = 0;
|
|
251
|
+
const OFF: u64 = 1;
|
|
252
|
+
const ON: u64 = 2;
|
|
253
|
+
|
|
254
|
+
/// Is phase attribution recording?
|
|
255
|
+
///
|
|
256
|
+
/// Resolved once from `FELTDB_WORKLOAD_DIAGNOSTICS`, then a relaxed load. The
|
|
257
|
+
/// environment is read at most once per process, so a hot path never touches the
|
|
258
|
+
/// environment and never allocates.
|
|
259
|
+
#[inline]
|
|
260
|
+
pub fn enabled() -> bool {
|
|
261
|
+
match ENABLED.load(Ordering::Relaxed) {
|
|
262
|
+
ON => true,
|
|
263
|
+
OFF => false,
|
|
264
|
+
_ => {
|
|
265
|
+
let resolved = if std::env::var("FELTDB_WORKLOAD_DIAGNOSTICS").as_deref() == Ok("1") {
|
|
266
|
+
ON
|
|
267
|
+
} else {
|
|
268
|
+
OFF
|
|
269
|
+
};
|
|
270
|
+
ENABLED.store(resolved, Ordering::Relaxed);
|
|
271
|
+
resolved == ON
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/// Turn recording on or off for the current process, for tests that need to
|
|
277
|
+
/// measure the instrumentation itself rather than through it.
|
|
278
|
+
pub fn set_enabled(on: bool) {
|
|
279
|
+
ENABLED.store(if on { ON } else { OFF }, Ordering::Relaxed);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/// An open span. Dropping it records the elapsed time against its phase.
|
|
283
|
+
///
|
|
284
|
+
/// Constructed only when recording is on, so the disabled path allocates
|
|
285
|
+
/// nothing and never reads the clock.
|
|
286
|
+
#[derive(Debug)]
|
|
287
|
+
pub struct Span {
|
|
288
|
+
phase: Phase,
|
|
289
|
+
began: Instant,
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
impl Drop for Span {
|
|
293
|
+
fn drop(&mut self) {
|
|
294
|
+
let elapsed = self.began.elapsed().as_nanos().min(u64::MAX as u128) as u64;
|
|
295
|
+
let index = self.phase as usize;
|
|
296
|
+
CALLS[index].fetch_add(1, Ordering::Relaxed);
|
|
297
|
+
NANOS[index].fetch_add(elapsed, Ordering::Relaxed);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/// Open a span, or `None` when recording is off.
|
|
302
|
+
///
|
|
303
|
+
/// The `Option` is the whole disabled-path cost: no clock read, no atomic
|
|
304
|
+
/// write, and a branch the predictor gets right every time.
|
|
305
|
+
#[inline]
|
|
306
|
+
pub fn span(phase: Phase) -> Option<Span> {
|
|
307
|
+
enabled().then(|| Span {
|
|
308
|
+
phase,
|
|
309
|
+
began: Instant::now(),
|
|
310
|
+
})
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/// Record a duration measured elsewhere, such as a lock wait that has to be
|
|
314
|
+
/// timed across an acquisition rather than around a closure.
|
|
315
|
+
#[inline]
|
|
316
|
+
pub fn record(phase: Phase, nanos: u64) {
|
|
317
|
+
if !enabled() {
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
let index = phase as usize;
|
|
321
|
+
CALLS[index].fetch_add(1, Ordering::Relaxed);
|
|
322
|
+
NANOS[index].fetch_add(nanos, Ordering::Relaxed);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
/// Time one closure into a phase and return its value.
|
|
326
|
+
#[inline]
|
|
327
|
+
pub fn timed<T>(phase: Phase, body: impl FnOnce() -> T) -> T {
|
|
328
|
+
let _span = span(phase);
|
|
329
|
+
body()
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/// What one phase accumulated: how often, and for how long.
|
|
333
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
334
|
+
pub struct PhaseCounter {
|
|
335
|
+
pub phase: Phase,
|
|
336
|
+
pub calls: u64,
|
|
337
|
+
pub nanos: u64,
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
impl PhaseCounter {
|
|
341
|
+
pub fn mean_nanos(&self) -> u64 {
|
|
342
|
+
self.nanos.checked_div(self.calls).unwrap_or(0)
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/// Read every phase counter. Monotonic since process start or the last
|
|
347
|
+
/// [`reset`], so a caller compares deltas across a measured window.
|
|
348
|
+
pub fn counters() -> Vec<PhaseCounter> {
|
|
349
|
+
Phase::ALL
|
|
350
|
+
.iter()
|
|
351
|
+
.map(|&phase| PhaseCounter {
|
|
352
|
+
phase,
|
|
353
|
+
calls: CALLS[phase as usize].load(Ordering::Relaxed),
|
|
354
|
+
nanos: NANOS[phase as usize].load(Ordering::Relaxed),
|
|
355
|
+
})
|
|
356
|
+
.collect()
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/// Zero every counter, so a phase of a measurement starts from a clean base.
|
|
360
|
+
pub fn reset() {
|
|
361
|
+
for index in 0..PHASES {
|
|
362
|
+
CALLS[index].store(0, Ordering::Relaxed);
|
|
363
|
+
NANOS[index].store(0, Ordering::Relaxed);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
#[cfg(test)]
|
|
368
|
+
mod tests {
|
|
369
|
+
use super::*;
|
|
370
|
+
use std::sync::{Mutex, MutexGuard, OnceLock};
|
|
371
|
+
|
|
372
|
+
/// Counters are process-global; a counter assertion is only meaningful
|
|
373
|
+
/// while no other test in this module is recording.
|
|
374
|
+
fn exclusive() -> MutexGuard<'static, ()> {
|
|
375
|
+
static SERIAL: OnceLock<Mutex<()>> = OnceLock::new();
|
|
376
|
+
SERIAL
|
|
377
|
+
.get_or_init(|| Mutex::new(()))
|
|
378
|
+
.lock()
|
|
379
|
+
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
fn counter(phase: Phase) -> PhaseCounter {
|
|
383
|
+
counters()
|
|
384
|
+
.into_iter()
|
|
385
|
+
.find(|counter| counter.phase == phase)
|
|
386
|
+
.expect("every phase is reported")
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
#[test]
|
|
390
|
+
fn a_disabled_span_records_nothing() {
|
|
391
|
+
let _serial = exclusive();
|
|
392
|
+
set_enabled(false);
|
|
393
|
+
reset();
|
|
394
|
+
|
|
395
|
+
for _ in 0..1_000 {
|
|
396
|
+
let _span = span(Phase::Ordering);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
assert_eq!(counter(Phase::Ordering).calls, 0);
|
|
400
|
+
assert!(
|
|
401
|
+
span(Phase::Ordering).is_none(),
|
|
402
|
+
"a disabled span is not constructed, so it reads no clock",
|
|
403
|
+
);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
#[test]
|
|
407
|
+
fn an_enabled_span_records_calls_and_elapsed_time() {
|
|
408
|
+
let _serial = exclusive();
|
|
409
|
+
set_enabled(true);
|
|
410
|
+
reset();
|
|
411
|
+
|
|
412
|
+
for _ in 0..8 {
|
|
413
|
+
timed(Phase::IndexLookup, || std::hint::black_box(0u64));
|
|
414
|
+
}
|
|
415
|
+
let observed = counter(Phase::IndexLookup);
|
|
416
|
+
assert_eq!(observed.calls, 8);
|
|
417
|
+
assert!(observed.nanos > 0, "a span of real work takes real time");
|
|
418
|
+
|
|
419
|
+
set_enabled(false);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
#[test]
|
|
423
|
+
fn every_phase_is_reported_and_nesting_is_declared() {
|
|
424
|
+
let _serial = exclusive();
|
|
425
|
+
let reported: Vec<&str> = counters()
|
|
426
|
+
.into_iter()
|
|
427
|
+
.map(|counter| counter.phase.name())
|
|
428
|
+
.collect();
|
|
429
|
+
assert_eq!(reported.len(), Phase::ALL.len());
|
|
430
|
+
// A phase must not be its own ancestor, or a report could not
|
|
431
|
+
// distinguish exclusive time from nested time.
|
|
432
|
+
for phase in Phase::ALL {
|
|
433
|
+
let mut ancestor = phase.parent();
|
|
434
|
+
let mut depth = 0;
|
|
435
|
+
while let Some(current) = ancestor {
|
|
436
|
+
assert_ne!(current, phase, "{} is its own ancestor", phase.name());
|
|
437
|
+
depth += 1;
|
|
438
|
+
assert!(depth < PHASES, "phase nesting must terminate");
|
|
439
|
+
ancestor = current.parent();
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
//! PR34 — bounded query execution without collection materialization.
|
|
2
|
+
//!
|
|
3
|
+
//! `FeltDb::query_collection` is the state-access primitive the bounded query
|
|
4
|
+
//! surface executes on. These tests pin the two properties the query path
|
|
5
|
+
//! depends on: it observes every live record of a collection in record-key
|
|
6
|
+
//! order, and it clones only the records its predicate retains.
|
|
7
|
+
|
|
8
|
+
use feltdb::{query_execution_diagnostics, FeltDb, StoredRow};
|
|
9
|
+
use serde_json::json;
|
|
10
|
+
use std::sync::{Mutex, MutexGuard, OnceLock};
|
|
11
|
+
use tempfile::TempDir;
|
|
12
|
+
|
|
13
|
+
/// The read counters are process-global, and the test binary runs its tests in
|
|
14
|
+
/// threads of one process, so a counter assertion is only meaningful while no
|
|
15
|
+
/// other test in this file is reading a collection.
|
|
16
|
+
fn exclusive() -> MutexGuard<'static, ()> {
|
|
17
|
+
static SERIAL: OnceLock<Mutex<()>> = OnceLock::new();
|
|
18
|
+
SERIAL
|
|
19
|
+
.get_or_init(|| Mutex::new(()))
|
|
20
|
+
.lock()
|
|
21
|
+
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
fn seeded(records: usize) -> (TempDir, FeltDb) {
|
|
25
|
+
let directory = TempDir::new().expect("temp dir");
|
|
26
|
+
let db = FeltDb::open(directory.path().join("pr34.log")).expect("open db");
|
|
27
|
+
for index in 0..records {
|
|
28
|
+
db.insert(
|
|
29
|
+
&format!("items:record-{index:05}"),
|
|
30
|
+
json!({ "tenant": format!("tenant-{}", index % 4), "n": index }),
|
|
31
|
+
)
|
|
32
|
+
.expect("insert");
|
|
33
|
+
}
|
|
34
|
+
(directory, db)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
#[test]
|
|
38
|
+
fn scan_returns_only_matching_records() {
|
|
39
|
+
let _serial = exclusive();
|
|
40
|
+
let (_directory, db) = seeded(40);
|
|
41
|
+
|
|
42
|
+
let matches = db
|
|
43
|
+
.query_collection("items", None, |row| {
|
|
44
|
+
row.value.get("tenant") == Some(&json!("tenant-1"))
|
|
45
|
+
})
|
|
46
|
+
.expect("scan");
|
|
47
|
+
|
|
48
|
+
assert_eq!(matches.len(), 10, "one quarter of the collection matches");
|
|
49
|
+
assert!(matches
|
|
50
|
+
.iter()
|
|
51
|
+
.all(|row| row.value.get("tenant") == Some(&json!("tenant-1"))));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
#[test]
|
|
55
|
+
fn scan_of_an_empty_or_unknown_collection_is_empty() {
|
|
56
|
+
let _serial = exclusive();
|
|
57
|
+
let (_directory, db) = seeded(0);
|
|
58
|
+
|
|
59
|
+
assert!(db
|
|
60
|
+
.query_collection("items", None, |_| true)
|
|
61
|
+
.expect("scan")
|
|
62
|
+
.is_empty());
|
|
63
|
+
assert!(db
|
|
64
|
+
.query_collection("no-such-collection", None, |_| true)
|
|
65
|
+
.expect("scan")
|
|
66
|
+
.is_empty());
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
#[test]
|
|
70
|
+
fn scan_visits_records_in_record_key_order() {
|
|
71
|
+
let _serial = exclusive();
|
|
72
|
+
let (_directory, db) = seeded(16);
|
|
73
|
+
|
|
74
|
+
let keys: Vec<String> = db
|
|
75
|
+
.query_collection("items", None, |_| true)
|
|
76
|
+
.expect("scan")
|
|
77
|
+
.iter()
|
|
78
|
+
.map(|row: &StoredRow| row.key.clone())
|
|
79
|
+
.collect();
|
|
80
|
+
let mut sorted = keys.clone();
|
|
81
|
+
sorted.sort();
|
|
82
|
+
|
|
83
|
+
assert_eq!(keys, sorted, "traversal follows the collection's key order");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
#[test]
|
|
87
|
+
fn a_limit_stops_the_traversal_once_it_is_satisfied() {
|
|
88
|
+
let _serial = exclusive();
|
|
89
|
+
let (_directory, db) = seeded(500);
|
|
90
|
+
|
|
91
|
+
let mut visited = 0usize;
|
|
92
|
+
let matches = db
|
|
93
|
+
.query_collection("items", Some(5), |_| {
|
|
94
|
+
visited += 1;
|
|
95
|
+
true
|
|
96
|
+
})
|
|
97
|
+
.expect("scan");
|
|
98
|
+
|
|
99
|
+
assert_eq!(matches.len(), 5, "the limit bounds the result");
|
|
100
|
+
assert_eq!(
|
|
101
|
+
visited, 5,
|
|
102
|
+
"a satisfied limit stops the traversal instead of walking the collection"
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
#[test]
|
|
107
|
+
fn a_scan_clones_only_the_records_it_retains() {
|
|
108
|
+
let _serial = exclusive();
|
|
109
|
+
let (_directory, db) = seeded(2_000);
|
|
110
|
+
|
|
111
|
+
let before = query_execution_diagnostics::counters();
|
|
112
|
+
let matches = db
|
|
113
|
+
.query_collection("items", None, |row| row.value.get("n") == Some(&json!(7)))
|
|
114
|
+
.expect("scan");
|
|
115
|
+
let after = query_execution_diagnostics::counters();
|
|
116
|
+
|
|
117
|
+
assert_eq!(matches.len(), 1);
|
|
118
|
+
assert_eq!(
|
|
119
|
+
after.full_collection_materializations, before.full_collection_materializations,
|
|
120
|
+
"a scan must not fall back to full collection materialization"
|
|
121
|
+
);
|
|
122
|
+
assert_eq!(
|
|
123
|
+
after.scan_records_visited - before.scan_records_visited,
|
|
124
|
+
2_000,
|
|
125
|
+
"the predicate observes every record, borrowed"
|
|
126
|
+
);
|
|
127
|
+
assert_eq!(
|
|
128
|
+
after.scan_records_materialized - before.scan_records_materialized,
|
|
129
|
+
1,
|
|
130
|
+
"only the matching record is cloned"
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
#[test]
|
|
135
|
+
fn full_materialization_is_still_counted_for_readers_that_ask_for_it() {
|
|
136
|
+
let _serial = exclusive();
|
|
137
|
+
let (_directory, db) = seeded(32);
|
|
138
|
+
|
|
139
|
+
let before = query_execution_diagnostics::counters();
|
|
140
|
+
let rows = db.list_collection("items").expect("list");
|
|
141
|
+
let after = query_execution_diagnostics::counters();
|
|
142
|
+
|
|
143
|
+
assert_eq!(rows.len(), 32);
|
|
144
|
+
assert_eq!(
|
|
145
|
+
after.full_collection_materializations - before.full_collection_materializations,
|
|
146
|
+
1
|
|
147
|
+
);
|
|
148
|
+
assert_eq!(
|
|
149
|
+
after.full_collection_records_cloned - before.full_collection_records_cloned,
|
|
150
|
+
32,
|
|
151
|
+
"listing a collection clones every record in it"
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
#[test]
|
|
156
|
+
fn a_scan_result_survives_concurrent_mutation() {
|
|
157
|
+
let _serial = exclusive();
|
|
158
|
+
let (_directory, db) = seeded(200);
|
|
159
|
+
|
|
160
|
+
let matches = db
|
|
161
|
+
.query_collection("items", None, |row| {
|
|
162
|
+
row.value.get("tenant") == Some(&json!("tenant-0"))
|
|
163
|
+
})
|
|
164
|
+
.expect("scan");
|
|
165
|
+
|
|
166
|
+
// The scan returns owned records, so mutation after the lock is released
|
|
167
|
+
// cannot invalidate or rewrite what the caller already holds.
|
|
168
|
+
for index in 0..200 {
|
|
169
|
+
db.delete(&format!("items:record-{index:05}")).expect("delete");
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
assert_eq!(matches.len(), 50);
|
|
173
|
+
assert!(matches
|
|
174
|
+
.iter()
|
|
175
|
+
.all(|row| row.value.get("tenant") == Some(&json!("tenant-0"))));
|
|
176
|
+
assert!(db
|
|
177
|
+
.query_collection("items", None, |_| true)
|
|
178
|
+
.expect("scan")
|
|
179
|
+
.is_empty());
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/// PR35 addition: the indexed execution must not reach the materializing path.
|
|
183
|
+
///
|
|
184
|
+
/// PR34's claim was that a bounded query never enters `list_collection`. PR35
|
|
185
|
+
/// adds a second execution, and an optimization that quietly fell back into a
|
|
186
|
+
/// whole-collection clone would satisfy every correctness test in this
|
|
187
|
+
/// repository while undoing the thing PR34 established. So the property is
|
|
188
|
+
/// re-asserted against the execution that did not exist when it was first
|
|
189
|
+
/// proved: indexed execution materializes nothing, and does not scan either.
|
|
190
|
+
#[test]
|
|
191
|
+
fn an_indexed_query_neither_materializes_nor_scans_the_collection() {
|
|
192
|
+
let _serial = exclusive();
|
|
193
|
+
let (_directory, db) = seeded(400);
|
|
194
|
+
db.create_equality_index("items", "tenant")
|
|
195
|
+
.expect("declare index");
|
|
196
|
+
|
|
197
|
+
let before = query_execution_diagnostics::counters();
|
|
198
|
+
let tenant = json!("tenant-1");
|
|
199
|
+
let matches = db
|
|
200
|
+
.query_collection_by_equality("items", &[("tenant", &tenant)], |row| {
|
|
201
|
+
row.value.get("tenant") == Some(&json!("tenant-1"))
|
|
202
|
+
})
|
|
203
|
+
.expect("indexed query")
|
|
204
|
+
.expect("the field is indexed");
|
|
205
|
+
let after = query_execution_diagnostics::counters();
|
|
206
|
+
|
|
207
|
+
assert_eq!(matches.len(), 100, "one quarter of the collection matches");
|
|
208
|
+
assert_eq!(
|
|
209
|
+
after.full_collection_materializations - before.full_collection_materializations,
|
|
210
|
+
0,
|
|
211
|
+
"indexed execution must not enter the full-collection materialization path",
|
|
212
|
+
);
|
|
213
|
+
assert_eq!(
|
|
214
|
+
after.collection_scans - before.collection_scans,
|
|
215
|
+
0,
|
|
216
|
+
"indexed execution is not a scan",
|
|
217
|
+
);
|
|
218
|
+
assert_eq!(
|
|
219
|
+
after.scan_records_visited - before.scan_records_visited,
|
|
220
|
+
0,
|
|
221
|
+
"and therefore visits no record by traversal",
|
|
222
|
+
);
|
|
223
|
+
assert_eq!(
|
|
224
|
+
after.queries_indexed - before.queries_indexed,
|
|
225
|
+
1,
|
|
226
|
+
"the query is attributed to the indexed execution",
|
|
227
|
+
);
|
|
228
|
+
assert_eq!(
|
|
229
|
+
after.records_predicate_evaluated - before.records_predicate_evaluated,
|
|
230
|
+
100,
|
|
231
|
+
"the predicate runs against the candidate set, not the collection",
|
|
232
|
+
);
|
|
233
|
+
}
|