prometheus-client-mmap 0.21.0-x86_64-darwin → 0.22.0-x86_64-darwin
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ext/fast_mmaped_file_rs/src/mmap/inner.rs +708 -0
- data/ext/fast_mmaped_file_rs/src/mmap.rs +587 -25
- data/lib/2.7/fast_mmaped_file_rs.bundle +0 -0
- data/lib/3.0/fast_mmaped_file_rs.bundle +0 -0
- data/lib/3.1/fast_mmaped_file.bundle +0 -0
- data/lib/3.1/fast_mmaped_file_rs.bundle +0 -0
- data/lib/3.2/fast_mmaped_file.bundle +0 -0
- data/lib/3.2/fast_mmaped_file_rs.bundle +0 -0
- data/lib/prometheus/client/version.rb +1 -1
- metadata +7 -2
@@ -1,9 +1,28 @@
|
|
1
|
+
use magnus::exception::*;
|
2
|
+
use magnus::prelude::*;
|
3
|
+
use magnus::rb_sys::{AsRawValue, FromRawValue};
|
1
4
|
use magnus::typed_data::Obj;
|
2
5
|
use magnus::value::Fixnum;
|
3
|
-
use magnus::{Integer, RArray, RClass, RHash, RString, Value};
|
6
|
+
use magnus::{block::Proc, eval, scan_args, Error, Integer, RArray, RClass, RHash, RString, Value};
|
7
|
+
use rb_sys::rb_str_new_static;
|
8
|
+
use std::fs::File;
|
9
|
+
use std::io::{prelude::*, SeekFrom};
|
10
|
+
use std::mem;
|
11
|
+
use std::path::Path;
|
12
|
+
use std::ptr::NonNull;
|
13
|
+
use std::sync::RwLock;
|
4
14
|
|
15
|
+
use crate::err;
|
16
|
+
use crate::error::MmapError;
|
5
17
|
use crate::file_entry::FileEntry;
|
6
18
|
use crate::map::EntryMap;
|
19
|
+
use crate::raw_entry::RawEntry;
|
20
|
+
use crate::util::{self, CheckedOps};
|
21
|
+
use crate::Result;
|
22
|
+
use crate::HEADER_SIZE;
|
23
|
+
use inner::InnerMmap;
|
24
|
+
|
25
|
+
mod inner;
|
7
26
|
|
8
27
|
/// A Rust struct wrapped in a Ruby object, providing access to a memory-mapped
|
9
28
|
/// file used to store, update, and read out Prometheus metrics.
|
@@ -35,9 +54,21 @@ use crate::map::EntryMap;
|
|
35
54
|
/// | K2 Value |
|
36
55
|
/// +-+-+-+-+-+-+-+
|
37
56
|
/// ```
|
57
|
+
//
|
58
|
+
// The API imposed by `magnus` requires all methods to use shared borrows.
|
59
|
+
// This means we can't store any mutable state in the top-level struct,
|
60
|
+
// and must store the interior data behind a `RwLock`, which adds run-time
|
61
|
+
// checks that mutable operations have no concurrent read or writes.
|
62
|
+
//
|
63
|
+
// We are further limited by the need to support subclassing in Ruby, which
|
64
|
+
// requires us to define an allocation function for the class, the
|
65
|
+
// `magnus::class::define_alloc_func()` function. This needs a support the
|
66
|
+
// `Default` trait, so a `File` cannot directly help by the object being
|
67
|
+
// constructed. Having the `RwLock` hold an `Option` of the interior object
|
68
|
+
// resolves this.
|
38
69
|
#[derive(Debug, Default)]
|
39
70
|
#[magnus::wrap(class = "FastMmapedFileRs", free_immediately, size)]
|
40
|
-
pub struct MmapedFile;
|
71
|
+
pub struct MmapedFile(RwLock<Option<InnerMmap>>);
|
41
72
|
|
42
73
|
impl MmapedFile {
|
43
74
|
/// call-seq:
|
@@ -50,14 +81,43 @@ impl MmapedFile {
|
|
50
81
|
///
|
51
82
|
/// Creates a mapping that's shared with all other processes
|
52
83
|
/// mapping the same area of the file.
|
53
|
-
pub fn new(
|
54
|
-
|
84
|
+
pub fn new(klass: RClass, args: &[Value]) -> magnus::error::Result<Obj<Self>> {
|
85
|
+
let args = scan_args::scan_args::<(RString,), (), (), (), (), ()>(args)?;
|
86
|
+
let path = args.required.0;
|
87
|
+
|
88
|
+
let lock = MmapedFile(RwLock::new(None));
|
89
|
+
let obj = Obj::wrap_as(lock, klass);
|
90
|
+
|
91
|
+
let _: Value = obj.funcall("initialize", (path,))?;
|
92
|
+
|
93
|
+
Ok(obj)
|
55
94
|
}
|
56
95
|
|
57
96
|
/// Initialize a new `FastMmapedFileRs` object. This must be defined in
|
58
97
|
/// order for inheritance to work.
|
59
|
-
pub fn initialize(
|
60
|
-
|
98
|
+
pub fn initialize(rb_self: Obj<Self>, fname: String) -> magnus::error::Result<()> {
|
99
|
+
let file = File::options()
|
100
|
+
.read(true)
|
101
|
+
.write(true)
|
102
|
+
.open(&fname)
|
103
|
+
.map_err(|_| err!(arg_error(), "Can't open {}", fname))?;
|
104
|
+
|
105
|
+
let inner = InnerMmap::new(fname.into(), file)?;
|
106
|
+
rb_self.insert_inner(inner)?;
|
107
|
+
|
108
|
+
let weak_klass = RClass::from_value(eval("ObjectSpace::WeakMap")?)
|
109
|
+
.ok_or_else(|| err!(no_method_error(), "unable to create WeakMap"))?;
|
110
|
+
let weak_obj_tracker = weak_klass.new_instance(())?;
|
111
|
+
|
112
|
+
// We will need to iterate over strings backed by the mmapped file, but
|
113
|
+
// don't want to prevent the GC from reaping them when the Ruby code
|
114
|
+
// has finished with them. `ObjectSpace::WeakMap` allows us to track
|
115
|
+
// them without extending their lifetime.
|
116
|
+
//
|
117
|
+
// https://ruby-doc.org/core-3.0.0/ObjectSpace/WeakMap.html
|
118
|
+
rb_self.ivar_set("@weak_obj_tracker", weak_obj_tracker)?;
|
119
|
+
|
120
|
+
Ok(())
|
61
121
|
}
|
62
122
|
|
63
123
|
/// Read the list of files provided from Ruby and convert them to a Prometheus
|
@@ -89,8 +149,23 @@ impl MmapedFile {
|
|
89
149
|
/// self[start, length]
|
90
150
|
///
|
91
151
|
/// return a substring of <em>lenght</em> characters from <em>start</em>
|
92
|
-
pub fn slice(
|
93
|
-
|
152
|
+
pub fn slice(rb_self: Obj<Self>, args: &[Value]) -> magnus::error::Result<RString> {
|
153
|
+
// The C implemenation would trigger a GC cycle via `rb_gc_force_recycle`
|
154
|
+
// if the `MM_PROTECT` flag is set, but in practice this is never used.
|
155
|
+
// We omit this logic, particularly because `rb_gc_force_recycle` is a
|
156
|
+
// no-op as of Ruby 3.1.
|
157
|
+
|
158
|
+
let str = (*rb_self).str(rb_self)?;
|
159
|
+
let res = str.funcall("[]", args);
|
160
|
+
|
161
|
+
// The C implementation does this, perhaps to validate that the len we
|
162
|
+
// provided is actually being used.
|
163
|
+
(*rb_self).inner_mut(|inner| {
|
164
|
+
inner.set_len(str.len());
|
165
|
+
Ok(())
|
166
|
+
})?;
|
167
|
+
|
168
|
+
res
|
94
169
|
}
|
95
170
|
|
96
171
|
/// Document-method: msync
|
@@ -100,8 +175,23 @@ impl MmapedFile {
|
|
100
175
|
/// call-seq: msync
|
101
176
|
///
|
102
177
|
/// flush the file
|
103
|
-
pub fn sync(&self,
|
104
|
-
|
178
|
+
pub fn sync(&self, args: &[Value]) -> magnus::error::Result<()> {
|
179
|
+
use nix::sys::mman::MsFlags;
|
180
|
+
|
181
|
+
let mut ms_async = false;
|
182
|
+
let args = scan_args::scan_args::<(), (Option<i32>,), (), (), (), ()>(args)?;
|
183
|
+
|
184
|
+
if let Some(flag) = args.optional.0 {
|
185
|
+
let flag = MsFlags::from_bits(flag).unwrap_or(MsFlags::empty());
|
186
|
+
ms_async = flag.contains(MsFlags::MS_ASYNC);
|
187
|
+
}
|
188
|
+
|
189
|
+
// The `memmap2` crate does not support the `MS_INVALIDATE` flag. We ignore that
|
190
|
+
// flag if passed in, checking only for `MS_ASYNC`. In practice no arguments are ever
|
191
|
+
// passed to this function, but we do this to maintain compatibility with the
|
192
|
+
// C implementation.
|
193
|
+
self.inner_mut(|inner| inner.flush(ms_async))
|
194
|
+
.map_err(|e| e.into())
|
105
195
|
}
|
106
196
|
|
107
197
|
/// Document-method: munmap
|
@@ -110,42 +200,514 @@ impl MmapedFile {
|
|
110
200
|
/// call-seq: munmap
|
111
201
|
///
|
112
202
|
/// terminate the association
|
113
|
-
pub fn munmap(
|
114
|
-
|
203
|
+
pub fn munmap(rb_self: Obj<Self>) -> magnus::error::Result<()> {
|
204
|
+
let rs_self = &*rb_self;
|
205
|
+
|
206
|
+
rs_self.inner_mut(|inner| {
|
207
|
+
// truncate file to actual used size
|
208
|
+
inner.truncate_file()?;
|
209
|
+
|
210
|
+
// We are about to release the backing mmap for Ruby's String
|
211
|
+
// objects. If Ruby attempts to read from them the program will
|
212
|
+
// segfault. We update the length of all Strings to zero so Ruby
|
213
|
+
// does not attempt to access the now invalid address between now
|
214
|
+
// and when GC eventually reaps the objects.
|
215
|
+
//
|
216
|
+
// See the following for more detail:
|
217
|
+
// https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/39
|
218
|
+
// https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/issues/41
|
219
|
+
// https://gitlab.com/gitlab-org/ruby/gems/prometheus-client-mmap/-/merge_requests/80
|
220
|
+
inner.set_len(0);
|
221
|
+
Ok(())
|
222
|
+
})?;
|
223
|
+
|
224
|
+
// Update each String object to be zero-length.
|
225
|
+
rs_self.update_weak_map(rb_self)?;
|
226
|
+
|
227
|
+
// Remove the `InnerMmap` from the `RwLock`. This will drop
|
228
|
+
// end of this function, unmapping and closing the file.
|
229
|
+
let _ = rs_self.take_inner()?;
|
230
|
+
Ok(())
|
115
231
|
}
|
116
232
|
|
117
233
|
/// Fetch the `used` header from the `.db` file, the length
|
118
234
|
/// in bytes of the data written to the file.
|
119
235
|
pub fn load_used(&self) -> magnus::error::Result<Integer> {
|
120
|
-
|
236
|
+
let used = self.inner(|inner| inner.load_used())?;
|
237
|
+
|
238
|
+
Ok(Integer::from_u64(used as u64))
|
121
239
|
}
|
122
240
|
|
123
241
|
/// Update the `used` header for the `.db` file, the length
|
124
242
|
/// in bytes of the data written to the file.
|
125
|
-
pub fn save_used(
|
126
|
-
|
243
|
+
pub fn save_used(rb_self: Obj<Self>, used: Fixnum) -> magnus::error::Result<Fixnum> {
|
244
|
+
let rs_self = &*rb_self;
|
245
|
+
let used_uint = used.to_u32()?;
|
246
|
+
|
247
|
+
// If the underlying mmap is smaller than the header, then resize to fit.
|
248
|
+
// The file has already been expanded to page size when first opened, so
|
249
|
+
// even if the map is less than HEADER_SIZE, we're not at risk of a
|
250
|
+
// SIGBUS.
|
251
|
+
if rs_self.capacity() < HEADER_SIZE {
|
252
|
+
rs_self.expand_to_fit(rb_self, HEADER_SIZE)?;
|
253
|
+
}
|
254
|
+
|
255
|
+
rs_self.inner_mut(|inner| inner.save_used(used_uint))?;
|
256
|
+
|
257
|
+
Ok(used)
|
127
258
|
}
|
128
259
|
|
129
260
|
/// Fetch the value associated with a key from the mmap.
|
130
261
|
/// If no entry is present, initialize with the default
|
131
262
|
/// value provided.
|
132
263
|
pub fn fetch_entry(
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
264
|
+
rb_self: Obj<Self>,
|
265
|
+
positions: RHash,
|
266
|
+
key: RString,
|
267
|
+
default_value: f64,
|
137
268
|
) -> magnus::error::Result<f64> {
|
138
|
-
|
269
|
+
let rs_self = &*rb_self;
|
270
|
+
let position: Option<Fixnum> = positions.lookup(key)?;
|
271
|
+
|
272
|
+
if let Some(pos) = position {
|
273
|
+
let pos = pos.to_usize()?;
|
274
|
+
return rs_self
|
275
|
+
.inner(|inner| inner.load_value(pos))
|
276
|
+
.map_err(|e| e.into());
|
277
|
+
}
|
278
|
+
|
279
|
+
rs_self.check_expand(rb_self, key.len())?;
|
280
|
+
|
281
|
+
let value_offset: usize = rs_self.inner_mut(|inner| {
|
282
|
+
// SAFETY: We must not call any Ruby code for the lifetime of this borrow.
|
283
|
+
unsafe { inner.initialize_entry(key.as_slice(), default_value) }
|
284
|
+
})?;
|
285
|
+
|
286
|
+
// CAST: no-op on 64-bit, widening on 32-bit.
|
287
|
+
positions.aset(key, Integer::from_u64(value_offset as u64))?;
|
288
|
+
|
289
|
+
rs_self.load_value(value_offset)
|
139
290
|
}
|
140
291
|
|
141
292
|
/// Update the value of an existing entry, if present. Otherwise create a new entry
|
142
293
|
/// for the key.
|
143
294
|
pub fn upsert_entry(
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
295
|
+
rb_self: Obj<Self>,
|
296
|
+
positions: RHash,
|
297
|
+
key: RString,
|
298
|
+
value: f64,
|
148
299
|
) -> magnus::error::Result<f64> {
|
149
|
-
|
300
|
+
let rs_self = &*rb_self;
|
301
|
+
let position: Option<Fixnum> = positions.lookup(key)?;
|
302
|
+
|
303
|
+
if let Some(pos) = position {
|
304
|
+
let pos = pos.to_usize()?;
|
305
|
+
return rs_self
|
306
|
+
.inner_mut(|inner| {
|
307
|
+
inner.save_value(pos, value)?;
|
308
|
+
|
309
|
+
// TODO just return `value` here instead of loading it?
|
310
|
+
// This is how the C implementation did it, but I don't
|
311
|
+
// see what the extra load gains us.
|
312
|
+
inner.load_value(pos)
|
313
|
+
})
|
314
|
+
.map_err(|e| e.into());
|
315
|
+
}
|
316
|
+
|
317
|
+
rs_self.check_expand(rb_self, key.len())?;
|
318
|
+
|
319
|
+
let value_offset: usize = rs_self.inner_mut(|inner| {
|
320
|
+
// SAFETY: We must not call any Ruby code for the lifetime of this borrow.
|
321
|
+
unsafe { inner.initialize_entry(key.as_slice(), value) }
|
322
|
+
})?;
|
323
|
+
|
324
|
+
// CAST: no-op on 64-bit, widening on 32-bit.
|
325
|
+
positions.aset(key, Integer::from_u64(value_offset as u64))?;
|
326
|
+
|
327
|
+
rs_self.load_value(value_offset)
|
328
|
+
}
|
329
|
+
|
330
|
+
/// Creates a Ruby String containing the section of the mmapped file that
|
331
|
+
/// has been written to.
|
332
|
+
fn str(&self, rb_self: Obj<Self>) -> magnus::error::Result<RString> {
|
333
|
+
let val_id = (*rb_self).inner(|inner| {
|
334
|
+
let ptr = inner.as_ptr();
|
335
|
+
let len = inner.len();
|
336
|
+
|
337
|
+
// SAFETY: This is safe so long as the data provided to Ruby meets its
|
338
|
+
// requirements. When unmapping the file this will no longer be the
|
339
|
+
// case, see the comment on `munmap` how we handle this.
|
340
|
+
Ok(unsafe { rb_str_new_static(ptr as _, len as _) })
|
341
|
+
})?;
|
342
|
+
|
343
|
+
let val = unsafe { Value::from_raw(val_id) };
|
344
|
+
|
345
|
+
// UNWRAP: We created this value as a string above.
|
346
|
+
let str = RString::from_value(val).unwrap();
|
347
|
+
|
348
|
+
let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
|
349
|
+
|
350
|
+
// Use the string's Id as the key in the `WeakMap`.
|
351
|
+
let key = str.as_raw();
|
352
|
+
let _: Value = tracker.funcall("[]=", (key, str))?;
|
353
|
+
|
354
|
+
Ok(str)
|
355
|
+
}
|
356
|
+
|
357
|
+
/// If we reallocate, any live Ruby strings provided by the `str()` method
|
358
|
+
/// will be will be invalidated. We need to iterate over them using and
|
359
|
+
/// update their heap pointers to the newly allocated memory region.
|
360
|
+
fn update_weak_map(&self, rb_self: Obj<Self>) -> magnus::error::Result<()> {
|
361
|
+
let tracker: Value = rb_self.ivar_get("@weak_obj_tracker")?;
|
362
|
+
|
363
|
+
let (new_ptr, new_len) = (*rb_self).inner(|inner| {
|
364
|
+
// Pointers are not `Send`, convert it to usize to allow capture in closure.
|
365
|
+
let new_ptr = inner.as_ptr() as usize;
|
366
|
+
let new_len = util::cast_chk::<_, i64>(inner.len(), "mmap len")?;
|
367
|
+
|
368
|
+
Ok((new_ptr, new_len))
|
369
|
+
})?;
|
370
|
+
|
371
|
+
// Allocate a block with a closure that updates the string details.
|
372
|
+
let block = Proc::from_fn(move |args, _block| {
|
373
|
+
let val = args
|
374
|
+
.get(0)
|
375
|
+
.ok_or_else(|| err!(arg_error(), "no argument received from `each_value`"))?;
|
376
|
+
let str = RString::from_value(*val)
|
377
|
+
.ok_or_else(|| err!(arg_error(), "weakmap value was not a string"))?;
|
378
|
+
|
379
|
+
// SAFETY: We're messing with Ruby's internals here, YOLO.
|
380
|
+
unsafe {
|
381
|
+
// Convert the magnus wrapper type to a raw string exposed by `rb_sys`,
|
382
|
+
// which provides access to its internals.
|
383
|
+
let mut raw_str = Self::rb_string_internal(str);
|
384
|
+
|
385
|
+
// Update the string to point to the new mmapped file.
|
386
|
+
// We're matching the behavior of Ruby's `str_new_static` function.
|
387
|
+
// See https://github.com/ruby/ruby/blob/e51014f9c05aa65cbf203442d37fef7c12390015/string.c#L1030-L1053
|
388
|
+
raw_str.as_mut().as_.heap.ptr = new_ptr as _;
|
389
|
+
raw_str.as_mut().as_.heap.len = new_len;
|
390
|
+
raw_str.as_mut().as_.heap.aux.capa = new_len;
|
391
|
+
}
|
392
|
+
Ok(())
|
393
|
+
});
|
394
|
+
|
395
|
+
// Execute the block.
|
396
|
+
let _: Value = tracker.funcall_with_block("each_value", (), block)?;
|
397
|
+
Ok(())
|
398
|
+
}
|
399
|
+
|
400
|
+
/// Check that the mmap is large enough to contain the value to be added,
|
401
|
+
/// and expand it to fit if necessary.
|
402
|
+
fn check_expand(&self, rb_self: Obj<Self>, key_len: usize) -> magnus::error::Result<()> {
|
403
|
+
// CAST: no-op on 32-bit, widening on 64-bit.
|
404
|
+
let used = self.inner(|inner| inner.load_used())? as usize;
|
405
|
+
let entry_len = RawEntry::calc_total_len(key_len)?;
|
406
|
+
|
407
|
+
// We need the mmapped region to contain at least one byte beyond the
|
408
|
+
// written data to create a NUL- terminated C string. Validate that
|
409
|
+
// new length does not exactly match the length of the mmap or exceed
|
410
|
+
// it.
|
411
|
+
while self.capacity() <= used.add_chk(entry_len)? {
|
412
|
+
self.expand_to_fit(rb_self, self.capacity().mul_chk(2)?)?;
|
413
|
+
}
|
414
|
+
|
415
|
+
Ok(())
|
416
|
+
}
|
417
|
+
|
418
|
+
/// Expand the underlying file until it is long enough to fit `target_cap`.
|
419
|
+
/// This will remove the existing mmap, expand the file, then update any
|
420
|
+
/// strings held by the `WeakMap` to point to the newly mmapped address.
|
421
|
+
fn expand_to_fit(&self, rb_self: Obj<Self>, target_cap: usize) -> magnus::error::Result<()> {
|
422
|
+
if target_cap < self.capacity() {
|
423
|
+
return Err(err!(arg_error(), "Can't reduce the size of mmap"));
|
424
|
+
}
|
425
|
+
|
426
|
+
let mut new_cap = self.capacity();
|
427
|
+
while new_cap < target_cap {
|
428
|
+
new_cap = new_cap.mul_chk(2)?;
|
429
|
+
}
|
430
|
+
|
431
|
+
if new_cap != self.capacity() {
|
432
|
+
// Drop the old mmap.
|
433
|
+
let (mut file, path) = self.take_inner()?.munmap();
|
434
|
+
|
435
|
+
self.expand_file(&mut file, &path, target_cap)?;
|
436
|
+
|
437
|
+
// Re-mmap the expanded file.
|
438
|
+
let new_inner = InnerMmap::reestablish(path, file, target_cap)?;
|
439
|
+
|
440
|
+
self.insert_inner(new_inner)?;
|
441
|
+
|
442
|
+
return self.update_weak_map(rb_self);
|
443
|
+
}
|
444
|
+
|
445
|
+
Ok(())
|
446
|
+
}
|
447
|
+
|
448
|
+
/// Use lseek(2) to seek past the end of the file and write a NUL byte. This
|
449
|
+
/// creates a file hole that expands the size of the file without consuming
|
450
|
+
/// disk space until it is actually written to.
|
451
|
+
fn expand_file(&self, file: &mut File, path: &Path, len: usize) -> Result<()> {
|
452
|
+
if len == 0 {
|
453
|
+
return Err(MmapError::overflowed(0, -1, "adding"));
|
454
|
+
}
|
455
|
+
|
456
|
+
// CAST: no-op on 64-bit, widening on 32-bit.
|
457
|
+
let len = len as u64;
|
458
|
+
|
459
|
+
match file.seek(SeekFrom::Start(len - 1)) {
|
460
|
+
Ok(_) => {}
|
461
|
+
Err(_) => {
|
462
|
+
return Err(MmapError::WithErrno(format!("Can't lseek {}", len - 1)));
|
463
|
+
}
|
464
|
+
}
|
465
|
+
|
466
|
+
match file.write(&[0x0]) {
|
467
|
+
Ok(1) => {}
|
468
|
+
_ => {
|
469
|
+
return Err(MmapError::WithErrno(format!(
|
470
|
+
"Can't extend {}",
|
471
|
+
path.display()
|
472
|
+
)));
|
473
|
+
}
|
474
|
+
}
|
475
|
+
|
476
|
+
Ok(())
|
477
|
+
}
|
478
|
+
|
479
|
+
/// The total capacity of the underlying mmap.
|
480
|
+
#[inline]
|
481
|
+
fn capacity(&self) -> usize {
|
482
|
+
// UNWRAP: This is actually infallible, but we need to
|
483
|
+
// wrap it in a `Result` for use with `inner()`.
|
484
|
+
self.inner(|inner| Ok(inner.capacity())).unwrap()
|
485
|
+
}
|
486
|
+
|
487
|
+
fn load_value(&self, position: usize) -> magnus::error::Result<f64> {
|
488
|
+
self.inner(|inner| inner.load_value(position))
|
489
|
+
.map_err(|e| e.into())
|
490
|
+
}
|
491
|
+
|
492
|
+
/// Takes a closure with immutable access to InnerMmap. Will fail if the inner
|
493
|
+
/// object has a mutable borrow or has been dropped.
|
494
|
+
fn inner<F, T>(&self, func: F) -> Result<T>
|
495
|
+
where
|
496
|
+
F: FnOnce(&InnerMmap) -> Result<T>,
|
497
|
+
{
|
498
|
+
let inner_opt = self.0.try_read().map_err(|_| MmapError::ConcurrentAccess)?;
|
499
|
+
|
500
|
+
let inner = inner_opt.as_ref().ok_or(MmapError::UnmappedFile)?;
|
501
|
+
|
502
|
+
func(inner)
|
503
|
+
}
|
504
|
+
|
505
|
+
/// Takes a closure with mutable access to InnerMmap. Will fail if the inner
|
506
|
+
/// object has an existing mutable borrow, or has been dropped.
|
507
|
+
fn inner_mut<F, T>(&self, func: F) -> Result<T>
|
508
|
+
where
|
509
|
+
F: FnOnce(&mut InnerMmap) -> Result<T>,
|
510
|
+
{
|
511
|
+
let mut inner_opt = self
|
512
|
+
.0
|
513
|
+
.try_write()
|
514
|
+
.map_err(|_| MmapError::ConcurrentAccess)?;
|
515
|
+
|
516
|
+
let inner = inner_opt.as_mut().ok_or(MmapError::UnmappedFile)?;
|
517
|
+
|
518
|
+
func(inner)
|
519
|
+
}
|
520
|
+
|
521
|
+
/// Take ownership of the `InnerMmap` from the `RwLock`.
|
522
|
+
/// Will fail if a mutable borrow is already held or the inner
|
523
|
+
/// object has been dropped.
|
524
|
+
fn take_inner(&self) -> Result<InnerMmap> {
|
525
|
+
let mut inner_opt = self
|
526
|
+
.0
|
527
|
+
.try_write()
|
528
|
+
.map_err(|_| MmapError::ConcurrentAccess)?;
|
529
|
+
match (*inner_opt).take() {
|
530
|
+
Some(i) => Ok(i),
|
531
|
+
None => Err(MmapError::UnmappedFile),
|
532
|
+
}
|
533
|
+
}
|
534
|
+
|
535
|
+
/// Move `new_inner` into the `RwLock`.
|
536
|
+
/// Will return an error if a mutable borrow is already held.
|
537
|
+
fn insert_inner(&self, new_inner: InnerMmap) -> Result<()> {
|
538
|
+
let mut inner_opt = self
|
539
|
+
.0
|
540
|
+
.try_write()
|
541
|
+
.map_err(|_| MmapError::ConcurrentAccess)?;
|
542
|
+
(*inner_opt).replace(new_inner);
|
543
|
+
|
544
|
+
Ok(())
|
545
|
+
}
|
546
|
+
|
547
|
+
/// Convert `magnus::RString` into the raw binding used by `rb_sys::RString`.
|
548
|
+
/// We need this to manually change the pointer and length values for strings
|
549
|
+
/// when moving the mmap to a new file.
|
550
|
+
unsafe fn rb_string_internal(rb_str: RString) -> NonNull<rb_sys::RString> {
|
551
|
+
mem::transmute::<RString, NonNull<rb_sys::RString>>(rb_str)
|
552
|
+
}
|
553
|
+
}
|
554
|
+
|
555
|
+
#[cfg(test)]
|
556
|
+
mod test {
|
557
|
+
use magnus::error::Error;
|
558
|
+
use magnus::eval;
|
559
|
+
use magnus::Range;
|
560
|
+
use nix::unistd::{sysconf, SysconfVar};
|
561
|
+
use std::mem::size_of;
|
562
|
+
|
563
|
+
use super::*;
|
564
|
+
use crate::raw_entry::RawEntry;
|
565
|
+
use crate::testhelper::TestFile;
|
566
|
+
|
567
|
+
/// Create a wrapped MmapedFile object.
|
568
|
+
fn create_obj() -> Obj<MmapedFile> {
|
569
|
+
let TestFile {
|
570
|
+
file: _file,
|
571
|
+
path,
|
572
|
+
dir: _dir,
|
573
|
+
} = TestFile::new(&[0u8; 8]);
|
574
|
+
|
575
|
+
let path_str = path.display().to_string();
|
576
|
+
let rpath = RString::new(&path_str);
|
577
|
+
|
578
|
+
eval!("FastMmapedFileRs.new(path)", path = rpath).unwrap()
|
579
|
+
}
|
580
|
+
|
581
|
+
/// Add three entries to the mmap. Expected length is 56, 3x 16-byte
|
582
|
+
/// entries with 8-byte header.
|
583
|
+
fn populate_entries(rb_self: &Obj<MmapedFile>) -> RHash {
|
584
|
+
let positions = RHash::from_value(eval("{}").unwrap()).unwrap();
|
585
|
+
|
586
|
+
MmapedFile::upsert_entry(*rb_self, positions, RString::new("a"), 0.0).unwrap();
|
587
|
+
MmapedFile::upsert_entry(*rb_self, positions, RString::new("b"), 1.0).unwrap();
|
588
|
+
MmapedFile::upsert_entry(*rb_self, positions, RString::new("c"), 2.0).unwrap();
|
589
|
+
|
590
|
+
positions
|
591
|
+
}
|
592
|
+
|
593
|
+
#[test]
|
594
|
+
fn test_new() {
|
595
|
+
let _cleanup = unsafe { magnus::embed::init() };
|
596
|
+
let ruby = magnus::Ruby::get().unwrap();
|
597
|
+
crate::init(&ruby).unwrap();
|
598
|
+
|
599
|
+
let TestFile {
|
600
|
+
file,
|
601
|
+
path,
|
602
|
+
dir: _dir,
|
603
|
+
} = TestFile::new(&[0u8; 8]);
|
604
|
+
|
605
|
+
let path_str = path.display().to_string();
|
606
|
+
let rpath = RString::new(&path_str);
|
607
|
+
|
608
|
+
// Object created successfully
|
609
|
+
let result: std::result::Result<Obj<MmapedFile>, Error> =
|
610
|
+
eval!("FastMmapedFileRs.new(path)", path = rpath);
|
611
|
+
assert!(result.is_ok());
|
612
|
+
|
613
|
+
// Weak map added
|
614
|
+
let obj = result.unwrap();
|
615
|
+
let weak_tracker: Value = obj.ivar_get("@weak_obj_tracker").unwrap();
|
616
|
+
assert_eq!("ObjectSpace::WeakMap", weak_tracker.class().inspect());
|
617
|
+
|
618
|
+
// File expanded to page size
|
619
|
+
let page_size = sysconf(SysconfVar::PAGE_SIZE).unwrap().unwrap() as u64;
|
620
|
+
let stat = file.metadata().unwrap();
|
621
|
+
assert_eq!(page_size, stat.len());
|
622
|
+
|
623
|
+
// Used set to header size
|
624
|
+
assert_eq!(
|
625
|
+
HEADER_SIZE as u64,
|
626
|
+
obj.load_used().unwrap().to_u64().unwrap()
|
627
|
+
);
|
628
|
+
}
|
629
|
+
|
630
|
+
#[test]
|
631
|
+
fn test_slice() {
|
632
|
+
let _cleanup = unsafe { magnus::embed::init() };
|
633
|
+
let ruby = magnus::Ruby::get().unwrap();
|
634
|
+
crate::init(&ruby).unwrap();
|
635
|
+
|
636
|
+
let obj = create_obj();
|
637
|
+
let _ = populate_entries(&obj);
|
638
|
+
|
639
|
+
// Validate header updated with new length
|
640
|
+
let header_range = Range::new(0, HEADER_SIZE, true).unwrap().as_value();
|
641
|
+
let header_slice = MmapedFile::slice(obj, &[header_range]).unwrap();
|
642
|
+
assert_eq!([56, 0, 0, 0, 0, 0, 0, 0], unsafe {
|
643
|
+
header_slice.as_slice()
|
644
|
+
});
|
645
|
+
|
646
|
+
let value_range = Range::new(HEADER_SIZE, 24, true).unwrap().as_value();
|
647
|
+
let value_slice = MmapedFile::slice(obj, &[value_range]).unwrap();
|
648
|
+
|
649
|
+
// Validate string length
|
650
|
+
assert_eq!(1u32.to_ne_bytes(), unsafe { &value_slice.as_slice()[0..4] });
|
651
|
+
|
652
|
+
// Validate string and padding
|
653
|
+
assert_eq!("a ", unsafe {
|
654
|
+
String::from_utf8_lossy(&value_slice.as_slice()[4..8])
|
655
|
+
});
|
656
|
+
|
657
|
+
// Validate value
|
658
|
+
assert_eq!(0.0f64.to_ne_bytes(), unsafe {
|
659
|
+
&value_slice.as_slice()[8..16]
|
660
|
+
});
|
661
|
+
}
|
662
|
+
|
663
|
+
#[test]
|
664
|
+
fn test_slice_resize() {
|
665
|
+
let _cleanup = unsafe { magnus::embed::init() };
|
666
|
+
let ruby = magnus::Ruby::get().unwrap();
|
667
|
+
crate::init(&ruby).unwrap();
|
668
|
+
|
669
|
+
let obj = create_obj();
|
670
|
+
let _ = populate_entries(&obj);
|
671
|
+
|
672
|
+
let rs_self = &*obj;
|
673
|
+
|
674
|
+
let value_range = Range::new(HEADER_SIZE, 24, false).unwrap().as_value();
|
675
|
+
let value_slice = MmapedFile::slice(obj, &[value_range]).unwrap();
|
676
|
+
|
677
|
+
rs_self.expand_to_fit(obj, rs_self.capacity() * 2).unwrap();
|
678
|
+
|
679
|
+
// If we haven't updated the pointer to the newly remapped file this will segfault.
|
680
|
+
let _: Value = eval!("puts str", str = value_slice).unwrap();
|
681
|
+
}
|
682
|
+
|
683
|
+
#[test]
|
684
|
+
fn test_dont_fill_mmap() {
|
685
|
+
let _cleanup = unsafe { magnus::embed::init() };
|
686
|
+
let ruby = magnus::Ruby::get().unwrap();
|
687
|
+
crate::init(&ruby).unwrap();
|
688
|
+
|
689
|
+
let obj = create_obj();
|
690
|
+
let positions = populate_entries(&obj);
|
691
|
+
|
692
|
+
let rs_self = &*obj;
|
693
|
+
|
694
|
+
rs_self.expand_to_fit(obj, 1024).unwrap();
|
695
|
+
|
696
|
+
let current_used = rs_self.inner(|inner| inner.load_used()).unwrap() as usize;
|
697
|
+
let current_cap = rs_self.inner(|inner| Ok(inner.len())).unwrap();
|
698
|
+
|
699
|
+
// Create a new entry that exactly fills the capacity of the mmap.
|
700
|
+
let val_len =
|
701
|
+
current_cap - current_used - HEADER_SIZE - size_of::<f64>() - size_of::<u32>();
|
702
|
+
assert_eq!(
|
703
|
+
current_cap,
|
704
|
+
RawEntry::calc_total_len(val_len).unwrap() + current_used
|
705
|
+
);
|
706
|
+
|
707
|
+
let str = String::from_utf8(vec![b'A'; val_len]).unwrap();
|
708
|
+
MmapedFile::upsert_entry(obj, positions, RString::new(&str), 1.0).unwrap();
|
709
|
+
|
710
|
+
// Validate that we have expanded the mmap, ensuring a trailing NUL.
|
711
|
+
assert!(rs_self.capacity() > current_cap);
|
150
712
|
}
|
151
713
|
}
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|