ruby-fst 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -0
- data/Cargo.lock +1 -1
- data/README.md +10 -2
- data/ext/ruby_fst/Cargo.toml +3 -2
- data/ext/ruby_fst/src/lib.rs +444 -461
- data/lib/ruby_fst/version.rb +1 -1
- data/lib/ruby_fst.rb +18 -2
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: eaf4d02399b3d1c106a6f0de243e84e644982fb3af15cb5b2e95416580a1cf6f
|
|
4
|
+
data.tar.gz: 247642cba615720e22d7f92ab429ed3a6262bc885fe0e8ef30d09f57a0105251
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 981010f25a4534daeee239017575822f1a64365e02cb80d80fc71504b0638c92ead519c6175751c8239c5cb28bf0bd252cde3c6dfabd11c758e5b7bcefcbf05c
|
|
7
|
+
data.tar.gz: fa2698b0f96ef514d17fc98071d09c9410b648b4e2853ef3aa7533cf49abcdc7490452c5ffb33841648d9cbd7007dfa32df352b77c05339a84f2e30ed0e0ceeb
|
data/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,29 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.3.0] — 2026-10-26
|
|
10
|
+
|
|
11
|
+
### Changed
|
|
12
|
+
- I/O failures now raise `IOError` instead of `RuntimeError`, and Levenshtein queries too large to
|
|
13
|
+
compile raise `ArgumentError`. Other errors raise the new `RubyFst::Error`, a `RuntimeError`
|
|
14
|
+
subclass, so existing `rescue RuntimeError` for builder and format errors keeps working.
|
|
15
|
+
- `new` and `from_path` verify the FST checksum. Corrupt data now raises `RubyFst::Error`; before,
|
|
16
|
+
it could return wrong results or terminate the process. FSTs written by fst < 0.4 have no
|
|
17
|
+
checksum and still load unverified. `from_path_mmap` does not verify.
|
|
18
|
+
- Paths accept `Pathname`, and on Unix non-UTF-8 file names.
|
|
19
|
+
- `each` and `search_levenshtein` return an `Enumerator` when called without a block.
|
|
20
|
+
- In-memory FSTs report their size to Ruby's GC.
|
|
21
|
+
|
|
22
|
+
- `save` writes a temporary file and renames it into place, so readers never see a partial file.
|
|
23
|
+
Symlinks are followed and permissions kept, but the replacement is a new file: hard links to the
|
|
24
|
+
old one are broken and ownership becomes the saving user's. Where a rename can't work (a
|
|
25
|
+
non-regular target such as a FIFO, or an unwritable directory) it writes in place as before. A
|
|
26
|
+
crash mid-save can leave a `<path>.tmp-<pid>` file behind.
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
- Saving over a memory-mapped file no longer crashes the process with `SIGBUS` on Linux and macOS.
|
|
30
|
+
Windows can't replace a mapped file, so `save` raises `IOError` there.
|
|
31
|
+
|
|
9
32
|
## [0.2.2] — 2026-09-24
|
|
10
33
|
|
|
11
34
|
### Fixed
|
data/Cargo.lock
CHANGED
data/README.md
CHANGED
|
@@ -25,7 +25,7 @@ gem 'ruby-fst'
|
|
|
25
25
|
|
|
26
26
|
All FST keys are arbitrary byte strings (`Encoding::BINARY`). When a key is returned from the gem (e.g. via `each`, `get_le`, `range`) it always carries the `Encoding::BINARY` encoding — be careful when interpolating into UTF-8 strings or `puts`-ing keys that contain non-ASCII bytes.
|
|
27
27
|
|
|
28
|
-
Keys must be inserted into a builder in **strictly ascending lexicographic byte order**, with no duplicates. Out-of-order or duplicate inserts raise
|
|
28
|
+
Keys must be inserted into a builder in **strictly ascending lexicographic byte order**, with no duplicates. Out-of-order or duplicate inserts raise `RubyFst::Error`.
|
|
29
29
|
|
|
30
30
|
## Map
|
|
31
31
|
|
|
@@ -156,7 +156,15 @@ map = RubyFst::Map.from_path('/path/to/file.fst')
|
|
|
156
156
|
map = RubyFst::Map.from_path_mmap('/path/to/file.fst')
|
|
157
157
|
```
|
|
158
158
|
|
|
159
|
-
|
|
159
|
+
`new` and `from_path` verify the FST's checksum (FSTs written before fst 0.4 have none and load unverified) and raise `RubyFst::Error` on corrupt or invalid data. `from_path_mmap` skips the check so that opening a large file stays cheap, which means a corrupt file can return wrong results or terminate the process — only memory-map files you trust.
|
|
160
|
+
|
|
161
|
+
When using `from_path_mmap`, the file must remain unchanged on disk for the lifetime of the resulting `Map` or `Set` — modifying or truncating it in place causes undefined behavior. `save` writes to a temporary file beside the target and renames it into place, so on Linux and macOS saving over a mapped file is safe: the existing mapping keeps reading the old contents. Windows refuses to replace a file while it is mapped, so there `save` raises `IOError` instead.
|
|
162
|
+
|
|
163
|
+
## Errors
|
|
164
|
+
|
|
165
|
+
- `IOError` — a file couldn't be read, mapped, or written.
|
|
166
|
+
- `ArgumentError` — a Levenshtein query too large to build an automaton for.
|
|
167
|
+
- `RubyFst::Error` (a `RuntimeError` subclass) — invalid or corrupt FST data, out-of-order builder inserts, or a builder used after `finish`.
|
|
160
168
|
|
|
161
169
|
## Development
|
|
162
170
|
|
data/ext/ruby_fst/Cargo.toml
CHANGED
data/ext/ruby_fst/src/lib.rs
CHANGED
|
@@ -1,28 +1,56 @@
|
|
|
1
1
|
use std::cell::RefCell;
|
|
2
|
+
use std::ffi::OsString;
|
|
2
3
|
use std::fs::{self, File};
|
|
4
|
+
use std::io::{self, Write};
|
|
5
|
+
use std::path::{Path, PathBuf};
|
|
3
6
|
|
|
4
|
-
use fst::automaton::Levenshtein;
|
|
5
|
-
use fst::raw::{CompiledAddr, Fst, Node, Output};
|
|
7
|
+
use fst::automaton::{Automaton, Levenshtein};
|
|
8
|
+
use fst::raw::{CompiledAddr, Fst, Node, Output, Stream};
|
|
6
9
|
use fst::{IntoStreamer, Streamer};
|
|
7
|
-
use magnus::
|
|
8
|
-
use magnus::{
|
|
10
|
+
use magnus::value::Lazy;
|
|
11
|
+
use magnus::{
|
|
12
|
+
DataTypeFunctions, Error, ExceptionClass, RModule, RString, Ruby, TypedData, Value, function,
|
|
13
|
+
method, prelude::*,
|
|
14
|
+
};
|
|
9
15
|
use memmap2::Mmap;
|
|
10
16
|
|
|
11
|
-
|
|
12
|
-
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// Errors
|
|
19
|
+
//
|
|
20
|
+
// I/O failures raise IOError, a rejected Levenshtein query raises
|
|
21
|
+
// ArgumentError, and everything else (invalid or corrupt FST data, misuse of
|
|
22
|
+
// a builder) raises RubyFst::Error, a RuntimeError subclass.
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
static FST_ERROR: Lazy<ExceptionClass> = Lazy::new(|ruby| {
|
|
26
|
+
ruby.class_object()
|
|
27
|
+
.const_get::<_, RModule>("RubyFst")
|
|
28
|
+
.and_then(|module| module.const_get("Error"))
|
|
29
|
+
.expect("RubyFst::Error is defined in init")
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
fn fst_error(ruby: &Ruby, msg: impl std::fmt::Display) -> Error {
|
|
33
|
+
Error::new(ruby.get_inner(&FST_ERROR), msg.to_string())
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
fn io_error(ruby: &Ruby, path: &Path, err: io::Error) -> Error {
|
|
37
|
+
Error::new(
|
|
38
|
+
ruby.exception_io_error(),
|
|
39
|
+
format!("{}: {err}", path.display()),
|
|
40
|
+
)
|
|
13
41
|
}
|
|
14
42
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
43
|
+
fn map_fst_error(ruby: &Ruby, err: fst::Error) -> Error {
|
|
44
|
+
match err {
|
|
45
|
+
fst::Error::Io(err) => Error::new(ruby.exception_io_error(), err.to_string()),
|
|
46
|
+
fst::Error::Fst(err) => fst_error(ruby, err),
|
|
47
|
+
}
|
|
20
48
|
}
|
|
21
49
|
|
|
22
50
|
// ---------------------------------------------------------------------------
|
|
23
|
-
// Storage: backing buffer for an FST
|
|
24
|
-
// memory map.
|
|
25
|
-
//
|
|
51
|
+
// Storage: the backing buffer for an FST, either an owned heap allocation or a
|
|
52
|
+
// memory map. A heap buffer is reported to Ruby's GC so that large FSTs count
|
|
53
|
+
// towards its memory pressure; a map is backed by the file, so it isn't.
|
|
26
54
|
// ---------------------------------------------------------------------------
|
|
27
55
|
|
|
28
56
|
enum Storage {
|
|
@@ -30,36 +58,122 @@ enum Storage {
|
|
|
30
58
|
Mmap(Mmap),
|
|
31
59
|
}
|
|
32
60
|
|
|
61
|
+
impl Storage {
|
|
62
|
+
fn mem(ruby: &Ruby, bytes: Vec<u8>) -> Self {
|
|
63
|
+
ruby.gc_adjust_memory_usage(heap_size(&bytes));
|
|
64
|
+
Storage::Mem(bytes)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
fn read(ruby: &Ruby, path: &Path) -> Result<Self, Error> {
|
|
68
|
+
let bytes = fs::read(path).map_err(|e| io_error(ruby, path, e))?;
|
|
69
|
+
Ok(Self::mem(ruby, bytes))
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
fn mmap(ruby: &Ruby, path: &Path) -> Result<Self, Error> {
|
|
73
|
+
let file = File::open(path).map_err(|e| io_error(ruby, path, e))?;
|
|
74
|
+
// SAFETY: modifying or truncating a mapped file is undefined behavior,
|
|
75
|
+
// and nothing here can prevent another process from doing it. The
|
|
76
|
+
// README makes keeping the file unchanged the caller's responsibility,
|
|
77
|
+
// and `save` replaces files by rename, so it never truncates a mapping.
|
|
78
|
+
let map = unsafe { Mmap::map(&file) }.map_err(|e| io_error(ruby, path, e))?;
|
|
79
|
+
Ok(Storage::Mmap(map))
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
33
83
|
impl AsRef<[u8]> for Storage {
|
|
34
84
|
fn as_ref(&self) -> &[u8] {
|
|
35
85
|
match self {
|
|
36
|
-
Storage::Mem(
|
|
37
|
-
Storage::Mmap(
|
|
86
|
+
Storage::Mem(bytes) => bytes,
|
|
87
|
+
Storage::Mmap(map) => map,
|
|
38
88
|
}
|
|
39
89
|
}
|
|
40
90
|
}
|
|
41
91
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
92
|
+
impl Drop for Storage {
|
|
93
|
+
fn drop(&mut self) {
|
|
94
|
+
if let Storage::Mem(bytes) = self {
|
|
95
|
+
// Dropped during GC finalization, which runs on a Ruby thread.
|
|
96
|
+
if let Ok(ruby) = Ruby::get() {
|
|
97
|
+
ruby.gc_adjust_memory_usage(-heap_size(bytes));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
45
101
|
}
|
|
46
102
|
|
|
47
|
-
fn
|
|
48
|
-
|
|
49
|
-
// SAFETY: callers opt in to mmap and accept the contract that the file
|
|
50
|
-
// must not be modified or truncated while the FST is alive. We document
|
|
51
|
-
// this in the Ruby-level docs.
|
|
52
|
-
let mmap = unsafe { Mmap::map(&file) }.map_err(err)?;
|
|
53
|
-
Ok(Storage::Mmap(mmap))
|
|
103
|
+
fn heap_size(bytes: &[u8]) -> isize {
|
|
104
|
+
isize::try_from(bytes.len()).unwrap_or(isize::MAX)
|
|
54
105
|
}
|
|
55
106
|
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
107
|
+
// Opens an FST over `storage`. Heap buffers are checksummed first, because the
|
|
108
|
+
// fst crate panics on corrupt data and a panic here terminates the Ruby
|
|
109
|
+
// process. FSTs written before fst 0.4 carry no checksum and are accepted
|
|
110
|
+
// unverified, as they always were. Memory maps skip the check so that opening
|
|
111
|
+
// stays lazy; the README documents that they must only be used on trusted files.
|
|
112
|
+
fn open_fst(ruby: &Ruby, storage: Storage) -> Result<Fst<Storage>, Error> {
|
|
113
|
+
let verify = matches!(storage, Storage::Mem(_));
|
|
114
|
+
let fst = Fst::new(storage).map_err(|e| fst_error(ruby, e))?;
|
|
115
|
+
if verify {
|
|
116
|
+
match fst.verify() {
|
|
117
|
+
Ok(()) | Err(fst::Error::Fst(fst::raw::Error::ChecksumMissing)) => {}
|
|
118
|
+
Err(e) => return Err(fst_error(ruby, e)),
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
Ok(fst)
|
|
122
|
+
}
|
|
60
123
|
|
|
61
|
-
|
|
62
|
-
|
|
124
|
+
// Runs `f` over the bytes of a Ruby string without copying them. `f` must not
|
|
125
|
+
// call into Ruby; pass it pure-Rust work and build Ruby objects from its result.
|
|
126
|
+
fn with_bytes<R>(s: RString, f: impl FnOnce(&[u8]) -> R + Send) -> R {
|
|
127
|
+
// SAFETY: the slice borrows memory owned by Ruby, which is sound only while
|
|
128
|
+
// no Ruby code runs, since a yield could mutate the string. The `Send` bound
|
|
129
|
+
// stops `f` from capturing a `Ruby` handle or any Ruby value (all `!Send`),
|
|
130
|
+
// and the callers in this file don't fetch one with `Ruby::get` either, so
|
|
131
|
+
// nothing in `f` can run Ruby code. The higher-ranked bound stops the result
|
|
132
|
+
// borrowing the slice, so it can't outlive this call; `s` is a method
|
|
133
|
+
// argument, so Ruby keeps it alive for the duration.
|
|
134
|
+
f(unsafe { s.as_slice() })
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Writes `bytes` to a temporary file beside the target and renames it into
|
|
138
|
+
// place, so a reader never sees a partial file and a live memory map of the old
|
|
139
|
+
// file is never truncated underneath it. A symlink is resolved first so its
|
|
140
|
+
// target is replaced rather than the link, and an existing file's permissions
|
|
141
|
+
// carry over. Where a rename can't work — the target isn't a regular file, or
|
|
142
|
+
// its directory isn't writable — this writes in place, as `fs::write` would.
|
|
143
|
+
fn write_atomically(path: &Path, bytes: &[u8]) -> io::Result<()> {
|
|
144
|
+
let target = match fs::canonicalize(path) {
|
|
145
|
+
Ok(target) => target,
|
|
146
|
+
Err(e) if e.kind() == io::ErrorKind::NotFound && !path.is_symlink() => path.to_path_buf(),
|
|
147
|
+
Err(_) => return fs::write(path, bytes),
|
|
148
|
+
};
|
|
149
|
+
let existing = fs::metadata(&target).ok();
|
|
150
|
+
if existing.as_ref().is_some_and(|meta| !meta.is_file()) {
|
|
151
|
+
return fs::write(&target, bytes);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
let mut tmp_name = OsString::from(target.as_os_str());
|
|
155
|
+
tmp_name.push(format!(".tmp-{}", std::process::id()));
|
|
156
|
+
let tmp_path = PathBuf::from(tmp_name);
|
|
157
|
+
let mut file = match File::create(&tmp_path) {
|
|
158
|
+
Ok(file) => file,
|
|
159
|
+
Err(e) if e.kind() == io::ErrorKind::PermissionDenied => {
|
|
160
|
+
return fs::write(&target, bytes);
|
|
161
|
+
}
|
|
162
|
+
Err(e) => return Err(e),
|
|
163
|
+
};
|
|
164
|
+
|
|
165
|
+
let result = file
|
|
166
|
+
.write_all(bytes)
|
|
167
|
+
.and_then(|()| match &existing {
|
|
168
|
+
Some(meta) => file.set_permissions(meta.permissions()),
|
|
169
|
+
None => Ok(()),
|
|
170
|
+
})
|
|
171
|
+
.and_then(|()| file.sync_all())
|
|
172
|
+
.and_then(|()| fs::rename(&tmp_path, &target));
|
|
173
|
+
if result.is_err() {
|
|
174
|
+
let _ = fs::remove_file(&tmp_path);
|
|
175
|
+
}
|
|
176
|
+
result
|
|
63
177
|
}
|
|
64
178
|
|
|
65
179
|
// Returns the exclusive upper bound for a prefix scan: the smallest byte
|
|
@@ -79,291 +193,230 @@ fn prefix_upper_bound(prefix: &[u8]) -> Option<Vec<u8>> {
|
|
|
79
193
|
}
|
|
80
194
|
|
|
81
195
|
// ---------------------------------------------------------------------------
|
|
82
|
-
// Map
|
|
196
|
+
// Map and Set share one implementation over the raw FST; they differ only in
|
|
197
|
+
// what they yield for each entry. The methods below are generic over that
|
|
198
|
+
// difference and registered once per class in `init`.
|
|
83
199
|
// ---------------------------------------------------------------------------
|
|
84
200
|
|
|
85
|
-
#[
|
|
201
|
+
#[derive(TypedData)]
|
|
202
|
+
#[magnus(class = "RubyFst::Map", free_immediately, size)]
|
|
86
203
|
struct FstMap {
|
|
87
|
-
|
|
204
|
+
fst: Fst<Storage>,
|
|
88
205
|
}
|
|
89
206
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
207
|
+
#[derive(TypedData)]
|
|
208
|
+
#[magnus(class = "RubyFst::Set", free_immediately, size)]
|
|
209
|
+
struct FstSet {
|
|
210
|
+
fst: Fst<Storage>,
|
|
211
|
+
}
|
|
95
212
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
213
|
+
trait FstObject: TypedData + Sized {
|
|
214
|
+
fn from_fst(fst: Fst<Storage>) -> Self;
|
|
215
|
+
fn fst(&self) -> &Fst<Storage>;
|
|
216
|
+
fn yield_entry(ruby: &Ruby, key: &[u8], value: u64) -> Result<(), Error>;
|
|
217
|
+
}
|
|
100
218
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
219
|
+
impl FstObject for FstMap {
|
|
220
|
+
fn from_fst(fst: Fst<Storage>) -> Self {
|
|
221
|
+
Self { fst }
|
|
104
222
|
}
|
|
105
223
|
|
|
106
|
-
fn
|
|
107
|
-
|
|
108
|
-
self.inner.get(key)
|
|
224
|
+
fn fst(&self) -> &Fst<Storage> {
|
|
225
|
+
&self.fst
|
|
109
226
|
}
|
|
110
227
|
|
|
111
|
-
fn
|
|
112
|
-
|
|
113
|
-
|
|
228
|
+
fn yield_entry(ruby: &Ruby, key: &[u8], value: u64) -> Result<(), Error> {
|
|
229
|
+
ruby.yield_values::<_, Value>((ruby.str_from_slice(key), value))
|
|
230
|
+
.map(drop)
|
|
114
231
|
}
|
|
232
|
+
}
|
|
115
233
|
|
|
116
|
-
|
|
117
|
-
|
|
234
|
+
impl FstObject for FstSet {
|
|
235
|
+
fn from_fst(fst: Fst<Storage>) -> Self {
|
|
236
|
+
Self { fst }
|
|
118
237
|
}
|
|
119
238
|
|
|
120
|
-
fn
|
|
121
|
-
self.
|
|
239
|
+
fn fst(&self) -> &Fst<Storage> {
|
|
240
|
+
&self.fst
|
|
122
241
|
}
|
|
123
242
|
|
|
124
|
-
fn
|
|
125
|
-
ruby(
|
|
243
|
+
fn yield_entry(ruby: &Ruby, key: &[u8], _value: u64) -> Result<(), Error> {
|
|
244
|
+
ruby.yield_value::<_, Value>(ruby.str_from_slice(key))
|
|
245
|
+
.map(drop)
|
|
126
246
|
}
|
|
247
|
+
}
|
|
127
248
|
|
|
128
|
-
|
|
129
|
-
|
|
249
|
+
impl DataTypeFunctions for FstMap {
|
|
250
|
+
fn size(&self) -> usize {
|
|
251
|
+
size_of_val(self) + heap_bytes(&self.fst)
|
|
130
252
|
}
|
|
253
|
+
}
|
|
131
254
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
match floor_lookup(self.inner.as_fst(), key) {
|
|
136
|
-
Some((found_key, value)) => {
|
|
137
|
-
let arr = r.ary_new_capa(2);
|
|
138
|
-
arr.push(r.str_from_slice(&found_key))?;
|
|
139
|
-
arr.push(value)?;
|
|
140
|
-
Ok(Some(arr))
|
|
141
|
-
}
|
|
142
|
-
None => Ok(None),
|
|
143
|
-
}
|
|
255
|
+
impl DataTypeFunctions for FstSet {
|
|
256
|
+
fn size(&self) -> usize {
|
|
257
|
+
size_of_val(self) + heap_bytes(&self.fst)
|
|
144
258
|
}
|
|
259
|
+
}
|
|
145
260
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
261
|
+
fn heap_bytes(fst: &Fst<Storage>) -> usize {
|
|
262
|
+
match fst.as_inner() {
|
|
263
|
+
Storage::Mem(bytes) => bytes.len(),
|
|
264
|
+
Storage::Mmap(_) => 0,
|
|
149
265
|
}
|
|
266
|
+
}
|
|
150
267
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
match stream.next() {
|
|
156
|
-
Some((k, v)) => {
|
|
157
|
-
let arr = r.ary_new_capa(2);
|
|
158
|
-
arr.push(r.str_from_slice(k))?;
|
|
159
|
-
arr.push(v)?;
|
|
160
|
-
Ok(Some(arr))
|
|
161
|
-
}
|
|
162
|
-
None => Ok(None),
|
|
163
|
-
}
|
|
164
|
-
}
|
|
268
|
+
fn new<T: FstObject>(ruby: &Ruby, bytes: RString) -> Result<T, Error> {
|
|
269
|
+
let bytes = with_bytes(bytes, <[u8]>::to_vec);
|
|
270
|
+
open_fst(ruby, Storage::mem(ruby, bytes)).map(T::from_fst)
|
|
271
|
+
}
|
|
165
272
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
stream.next().map(|(_, v)| v)
|
|
170
|
-
}
|
|
273
|
+
fn from_path<T: FstObject>(ruby: &Ruby, path: PathBuf) -> Result<T, Error> {
|
|
274
|
+
open_fst(ruby, Storage::read(ruby, &path)?).map(T::from_fst)
|
|
275
|
+
}
|
|
171
276
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
while let Some((key, value)) = stream.next() {
|
|
176
|
-
let rb_key = r.str_from_slice(key);
|
|
177
|
-
let _: Value = r.yield_values((rb_key, value))?;
|
|
178
|
-
}
|
|
179
|
-
Ok(())
|
|
180
|
-
}
|
|
277
|
+
fn from_path_mmap<T: FstObject>(ruby: &Ruby, path: PathBuf) -> Result<T, Error> {
|
|
278
|
+
open_fst(ruby, Storage::mmap(ruby, &path)?).map(T::from_fst)
|
|
279
|
+
}
|
|
181
280
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
let mut builder = self.inner.range();
|
|
187
|
-
if let Some(ref b) = ge_bytes {
|
|
188
|
-
builder = builder.ge(b);
|
|
189
|
-
}
|
|
190
|
-
if let Some(ref b) = le_bytes {
|
|
191
|
-
builder = builder.le(b);
|
|
192
|
-
}
|
|
193
|
-
let mut stream = builder.into_stream();
|
|
194
|
-
while let Some((key, value)) = stream.next() {
|
|
195
|
-
let rb_key = r.str_from_slice(key);
|
|
196
|
-
let _: Value = r.yield_values((rb_key, value))?;
|
|
197
|
-
}
|
|
198
|
-
Ok(())
|
|
199
|
-
}
|
|
281
|
+
fn contains<T: FstObject>(rb_self: &T, key: RString) -> bool {
|
|
282
|
+
let fst = rb_self.fst();
|
|
283
|
+
with_bytes(key, |key| fst.contains_key(key))
|
|
284
|
+
}
|
|
200
285
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
let upper = prefix_upper_bound(&prefix_bytes);
|
|
205
|
-
let mut builder = self.inner.range().ge(&prefix_bytes);
|
|
206
|
-
if let Some(ref u) = upper {
|
|
207
|
-
builder = builder.lt(u);
|
|
208
|
-
}
|
|
209
|
-
let mut stream = builder.into_stream();
|
|
210
|
-
while let Some((key, value)) = stream.next() {
|
|
211
|
-
let rb_key = r.str_from_slice(key);
|
|
212
|
-
let _: Value = r.yield_values((rb_key, value))?;
|
|
213
|
-
}
|
|
214
|
-
Ok(())
|
|
215
|
-
}
|
|
286
|
+
fn len<T: FstObject>(rb_self: &T) -> usize {
|
|
287
|
+
rb_self.fst().len()
|
|
288
|
+
}
|
|
216
289
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
let lev = Levenshtein::new(&query, distance).map_err(err)?;
|
|
220
|
-
let mut stream = self.inner.search(lev).into_stream();
|
|
221
|
-
while let Some((key, value)) = stream.next() {
|
|
222
|
-
let rb_key = r.str_from_slice(key);
|
|
223
|
-
let _: Value = r.yield_values((rb_key, value))?;
|
|
224
|
-
}
|
|
225
|
-
Ok(())
|
|
226
|
-
}
|
|
290
|
+
fn is_empty<T: FstObject>(rb_self: &T) -> bool {
|
|
291
|
+
rb_self.fst().is_empty()
|
|
227
292
|
}
|
|
228
293
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
294
|
+
fn to_bytes<T: FstObject>(ruby: &Ruby, rb_self: &T) -> RString {
|
|
295
|
+
ruby.str_from_slice(rb_self.fst().as_bytes())
|
|
296
|
+
}
|
|
232
297
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
inner: RefCell<Option<fst::MapBuilder<Vec<u8>>>>,
|
|
298
|
+
fn save<T: FstObject>(ruby: &Ruby, rb_self: &T, path: PathBuf) -> Result<(), Error> {
|
|
299
|
+
write_atomically(&path, rb_self.fst().as_bytes()).map_err(|e| io_error(ruby, &path, e))
|
|
236
300
|
}
|
|
237
301
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
302
|
+
fn yield_all<T: FstObject, A: Automaton>(
|
|
303
|
+
ruby: &Ruby,
|
|
304
|
+
mut stream: Stream<'_, A>,
|
|
305
|
+
) -> Result<(), Error> {
|
|
306
|
+
while let Some((key, output)) = stream.next() {
|
|
307
|
+
T::yield_entry(ruby, key, output.value())?;
|
|
243
308
|
}
|
|
309
|
+
Ok(())
|
|
310
|
+
}
|
|
244
311
|
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
let b = guard.as_mut().ok_or_else(|| err("builder already finished"))?;
|
|
249
|
-
b.insert(&key, value).map_err(err)
|
|
250
|
-
}
|
|
312
|
+
fn each<T: FstObject>(ruby: &Ruby, rb_self: &T) -> Result<(), Error> {
|
|
313
|
+
yield_all::<T, _>(ruby, rb_self.fst().stream())
|
|
314
|
+
}
|
|
251
315
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
316
|
+
fn range<T: FstObject>(
|
|
317
|
+
ruby: &Ruby,
|
|
318
|
+
rb_self: &T,
|
|
319
|
+
ge: Option<RString>,
|
|
320
|
+
le: Option<RString>,
|
|
321
|
+
) -> Result<(), Error> {
|
|
322
|
+
let mut builder = rb_self.fst().range();
|
|
323
|
+
if let Some(ge) = ge {
|
|
324
|
+
builder = with_bytes(ge, |bound| builder.ge(bound));
|
|
257
325
|
}
|
|
326
|
+
if let Some(le) = le {
|
|
327
|
+
builder = with_bytes(le, |bound| builder.le(bound));
|
|
328
|
+
}
|
|
329
|
+
yield_all::<T, _>(ruby, builder.into_stream())
|
|
258
330
|
}
|
|
259
331
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
332
|
+
fn starts_with<T: FstObject>(ruby: &Ruby, rb_self: &T, prefix: RString) -> Result<(), Error> {
|
|
333
|
+
let fst = rb_self.fst();
|
|
334
|
+
let builder = with_bytes(prefix, |prefix| {
|
|
335
|
+
let builder = fst.range().ge(prefix);
|
|
336
|
+
match prefix_upper_bound(prefix) {
|
|
337
|
+
Some(upper) => builder.lt(upper),
|
|
338
|
+
None => builder,
|
|
339
|
+
}
|
|
340
|
+
});
|
|
341
|
+
yield_all::<T, _>(ruby, builder.into_stream())
|
|
267
342
|
}
|
|
268
343
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
344
|
+
fn search_levenshtein<T: FstObject>(
|
|
345
|
+
ruby: &Ruby,
|
|
346
|
+
rb_self: &T,
|
|
347
|
+
query: String,
|
|
348
|
+
distance: u32,
|
|
349
|
+
) -> Result<(), Error> {
|
|
350
|
+
let automaton = Levenshtein::new(&query, distance)
|
|
351
|
+
.map_err(|e| Error::new(ruby.exception_arg_error(), e.to_string()))?;
|
|
352
|
+
yield_all::<T, _>(ruby, rb_self.fst().search(automaton).into_stream())
|
|
353
|
+
}
|
|
274
354
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
Ok(Self { inner })
|
|
278
|
-
}
|
|
355
|
+
// Map-only lookups. Keys come back as freshly allocated Ruby strings, so each
|
|
356
|
+
// lookup finishes with the borrowed key before building its result.
|
|
279
357
|
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
}
|
|
358
|
+
fn get(rb_self: &FstMap, key: RString) -> Option<u64> {
|
|
359
|
+
with_bytes(key, |key| rb_self.fst.get(key).map(Output::value))
|
|
360
|
+
}
|
|
284
361
|
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
362
|
+
fn get_le(ruby: &Ruby, rb_self: &FstMap, key: RString) -> Option<(RString, u64)> {
|
|
363
|
+
let (found, value) = with_bytes(key, |key| floor_entry(&rb_self.fst, key))?;
|
|
364
|
+
Some((ruby.str_from_slice(&found), value))
|
|
365
|
+
}
|
|
289
366
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
367
|
+
fn get_le_value(rb_self: &FstMap, key: RString) -> Option<u64> {
|
|
368
|
+
with_bytes(key, |key| floor_value(&rb_self.fst, key))
|
|
369
|
+
}
|
|
293
370
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
371
|
+
fn get_ge(ruby: &Ruby, rb_self: &FstMap, key: RString) -> Option<(RString, u64)> {
|
|
372
|
+
let mut stream = with_bytes(key, |key| rb_self.fst.range().ge(key)).into_stream();
|
|
373
|
+
let (found, output) = stream.next()?;
|
|
374
|
+
Some((ruby.str_from_slice(found), output.value()))
|
|
375
|
+
}
|
|
297
376
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
377
|
+
fn get_ge_value(rb_self: &FstMap, key: RString) -> Option<u64> {
|
|
378
|
+
let builder = with_bytes(key, |key| rb_self.fst.range().ge(key));
|
|
379
|
+
builder
|
|
380
|
+
.into_stream()
|
|
381
|
+
.next()
|
|
382
|
+
.map(|(_, output)| output.value())
|
|
383
|
+
}
|
|
301
384
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
385
|
+
// ---------------------------------------------------------------------------
|
|
386
|
+
// Builders
|
|
387
|
+
// ---------------------------------------------------------------------------
|
|
305
388
|
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
let rb_key = r.str_from_slice(key);
|
|
311
|
-
let _: Value = r.yield_value(rb_key)?;
|
|
312
|
-
}
|
|
313
|
-
Ok(())
|
|
314
|
-
}
|
|
389
|
+
#[magnus::wrap(class = "RubyFst::MapBuilder", free_immediately, size)]
|
|
390
|
+
struct FstMapBuilder {
|
|
391
|
+
inner: RefCell<Option<fst::MapBuilder<Vec<u8>>>>,
|
|
392
|
+
}
|
|
315
393
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
let mut builder = self.inner.range();
|
|
321
|
-
if let Some(ref b) = ge_bytes {
|
|
322
|
-
builder = builder.ge(b);
|
|
323
|
-
}
|
|
324
|
-
if let Some(ref b) = le_bytes {
|
|
325
|
-
builder = builder.le(b);
|
|
326
|
-
}
|
|
327
|
-
let mut stream = builder.into_stream();
|
|
328
|
-
while let Some(key) = stream.next() {
|
|
329
|
-
let rb_key = r.str_from_slice(key);
|
|
330
|
-
let _: Value = r.yield_value(rb_key)?;
|
|
394
|
+
impl FstMapBuilder {
|
|
395
|
+
fn new() -> Self {
|
|
396
|
+
Self {
|
|
397
|
+
inner: RefCell::new(Some(fst::MapBuilder::memory())),
|
|
331
398
|
}
|
|
332
|
-
Ok(())
|
|
333
399
|
}
|
|
334
400
|
|
|
335
|
-
fn
|
|
336
|
-
let
|
|
337
|
-
let
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
builder = builder.lt(u);
|
|
342
|
-
}
|
|
343
|
-
let mut stream = builder.into_stream();
|
|
344
|
-
while let Some(key) = stream.next() {
|
|
345
|
-
let rb_key = r.str_from_slice(key);
|
|
346
|
-
let _: Value = r.yield_value(rb_key)?;
|
|
347
|
-
}
|
|
348
|
-
Ok(())
|
|
401
|
+
fn insert(ruby: &Ruby, rb_self: &Self, key: RString, value: u64) -> Result<(), Error> {
|
|
402
|
+
let mut guard = rb_self.inner.borrow_mut();
|
|
403
|
+
let builder = guard
|
|
404
|
+
.as_mut()
|
|
405
|
+
.ok_or_else(|| fst_error(ruby, "builder already finished"))?;
|
|
406
|
+
with_bytes(key, |key| builder.insert(key, value)).map_err(|e| map_fst_error(ruby, e))
|
|
349
407
|
}
|
|
350
408
|
|
|
351
|
-
fn
|
|
352
|
-
let
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
Ok(())
|
|
409
|
+
fn finish(ruby: &Ruby, rb_self: &Self) -> Result<RString, Error> {
|
|
410
|
+
let builder = rb_self
|
|
411
|
+
.inner
|
|
412
|
+
.borrow_mut()
|
|
413
|
+
.take()
|
|
414
|
+
.ok_or_else(|| fst_error(ruby, "builder already finished"))?;
|
|
415
|
+
let bytes = builder.into_inner().map_err(|e| map_fst_error(ruby, e))?;
|
|
416
|
+
Ok(ruby.str_from_slice(&bytes))
|
|
360
417
|
}
|
|
361
418
|
}
|
|
362
419
|
|
|
363
|
-
// ---------------------------------------------------------------------------
|
|
364
|
-
// SetBuilder
|
|
365
|
-
// ---------------------------------------------------------------------------
|
|
366
|
-
|
|
367
420
|
#[magnus::wrap(class = "RubyFst::SetBuilder", free_immediately, size)]
|
|
368
421
|
struct FstSetBuilder {
|
|
369
422
|
inner: RefCell<Option<fst::SetBuilder<Vec<u8>>>>,
|
|
@@ -376,47 +429,91 @@ impl FstSetBuilder {
|
|
|
376
429
|
}
|
|
377
430
|
}
|
|
378
431
|
|
|
379
|
-
fn insert(&
|
|
380
|
-
let
|
|
381
|
-
let
|
|
382
|
-
|
|
383
|
-
|
|
432
|
+
fn insert(ruby: &Ruby, rb_self: &Self, key: RString) -> Result<(), Error> {
|
|
433
|
+
let mut guard = rb_self.inner.borrow_mut();
|
|
434
|
+
let builder = guard
|
|
435
|
+
.as_mut()
|
|
436
|
+
.ok_or_else(|| fst_error(ruby, "builder already finished"))?;
|
|
437
|
+
with_bytes(key, |key| builder.insert(key)).map_err(|e| map_fst_error(ruby, e))
|
|
384
438
|
}
|
|
385
439
|
|
|
386
|
-
fn finish(&
|
|
387
|
-
let
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
440
|
+
fn finish(ruby: &Ruby, rb_self: &Self) -> Result<RString, Error> {
|
|
441
|
+
let builder = rb_self
|
|
442
|
+
.inner
|
|
443
|
+
.borrow_mut()
|
|
444
|
+
.take()
|
|
445
|
+
.ok_or_else(|| fst_error(ruby, "builder already finished"))?;
|
|
446
|
+
let bytes = builder.into_inner().map_err(|e| map_fst_error(ruby, e))?;
|
|
447
|
+
Ok(ruby.str_from_slice(&bytes))
|
|
391
448
|
}
|
|
392
449
|
}
|
|
393
450
|
|
|
394
451
|
// ---------------------------------------------------------------------------
|
|
395
452
|
// Floor lookup (get_le): greatest key <= query.
|
|
396
453
|
//
|
|
397
|
-
// The
|
|
398
|
-
//
|
|
399
|
-
//
|
|
400
|
-
//
|
|
401
|
-
//
|
|
454
|
+
// The fst crate exposes `range().ge(...)` natively but not a floor operation,
|
|
455
|
+
// so this walks the FST manually. While descending along the query it keeps
|
|
456
|
+
// the best candidate seen so far: the rightmost transition strictly below the
|
|
457
|
+
// query byte at that depth, or else the prefix itself if it is a key. A deeper
|
|
458
|
+
// candidate always beats a shallower one, because it agrees with the query for
|
|
459
|
+
// longer, so only the latest needs keeping. An exact match beats them all.
|
|
402
460
|
// ---------------------------------------------------------------------------
|
|
403
461
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
462
|
+
enum Floor {
|
|
463
|
+
// `key[..len]` is itself in the FST, with this output.
|
|
464
|
+
Prefix {
|
|
465
|
+
len: usize,
|
|
466
|
+
output: Output,
|
|
467
|
+
},
|
|
468
|
+
// `key[..len]`, then `input`, then the rightmost path below `addr`.
|
|
469
|
+
Branch {
|
|
470
|
+
len: usize,
|
|
471
|
+
input: u8,
|
|
472
|
+
addr: CompiledAddr,
|
|
473
|
+
output: Output,
|
|
474
|
+
},
|
|
411
475
|
}
|
|
412
476
|
|
|
413
|
-
fn
|
|
414
|
-
let
|
|
415
|
-
|
|
416
|
-
|
|
477
|
+
fn find_floor<D: AsRef<[u8]>>(fst: &Fst<D>, key: &[u8]) -> Option<Floor> {
|
|
478
|
+
let mut node = fst.root();
|
|
479
|
+
let mut output = Output::zero();
|
|
480
|
+
let mut best = None;
|
|
481
|
+
|
|
482
|
+
for (depth, &byte) in key.iter().enumerate() {
|
|
483
|
+
if let Some(i) = max_lesser_transition(&node, byte) {
|
|
484
|
+
let t = node.transition(i);
|
|
485
|
+
best = Some(Floor::Branch {
|
|
486
|
+
len: depth,
|
|
487
|
+
input: t.inp,
|
|
488
|
+
addr: t.addr,
|
|
489
|
+
output: output.cat(t.out),
|
|
490
|
+
});
|
|
491
|
+
} else if node.is_final() {
|
|
492
|
+
best = Some(Floor::Prefix {
|
|
493
|
+
len: depth,
|
|
494
|
+
output: output.cat(node.final_output()),
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
let Some(i) = node.find_input(byte) else {
|
|
499
|
+
return best;
|
|
500
|
+
};
|
|
501
|
+
let t = node.transition(i);
|
|
502
|
+
output = output.cat(t.out);
|
|
503
|
+
node = fst.node(t.addr);
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
if node.is_final() {
|
|
507
|
+
return Some(Floor::Prefix {
|
|
508
|
+
len: key.len(),
|
|
509
|
+
output: output.cat(node.final_output()),
|
|
510
|
+
});
|
|
417
511
|
}
|
|
418
|
-
|
|
419
|
-
|
|
512
|
+
best
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
fn max_lesser_transition(node: &Node, byte: u8) -> Option<usize> {
|
|
516
|
+
let (mut lo, mut hi) = (0, node.len());
|
|
420
517
|
while lo < hi {
|
|
421
518
|
let mid = lo + (hi - lo) / 2;
|
|
422
519
|
if node.transition(mid).inp < byte {
|
|
@@ -428,211 +525,97 @@ fn find_max_lesser(node: &Node, byte: u8) -> Option<usize> {
|
|
|
428
525
|
lo.checked_sub(1)
|
|
429
526
|
}
|
|
430
527
|
|
|
431
|
-
|
|
528
|
+
// Follows the last transition from `addr` down to a leaf, passing each input
|
|
529
|
+
// byte to `on_byte`, and returns the accumulated value.
|
|
530
|
+
fn rightmost<D: AsRef<[u8]>>(
|
|
432
531
|
fst: &Fst<D>,
|
|
433
532
|
addr: CompiledAddr,
|
|
434
|
-
output: Output,
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
let mut out = output;
|
|
438
|
-
let mut suffix = Vec::new();
|
|
439
|
-
|
|
440
|
-
while !node.is_empty() {
|
|
441
|
-
let last = node.len() - 1;
|
|
442
|
-
let t = node.transition(last);
|
|
443
|
-
suffix.push(t.inp);
|
|
444
|
-
out = out.cat(t.out);
|
|
445
|
-
node = fst.node(t.addr);
|
|
446
|
-
}
|
|
447
|
-
|
|
448
|
-
(suffix, out.cat(node.final_output()).value())
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
fn rightmost_value<D: AsRef<[u8]>>(fst: &Fst<D>, addr: CompiledAddr, output: Output) -> u64 {
|
|
533
|
+
mut output: Output,
|
|
534
|
+
mut on_byte: impl FnMut(u8),
|
|
535
|
+
) -> u64 {
|
|
452
536
|
let mut node = fst.node(addr);
|
|
453
|
-
let mut out = output;
|
|
454
|
-
|
|
455
537
|
while !node.is_empty() {
|
|
456
|
-
let
|
|
457
|
-
|
|
458
|
-
|
|
538
|
+
let t = node.transition(node.len() - 1);
|
|
539
|
+
on_byte(t.inp);
|
|
540
|
+
output = output.cat(t.out);
|
|
459
541
|
node = fst.node(t.addr);
|
|
460
542
|
}
|
|
461
|
-
|
|
462
|
-
out.cat(node.final_output()).value()
|
|
543
|
+
output.cat(node.final_output()).value()
|
|
463
544
|
}
|
|
464
545
|
|
|
465
|
-
fn
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
None
|
|
473
|
-
};
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
let mut node = root;
|
|
477
|
-
let mut output = Output::zero();
|
|
478
|
-
let mut stack: Vec<Frame> = Vec::with_capacity(key.len());
|
|
479
|
-
let mut matched: usize = 0;
|
|
480
|
-
|
|
481
|
-
for &byte in key.iter() {
|
|
482
|
-
let lesser = find_max_lesser(&node, byte);
|
|
483
|
-
|
|
484
|
-
stack.push(Frame {
|
|
485
|
-
node_addr: node.addr(),
|
|
546
|
+
fn floor_entry<D: AsRef<[u8]>>(fst: &Fst<D>, key: &[u8]) -> Option<(Vec<u8>, u64)> {
|
|
547
|
+
Some(match find_floor(fst, key)? {
|
|
548
|
+
Floor::Prefix { len, output } => (key[..len].to_vec(), output.value()),
|
|
549
|
+
Floor::Branch {
|
|
550
|
+
len,
|
|
551
|
+
input,
|
|
552
|
+
addr,
|
|
486
553
|
output,
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
match node.find_input(byte) {
|
|
494
|
-
Some(idx) => {
|
|
495
|
-
let t = node.transition(idx);
|
|
496
|
-
output = output.cat(t.out);
|
|
497
|
-
node = fst.node(t.addr);
|
|
498
|
-
matched += 1;
|
|
499
|
-
}
|
|
500
|
-
None => break,
|
|
501
|
-
}
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
if matched == key.len() && node.is_final() {
|
|
505
|
-
return Some((key.to_vec(), output.cat(node.final_output()).value()));
|
|
506
|
-
}
|
|
507
|
-
|
|
508
|
-
while let Some(frame) = stack.pop() {
|
|
509
|
-
if let Some(j) = frame.max_lesser_idx {
|
|
510
|
-
let frame_node = fst.node(frame.node_addr);
|
|
511
|
-
let t = frame_node.transition(j);
|
|
512
|
-
let mut result = key[..frame.prefix_len].to_vec();
|
|
513
|
-
result.push(t.inp);
|
|
514
|
-
let branch_output = frame.output.cat(t.out);
|
|
515
|
-
let (suffix, val) = rightmost_to_leaf(fst, t.addr, branch_output);
|
|
516
|
-
result.extend(suffix);
|
|
517
|
-
return Some((result, val));
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
if frame.is_final {
|
|
521
|
-
return Some((key[..frame.prefix_len].to_vec(), frame.final_value));
|
|
554
|
+
} => {
|
|
555
|
+
let mut found = key[..len].to_vec();
|
|
556
|
+
found.push(input);
|
|
557
|
+
let value = rightmost(fst, addr, output, |byte| found.push(byte));
|
|
558
|
+
(found, value)
|
|
522
559
|
}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
None
|
|
560
|
+
})
|
|
526
561
|
}
|
|
527
562
|
|
|
528
563
|
fn floor_value<D: AsRef<[u8]>>(fst: &Fst<D>, key: &[u8]) -> Option<u64> {
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
Some(root.final_output().value())
|
|
534
|
-
} else {
|
|
535
|
-
None
|
|
536
|
-
};
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
let mut node = root;
|
|
540
|
-
let mut output = Output::zero();
|
|
541
|
-
let mut stack: Vec<Frame> = Vec::with_capacity(key.len());
|
|
542
|
-
let mut matched: usize = 0;
|
|
543
|
-
|
|
544
|
-
for &byte in key.iter() {
|
|
545
|
-
let lesser = find_max_lesser(&node, byte);
|
|
546
|
-
|
|
547
|
-
stack.push(Frame {
|
|
548
|
-
node_addr: node.addr(),
|
|
549
|
-
output,
|
|
550
|
-
prefix_len: matched,
|
|
551
|
-
max_lesser_idx: lesser,
|
|
552
|
-
is_final: node.is_final(),
|
|
553
|
-
final_value: output.cat(node.final_output()).value(),
|
|
554
|
-
});
|
|
555
|
-
|
|
556
|
-
match node.find_input(byte) {
|
|
557
|
-
Some(idx) => {
|
|
558
|
-
let t = node.transition(idx);
|
|
559
|
-
output = output.cat(t.out);
|
|
560
|
-
node = fst.node(t.addr);
|
|
561
|
-
matched += 1;
|
|
562
|
-
}
|
|
563
|
-
None => break,
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
if matched == key.len() && node.is_final() {
|
|
568
|
-
return Some(output.cat(node.final_output()).value());
|
|
569
|
-
}
|
|
570
|
-
|
|
571
|
-
while let Some(frame) = stack.pop() {
|
|
572
|
-
if let Some(j) = frame.max_lesser_idx {
|
|
573
|
-
let frame_node = fst.node(frame.node_addr);
|
|
574
|
-
let t = frame_node.transition(j);
|
|
575
|
-
let branch_output = frame.output.cat(t.out);
|
|
576
|
-
return Some(rightmost_value(fst, t.addr, branch_output));
|
|
577
|
-
}
|
|
578
|
-
|
|
579
|
-
if frame.is_final {
|
|
580
|
-
return Some(frame.final_value);
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
None
|
|
564
|
+
Some(match find_floor(fst, key)? {
|
|
565
|
+
Floor::Prefix { output, .. } => output.value(),
|
|
566
|
+
Floor::Branch { addr, output, .. } => rightmost(fst, addr, output, |_| {}),
|
|
567
|
+
})
|
|
585
568
|
}
|
|
586
569
|
|
|
587
570
|
// ---------------------------------------------------------------------------
|
|
588
571
|
// Init
|
|
589
572
|
// ---------------------------------------------------------------------------
|
|
590
573
|
|
|
574
|
+
// `method!` can't name a generic parameter from an enclosing function, so the
|
|
575
|
+
// shared methods are registered through a macro instead.
|
|
576
|
+
macro_rules! define_fst_methods {
|
|
577
|
+
($class:expr, $type:ty) => {{
|
|
578
|
+
let class = $class;
|
|
579
|
+
class.define_singleton_method("new", function!(new::<$type>, 1))?;
|
|
580
|
+
class.define_singleton_method("from_path", function!(from_path::<$type>, 1))?;
|
|
581
|
+
class.define_singleton_method("from_path_mmap", function!(from_path_mmap::<$type>, 1))?;
|
|
582
|
+
class.define_method("contains?", method!(contains::<$type>, 1))?;
|
|
583
|
+
class.define_method("length", method!(len::<$type>, 0))?;
|
|
584
|
+
class.define_method("size", method!(len::<$type>, 0))?;
|
|
585
|
+
class.define_method("empty?", method!(is_empty::<$type>, 0))?;
|
|
586
|
+
class.define_method("to_bytes", method!(to_bytes::<$type>, 0))?;
|
|
587
|
+
class.define_method("save", method!(save::<$type>, 1))?;
|
|
588
|
+
class.define_method("each", method!(each::<$type>, 0))?;
|
|
589
|
+
class.define_method("range", method!(range::<$type>, 2))?;
|
|
590
|
+
class.define_method("starts_with", method!(starts_with::<$type>, 1))?;
|
|
591
|
+
class.define_method(
|
|
592
|
+
"search_levenshtein",
|
|
593
|
+
method!(search_levenshtein::<$type>, 2),
|
|
594
|
+
)?;
|
|
595
|
+
class
|
|
596
|
+
}};
|
|
597
|
+
}
|
|
598
|
+
|
|
591
599
|
#[magnus::init]
|
|
592
600
|
fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
593
601
|
let module = ruby.define_module("RubyFst")?;
|
|
602
|
+
module.define_error("Error", ruby.exception_runtime_error())?;
|
|
594
603
|
|
|
595
|
-
let map_class = module.define_class("Map", ruby.class_object())
|
|
596
|
-
map_class.
|
|
597
|
-
map_class.
|
|
598
|
-
map_class.
|
|
599
|
-
map_class.define_method("
|
|
600
|
-
map_class.define_method("
|
|
601
|
-
map_class.define_method("
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
map_class.define_method("empty?", method!(FstMap::is_empty, 0))?;
|
|
605
|
-
map_class.define_method("to_bytes", method!(FstMap::to_bytes, 0))?;
|
|
606
|
-
map_class.define_method("save", method!(FstMap::save, 1))?;
|
|
607
|
-
map_class.define_method("get_le", method!(FstMap::get_le, 1))?;
|
|
608
|
-
map_class.define_method("get_le_value", method!(FstMap::get_le_value, 1))?;
|
|
609
|
-
map_class.define_method("get_ge", method!(FstMap::get_ge, 1))?;
|
|
610
|
-
map_class.define_method("get_ge_value", method!(FstMap::get_ge_value, 1))?;
|
|
611
|
-
map_class.define_method("each", method!(FstMap::each, 0))?;
|
|
612
|
-
map_class.define_method("range", method!(FstMap::range, 2))?;
|
|
613
|
-
map_class.define_method("starts_with", method!(FstMap::starts_with, 1))?;
|
|
614
|
-
map_class.define_method("search_levenshtein", method!(FstMap::search_levenshtein, 2))?;
|
|
604
|
+
let map_class = define_fst_methods!(module.define_class("Map", ruby.class_object())?, FstMap);
|
|
605
|
+
map_class.define_method("get", method!(get, 1))?;
|
|
606
|
+
map_class.define_method("[]", method!(get, 1))?;
|
|
607
|
+
map_class.define_method("get_le", method!(get_le, 1))?;
|
|
608
|
+
map_class.define_method("get_le_value", method!(get_le_value, 1))?;
|
|
609
|
+
map_class.define_method("get_ge", method!(get_ge, 1))?;
|
|
610
|
+
map_class.define_method("get_ge_value", method!(get_ge_value, 1))?;
|
|
611
|
+
|
|
612
|
+
define_fst_methods!(module.define_class("Set", ruby.class_object())?, FstSet);
|
|
615
613
|
|
|
616
614
|
let map_builder = module.define_class("MapBuilder", ruby.class_object())?;
|
|
617
615
|
map_builder.define_singleton_method("new", function!(FstMapBuilder::new, 0))?;
|
|
618
616
|
map_builder.define_method("insert", method!(FstMapBuilder::insert, 2))?;
|
|
619
617
|
map_builder.define_method("finish", method!(FstMapBuilder::finish, 0))?;
|
|
620
618
|
|
|
621
|
-
let set_class = module.define_class("Set", ruby.class_object())?;
|
|
622
|
-
set_class.define_singleton_method("new", function!(FstSet::new, 1))?;
|
|
623
|
-
set_class.define_singleton_method("from_path", function!(FstSet::from_path, 1))?;
|
|
624
|
-
set_class.define_singleton_method("from_path_mmap", function!(FstSet::from_path_mmap, 1))?;
|
|
625
|
-
set_class.define_method("contains?", method!(FstSet::contains, 1))?;
|
|
626
|
-
set_class.define_method("length", method!(FstSet::len, 0))?;
|
|
627
|
-
set_class.define_method("size", method!(FstSet::len, 0))?;
|
|
628
|
-
set_class.define_method("empty?", method!(FstSet::is_empty, 0))?;
|
|
629
|
-
set_class.define_method("to_bytes", method!(FstSet::to_bytes, 0))?;
|
|
630
|
-
set_class.define_method("save", method!(FstSet::save, 1))?;
|
|
631
|
-
set_class.define_method("each", method!(FstSet::each, 0))?;
|
|
632
|
-
set_class.define_method("range", method!(FstSet::range, 2))?;
|
|
633
|
-
set_class.define_method("starts_with", method!(FstSet::starts_with, 1))?;
|
|
634
|
-
set_class.define_method("search_levenshtein", method!(FstSet::search_levenshtein, 2))?;
|
|
635
|
-
|
|
636
619
|
let set_builder = module.define_class("SetBuilder", ruby.class_object())?;
|
|
637
620
|
set_builder.define_singleton_method("new", function!(FstSetBuilder::new, 0))?;
|
|
638
621
|
set_builder.define_method("insert", method!(FstSetBuilder::insert, 1))?;
|
data/lib/ruby_fst/version.rb
CHANGED
data/lib/ruby_fst.rb
CHANGED
|
@@ -9,6 +9,12 @@ end
|
|
|
9
9
|
|
|
10
10
|
module RubyFst
|
|
11
11
|
module RangeQuery
|
|
12
|
+
def each(&block)
|
|
13
|
+
return enum_for(:each) { size } unless block
|
|
14
|
+
|
|
15
|
+
_each(&block)
|
|
16
|
+
end
|
|
17
|
+
|
|
12
18
|
def range(ge: nil, le: nil, &block)
|
|
13
19
|
return enum_for(:range, ge:, le:) unless block
|
|
14
20
|
|
|
@@ -20,14 +26,22 @@ module RubyFst
|
|
|
20
26
|
|
|
21
27
|
_starts_with(prefix, &block)
|
|
22
28
|
end
|
|
29
|
+
|
|
30
|
+
def search_levenshtein(query, distance, &block)
|
|
31
|
+
return enum_for(:search_levenshtein, query, distance) unless block
|
|
32
|
+
|
|
33
|
+
_search_levenshtein(query, distance, &block)
|
|
34
|
+
end
|
|
23
35
|
end
|
|
24
36
|
|
|
25
37
|
class Map
|
|
26
38
|
include Enumerable
|
|
27
39
|
|
|
40
|
+
alias _each each
|
|
28
41
|
alias _range range
|
|
29
42
|
alias _starts_with starts_with
|
|
30
|
-
|
|
43
|
+
alias _search_levenshtein search_levenshtein
|
|
44
|
+
private :_each, :_range, :_starts_with, :_search_levenshtein
|
|
31
45
|
|
|
32
46
|
prepend RangeQuery
|
|
33
47
|
end
|
|
@@ -35,9 +49,11 @@ module RubyFst
|
|
|
35
49
|
class Set
|
|
36
50
|
include Enumerable
|
|
37
51
|
|
|
52
|
+
alias _each each
|
|
38
53
|
alias _range range
|
|
39
54
|
alias _starts_with starts_with
|
|
40
|
-
|
|
55
|
+
alias _search_levenshtein search_levenshtein
|
|
56
|
+
private :_each, :_range, :_starts_with, :_search_levenshtein
|
|
41
57
|
|
|
42
58
|
prepend RangeQuery
|
|
43
59
|
end
|