lindera 6.2.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +8 -6
- data/README.md +3 -3
- data/extconf.rb +6 -4
- data/src/dictionary.rs +130 -16
- data/src/metadata.rs +36 -38
- data/src/mode.rs +212 -154
- data/src/schema.rs +20 -9
- data/src/token.rs +16 -9
- data/src/tokenizer.rs +270 -31
- data/src/trainer.rs +172 -158
- data/src/util.rs +65 -3
- metadata +1 -43
data/src/mode.rs
CHANGED
|
@@ -1,130 +1,172 @@
|
|
|
1
1
|
//! Tokenization modes and penalty configurations.
|
|
2
2
|
//!
|
|
3
3
|
//! This module defines the different tokenization modes available and their
|
|
4
|
-
//! penalty configurations for controlling segmentation behavior.
|
|
4
|
+
//! penalty configurations for controlling segmentation behavior. A
|
|
5
|
+
//! `Lindera::Mode` holds the penalty of decompose mode, as the Rust
|
|
6
|
+
//! `lindera::mode::Mode` does, and is what `TokenizerBuilder#set_mode` and
|
|
7
|
+
//! `Tokenizer.new` take (#1218).
|
|
5
8
|
|
|
6
9
|
use magnus::prelude::*;
|
|
7
|
-
use magnus::
|
|
10
|
+
use magnus::scan_args::scan_args;
|
|
11
|
+
use magnus::{Error, Integer, Ruby, TryConvert, Value, function, method};
|
|
8
12
|
|
|
9
13
|
use lindera::mode::{Mode as LinderaMode, Penalty as LinderaPenalty};
|
|
10
14
|
|
|
11
|
-
|
|
15
|
+
use crate::error::to_magnus_error;
|
|
16
|
+
use crate::util::optional_integer_arg;
|
|
17
|
+
|
|
18
|
+
/// Tokenization mode: normal, or decompose with a `Lindera::Penalty`.
|
|
12
19
|
///
|
|
13
|
-
///
|
|
20
|
+
/// The Ruby counterpart of `lindera::mode::Mode`. It has no `new`;
|
|
21
|
+
/// `Mode.normal`, `Mode.decompose` and `Mode.from_str` create one. A mode
|
|
22
|
+
/// cannot be changed, and `==` compares the name and the penalty. Like the
|
|
23
|
+
/// Rust `Mode`, it does not define `eql?` and `hash`.
|
|
14
24
|
#[magnus::wrap(class = "Lindera::Mode", free_immediately, size)]
|
|
15
|
-
#[derive(Debug, Clone,
|
|
25
|
+
#[derive(Debug, Clone, PartialEq)]
|
|
16
26
|
pub struct RbMode {
|
|
17
|
-
///
|
|
18
|
-
inner:
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/// Internal enum for mode kind.
|
|
22
|
-
#[derive(Debug, Clone, Copy)]
|
|
23
|
-
enum RbModeKind {
|
|
24
|
-
/// Standard tokenization based on dictionary cost.
|
|
25
|
-
Normal,
|
|
26
|
-
/// Decompose compound words using penalty-based segmentation.
|
|
27
|
-
Decompose,
|
|
27
|
+
/// The mode, with its penalty for decompose mode.
|
|
28
|
+
inner: LinderaMode,
|
|
28
29
|
}
|
|
29
30
|
|
|
30
31
|
impl RbMode {
|
|
31
|
-
///
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
/// Returns the wrapped `lindera::mode::Mode`.
|
|
33
|
+
pub fn as_lindera(&self) -> &LinderaMode {
|
|
34
|
+
&self.inner
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/// Creates normal mode.
|
|
34
38
|
///
|
|
35
|
-
///
|
|
39
|
+
/// Ruby signature: `Mode.normal`.
|
|
36
40
|
///
|
|
37
41
|
/// # Returns
|
|
38
42
|
///
|
|
39
|
-
///
|
|
40
|
-
fn
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
Some("normal") | Some("Normal") | None => RbModeKind::Normal,
|
|
45
|
-
Some(s) => {
|
|
46
|
-
return Err(Error::new(
|
|
47
|
-
ruby.exception_arg_error(),
|
|
48
|
-
format!("Invalid mode: {s}. Must be 'normal' or 'decompose'"),
|
|
49
|
-
));
|
|
50
|
-
}
|
|
51
|
-
};
|
|
52
|
-
Ok(Self { inner: kind })
|
|
43
|
+
/// Normal mode.
|
|
44
|
+
fn normal() -> Self {
|
|
45
|
+
Self {
|
|
46
|
+
inner: LinderaMode::Normal,
|
|
47
|
+
}
|
|
53
48
|
}
|
|
54
49
|
|
|
55
|
-
///
|
|
50
|
+
/// Creates decompose mode.
|
|
51
|
+
///
|
|
52
|
+
/// Ruby signature: `Mode.decompose(penalty = nil)`.
|
|
53
|
+
///
|
|
54
|
+
/// # Arguments
|
|
55
|
+
///
|
|
56
|
+
/// * `args` - The Ruby arguments:
|
|
57
|
+
/// * `penalty` - The `Lindera::Penalty`. `nil` or omitted uses the
|
|
58
|
+
/// default penalty.
|
|
56
59
|
///
|
|
57
60
|
/// # Returns
|
|
58
61
|
///
|
|
59
|
-
///
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
/// Decompose mode with the penalty, an `ArgumentError` for more than one
|
|
63
|
+
/// argument, or a `TypeError` for a penalty that is not a
|
|
64
|
+
/// `Lindera::Penalty`.
|
|
65
|
+
fn decompose(args: &[Value]) -> Result<Self, Error> {
|
|
66
|
+
let args = scan_args::<(), (Option<Option<&RbPenalty>>,), (), (), (), ()>(args)?;
|
|
67
|
+
let (penalty,) = args.optional;
|
|
68
|
+
Ok(Self::decompose_with(
|
|
69
|
+
penalty.flatten().map(|penalty| (*penalty).into()),
|
|
70
|
+
))
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/// Creates decompose mode with `penalty`, or with the default penalty for
|
|
74
|
+
/// `None`.
|
|
75
|
+
fn decompose_with(penalty: Option<LinderaPenalty>) -> Self {
|
|
76
|
+
Self {
|
|
77
|
+
inner: LinderaMode::Decompose(penalty.unwrap_or_default()),
|
|
65
78
|
}
|
|
66
79
|
}
|
|
67
80
|
|
|
68
|
-
///
|
|
81
|
+
/// Creates a mode from its name, as the Rust `Mode` parses one.
|
|
69
82
|
///
|
|
70
|
-
///
|
|
83
|
+
/// Ruby signature: `Mode.from_str(name)`.
|
|
71
84
|
///
|
|
72
|
-
///
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
/// Returns the name of the mode.
|
|
85
|
+
/// # Arguments
|
|
86
|
+
///
|
|
87
|
+
/// * `ruby` - Ruby runtime handle.
|
|
88
|
+
/// * `name` - `"normal"` or `"decompose"`, case sensitive.
|
|
78
89
|
///
|
|
79
90
|
/// # Returns
|
|
80
91
|
///
|
|
81
|
-
///
|
|
92
|
+
/// The mode, with the default penalty for `"decompose"`, or a
|
|
93
|
+
/// `RuntimeError` for any other name, as `set_mode` raised for it in v6.
|
|
94
|
+
fn parse(ruby: &Ruby, name: String) -> Result<Self, Error> {
|
|
95
|
+
let inner = lindera_binding::parse_mode(&name)
|
|
96
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
97
|
+
Ok(Self { inner })
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// Returns the name of the mode: `"normal"` or `"decompose"`.
|
|
82
101
|
fn name(&self) -> &str {
|
|
83
|
-
self.
|
|
102
|
+
self.inner.as_str()
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/// Returns the name of the mode, as `name` does.
|
|
106
|
+
#[allow(clippy::wrong_self_convention)]
|
|
107
|
+
fn to_s(&self) -> &str {
|
|
108
|
+
self.name()
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/// Returns the inspect representation of the mode, with the penalty of
|
|
112
|
+
/// decompose mode.
|
|
113
|
+
fn inspect(&self) -> String {
|
|
114
|
+
match &self.inner {
|
|
115
|
+
LinderaMode::Normal => "#<Lindera::Mode: normal>".to_string(),
|
|
116
|
+
LinderaMode::Decompose(penalty) => format!(
|
|
117
|
+
"#<Lindera::Mode: decompose {}>",
|
|
118
|
+
RbPenalty::from(penalty.clone()).to_s()
|
|
119
|
+
),
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/// Returns the penalty of decompose mode, as a copy, or `nil` for normal
|
|
124
|
+
/// mode.
|
|
125
|
+
fn penalty(&self) -> Option<RbPenalty> {
|
|
126
|
+
match &self.inner {
|
|
127
|
+
LinderaMode::Normal => None,
|
|
128
|
+
LinderaMode::Decompose(penalty) => Some(penalty.clone().into()),
|
|
129
|
+
}
|
|
84
130
|
}
|
|
85
131
|
|
|
86
132
|
/// Returns true if the mode is normal.
|
|
87
|
-
///
|
|
88
|
-
/// # Returns
|
|
89
|
-
///
|
|
90
|
-
/// A boolean indicating if this is normal mode.
|
|
91
133
|
fn is_normal(&self) -> bool {
|
|
92
|
-
matches!(self.inner,
|
|
134
|
+
matches!(self.inner, LinderaMode::Normal)
|
|
93
135
|
}
|
|
94
136
|
|
|
95
137
|
/// Returns true if the mode is decompose.
|
|
96
|
-
///
|
|
97
|
-
/// # Returns
|
|
98
|
-
///
|
|
99
|
-
/// A boolean indicating if this is decompose mode.
|
|
100
138
|
fn is_decompose(&self) -> bool {
|
|
101
|
-
matches!(self.inner,
|
|
139
|
+
matches!(self.inner, LinderaMode::Decompose(_))
|
|
102
140
|
}
|
|
103
|
-
}
|
|
104
141
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
142
|
+
/// Ruby's `==`: whether `other` is a `Lindera::Mode` with the same name
|
|
143
|
+
/// and penalty.
|
|
144
|
+
///
|
|
145
|
+
/// # Arguments
|
|
146
|
+
///
|
|
147
|
+
/// * `other` - Any Ruby value.
|
|
148
|
+
///
|
|
149
|
+
/// # Returns
|
|
150
|
+
///
|
|
151
|
+
/// `true` for an equal mode, `false` otherwise, other types included.
|
|
152
|
+
fn is_equal(&self, other: Value) -> bool {
|
|
153
|
+
match <&RbMode>::try_convert(other) {
|
|
154
|
+
Ok(other) => self == other,
|
|
155
|
+
Err(_) => false,
|
|
110
156
|
}
|
|
111
157
|
}
|
|
112
158
|
}
|
|
113
159
|
|
|
114
160
|
impl From<LinderaMode> for RbMode {
|
|
115
|
-
fn from(
|
|
116
|
-
|
|
117
|
-
LinderaMode::Normal => RbModeKind::Normal,
|
|
118
|
-
LinderaMode::Decompose(_) => RbModeKind::Decompose,
|
|
119
|
-
};
|
|
120
|
-
RbMode { inner: kind }
|
|
161
|
+
fn from(inner: LinderaMode) -> Self {
|
|
162
|
+
Self { inner }
|
|
121
163
|
}
|
|
122
164
|
}
|
|
123
165
|
|
|
124
166
|
/// Penalty configuration for decompose mode.
|
|
125
167
|
///
|
|
126
168
|
/// Controls how aggressively compound words are decomposed based on
|
|
127
|
-
/// character type and length thresholds.
|
|
169
|
+
/// character type and length thresholds. Pass it to `Mode.decompose`.
|
|
128
170
|
#[magnus::wrap(class = "Lindera::Penalty", free_immediately, size)]
|
|
129
171
|
#[derive(Debug, Clone, Copy)]
|
|
130
172
|
pub struct RbPenalty {
|
|
@@ -143,26 +185,53 @@ impl RbPenalty {
|
|
|
143
185
|
///
|
|
144
186
|
/// # Arguments
|
|
145
187
|
///
|
|
146
|
-
/// * `
|
|
147
|
-
/// * `
|
|
148
|
-
///
|
|
149
|
-
/// * `
|
|
188
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
189
|
+
/// * `kanji_penalty_length_threshold` - Kanji penalty length threshold,
|
|
190
|
+
/// 0 or more (default: 2).
|
|
191
|
+
/// * `kanji_penalty_length_penalty` - Kanji penalty value, a 32-bit
|
|
192
|
+
/// integer (default: 3000).
|
|
193
|
+
/// * `other_penalty_length_threshold` - Other penalty length threshold,
|
|
194
|
+
/// 0 or more (default: 7).
|
|
195
|
+
/// * `other_penalty_length_penalty` - Other penalty value, a 32-bit
|
|
196
|
+
/// integer (default: 1700).
|
|
150
197
|
///
|
|
151
198
|
/// # Returns
|
|
152
199
|
///
|
|
153
|
-
/// A new `RbPenalty` instance
|
|
200
|
+
/// A new `RbPenalty` instance, or a `RangeError` for a value outside the
|
|
201
|
+
/// range of its setting.
|
|
154
202
|
fn new(
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
203
|
+
ruby: &Ruby,
|
|
204
|
+
kanji_penalty_length_threshold: Option<Integer>,
|
|
205
|
+
kanji_penalty_length_penalty: Option<Integer>,
|
|
206
|
+
other_penalty_length_threshold: Option<Integer>,
|
|
207
|
+
other_penalty_length_penalty: Option<Integer>,
|
|
208
|
+
) -> Result<Self, Error> {
|
|
209
|
+
Ok(Self {
|
|
210
|
+
kanji_penalty_length_threshold: optional_integer_arg(
|
|
211
|
+
ruby,
|
|
212
|
+
"kanji_penalty_length_threshold",
|
|
213
|
+
kanji_penalty_length_threshold,
|
|
214
|
+
)?
|
|
215
|
+
.unwrap_or(2),
|
|
216
|
+
kanji_penalty_length_penalty: optional_integer_arg(
|
|
217
|
+
ruby,
|
|
218
|
+
"kanji_penalty_length_penalty",
|
|
219
|
+
kanji_penalty_length_penalty,
|
|
220
|
+
)?
|
|
221
|
+
.unwrap_or(3000),
|
|
222
|
+
other_penalty_length_threshold: optional_integer_arg(
|
|
223
|
+
ruby,
|
|
224
|
+
"other_penalty_length_threshold",
|
|
225
|
+
other_penalty_length_threshold,
|
|
226
|
+
)?
|
|
227
|
+
.unwrap_or(7),
|
|
228
|
+
other_penalty_length_penalty: optional_integer_arg(
|
|
229
|
+
ruby,
|
|
230
|
+
"other_penalty_length_penalty",
|
|
231
|
+
other_penalty_length_penalty,
|
|
232
|
+
)?
|
|
233
|
+
.unwrap_or(1700),
|
|
234
|
+
})
|
|
166
235
|
}
|
|
167
236
|
|
|
168
237
|
/// Returns the kanji penalty length threshold.
|
|
@@ -237,12 +306,19 @@ impl From<LinderaPenalty> for RbPenalty {
|
|
|
237
306
|
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
238
307
|
pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
239
308
|
let mode_class = module.define_class("Mode", ruby.class_object())?;
|
|
240
|
-
|
|
309
|
+
// magnus undefines the allocator only when the class first wraps a
|
|
310
|
+
// value, so until then `Mode.new` would return an empty object.
|
|
311
|
+
mode_class.undef_default_alloc_func();
|
|
312
|
+
mode_class.define_singleton_method("normal", function!(RbMode::normal, 0))?;
|
|
313
|
+
mode_class.define_singleton_method("decompose", function!(RbMode::decompose, -1))?;
|
|
314
|
+
mode_class.define_singleton_method("from_str", function!(RbMode::parse, 1))?;
|
|
241
315
|
mode_class.define_method("to_s", method!(RbMode::to_s, 0))?;
|
|
242
316
|
mode_class.define_method("inspect", method!(RbMode::inspect, 0))?;
|
|
243
317
|
mode_class.define_method("name", method!(RbMode::name, 0))?;
|
|
318
|
+
mode_class.define_method("penalty", method!(RbMode::penalty, 0))?;
|
|
244
319
|
mode_class.define_method("normal?", method!(RbMode::is_normal, 0))?;
|
|
245
320
|
mode_class.define_method("decompose?", method!(RbMode::is_decompose, 0))?;
|
|
321
|
+
mode_class.define_method("==", method!(RbMode::is_equal, 1))?;
|
|
246
322
|
|
|
247
323
|
let penalty_class = module.define_class("Penalty", ruby.class_object())?;
|
|
248
324
|
penalty_class.define_singleton_method("new", function!(RbPenalty::new, 4))?;
|
|
@@ -273,54 +349,59 @@ mod tests {
|
|
|
273
349
|
use super::*;
|
|
274
350
|
|
|
275
351
|
#[test]
|
|
276
|
-
fn
|
|
277
|
-
|
|
278
|
-
|
|
352
|
+
fn test_rb_mode_factories() {
|
|
353
|
+
assert_eq!(RbMode::normal().as_lindera(), &LinderaMode::Normal);
|
|
354
|
+
assert_eq!(
|
|
355
|
+
RbMode::decompose_with(None).as_lindera(),
|
|
356
|
+
&LinderaMode::Decompose(LinderaPenalty::default())
|
|
357
|
+
);
|
|
358
|
+
let penalty = LinderaPenalty {
|
|
359
|
+
kanji_penalty_length_threshold: 3,
|
|
360
|
+
kanji_penalty_length_penalty: 0,
|
|
361
|
+
other_penalty_length_threshold: 4,
|
|
362
|
+
other_penalty_length_penalty: 3000,
|
|
279
363
|
};
|
|
280
|
-
let
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
#[test]
|
|
285
|
-
fn test_rb_mode_decompose_to_lindera_mode() {
|
|
286
|
-
let rb_mode = RbMode {
|
|
287
|
-
inner: RbModeKind::Decompose,
|
|
364
|
+
let mode = RbMode::decompose_with(Some(penalty.clone()));
|
|
365
|
+
assert_eq!(mode.as_lindera(), &LinderaMode::Decompose(penalty));
|
|
366
|
+
let Some(back) = mode.penalty() else {
|
|
367
|
+
panic!("decompose mode has no penalty");
|
|
288
368
|
};
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
let default_penalty = LinderaPenalty::default();
|
|
293
|
-
assert_eq!(
|
|
294
|
-
penalty.kanji_penalty_length_threshold,
|
|
295
|
-
default_penalty.kanji_penalty_length_threshold
|
|
296
|
-
);
|
|
297
|
-
assert_eq!(
|
|
298
|
-
penalty.kanji_penalty_length_penalty,
|
|
299
|
-
default_penalty.kanji_penalty_length_penalty
|
|
300
|
-
);
|
|
301
|
-
assert_eq!(
|
|
302
|
-
penalty.other_penalty_length_threshold,
|
|
303
|
-
default_penalty.other_penalty_length_threshold
|
|
304
|
-
);
|
|
305
|
-
assert_eq!(
|
|
306
|
-
penalty.other_penalty_length_penalty,
|
|
307
|
-
default_penalty.other_penalty_length_penalty
|
|
308
|
-
);
|
|
309
|
-
}
|
|
369
|
+
assert_eq!(back.kanji_penalty_length_threshold, 3);
|
|
370
|
+
assert_eq!(back.other_penalty_length_penalty, 3000);
|
|
371
|
+
assert!(RbMode::normal().penalty().is_none());
|
|
310
372
|
}
|
|
311
373
|
|
|
312
374
|
#[test]
|
|
313
|
-
fn
|
|
314
|
-
let
|
|
315
|
-
let
|
|
316
|
-
|
|
375
|
+
fn test_rb_mode_names_and_kinds() {
|
|
376
|
+
let normal = RbMode::normal();
|
|
377
|
+
let decompose = RbMode::decompose_with(None);
|
|
378
|
+
assert_eq!(normal.name(), "normal");
|
|
379
|
+
assert_eq!(decompose.to_s(), "decompose");
|
|
380
|
+
assert!(normal.is_normal() && !normal.is_decompose());
|
|
381
|
+
assert!(decompose.is_decompose() && !decompose.is_normal());
|
|
382
|
+
assert_eq!(normal.inspect(), "#<Lindera::Mode: normal>");
|
|
383
|
+
assert!(
|
|
384
|
+
decompose
|
|
385
|
+
.inspect()
|
|
386
|
+
.starts_with("#<Lindera::Mode: decompose Penalty(")
|
|
387
|
+
);
|
|
317
388
|
}
|
|
318
389
|
|
|
390
|
+
/// Two modes are equal when their names and penalties are, as the Rust
|
|
391
|
+
/// `Mode` compares.
|
|
319
392
|
#[test]
|
|
320
|
-
fn
|
|
321
|
-
let
|
|
322
|
-
|
|
323
|
-
|
|
393
|
+
fn test_rb_mode_equality_compares_the_penalty() {
|
|
394
|
+
let zero = LinderaPenalty {
|
|
395
|
+
kanji_penalty_length_penalty: 0,
|
|
396
|
+
other_penalty_length_penalty: 0,
|
|
397
|
+
..LinderaPenalty::default()
|
|
398
|
+
};
|
|
399
|
+
assert_eq!(RbMode::decompose_with(None), RbMode::decompose_with(None));
|
|
400
|
+
assert_ne!(
|
|
401
|
+
RbMode::decompose_with(None),
|
|
402
|
+
RbMode::decompose_with(Some(zero))
|
|
403
|
+
);
|
|
404
|
+
assert_ne!(RbMode::normal(), RbMode::decompose_with(None));
|
|
324
405
|
}
|
|
325
406
|
|
|
326
407
|
#[test]
|
|
@@ -375,27 +456,4 @@ mod tests {
|
|
|
375
456
|
default_lindera.other_penalty_length_penalty
|
|
376
457
|
);
|
|
377
458
|
}
|
|
378
|
-
|
|
379
|
-
#[test]
|
|
380
|
-
fn test_mode_roundtrip_normal() {
|
|
381
|
-
let original = LinderaMode::Normal;
|
|
382
|
-
let rb: RbMode = original.into();
|
|
383
|
-
let back: LinderaMode = rb.into();
|
|
384
|
-
assert!(matches!(back, LinderaMode::Normal));
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
#[test]
|
|
388
|
-
fn test_mode_roundtrip_decompose() {
|
|
389
|
-
let penalty = LinderaPenalty {
|
|
390
|
-
kanji_penalty_length_threshold: 5,
|
|
391
|
-
kanji_penalty_length_penalty: 4000,
|
|
392
|
-
other_penalty_length_threshold: 9,
|
|
393
|
-
other_penalty_length_penalty: 2000,
|
|
394
|
-
};
|
|
395
|
-
let original = LinderaMode::Decompose(penalty);
|
|
396
|
-
let rb: RbMode = original.into();
|
|
397
|
-
let back: LinderaMode = rb.into();
|
|
398
|
-
// Note: RbMode loses the penalty details, so Decompose uses default penalty.
|
|
399
|
-
assert!(matches!(back, LinderaMode::Decompose(_)));
|
|
400
|
-
}
|
|
401
459
|
}
|
data/src/schema.rs
CHANGED
|
@@ -2,13 +2,15 @@
|
|
|
2
2
|
//!
|
|
3
3
|
//! This module provides schema structures that define the format and fields
|
|
4
4
|
//! of dictionary entries. The field-management logic is delegated to
|
|
5
|
-
//! [`
|
|
5
|
+
//! [`lindera_binding::CoreSchema`]; this module only adds the magnus wrappers.
|
|
6
6
|
|
|
7
7
|
use magnus::prelude::*;
|
|
8
|
-
use magnus::{Error, RArray, Ruby, function, method};
|
|
8
|
+
use magnus::{Error, Integer, RArray, Ruby, function, method};
|
|
9
9
|
|
|
10
10
|
use lindera::dictionary::{FieldDefinition, FieldType, Schema};
|
|
11
|
-
use
|
|
11
|
+
use lindera_binding::{CoreFieldDefinition, CoreFieldType, CoreSchema};
|
|
12
|
+
|
|
13
|
+
use crate::util::integer_arg;
|
|
12
14
|
|
|
13
15
|
/// Field type in dictionary schema.
|
|
14
16
|
///
|
|
@@ -177,12 +179,12 @@ impl From<RbFieldDefinition> for FieldDefinition {
|
|
|
177
179
|
|
|
178
180
|
/// Dictionary schema definition.
|
|
179
181
|
///
|
|
180
|
-
/// A thin magnus wrapper over [`
|
|
182
|
+
/// A thin magnus wrapper over [`lindera_binding::CoreSchema`], which owns
|
|
181
183
|
/// the field storage, the name-to-index map, and the field lookups.
|
|
182
184
|
#[magnus::wrap(class = "Lindera::Schema", free_immediately, size)]
|
|
183
185
|
#[derive(Debug, Clone)]
|
|
184
186
|
pub struct RbSchema {
|
|
185
|
-
/// The backing binding
|
|
187
|
+
/// The backing lindera-binding schema.
|
|
186
188
|
inner: CoreSchema,
|
|
187
189
|
}
|
|
188
190
|
|
|
@@ -244,13 +246,22 @@ impl RbSchema {
|
|
|
244
246
|
///
|
|
245
247
|
/// # Arguments
|
|
246
248
|
///
|
|
247
|
-
/// * `
|
|
249
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
250
|
+
/// * `rb_self` - The schema.
|
|
251
|
+
/// * `index` - Index of the field, 0 or more.
|
|
248
252
|
///
|
|
249
253
|
/// # Returns
|
|
250
254
|
///
|
|
251
|
-
/// The field name,
|
|
252
|
-
|
|
253
|
-
|
|
255
|
+
/// The field name, `None` if the index is past the last field, or a
|
|
256
|
+
/// `RangeError` for an index that is negative or larger than a `usize`
|
|
257
|
+
/// holds.
|
|
258
|
+
fn get_field_name(
|
|
259
|
+
ruby: &Ruby,
|
|
260
|
+
rb_self: &Self,
|
|
261
|
+
index: Integer,
|
|
262
|
+
) -> Result<Option<String>, Error> {
|
|
263
|
+
let index = integer_arg(ruby, "index", index)?;
|
|
264
|
+
Ok(rb_self.inner.get_field_name(index).map(str::to_string))
|
|
254
265
|
}
|
|
255
266
|
|
|
256
267
|
/// Returns the custom fields (fields after the first 4 standard fields).
|
data/src/token.rs
CHANGED
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
//! This module wraps Lindera tokens for use in Ruby.
|
|
4
4
|
|
|
5
5
|
use magnus::prelude::*;
|
|
6
|
-
use magnus::{Error, RHash, Ruby, method};
|
|
6
|
+
use magnus::{Error, Integer, RHash, Ruby, method};
|
|
7
7
|
|
|
8
8
|
use lindera::token::Token;
|
|
9
9
|
|
|
10
|
+
use crate::util::integer_arg;
|
|
11
|
+
|
|
10
12
|
/// Token object wrapping the Rust Token data.
|
|
11
13
|
///
|
|
12
14
|
/// This class provides access to token fields and details.
|
|
@@ -39,19 +41,19 @@ impl RbToken {
|
|
|
39
41
|
///
|
|
40
42
|
/// A new `RbToken` instance.
|
|
41
43
|
pub fn from_token(token: Token) -> Self {
|
|
42
|
-
Self::from_view(
|
|
44
|
+
Self::from_view(lindera_binding::TokenView::from_token(token))
|
|
43
45
|
}
|
|
44
46
|
|
|
45
|
-
/// Creates a new `RbToken` from a binding
|
|
47
|
+
/// Creates a new `RbToken` from a lindera-binding `TokenView`.
|
|
46
48
|
///
|
|
47
49
|
/// # Arguments
|
|
48
50
|
///
|
|
49
|
-
/// * `view` - Token view produced by the binding
|
|
51
|
+
/// * `view` - Token view produced by the lindera-binding tokenizer.
|
|
50
52
|
///
|
|
51
53
|
/// # Returns
|
|
52
54
|
///
|
|
53
55
|
/// A new `RbToken` instance.
|
|
54
|
-
pub fn from_view(view:
|
|
56
|
+
pub fn from_view(view: lindera_binding::TokenView) -> Self {
|
|
55
57
|
Self {
|
|
56
58
|
surface: view.surface,
|
|
57
59
|
byte_start: view.byte_start,
|
|
@@ -102,13 +104,18 @@ impl RbToken {
|
|
|
102
104
|
///
|
|
103
105
|
/// # Arguments
|
|
104
106
|
///
|
|
105
|
-
/// * `
|
|
107
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
108
|
+
/// * `rb_self` - The token.
|
|
109
|
+
/// * `index` - Index of the detail to retrieve, 0 or more.
|
|
106
110
|
///
|
|
107
111
|
/// # Returns
|
|
108
112
|
///
|
|
109
|
-
/// The detail string if found,
|
|
110
|
-
|
|
111
|
-
|
|
113
|
+
/// The detail string if found, nil if the index is past the last detail,
|
|
114
|
+
/// or a `RangeError` for an index that is negative or larger than a
|
|
115
|
+
/// `usize` holds.
|
|
116
|
+
fn get_detail(ruby: &Ruby, rb_self: &Self, index: Integer) -> Result<Option<String>, Error> {
|
|
117
|
+
let index: usize = integer_arg(ruby, "index", index)?;
|
|
118
|
+
Ok(rb_self.details.get(index).cloned())
|
|
112
119
|
}
|
|
113
120
|
|
|
114
121
|
/// Returns the token as a plain Hash keyed by Symbols.
|