lindera 6.1.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/mode.rs CHANGED
@@ -1,130 +1,172 @@
1
1
  //! Tokenization modes and penalty configurations.
2
2
  //!
3
3
  //! This module defines the different tokenization modes available and their
4
- //! penalty configurations for controlling segmentation behavior.
4
+ //! penalty configurations for controlling segmentation behavior. A
5
+ //! `Lindera::Mode` holds the penalty of decompose mode, as the Rust
6
+ //! `lindera::mode::Mode` does, and is what `TokenizerBuilder#set_mode` and
7
+ //! `Tokenizer.new` take (#1218).
5
8
 
6
9
  use magnus::prelude::*;
7
- use magnus::{Error, Ruby, function, method};
10
+ use magnus::scan_args::scan_args;
11
+ use magnus::{Error, Integer, Ruby, TryConvert, Value, function, method};
8
12
 
9
13
  use lindera::mode::{Mode as LinderaMode, Penalty as LinderaPenalty};
10
14
 
11
- /// Tokenization mode.
15
+ use crate::error::to_magnus_error;
16
+ use crate::util::optional_integer_arg;
17
+
18
+ /// Tokenization mode: normal, or decompose with a `Lindera::Penalty`.
12
19
  ///
13
- /// Determines how text is segmented into tokens.
20
+ /// The Ruby counterpart of `lindera::mode::Mode`. It has no `new`;
21
+ /// `Mode.normal`, `Mode.decompose` and `Mode.from_str` create one. A mode
22
+ /// cannot be changed, and `==` compares the name and the penalty. Like the
23
+ /// Rust `Mode`, it does not define `eql?` and `hash`.
14
24
  #[magnus::wrap(class = "Lindera::Mode", free_immediately, size)]
15
- #[derive(Debug, Clone, Copy)]
25
+ #[derive(Debug, Clone, PartialEq)]
16
26
  pub struct RbMode {
17
- /// Internal mode variant.
18
- inner: RbModeKind,
19
- }
20
-
21
- /// Internal enum for mode kind.
22
- #[derive(Debug, Clone, Copy)]
23
- enum RbModeKind {
24
- /// Standard tokenization based on dictionary cost.
25
- Normal,
26
- /// Decompose compound words using penalty-based segmentation.
27
- Decompose,
27
+ /// The mode, with its penalty for decompose mode.
28
+ inner: LinderaMode,
28
29
  }
29
30
 
30
31
  impl RbMode {
31
- /// Creates a new `RbMode` from a mode string.
32
- ///
33
- /// # Arguments
32
+ /// Returns the wrapped `lindera::mode::Mode`.
33
+ pub fn as_lindera(&self) -> &LinderaMode {
34
+ &self.inner
35
+ }
36
+
37
+ /// Creates normal mode.
34
38
  ///
35
- /// * `mode_str` - Mode string ("normal" or "decompose"). Defaults to "normal" if None.
39
+ /// Ruby signature: `Mode.normal`.
36
40
  ///
37
41
  /// # Returns
38
42
  ///
39
- /// A new `RbMode` instance.
40
- fn new(mode_str: Option<String>) -> Result<Self, Error> {
41
- let ruby = Ruby::get().expect("Ruby runtime not initialized");
42
- let kind = match mode_str.as_deref() {
43
- Some("decompose") | Some("Decompose") => RbModeKind::Decompose,
44
- Some("normal") | Some("Normal") | None => RbModeKind::Normal,
45
- Some(s) => {
46
- return Err(Error::new(
47
- ruby.exception_arg_error(),
48
- format!("Invalid mode: {s}. Must be 'normal' or 'decompose'"),
49
- ));
50
- }
51
- };
52
- Ok(Self { inner: kind })
43
+ /// Normal mode.
44
+ fn normal() -> Self {
45
+ Self {
46
+ inner: LinderaMode::Normal,
47
+ }
53
48
  }
54
49
 
55
- /// Returns the string representation of the mode.
50
+ /// Creates decompose mode.
51
+ ///
52
+ /// Ruby signature: `Mode.decompose(penalty = nil)`.
53
+ ///
54
+ /// # Arguments
55
+ ///
56
+ /// * `args` - The Ruby arguments:
57
+ /// * `penalty` - The `Lindera::Penalty`. `nil` or omitted uses the
58
+ /// default penalty.
56
59
  ///
57
60
  /// # Returns
58
61
  ///
59
- /// A string slice representing the mode.
60
- #[allow(clippy::wrong_self_convention)]
61
- fn to_s(&self) -> &str {
62
- match self.inner {
63
- RbModeKind::Normal => "normal",
64
- RbModeKind::Decompose => "decompose",
62
+ /// Decompose mode with the penalty, an `ArgumentError` for more than one
63
+ /// argument, or a `TypeError` for a penalty that is not a
64
+ /// `Lindera::Penalty`.
65
+ fn decompose(args: &[Value]) -> Result<Self, Error> {
66
+ let args = scan_args::<(), (Option<Option<&RbPenalty>>,), (), (), (), ()>(args)?;
67
+ let (penalty,) = args.optional;
68
+ Ok(Self::decompose_with(
69
+ penalty.flatten().map(|penalty| (*penalty).into()),
70
+ ))
71
+ }
72
+
73
+ /// Creates decompose mode with `penalty`, or with the default penalty for
74
+ /// `None`.
75
+ fn decompose_with(penalty: Option<LinderaPenalty>) -> Self {
76
+ Self {
77
+ inner: LinderaMode::Decompose(penalty.unwrap_or_default()),
65
78
  }
66
79
  }
67
80
 
68
- /// Returns the inspect representation of the mode.
81
+ /// Creates a mode from its name, as the Rust `Mode` parses one.
69
82
  ///
70
- /// # Returns
83
+ /// Ruby signature: `Mode.from_str(name)`.
71
84
  ///
72
- /// A string with the mode inspect representation.
73
- fn inspect(&self) -> String {
74
- format!("#<Lindera::Mode: {}>", self.to_s())
75
- }
76
-
77
- /// Returns the name of the mode.
85
+ /// # Arguments
86
+ ///
87
+ /// * `ruby` - Ruby runtime handle.
88
+ /// * `name` - `"normal"` or `"decompose"`, case sensitive.
78
89
  ///
79
90
  /// # Returns
80
91
  ///
81
- /// A string slice with the mode name.
92
+ /// The mode, with the default penalty for `"decompose"`, or a
93
+ /// `RuntimeError` for any other name, as `set_mode` raised for it in v6.
94
+ fn parse(ruby: &Ruby, name: String) -> Result<Self, Error> {
95
+ let inner = lindera_binding::parse_mode(&name)
96
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
97
+ Ok(Self { inner })
98
+ }
99
+
100
+ /// Returns the name of the mode: `"normal"` or `"decompose"`.
82
101
  fn name(&self) -> &str {
83
- self.to_s()
102
+ self.inner.as_str()
103
+ }
104
+
105
+ /// Returns the name of the mode, as `name` does.
106
+ #[allow(clippy::wrong_self_convention)]
107
+ fn to_s(&self) -> &str {
108
+ self.name()
109
+ }
110
+
111
+ /// Returns the inspect representation of the mode, with the penalty of
112
+ /// decompose mode.
113
+ fn inspect(&self) -> String {
114
+ match &self.inner {
115
+ LinderaMode::Normal => "#<Lindera::Mode: normal>".to_string(),
116
+ LinderaMode::Decompose(penalty) => format!(
117
+ "#<Lindera::Mode: decompose {}>",
118
+ RbPenalty::from(penalty.clone()).to_s()
119
+ ),
120
+ }
121
+ }
122
+
123
+ /// Returns the penalty of decompose mode, as a copy, or `nil` for normal
124
+ /// mode.
125
+ fn penalty(&self) -> Option<RbPenalty> {
126
+ match &self.inner {
127
+ LinderaMode::Normal => None,
128
+ LinderaMode::Decompose(penalty) => Some(penalty.clone().into()),
129
+ }
84
130
  }
85
131
 
86
132
  /// Returns true if the mode is normal.
87
- ///
88
- /// # Returns
89
- ///
90
- /// A boolean indicating if this is normal mode.
91
133
  fn is_normal(&self) -> bool {
92
- matches!(self.inner, RbModeKind::Normal)
134
+ matches!(self.inner, LinderaMode::Normal)
93
135
  }
94
136
 
95
137
  /// Returns true if the mode is decompose.
96
- ///
97
- /// # Returns
98
- ///
99
- /// A boolean indicating if this is decompose mode.
100
138
  fn is_decompose(&self) -> bool {
101
- matches!(self.inner, RbModeKind::Decompose)
139
+ matches!(self.inner, LinderaMode::Decompose(_))
102
140
  }
103
- }
104
141
 
105
- impl From<RbMode> for LinderaMode {
106
- fn from(mode: RbMode) -> Self {
107
- match mode.inner {
108
- RbModeKind::Normal => LinderaMode::Normal,
109
- RbModeKind::Decompose => LinderaMode::Decompose(LinderaPenalty::default()),
142
+ /// Ruby's `==`: whether `other` is a `Lindera::Mode` with the same name
143
+ /// and penalty.
144
+ ///
145
+ /// # Arguments
146
+ ///
147
+ /// * `other` - Any Ruby value.
148
+ ///
149
+ /// # Returns
150
+ ///
151
+ /// `true` for an equal mode, `false` otherwise, other types included.
152
+ fn is_equal(&self, other: Value) -> bool {
153
+ match <&RbMode>::try_convert(other) {
154
+ Ok(other) => self == other,
155
+ Err(_) => false,
110
156
  }
111
157
  }
112
158
  }
113
159
 
114
160
  impl From<LinderaMode> for RbMode {
115
- fn from(mode: LinderaMode) -> Self {
116
- let kind = match mode {
117
- LinderaMode::Normal => RbModeKind::Normal,
118
- LinderaMode::Decompose(_) => RbModeKind::Decompose,
119
- };
120
- RbMode { inner: kind }
161
+ fn from(inner: LinderaMode) -> Self {
162
+ Self { inner }
121
163
  }
122
164
  }
123
165
 
124
166
  /// Penalty configuration for decompose mode.
125
167
  ///
126
168
  /// Controls how aggressively compound words are decomposed based on
127
- /// character type and length thresholds.
169
+ /// character type and length thresholds. Pass it to `Mode.decompose`.
128
170
  #[magnus::wrap(class = "Lindera::Penalty", free_immediately, size)]
129
171
  #[derive(Debug, Clone, Copy)]
130
172
  pub struct RbPenalty {
@@ -143,26 +185,53 @@ impl RbPenalty {
143
185
  ///
144
186
  /// # Arguments
145
187
  ///
146
- /// * `kanji_threshold` - Kanji penalty length threshold (default: 2).
147
- /// * `kanji_penalty` - Kanji penalty value (default: 3000).
148
- /// * `other_threshold` - Other penalty length threshold (default: 7).
149
- /// * `other_penalty` - Other penalty value (default: 1700).
188
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
189
+ /// * `kanji_penalty_length_threshold` - Kanji penalty length threshold,
190
+ /// 0 or more (default: 2).
191
+ /// * `kanji_penalty_length_penalty` - Kanji penalty value, a 32-bit
192
+ /// integer (default: 3000).
193
+ /// * `other_penalty_length_threshold` - Other penalty length threshold,
194
+ /// 0 or more (default: 7).
195
+ /// * `other_penalty_length_penalty` - Other penalty value, a 32-bit
196
+ /// integer (default: 1700).
150
197
  ///
151
198
  /// # Returns
152
199
  ///
153
- /// A new `RbPenalty` instance.
200
+ /// A new `RbPenalty` instance, or a `RangeError` for a value outside the
201
+ /// range of its setting.
154
202
  fn new(
155
- kanji_threshold: Option<usize>,
156
- kanji_penalty: Option<i32>,
157
- other_threshold: Option<usize>,
158
- other_penalty: Option<i32>,
159
- ) -> Self {
160
- Self {
161
- kanji_penalty_length_threshold: kanji_threshold.unwrap_or(2),
162
- kanji_penalty_length_penalty: kanji_penalty.unwrap_or(3000),
163
- other_penalty_length_threshold: other_threshold.unwrap_or(7),
164
- other_penalty_length_penalty: other_penalty.unwrap_or(1700),
165
- }
203
+ ruby: &Ruby,
204
+ kanji_penalty_length_threshold: Option<Integer>,
205
+ kanji_penalty_length_penalty: Option<Integer>,
206
+ other_penalty_length_threshold: Option<Integer>,
207
+ other_penalty_length_penalty: Option<Integer>,
208
+ ) -> Result<Self, Error> {
209
+ Ok(Self {
210
+ kanji_penalty_length_threshold: optional_integer_arg(
211
+ ruby,
212
+ "kanji_penalty_length_threshold",
213
+ kanji_penalty_length_threshold,
214
+ )?
215
+ .unwrap_or(2),
216
+ kanji_penalty_length_penalty: optional_integer_arg(
217
+ ruby,
218
+ "kanji_penalty_length_penalty",
219
+ kanji_penalty_length_penalty,
220
+ )?
221
+ .unwrap_or(3000),
222
+ other_penalty_length_threshold: optional_integer_arg(
223
+ ruby,
224
+ "other_penalty_length_threshold",
225
+ other_penalty_length_threshold,
226
+ )?
227
+ .unwrap_or(7),
228
+ other_penalty_length_penalty: optional_integer_arg(
229
+ ruby,
230
+ "other_penalty_length_penalty",
231
+ other_penalty_length_penalty,
232
+ )?
233
+ .unwrap_or(1700),
234
+ })
166
235
  }
167
236
 
168
237
  /// Returns the kanji penalty length threshold.
@@ -237,12 +306,19 @@ impl From<LinderaPenalty> for RbPenalty {
237
306
  /// `Ok(())` on success, or a Magnus `Error` on failure.
238
307
  pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
239
308
  let mode_class = module.define_class("Mode", ruby.class_object())?;
240
- mode_class.define_singleton_method("new", function!(RbMode::new, 1))?;
309
+ // magnus undefines the allocator only when the class first wraps a
310
+ // value, so until then `Mode.new` would return an empty object.
311
+ mode_class.undef_default_alloc_func();
312
+ mode_class.define_singleton_method("normal", function!(RbMode::normal, 0))?;
313
+ mode_class.define_singleton_method("decompose", function!(RbMode::decompose, -1))?;
314
+ mode_class.define_singleton_method("from_str", function!(RbMode::parse, 1))?;
241
315
  mode_class.define_method("to_s", method!(RbMode::to_s, 0))?;
242
316
  mode_class.define_method("inspect", method!(RbMode::inspect, 0))?;
243
317
  mode_class.define_method("name", method!(RbMode::name, 0))?;
318
+ mode_class.define_method("penalty", method!(RbMode::penalty, 0))?;
244
319
  mode_class.define_method("normal?", method!(RbMode::is_normal, 0))?;
245
320
  mode_class.define_method("decompose?", method!(RbMode::is_decompose, 0))?;
321
+ mode_class.define_method("==", method!(RbMode::is_equal, 1))?;
246
322
 
247
323
  let penalty_class = module.define_class("Penalty", ruby.class_object())?;
248
324
  penalty_class.define_singleton_method("new", function!(RbPenalty::new, 4))?;
@@ -273,54 +349,59 @@ mod tests {
273
349
  use super::*;
274
350
 
275
351
  #[test]
276
- fn test_rb_mode_normal_to_lindera_mode() {
277
- let rb_mode = RbMode {
278
- inner: RbModeKind::Normal,
352
+ fn test_rb_mode_factories() {
353
+ assert_eq!(RbMode::normal().as_lindera(), &LinderaMode::Normal);
354
+ assert_eq!(
355
+ RbMode::decompose_with(None).as_lindera(),
356
+ &LinderaMode::Decompose(LinderaPenalty::default())
357
+ );
358
+ let penalty = LinderaPenalty {
359
+ kanji_penalty_length_threshold: 3,
360
+ kanji_penalty_length_penalty: 0,
361
+ other_penalty_length_threshold: 4,
362
+ other_penalty_length_penalty: 3000,
279
363
  };
280
- let lindera_mode: LinderaMode = rb_mode.into();
281
- assert!(matches!(lindera_mode, LinderaMode::Normal));
282
- }
283
-
284
- #[test]
285
- fn test_rb_mode_decompose_to_lindera_mode() {
286
- let rb_mode = RbMode {
287
- inner: RbModeKind::Decompose,
364
+ let mode = RbMode::decompose_with(Some(penalty.clone()));
365
+ assert_eq!(mode.as_lindera(), &LinderaMode::Decompose(penalty));
366
+ let Some(back) = mode.penalty() else {
367
+ panic!("decompose mode has no penalty");
288
368
  };
289
- let lindera_mode: LinderaMode = rb_mode.into();
290
- assert!(matches!(lindera_mode, LinderaMode::Decompose(_)));
291
- if let LinderaMode::Decompose(penalty) = lindera_mode {
292
- let default_penalty = LinderaPenalty::default();
293
- assert_eq!(
294
- penalty.kanji_penalty_length_threshold,
295
- default_penalty.kanji_penalty_length_threshold
296
- );
297
- assert_eq!(
298
- penalty.kanji_penalty_length_penalty,
299
- default_penalty.kanji_penalty_length_penalty
300
- );
301
- assert_eq!(
302
- penalty.other_penalty_length_threshold,
303
- default_penalty.other_penalty_length_threshold
304
- );
305
- assert_eq!(
306
- penalty.other_penalty_length_penalty,
307
- default_penalty.other_penalty_length_penalty
308
- );
309
- }
369
+ assert_eq!(back.kanji_penalty_length_threshold, 3);
370
+ assert_eq!(back.other_penalty_length_penalty, 3000);
371
+ assert!(RbMode::normal().penalty().is_none());
310
372
  }
311
373
 
312
374
  #[test]
313
- fn test_lindera_mode_normal_to_rb_mode() {
314
- let lindera_mode = LinderaMode::Normal;
315
- let rb_mode: RbMode = lindera_mode.into();
316
- assert!(matches!(rb_mode.inner, RbModeKind::Normal));
375
+ fn test_rb_mode_names_and_kinds() {
376
+ let normal = RbMode::normal();
377
+ let decompose = RbMode::decompose_with(None);
378
+ assert_eq!(normal.name(), "normal");
379
+ assert_eq!(decompose.to_s(), "decompose");
380
+ assert!(normal.is_normal() && !normal.is_decompose());
381
+ assert!(decompose.is_decompose() && !decompose.is_normal());
382
+ assert_eq!(normal.inspect(), "#<Lindera::Mode: normal>");
383
+ assert!(
384
+ decompose
385
+ .inspect()
386
+ .starts_with("#<Lindera::Mode: decompose Penalty(")
387
+ );
317
388
  }
318
389
 
390
+ /// Two modes are equal when their names and penalties are, as the Rust
391
+ /// `Mode` compares.
319
392
  #[test]
320
- fn test_lindera_mode_decompose_to_rb_mode() {
321
- let lindera_mode = LinderaMode::Decompose(LinderaPenalty::default());
322
- let rb_mode: RbMode = lindera_mode.into();
323
- assert!(matches!(rb_mode.inner, RbModeKind::Decompose));
393
+ fn test_rb_mode_equality_compares_the_penalty() {
394
+ let zero = LinderaPenalty {
395
+ kanji_penalty_length_penalty: 0,
396
+ other_penalty_length_penalty: 0,
397
+ ..LinderaPenalty::default()
398
+ };
399
+ assert_eq!(RbMode::decompose_with(None), RbMode::decompose_with(None));
400
+ assert_ne!(
401
+ RbMode::decompose_with(None),
402
+ RbMode::decompose_with(Some(zero))
403
+ );
404
+ assert_ne!(RbMode::normal(), RbMode::decompose_with(None));
324
405
  }
325
406
 
326
407
  #[test]
@@ -375,27 +456,4 @@ mod tests {
375
456
  default_lindera.other_penalty_length_penalty
376
457
  );
377
458
  }
378
-
379
- #[test]
380
- fn test_mode_roundtrip_normal() {
381
- let original = LinderaMode::Normal;
382
- let rb: RbMode = original.into();
383
- let back: LinderaMode = rb.into();
384
- assert!(matches!(back, LinderaMode::Normal));
385
- }
386
-
387
- #[test]
388
- fn test_mode_roundtrip_decompose() {
389
- let penalty = LinderaPenalty {
390
- kanji_penalty_length_threshold: 5,
391
- kanji_penalty_length_penalty: 4000,
392
- other_penalty_length_threshold: 9,
393
- other_penalty_length_penalty: 2000,
394
- };
395
- let original = LinderaMode::Decompose(penalty);
396
- let rb: RbMode = original.into();
397
- let back: LinderaMode = rb.into();
398
- // Note: RbMode loses the penalty details, so Decompose uses default penalty.
399
- assert!(matches!(back, LinderaMode::Decompose(_)));
400
- }
401
459
  }
data/src/schema.rs CHANGED
@@ -2,13 +2,15 @@
2
2
  //!
3
3
  //! This module provides schema structures that define the format and fields
4
4
  //! of dictionary entries. The field-management logic is delegated to
5
- //! [`lindera_binding_core::CoreSchema`]; this module only adds the magnus wrappers.
5
+ //! [`lindera_binding::CoreSchema`]; this module only adds the magnus wrappers.
6
6
 
7
7
  use magnus::prelude::*;
8
- use magnus::{Error, RArray, Ruby, function, method};
8
+ use magnus::{Error, Integer, RArray, Ruby, function, method};
9
9
 
10
10
  use lindera::dictionary::{FieldDefinition, FieldType, Schema};
11
- use lindera_binding_core::{CoreFieldDefinition, CoreFieldType, CoreSchema};
11
+ use lindera_binding::{CoreFieldDefinition, CoreFieldType, CoreSchema};
12
+
13
+ use crate::util::integer_arg;
12
14
 
13
15
  /// Field type in dictionary schema.
14
16
  ///
@@ -177,12 +179,12 @@ impl From<RbFieldDefinition> for FieldDefinition {
177
179
 
178
180
  /// Dictionary schema definition.
179
181
  ///
180
- /// A thin magnus wrapper over [`lindera_binding_core::CoreSchema`], which owns
182
+ /// A thin magnus wrapper over [`lindera_binding::CoreSchema`], which owns
181
183
  /// the field storage, the name-to-index map, and the field lookups.
182
184
  #[magnus::wrap(class = "Lindera::Schema", free_immediately, size)]
183
185
  #[derive(Debug, Clone)]
184
186
  pub struct RbSchema {
185
- /// The backing binding-core schema.
187
+ /// The backing lindera-binding schema.
186
188
  inner: CoreSchema,
187
189
  }
188
190
 
@@ -244,13 +246,22 @@ impl RbSchema {
244
246
  ///
245
247
  /// # Arguments
246
248
  ///
247
- /// * `index` - Index of the field.
249
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
250
+ /// * `rb_self` - The schema.
251
+ /// * `index` - Index of the field, 0 or more.
248
252
  ///
249
253
  /// # Returns
250
254
  ///
251
- /// The field name, or None if the index is out of bounds.
252
- fn get_field_name(&self, index: usize) -> Option<String> {
253
- self.inner.get_field_name(index).map(str::to_string)
255
+ /// The field name, `None` if the index is past the last field, or a
256
+ /// `RangeError` for an index that is negative or larger than a `usize`
257
+ /// holds.
258
+ fn get_field_name(
259
+ ruby: &Ruby,
260
+ rb_self: &Self,
261
+ index: Integer,
262
+ ) -> Result<Option<String>, Error> {
263
+ let index = integer_arg(ruby, "index", index)?;
264
+ Ok(rb_self.inner.get_field_name(index).map(str::to_string))
254
265
  }
255
266
 
256
267
  /// Returns the custom fields (fields after the first 4 standard fields).
data/src/token.rs CHANGED
@@ -3,10 +3,12 @@
3
3
  //! This module wraps Lindera tokens for use in Ruby.
4
4
 
5
5
  use magnus::prelude::*;
6
- use magnus::{Error, RHash, Ruby, method};
6
+ use magnus::{Error, Integer, RHash, Ruby, method};
7
7
 
8
8
  use lindera::token::Token;
9
9
 
10
+ use crate::util::integer_arg;
11
+
10
12
  /// Token object wrapping the Rust Token data.
11
13
  ///
12
14
  /// This class provides access to token fields and details.
@@ -39,19 +41,19 @@ impl RbToken {
39
41
  ///
40
42
  /// A new `RbToken` instance.
41
43
  pub fn from_token(token: Token) -> Self {
42
- Self::from_view(lindera_binding_core::TokenView::from_token(token))
44
+ Self::from_view(lindera_binding::TokenView::from_token(token))
43
45
  }
44
46
 
45
- /// Creates a new `RbToken` from a binding-core `TokenView`.
47
+ /// Creates a new `RbToken` from a lindera-binding `TokenView`.
46
48
  ///
47
49
  /// # Arguments
48
50
  ///
49
- /// * `view` - Token view produced by the binding-core tokenizer.
51
+ /// * `view` - Token view produced by the lindera-binding tokenizer.
50
52
  ///
51
53
  /// # Returns
52
54
  ///
53
55
  /// A new `RbToken` instance.
54
- pub fn from_view(view: lindera_binding_core::TokenView) -> Self {
56
+ pub fn from_view(view: lindera_binding::TokenView) -> Self {
55
57
  Self {
56
58
  surface: view.surface,
57
59
  byte_start: view.byte_start,
@@ -102,13 +104,18 @@ impl RbToken {
102
104
  ///
103
105
  /// # Arguments
104
106
  ///
105
- /// * `index` - Index of the detail to retrieve.
107
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
108
+ /// * `rb_self` - The token.
109
+ /// * `index` - Index of the detail to retrieve, 0 or more.
106
110
  ///
107
111
  /// # Returns
108
112
  ///
109
- /// The detail string if found, otherwise nil.
110
- fn get_detail(&self, index: usize) -> Option<String> {
111
- self.details.get(index).cloned()
113
+ /// The detail string if found, nil if the index is past the last detail,
114
+ /// or a `RangeError` for an index that is negative or larger than a
115
+ /// `usize` holds.
116
+ fn get_detail(ruby: &Ruby, rb_self: &Self, index: Integer) -> Result<Option<String>, Error> {
117
+ let index: usize = integer_arg(ruby, "index", index)?;
118
+ Ok(rb_self.details.get(index).cloned())
112
119
  }
113
120
 
114
121
  /// Returns the token as a plain Hash keyed by Symbols.