lindera 5.3.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/mode.rs ADDED
@@ -0,0 +1,401 @@
1
+ //! Tokenization modes and penalty configurations.
2
+ //!
3
+ //! This module defines the different tokenization modes available and their
4
+ //! penalty configurations for controlling segmentation behavior.
5
+
6
+ use magnus::prelude::*;
7
+ use magnus::{Error, Ruby, function, method};
8
+
9
+ use lindera::mode::{Mode as LinderaMode, Penalty as LinderaPenalty};
10
+
11
+ /// Tokenization mode.
12
+ ///
13
+ /// Determines how text is segmented into tokens.
14
+ #[magnus::wrap(class = "Lindera::Mode", free_immediately, size)]
15
+ #[derive(Debug, Clone, Copy)]
16
+ pub struct RbMode {
17
+ /// Internal mode variant.
18
+ inner: RbModeKind,
19
+ }
20
+
21
+ /// Internal enum for mode kind.
22
+ #[derive(Debug, Clone, Copy)]
23
+ enum RbModeKind {
24
+ /// Standard tokenization based on dictionary cost.
25
+ Normal,
26
+ /// Decompose compound words using penalty-based segmentation.
27
+ Decompose,
28
+ }
29
+
30
+ impl RbMode {
31
+ /// Creates a new `RbMode` from a mode string.
32
+ ///
33
+ /// # Arguments
34
+ ///
35
+ /// * `mode_str` - Mode string ("normal" or "decompose"). Defaults to "normal" if None.
36
+ ///
37
+ /// # Returns
38
+ ///
39
+ /// A new `RbMode` instance.
40
+ fn new(mode_str: Option<String>) -> Result<Self, Error> {
41
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
42
+ let kind = match mode_str.as_deref() {
43
+ Some("decompose") | Some("Decompose") => RbModeKind::Decompose,
44
+ Some("normal") | Some("Normal") | None => RbModeKind::Normal,
45
+ Some(s) => {
46
+ return Err(Error::new(
47
+ ruby.exception_arg_error(),
48
+ format!("Invalid mode: {s}. Must be 'normal' or 'decompose'"),
49
+ ));
50
+ }
51
+ };
52
+ Ok(Self { inner: kind })
53
+ }
54
+
55
+ /// Returns the string representation of the mode.
56
+ ///
57
+ /// # Returns
58
+ ///
59
+ /// A string slice representing the mode.
60
+ #[allow(clippy::wrong_self_convention)]
61
+ fn to_s(&self) -> &str {
62
+ match self.inner {
63
+ RbModeKind::Normal => "normal",
64
+ RbModeKind::Decompose => "decompose",
65
+ }
66
+ }
67
+
68
+ /// Returns the inspect representation of the mode.
69
+ ///
70
+ /// # Returns
71
+ ///
72
+ /// A string with the mode inspect representation.
73
+ fn inspect(&self) -> String {
74
+ format!("#<Lindera::Mode: {}>", self.to_s())
75
+ }
76
+
77
+ /// Returns the name of the mode.
78
+ ///
79
+ /// # Returns
80
+ ///
81
+ /// A string slice with the mode name.
82
+ fn name(&self) -> &str {
83
+ self.to_s()
84
+ }
85
+
86
+ /// Returns true if the mode is normal.
87
+ ///
88
+ /// # Returns
89
+ ///
90
+ /// A boolean indicating if this is normal mode.
91
+ fn is_normal(&self) -> bool {
92
+ matches!(self.inner, RbModeKind::Normal)
93
+ }
94
+
95
+ /// Returns true if the mode is decompose.
96
+ ///
97
+ /// # Returns
98
+ ///
99
+ /// A boolean indicating if this is decompose mode.
100
+ fn is_decompose(&self) -> bool {
101
+ matches!(self.inner, RbModeKind::Decompose)
102
+ }
103
+ }
104
+
105
+ impl From<RbMode> for LinderaMode {
106
+ fn from(mode: RbMode) -> Self {
107
+ match mode.inner {
108
+ RbModeKind::Normal => LinderaMode::Normal,
109
+ RbModeKind::Decompose => LinderaMode::Decompose(LinderaPenalty::default()),
110
+ }
111
+ }
112
+ }
113
+
114
+ impl From<LinderaMode> for RbMode {
115
+ fn from(mode: LinderaMode) -> Self {
116
+ let kind = match mode {
117
+ LinderaMode::Normal => RbModeKind::Normal,
118
+ LinderaMode::Decompose(_) => RbModeKind::Decompose,
119
+ };
120
+ RbMode { inner: kind }
121
+ }
122
+ }
123
+
124
+ /// Penalty configuration for decompose mode.
125
+ ///
126
+ /// Controls how aggressively compound words are decomposed based on
127
+ /// character type and length thresholds.
128
+ #[magnus::wrap(class = "Lindera::Penalty", free_immediately, size)]
129
+ #[derive(Debug, Clone, Copy)]
130
+ pub struct RbPenalty {
131
+ /// Length threshold for kanji penalty.
132
+ kanji_penalty_length_threshold: usize,
133
+ /// Penalty value for kanji.
134
+ kanji_penalty_length_penalty: i32,
135
+ /// Length threshold for other character penalty.
136
+ other_penalty_length_threshold: usize,
137
+ /// Penalty value for other characters.
138
+ other_penalty_length_penalty: i32,
139
+ }
140
+
141
+ impl RbPenalty {
142
+ /// Creates a new `RbPenalty` with optional parameters.
143
+ ///
144
+ /// # Arguments
145
+ ///
146
+ /// * `kanji_threshold` - Kanji penalty length threshold (default: 2).
147
+ /// * `kanji_penalty` - Kanji penalty value (default: 3000).
148
+ /// * `other_threshold` - Other penalty length threshold (default: 7).
149
+ /// * `other_penalty` - Other penalty value (default: 1700).
150
+ ///
151
+ /// # Returns
152
+ ///
153
+ /// A new `RbPenalty` instance.
154
+ fn new(
155
+ kanji_threshold: Option<usize>,
156
+ kanji_penalty: Option<i32>,
157
+ other_threshold: Option<usize>,
158
+ other_penalty: Option<i32>,
159
+ ) -> Self {
160
+ Self {
161
+ kanji_penalty_length_threshold: kanji_threshold.unwrap_or(2),
162
+ kanji_penalty_length_penalty: kanji_penalty.unwrap_or(3000),
163
+ other_penalty_length_threshold: other_threshold.unwrap_or(7),
164
+ other_penalty_length_penalty: other_penalty.unwrap_or(1700),
165
+ }
166
+ }
167
+
168
+ /// Returns the kanji penalty length threshold.
169
+ fn kanji_penalty_length_threshold(&self) -> usize {
170
+ self.kanji_penalty_length_threshold
171
+ }
172
+
173
+ /// Returns the kanji penalty length penalty.
174
+ fn kanji_penalty_length_penalty(&self) -> i32 {
175
+ self.kanji_penalty_length_penalty
176
+ }
177
+
178
+ /// Returns the other penalty length threshold.
179
+ fn other_penalty_length_threshold(&self) -> usize {
180
+ self.other_penalty_length_threshold
181
+ }
182
+
183
+ /// Returns the other penalty length penalty.
184
+ fn other_penalty_length_penalty(&self) -> i32 {
185
+ self.other_penalty_length_penalty
186
+ }
187
+
188
+ /// Returns a string representation of the penalty configuration.
189
+ #[allow(clippy::wrong_self_convention)]
190
+ fn to_s(&self) -> String {
191
+ format!(
192
+ "Penalty(kanji_threshold={}, kanji_penalty={}, other_threshold={}, other_penalty={})",
193
+ self.kanji_penalty_length_threshold,
194
+ self.kanji_penalty_length_penalty,
195
+ self.other_penalty_length_threshold,
196
+ self.other_penalty_length_penalty
197
+ )
198
+ }
199
+
200
+ /// Returns the inspect representation of the penalty.
201
+ fn inspect(&self) -> String {
202
+ format!("#<Lindera::Penalty: {}>", self.to_s())
203
+ }
204
+ }
205
+
206
+ impl From<RbPenalty> for LinderaPenalty {
207
+ fn from(penalty: RbPenalty) -> Self {
208
+ LinderaPenalty {
209
+ kanji_penalty_length_threshold: penalty.kanji_penalty_length_threshold,
210
+ kanji_penalty_length_penalty: penalty.kanji_penalty_length_penalty,
211
+ other_penalty_length_threshold: penalty.other_penalty_length_threshold,
212
+ other_penalty_length_penalty: penalty.other_penalty_length_penalty,
213
+ }
214
+ }
215
+ }
216
+
217
+ impl From<LinderaPenalty> for RbPenalty {
218
+ fn from(penalty: LinderaPenalty) -> Self {
219
+ RbPenalty {
220
+ kanji_penalty_length_threshold: penalty.kanji_penalty_length_threshold,
221
+ kanji_penalty_length_penalty: penalty.kanji_penalty_length_penalty,
222
+ other_penalty_length_threshold: penalty.other_penalty_length_threshold,
223
+ other_penalty_length_penalty: penalty.other_penalty_length_penalty,
224
+ }
225
+ }
226
+ }
227
+
228
+ /// Defines Mode and Penalty classes in the given Ruby module.
229
+ ///
230
+ /// # Arguments
231
+ ///
232
+ /// * `ruby` - Ruby runtime handle.
233
+ /// * `module` - Parent Ruby module.
234
+ ///
235
+ /// # Returns
236
+ ///
237
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
238
+ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
239
+ let mode_class = module.define_class("Mode", ruby.class_object())?;
240
+ mode_class.define_singleton_method("new", function!(RbMode::new, 1))?;
241
+ mode_class.define_method("to_s", method!(RbMode::to_s, 0))?;
242
+ mode_class.define_method("inspect", method!(RbMode::inspect, 0))?;
243
+ mode_class.define_method("name", method!(RbMode::name, 0))?;
244
+ mode_class.define_method("normal?", method!(RbMode::is_normal, 0))?;
245
+ mode_class.define_method("decompose?", method!(RbMode::is_decompose, 0))?;
246
+
247
+ let penalty_class = module.define_class("Penalty", ruby.class_object())?;
248
+ penalty_class.define_singleton_method("new", function!(RbPenalty::new, 4))?;
249
+ penalty_class.define_method(
250
+ "kanji_penalty_length_threshold",
251
+ method!(RbPenalty::kanji_penalty_length_threshold, 0),
252
+ )?;
253
+ penalty_class.define_method(
254
+ "kanji_penalty_length_penalty",
255
+ method!(RbPenalty::kanji_penalty_length_penalty, 0),
256
+ )?;
257
+ penalty_class.define_method(
258
+ "other_penalty_length_threshold",
259
+ method!(RbPenalty::other_penalty_length_threshold, 0),
260
+ )?;
261
+ penalty_class.define_method(
262
+ "other_penalty_length_penalty",
263
+ method!(RbPenalty::other_penalty_length_penalty, 0),
264
+ )?;
265
+ penalty_class.define_method("to_s", method!(RbPenalty::to_s, 0))?;
266
+ penalty_class.define_method("inspect", method!(RbPenalty::inspect, 0))?;
267
+
268
+ Ok(())
269
+ }
270
+
271
+ #[cfg(test)]
272
+ mod tests {
273
+ use super::*;
274
+
275
+ #[test]
276
+ fn test_rb_mode_normal_to_lindera_mode() {
277
+ let rb_mode = RbMode {
278
+ inner: RbModeKind::Normal,
279
+ };
280
+ let lindera_mode: LinderaMode = rb_mode.into();
281
+ assert!(matches!(lindera_mode, LinderaMode::Normal));
282
+ }
283
+
284
+ #[test]
285
+ fn test_rb_mode_decompose_to_lindera_mode() {
286
+ let rb_mode = RbMode {
287
+ inner: RbModeKind::Decompose,
288
+ };
289
+ let lindera_mode: LinderaMode = rb_mode.into();
290
+ assert!(matches!(lindera_mode, LinderaMode::Decompose(_)));
291
+ if let LinderaMode::Decompose(penalty) = lindera_mode {
292
+ let default_penalty = LinderaPenalty::default();
293
+ assert_eq!(
294
+ penalty.kanji_penalty_length_threshold,
295
+ default_penalty.kanji_penalty_length_threshold
296
+ );
297
+ assert_eq!(
298
+ penalty.kanji_penalty_length_penalty,
299
+ default_penalty.kanji_penalty_length_penalty
300
+ );
301
+ assert_eq!(
302
+ penalty.other_penalty_length_threshold,
303
+ default_penalty.other_penalty_length_threshold
304
+ );
305
+ assert_eq!(
306
+ penalty.other_penalty_length_penalty,
307
+ default_penalty.other_penalty_length_penalty
308
+ );
309
+ }
310
+ }
311
+
312
+ #[test]
313
+ fn test_lindera_mode_normal_to_rb_mode() {
314
+ let lindera_mode = LinderaMode::Normal;
315
+ let rb_mode: RbMode = lindera_mode.into();
316
+ assert!(matches!(rb_mode.inner, RbModeKind::Normal));
317
+ }
318
+
319
+ #[test]
320
+ fn test_lindera_mode_decompose_to_rb_mode() {
321
+ let lindera_mode = LinderaMode::Decompose(LinderaPenalty::default());
322
+ let rb_mode: RbMode = lindera_mode.into();
323
+ assert!(matches!(rb_mode.inner, RbModeKind::Decompose));
324
+ }
325
+
326
+ #[test]
327
+ fn test_rb_penalty_to_lindera_penalty() {
328
+ let rb_penalty = RbPenalty {
329
+ kanji_penalty_length_threshold: 3,
330
+ kanji_penalty_length_penalty: 5000,
331
+ other_penalty_length_threshold: 10,
332
+ other_penalty_length_penalty: 2500,
333
+ };
334
+ let lindera_penalty: LinderaPenalty = rb_penalty.into();
335
+ assert_eq!(lindera_penalty.kanji_penalty_length_threshold, 3);
336
+ assert_eq!(lindera_penalty.kanji_penalty_length_penalty, 5000);
337
+ assert_eq!(lindera_penalty.other_penalty_length_threshold, 10);
338
+ assert_eq!(lindera_penalty.other_penalty_length_penalty, 2500);
339
+ }
340
+
341
+ #[test]
342
+ fn test_lindera_penalty_to_rb_penalty() {
343
+ let lindera_penalty = LinderaPenalty {
344
+ kanji_penalty_length_threshold: 4,
345
+ kanji_penalty_length_penalty: 6000,
346
+ other_penalty_length_threshold: 8,
347
+ other_penalty_length_penalty: 1500,
348
+ };
349
+ let rb_penalty: RbPenalty = lindera_penalty.into();
350
+ assert_eq!(rb_penalty.kanji_penalty_length_threshold, 4);
351
+ assert_eq!(rb_penalty.kanji_penalty_length_penalty, 6000);
352
+ assert_eq!(rb_penalty.other_penalty_length_threshold, 8);
353
+ assert_eq!(rb_penalty.other_penalty_length_penalty, 1500);
354
+ }
355
+
356
+ #[test]
357
+ fn test_rb_penalty_default_values_roundtrip() {
358
+ let default_lindera = LinderaPenalty::default();
359
+ let rb_penalty: RbPenalty = default_lindera.clone().into();
360
+ let roundtripped: LinderaPenalty = rb_penalty.into();
361
+ assert_eq!(
362
+ roundtripped.kanji_penalty_length_threshold,
363
+ default_lindera.kanji_penalty_length_threshold
364
+ );
365
+ assert_eq!(
366
+ roundtripped.kanji_penalty_length_penalty,
367
+ default_lindera.kanji_penalty_length_penalty
368
+ );
369
+ assert_eq!(
370
+ roundtripped.other_penalty_length_threshold,
371
+ default_lindera.other_penalty_length_threshold
372
+ );
373
+ assert_eq!(
374
+ roundtripped.other_penalty_length_penalty,
375
+ default_lindera.other_penalty_length_penalty
376
+ );
377
+ }
378
+
379
+ #[test]
380
+ fn test_mode_roundtrip_normal() {
381
+ let original = LinderaMode::Normal;
382
+ let rb: RbMode = original.into();
383
+ let back: LinderaMode = rb.into();
384
+ assert!(matches!(back, LinderaMode::Normal));
385
+ }
386
+
387
+ #[test]
388
+ fn test_mode_roundtrip_decompose() {
389
+ let penalty = LinderaPenalty {
390
+ kanji_penalty_length_threshold: 5,
391
+ kanji_penalty_length_penalty: 4000,
392
+ other_penalty_length_threshold: 9,
393
+ other_penalty_length_penalty: 2000,
394
+ };
395
+ let original = LinderaMode::Decompose(penalty);
396
+ let rb: RbMode = original.into();
397
+ let back: LinderaMode = rb.into();
398
+ // Note: RbMode loses the penalty details, so Decompose uses default penalty.
399
+ assert!(matches!(back, LinderaMode::Decompose(_)));
400
+ }
401
+ }