lindera 6.0.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +71 -37
- data/LICENSE +21 -0
- data/README.md +23 -6
- data/extconf.rb +13 -0
- data/src/character_filter.rs +3 -0
- data/src/dictionary.rs +239 -0
- data/src/error.rs +19 -0
- data/src/lib.rs +90 -0
- data/src/metadata.rs +423 -0
- data/src/mode.rs +401 -0
- data/src/schema.rs +568 -0
- data/src/token.rs +188 -0
- data/src/token_filter.rs +3 -0
- data/src/tokenizer.rs +373 -0
- data/src/trainer.rs +283 -0
- data/src/util.rs +168 -0
- metadata +17 -4
- data/ext/lindera_ruby/extconf.rb +0 -8
data/src/mode.rs
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
//! Tokenization modes and penalty configurations.
|
|
2
|
+
//!
|
|
3
|
+
//! This module defines the different tokenization modes available and their
|
|
4
|
+
//! penalty configurations for controlling segmentation behavior.
|
|
5
|
+
|
|
6
|
+
use magnus::prelude::*;
|
|
7
|
+
use magnus::{Error, Ruby, function, method};
|
|
8
|
+
|
|
9
|
+
use lindera::mode::{Mode as LinderaMode, Penalty as LinderaPenalty};
|
|
10
|
+
|
|
11
|
+
/// Tokenization mode.
|
|
12
|
+
///
|
|
13
|
+
/// Determines how text is segmented into tokens.
|
|
14
|
+
#[magnus::wrap(class = "Lindera::Mode", free_immediately, size)]
|
|
15
|
+
#[derive(Debug, Clone, Copy)]
|
|
16
|
+
pub struct RbMode {
|
|
17
|
+
/// Internal mode variant.
|
|
18
|
+
inner: RbModeKind,
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/// Internal enum for mode kind.
|
|
22
|
+
#[derive(Debug, Clone, Copy)]
|
|
23
|
+
enum RbModeKind {
|
|
24
|
+
/// Standard tokenization based on dictionary cost.
|
|
25
|
+
Normal,
|
|
26
|
+
/// Decompose compound words using penalty-based segmentation.
|
|
27
|
+
Decompose,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
impl RbMode {
|
|
31
|
+
/// Creates a new `RbMode` from a mode string.
|
|
32
|
+
///
|
|
33
|
+
/// # Arguments
|
|
34
|
+
///
|
|
35
|
+
/// * `mode_str` - Mode string ("normal" or "decompose"). Defaults to "normal" if None.
|
|
36
|
+
///
|
|
37
|
+
/// # Returns
|
|
38
|
+
///
|
|
39
|
+
/// A new `RbMode` instance.
|
|
40
|
+
fn new(mode_str: Option<String>) -> Result<Self, Error> {
|
|
41
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
42
|
+
let kind = match mode_str.as_deref() {
|
|
43
|
+
Some("decompose") | Some("Decompose") => RbModeKind::Decompose,
|
|
44
|
+
Some("normal") | Some("Normal") | None => RbModeKind::Normal,
|
|
45
|
+
Some(s) => {
|
|
46
|
+
return Err(Error::new(
|
|
47
|
+
ruby.exception_arg_error(),
|
|
48
|
+
format!("Invalid mode: {s}. Must be 'normal' or 'decompose'"),
|
|
49
|
+
));
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
Ok(Self { inner: kind })
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/// Returns the string representation of the mode.
|
|
56
|
+
///
|
|
57
|
+
/// # Returns
|
|
58
|
+
///
|
|
59
|
+
/// A string slice representing the mode.
|
|
60
|
+
#[allow(clippy::wrong_self_convention)]
|
|
61
|
+
fn to_s(&self) -> &str {
|
|
62
|
+
match self.inner {
|
|
63
|
+
RbModeKind::Normal => "normal",
|
|
64
|
+
RbModeKind::Decompose => "decompose",
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/// Returns the inspect representation of the mode.
|
|
69
|
+
///
|
|
70
|
+
/// # Returns
|
|
71
|
+
///
|
|
72
|
+
/// A string with the mode inspect representation.
|
|
73
|
+
fn inspect(&self) -> String {
|
|
74
|
+
format!("#<Lindera::Mode: {}>", self.to_s())
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/// Returns the name of the mode.
|
|
78
|
+
///
|
|
79
|
+
/// # Returns
|
|
80
|
+
///
|
|
81
|
+
/// A string slice with the mode name.
|
|
82
|
+
fn name(&self) -> &str {
|
|
83
|
+
self.to_s()
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// Returns true if the mode is normal.
|
|
87
|
+
///
|
|
88
|
+
/// # Returns
|
|
89
|
+
///
|
|
90
|
+
/// A boolean indicating if this is normal mode.
|
|
91
|
+
fn is_normal(&self) -> bool {
|
|
92
|
+
matches!(self.inner, RbModeKind::Normal)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/// Returns true if the mode is decompose.
|
|
96
|
+
///
|
|
97
|
+
/// # Returns
|
|
98
|
+
///
|
|
99
|
+
/// A boolean indicating if this is decompose mode.
|
|
100
|
+
fn is_decompose(&self) -> bool {
|
|
101
|
+
matches!(self.inner, RbModeKind::Decompose)
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
impl From<RbMode> for LinderaMode {
|
|
106
|
+
fn from(mode: RbMode) -> Self {
|
|
107
|
+
match mode.inner {
|
|
108
|
+
RbModeKind::Normal => LinderaMode::Normal,
|
|
109
|
+
RbModeKind::Decompose => LinderaMode::Decompose(LinderaPenalty::default()),
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
impl From<LinderaMode> for RbMode {
|
|
115
|
+
fn from(mode: LinderaMode) -> Self {
|
|
116
|
+
let kind = match mode {
|
|
117
|
+
LinderaMode::Normal => RbModeKind::Normal,
|
|
118
|
+
LinderaMode::Decompose(_) => RbModeKind::Decompose,
|
|
119
|
+
};
|
|
120
|
+
RbMode { inner: kind }
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/// Penalty configuration for decompose mode.
|
|
125
|
+
///
|
|
126
|
+
/// Controls how aggressively compound words are decomposed based on
|
|
127
|
+
/// character type and length thresholds.
|
|
128
|
+
#[magnus::wrap(class = "Lindera::Penalty", free_immediately, size)]
|
|
129
|
+
#[derive(Debug, Clone, Copy)]
|
|
130
|
+
pub struct RbPenalty {
|
|
131
|
+
/// Length threshold for kanji penalty.
|
|
132
|
+
kanji_penalty_length_threshold: usize,
|
|
133
|
+
/// Penalty value for kanji.
|
|
134
|
+
kanji_penalty_length_penalty: i32,
|
|
135
|
+
/// Length threshold for other character penalty.
|
|
136
|
+
other_penalty_length_threshold: usize,
|
|
137
|
+
/// Penalty value for other characters.
|
|
138
|
+
other_penalty_length_penalty: i32,
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
impl RbPenalty {
|
|
142
|
+
/// Creates a new `RbPenalty` with optional parameters.
|
|
143
|
+
///
|
|
144
|
+
/// # Arguments
|
|
145
|
+
///
|
|
146
|
+
/// * `kanji_threshold` - Kanji penalty length threshold (default: 2).
|
|
147
|
+
/// * `kanji_penalty` - Kanji penalty value (default: 3000).
|
|
148
|
+
/// * `other_threshold` - Other penalty length threshold (default: 7).
|
|
149
|
+
/// * `other_penalty` - Other penalty value (default: 1700).
|
|
150
|
+
///
|
|
151
|
+
/// # Returns
|
|
152
|
+
///
|
|
153
|
+
/// A new `RbPenalty` instance.
|
|
154
|
+
fn new(
|
|
155
|
+
kanji_threshold: Option<usize>,
|
|
156
|
+
kanji_penalty: Option<i32>,
|
|
157
|
+
other_threshold: Option<usize>,
|
|
158
|
+
other_penalty: Option<i32>,
|
|
159
|
+
) -> Self {
|
|
160
|
+
Self {
|
|
161
|
+
kanji_penalty_length_threshold: kanji_threshold.unwrap_or(2),
|
|
162
|
+
kanji_penalty_length_penalty: kanji_penalty.unwrap_or(3000),
|
|
163
|
+
other_penalty_length_threshold: other_threshold.unwrap_or(7),
|
|
164
|
+
other_penalty_length_penalty: other_penalty.unwrap_or(1700),
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/// Returns the kanji penalty length threshold.
|
|
169
|
+
fn kanji_penalty_length_threshold(&self) -> usize {
|
|
170
|
+
self.kanji_penalty_length_threshold
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/// Returns the kanji penalty length penalty.
|
|
174
|
+
fn kanji_penalty_length_penalty(&self) -> i32 {
|
|
175
|
+
self.kanji_penalty_length_penalty
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/// Returns the other penalty length threshold.
|
|
179
|
+
fn other_penalty_length_threshold(&self) -> usize {
|
|
180
|
+
self.other_penalty_length_threshold
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/// Returns the other penalty length penalty.
|
|
184
|
+
fn other_penalty_length_penalty(&self) -> i32 {
|
|
185
|
+
self.other_penalty_length_penalty
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/// Returns a string representation of the penalty configuration.
|
|
189
|
+
#[allow(clippy::wrong_self_convention)]
|
|
190
|
+
fn to_s(&self) -> String {
|
|
191
|
+
format!(
|
|
192
|
+
"Penalty(kanji_threshold={}, kanji_penalty={}, other_threshold={}, other_penalty={})",
|
|
193
|
+
self.kanji_penalty_length_threshold,
|
|
194
|
+
self.kanji_penalty_length_penalty,
|
|
195
|
+
self.other_penalty_length_threshold,
|
|
196
|
+
self.other_penalty_length_penalty
|
|
197
|
+
)
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/// Returns the inspect representation of the penalty.
|
|
201
|
+
fn inspect(&self) -> String {
|
|
202
|
+
format!("#<Lindera::Penalty: {}>", self.to_s())
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
impl From<RbPenalty> for LinderaPenalty {
|
|
207
|
+
fn from(penalty: RbPenalty) -> Self {
|
|
208
|
+
LinderaPenalty {
|
|
209
|
+
kanji_penalty_length_threshold: penalty.kanji_penalty_length_threshold,
|
|
210
|
+
kanji_penalty_length_penalty: penalty.kanji_penalty_length_penalty,
|
|
211
|
+
other_penalty_length_threshold: penalty.other_penalty_length_threshold,
|
|
212
|
+
other_penalty_length_penalty: penalty.other_penalty_length_penalty,
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
impl From<LinderaPenalty> for RbPenalty {
|
|
218
|
+
fn from(penalty: LinderaPenalty) -> Self {
|
|
219
|
+
RbPenalty {
|
|
220
|
+
kanji_penalty_length_threshold: penalty.kanji_penalty_length_threshold,
|
|
221
|
+
kanji_penalty_length_penalty: penalty.kanji_penalty_length_penalty,
|
|
222
|
+
other_penalty_length_threshold: penalty.other_penalty_length_threshold,
|
|
223
|
+
other_penalty_length_penalty: penalty.other_penalty_length_penalty,
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/// Defines Mode and Penalty classes in the given Ruby module.
|
|
229
|
+
///
|
|
230
|
+
/// # Arguments
|
|
231
|
+
///
|
|
232
|
+
/// * `ruby` - Ruby runtime handle.
|
|
233
|
+
/// * `module` - Parent Ruby module.
|
|
234
|
+
///
|
|
235
|
+
/// # Returns
|
|
236
|
+
///
|
|
237
|
+
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
238
|
+
pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
239
|
+
let mode_class = module.define_class("Mode", ruby.class_object())?;
|
|
240
|
+
mode_class.define_singleton_method("new", function!(RbMode::new, 1))?;
|
|
241
|
+
mode_class.define_method("to_s", method!(RbMode::to_s, 0))?;
|
|
242
|
+
mode_class.define_method("inspect", method!(RbMode::inspect, 0))?;
|
|
243
|
+
mode_class.define_method("name", method!(RbMode::name, 0))?;
|
|
244
|
+
mode_class.define_method("normal?", method!(RbMode::is_normal, 0))?;
|
|
245
|
+
mode_class.define_method("decompose?", method!(RbMode::is_decompose, 0))?;
|
|
246
|
+
|
|
247
|
+
let penalty_class = module.define_class("Penalty", ruby.class_object())?;
|
|
248
|
+
penalty_class.define_singleton_method("new", function!(RbPenalty::new, 4))?;
|
|
249
|
+
penalty_class.define_method(
|
|
250
|
+
"kanji_penalty_length_threshold",
|
|
251
|
+
method!(RbPenalty::kanji_penalty_length_threshold, 0),
|
|
252
|
+
)?;
|
|
253
|
+
penalty_class.define_method(
|
|
254
|
+
"kanji_penalty_length_penalty",
|
|
255
|
+
method!(RbPenalty::kanji_penalty_length_penalty, 0),
|
|
256
|
+
)?;
|
|
257
|
+
penalty_class.define_method(
|
|
258
|
+
"other_penalty_length_threshold",
|
|
259
|
+
method!(RbPenalty::other_penalty_length_threshold, 0),
|
|
260
|
+
)?;
|
|
261
|
+
penalty_class.define_method(
|
|
262
|
+
"other_penalty_length_penalty",
|
|
263
|
+
method!(RbPenalty::other_penalty_length_penalty, 0),
|
|
264
|
+
)?;
|
|
265
|
+
penalty_class.define_method("to_s", method!(RbPenalty::to_s, 0))?;
|
|
266
|
+
penalty_class.define_method("inspect", method!(RbPenalty::inspect, 0))?;
|
|
267
|
+
|
|
268
|
+
Ok(())
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
#[cfg(test)]
|
|
272
|
+
mod tests {
|
|
273
|
+
use super::*;
|
|
274
|
+
|
|
275
|
+
#[test]
|
|
276
|
+
fn test_rb_mode_normal_to_lindera_mode() {
|
|
277
|
+
let rb_mode = RbMode {
|
|
278
|
+
inner: RbModeKind::Normal,
|
|
279
|
+
};
|
|
280
|
+
let lindera_mode: LinderaMode = rb_mode.into();
|
|
281
|
+
assert!(matches!(lindera_mode, LinderaMode::Normal));
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
#[test]
|
|
285
|
+
fn test_rb_mode_decompose_to_lindera_mode() {
|
|
286
|
+
let rb_mode = RbMode {
|
|
287
|
+
inner: RbModeKind::Decompose,
|
|
288
|
+
};
|
|
289
|
+
let lindera_mode: LinderaMode = rb_mode.into();
|
|
290
|
+
assert!(matches!(lindera_mode, LinderaMode::Decompose(_)));
|
|
291
|
+
if let LinderaMode::Decompose(penalty) = lindera_mode {
|
|
292
|
+
let default_penalty = LinderaPenalty::default();
|
|
293
|
+
assert_eq!(
|
|
294
|
+
penalty.kanji_penalty_length_threshold,
|
|
295
|
+
default_penalty.kanji_penalty_length_threshold
|
|
296
|
+
);
|
|
297
|
+
assert_eq!(
|
|
298
|
+
penalty.kanji_penalty_length_penalty,
|
|
299
|
+
default_penalty.kanji_penalty_length_penalty
|
|
300
|
+
);
|
|
301
|
+
assert_eq!(
|
|
302
|
+
penalty.other_penalty_length_threshold,
|
|
303
|
+
default_penalty.other_penalty_length_threshold
|
|
304
|
+
);
|
|
305
|
+
assert_eq!(
|
|
306
|
+
penalty.other_penalty_length_penalty,
|
|
307
|
+
default_penalty.other_penalty_length_penalty
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
#[test]
|
|
313
|
+
fn test_lindera_mode_normal_to_rb_mode() {
|
|
314
|
+
let lindera_mode = LinderaMode::Normal;
|
|
315
|
+
let rb_mode: RbMode = lindera_mode.into();
|
|
316
|
+
assert!(matches!(rb_mode.inner, RbModeKind::Normal));
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
#[test]
|
|
320
|
+
fn test_lindera_mode_decompose_to_rb_mode() {
|
|
321
|
+
let lindera_mode = LinderaMode::Decompose(LinderaPenalty::default());
|
|
322
|
+
let rb_mode: RbMode = lindera_mode.into();
|
|
323
|
+
assert!(matches!(rb_mode.inner, RbModeKind::Decompose));
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
#[test]
|
|
327
|
+
fn test_rb_penalty_to_lindera_penalty() {
|
|
328
|
+
let rb_penalty = RbPenalty {
|
|
329
|
+
kanji_penalty_length_threshold: 3,
|
|
330
|
+
kanji_penalty_length_penalty: 5000,
|
|
331
|
+
other_penalty_length_threshold: 10,
|
|
332
|
+
other_penalty_length_penalty: 2500,
|
|
333
|
+
};
|
|
334
|
+
let lindera_penalty: LinderaPenalty = rb_penalty.into();
|
|
335
|
+
assert_eq!(lindera_penalty.kanji_penalty_length_threshold, 3);
|
|
336
|
+
assert_eq!(lindera_penalty.kanji_penalty_length_penalty, 5000);
|
|
337
|
+
assert_eq!(lindera_penalty.other_penalty_length_threshold, 10);
|
|
338
|
+
assert_eq!(lindera_penalty.other_penalty_length_penalty, 2500);
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
#[test]
|
|
342
|
+
fn test_lindera_penalty_to_rb_penalty() {
|
|
343
|
+
let lindera_penalty = LinderaPenalty {
|
|
344
|
+
kanji_penalty_length_threshold: 4,
|
|
345
|
+
kanji_penalty_length_penalty: 6000,
|
|
346
|
+
other_penalty_length_threshold: 8,
|
|
347
|
+
other_penalty_length_penalty: 1500,
|
|
348
|
+
};
|
|
349
|
+
let rb_penalty: RbPenalty = lindera_penalty.into();
|
|
350
|
+
assert_eq!(rb_penalty.kanji_penalty_length_threshold, 4);
|
|
351
|
+
assert_eq!(rb_penalty.kanji_penalty_length_penalty, 6000);
|
|
352
|
+
assert_eq!(rb_penalty.other_penalty_length_threshold, 8);
|
|
353
|
+
assert_eq!(rb_penalty.other_penalty_length_penalty, 1500);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
#[test]
|
|
357
|
+
fn test_rb_penalty_default_values_roundtrip() {
|
|
358
|
+
let default_lindera = LinderaPenalty::default();
|
|
359
|
+
let rb_penalty: RbPenalty = default_lindera.clone().into();
|
|
360
|
+
let roundtripped: LinderaPenalty = rb_penalty.into();
|
|
361
|
+
assert_eq!(
|
|
362
|
+
roundtripped.kanji_penalty_length_threshold,
|
|
363
|
+
default_lindera.kanji_penalty_length_threshold
|
|
364
|
+
);
|
|
365
|
+
assert_eq!(
|
|
366
|
+
roundtripped.kanji_penalty_length_penalty,
|
|
367
|
+
default_lindera.kanji_penalty_length_penalty
|
|
368
|
+
);
|
|
369
|
+
assert_eq!(
|
|
370
|
+
roundtripped.other_penalty_length_threshold,
|
|
371
|
+
default_lindera.other_penalty_length_threshold
|
|
372
|
+
);
|
|
373
|
+
assert_eq!(
|
|
374
|
+
roundtripped.other_penalty_length_penalty,
|
|
375
|
+
default_lindera.other_penalty_length_penalty
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
#[test]
|
|
380
|
+
fn test_mode_roundtrip_normal() {
|
|
381
|
+
let original = LinderaMode::Normal;
|
|
382
|
+
let rb: RbMode = original.into();
|
|
383
|
+
let back: LinderaMode = rb.into();
|
|
384
|
+
assert!(matches!(back, LinderaMode::Normal));
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
#[test]
|
|
388
|
+
fn test_mode_roundtrip_decompose() {
|
|
389
|
+
let penalty = LinderaPenalty {
|
|
390
|
+
kanji_penalty_length_threshold: 5,
|
|
391
|
+
kanji_penalty_length_penalty: 4000,
|
|
392
|
+
other_penalty_length_threshold: 9,
|
|
393
|
+
other_penalty_length_penalty: 2000,
|
|
394
|
+
};
|
|
395
|
+
let original = LinderaMode::Decompose(penalty);
|
|
396
|
+
let rb: RbMode = original.into();
|
|
397
|
+
let back: LinderaMode = rb.into();
|
|
398
|
+
// Note: RbMode loses the penalty details, so Decompose uses default penalty.
|
|
399
|
+
assert!(matches!(back, LinderaMode::Decompose(_)));
|
|
400
|
+
}
|
|
401
|
+
}
|