lindera 5.3.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/token.rs ADDED
@@ -0,0 +1,188 @@
1
+ //! Token representation for morphological analysis results.
2
+ //!
3
+ //! This module wraps Lindera tokens for use in Ruby.
4
+
5
+ use magnus::prelude::*;
6
+ use magnus::{Error, RHash, Ruby, method};
7
+
8
+ use lindera::token::Token;
9
+
10
+ /// Token object wrapping the Rust Token data.
11
+ ///
12
+ /// This class provides access to token fields and details.
13
+ #[magnus::wrap(class = "Lindera::Token", free_immediately, size)]
14
+ pub struct RbToken {
15
+ /// Surface form of the token.
16
+ surface: String,
17
+ /// Start byte position in the original text.
18
+ byte_start: usize,
19
+ /// End byte position in the original text.
20
+ byte_end: usize,
21
+ /// Position index of the token.
22
+ position: usize,
23
+ /// Word ID in the dictionary.
24
+ word_id: u32,
25
+ /// Whether this token is an unknown word.
26
+ is_unknown: bool,
27
+ /// Morphological details of the token.
28
+ details: Vec<String>,
29
+ }
30
+
31
+ impl RbToken {
32
+ /// Creates a new `RbToken` from a Lindera `Token`.
33
+ ///
34
+ /// # Arguments
35
+ ///
36
+ /// * `token` - Lindera token to convert.
37
+ ///
38
+ /// # Returns
39
+ ///
40
+ /// A new `RbToken` instance.
41
+ pub fn from_token(token: Token) -> Self {
42
+ Self::from_view(lindera_binding_core::TokenView::from_token(token))
43
+ }
44
+
45
+ /// Creates a new `RbToken` from a binding-core `TokenView`.
46
+ ///
47
+ /// # Arguments
48
+ ///
49
+ /// * `view` - Token view produced by the binding-core tokenizer.
50
+ ///
51
+ /// # Returns
52
+ ///
53
+ /// A new `RbToken` instance.
54
+ pub fn from_view(view: lindera_binding_core::TokenView) -> Self {
55
+ Self {
56
+ surface: view.surface,
57
+ byte_start: view.byte_start,
58
+ byte_end: view.byte_end,
59
+ position: view.position,
60
+ word_id: view.word_id,
61
+ is_unknown: view.is_unknown,
62
+ details: view.details,
63
+ }
64
+ }
65
+
66
+ /// Returns the surface form of the token.
67
+ fn surface(&self) -> String {
68
+ self.surface.clone()
69
+ }
70
+
71
+ /// Returns the start byte position.
72
+ fn byte_start(&self) -> usize {
73
+ self.byte_start
74
+ }
75
+
76
+ /// Returns the end byte position.
77
+ fn byte_end(&self) -> usize {
78
+ self.byte_end
79
+ }
80
+
81
+ /// Returns the position index.
82
+ fn position(&self) -> usize {
83
+ self.position
84
+ }
85
+
86
+ /// Returns the word ID.
87
+ fn word_id(&self) -> u32 {
88
+ self.word_id
89
+ }
90
+
91
+ /// Returns whether this token is an unknown word.
92
+ fn is_unknown(&self) -> bool {
93
+ self.is_unknown
94
+ }
95
+
96
+ /// Returns the morphological details of the token.
97
+ fn details(&self) -> Vec<String> {
98
+ self.details.clone()
99
+ }
100
+
101
+ /// Returns the detail at the specified index.
102
+ ///
103
+ /// # Arguments
104
+ ///
105
+ /// * `index` - Index of the detail to retrieve.
106
+ ///
107
+ /// # Returns
108
+ ///
109
+ /// The detail string if found, otherwise nil.
110
+ fn get_detail(&self, index: usize) -> Option<String> {
111
+ self.details.get(index).cloned()
112
+ }
113
+
114
+ /// Returns the token as a plain Hash keyed by Symbols.
115
+ ///
116
+ /// Values keep their natural Ruby types (Integer for the positions and
117
+ /// word id, true/false for `:is_unknown`, Array of String for
118
+ /// `:details`), so the result works directly with `JSON.generate`.
119
+ ///
120
+ /// # Returns
121
+ ///
122
+ /// A Hash with the keys `:surface`, `:byte_start`, `:byte_end`,
123
+ /// `:position`, `:word_id`, `:is_unknown`, and `:details`, or a Magnus
124
+ /// `Error` if the Hash cannot be built.
125
+ fn to_hash(ruby: &Ruby, rb_self: &Self) -> Result<RHash, Error> {
126
+ let this = rb_self;
127
+ let hash = ruby.hash_new();
128
+ hash.aset(ruby.to_symbol("surface"), this.surface.as_str())?;
129
+ hash.aset(ruby.to_symbol("byte_start"), this.byte_start)?;
130
+ hash.aset(ruby.to_symbol("byte_end"), this.byte_end)?;
131
+ hash.aset(ruby.to_symbol("position"), this.position)?;
132
+ hash.aset(ruby.to_symbol("word_id"), this.word_id)?;
133
+ hash.aset(ruby.to_symbol("is_unknown"), this.is_unknown)?;
134
+ let details = ruby.ary_new();
135
+ for detail in &this.details {
136
+ details.push(detail.as_str())?;
137
+ }
138
+ hash.aset(ruby.to_symbol("details"), details)?;
139
+ Ok(hash)
140
+ }
141
+
142
+ /// Returns the string representation of the token.
143
+ fn to_s(&self) -> String {
144
+ self.surface.clone()
145
+ }
146
+
147
+ /// Returns the inspect representation of the token.
148
+ fn inspect(&self) -> String {
149
+ format!(
150
+ "#<Lindera::Token surface='{}', start={}, end={}, position={}, word_id={}, unknown={}>",
151
+ self.surface,
152
+ self.byte_start,
153
+ self.byte_end,
154
+ self.position,
155
+ self.word_id,
156
+ self.is_unknown
157
+ )
158
+ }
159
+ }
160
+
161
+ /// Defines the Token class in the given Ruby module.
162
+ ///
163
+ /// # Arguments
164
+ ///
165
+ /// * `ruby` - Ruby runtime handle.
166
+ /// * `module` - Parent Ruby module.
167
+ ///
168
+ /// # Returns
169
+ ///
170
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
171
+ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
172
+ let token_class = module.define_class("Token", ruby.class_object())?;
173
+ token_class.define_method("surface", method!(RbToken::surface, 0))?;
174
+ token_class.define_method("byte_start", method!(RbToken::byte_start, 0))?;
175
+ token_class.define_method("byte_end", method!(RbToken::byte_end, 0))?;
176
+ token_class.define_method("position", method!(RbToken::position, 0))?;
177
+ token_class.define_method("word_id", method!(RbToken::word_id, 0))?;
178
+ token_class.define_method("unknown?", method!(RbToken::is_unknown, 0))?;
179
+ token_class.define_method("details", method!(RbToken::details, 0))?;
180
+ token_class.define_method("get_detail", method!(RbToken::get_detail, 1))?;
181
+ token_class.define_method("to_hash", method!(RbToken::to_hash, 0))?;
182
+ token_class.define_method("to_h", method!(RbToken::to_hash, 0))?;
183
+
184
+ token_class.define_method("to_s", method!(RbToken::to_s, 0))?;
185
+ token_class.define_method("inspect", method!(RbToken::inspect, 0))?;
186
+
187
+ Ok(())
188
+ }
@@ -0,0 +1,3 @@
1
+ //! Token filters for post-processing tokens.
2
+ //!
3
+ //! This module is a placeholder for future token filter implementations.
data/src/tokenizer.rs ADDED
@@ -0,0 +1,373 @@
1
+ //! Tokenizer implementation for morphological analysis.
2
+ //!
3
+ //! This module provides a builder pattern for creating tokenizers and the tokenizer itself.
4
+ //! The build-flow orchestration is delegated to
5
+ //! [`lindera_binding_core::CoreTokenizerBuilder`] / [`lindera_binding_core::CoreTokenizer`];
6
+ //! this module only adds the magnus wrappers and the Ruby-hash conversion.
7
+
8
+ use std::cell::RefCell;
9
+ use std::path::Path;
10
+
11
+ use magnus::prelude::*;
12
+ use magnus::{Error, RArray, RHash, Ruby, Value, function, method};
13
+
14
+ use lindera_binding_core::{CoreTokenizer, CoreTokenizerBuilder};
15
+
16
+ use crate::dictionary::{RbDictionary, RbUserDictionary};
17
+ use crate::error::to_magnus_error;
18
+ use crate::token::RbToken;
19
+ use crate::util::{rb_hash_to_json, rb_value_to_json};
20
+
21
+ /// Builder for creating a `Tokenizer` with custom configuration.
22
+ ///
23
+ /// The builder pattern allows for fluent configuration of tokenizer parameters.
24
+ /// Uses `RefCell` for interior mutability since Magnus `method!` requires `&self`.
25
+ #[magnus::wrap(class = "Lindera::TokenizerBuilder", free_immediately, size)]
26
+ pub struct RbTokenizerBuilder {
27
+ /// Inner binding-core tokenizer builder (wrapped in RefCell for interior mutability).
28
+ inner: RefCell<CoreTokenizerBuilder>,
29
+ }
30
+
31
+ impl RbTokenizerBuilder {
32
+ /// Creates a new `RbTokenizerBuilder` with default configuration.
33
+ ///
34
+ /// # Returns
35
+ ///
36
+ /// A new `RbTokenizerBuilder` instance.
37
+ fn new() -> Result<Self, Error> {
38
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
39
+ let inner =
40
+ CoreTokenizerBuilder::new().map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
41
+ Ok(Self {
42
+ inner: RefCell::new(inner),
43
+ })
44
+ }
45
+
46
+ /// Loads configuration from a file.
47
+ ///
48
+ /// # Arguments
49
+ ///
50
+ /// * `file_path` - Path to the configuration file.
51
+ ///
52
+ /// # Returns
53
+ ///
54
+ /// A new `RbTokenizerBuilder` with the loaded configuration.
55
+ fn from_file(file_path: String) -> Result<Self, Error> {
56
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
57
+ let inner = CoreTokenizerBuilder::from_file(Path::new(&file_path))
58
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
59
+ Ok(Self {
60
+ inner: RefCell::new(inner),
61
+ })
62
+ }
63
+
64
+ /// Sets the tokenization mode.
65
+ ///
66
+ /// # Arguments
67
+ ///
68
+ /// * `mode` - Mode string ("normal" or "decompose").
69
+ fn set_mode(&self, mode: String) -> Result<(), Error> {
70
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
71
+ let mut builder = self.inner.borrow_mut();
72
+ builder
73
+ .set_mode(&mode)
74
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
75
+ Ok(())
76
+ }
77
+
78
+ /// Sets the dictionary path.
79
+ ///
80
+ /// # Arguments
81
+ ///
82
+ /// * `path` - Path to the dictionary directory.
83
+ fn set_dictionary(&self, path: String) {
84
+ self.inner.borrow_mut().set_dictionary(&path);
85
+ }
86
+
87
+ /// Sets the user dictionary URI.
88
+ ///
89
+ /// # Arguments
90
+ ///
91
+ /// * `uri` - URI to the user dictionary.
92
+ fn set_user_dictionary(&self, uri: String) {
93
+ self.inner.borrow_mut().set_user_dictionary(&uri);
94
+ }
95
+
96
+ /// Sets whether to keep whitespace in tokenization results.
97
+ ///
98
+ /// # Arguments
99
+ ///
100
+ /// * `keep_whitespace` - If true, whitespace tokens will be included.
101
+ fn set_keep_whitespace(&self, keep_whitespace: bool) {
102
+ self.inner.borrow_mut().set_keep_whitespace(keep_whitespace);
103
+ }
104
+
105
+ /// Sets the left-space penalty (Korean), with the same meaning as
106
+ /// `segmenter.space_penalty` in a YAML config.
107
+ ///
108
+ /// `nil` restores the default (the rules the dictionary ships, if any;
109
+ /// ko-dic ships mecab-ko-dic's), `false` turns the penalty off, `true`
110
+ /// requires the dictionary's rules (`build` then fails for a dictionary
111
+ /// that ships none), and a hash such as
112
+ /// `{ 'rules' => [{ 'pos' => ['JKS'], 'cost' => 6000 }] }` applies those
113
+ /// rules instead.
114
+ ///
115
+ /// # Arguments
116
+ ///
117
+ /// * `ruby` - Ruby runtime handle.
118
+ /// * `rb_self` - The builder.
119
+ /// * `value` - `nil`, `true`, `false`, or a hash with string keys holding `"rules"`.
120
+ ///
121
+ /// # Returns
122
+ ///
123
+ /// `Ok(())` on success, a `TypeError` when `value` holds something that
124
+ /// cannot be converted to JSON (such as a Symbol or a non-finite Float),
125
+ /// or a `RuntimeError` when it converts but is not one of the forms
126
+ /// above.
127
+ fn set_space_penalty(ruby: &Ruby, rb_self: &Self, value: Value) -> Result<(), Error> {
128
+ let setting = rb_value_to_json(ruby, value)?;
129
+ rb_self
130
+ .inner
131
+ .borrow_mut()
132
+ .set_space_penalty(&setting)
133
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
134
+ Ok(())
135
+ }
136
+
137
+ /// Appends a character filter to the filter pipeline.
138
+ ///
139
+ /// # Arguments
140
+ ///
141
+ /// * `kind` - Type of character filter to add.
142
+ /// * `args` - Optional hash of filter arguments.
143
+ fn append_character_filter(&self, kind: String, args: Option<RHash>) -> Result<(), Error> {
144
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
145
+ let filter_args = if let Some(hash) = args {
146
+ rb_hash_to_json(&ruby, hash)?
147
+ } else {
148
+ serde_json::Value::Object(serde_json::Map::new())
149
+ };
150
+ self.inner
151
+ .borrow_mut()
152
+ .append_character_filter(&kind, &filter_args);
153
+ Ok(())
154
+ }
155
+
156
+ /// Appends a token filter to the filter pipeline.
157
+ ///
158
+ /// # Arguments
159
+ ///
160
+ /// * `kind` - Type of token filter to add.
161
+ /// * `args` - Optional hash of filter arguments.
162
+ fn append_token_filter(&self, kind: String, args: Option<RHash>) -> Result<(), Error> {
163
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
164
+ let filter_args = if let Some(hash) = args {
165
+ rb_hash_to_json(&ruby, hash)?
166
+ } else {
167
+ serde_json::Value::Object(serde_json::Map::new())
168
+ };
169
+ self.inner
170
+ .borrow_mut()
171
+ .append_token_filter(&kind, &filter_args);
172
+ Ok(())
173
+ }
174
+
175
+ /// Builds the tokenizer with the configured settings.
176
+ ///
177
+ /// # Returns
178
+ ///
179
+ /// A configured `RbTokenizer` instance.
180
+ fn build(&self) -> Result<RbTokenizer, Error> {
181
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
182
+ let inner = self
183
+ .inner
184
+ .borrow()
185
+ .build()
186
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
187
+ Ok(RbTokenizer { inner })
188
+ }
189
+ }
190
+
191
+ /// Tokenizer for performing morphological analysis.
192
+ ///
193
+ /// The tokenizer processes text and returns tokens with their morphological features.
194
+ #[magnus::wrap(class = "Lindera::Tokenizer", free_immediately, size)]
195
+ pub struct RbTokenizer {
196
+ /// Inner binding-core tokenizer.
197
+ inner: CoreTokenizer,
198
+ }
199
+
200
+ /// Creates a new tokenizer with the given dictionary and mode.
201
+ ///
202
+ /// # Arguments
203
+ ///
204
+ /// * `dictionary` - Dictionary to use.
205
+ /// * `mode` - Tokenization mode ("normal" or "decompose"). Defaults to "normal".
206
+ /// * `user_dictionary` - Optional user dictionary.
207
+ ///
208
+ /// # Returns
209
+ ///
210
+ /// A new `RbTokenizer` instance.
211
+ fn tokenizer_new(
212
+ dictionary: &RbDictionary,
213
+ mode: Option<String>,
214
+ user_dictionary: Option<&RbUserDictionary>,
215
+ ) -> Result<RbTokenizer, Error> {
216
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
217
+ let mode_str = mode.as_deref().unwrap_or("normal");
218
+
219
+ let dict = dictionary.inner.clone();
220
+ let user_dict = user_dictionary.map(|d| d.inner.clone());
221
+
222
+ let inner = CoreTokenizer::from_segmenter(mode_str, dict, user_dict)
223
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
224
+
225
+ Ok(RbTokenizer { inner })
226
+ }
227
+
228
+ impl RbTokenizer {
229
+ /// Tokenizes the given text.
230
+ ///
231
+ /// # Arguments
232
+ ///
233
+ /// * `text` - Text to tokenize.
234
+ ///
235
+ /// # Returns
236
+ ///
237
+ /// An array of Token objects.
238
+ fn tokenize(&self, text: String) -> Result<RArray, Error> {
239
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
240
+ let views = self
241
+ .inner
242
+ .tokenize(&text)
243
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
244
+
245
+ let arr = ruby.ary_new_capa(views.len());
246
+ for view in views {
247
+ arr.push(ruby.into_value(RbToken::from_view(view)))?;
248
+ }
249
+ Ok(arr)
250
+ }
251
+
252
+ /// Tokenizes the given text and returns only the token surfaces.
253
+ ///
254
+ /// This is the fast path for wakati-style use: no Token objects are
255
+ /// created and no morphological details are loaded, so it is
256
+ /// significantly faster than `tokenize` when only the surface strings
257
+ /// are needed. The surfaces equal `tokenize(text).map(&:surface)`.
258
+ ///
259
+ /// # Arguments
260
+ ///
261
+ /// * `text` - Text to tokenize.
262
+ ///
263
+ /// # Returns
264
+ ///
265
+ /// An array of surface strings, in reading order.
266
+ fn tokenize_surfaces(&self, text: String) -> Result<RArray, Error> {
267
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
268
+ let views = self
269
+ .inner
270
+ .tokenize_surfaces(&text)
271
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
272
+
273
+ let arr = ruby.ary_new_capa(views.len());
274
+ for view in views {
275
+ arr.push(view.surface)?;
276
+ }
277
+ Ok(arr)
278
+ }
279
+
280
+ /// Tokenizes the given text and returns N-best results.
281
+ ///
282
+ /// # Arguments
283
+ ///
284
+ /// * `text` - Text to tokenize.
285
+ /// * `n` - Number of N-best results.
286
+ /// * `unique` - Whether to deduplicate results (default: false).
287
+ /// * `cost_threshold` - Optional cost threshold.
288
+ ///
289
+ /// # Returns
290
+ ///
291
+ /// An array of [tokens, cost] pairs.
292
+ fn tokenize_nbest(
293
+ &self,
294
+ text: String,
295
+ n: usize,
296
+ unique: Option<bool>,
297
+ cost_threshold: Option<i64>,
298
+ ) -> Result<RArray, Error> {
299
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
300
+ let results = self
301
+ .inner
302
+ .tokenize_nbest(&text, n, unique.unwrap_or(false), cost_threshold)
303
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
304
+
305
+ let rb_results = ruby.ary_new_capa(results.len());
306
+ for (views, cost) in results {
307
+ let token_arr = ruby.ary_new_capa(views.len());
308
+ for view in views {
309
+ token_arr.push(ruby.into_value(RbToken::from_view(view)))?;
310
+ }
311
+ let pair = ruby.ary_new_capa(2);
312
+ pair.push(token_arr)?;
313
+ pair.push(cost)?;
314
+ rb_results.push(pair)?;
315
+ }
316
+
317
+ Ok(rb_results)
318
+ }
319
+ }
320
+
321
+ /// Defines TokenizerBuilder and Tokenizer classes in the given Ruby module.
322
+ ///
323
+ /// # Arguments
324
+ ///
325
+ /// * `ruby` - Ruby runtime handle.
326
+ /// * `module` - Parent Ruby module.
327
+ ///
328
+ /// # Returns
329
+ ///
330
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
331
+ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
332
+ let builder_class = module.define_class("TokenizerBuilder", ruby.class_object())?;
333
+ builder_class.define_singleton_method("new", function!(RbTokenizerBuilder::new, 0))?;
334
+ builder_class
335
+ .define_singleton_method("from_file", function!(RbTokenizerBuilder::from_file, 1))?;
336
+ builder_class.define_method("set_mode", method!(RbTokenizerBuilder::set_mode, 1))?;
337
+ builder_class.define_method(
338
+ "set_dictionary",
339
+ method!(RbTokenizerBuilder::set_dictionary, 1),
340
+ )?;
341
+ builder_class.define_method(
342
+ "set_user_dictionary",
343
+ method!(RbTokenizerBuilder::set_user_dictionary, 1),
344
+ )?;
345
+ builder_class.define_method(
346
+ "set_keep_whitespace",
347
+ method!(RbTokenizerBuilder::set_keep_whitespace, 1),
348
+ )?;
349
+ builder_class.define_method(
350
+ "set_space_penalty",
351
+ method!(RbTokenizerBuilder::set_space_penalty, 1),
352
+ )?;
353
+ builder_class.define_method(
354
+ "append_character_filter",
355
+ method!(RbTokenizerBuilder::append_character_filter, 2),
356
+ )?;
357
+ builder_class.define_method(
358
+ "append_token_filter",
359
+ method!(RbTokenizerBuilder::append_token_filter, 2),
360
+ )?;
361
+ builder_class.define_method("build", method!(RbTokenizerBuilder::build, 0))?;
362
+
363
+ let tokenizer_class = module.define_class("Tokenizer", ruby.class_object())?;
364
+ tokenizer_class.define_singleton_method("new", function!(tokenizer_new, 3))?;
365
+ tokenizer_class.define_method("tokenize", method!(RbTokenizer::tokenize, 1))?;
366
+ tokenizer_class.define_method(
367
+ "tokenize_surfaces",
368
+ method!(RbTokenizer::tokenize_surfaces, 1),
369
+ )?;
370
+ tokenizer_class.define_method("tokenize_nbest", method!(RbTokenizer::tokenize_nbest, 4))?;
371
+
372
+ Ok(())
373
+ }