lindera 6.0.0 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/token.rs ADDED
@@ -0,0 +1,188 @@
1
+ //! Token representation for morphological analysis results.
2
+ //!
3
+ //! This module wraps Lindera tokens for use in Ruby.
4
+
5
+ use magnus::prelude::*;
6
+ use magnus::{Error, RHash, Ruby, method};
7
+
8
+ use lindera::token::Token;
9
+
10
+ /// Token object wrapping the Rust Token data.
11
+ ///
12
+ /// This class provides access to token fields and details.
13
+ #[magnus::wrap(class = "Lindera::Token", free_immediately, size)]
14
+ pub struct RbToken {
15
+ /// Surface form of the token.
16
+ surface: String,
17
+ /// Start byte position in the original text.
18
+ byte_start: usize,
19
+ /// End byte position in the original text.
20
+ byte_end: usize,
21
+ /// Position index of the token.
22
+ position: usize,
23
+ /// Word ID in the dictionary.
24
+ word_id: u32,
25
+ /// Whether this token is an unknown word.
26
+ is_unknown: bool,
27
+ /// Morphological details of the token.
28
+ details: Vec<String>,
29
+ }
30
+
31
+ impl RbToken {
32
+ /// Creates a new `RbToken` from a Lindera `Token`.
33
+ ///
34
+ /// # Arguments
35
+ ///
36
+ /// * `token` - Lindera token to convert.
37
+ ///
38
+ /// # Returns
39
+ ///
40
+ /// A new `RbToken` instance.
41
+ pub fn from_token(token: Token) -> Self {
42
+ Self::from_view(lindera_binding_core::TokenView::from_token(token))
43
+ }
44
+
45
+ /// Creates a new `RbToken` from a binding-core `TokenView`.
46
+ ///
47
+ /// # Arguments
48
+ ///
49
+ /// * `view` - Token view produced by the binding-core tokenizer.
50
+ ///
51
+ /// # Returns
52
+ ///
53
+ /// A new `RbToken` instance.
54
+ pub fn from_view(view: lindera_binding_core::TokenView) -> Self {
55
+ Self {
56
+ surface: view.surface,
57
+ byte_start: view.byte_start,
58
+ byte_end: view.byte_end,
59
+ position: view.position,
60
+ word_id: view.word_id,
61
+ is_unknown: view.is_unknown,
62
+ details: view.details,
63
+ }
64
+ }
65
+
66
+ /// Returns the surface form of the token.
67
+ fn surface(&self) -> String {
68
+ self.surface.clone()
69
+ }
70
+
71
+ /// Returns the start byte position.
72
+ fn byte_start(&self) -> usize {
73
+ self.byte_start
74
+ }
75
+
76
+ /// Returns the end byte position.
77
+ fn byte_end(&self) -> usize {
78
+ self.byte_end
79
+ }
80
+
81
+ /// Returns the position index.
82
+ fn position(&self) -> usize {
83
+ self.position
84
+ }
85
+
86
+ /// Returns the word ID.
87
+ fn word_id(&self) -> u32 {
88
+ self.word_id
89
+ }
90
+
91
+ /// Returns whether this token is an unknown word.
92
+ fn is_unknown(&self) -> bool {
93
+ self.is_unknown
94
+ }
95
+
96
+ /// Returns the morphological details of the token.
97
+ fn details(&self) -> Vec<String> {
98
+ self.details.clone()
99
+ }
100
+
101
+ /// Returns the detail at the specified index.
102
+ ///
103
+ /// # Arguments
104
+ ///
105
+ /// * `index` - Index of the detail to retrieve.
106
+ ///
107
+ /// # Returns
108
+ ///
109
+ /// The detail string if found, otherwise nil.
110
+ fn get_detail(&self, index: usize) -> Option<String> {
111
+ self.details.get(index).cloned()
112
+ }
113
+
114
+ /// Returns the token as a plain Hash keyed by Symbols.
115
+ ///
116
+ /// Values keep their natural Ruby types (Integer for the positions and
117
+ /// word id, true/false for `:is_unknown`, Array of String for
118
+ /// `:details`), so the result works directly with `JSON.generate`.
119
+ ///
120
+ /// # Returns
121
+ ///
122
+ /// A Hash with the keys `:surface`, `:byte_start`, `:byte_end`,
123
+ /// `:position`, `:word_id`, `:is_unknown`, and `:details`, or a Magnus
124
+ /// `Error` if the Hash cannot be built.
125
+ fn to_hash(ruby: &Ruby, rb_self: &Self) -> Result<RHash, Error> {
126
+ let this = rb_self;
127
+ let hash = ruby.hash_new();
128
+ hash.aset(ruby.to_symbol("surface"), this.surface.as_str())?;
129
+ hash.aset(ruby.to_symbol("byte_start"), this.byte_start)?;
130
+ hash.aset(ruby.to_symbol("byte_end"), this.byte_end)?;
131
+ hash.aset(ruby.to_symbol("position"), this.position)?;
132
+ hash.aset(ruby.to_symbol("word_id"), this.word_id)?;
133
+ hash.aset(ruby.to_symbol("is_unknown"), this.is_unknown)?;
134
+ let details = ruby.ary_new();
135
+ for detail in &this.details {
136
+ details.push(detail.as_str())?;
137
+ }
138
+ hash.aset(ruby.to_symbol("details"), details)?;
139
+ Ok(hash)
140
+ }
141
+
142
+ /// Returns the string representation of the token.
143
+ fn to_s(&self) -> String {
144
+ self.surface.clone()
145
+ }
146
+
147
+ /// Returns the inspect representation of the token.
148
+ fn inspect(&self) -> String {
149
+ format!(
150
+ "#<Lindera::Token surface='{}', start={}, end={}, position={}, word_id={}, unknown={}>",
151
+ self.surface,
152
+ self.byte_start,
153
+ self.byte_end,
154
+ self.position,
155
+ self.word_id,
156
+ self.is_unknown
157
+ )
158
+ }
159
+ }
160
+
161
+ /// Defines the Token class in the given Ruby module.
162
+ ///
163
+ /// # Arguments
164
+ ///
165
+ /// * `ruby` - Ruby runtime handle.
166
+ /// * `module` - Parent Ruby module.
167
+ ///
168
+ /// # Returns
169
+ ///
170
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
171
+ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
172
+ let token_class = module.define_class("Token", ruby.class_object())?;
173
+ token_class.define_method("surface", method!(RbToken::surface, 0))?;
174
+ token_class.define_method("byte_start", method!(RbToken::byte_start, 0))?;
175
+ token_class.define_method("byte_end", method!(RbToken::byte_end, 0))?;
176
+ token_class.define_method("position", method!(RbToken::position, 0))?;
177
+ token_class.define_method("word_id", method!(RbToken::word_id, 0))?;
178
+ token_class.define_method("unknown?", method!(RbToken::is_unknown, 0))?;
179
+ token_class.define_method("details", method!(RbToken::details, 0))?;
180
+ token_class.define_method("get_detail", method!(RbToken::get_detail, 1))?;
181
+ token_class.define_method("to_hash", method!(RbToken::to_hash, 0))?;
182
+ token_class.define_method("to_h", method!(RbToken::to_hash, 0))?;
183
+
184
+ token_class.define_method("to_s", method!(RbToken::to_s, 0))?;
185
+ token_class.define_method("inspect", method!(RbToken::inspect, 0))?;
186
+
187
+ Ok(())
188
+ }
@@ -0,0 +1,3 @@
1
+ //! Token filters for post-processing tokens.
2
+ //!
3
+ //! This module is a placeholder for future token filter implementations.
data/src/tokenizer.rs ADDED
@@ -0,0 +1,401 @@
1
+ //! Tokenizer implementation for morphological analysis.
2
+ //!
3
+ //! This module provides a builder pattern for creating tokenizers and the tokenizer itself.
4
+ //! The build-flow orchestration is delegated to
5
+ //! [`lindera_binding_core::CoreTokenizerBuilder`] / [`lindera_binding_core::CoreTokenizer`];
6
+ //! this module only adds the magnus wrappers and the Ruby-hash conversion.
7
+
8
+ use std::cell::RefCell;
9
+ use std::path::Path;
10
+
11
+ use magnus::prelude::*;
12
+ use magnus::scan_args::{get_kwargs, scan_args};
13
+ use magnus::{Error, RArray, RHash, Ruby, Value, function, method};
14
+
15
+ use lindera_binding_core::{CoreTokenizer, CoreTokenizerBuilder};
16
+
17
+ use crate::dictionary::{RbDictionary, RbUserDictionary};
18
+ use crate::error::to_magnus_error;
19
+ use crate::token::RbToken;
20
+ use crate::util::{rb_hash_to_json, rb_value_to_json};
21
+
22
+ /// Builder for creating a `Tokenizer` with custom configuration.
23
+ ///
24
+ /// The builder pattern allows for fluent configuration of tokenizer parameters.
25
+ /// Uses `RefCell` for interior mutability since Magnus `method!` requires `&self`.
26
+ #[magnus::wrap(class = "Lindera::TokenizerBuilder", free_immediately, size)]
27
+ pub struct RbTokenizerBuilder {
28
+ /// Inner binding-core tokenizer builder (wrapped in RefCell for interior mutability).
29
+ inner: RefCell<CoreTokenizerBuilder>,
30
+ }
31
+
32
+ impl RbTokenizerBuilder {
33
+ /// Creates a new `RbTokenizerBuilder` with default configuration.
34
+ ///
35
+ /// # Returns
36
+ ///
37
+ /// A new `RbTokenizerBuilder` instance.
38
+ fn new() -> Result<Self, Error> {
39
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
40
+ let inner =
41
+ CoreTokenizerBuilder::new().map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
42
+ Ok(Self {
43
+ inner: RefCell::new(inner),
44
+ })
45
+ }
46
+
47
+ /// Loads configuration from a file.
48
+ ///
49
+ /// # Arguments
50
+ ///
51
+ /// * `file_path` - Path to the configuration file.
52
+ ///
53
+ /// # Returns
54
+ ///
55
+ /// A new `RbTokenizerBuilder` with the loaded configuration.
56
+ fn from_file(file_path: String) -> Result<Self, Error> {
57
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
58
+ let inner = CoreTokenizerBuilder::from_file(Path::new(&file_path))
59
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
60
+ Ok(Self {
61
+ inner: RefCell::new(inner),
62
+ })
63
+ }
64
+
65
+ /// Sets the tokenization mode.
66
+ ///
67
+ /// # Arguments
68
+ ///
69
+ /// * `mode` - Mode string ("normal" or "decompose").
70
+ fn set_mode(&self, mode: String) -> Result<(), Error> {
71
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
72
+ let mut builder = self.inner.borrow_mut();
73
+ builder
74
+ .set_mode(&mode)
75
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
76
+ Ok(())
77
+ }
78
+
79
+ /// Sets the dictionary path.
80
+ ///
81
+ /// # Arguments
82
+ ///
83
+ /// * `path` - Path to the dictionary directory.
84
+ fn set_dictionary(&self, path: String) {
85
+ self.inner.borrow_mut().set_dictionary(&path);
86
+ }
87
+
88
+ /// Sets the user dictionary URI.
89
+ ///
90
+ /// # Arguments
91
+ ///
92
+ /// * `uri` - URI to the user dictionary.
93
+ fn set_user_dictionary(&self, uri: String) {
94
+ self.inner.borrow_mut().set_user_dictionary(&uri);
95
+ }
96
+
97
+ /// Sets whether to keep whitespace in tokenization results.
98
+ ///
99
+ /// # Arguments
100
+ ///
101
+ /// * `keep_whitespace` - If true, whitespace tokens will be included.
102
+ fn set_keep_whitespace(&self, keep_whitespace: bool) {
103
+ self.inner.borrow_mut().set_keep_whitespace(keep_whitespace);
104
+ }
105
+
106
+ /// Sets the left-space penalty (Korean), with the same meaning as
107
+ /// `segmenter.space_penalty` in a YAML config.
108
+ ///
109
+ /// `nil` restores the default (the rules the dictionary ships, if any;
110
+ /// ko-dic ships mecab-ko-dic's), `false` turns the penalty off, `true`
111
+ /// requires the dictionary's rules (`build` then fails for a dictionary
112
+ /// that ships none), and a hash such as
113
+ /// `{ 'rules' => [{ 'pos' => ['JKS'], 'cost' => 6000 }] }` applies those
114
+ /// rules instead.
115
+ ///
116
+ /// # Arguments
117
+ ///
118
+ /// * `ruby` - Ruby runtime handle.
119
+ /// * `rb_self` - The builder.
120
+ /// * `value` - `nil`, `true`, `false`, or a hash with string keys holding `"rules"`.
121
+ ///
122
+ /// # Returns
123
+ ///
124
+ /// `Ok(())` on success, a `TypeError` when `value` holds something that
125
+ /// cannot be converted to JSON (such as a Symbol or a non-finite Float),
126
+ /// or a `RuntimeError` when it converts but is not one of the forms
127
+ /// above.
128
+ fn set_space_penalty(ruby: &Ruby, rb_self: &Self, value: Value) -> Result<(), Error> {
129
+ let setting = rb_value_to_json(ruby, value)?;
130
+ rb_self
131
+ .inner
132
+ .borrow_mut()
133
+ .set_space_penalty(&setting)
134
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
135
+ Ok(())
136
+ }
137
+
138
+ /// Appends a character filter to the filter pipeline.
139
+ ///
140
+ /// # Arguments
141
+ ///
142
+ /// * `kind` - Type of character filter to add.
143
+ /// * `args` - Optional hash of filter arguments.
144
+ fn append_character_filter(&self, kind: String, args: Option<RHash>) -> Result<(), Error> {
145
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
146
+ let filter_args = if let Some(hash) = args {
147
+ rb_hash_to_json(&ruby, hash)?
148
+ } else {
149
+ serde_json::Value::Object(serde_json::Map::new())
150
+ };
151
+ self.inner
152
+ .borrow_mut()
153
+ .append_character_filter(&kind, &filter_args);
154
+ Ok(())
155
+ }
156
+
157
+ /// Appends a token filter to the filter pipeline.
158
+ ///
159
+ /// # Arguments
160
+ ///
161
+ /// * `kind` - Type of token filter to add.
162
+ /// * `args` - Optional hash of filter arguments.
163
+ fn append_token_filter(&self, kind: String, args: Option<RHash>) -> Result<(), Error> {
164
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
165
+ let filter_args = if let Some(hash) = args {
166
+ rb_hash_to_json(&ruby, hash)?
167
+ } else {
168
+ serde_json::Value::Object(serde_json::Map::new())
169
+ };
170
+ self.inner
171
+ .borrow_mut()
172
+ .append_token_filter(&kind, &filter_args);
173
+ Ok(())
174
+ }
175
+
176
+ /// Builds the tokenizer with the configured settings.
177
+ ///
178
+ /// # Returns
179
+ ///
180
+ /// A configured `RbTokenizer` instance.
181
+ fn build(&self) -> Result<RbTokenizer, Error> {
182
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
183
+ let inner = self
184
+ .inner
185
+ .borrow()
186
+ .build()
187
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
188
+ Ok(RbTokenizer { inner })
189
+ }
190
+ }
191
+
192
+ /// Tokenizer for performing morphological analysis.
193
+ ///
194
+ /// The tokenizer processes text and returns tokens with their morphological features.
195
+ #[magnus::wrap(class = "Lindera::Tokenizer", free_immediately, size)]
196
+ pub struct RbTokenizer {
197
+ /// Inner binding-core tokenizer.
198
+ inner: CoreTokenizer,
199
+ }
200
+
201
+ /// Creates a new tokenizer with the given dictionary and mode.
202
+ ///
203
+ /// Ruby signature: `Tokenizer.new(dictionary, mode = nil, user_dictionary = nil,
204
+ /// space_penalty: nil)`.
205
+ ///
206
+ /// # Arguments
207
+ ///
208
+ /// * `args` - The Ruby arguments:
209
+ /// * `dictionary` - Dictionary to use.
210
+ /// * `mode` - Tokenization mode ("normal" or "decompose"). `nil` or
211
+ /// omitted means "normal".
212
+ /// * `user_dictionary` - Optional user dictionary.
213
+ /// * `space_penalty:` - The left-space penalty, in the forms
214
+ /// `TokenizerBuilder#set_space_penalty` accepts. `nil` or omitted keeps
215
+ /// the rules the dictionary ships.
216
+ ///
217
+ /// # Returns
218
+ ///
219
+ /// A new `RbTokenizer` instance, an `ArgumentError` for a wrong number of
220
+ /// arguments or an unknown keyword, or the errors `set_space_penalty` and
221
+ /// `build` raise for an invalid space-penalty setting.
222
+ fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
223
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
224
+ let args = scan_args::<
225
+ (&RbDictionary,),
226
+ (Option<Option<String>>, Option<Option<&RbUserDictionary>>),
227
+ (),
228
+ (),
229
+ RHash,
230
+ (),
231
+ >(args)?;
232
+ let (dictionary,) = args.required;
233
+ let (mode, user_dictionary) = args.optional;
234
+ let keywords =
235
+ get_kwargs::<_, (), (Option<Value>,), ()>(args.keywords, &[], &["space_penalty"])?;
236
+ let (space_penalty,) = keywords.optional;
237
+
238
+ let mode = mode.flatten();
239
+ let user_dictionary = user_dictionary.flatten();
240
+ let mode_str = mode.as_deref().unwrap_or("normal");
241
+ let space_penalty = match space_penalty {
242
+ Some(value) => rb_value_to_json(&ruby, value)?,
243
+ None => serde_json::Value::Null,
244
+ };
245
+
246
+ let dict = dictionary.inner.clone();
247
+ let user_dict = user_dictionary.map(|d| d.inner.clone());
248
+
249
+ let inner =
250
+ CoreTokenizer::from_segmenter_with_space_penalty(mode_str, dict, user_dict, &space_penalty)
251
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
252
+
253
+ Ok(RbTokenizer { inner })
254
+ }
255
+
256
+ impl RbTokenizer {
257
+ /// Tokenizes the given text.
258
+ ///
259
+ /// # Arguments
260
+ ///
261
+ /// * `text` - Text to tokenize.
262
+ ///
263
+ /// # Returns
264
+ ///
265
+ /// An array of Token objects.
266
+ fn tokenize(&self, text: String) -> Result<RArray, Error> {
267
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
268
+ let views = self
269
+ .inner
270
+ .tokenize(&text)
271
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
272
+
273
+ let arr = ruby.ary_new_capa(views.len());
274
+ for view in views {
275
+ arr.push(ruby.into_value(RbToken::from_view(view)))?;
276
+ }
277
+ Ok(arr)
278
+ }
279
+
280
+ /// Tokenizes the given text and returns only the token surfaces.
281
+ ///
282
+ /// This is the fast path for wakati-style use: no Token objects are
283
+ /// created and no morphological details are loaded, so it is
284
+ /// significantly faster than `tokenize` when only the surface strings
285
+ /// are needed. The surfaces equal `tokenize(text).map(&:surface)`.
286
+ ///
287
+ /// # Arguments
288
+ ///
289
+ /// * `text` - Text to tokenize.
290
+ ///
291
+ /// # Returns
292
+ ///
293
+ /// An array of surface strings, in reading order.
294
+ fn tokenize_surfaces(&self, text: String) -> Result<RArray, Error> {
295
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
296
+ let views = self
297
+ .inner
298
+ .tokenize_surfaces(&text)
299
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
300
+
301
+ let arr = ruby.ary_new_capa(views.len());
302
+ for view in views {
303
+ arr.push(view.surface)?;
304
+ }
305
+ Ok(arr)
306
+ }
307
+
308
+ /// Tokenizes the given text and returns N-best results.
309
+ ///
310
+ /// # Arguments
311
+ ///
312
+ /// * `text` - Text to tokenize.
313
+ /// * `n` - Number of N-best results.
314
+ /// * `unique` - Whether to deduplicate results (default: false).
315
+ /// * `cost_threshold` - Optional cost threshold.
316
+ ///
317
+ /// # Returns
318
+ ///
319
+ /// An array of [tokens, cost] pairs.
320
+ fn tokenize_nbest(
321
+ &self,
322
+ text: String,
323
+ n: usize,
324
+ unique: Option<bool>,
325
+ cost_threshold: Option<i64>,
326
+ ) -> Result<RArray, Error> {
327
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
328
+ let results = self
329
+ .inner
330
+ .tokenize_nbest(&text, n, unique.unwrap_or(false), cost_threshold)
331
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
332
+
333
+ let rb_results = ruby.ary_new_capa(results.len());
334
+ for (views, cost) in results {
335
+ let token_arr = ruby.ary_new_capa(views.len());
336
+ for view in views {
337
+ token_arr.push(ruby.into_value(RbToken::from_view(view)))?;
338
+ }
339
+ let pair = ruby.ary_new_capa(2);
340
+ pair.push(token_arr)?;
341
+ pair.push(cost)?;
342
+ rb_results.push(pair)?;
343
+ }
344
+
345
+ Ok(rb_results)
346
+ }
347
+ }
348
+
349
+ /// Defines TokenizerBuilder and Tokenizer classes in the given Ruby module.
350
+ ///
351
+ /// # Arguments
352
+ ///
353
+ /// * `ruby` - Ruby runtime handle.
354
+ /// * `module` - Parent Ruby module.
355
+ ///
356
+ /// # Returns
357
+ ///
358
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
359
+ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
360
+ let builder_class = module.define_class("TokenizerBuilder", ruby.class_object())?;
361
+ builder_class.define_singleton_method("new", function!(RbTokenizerBuilder::new, 0))?;
362
+ builder_class
363
+ .define_singleton_method("from_file", function!(RbTokenizerBuilder::from_file, 1))?;
364
+ builder_class.define_method("set_mode", method!(RbTokenizerBuilder::set_mode, 1))?;
365
+ builder_class.define_method(
366
+ "set_dictionary",
367
+ method!(RbTokenizerBuilder::set_dictionary, 1),
368
+ )?;
369
+ builder_class.define_method(
370
+ "set_user_dictionary",
371
+ method!(RbTokenizerBuilder::set_user_dictionary, 1),
372
+ )?;
373
+ builder_class.define_method(
374
+ "set_keep_whitespace",
375
+ method!(RbTokenizerBuilder::set_keep_whitespace, 1),
376
+ )?;
377
+ builder_class.define_method(
378
+ "set_space_penalty",
379
+ method!(RbTokenizerBuilder::set_space_penalty, 1),
380
+ )?;
381
+ builder_class.define_method(
382
+ "append_character_filter",
383
+ method!(RbTokenizerBuilder::append_character_filter, 2),
384
+ )?;
385
+ builder_class.define_method(
386
+ "append_token_filter",
387
+ method!(RbTokenizerBuilder::append_token_filter, 2),
388
+ )?;
389
+ builder_class.define_method("build", method!(RbTokenizerBuilder::build, 0))?;
390
+
391
+ let tokenizer_class = module.define_class("Tokenizer", ruby.class_object())?;
392
+ tokenizer_class.define_singleton_method("new", function!(tokenizer_new, -1))?;
393
+ tokenizer_class.define_method("tokenize", method!(RbTokenizer::tokenize, 1))?;
394
+ tokenizer_class.define_method(
395
+ "tokenize_surfaces",
396
+ method!(RbTokenizer::tokenize_surfaces, 1),
397
+ )?;
398
+ tokenizer_class.define_method("tokenize_nbest", method!(RbTokenizer::tokenize_nbest, 4))?;
399
+
400
+ Ok(())
401
+ }