lindera 6.1.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/tokenizer.rs CHANGED
@@ -2,21 +2,25 @@
2
2
  //!
3
3
  //! This module provides a builder pattern for creating tokenizers and the tokenizer itself.
4
4
  //! The build-flow orchestration is delegated to
5
- //! [`lindera_binding_core::CoreTokenizerBuilder`] / [`lindera_binding_core::CoreTokenizer`];
5
+ //! [`lindera_binding::CoreTokenizerBuilder`] / [`lindera_binding::CoreTokenizer`];
6
6
  //! this module only adds the magnus wrappers and the Ruby-hash conversion.
7
7
 
8
8
  use std::cell::RefCell;
9
+ use std::num::NonZeroUsize;
9
10
  use std::path::Path;
10
11
 
11
12
  use magnus::prelude::*;
12
- use magnus::{Error, RArray, RHash, Ruby, Value, function, method};
13
+ use magnus::scan_args::{get_kwargs, scan_args};
14
+ use magnus::{Error, Integer, RArray, RHash, Ruby, Value, function, method};
13
15
 
14
- use lindera_binding_core::{CoreTokenizer, CoreTokenizerBuilder};
16
+ use lindera_binding::argument::Positive;
17
+ use lindera_binding::{CoreTokenizer, CoreTokenizerBuilder, TokenView};
15
18
 
16
19
  use crate::dictionary::{RbDictionary, RbUserDictionary};
17
20
  use crate::error::to_magnus_error;
21
+ use crate::mode::RbMode;
18
22
  use crate::token::RbToken;
19
- use crate::util::{rb_hash_to_json, rb_value_to_json};
23
+ use crate::util::{integer_arg, optional_integer_arg, rb_hash_to_json, rb_value_to_json};
20
24
 
21
25
  /// Builder for creating a `Tokenizer` with custom configuration.
22
26
  ///
@@ -24,7 +28,7 @@ use crate::util::{rb_hash_to_json, rb_value_to_json};
24
28
  /// Uses `RefCell` for interior mutability since Magnus `method!` requires `&self`.
25
29
  #[magnus::wrap(class = "Lindera::TokenizerBuilder", free_immediately, size)]
26
30
  pub struct RbTokenizerBuilder {
27
- /// Inner binding-core tokenizer builder (wrapped in RefCell for interior mutability).
31
+ /// Inner lindera-binding tokenizer builder (wrapped in RefCell for interior mutability).
28
32
  inner: RefCell<CoreTokenizerBuilder>,
29
33
  }
30
34
 
@@ -63,16 +67,14 @@ impl RbTokenizerBuilder {
63
67
 
64
68
  /// Sets the tokenization mode.
65
69
  ///
70
+ /// Ruby signature: `set_mode(mode)`. magnus raises a `TypeError` for a
71
+ /// `mode` that is not a `Lindera::Mode`.
72
+ ///
66
73
  /// # Arguments
67
74
  ///
68
- /// * `mode` - Mode string ("normal" or "decompose").
69
- fn set_mode(&self, mode: String) -> Result<(), Error> {
70
- let ruby = Ruby::get().expect("Ruby runtime not initialized");
71
- let mut builder = self.inner.borrow_mut();
72
- builder
73
- .set_mode(&mode)
74
- .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
75
- Ok(())
75
+ /// * `mode` - The `Lindera::Mode`, with its penalty for decompose mode.
76
+ fn set_mode(&self, mode: &RbMode) {
77
+ self.inner.borrow_mut().set_mode(mode.as_lindera());
76
78
  }
77
79
 
78
80
  /// Sets the dictionary path.
@@ -95,6 +97,10 @@ impl RbTokenizerBuilder {
95
97
 
96
98
  /// Sets whether to keep whitespace in tokenization results.
97
99
  ///
100
+ /// Kept whitespace also stays in the lattice, so the other tokens can
101
+ /// differ from those of the default output, which skips whitespace as
102
+ /// MeCab does.
103
+ ///
98
104
  /// # Arguments
99
105
  ///
100
106
  /// * `keep_whitespace` - If true, whitespace tokens will be included.
@@ -102,6 +108,44 @@ impl RbTokenizerBuilder {
102
108
  self.inner.borrow_mut().set_keep_whitespace(keep_whitespace);
103
109
  }
104
110
 
111
+ /// Sets whether whitespace is skipped in the lattice, with the same
112
+ /// meaning as `segmenter.skip_whitespace` in a YAML config.
113
+ ///
114
+ /// `true` skips whitespace, so the words on either side connect directly,
115
+ /// as MeCab does; `false` keeps whitespace in the lattice as a node of
116
+ /// its own, as before v7 (it is still dropped from the output); `nil`
117
+ /// goes back to the default of the dictionary's `metadata.json`
118
+ /// (skipping, except with SudachiDict). No effect while whitespace tokens
119
+ /// are kept (`set_keep_whitespace(true)`).
120
+ ///
121
+ /// # Arguments
122
+ ///
123
+ /// * `skip_whitespace` - The setting, or `nil` for the dictionary's
124
+ /// default. Any other value counts by Ruby truthiness, as for
125
+ /// `set_keep_whitespace`.
126
+ fn set_skip_whitespace(&self, skip_whitespace: Option<bool>) {
127
+ self.inner.borrow_mut().set_skip_whitespace(skip_whitespace);
128
+ }
129
+
130
+ /// Sets the break characters, with the same meaning as
131
+ /// `segmenter.break_characters` in a YAML config.
132
+ ///
133
+ /// Each character of the string is a break character: no unknown word,
134
+ /// nor, in decompose mode, any system-dictionary word, crosses a
135
+ /// boundary between a break character and another character. An empty
136
+ /// string turns the rule off, which gives MeCab's segmentation; `nil`
137
+ /// goes back to the default, `・`.
138
+ ///
139
+ /// # Arguments
140
+ ///
141
+ /// * `break_characters` - The break characters, or `nil` for the
142
+ /// default. A value that is not a string raises a `TypeError`.
143
+ fn set_break_characters(&self, break_characters: Option<String>) {
144
+ self.inner
145
+ .borrow_mut()
146
+ .set_break_characters(break_characters.as_deref());
147
+ }
148
+
105
149
  /// Sets the left-space penalty (Korean), with the same meaning as
106
150
  /// `segmenter.space_penalty` in a YAML config.
107
151
  ///
@@ -193,34 +237,64 @@ impl RbTokenizerBuilder {
193
237
  /// The tokenizer processes text and returns tokens with their morphological features.
194
238
  #[magnus::wrap(class = "Lindera::Tokenizer", free_immediately, size)]
195
239
  pub struct RbTokenizer {
196
- /// Inner binding-core tokenizer.
240
+ /// Inner lindera-binding tokenizer.
197
241
  inner: CoreTokenizer,
198
242
  }
199
243
 
200
244
  /// Creates a new tokenizer with the given dictionary and mode.
201
245
  ///
246
+ /// Ruby signature: `Tokenizer.new(dictionary, mode = nil, user_dictionary = nil,
247
+ /// space_penalty: nil)`.
248
+ ///
202
249
  /// # Arguments
203
250
  ///
204
- /// * `dictionary` - Dictionary to use.
205
- /// * `mode` - Tokenization mode ("normal" or "decompose"). Defaults to "normal".
206
- /// * `user_dictionary` - Optional user dictionary.
251
+ /// * `args` - The Ruby arguments:
252
+ /// * `dictionary` - Dictionary to use.
253
+ /// * `mode` - The `Lindera::Mode`, as `TokenizerBuilder#set_mode` takes
254
+ /// it. `nil` or omitted means normal mode.
255
+ /// * `user_dictionary` - Optional user dictionary.
256
+ /// * `space_penalty:` - The left-space penalty, in the forms
257
+ /// `TokenizerBuilder#set_space_penalty` accepts. `nil` or omitted keeps
258
+ /// the rules the dictionary ships.
207
259
  ///
208
260
  /// # Returns
209
261
  ///
210
- /// A new `RbTokenizer` instance.
211
- fn tokenizer_new(
212
- dictionary: &RbDictionary,
213
- mode: Option<String>,
214
- user_dictionary: Option<&RbUserDictionary>,
215
- ) -> Result<RbTokenizer, Error> {
262
+ /// A new `RbTokenizer` instance, an `ArgumentError` for a wrong number of
263
+ /// arguments or an unknown keyword, a `TypeError` for a `mode` that is not a
264
+ /// `Lindera::Mode`, the errors `set_space_penalty` and `build` raise for an
265
+ /// invalid space-penalty setting, or a `RuntimeError` for a user dictionary
266
+ /// with a context ID outside the dictionary's connection matrix.
267
+ fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
216
268
  let ruby = Ruby::get().expect("Ruby runtime not initialized");
217
- let mode_str = mode.as_deref().unwrap_or("normal");
269
+ let args = scan_args::<
270
+ (&RbDictionary,),
271
+ (Option<Option<&RbMode>>, Option<Option<&RbUserDictionary>>),
272
+ (),
273
+ (),
274
+ RHash,
275
+ (),
276
+ >(args)?;
277
+ let (dictionary,) = args.required;
278
+ let (mode, user_dictionary) = args.optional;
279
+ let keywords =
280
+ get_kwargs::<_, (), (Option<Value>,), ()>(args.keywords, &[], &["space_penalty"])?;
281
+ let (space_penalty,) = keywords.optional;
282
+
283
+ let mode = mode.flatten().map_or(lindera::mode::Mode::Normal, |mode| {
284
+ mode.as_lindera().clone()
285
+ });
286
+ let user_dictionary = user_dictionary.flatten();
287
+ let space_penalty = match space_penalty {
288
+ Some(value) => rb_value_to_json(&ruby, value)?,
289
+ None => serde_json::Value::Null,
290
+ };
218
291
 
219
292
  let dict = dictionary.inner.clone();
220
293
  let user_dict = user_dictionary.map(|d| d.inner.clone());
221
294
 
222
- let inner = CoreTokenizer::from_segmenter(mode_str, dict, user_dict)
223
- .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
295
+ let inner =
296
+ CoreTokenizer::from_segmenter_with_space_penalty(mode, dict, user_dict, &space_penalty)
297
+ .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
224
298
 
225
299
  Ok(RbTokenizer { inner })
226
300
  }
@@ -281,26 +355,31 @@ impl RbTokenizer {
281
355
  ///
282
356
  /// # Arguments
283
357
  ///
358
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
359
+ /// * `rb_self` - The tokenizer.
284
360
  /// * `text` - Text to tokenize.
285
- /// * `n` - Number of N-best results.
361
+ /// * `n` - Number of N-best results, 0 or more.
286
362
  /// * `unique` - Whether to deduplicate results (default: false).
287
- /// * `cost_threshold` - Optional cost threshold.
363
+ /// * `cost_threshold` - Optional cost threshold, a 64-bit integer.
288
364
  ///
289
365
  /// # Returns
290
366
  ///
291
- /// An array of [tokens, cost] pairs.
367
+ /// An array of [tokens, cost] pairs, or a `RangeError` for an `n` or a
368
+ /// `cost_threshold` outside its range.
292
369
  fn tokenize_nbest(
293
- &self,
370
+ ruby: &Ruby,
371
+ rb_self: &Self,
294
372
  text: String,
295
- n: usize,
373
+ n: Integer,
296
374
  unique: Option<bool>,
297
- cost_threshold: Option<i64>,
375
+ cost_threshold: Option<Integer>,
298
376
  ) -> Result<RArray, Error> {
299
- let ruby = Ruby::get().expect("Ruby runtime not initialized");
300
- let results = self
377
+ let n = integer_arg(ruby, "n", n)?;
378
+ let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
379
+ let results = rb_self
301
380
  .inner
302
381
  .tokenize_nbest(&text, n, unique.unwrap_or(false), cost_threshold)
303
- .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
382
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
304
383
 
305
384
  let rb_results = ruby.ary_new_capa(results.len());
306
385
  for (views, cost) in results {
@@ -316,6 +395,177 @@ impl RbTokenizer {
316
395
 
317
396
  Ok(rb_results)
318
397
  }
398
+
399
+ /// Tokenizes every text of `texts`, each as `tokenize` does, on up to
400
+ /// `max_threads` threads, and returns their tokens in the order of
401
+ /// `texts`: `tokenize_batch(texts, max_threads = nil)`.
402
+ ///
403
+ /// The tokenization runs on several threads while the call holds the
404
+ /// GVL, so other Ruby threads wait for it. The results are the same for
405
+ /// every number of threads.
406
+ ///
407
+ /// # Arguments
408
+ ///
409
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
410
+ /// * `rb_self` - The tokenizer.
411
+ /// * `args` - `texts`, an array of strings, and `max_threads`, an
412
+ /// integer of 1 or more, or `nil` (the default) for the number of
413
+ /// CPUs.
414
+ ///
415
+ /// # Returns
416
+ ///
417
+ /// An array with the Token objects of each text, an `ArgumentError` for
418
+ /// a wrong number of arguments, a `TypeError` for `texts` that is not an
419
+ /// array of strings, a `RangeError` for a `max_threads` outside its
420
+ /// range, or a `RuntimeError` for the first text, in the order of
421
+ /// `texts`, whose tokenization fails.
422
+ fn tokenize_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
423
+ let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
424
+ let (texts,) = args.required;
425
+ let (max_threads,) = args.optional;
426
+ let max_threads = batch_threads(ruby, max_threads.flatten())?;
427
+ let batch = rb_self
428
+ .inner
429
+ .tokenize_batch(&texts, max_threads)
430
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
431
+
432
+ let rb_batch = ruby.ary_new_capa(batch.len());
433
+ for views in batch {
434
+ rb_batch.push(token_array(ruby, views)?)?;
435
+ }
436
+ Ok(rb_batch)
437
+ }
438
+
439
+ /// Tokenizes every text of `texts` as `tokenize_surfaces` does, on up to
440
+ /// `max_threads` threads, as `tokenize_batch` does:
441
+ /// `tokenize_surfaces_batch(texts, max_threads = nil)`.
442
+ ///
443
+ /// # Arguments
444
+ ///
445
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
446
+ /// * `rb_self` - The tokenizer.
447
+ /// * `args` - `texts` and `max_threads`, as `tokenize_batch` takes them.
448
+ ///
449
+ /// # Returns
450
+ ///
451
+ /// An array with the surface strings of each text, or the errors
452
+ /// `tokenize_batch` raises.
453
+ fn tokenize_surfaces_batch(
454
+ ruby: &Ruby,
455
+ rb_self: &Self,
456
+ args: &[Value],
457
+ ) -> Result<RArray, Error> {
458
+ let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
459
+ let (texts,) = args.required;
460
+ let (max_threads,) = args.optional;
461
+ let max_threads = batch_threads(ruby, max_threads.flatten())?;
462
+ let batch = rb_self
463
+ .inner
464
+ .tokenize_surfaces_batch(&texts, max_threads)
465
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
466
+
467
+ let rb_batch = ruby.ary_new_capa(batch.len());
468
+ for views in batch {
469
+ let surfaces = ruby.ary_new_capa(views.len());
470
+ for view in views {
471
+ surfaces.push(view.surface)?;
472
+ }
473
+ rb_batch.push(surfaces)?;
474
+ }
475
+ Ok(rb_batch)
476
+ }
477
+
478
+ /// Tokenizes every text of `texts` as `tokenize_nbest` does, on up to
479
+ /// `max_threads` threads, as `tokenize_batch` does:
480
+ /// `tokenize_nbest_batch(texts, n, unique, cost_threshold, max_threads = nil)`.
481
+ ///
482
+ /// # Arguments
483
+ ///
484
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
485
+ /// * `rb_self` - The tokenizer.
486
+ /// * `args` - `texts`; `n`, `unique` and `cost_threshold`, as
487
+ /// `tokenize_nbest` takes them; and `max_threads`, as
488
+ /// `tokenize_batch` takes it.
489
+ ///
490
+ /// # Returns
491
+ ///
492
+ /// An array with the [tokens, cost] pairs of each text, a `RangeError`
493
+ /// for an `n`, a `cost_threshold` or a `max_threads` outside its range,
494
+ /// or the other errors `tokenize_batch` raises.
495
+ fn tokenize_nbest_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
496
+ let args = scan_args::<
497
+ (Vec<String>, Integer, Option<bool>, Option<Integer>),
498
+ (Option<Option<Integer>>,),
499
+ (),
500
+ (),
501
+ (),
502
+ (),
503
+ >(args)?;
504
+ let (texts, n, unique, cost_threshold) = args.required;
505
+ let (max_threads,) = args.optional;
506
+ let n = integer_arg(ruby, "n", n)?;
507
+ let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
508
+ let max_threads = batch_threads(ruby, max_threads.flatten())?;
509
+ let batch = rb_self
510
+ .inner
511
+ .tokenize_nbest_batch(
512
+ &texts,
513
+ n,
514
+ unique.unwrap_or(false),
515
+ cost_threshold,
516
+ max_threads,
517
+ )
518
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
519
+
520
+ let rb_batch = ruby.ary_new_capa(batch.len());
521
+ for results in batch {
522
+ let rb_results = ruby.ary_new_capa(results.len());
523
+ for (views, cost) in results {
524
+ let pair = ruby.ary_new_capa(2);
525
+ pair.push(token_array(ruby, views)?)?;
526
+ pair.push(cost)?;
527
+ rb_results.push(pair)?;
528
+ }
529
+ rb_batch.push(rb_results)?;
530
+ }
531
+ Ok(rb_batch)
532
+ }
533
+ }
534
+
535
+ /// Makes the Token objects of one text into an array.
536
+ ///
537
+ /// # Arguments
538
+ ///
539
+ /// * `ruby` - Ruby runtime handle.
540
+ /// * `views` - The tokens of the text.
541
+ ///
542
+ /// # Returns
543
+ ///
544
+ /// The array of Token objects.
545
+ fn token_array(ruby: &Ruby, views: Vec<TokenView>) -> Result<RArray, Error> {
546
+ let arr = ruby.ary_new_capa(views.len());
547
+ for view in views {
548
+ arr.push(ruby.into_value(RbToken::from_view(view)))?;
549
+ }
550
+ Ok(arr)
551
+ }
552
+
553
+ /// Converts the `max_threads` argument of the batch methods, as `train`
554
+ /// checks its own.
555
+ ///
556
+ /// # Arguments
557
+ ///
558
+ /// * `ruby` - Ruby runtime handle.
559
+ /// * `max_threads` - The argument, `None` for `nil` or omitted.
560
+ ///
561
+ /// # Returns
562
+ ///
563
+ /// The thread count, `None` for the default, or a `RangeError` when it is
564
+ /// not an integer of 1 or more.
565
+ fn batch_threads(ruby: &Ruby, max_threads: Option<Integer>) -> Result<Option<NonZeroUsize>, Error> {
566
+ let max_threads: Option<Positive<usize>> =
567
+ optional_integer_arg(ruby, "max_threads", max_threads)?;
568
+ Ok(max_threads.and_then(|Positive(n)| NonZeroUsize::new(n)))
319
569
  }
320
570
 
321
571
  /// Defines TokenizerBuilder and Tokenizer classes in the given Ruby module.
@@ -346,6 +596,14 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
346
596
  "set_keep_whitespace",
347
597
  method!(RbTokenizerBuilder::set_keep_whitespace, 1),
348
598
  )?;
599
+ builder_class.define_method(
600
+ "set_skip_whitespace",
601
+ method!(RbTokenizerBuilder::set_skip_whitespace, 1),
602
+ )?;
603
+ builder_class.define_method(
604
+ "set_break_characters",
605
+ method!(RbTokenizerBuilder::set_break_characters, 1),
606
+ )?;
349
607
  builder_class.define_method(
350
608
  "set_space_penalty",
351
609
  method!(RbTokenizerBuilder::set_space_penalty, 1),
@@ -361,13 +619,22 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
361
619
  builder_class.define_method("build", method!(RbTokenizerBuilder::build, 0))?;
362
620
 
363
621
  let tokenizer_class = module.define_class("Tokenizer", ruby.class_object())?;
364
- tokenizer_class.define_singleton_method("new", function!(tokenizer_new, 3))?;
622
+ tokenizer_class.define_singleton_method("new", function!(tokenizer_new, -1))?;
365
623
  tokenizer_class.define_method("tokenize", method!(RbTokenizer::tokenize, 1))?;
366
624
  tokenizer_class.define_method(
367
625
  "tokenize_surfaces",
368
626
  method!(RbTokenizer::tokenize_surfaces, 1),
369
627
  )?;
370
628
  tokenizer_class.define_method("tokenize_nbest", method!(RbTokenizer::tokenize_nbest, 4))?;
629
+ tokenizer_class.define_method("tokenize_batch", method!(RbTokenizer::tokenize_batch, -1))?;
630
+ tokenizer_class.define_method(
631
+ "tokenize_surfaces_batch",
632
+ method!(RbTokenizer::tokenize_surfaces_batch, -1),
633
+ )?;
634
+ tokenizer_class.define_method(
635
+ "tokenize_nbest_batch",
636
+ method!(RbTokenizer::tokenize_nbest_batch, -1),
637
+ )?;
371
638
 
372
639
  Ok(())
373
640
  }