lindera 6.2.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/tokenizer.rs CHANGED
@@ -2,22 +2,25 @@
2
2
  //!
3
3
  //! This module provides a builder pattern for creating tokenizers and the tokenizer itself.
4
4
  //! The build-flow orchestration is delegated to
5
- //! [`lindera_binding_core::CoreTokenizerBuilder`] / [`lindera_binding_core::CoreTokenizer`];
5
+ //! [`lindera_binding::CoreTokenizerBuilder`] / [`lindera_binding::CoreTokenizer`];
6
6
  //! this module only adds the magnus wrappers and the Ruby-hash conversion.
7
7
 
8
8
  use std::cell::RefCell;
9
+ use std::num::NonZeroUsize;
9
10
  use std::path::Path;
10
11
 
11
12
  use magnus::prelude::*;
12
13
  use magnus::scan_args::{get_kwargs, scan_args};
13
- use magnus::{Error, RArray, RHash, Ruby, Value, function, method};
14
+ use magnus::{Error, Integer, RArray, RHash, Ruby, Value, function, method};
14
15
 
15
- use lindera_binding_core::{CoreTokenizer, CoreTokenizerBuilder};
16
+ use lindera_binding::argument::Positive;
17
+ use lindera_binding::{CoreTokenizer, CoreTokenizerBuilder, TokenView};
16
18
 
17
19
  use crate::dictionary::{RbDictionary, RbUserDictionary};
18
20
  use crate::error::to_magnus_error;
21
+ use crate::mode::RbMode;
19
22
  use crate::token::RbToken;
20
- use crate::util::{rb_hash_to_json, rb_value_to_json};
23
+ use crate::util::{integer_arg, optional_integer_arg, rb_hash_to_json, rb_value_to_json};
21
24
 
22
25
  /// Builder for creating a `Tokenizer` with custom configuration.
23
26
  ///
@@ -25,7 +28,7 @@ use crate::util::{rb_hash_to_json, rb_value_to_json};
25
28
  /// Uses `RefCell` for interior mutability since Magnus `method!` requires `&self`.
26
29
  #[magnus::wrap(class = "Lindera::TokenizerBuilder", free_immediately, size)]
27
30
  pub struct RbTokenizerBuilder {
28
- /// Inner binding-core tokenizer builder (wrapped in RefCell for interior mutability).
31
+ /// Inner lindera-binding tokenizer builder (wrapped in RefCell for interior mutability).
29
32
  inner: RefCell<CoreTokenizerBuilder>,
30
33
  }
31
34
 
@@ -64,16 +67,14 @@ impl RbTokenizerBuilder {
64
67
 
65
68
  /// Sets the tokenization mode.
66
69
  ///
70
+ /// Ruby signature: `set_mode(mode)`. magnus raises a `TypeError` for a
71
+ /// `mode` that is not a `Lindera::Mode`.
72
+ ///
67
73
  /// # Arguments
68
74
  ///
69
- /// * `mode` - Mode string ("normal" or "decompose").
70
- fn set_mode(&self, mode: String) -> Result<(), Error> {
71
- let ruby = Ruby::get().expect("Ruby runtime not initialized");
72
- let mut builder = self.inner.borrow_mut();
73
- builder
74
- .set_mode(&mode)
75
- .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
76
- Ok(())
75
+ /// * `mode` - The `Lindera::Mode`, with its penalty for decompose mode.
76
+ fn set_mode(&self, mode: &RbMode) {
77
+ self.inner.borrow_mut().set_mode(mode.as_lindera());
77
78
  }
78
79
 
79
80
  /// Sets the dictionary path.
@@ -96,6 +97,10 @@ impl RbTokenizerBuilder {
96
97
 
97
98
  /// Sets whether to keep whitespace in tokenization results.
98
99
  ///
100
+ /// Kept whitespace also stays in the lattice, so the other tokens can
101
+ /// differ from those of the default output, which skips whitespace as
102
+ /// MeCab does.
103
+ ///
99
104
  /// # Arguments
100
105
  ///
101
106
  /// * `keep_whitespace` - If true, whitespace tokens will be included.
@@ -103,6 +108,44 @@ impl RbTokenizerBuilder {
103
108
  self.inner.borrow_mut().set_keep_whitespace(keep_whitespace);
104
109
  }
105
110
 
111
+ /// Sets whether whitespace is skipped in the lattice, with the same
112
+ /// meaning as `segmenter.skip_whitespace` in a YAML config.
113
+ ///
114
+ /// `true` skips whitespace, so the words on either side connect directly,
115
+ /// as MeCab does; `false` keeps whitespace in the lattice as a node of
116
+ /// its own, as before v7 (it is still dropped from the output); `nil`
117
+ /// goes back to the default of the dictionary's `metadata.json`
118
+ /// (skipping, except with SudachiDict). No effect while whitespace tokens
119
+ /// are kept (`set_keep_whitespace(true)`).
120
+ ///
121
+ /// # Arguments
122
+ ///
123
+ /// * `skip_whitespace` - The setting, or `nil` for the dictionary's
124
+ /// default. Any other value counts by Ruby truthiness, as for
125
+ /// `set_keep_whitespace`.
126
+ fn set_skip_whitespace(&self, skip_whitespace: Option<bool>) {
127
+ self.inner.borrow_mut().set_skip_whitespace(skip_whitespace);
128
+ }
129
+
130
+ /// Sets the break characters, with the same meaning as
131
+ /// `segmenter.break_characters` in a YAML config.
132
+ ///
133
+ /// Each character of the string is a break character: no unknown word,
134
+ /// nor, in decompose mode, any system-dictionary word, crosses a
135
+ /// boundary between a break character and another character. An empty
136
+ /// string turns the rule off, which gives MeCab's segmentation; `nil`
137
+ /// goes back to the default, `・`.
138
+ ///
139
+ /// # Arguments
140
+ ///
141
+ /// * `break_characters` - The break characters, or `nil` for the
142
+ /// default. A value that is not a string raises a `TypeError`.
143
+ fn set_break_characters(&self, break_characters: Option<String>) {
144
+ self.inner
145
+ .borrow_mut()
146
+ .set_break_characters(break_characters.as_deref());
147
+ }
148
+
106
149
  /// Sets the left-space penalty (Korean), with the same meaning as
107
150
  /// `segmenter.space_penalty` in a YAML config.
108
151
  ///
@@ -194,7 +237,7 @@ impl RbTokenizerBuilder {
194
237
  /// The tokenizer processes text and returns tokens with their morphological features.
195
238
  #[magnus::wrap(class = "Lindera::Tokenizer", free_immediately, size)]
196
239
  pub struct RbTokenizer {
197
- /// Inner binding-core tokenizer.
240
+ /// Inner lindera-binding tokenizer.
198
241
  inner: CoreTokenizer,
199
242
  }
200
243
 
@@ -207,8 +250,8 @@ pub struct RbTokenizer {
207
250
  ///
208
251
  /// * `args` - The Ruby arguments:
209
252
  /// * `dictionary` - Dictionary to use.
210
- /// * `mode` - Tokenization mode ("normal" or "decompose"). `nil` or
211
- /// omitted means "normal".
253
+ /// * `mode` - The `Lindera::Mode`, as `TokenizerBuilder#set_mode` takes
254
+ /// it. `nil` or omitted means normal mode.
212
255
  /// * `user_dictionary` - Optional user dictionary.
213
256
  /// * `space_penalty:` - The left-space penalty, in the forms
214
257
  /// `TokenizerBuilder#set_space_penalty` accepts. `nil` or omitted keeps
@@ -217,13 +260,15 @@ pub struct RbTokenizer {
217
260
  /// # Returns
218
261
  ///
219
262
  /// A new `RbTokenizer` instance, an `ArgumentError` for a wrong number of
220
- /// arguments or an unknown keyword, or the errors `set_space_penalty` and
221
- /// `build` raise for an invalid space-penalty setting.
263
+ /// arguments or an unknown keyword, a `TypeError` for a `mode` that is not a
264
+ /// `Lindera::Mode`, the errors `set_space_penalty` and `build` raise for an
265
+ /// invalid space-penalty setting, or a `RuntimeError` for a user dictionary
266
+ /// with a context ID outside the dictionary's connection matrix.
222
267
  fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
223
268
  let ruby = Ruby::get().expect("Ruby runtime not initialized");
224
269
  let args = scan_args::<
225
270
  (&RbDictionary,),
226
- (Option<Option<String>>, Option<Option<&RbUserDictionary>>),
271
+ (Option<Option<&RbMode>>, Option<Option<&RbUserDictionary>>),
227
272
  (),
228
273
  (),
229
274
  RHash,
@@ -235,9 +280,10 @@ fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
235
280
  get_kwargs::<_, (), (Option<Value>,), ()>(args.keywords, &[], &["space_penalty"])?;
236
281
  let (space_penalty,) = keywords.optional;
237
282
 
238
- let mode = mode.flatten();
283
+ let mode = mode.flatten().map_or(lindera::mode::Mode::Normal, |mode| {
284
+ mode.as_lindera().clone()
285
+ });
239
286
  let user_dictionary = user_dictionary.flatten();
240
- let mode_str = mode.as_deref().unwrap_or("normal");
241
287
  let space_penalty = match space_penalty {
242
288
  Some(value) => rb_value_to_json(&ruby, value)?,
243
289
  None => serde_json::Value::Null,
@@ -247,7 +293,7 @@ fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
247
293
  let user_dict = user_dictionary.map(|d| d.inner.clone());
248
294
 
249
295
  let inner =
250
- CoreTokenizer::from_segmenter_with_space_penalty(mode_str, dict, user_dict, &space_penalty)
296
+ CoreTokenizer::from_segmenter_with_space_penalty(mode, dict, user_dict, &space_penalty)
251
297
  .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
252
298
 
253
299
  Ok(RbTokenizer { inner })
@@ -309,26 +355,31 @@ impl RbTokenizer {
309
355
  ///
310
356
  /// # Arguments
311
357
  ///
358
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
359
+ /// * `rb_self` - The tokenizer.
312
360
  /// * `text` - Text to tokenize.
313
- /// * `n` - Number of N-best results.
361
+ /// * `n` - Number of N-best results, 0 or more.
314
362
  /// * `unique` - Whether to deduplicate results (default: false).
315
- /// * `cost_threshold` - Optional cost threshold.
363
+ /// * `cost_threshold` - Optional cost threshold, a 64-bit integer.
316
364
  ///
317
365
  /// # Returns
318
366
  ///
319
- /// An array of [tokens, cost] pairs.
367
+ /// An array of [tokens, cost] pairs, or a `RangeError` for an `n` or a
368
+ /// `cost_threshold` outside its range.
320
369
  fn tokenize_nbest(
321
- &self,
370
+ ruby: &Ruby,
371
+ rb_self: &Self,
322
372
  text: String,
323
- n: usize,
373
+ n: Integer,
324
374
  unique: Option<bool>,
325
- cost_threshold: Option<i64>,
375
+ cost_threshold: Option<Integer>,
326
376
  ) -> Result<RArray, Error> {
327
- let ruby = Ruby::get().expect("Ruby runtime not initialized");
328
- let results = self
377
+ let n = integer_arg(ruby, "n", n)?;
378
+ let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
379
+ let results = rb_self
329
380
  .inner
330
381
  .tokenize_nbest(&text, n, unique.unwrap_or(false), cost_threshold)
331
- .map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
382
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
332
383
 
333
384
  let rb_results = ruby.ary_new_capa(results.len());
334
385
  for (views, cost) in results {
@@ -344,6 +395,177 @@ impl RbTokenizer {
344
395
 
345
396
  Ok(rb_results)
346
397
  }
398
+
399
+ /// Tokenizes every text of `texts`, each as `tokenize` does, on up to
400
+ /// `max_threads` threads, and returns their tokens in the order of
401
+ /// `texts`: `tokenize_batch(texts, max_threads = nil)`.
402
+ ///
403
+ /// The tokenization runs on several threads while the call holds the
404
+ /// GVL, so other Ruby threads wait for it. The results are the same for
405
+ /// every number of threads.
406
+ ///
407
+ /// # Arguments
408
+ ///
409
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
410
+ /// * `rb_self` - The tokenizer.
411
+ /// * `args` - `texts`, an array of strings, and `max_threads`, an
412
+ /// integer of 1 or more, or `nil` (the default) for the number of
413
+ /// CPUs.
414
+ ///
415
+ /// # Returns
416
+ ///
417
+ /// An array with the Token objects of each text, an `ArgumentError` for
418
+ /// a wrong number of arguments, a `TypeError` for `texts` that is not an
419
+ /// array of strings, a `RangeError` for a `max_threads` outside its
420
+ /// range, or a `RuntimeError` for the first text, in the order of
421
+ /// `texts`, whose tokenization fails.
422
+ fn tokenize_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
423
+ let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
424
+ let (texts,) = args.required;
425
+ let (max_threads,) = args.optional;
426
+ let max_threads = batch_threads(ruby, max_threads.flatten())?;
427
+ let batch = rb_self
428
+ .inner
429
+ .tokenize_batch(&texts, max_threads)
430
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
431
+
432
+ let rb_batch = ruby.ary_new_capa(batch.len());
433
+ for views in batch {
434
+ rb_batch.push(token_array(ruby, views)?)?;
435
+ }
436
+ Ok(rb_batch)
437
+ }
438
+
439
+ /// Tokenizes every text of `texts` as `tokenize_surfaces` does, on up to
440
+ /// `max_threads` threads, as `tokenize_batch` does:
441
+ /// `tokenize_surfaces_batch(texts, max_threads = nil)`.
442
+ ///
443
+ /// # Arguments
444
+ ///
445
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
446
+ /// * `rb_self` - The tokenizer.
447
+ /// * `args` - `texts` and `max_threads`, as `tokenize_batch` takes them.
448
+ ///
449
+ /// # Returns
450
+ ///
451
+ /// An array with the surface strings of each text, or the errors
452
+ /// `tokenize_batch` raises.
453
+ fn tokenize_surfaces_batch(
454
+ ruby: &Ruby,
455
+ rb_self: &Self,
456
+ args: &[Value],
457
+ ) -> Result<RArray, Error> {
458
+ let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
459
+ let (texts,) = args.required;
460
+ let (max_threads,) = args.optional;
461
+ let max_threads = batch_threads(ruby, max_threads.flatten())?;
462
+ let batch = rb_self
463
+ .inner
464
+ .tokenize_surfaces_batch(&texts, max_threads)
465
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
466
+
467
+ let rb_batch = ruby.ary_new_capa(batch.len());
468
+ for views in batch {
469
+ let surfaces = ruby.ary_new_capa(views.len());
470
+ for view in views {
471
+ surfaces.push(view.surface)?;
472
+ }
473
+ rb_batch.push(surfaces)?;
474
+ }
475
+ Ok(rb_batch)
476
+ }
477
+
478
+ /// Tokenizes every text of `texts` as `tokenize_nbest` does, on up to
479
+ /// `max_threads` threads, as `tokenize_batch` does:
480
+ /// `tokenize_nbest_batch(texts, n, unique, cost_threshold, max_threads = nil)`.
481
+ ///
482
+ /// # Arguments
483
+ ///
484
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
485
+ /// * `rb_self` - The tokenizer.
486
+ /// * `args` - `texts`; `n`, `unique` and `cost_threshold`, as
487
+ /// `tokenize_nbest` takes them; and `max_threads`, as
488
+ /// `tokenize_batch` takes it.
489
+ ///
490
+ /// # Returns
491
+ ///
492
+ /// An array with the [tokens, cost] pairs of each text, a `RangeError`
493
+ /// for an `n`, a `cost_threshold` or a `max_threads` outside its range,
494
+ /// or the other errors `tokenize_batch` raises.
495
+ fn tokenize_nbest_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
496
+ let args = scan_args::<
497
+ (Vec<String>, Integer, Option<bool>, Option<Integer>),
498
+ (Option<Option<Integer>>,),
499
+ (),
500
+ (),
501
+ (),
502
+ (),
503
+ >(args)?;
504
+ let (texts, n, unique, cost_threshold) = args.required;
505
+ let (max_threads,) = args.optional;
506
+ let n = integer_arg(ruby, "n", n)?;
507
+ let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
508
+ let max_threads = batch_threads(ruby, max_threads.flatten())?;
509
+ let batch = rb_self
510
+ .inner
511
+ .tokenize_nbest_batch(
512
+ &texts,
513
+ n,
514
+ unique.unwrap_or(false),
515
+ cost_threshold,
516
+ max_threads,
517
+ )
518
+ .map_err(|err| to_magnus_error(ruby, err.to_string()))?;
519
+
520
+ let rb_batch = ruby.ary_new_capa(batch.len());
521
+ for results in batch {
522
+ let rb_results = ruby.ary_new_capa(results.len());
523
+ for (views, cost) in results {
524
+ let pair = ruby.ary_new_capa(2);
525
+ pair.push(token_array(ruby, views)?)?;
526
+ pair.push(cost)?;
527
+ rb_results.push(pair)?;
528
+ }
529
+ rb_batch.push(rb_results)?;
530
+ }
531
+ Ok(rb_batch)
532
+ }
533
+ }
534
+
535
+ /// Makes the Token objects of one text into an array.
536
+ ///
537
+ /// # Arguments
538
+ ///
539
+ /// * `ruby` - Ruby runtime handle.
540
+ /// * `views` - The tokens of the text.
541
+ ///
542
+ /// # Returns
543
+ ///
544
+ /// The array of Token objects.
545
+ fn token_array(ruby: &Ruby, views: Vec<TokenView>) -> Result<RArray, Error> {
546
+ let arr = ruby.ary_new_capa(views.len());
547
+ for view in views {
548
+ arr.push(ruby.into_value(RbToken::from_view(view)))?;
549
+ }
550
+ Ok(arr)
551
+ }
552
+
553
+ /// Converts the `max_threads` argument of the batch methods, as `train`
554
+ /// checks its own.
555
+ ///
556
+ /// # Arguments
557
+ ///
558
+ /// * `ruby` - Ruby runtime handle.
559
+ /// * `max_threads` - The argument, `None` for `nil` or omitted.
560
+ ///
561
+ /// # Returns
562
+ ///
563
+ /// The thread count, `None` for the default, or a `RangeError` when it is
564
+ /// not an integer of 1 or more.
565
+ fn batch_threads(ruby: &Ruby, max_threads: Option<Integer>) -> Result<Option<NonZeroUsize>, Error> {
566
+ let max_threads: Option<Positive<usize>> =
567
+ optional_integer_arg(ruby, "max_threads", max_threads)?;
568
+ Ok(max_threads.and_then(|Positive(n)| NonZeroUsize::new(n)))
347
569
  }
348
570
 
349
571
  /// Defines TokenizerBuilder and Tokenizer classes in the given Ruby module.
@@ -374,6 +596,14 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
374
596
  "set_keep_whitespace",
375
597
  method!(RbTokenizerBuilder::set_keep_whitespace, 1),
376
598
  )?;
599
+ builder_class.define_method(
600
+ "set_skip_whitespace",
601
+ method!(RbTokenizerBuilder::set_skip_whitespace, 1),
602
+ )?;
603
+ builder_class.define_method(
604
+ "set_break_characters",
605
+ method!(RbTokenizerBuilder::set_break_characters, 1),
606
+ )?;
377
607
  builder_class.define_method(
378
608
  "set_space_penalty",
379
609
  method!(RbTokenizerBuilder::set_space_penalty, 1),
@@ -396,6 +626,15 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
396
626
  method!(RbTokenizer::tokenize_surfaces, 1),
397
627
  )?;
398
628
  tokenizer_class.define_method("tokenize_nbest", method!(RbTokenizer::tokenize_nbest, 4))?;
629
+ tokenizer_class.define_method("tokenize_batch", method!(RbTokenizer::tokenize_batch, -1))?;
630
+ tokenizer_class.define_method(
631
+ "tokenize_surfaces_batch",
632
+ method!(RbTokenizer::tokenize_surfaces_batch, -1),
633
+ )?;
634
+ tokenizer_class.define_method(
635
+ "tokenize_nbest_batch",
636
+ method!(RbTokenizer::tokenize_nbest_batch, -1),
637
+ )?;
399
638
 
400
639
  Ok(())
401
640
  }