lindera 6.2.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +8 -6
- data/README.md +3 -3
- data/extconf.rb +6 -4
- data/src/dictionary.rs +130 -16
- data/src/metadata.rs +36 -38
- data/src/mode.rs +212 -154
- data/src/schema.rs +20 -9
- data/src/token.rs +16 -9
- data/src/tokenizer.rs +270 -31
- data/src/trainer.rs +172 -158
- data/src/util.rs +65 -3
- metadata +1 -43
data/src/tokenizer.rs
CHANGED
|
@@ -2,22 +2,25 @@
|
|
|
2
2
|
//!
|
|
3
3
|
//! This module provides a builder pattern for creating tokenizers and the tokenizer itself.
|
|
4
4
|
//! The build-flow orchestration is delegated to
|
|
5
|
-
//! [`
|
|
5
|
+
//! [`lindera_binding::CoreTokenizerBuilder`] / [`lindera_binding::CoreTokenizer`];
|
|
6
6
|
//! this module only adds the magnus wrappers and the Ruby-hash conversion.
|
|
7
7
|
|
|
8
8
|
use std::cell::RefCell;
|
|
9
|
+
use std::num::NonZeroUsize;
|
|
9
10
|
use std::path::Path;
|
|
10
11
|
|
|
11
12
|
use magnus::prelude::*;
|
|
12
13
|
use magnus::scan_args::{get_kwargs, scan_args};
|
|
13
|
-
use magnus::{Error, RArray, RHash, Ruby, Value, function, method};
|
|
14
|
+
use magnus::{Error, Integer, RArray, RHash, Ruby, Value, function, method};
|
|
14
15
|
|
|
15
|
-
use
|
|
16
|
+
use lindera_binding::argument::Positive;
|
|
17
|
+
use lindera_binding::{CoreTokenizer, CoreTokenizerBuilder, TokenView};
|
|
16
18
|
|
|
17
19
|
use crate::dictionary::{RbDictionary, RbUserDictionary};
|
|
18
20
|
use crate::error::to_magnus_error;
|
|
21
|
+
use crate::mode::RbMode;
|
|
19
22
|
use crate::token::RbToken;
|
|
20
|
-
use crate::util::{rb_hash_to_json, rb_value_to_json};
|
|
23
|
+
use crate::util::{integer_arg, optional_integer_arg, rb_hash_to_json, rb_value_to_json};
|
|
21
24
|
|
|
22
25
|
/// Builder for creating a `Tokenizer` with custom configuration.
|
|
23
26
|
///
|
|
@@ -25,7 +28,7 @@ use crate::util::{rb_hash_to_json, rb_value_to_json};
|
|
|
25
28
|
/// Uses `RefCell` for interior mutability since Magnus `method!` requires `&self`.
|
|
26
29
|
#[magnus::wrap(class = "Lindera::TokenizerBuilder", free_immediately, size)]
|
|
27
30
|
pub struct RbTokenizerBuilder {
|
|
28
|
-
/// Inner binding
|
|
31
|
+
/// Inner lindera-binding tokenizer builder (wrapped in RefCell for interior mutability).
|
|
29
32
|
inner: RefCell<CoreTokenizerBuilder>,
|
|
30
33
|
}
|
|
31
34
|
|
|
@@ -64,16 +67,14 @@ impl RbTokenizerBuilder {
|
|
|
64
67
|
|
|
65
68
|
/// Sets the tokenization mode.
|
|
66
69
|
///
|
|
70
|
+
/// Ruby signature: `set_mode(mode)`. magnus raises a `TypeError` for a
|
|
71
|
+
/// `mode` that is not a `Lindera::Mode`.
|
|
72
|
+
///
|
|
67
73
|
/// # Arguments
|
|
68
74
|
///
|
|
69
|
-
/// * `mode` - Mode
|
|
70
|
-
fn set_mode(&self, mode:
|
|
71
|
-
|
|
72
|
-
let mut builder = self.inner.borrow_mut();
|
|
73
|
-
builder
|
|
74
|
-
.set_mode(&mode)
|
|
75
|
-
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
76
|
-
Ok(())
|
|
75
|
+
/// * `mode` - The `Lindera::Mode`, with its penalty for decompose mode.
|
|
76
|
+
fn set_mode(&self, mode: &RbMode) {
|
|
77
|
+
self.inner.borrow_mut().set_mode(mode.as_lindera());
|
|
77
78
|
}
|
|
78
79
|
|
|
79
80
|
/// Sets the dictionary path.
|
|
@@ -96,6 +97,10 @@ impl RbTokenizerBuilder {
|
|
|
96
97
|
|
|
97
98
|
/// Sets whether to keep whitespace in tokenization results.
|
|
98
99
|
///
|
|
100
|
+
/// Kept whitespace also stays in the lattice, so the other tokens can
|
|
101
|
+
/// differ from those of the default output, which skips whitespace as
|
|
102
|
+
/// MeCab does.
|
|
103
|
+
///
|
|
99
104
|
/// # Arguments
|
|
100
105
|
///
|
|
101
106
|
/// * `keep_whitespace` - If true, whitespace tokens will be included.
|
|
@@ -103,6 +108,44 @@ impl RbTokenizerBuilder {
|
|
|
103
108
|
self.inner.borrow_mut().set_keep_whitespace(keep_whitespace);
|
|
104
109
|
}
|
|
105
110
|
|
|
111
|
+
/// Sets whether whitespace is skipped in the lattice, with the same
|
|
112
|
+
/// meaning as `segmenter.skip_whitespace` in a YAML config.
|
|
113
|
+
///
|
|
114
|
+
/// `true` skips whitespace, so the words on either side connect directly,
|
|
115
|
+
/// as MeCab does; `false` keeps whitespace in the lattice as a node of
|
|
116
|
+
/// its own, as before v7 (it is still dropped from the output); `nil`
|
|
117
|
+
/// goes back to the default of the dictionary's `metadata.json`
|
|
118
|
+
/// (skipping, except with SudachiDict). No effect while whitespace tokens
|
|
119
|
+
/// are kept (`set_keep_whitespace(true)`).
|
|
120
|
+
///
|
|
121
|
+
/// # Arguments
|
|
122
|
+
///
|
|
123
|
+
/// * `skip_whitespace` - The setting, or `nil` for the dictionary's
|
|
124
|
+
/// default. Any other value counts by Ruby truthiness, as for
|
|
125
|
+
/// `set_keep_whitespace`.
|
|
126
|
+
fn set_skip_whitespace(&self, skip_whitespace: Option<bool>) {
|
|
127
|
+
self.inner.borrow_mut().set_skip_whitespace(skip_whitespace);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/// Sets the break characters, with the same meaning as
|
|
131
|
+
/// `segmenter.break_characters` in a YAML config.
|
|
132
|
+
///
|
|
133
|
+
/// Each character of the string is a break character: no unknown word,
|
|
134
|
+
/// nor, in decompose mode, any system-dictionary word, crosses a
|
|
135
|
+
/// boundary between a break character and another character. An empty
|
|
136
|
+
/// string turns the rule off, which gives MeCab's segmentation; `nil`
|
|
137
|
+
/// goes back to the default, `・`.
|
|
138
|
+
///
|
|
139
|
+
/// # Arguments
|
|
140
|
+
///
|
|
141
|
+
/// * `break_characters` - The break characters, or `nil` for the
|
|
142
|
+
/// default. A value that is not a string raises a `TypeError`.
|
|
143
|
+
fn set_break_characters(&self, break_characters: Option<String>) {
|
|
144
|
+
self.inner
|
|
145
|
+
.borrow_mut()
|
|
146
|
+
.set_break_characters(break_characters.as_deref());
|
|
147
|
+
}
|
|
148
|
+
|
|
106
149
|
/// Sets the left-space penalty (Korean), with the same meaning as
|
|
107
150
|
/// `segmenter.space_penalty` in a YAML config.
|
|
108
151
|
///
|
|
@@ -194,7 +237,7 @@ impl RbTokenizerBuilder {
|
|
|
194
237
|
/// The tokenizer processes text and returns tokens with their morphological features.
|
|
195
238
|
#[magnus::wrap(class = "Lindera::Tokenizer", free_immediately, size)]
|
|
196
239
|
pub struct RbTokenizer {
|
|
197
|
-
/// Inner binding
|
|
240
|
+
/// Inner lindera-binding tokenizer.
|
|
198
241
|
inner: CoreTokenizer,
|
|
199
242
|
}
|
|
200
243
|
|
|
@@ -207,8 +250,8 @@ pub struct RbTokenizer {
|
|
|
207
250
|
///
|
|
208
251
|
/// * `args` - The Ruby arguments:
|
|
209
252
|
/// * `dictionary` - Dictionary to use.
|
|
210
|
-
/// * `mode` -
|
|
211
|
-
/// omitted means
|
|
253
|
+
/// * `mode` - The `Lindera::Mode`, as `TokenizerBuilder#set_mode` takes
|
|
254
|
+
/// it. `nil` or omitted means normal mode.
|
|
212
255
|
/// * `user_dictionary` - Optional user dictionary.
|
|
213
256
|
/// * `space_penalty:` - The left-space penalty, in the forms
|
|
214
257
|
/// `TokenizerBuilder#set_space_penalty` accepts. `nil` or omitted keeps
|
|
@@ -217,13 +260,15 @@ pub struct RbTokenizer {
|
|
|
217
260
|
/// # Returns
|
|
218
261
|
///
|
|
219
262
|
/// A new `RbTokenizer` instance, an `ArgumentError` for a wrong number of
|
|
220
|
-
/// arguments or an unknown keyword,
|
|
221
|
-
/// `build` raise for an
|
|
263
|
+
/// arguments or an unknown keyword, a `TypeError` for a `mode` that is not a
|
|
264
|
+
/// `Lindera::Mode`, the errors `set_space_penalty` and `build` raise for an
|
|
265
|
+
/// invalid space-penalty setting, or a `RuntimeError` for a user dictionary
|
|
266
|
+
/// with a context ID outside the dictionary's connection matrix.
|
|
222
267
|
fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
|
|
223
268
|
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
224
269
|
let args = scan_args::<
|
|
225
270
|
(&RbDictionary,),
|
|
226
|
-
(Option<Option
|
|
271
|
+
(Option<Option<&RbMode>>, Option<Option<&RbUserDictionary>>),
|
|
227
272
|
(),
|
|
228
273
|
(),
|
|
229
274
|
RHash,
|
|
@@ -235,9 +280,10 @@ fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
|
|
|
235
280
|
get_kwargs::<_, (), (Option<Value>,), ()>(args.keywords, &[], &["space_penalty"])?;
|
|
236
281
|
let (space_penalty,) = keywords.optional;
|
|
237
282
|
|
|
238
|
-
let mode = mode.flatten()
|
|
283
|
+
let mode = mode.flatten().map_or(lindera::mode::Mode::Normal, |mode| {
|
|
284
|
+
mode.as_lindera().clone()
|
|
285
|
+
});
|
|
239
286
|
let user_dictionary = user_dictionary.flatten();
|
|
240
|
-
let mode_str = mode.as_deref().unwrap_or("normal");
|
|
241
287
|
let space_penalty = match space_penalty {
|
|
242
288
|
Some(value) => rb_value_to_json(&ruby, value)?,
|
|
243
289
|
None => serde_json::Value::Null,
|
|
@@ -247,7 +293,7 @@ fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
|
|
|
247
293
|
let user_dict = user_dictionary.map(|d| d.inner.clone());
|
|
248
294
|
|
|
249
295
|
let inner =
|
|
250
|
-
CoreTokenizer::from_segmenter_with_space_penalty(
|
|
296
|
+
CoreTokenizer::from_segmenter_with_space_penalty(mode, dict, user_dict, &space_penalty)
|
|
251
297
|
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
252
298
|
|
|
253
299
|
Ok(RbTokenizer { inner })
|
|
@@ -309,26 +355,31 @@ impl RbTokenizer {
|
|
|
309
355
|
///
|
|
310
356
|
/// # Arguments
|
|
311
357
|
///
|
|
358
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
359
|
+
/// * `rb_self` - The tokenizer.
|
|
312
360
|
/// * `text` - Text to tokenize.
|
|
313
|
-
/// * `n` - Number of N-best results.
|
|
361
|
+
/// * `n` - Number of N-best results, 0 or more.
|
|
314
362
|
/// * `unique` - Whether to deduplicate results (default: false).
|
|
315
|
-
/// * `cost_threshold` - Optional cost threshold.
|
|
363
|
+
/// * `cost_threshold` - Optional cost threshold, a 64-bit integer.
|
|
316
364
|
///
|
|
317
365
|
/// # Returns
|
|
318
366
|
///
|
|
319
|
-
/// An array of [tokens, cost] pairs
|
|
367
|
+
/// An array of [tokens, cost] pairs, or a `RangeError` for an `n` or a
|
|
368
|
+
/// `cost_threshold` outside its range.
|
|
320
369
|
fn tokenize_nbest(
|
|
321
|
-
&
|
|
370
|
+
ruby: &Ruby,
|
|
371
|
+
rb_self: &Self,
|
|
322
372
|
text: String,
|
|
323
|
-
n:
|
|
373
|
+
n: Integer,
|
|
324
374
|
unique: Option<bool>,
|
|
325
|
-
cost_threshold: Option<
|
|
375
|
+
cost_threshold: Option<Integer>,
|
|
326
376
|
) -> Result<RArray, Error> {
|
|
327
|
-
let
|
|
328
|
-
let
|
|
377
|
+
let n = integer_arg(ruby, "n", n)?;
|
|
378
|
+
let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
|
|
379
|
+
let results = rb_self
|
|
329
380
|
.inner
|
|
330
381
|
.tokenize_nbest(&text, n, unique.unwrap_or(false), cost_threshold)
|
|
331
|
-
.map_err(|err| to_magnus_error(
|
|
382
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
332
383
|
|
|
333
384
|
let rb_results = ruby.ary_new_capa(results.len());
|
|
334
385
|
for (views, cost) in results {
|
|
@@ -344,6 +395,177 @@ impl RbTokenizer {
|
|
|
344
395
|
|
|
345
396
|
Ok(rb_results)
|
|
346
397
|
}
|
|
398
|
+
|
|
399
|
+
/// Tokenizes every text of `texts`, each as `tokenize` does, on up to
|
|
400
|
+
/// `max_threads` threads, and returns their tokens in the order of
|
|
401
|
+
/// `texts`: `tokenize_batch(texts, max_threads = nil)`.
|
|
402
|
+
///
|
|
403
|
+
/// The tokenization runs on several threads while the call holds the
|
|
404
|
+
/// GVL, so other Ruby threads wait for it. The results are the same for
|
|
405
|
+
/// every number of threads.
|
|
406
|
+
///
|
|
407
|
+
/// # Arguments
|
|
408
|
+
///
|
|
409
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
410
|
+
/// * `rb_self` - The tokenizer.
|
|
411
|
+
/// * `args` - `texts`, an array of strings, and `max_threads`, an
|
|
412
|
+
/// integer of 1 or more, or `nil` (the default) for the number of
|
|
413
|
+
/// CPUs.
|
|
414
|
+
///
|
|
415
|
+
/// # Returns
|
|
416
|
+
///
|
|
417
|
+
/// An array with the Token objects of each text, an `ArgumentError` for
|
|
418
|
+
/// a wrong number of arguments, a `TypeError` for `texts` that is not an
|
|
419
|
+
/// array of strings, a `RangeError` for a `max_threads` outside its
|
|
420
|
+
/// range, or a `RuntimeError` for the first text, in the order of
|
|
421
|
+
/// `texts`, whose tokenization fails.
|
|
422
|
+
fn tokenize_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
|
|
423
|
+
let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
|
|
424
|
+
let (texts,) = args.required;
|
|
425
|
+
let (max_threads,) = args.optional;
|
|
426
|
+
let max_threads = batch_threads(ruby, max_threads.flatten())?;
|
|
427
|
+
let batch = rb_self
|
|
428
|
+
.inner
|
|
429
|
+
.tokenize_batch(&texts, max_threads)
|
|
430
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
431
|
+
|
|
432
|
+
let rb_batch = ruby.ary_new_capa(batch.len());
|
|
433
|
+
for views in batch {
|
|
434
|
+
rb_batch.push(token_array(ruby, views)?)?;
|
|
435
|
+
}
|
|
436
|
+
Ok(rb_batch)
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/// Tokenizes every text of `texts` as `tokenize_surfaces` does, on up to
|
|
440
|
+
/// `max_threads` threads, as `tokenize_batch` does:
|
|
441
|
+
/// `tokenize_surfaces_batch(texts, max_threads = nil)`.
|
|
442
|
+
///
|
|
443
|
+
/// # Arguments
|
|
444
|
+
///
|
|
445
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
446
|
+
/// * `rb_self` - The tokenizer.
|
|
447
|
+
/// * `args` - `texts` and `max_threads`, as `tokenize_batch` takes them.
|
|
448
|
+
///
|
|
449
|
+
/// # Returns
|
|
450
|
+
///
|
|
451
|
+
/// An array with the surface strings of each text, or the errors
|
|
452
|
+
/// `tokenize_batch` raises.
|
|
453
|
+
fn tokenize_surfaces_batch(
|
|
454
|
+
ruby: &Ruby,
|
|
455
|
+
rb_self: &Self,
|
|
456
|
+
args: &[Value],
|
|
457
|
+
) -> Result<RArray, Error> {
|
|
458
|
+
let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
|
|
459
|
+
let (texts,) = args.required;
|
|
460
|
+
let (max_threads,) = args.optional;
|
|
461
|
+
let max_threads = batch_threads(ruby, max_threads.flatten())?;
|
|
462
|
+
let batch = rb_self
|
|
463
|
+
.inner
|
|
464
|
+
.tokenize_surfaces_batch(&texts, max_threads)
|
|
465
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
466
|
+
|
|
467
|
+
let rb_batch = ruby.ary_new_capa(batch.len());
|
|
468
|
+
for views in batch {
|
|
469
|
+
let surfaces = ruby.ary_new_capa(views.len());
|
|
470
|
+
for view in views {
|
|
471
|
+
surfaces.push(view.surface)?;
|
|
472
|
+
}
|
|
473
|
+
rb_batch.push(surfaces)?;
|
|
474
|
+
}
|
|
475
|
+
Ok(rb_batch)
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/// Tokenizes every text of `texts` as `tokenize_nbest` does, on up to
|
|
479
|
+
/// `max_threads` threads, as `tokenize_batch` does:
|
|
480
|
+
/// `tokenize_nbest_batch(texts, n, unique, cost_threshold, max_threads = nil)`.
|
|
481
|
+
///
|
|
482
|
+
/// # Arguments
|
|
483
|
+
///
|
|
484
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
485
|
+
/// * `rb_self` - The tokenizer.
|
|
486
|
+
/// * `args` - `texts`; `n`, `unique` and `cost_threshold`, as
|
|
487
|
+
/// `tokenize_nbest` takes them; and `max_threads`, as
|
|
488
|
+
/// `tokenize_batch` takes it.
|
|
489
|
+
///
|
|
490
|
+
/// # Returns
|
|
491
|
+
///
|
|
492
|
+
/// An array with the [tokens, cost] pairs of each text, a `RangeError`
|
|
493
|
+
/// for an `n`, a `cost_threshold` or a `max_threads` outside its range,
|
|
494
|
+
/// or the other errors `tokenize_batch` raises.
|
|
495
|
+
fn tokenize_nbest_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
|
|
496
|
+
let args = scan_args::<
|
|
497
|
+
(Vec<String>, Integer, Option<bool>, Option<Integer>),
|
|
498
|
+
(Option<Option<Integer>>,),
|
|
499
|
+
(),
|
|
500
|
+
(),
|
|
501
|
+
(),
|
|
502
|
+
(),
|
|
503
|
+
>(args)?;
|
|
504
|
+
let (texts, n, unique, cost_threshold) = args.required;
|
|
505
|
+
let (max_threads,) = args.optional;
|
|
506
|
+
let n = integer_arg(ruby, "n", n)?;
|
|
507
|
+
let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
|
|
508
|
+
let max_threads = batch_threads(ruby, max_threads.flatten())?;
|
|
509
|
+
let batch = rb_self
|
|
510
|
+
.inner
|
|
511
|
+
.tokenize_nbest_batch(
|
|
512
|
+
&texts,
|
|
513
|
+
n,
|
|
514
|
+
unique.unwrap_or(false),
|
|
515
|
+
cost_threshold,
|
|
516
|
+
max_threads,
|
|
517
|
+
)
|
|
518
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
519
|
+
|
|
520
|
+
let rb_batch = ruby.ary_new_capa(batch.len());
|
|
521
|
+
for results in batch {
|
|
522
|
+
let rb_results = ruby.ary_new_capa(results.len());
|
|
523
|
+
for (views, cost) in results {
|
|
524
|
+
let pair = ruby.ary_new_capa(2);
|
|
525
|
+
pair.push(token_array(ruby, views)?)?;
|
|
526
|
+
pair.push(cost)?;
|
|
527
|
+
rb_results.push(pair)?;
|
|
528
|
+
}
|
|
529
|
+
rb_batch.push(rb_results)?;
|
|
530
|
+
}
|
|
531
|
+
Ok(rb_batch)
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/// Makes the Token objects of one text into an array.
|
|
536
|
+
///
|
|
537
|
+
/// # Arguments
|
|
538
|
+
///
|
|
539
|
+
/// * `ruby` - Ruby runtime handle.
|
|
540
|
+
/// * `views` - The tokens of the text.
|
|
541
|
+
///
|
|
542
|
+
/// # Returns
|
|
543
|
+
///
|
|
544
|
+
/// The array of Token objects.
|
|
545
|
+
fn token_array(ruby: &Ruby, views: Vec<TokenView>) -> Result<RArray, Error> {
|
|
546
|
+
let arr = ruby.ary_new_capa(views.len());
|
|
547
|
+
for view in views {
|
|
548
|
+
arr.push(ruby.into_value(RbToken::from_view(view)))?;
|
|
549
|
+
}
|
|
550
|
+
Ok(arr)
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/// Converts the `max_threads` argument of the batch methods, as `train`
|
|
554
|
+
/// checks its own.
|
|
555
|
+
///
|
|
556
|
+
/// # Arguments
|
|
557
|
+
///
|
|
558
|
+
/// * `ruby` - Ruby runtime handle.
|
|
559
|
+
/// * `max_threads` - The argument, `None` for `nil` or omitted.
|
|
560
|
+
///
|
|
561
|
+
/// # Returns
|
|
562
|
+
///
|
|
563
|
+
/// The thread count, `None` for the default, or a `RangeError` when it is
|
|
564
|
+
/// not an integer of 1 or more.
|
|
565
|
+
fn batch_threads(ruby: &Ruby, max_threads: Option<Integer>) -> Result<Option<NonZeroUsize>, Error> {
|
|
566
|
+
let max_threads: Option<Positive<usize>> =
|
|
567
|
+
optional_integer_arg(ruby, "max_threads", max_threads)?;
|
|
568
|
+
Ok(max_threads.and_then(|Positive(n)| NonZeroUsize::new(n)))
|
|
347
569
|
}
|
|
348
570
|
|
|
349
571
|
/// Defines TokenizerBuilder and Tokenizer classes in the given Ruby module.
|
|
@@ -374,6 +596,14 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
|
374
596
|
"set_keep_whitespace",
|
|
375
597
|
method!(RbTokenizerBuilder::set_keep_whitespace, 1),
|
|
376
598
|
)?;
|
|
599
|
+
builder_class.define_method(
|
|
600
|
+
"set_skip_whitespace",
|
|
601
|
+
method!(RbTokenizerBuilder::set_skip_whitespace, 1),
|
|
602
|
+
)?;
|
|
603
|
+
builder_class.define_method(
|
|
604
|
+
"set_break_characters",
|
|
605
|
+
method!(RbTokenizerBuilder::set_break_characters, 1),
|
|
606
|
+
)?;
|
|
377
607
|
builder_class.define_method(
|
|
378
608
|
"set_space_penalty",
|
|
379
609
|
method!(RbTokenizerBuilder::set_space_penalty, 1),
|
|
@@ -396,6 +626,15 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
|
396
626
|
method!(RbTokenizer::tokenize_surfaces, 1),
|
|
397
627
|
)?;
|
|
398
628
|
tokenizer_class.define_method("tokenize_nbest", method!(RbTokenizer::tokenize_nbest, 4))?;
|
|
629
|
+
tokenizer_class.define_method("tokenize_batch", method!(RbTokenizer::tokenize_batch, -1))?;
|
|
630
|
+
tokenizer_class.define_method(
|
|
631
|
+
"tokenize_surfaces_batch",
|
|
632
|
+
method!(RbTokenizer::tokenize_surfaces_batch, -1),
|
|
633
|
+
)?;
|
|
634
|
+
tokenizer_class.define_method(
|
|
635
|
+
"tokenize_nbest_batch",
|
|
636
|
+
method!(RbTokenizer::tokenize_nbest_batch, -1),
|
|
637
|
+
)?;
|
|
399
638
|
|
|
400
639
|
Ok(())
|
|
401
640
|
}
|