lindera 6.1.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +8 -6
- data/README.md +3 -3
- data/extconf.rb +6 -4
- data/src/dictionary.rs +130 -16
- data/src/metadata.rs +36 -38
- data/src/mode.rs +212 -154
- data/src/schema.rs +20 -9
- data/src/token.rs +16 -9
- data/src/tokenizer.rs +303 -36
- data/src/trainer.rs +172 -158
- data/src/util.rs +142 -11
- metadata +1 -43
data/src/tokenizer.rs
CHANGED
|
@@ -2,21 +2,25 @@
|
|
|
2
2
|
//!
|
|
3
3
|
//! This module provides a builder pattern for creating tokenizers and the tokenizer itself.
|
|
4
4
|
//! The build-flow orchestration is delegated to
|
|
5
|
-
//! [`
|
|
5
|
+
//! [`lindera_binding::CoreTokenizerBuilder`] / [`lindera_binding::CoreTokenizer`];
|
|
6
6
|
//! this module only adds the magnus wrappers and the Ruby-hash conversion.
|
|
7
7
|
|
|
8
8
|
use std::cell::RefCell;
|
|
9
|
+
use std::num::NonZeroUsize;
|
|
9
10
|
use std::path::Path;
|
|
10
11
|
|
|
11
12
|
use magnus::prelude::*;
|
|
12
|
-
use magnus::{
|
|
13
|
+
use magnus::scan_args::{get_kwargs, scan_args};
|
|
14
|
+
use magnus::{Error, Integer, RArray, RHash, Ruby, Value, function, method};
|
|
13
15
|
|
|
14
|
-
use
|
|
16
|
+
use lindera_binding::argument::Positive;
|
|
17
|
+
use lindera_binding::{CoreTokenizer, CoreTokenizerBuilder, TokenView};
|
|
15
18
|
|
|
16
19
|
use crate::dictionary::{RbDictionary, RbUserDictionary};
|
|
17
20
|
use crate::error::to_magnus_error;
|
|
21
|
+
use crate::mode::RbMode;
|
|
18
22
|
use crate::token::RbToken;
|
|
19
|
-
use crate::util::{rb_hash_to_json, rb_value_to_json};
|
|
23
|
+
use crate::util::{integer_arg, optional_integer_arg, rb_hash_to_json, rb_value_to_json};
|
|
20
24
|
|
|
21
25
|
/// Builder for creating a `Tokenizer` with custom configuration.
|
|
22
26
|
///
|
|
@@ -24,7 +28,7 @@ use crate::util::{rb_hash_to_json, rb_value_to_json};
|
|
|
24
28
|
/// Uses `RefCell` for interior mutability since Magnus `method!` requires `&self`.
|
|
25
29
|
#[magnus::wrap(class = "Lindera::TokenizerBuilder", free_immediately, size)]
|
|
26
30
|
pub struct RbTokenizerBuilder {
|
|
27
|
-
/// Inner binding
|
|
31
|
+
/// Inner lindera-binding tokenizer builder (wrapped in RefCell for interior mutability).
|
|
28
32
|
inner: RefCell<CoreTokenizerBuilder>,
|
|
29
33
|
}
|
|
30
34
|
|
|
@@ -63,16 +67,14 @@ impl RbTokenizerBuilder {
|
|
|
63
67
|
|
|
64
68
|
/// Sets the tokenization mode.
|
|
65
69
|
///
|
|
70
|
+
/// Ruby signature: `set_mode(mode)`. magnus raises a `TypeError` for a
|
|
71
|
+
/// `mode` that is not a `Lindera::Mode`.
|
|
72
|
+
///
|
|
66
73
|
/// # Arguments
|
|
67
74
|
///
|
|
68
|
-
/// * `mode` - Mode
|
|
69
|
-
fn set_mode(&self, mode:
|
|
70
|
-
|
|
71
|
-
let mut builder = self.inner.borrow_mut();
|
|
72
|
-
builder
|
|
73
|
-
.set_mode(&mode)
|
|
74
|
-
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
75
|
-
Ok(())
|
|
75
|
+
/// * `mode` - The `Lindera::Mode`, with its penalty for decompose mode.
|
|
76
|
+
fn set_mode(&self, mode: &RbMode) {
|
|
77
|
+
self.inner.borrow_mut().set_mode(mode.as_lindera());
|
|
76
78
|
}
|
|
77
79
|
|
|
78
80
|
/// Sets the dictionary path.
|
|
@@ -95,6 +97,10 @@ impl RbTokenizerBuilder {
|
|
|
95
97
|
|
|
96
98
|
/// Sets whether to keep whitespace in tokenization results.
|
|
97
99
|
///
|
|
100
|
+
/// Kept whitespace also stays in the lattice, so the other tokens can
|
|
101
|
+
/// differ from those of the default output, which skips whitespace as
|
|
102
|
+
/// MeCab does.
|
|
103
|
+
///
|
|
98
104
|
/// # Arguments
|
|
99
105
|
///
|
|
100
106
|
/// * `keep_whitespace` - If true, whitespace tokens will be included.
|
|
@@ -102,6 +108,44 @@ impl RbTokenizerBuilder {
|
|
|
102
108
|
self.inner.borrow_mut().set_keep_whitespace(keep_whitespace);
|
|
103
109
|
}
|
|
104
110
|
|
|
111
|
+
/// Sets whether whitespace is skipped in the lattice, with the same
|
|
112
|
+
/// meaning as `segmenter.skip_whitespace` in a YAML config.
|
|
113
|
+
///
|
|
114
|
+
/// `true` skips whitespace, so the words on either side connect directly,
|
|
115
|
+
/// as MeCab does; `false` keeps whitespace in the lattice as a node of
|
|
116
|
+
/// its own, as before v7 (it is still dropped from the output); `nil`
|
|
117
|
+
/// goes back to the default of the dictionary's `metadata.json`
|
|
118
|
+
/// (skipping, except with SudachiDict). No effect while whitespace tokens
|
|
119
|
+
/// are kept (`set_keep_whitespace(true)`).
|
|
120
|
+
///
|
|
121
|
+
/// # Arguments
|
|
122
|
+
///
|
|
123
|
+
/// * `skip_whitespace` - The setting, or `nil` for the dictionary's
|
|
124
|
+
/// default. Any other value counts by Ruby truthiness, as for
|
|
125
|
+
/// `set_keep_whitespace`.
|
|
126
|
+
fn set_skip_whitespace(&self, skip_whitespace: Option<bool>) {
|
|
127
|
+
self.inner.borrow_mut().set_skip_whitespace(skip_whitespace);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/// Sets the break characters, with the same meaning as
|
|
131
|
+
/// `segmenter.break_characters` in a YAML config.
|
|
132
|
+
///
|
|
133
|
+
/// Each character of the string is a break character: no unknown word,
|
|
134
|
+
/// nor, in decompose mode, any system-dictionary word, crosses a
|
|
135
|
+
/// boundary between a break character and another character. An empty
|
|
136
|
+
/// string turns the rule off, which gives MeCab's segmentation; `nil`
|
|
137
|
+
/// goes back to the default, `・`.
|
|
138
|
+
///
|
|
139
|
+
/// # Arguments
|
|
140
|
+
///
|
|
141
|
+
/// * `break_characters` - The break characters, or `nil` for the
|
|
142
|
+
/// default. A value that is not a string raises a `TypeError`.
|
|
143
|
+
fn set_break_characters(&self, break_characters: Option<String>) {
|
|
144
|
+
self.inner
|
|
145
|
+
.borrow_mut()
|
|
146
|
+
.set_break_characters(break_characters.as_deref());
|
|
147
|
+
}
|
|
148
|
+
|
|
105
149
|
/// Sets the left-space penalty (Korean), with the same meaning as
|
|
106
150
|
/// `segmenter.space_penalty` in a YAML config.
|
|
107
151
|
///
|
|
@@ -193,34 +237,64 @@ impl RbTokenizerBuilder {
|
|
|
193
237
|
/// The tokenizer processes text and returns tokens with their morphological features.
|
|
194
238
|
#[magnus::wrap(class = "Lindera::Tokenizer", free_immediately, size)]
|
|
195
239
|
pub struct RbTokenizer {
|
|
196
|
-
/// Inner binding
|
|
240
|
+
/// Inner lindera-binding tokenizer.
|
|
197
241
|
inner: CoreTokenizer,
|
|
198
242
|
}
|
|
199
243
|
|
|
200
244
|
/// Creates a new tokenizer with the given dictionary and mode.
|
|
201
245
|
///
|
|
246
|
+
/// Ruby signature: `Tokenizer.new(dictionary, mode = nil, user_dictionary = nil,
|
|
247
|
+
/// space_penalty: nil)`.
|
|
248
|
+
///
|
|
202
249
|
/// # Arguments
|
|
203
250
|
///
|
|
204
|
-
/// * `
|
|
205
|
-
///
|
|
206
|
-
///
|
|
251
|
+
/// * `args` - The Ruby arguments:
|
|
252
|
+
/// * `dictionary` - Dictionary to use.
|
|
253
|
+
/// * `mode` - The `Lindera::Mode`, as `TokenizerBuilder#set_mode` takes
|
|
254
|
+
/// it. `nil` or omitted means normal mode.
|
|
255
|
+
/// * `user_dictionary` - Optional user dictionary.
|
|
256
|
+
/// * `space_penalty:` - The left-space penalty, in the forms
|
|
257
|
+
/// `TokenizerBuilder#set_space_penalty` accepts. `nil` or omitted keeps
|
|
258
|
+
/// the rules the dictionary ships.
|
|
207
259
|
///
|
|
208
260
|
/// # Returns
|
|
209
261
|
///
|
|
210
|
-
/// A new `RbTokenizer` instance
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
) -> Result<RbTokenizer, Error> {
|
|
262
|
+
/// A new `RbTokenizer` instance, an `ArgumentError` for a wrong number of
|
|
263
|
+
/// arguments or an unknown keyword, a `TypeError` for a `mode` that is not a
|
|
264
|
+
/// `Lindera::Mode`, the errors `set_space_penalty` and `build` raise for an
|
|
265
|
+
/// invalid space-penalty setting, or a `RuntimeError` for a user dictionary
|
|
266
|
+
/// with a context ID outside the dictionary's connection matrix.
|
|
267
|
+
fn tokenizer_new(args: &[Value]) -> Result<RbTokenizer, Error> {
|
|
216
268
|
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
217
|
-
let
|
|
269
|
+
let args = scan_args::<
|
|
270
|
+
(&RbDictionary,),
|
|
271
|
+
(Option<Option<&RbMode>>, Option<Option<&RbUserDictionary>>),
|
|
272
|
+
(),
|
|
273
|
+
(),
|
|
274
|
+
RHash,
|
|
275
|
+
(),
|
|
276
|
+
>(args)?;
|
|
277
|
+
let (dictionary,) = args.required;
|
|
278
|
+
let (mode, user_dictionary) = args.optional;
|
|
279
|
+
let keywords =
|
|
280
|
+
get_kwargs::<_, (), (Option<Value>,), ()>(args.keywords, &[], &["space_penalty"])?;
|
|
281
|
+
let (space_penalty,) = keywords.optional;
|
|
282
|
+
|
|
283
|
+
let mode = mode.flatten().map_or(lindera::mode::Mode::Normal, |mode| {
|
|
284
|
+
mode.as_lindera().clone()
|
|
285
|
+
});
|
|
286
|
+
let user_dictionary = user_dictionary.flatten();
|
|
287
|
+
let space_penalty = match space_penalty {
|
|
288
|
+
Some(value) => rb_value_to_json(&ruby, value)?,
|
|
289
|
+
None => serde_json::Value::Null,
|
|
290
|
+
};
|
|
218
291
|
|
|
219
292
|
let dict = dictionary.inner.clone();
|
|
220
293
|
let user_dict = user_dictionary.map(|d| d.inner.clone());
|
|
221
294
|
|
|
222
|
-
let inner =
|
|
223
|
-
|
|
295
|
+
let inner =
|
|
296
|
+
CoreTokenizer::from_segmenter_with_space_penalty(mode, dict, user_dict, &space_penalty)
|
|
297
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
224
298
|
|
|
225
299
|
Ok(RbTokenizer { inner })
|
|
226
300
|
}
|
|
@@ -281,26 +355,31 @@ impl RbTokenizer {
|
|
|
281
355
|
///
|
|
282
356
|
/// # Arguments
|
|
283
357
|
///
|
|
358
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
359
|
+
/// * `rb_self` - The tokenizer.
|
|
284
360
|
/// * `text` - Text to tokenize.
|
|
285
|
-
/// * `n` - Number of N-best results.
|
|
361
|
+
/// * `n` - Number of N-best results, 0 or more.
|
|
286
362
|
/// * `unique` - Whether to deduplicate results (default: false).
|
|
287
|
-
/// * `cost_threshold` - Optional cost threshold.
|
|
363
|
+
/// * `cost_threshold` - Optional cost threshold, a 64-bit integer.
|
|
288
364
|
///
|
|
289
365
|
/// # Returns
|
|
290
366
|
///
|
|
291
|
-
/// An array of [tokens, cost] pairs
|
|
367
|
+
/// An array of [tokens, cost] pairs, or a `RangeError` for an `n` or a
|
|
368
|
+
/// `cost_threshold` outside its range.
|
|
292
369
|
fn tokenize_nbest(
|
|
293
|
-
&
|
|
370
|
+
ruby: &Ruby,
|
|
371
|
+
rb_self: &Self,
|
|
294
372
|
text: String,
|
|
295
|
-
n:
|
|
373
|
+
n: Integer,
|
|
296
374
|
unique: Option<bool>,
|
|
297
|
-
cost_threshold: Option<
|
|
375
|
+
cost_threshold: Option<Integer>,
|
|
298
376
|
) -> Result<RArray, Error> {
|
|
299
|
-
let
|
|
300
|
-
let
|
|
377
|
+
let n = integer_arg(ruby, "n", n)?;
|
|
378
|
+
let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
|
|
379
|
+
let results = rb_self
|
|
301
380
|
.inner
|
|
302
381
|
.tokenize_nbest(&text, n, unique.unwrap_or(false), cost_threshold)
|
|
303
|
-
.map_err(|err| to_magnus_error(
|
|
382
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
304
383
|
|
|
305
384
|
let rb_results = ruby.ary_new_capa(results.len());
|
|
306
385
|
for (views, cost) in results {
|
|
@@ -316,6 +395,177 @@ impl RbTokenizer {
|
|
|
316
395
|
|
|
317
396
|
Ok(rb_results)
|
|
318
397
|
}
|
|
398
|
+
|
|
399
|
+
/// Tokenizes every text of `texts`, each as `tokenize` does, on up to
|
|
400
|
+
/// `max_threads` threads, and returns their tokens in the order of
|
|
401
|
+
/// `texts`: `tokenize_batch(texts, max_threads = nil)`.
|
|
402
|
+
///
|
|
403
|
+
/// The tokenization runs on several threads while the call holds the
|
|
404
|
+
/// GVL, so other Ruby threads wait for it. The results are the same for
|
|
405
|
+
/// every number of threads.
|
|
406
|
+
///
|
|
407
|
+
/// # Arguments
|
|
408
|
+
///
|
|
409
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
410
|
+
/// * `rb_self` - The tokenizer.
|
|
411
|
+
/// * `args` - `texts`, an array of strings, and `max_threads`, an
|
|
412
|
+
/// integer of 1 or more, or `nil` (the default) for the number of
|
|
413
|
+
/// CPUs.
|
|
414
|
+
///
|
|
415
|
+
/// # Returns
|
|
416
|
+
///
|
|
417
|
+
/// An array with the Token objects of each text, an `ArgumentError` for
|
|
418
|
+
/// a wrong number of arguments, a `TypeError` for `texts` that is not an
|
|
419
|
+
/// array of strings, a `RangeError` for a `max_threads` outside its
|
|
420
|
+
/// range, or a `RuntimeError` for the first text, in the order of
|
|
421
|
+
/// `texts`, whose tokenization fails.
|
|
422
|
+
fn tokenize_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
|
|
423
|
+
let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
|
|
424
|
+
let (texts,) = args.required;
|
|
425
|
+
let (max_threads,) = args.optional;
|
|
426
|
+
let max_threads = batch_threads(ruby, max_threads.flatten())?;
|
|
427
|
+
let batch = rb_self
|
|
428
|
+
.inner
|
|
429
|
+
.tokenize_batch(&texts, max_threads)
|
|
430
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
431
|
+
|
|
432
|
+
let rb_batch = ruby.ary_new_capa(batch.len());
|
|
433
|
+
for views in batch {
|
|
434
|
+
rb_batch.push(token_array(ruby, views)?)?;
|
|
435
|
+
}
|
|
436
|
+
Ok(rb_batch)
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/// Tokenizes every text of `texts` as `tokenize_surfaces` does, on up to
|
|
440
|
+
/// `max_threads` threads, as `tokenize_batch` does:
|
|
441
|
+
/// `tokenize_surfaces_batch(texts, max_threads = nil)`.
|
|
442
|
+
///
|
|
443
|
+
/// # Arguments
|
|
444
|
+
///
|
|
445
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
446
|
+
/// * `rb_self` - The tokenizer.
|
|
447
|
+
/// * `args` - `texts` and `max_threads`, as `tokenize_batch` takes them.
|
|
448
|
+
///
|
|
449
|
+
/// # Returns
|
|
450
|
+
///
|
|
451
|
+
/// An array with the surface strings of each text, or the errors
|
|
452
|
+
/// `tokenize_batch` raises.
|
|
453
|
+
fn tokenize_surfaces_batch(
|
|
454
|
+
ruby: &Ruby,
|
|
455
|
+
rb_self: &Self,
|
|
456
|
+
args: &[Value],
|
|
457
|
+
) -> Result<RArray, Error> {
|
|
458
|
+
let args = scan_args::<(Vec<String>,), (Option<Option<Integer>>,), (), (), (), ()>(args)?;
|
|
459
|
+
let (texts,) = args.required;
|
|
460
|
+
let (max_threads,) = args.optional;
|
|
461
|
+
let max_threads = batch_threads(ruby, max_threads.flatten())?;
|
|
462
|
+
let batch = rb_self
|
|
463
|
+
.inner
|
|
464
|
+
.tokenize_surfaces_batch(&texts, max_threads)
|
|
465
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
466
|
+
|
|
467
|
+
let rb_batch = ruby.ary_new_capa(batch.len());
|
|
468
|
+
for views in batch {
|
|
469
|
+
let surfaces = ruby.ary_new_capa(views.len());
|
|
470
|
+
for view in views {
|
|
471
|
+
surfaces.push(view.surface)?;
|
|
472
|
+
}
|
|
473
|
+
rb_batch.push(surfaces)?;
|
|
474
|
+
}
|
|
475
|
+
Ok(rb_batch)
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
/// Tokenizes every text of `texts` as `tokenize_nbest` does, on up to
|
|
479
|
+
/// `max_threads` threads, as `tokenize_batch` does:
|
|
480
|
+
/// `tokenize_nbest_batch(texts, n, unique, cost_threshold, max_threads = nil)`.
|
|
481
|
+
///
|
|
482
|
+
/// # Arguments
|
|
483
|
+
///
|
|
484
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
485
|
+
/// * `rb_self` - The tokenizer.
|
|
486
|
+
/// * `args` - `texts`; `n`, `unique` and `cost_threshold`, as
|
|
487
|
+
/// `tokenize_nbest` takes them; and `max_threads`, as
|
|
488
|
+
/// `tokenize_batch` takes it.
|
|
489
|
+
///
|
|
490
|
+
/// # Returns
|
|
491
|
+
///
|
|
492
|
+
/// An array with the [tokens, cost] pairs of each text, a `RangeError`
|
|
493
|
+
/// for an `n`, a `cost_threshold` or a `max_threads` outside its range,
|
|
494
|
+
/// or the other errors `tokenize_batch` raises.
|
|
495
|
+
fn tokenize_nbest_batch(ruby: &Ruby, rb_self: &Self, args: &[Value]) -> Result<RArray, Error> {
|
|
496
|
+
let args = scan_args::<
|
|
497
|
+
(Vec<String>, Integer, Option<bool>, Option<Integer>),
|
|
498
|
+
(Option<Option<Integer>>,),
|
|
499
|
+
(),
|
|
500
|
+
(),
|
|
501
|
+
(),
|
|
502
|
+
(),
|
|
503
|
+
>(args)?;
|
|
504
|
+
let (texts, n, unique, cost_threshold) = args.required;
|
|
505
|
+
let (max_threads,) = args.optional;
|
|
506
|
+
let n = integer_arg(ruby, "n", n)?;
|
|
507
|
+
let cost_threshold = optional_integer_arg(ruby, "cost_threshold", cost_threshold)?;
|
|
508
|
+
let max_threads = batch_threads(ruby, max_threads.flatten())?;
|
|
509
|
+
let batch = rb_self
|
|
510
|
+
.inner
|
|
511
|
+
.tokenize_nbest_batch(
|
|
512
|
+
&texts,
|
|
513
|
+
n,
|
|
514
|
+
unique.unwrap_or(false),
|
|
515
|
+
cost_threshold,
|
|
516
|
+
max_threads,
|
|
517
|
+
)
|
|
518
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
519
|
+
|
|
520
|
+
let rb_batch = ruby.ary_new_capa(batch.len());
|
|
521
|
+
for results in batch {
|
|
522
|
+
let rb_results = ruby.ary_new_capa(results.len());
|
|
523
|
+
for (views, cost) in results {
|
|
524
|
+
let pair = ruby.ary_new_capa(2);
|
|
525
|
+
pair.push(token_array(ruby, views)?)?;
|
|
526
|
+
pair.push(cost)?;
|
|
527
|
+
rb_results.push(pair)?;
|
|
528
|
+
}
|
|
529
|
+
rb_batch.push(rb_results)?;
|
|
530
|
+
}
|
|
531
|
+
Ok(rb_batch)
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
/// Makes the Token objects of one text into an array.
|
|
536
|
+
///
|
|
537
|
+
/// # Arguments
|
|
538
|
+
///
|
|
539
|
+
/// * `ruby` - Ruby runtime handle.
|
|
540
|
+
/// * `views` - The tokens of the text.
|
|
541
|
+
///
|
|
542
|
+
/// # Returns
|
|
543
|
+
///
|
|
544
|
+
/// The array of Token objects.
|
|
545
|
+
fn token_array(ruby: &Ruby, views: Vec<TokenView>) -> Result<RArray, Error> {
|
|
546
|
+
let arr = ruby.ary_new_capa(views.len());
|
|
547
|
+
for view in views {
|
|
548
|
+
arr.push(ruby.into_value(RbToken::from_view(view)))?;
|
|
549
|
+
}
|
|
550
|
+
Ok(arr)
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
/// Converts the `max_threads` argument of the batch methods, as `train`
|
|
554
|
+
/// checks its own.
|
|
555
|
+
///
|
|
556
|
+
/// # Arguments
|
|
557
|
+
///
|
|
558
|
+
/// * `ruby` - Ruby runtime handle.
|
|
559
|
+
/// * `max_threads` - The argument, `None` for `nil` or omitted.
|
|
560
|
+
///
|
|
561
|
+
/// # Returns
|
|
562
|
+
///
|
|
563
|
+
/// The thread count, `None` for the default, or a `RangeError` when it is
|
|
564
|
+
/// not an integer of 1 or more.
|
|
565
|
+
fn batch_threads(ruby: &Ruby, max_threads: Option<Integer>) -> Result<Option<NonZeroUsize>, Error> {
|
|
566
|
+
let max_threads: Option<Positive<usize>> =
|
|
567
|
+
optional_integer_arg(ruby, "max_threads", max_threads)?;
|
|
568
|
+
Ok(max_threads.and_then(|Positive(n)| NonZeroUsize::new(n)))
|
|
319
569
|
}
|
|
320
570
|
|
|
321
571
|
/// Defines TokenizerBuilder and Tokenizer classes in the given Ruby module.
|
|
@@ -346,6 +596,14 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
|
346
596
|
"set_keep_whitespace",
|
|
347
597
|
method!(RbTokenizerBuilder::set_keep_whitespace, 1),
|
|
348
598
|
)?;
|
|
599
|
+
builder_class.define_method(
|
|
600
|
+
"set_skip_whitespace",
|
|
601
|
+
method!(RbTokenizerBuilder::set_skip_whitespace, 1),
|
|
602
|
+
)?;
|
|
603
|
+
builder_class.define_method(
|
|
604
|
+
"set_break_characters",
|
|
605
|
+
method!(RbTokenizerBuilder::set_break_characters, 1),
|
|
606
|
+
)?;
|
|
349
607
|
builder_class.define_method(
|
|
350
608
|
"set_space_penalty",
|
|
351
609
|
method!(RbTokenizerBuilder::set_space_penalty, 1),
|
|
@@ -361,13 +619,22 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
|
361
619
|
builder_class.define_method("build", method!(RbTokenizerBuilder::build, 0))?;
|
|
362
620
|
|
|
363
621
|
let tokenizer_class = module.define_class("Tokenizer", ruby.class_object())?;
|
|
364
|
-
tokenizer_class.define_singleton_method("new", function!(tokenizer_new,
|
|
622
|
+
tokenizer_class.define_singleton_method("new", function!(tokenizer_new, -1))?;
|
|
365
623
|
tokenizer_class.define_method("tokenize", method!(RbTokenizer::tokenize, 1))?;
|
|
366
624
|
tokenizer_class.define_method(
|
|
367
625
|
"tokenize_surfaces",
|
|
368
626
|
method!(RbTokenizer::tokenize_surfaces, 1),
|
|
369
627
|
)?;
|
|
370
628
|
tokenizer_class.define_method("tokenize_nbest", method!(RbTokenizer::tokenize_nbest, 4))?;
|
|
629
|
+
tokenizer_class.define_method("tokenize_batch", method!(RbTokenizer::tokenize_batch, -1))?;
|
|
630
|
+
tokenizer_class.define_method(
|
|
631
|
+
"tokenize_surfaces_batch",
|
|
632
|
+
method!(RbTokenizer::tokenize_surfaces_batch, -1),
|
|
633
|
+
)?;
|
|
634
|
+
tokenizer_class.define_method(
|
|
635
|
+
"tokenize_nbest_batch",
|
|
636
|
+
method!(RbTokenizer::tokenize_nbest_batch, -1),
|
|
637
|
+
)?;
|
|
371
638
|
|
|
372
639
|
Ok(())
|
|
373
640
|
}
|