lindera 6.0.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +71 -37
- data/LICENSE +21 -0
- data/README.md +23 -6
- data/extconf.rb +13 -0
- data/src/character_filter.rs +3 -0
- data/src/dictionary.rs +239 -0
- data/src/error.rs +19 -0
- data/src/lib.rs +90 -0
- data/src/metadata.rs +423 -0
- data/src/mode.rs +401 -0
- data/src/schema.rs +568 -0
- data/src/token.rs +188 -0
- data/src/token_filter.rs +3 -0
- data/src/tokenizer.rs +373 -0
- data/src/trainer.rs +283 -0
- data/src/util.rs +168 -0
- metadata +17 -4
- data/ext/lindera_ruby/extconf.rb +0 -8
data/src/token.rs
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
//! Token representation for morphological analysis results.
|
|
2
|
+
//!
|
|
3
|
+
//! This module wraps Lindera tokens for use in Ruby.
|
|
4
|
+
|
|
5
|
+
use magnus::prelude::*;
|
|
6
|
+
use magnus::{Error, RHash, Ruby, method};
|
|
7
|
+
|
|
8
|
+
use lindera::token::Token;
|
|
9
|
+
|
|
10
|
+
/// Token object wrapping the Rust Token data.
|
|
11
|
+
///
|
|
12
|
+
/// This class provides access to token fields and details.
|
|
13
|
+
#[magnus::wrap(class = "Lindera::Token", free_immediately, size)]
|
|
14
|
+
pub struct RbToken {
|
|
15
|
+
/// Surface form of the token.
|
|
16
|
+
surface: String,
|
|
17
|
+
/// Start byte position in the original text.
|
|
18
|
+
byte_start: usize,
|
|
19
|
+
/// End byte position in the original text.
|
|
20
|
+
byte_end: usize,
|
|
21
|
+
/// Position index of the token.
|
|
22
|
+
position: usize,
|
|
23
|
+
/// Word ID in the dictionary.
|
|
24
|
+
word_id: u32,
|
|
25
|
+
/// Whether this token is an unknown word.
|
|
26
|
+
is_unknown: bool,
|
|
27
|
+
/// Morphological details of the token.
|
|
28
|
+
details: Vec<String>,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
impl RbToken {
|
|
32
|
+
/// Creates a new `RbToken` from a Lindera `Token`.
|
|
33
|
+
///
|
|
34
|
+
/// # Arguments
|
|
35
|
+
///
|
|
36
|
+
/// * `token` - Lindera token to convert.
|
|
37
|
+
///
|
|
38
|
+
/// # Returns
|
|
39
|
+
///
|
|
40
|
+
/// A new `RbToken` instance.
|
|
41
|
+
pub fn from_token(token: Token) -> Self {
|
|
42
|
+
Self::from_view(lindera_binding_core::TokenView::from_token(token))
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/// Creates a new `RbToken` from a binding-core `TokenView`.
|
|
46
|
+
///
|
|
47
|
+
/// # Arguments
|
|
48
|
+
///
|
|
49
|
+
/// * `view` - Token view produced by the binding-core tokenizer.
|
|
50
|
+
///
|
|
51
|
+
/// # Returns
|
|
52
|
+
///
|
|
53
|
+
/// A new `RbToken` instance.
|
|
54
|
+
pub fn from_view(view: lindera_binding_core::TokenView) -> Self {
|
|
55
|
+
Self {
|
|
56
|
+
surface: view.surface,
|
|
57
|
+
byte_start: view.byte_start,
|
|
58
|
+
byte_end: view.byte_end,
|
|
59
|
+
position: view.position,
|
|
60
|
+
word_id: view.word_id,
|
|
61
|
+
is_unknown: view.is_unknown,
|
|
62
|
+
details: view.details,
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/// Returns the surface form of the token.
|
|
67
|
+
fn surface(&self) -> String {
|
|
68
|
+
self.surface.clone()
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/// Returns the start byte position.
|
|
72
|
+
fn byte_start(&self) -> usize {
|
|
73
|
+
self.byte_start
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/// Returns the end byte position.
|
|
77
|
+
fn byte_end(&self) -> usize {
|
|
78
|
+
self.byte_end
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// Returns the position index.
|
|
82
|
+
fn position(&self) -> usize {
|
|
83
|
+
self.position
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/// Returns the word ID.
|
|
87
|
+
fn word_id(&self) -> u32 {
|
|
88
|
+
self.word_id
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/// Returns whether this token is an unknown word.
|
|
92
|
+
fn is_unknown(&self) -> bool {
|
|
93
|
+
self.is_unknown
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/// Returns the morphological details of the token.
|
|
97
|
+
fn details(&self) -> Vec<String> {
|
|
98
|
+
self.details.clone()
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/// Returns the detail at the specified index.
|
|
102
|
+
///
|
|
103
|
+
/// # Arguments
|
|
104
|
+
///
|
|
105
|
+
/// * `index` - Index of the detail to retrieve.
|
|
106
|
+
///
|
|
107
|
+
/// # Returns
|
|
108
|
+
///
|
|
109
|
+
/// The detail string if found, otherwise nil.
|
|
110
|
+
fn get_detail(&self, index: usize) -> Option<String> {
|
|
111
|
+
self.details.get(index).cloned()
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/// Returns the token as a plain Hash keyed by Symbols.
|
|
115
|
+
///
|
|
116
|
+
/// Values keep their natural Ruby types (Integer for the positions and
|
|
117
|
+
/// word id, true/false for `:is_unknown`, Array of String for
|
|
118
|
+
/// `:details`), so the result works directly with `JSON.generate`.
|
|
119
|
+
///
|
|
120
|
+
/// # Returns
|
|
121
|
+
///
|
|
122
|
+
/// A Hash with the keys `:surface`, `:byte_start`, `:byte_end`,
|
|
123
|
+
/// `:position`, `:word_id`, `:is_unknown`, and `:details`, or a Magnus
|
|
124
|
+
/// `Error` if the Hash cannot be built.
|
|
125
|
+
fn to_hash(ruby: &Ruby, rb_self: &Self) -> Result<RHash, Error> {
|
|
126
|
+
let this = rb_self;
|
|
127
|
+
let hash = ruby.hash_new();
|
|
128
|
+
hash.aset(ruby.to_symbol("surface"), this.surface.as_str())?;
|
|
129
|
+
hash.aset(ruby.to_symbol("byte_start"), this.byte_start)?;
|
|
130
|
+
hash.aset(ruby.to_symbol("byte_end"), this.byte_end)?;
|
|
131
|
+
hash.aset(ruby.to_symbol("position"), this.position)?;
|
|
132
|
+
hash.aset(ruby.to_symbol("word_id"), this.word_id)?;
|
|
133
|
+
hash.aset(ruby.to_symbol("is_unknown"), this.is_unknown)?;
|
|
134
|
+
let details = ruby.ary_new();
|
|
135
|
+
for detail in &this.details {
|
|
136
|
+
details.push(detail.as_str())?;
|
|
137
|
+
}
|
|
138
|
+
hash.aset(ruby.to_symbol("details"), details)?;
|
|
139
|
+
Ok(hash)
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/// Returns the string representation of the token.
|
|
143
|
+
fn to_s(&self) -> String {
|
|
144
|
+
self.surface.clone()
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/// Returns the inspect representation of the token.
|
|
148
|
+
fn inspect(&self) -> String {
|
|
149
|
+
format!(
|
|
150
|
+
"#<Lindera::Token surface='{}', start={}, end={}, position={}, word_id={}, unknown={}>",
|
|
151
|
+
self.surface,
|
|
152
|
+
self.byte_start,
|
|
153
|
+
self.byte_end,
|
|
154
|
+
self.position,
|
|
155
|
+
self.word_id,
|
|
156
|
+
self.is_unknown
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/// Defines the Token class in the given Ruby module.
|
|
162
|
+
///
|
|
163
|
+
/// # Arguments
|
|
164
|
+
///
|
|
165
|
+
/// * `ruby` - Ruby runtime handle.
|
|
166
|
+
/// * `module` - Parent Ruby module.
|
|
167
|
+
///
|
|
168
|
+
/// # Returns
|
|
169
|
+
///
|
|
170
|
+
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
171
|
+
pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
172
|
+
let token_class = module.define_class("Token", ruby.class_object())?;
|
|
173
|
+
token_class.define_method("surface", method!(RbToken::surface, 0))?;
|
|
174
|
+
token_class.define_method("byte_start", method!(RbToken::byte_start, 0))?;
|
|
175
|
+
token_class.define_method("byte_end", method!(RbToken::byte_end, 0))?;
|
|
176
|
+
token_class.define_method("position", method!(RbToken::position, 0))?;
|
|
177
|
+
token_class.define_method("word_id", method!(RbToken::word_id, 0))?;
|
|
178
|
+
token_class.define_method("unknown?", method!(RbToken::is_unknown, 0))?;
|
|
179
|
+
token_class.define_method("details", method!(RbToken::details, 0))?;
|
|
180
|
+
token_class.define_method("get_detail", method!(RbToken::get_detail, 1))?;
|
|
181
|
+
token_class.define_method("to_hash", method!(RbToken::to_hash, 0))?;
|
|
182
|
+
token_class.define_method("to_h", method!(RbToken::to_hash, 0))?;
|
|
183
|
+
|
|
184
|
+
token_class.define_method("to_s", method!(RbToken::to_s, 0))?;
|
|
185
|
+
token_class.define_method("inspect", method!(RbToken::inspect, 0))?;
|
|
186
|
+
|
|
187
|
+
Ok(())
|
|
188
|
+
}
|
data/src/token_filter.rs
ADDED
data/src/tokenizer.rs
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
//! Tokenizer implementation for morphological analysis.
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides a builder pattern for creating tokenizers and the tokenizer itself.
|
|
4
|
+
//! The build-flow orchestration is delegated to
|
|
5
|
+
//! [`lindera_binding_core::CoreTokenizerBuilder`] / [`lindera_binding_core::CoreTokenizer`];
|
|
6
|
+
//! this module only adds the magnus wrappers and the Ruby-hash conversion.
|
|
7
|
+
|
|
8
|
+
use std::cell::RefCell;
|
|
9
|
+
use std::path::Path;
|
|
10
|
+
|
|
11
|
+
use magnus::prelude::*;
|
|
12
|
+
use magnus::{Error, RArray, RHash, Ruby, Value, function, method};
|
|
13
|
+
|
|
14
|
+
use lindera_binding_core::{CoreTokenizer, CoreTokenizerBuilder};
|
|
15
|
+
|
|
16
|
+
use crate::dictionary::{RbDictionary, RbUserDictionary};
|
|
17
|
+
use crate::error::to_magnus_error;
|
|
18
|
+
use crate::token::RbToken;
|
|
19
|
+
use crate::util::{rb_hash_to_json, rb_value_to_json};
|
|
20
|
+
|
|
21
|
+
/// Builder for creating a `Tokenizer` with custom configuration.
|
|
22
|
+
///
|
|
23
|
+
/// The builder pattern allows for fluent configuration of tokenizer parameters.
|
|
24
|
+
/// Uses `RefCell` for interior mutability since Magnus `method!` requires `&self`.
|
|
25
|
+
#[magnus::wrap(class = "Lindera::TokenizerBuilder", free_immediately, size)]
|
|
26
|
+
pub struct RbTokenizerBuilder {
|
|
27
|
+
/// Inner binding-core tokenizer builder (wrapped in RefCell for interior mutability).
|
|
28
|
+
inner: RefCell<CoreTokenizerBuilder>,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
impl RbTokenizerBuilder {
|
|
32
|
+
/// Creates a new `RbTokenizerBuilder` with default configuration.
|
|
33
|
+
///
|
|
34
|
+
/// # Returns
|
|
35
|
+
///
|
|
36
|
+
/// A new `RbTokenizerBuilder` instance.
|
|
37
|
+
fn new() -> Result<Self, Error> {
|
|
38
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
39
|
+
let inner =
|
|
40
|
+
CoreTokenizerBuilder::new().map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
41
|
+
Ok(Self {
|
|
42
|
+
inner: RefCell::new(inner),
|
|
43
|
+
})
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/// Loads configuration from a file.
|
|
47
|
+
///
|
|
48
|
+
/// # Arguments
|
|
49
|
+
///
|
|
50
|
+
/// * `file_path` - Path to the configuration file.
|
|
51
|
+
///
|
|
52
|
+
/// # Returns
|
|
53
|
+
///
|
|
54
|
+
/// A new `RbTokenizerBuilder` with the loaded configuration.
|
|
55
|
+
fn from_file(file_path: String) -> Result<Self, Error> {
|
|
56
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
57
|
+
let inner = CoreTokenizerBuilder::from_file(Path::new(&file_path))
|
|
58
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
59
|
+
Ok(Self {
|
|
60
|
+
inner: RefCell::new(inner),
|
|
61
|
+
})
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/// Sets the tokenization mode.
|
|
65
|
+
///
|
|
66
|
+
/// # Arguments
|
|
67
|
+
///
|
|
68
|
+
/// * `mode` - Mode string ("normal" or "decompose").
|
|
69
|
+
fn set_mode(&self, mode: String) -> Result<(), Error> {
|
|
70
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
71
|
+
let mut builder = self.inner.borrow_mut();
|
|
72
|
+
builder
|
|
73
|
+
.set_mode(&mode)
|
|
74
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
75
|
+
Ok(())
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/// Sets the dictionary path.
|
|
79
|
+
///
|
|
80
|
+
/// # Arguments
|
|
81
|
+
///
|
|
82
|
+
/// * `path` - Path to the dictionary directory.
|
|
83
|
+
fn set_dictionary(&self, path: String) {
|
|
84
|
+
self.inner.borrow_mut().set_dictionary(&path);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/// Sets the user dictionary URI.
|
|
88
|
+
///
|
|
89
|
+
/// # Arguments
|
|
90
|
+
///
|
|
91
|
+
/// * `uri` - URI to the user dictionary.
|
|
92
|
+
fn set_user_dictionary(&self, uri: String) {
|
|
93
|
+
self.inner.borrow_mut().set_user_dictionary(&uri);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/// Sets whether to keep whitespace in tokenization results.
|
|
97
|
+
///
|
|
98
|
+
/// # Arguments
|
|
99
|
+
///
|
|
100
|
+
/// * `keep_whitespace` - If true, whitespace tokens will be included.
|
|
101
|
+
fn set_keep_whitespace(&self, keep_whitespace: bool) {
|
|
102
|
+
self.inner.borrow_mut().set_keep_whitespace(keep_whitespace);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/// Sets the left-space penalty (Korean), with the same meaning as
|
|
106
|
+
/// `segmenter.space_penalty` in a YAML config.
|
|
107
|
+
///
|
|
108
|
+
/// `nil` restores the default (the rules the dictionary ships, if any;
|
|
109
|
+
/// ko-dic ships mecab-ko-dic's), `false` turns the penalty off, `true`
|
|
110
|
+
/// requires the dictionary's rules (`build` then fails for a dictionary
|
|
111
|
+
/// that ships none), and a hash such as
|
|
112
|
+
/// `{ 'rules' => [{ 'pos' => ['JKS'], 'cost' => 6000 }] }` applies those
|
|
113
|
+
/// rules instead.
|
|
114
|
+
///
|
|
115
|
+
/// # Arguments
|
|
116
|
+
///
|
|
117
|
+
/// * `ruby` - Ruby runtime handle.
|
|
118
|
+
/// * `rb_self` - The builder.
|
|
119
|
+
/// * `value` - `nil`, `true`, `false`, or a hash with string keys holding `"rules"`.
|
|
120
|
+
///
|
|
121
|
+
/// # Returns
|
|
122
|
+
///
|
|
123
|
+
/// `Ok(())` on success, a `TypeError` when `value` holds something that
|
|
124
|
+
/// cannot be converted to JSON (such as a Symbol or a non-finite Float),
|
|
125
|
+
/// or a `RuntimeError` when it converts but is not one of the forms
|
|
126
|
+
/// above.
|
|
127
|
+
fn set_space_penalty(ruby: &Ruby, rb_self: &Self, value: Value) -> Result<(), Error> {
|
|
128
|
+
let setting = rb_value_to_json(ruby, value)?;
|
|
129
|
+
rb_self
|
|
130
|
+
.inner
|
|
131
|
+
.borrow_mut()
|
|
132
|
+
.set_space_penalty(&setting)
|
|
133
|
+
.map_err(|err| to_magnus_error(ruby, err.to_string()))?;
|
|
134
|
+
Ok(())
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/// Appends a character filter to the filter pipeline.
|
|
138
|
+
///
|
|
139
|
+
/// # Arguments
|
|
140
|
+
///
|
|
141
|
+
/// * `kind` - Type of character filter to add.
|
|
142
|
+
/// * `args` - Optional hash of filter arguments.
|
|
143
|
+
fn append_character_filter(&self, kind: String, args: Option<RHash>) -> Result<(), Error> {
|
|
144
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
145
|
+
let filter_args = if let Some(hash) = args {
|
|
146
|
+
rb_hash_to_json(&ruby, hash)?
|
|
147
|
+
} else {
|
|
148
|
+
serde_json::Value::Object(serde_json::Map::new())
|
|
149
|
+
};
|
|
150
|
+
self.inner
|
|
151
|
+
.borrow_mut()
|
|
152
|
+
.append_character_filter(&kind, &filter_args);
|
|
153
|
+
Ok(())
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/// Appends a token filter to the filter pipeline.
|
|
157
|
+
///
|
|
158
|
+
/// # Arguments
|
|
159
|
+
///
|
|
160
|
+
/// * `kind` - Type of token filter to add.
|
|
161
|
+
/// * `args` - Optional hash of filter arguments.
|
|
162
|
+
fn append_token_filter(&self, kind: String, args: Option<RHash>) -> Result<(), Error> {
|
|
163
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
164
|
+
let filter_args = if let Some(hash) = args {
|
|
165
|
+
rb_hash_to_json(&ruby, hash)?
|
|
166
|
+
} else {
|
|
167
|
+
serde_json::Value::Object(serde_json::Map::new())
|
|
168
|
+
};
|
|
169
|
+
self.inner
|
|
170
|
+
.borrow_mut()
|
|
171
|
+
.append_token_filter(&kind, &filter_args);
|
|
172
|
+
Ok(())
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/// Builds the tokenizer with the configured settings.
|
|
176
|
+
///
|
|
177
|
+
/// # Returns
|
|
178
|
+
///
|
|
179
|
+
/// A configured `RbTokenizer` instance.
|
|
180
|
+
fn build(&self) -> Result<RbTokenizer, Error> {
|
|
181
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
182
|
+
let inner = self
|
|
183
|
+
.inner
|
|
184
|
+
.borrow()
|
|
185
|
+
.build()
|
|
186
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
187
|
+
Ok(RbTokenizer { inner })
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/// Tokenizer for performing morphological analysis.
|
|
192
|
+
///
|
|
193
|
+
/// The tokenizer processes text and returns tokens with their morphological features.
|
|
194
|
+
#[magnus::wrap(class = "Lindera::Tokenizer", free_immediately, size)]
|
|
195
|
+
pub struct RbTokenizer {
|
|
196
|
+
/// Inner binding-core tokenizer.
|
|
197
|
+
inner: CoreTokenizer,
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/// Creates a new tokenizer with the given dictionary and mode.
|
|
201
|
+
///
|
|
202
|
+
/// # Arguments
|
|
203
|
+
///
|
|
204
|
+
/// * `dictionary` - Dictionary to use.
|
|
205
|
+
/// * `mode` - Tokenization mode ("normal" or "decompose"). Defaults to "normal".
|
|
206
|
+
/// * `user_dictionary` - Optional user dictionary.
|
|
207
|
+
///
|
|
208
|
+
/// # Returns
|
|
209
|
+
///
|
|
210
|
+
/// A new `RbTokenizer` instance.
|
|
211
|
+
fn tokenizer_new(
|
|
212
|
+
dictionary: &RbDictionary,
|
|
213
|
+
mode: Option<String>,
|
|
214
|
+
user_dictionary: Option<&RbUserDictionary>,
|
|
215
|
+
) -> Result<RbTokenizer, Error> {
|
|
216
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
217
|
+
let mode_str = mode.as_deref().unwrap_or("normal");
|
|
218
|
+
|
|
219
|
+
let dict = dictionary.inner.clone();
|
|
220
|
+
let user_dict = user_dictionary.map(|d| d.inner.clone());
|
|
221
|
+
|
|
222
|
+
let inner = CoreTokenizer::from_segmenter(mode_str, dict, user_dict)
|
|
223
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
224
|
+
|
|
225
|
+
Ok(RbTokenizer { inner })
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
impl RbTokenizer {
|
|
229
|
+
/// Tokenizes the given text.
|
|
230
|
+
///
|
|
231
|
+
/// # Arguments
|
|
232
|
+
///
|
|
233
|
+
/// * `text` - Text to tokenize.
|
|
234
|
+
///
|
|
235
|
+
/// # Returns
|
|
236
|
+
///
|
|
237
|
+
/// An array of Token objects.
|
|
238
|
+
fn tokenize(&self, text: String) -> Result<RArray, Error> {
|
|
239
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
240
|
+
let views = self
|
|
241
|
+
.inner
|
|
242
|
+
.tokenize(&text)
|
|
243
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
244
|
+
|
|
245
|
+
let arr = ruby.ary_new_capa(views.len());
|
|
246
|
+
for view in views {
|
|
247
|
+
arr.push(ruby.into_value(RbToken::from_view(view)))?;
|
|
248
|
+
}
|
|
249
|
+
Ok(arr)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/// Tokenizes the given text and returns only the token surfaces.
|
|
253
|
+
///
|
|
254
|
+
/// This is the fast path for wakati-style use: no Token objects are
|
|
255
|
+
/// created and no morphological details are loaded, so it is
|
|
256
|
+
/// significantly faster than `tokenize` when only the surface strings
|
|
257
|
+
/// are needed. The surfaces equal `tokenize(text).map(&:surface)`.
|
|
258
|
+
///
|
|
259
|
+
/// # Arguments
|
|
260
|
+
///
|
|
261
|
+
/// * `text` - Text to tokenize.
|
|
262
|
+
///
|
|
263
|
+
/// # Returns
|
|
264
|
+
///
|
|
265
|
+
/// An array of surface strings, in reading order.
|
|
266
|
+
fn tokenize_surfaces(&self, text: String) -> Result<RArray, Error> {
|
|
267
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
268
|
+
let views = self
|
|
269
|
+
.inner
|
|
270
|
+
.tokenize_surfaces(&text)
|
|
271
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
272
|
+
|
|
273
|
+
let arr = ruby.ary_new_capa(views.len());
|
|
274
|
+
for view in views {
|
|
275
|
+
arr.push(view.surface)?;
|
|
276
|
+
}
|
|
277
|
+
Ok(arr)
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/// Tokenizes the given text and returns N-best results.
|
|
281
|
+
///
|
|
282
|
+
/// # Arguments
|
|
283
|
+
///
|
|
284
|
+
/// * `text` - Text to tokenize.
|
|
285
|
+
/// * `n` - Number of N-best results.
|
|
286
|
+
/// * `unique` - Whether to deduplicate results (default: false).
|
|
287
|
+
/// * `cost_threshold` - Optional cost threshold.
|
|
288
|
+
///
|
|
289
|
+
/// # Returns
|
|
290
|
+
///
|
|
291
|
+
/// An array of [tokens, cost] pairs.
|
|
292
|
+
fn tokenize_nbest(
|
|
293
|
+
&self,
|
|
294
|
+
text: String,
|
|
295
|
+
n: usize,
|
|
296
|
+
unique: Option<bool>,
|
|
297
|
+
cost_threshold: Option<i64>,
|
|
298
|
+
) -> Result<RArray, Error> {
|
|
299
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
300
|
+
let results = self
|
|
301
|
+
.inner
|
|
302
|
+
.tokenize_nbest(&text, n, unique.unwrap_or(false), cost_threshold)
|
|
303
|
+
.map_err(|err| to_magnus_error(&ruby, err.to_string()))?;
|
|
304
|
+
|
|
305
|
+
let rb_results = ruby.ary_new_capa(results.len());
|
|
306
|
+
for (views, cost) in results {
|
|
307
|
+
let token_arr = ruby.ary_new_capa(views.len());
|
|
308
|
+
for view in views {
|
|
309
|
+
token_arr.push(ruby.into_value(RbToken::from_view(view)))?;
|
|
310
|
+
}
|
|
311
|
+
let pair = ruby.ary_new_capa(2);
|
|
312
|
+
pair.push(token_arr)?;
|
|
313
|
+
pair.push(cost)?;
|
|
314
|
+
rb_results.push(pair)?;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
Ok(rb_results)
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/// Defines TokenizerBuilder and Tokenizer classes in the given Ruby module.
|
|
322
|
+
///
|
|
323
|
+
/// # Arguments
|
|
324
|
+
///
|
|
325
|
+
/// * `ruby` - Ruby runtime handle.
|
|
326
|
+
/// * `module` - Parent Ruby module.
|
|
327
|
+
///
|
|
328
|
+
/// # Returns
|
|
329
|
+
///
|
|
330
|
+
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
331
|
+
pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
332
|
+
let builder_class = module.define_class("TokenizerBuilder", ruby.class_object())?;
|
|
333
|
+
builder_class.define_singleton_method("new", function!(RbTokenizerBuilder::new, 0))?;
|
|
334
|
+
builder_class
|
|
335
|
+
.define_singleton_method("from_file", function!(RbTokenizerBuilder::from_file, 1))?;
|
|
336
|
+
builder_class.define_method("set_mode", method!(RbTokenizerBuilder::set_mode, 1))?;
|
|
337
|
+
builder_class.define_method(
|
|
338
|
+
"set_dictionary",
|
|
339
|
+
method!(RbTokenizerBuilder::set_dictionary, 1),
|
|
340
|
+
)?;
|
|
341
|
+
builder_class.define_method(
|
|
342
|
+
"set_user_dictionary",
|
|
343
|
+
method!(RbTokenizerBuilder::set_user_dictionary, 1),
|
|
344
|
+
)?;
|
|
345
|
+
builder_class.define_method(
|
|
346
|
+
"set_keep_whitespace",
|
|
347
|
+
method!(RbTokenizerBuilder::set_keep_whitespace, 1),
|
|
348
|
+
)?;
|
|
349
|
+
builder_class.define_method(
|
|
350
|
+
"set_space_penalty",
|
|
351
|
+
method!(RbTokenizerBuilder::set_space_penalty, 1),
|
|
352
|
+
)?;
|
|
353
|
+
builder_class.define_method(
|
|
354
|
+
"append_character_filter",
|
|
355
|
+
method!(RbTokenizerBuilder::append_character_filter, 2),
|
|
356
|
+
)?;
|
|
357
|
+
builder_class.define_method(
|
|
358
|
+
"append_token_filter",
|
|
359
|
+
method!(RbTokenizerBuilder::append_token_filter, 2),
|
|
360
|
+
)?;
|
|
361
|
+
builder_class.define_method("build", method!(RbTokenizerBuilder::build, 0))?;
|
|
362
|
+
|
|
363
|
+
let tokenizer_class = module.define_class("Tokenizer", ruby.class_object())?;
|
|
364
|
+
tokenizer_class.define_singleton_method("new", function!(tokenizer_new, 3))?;
|
|
365
|
+
tokenizer_class.define_method("tokenize", method!(RbTokenizer::tokenize, 1))?;
|
|
366
|
+
tokenizer_class.define_method(
|
|
367
|
+
"tokenize_surfaces",
|
|
368
|
+
method!(RbTokenizer::tokenize_surfaces, 1),
|
|
369
|
+
)?;
|
|
370
|
+
tokenizer_class.define_method("tokenize_nbest", method!(RbTokenizer::tokenize_nbest, 4))?;
|
|
371
|
+
|
|
372
|
+
Ok(())
|
|
373
|
+
}
|