lindera 6.1.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +8 -6
- data/README.md +3 -3
- data/extconf.rb +6 -4
- data/src/dictionary.rs +130 -16
- data/src/metadata.rs +36 -38
- data/src/mode.rs +212 -154
- data/src/schema.rs +20 -9
- data/src/token.rs +16 -9
- data/src/tokenizer.rs +303 -36
- data/src/trainer.rs +172 -158
- data/src/util.rs +142 -11
- metadata +1 -43
data/src/trainer.rs
CHANGED
|
@@ -3,43 +3,125 @@
|
|
|
3
3
|
//! This module provides functions for training custom morphological analysis models
|
|
4
4
|
//! from annotated corpora.
|
|
5
5
|
|
|
6
|
+
// The lines go through `console`, which does not panic when a standard stream
|
|
7
|
+
// is full or closed, as `println!` does (#1312).
|
|
8
|
+
#![cfg_attr(not(test), deny(clippy::print_stdout, clippy::print_stderr))]
|
|
9
|
+
|
|
6
10
|
use std::fs::File;
|
|
7
11
|
use std::path::Path;
|
|
8
12
|
|
|
9
|
-
use magnus::{
|
|
13
|
+
use magnus::scan_args::{get_kwargs, scan_args};
|
|
14
|
+
use magnus::{Error, Integer, RHash, Ruby, Value, function};
|
|
10
15
|
|
|
11
|
-
use lindera::dictionary::trainer::{
|
|
16
|
+
use lindera::dictionary::trainer::{
|
|
17
|
+
Corpus, DEFAULT_ELASTIC_NET_L1_RATIO, DEFAULT_ETA, DEFAULT_LAMBDA, Model,
|
|
18
|
+
ParseRegularizationTypeError, RegularizationType, SerializableModel, Trainer, TrainerConfig,
|
|
19
|
+
console::{self, Stream},
|
|
20
|
+
};
|
|
21
|
+
use lindera_binding::argument::{Positive, non_negative_float_argument, unit_interval_argument};
|
|
12
22
|
|
|
13
23
|
use crate::error::to_magnus_error;
|
|
24
|
+
use crate::util::optional_integer_arg;
|
|
14
25
|
|
|
15
26
|
/// Trains a morphological analysis model from an annotated corpus.
|
|
16
27
|
///
|
|
28
|
+
/// Ruby signature: `Lindera.train(seed, corpus, char_def, unk_def,
|
|
29
|
+
/// feature_def, rewrite_def, output, lambda = nil, max_iter = nil,
|
|
30
|
+
/// max_threads = nil, regularization: nil, elastic_net_l1_ratio: nil,
|
|
31
|
+
/// eta: nil)`.
|
|
32
|
+
///
|
|
17
33
|
/// # Arguments
|
|
18
34
|
///
|
|
19
|
-
/// * `
|
|
20
|
-
/// * `
|
|
21
|
-
///
|
|
22
|
-
///
|
|
23
|
-
///
|
|
24
|
-
///
|
|
25
|
-
///
|
|
26
|
-
///
|
|
27
|
-
///
|
|
28
|
-
///
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
35
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
36
|
+
/// * `args` - The Ruby arguments:
|
|
37
|
+
/// * `seed` - Seed lexicon file path (CSV format).
|
|
38
|
+
/// * `corpus` - Training corpus file path (annotated text).
|
|
39
|
+
/// * `char_def` - Character definition file path (char.def).
|
|
40
|
+
/// * `unk_def` - Unknown word definition file path (unk.def).
|
|
41
|
+
/// * `feature_def` - Feature definition file path (feature.def).
|
|
42
|
+
/// * `rewrite_def` - Rewrite rule definition file path (rewrite.def).
|
|
43
|
+
/// * `output` - Output model file path.
|
|
44
|
+
/// * `lambda` - Regularization coefficient, finite and >= 0, default
|
|
45
|
+
/// (also for `nil`): 1.0; with L2, MeCab's `-c C` is `1 / C`, so 1.0 is
|
|
46
|
+
/// MeCab's `-c 1.0`.
|
|
47
|
+
/// Zero disables regularization for every type.
|
|
48
|
+
/// * `max_iter` - Maximum number of iterations, 1 or more, default: 100.
|
|
49
|
+
/// * `max_threads` - Maximum number of threads, 1 or more (nil =
|
|
50
|
+
/// auto-detect CPU cores).
|
|
51
|
+
/// * `regularization:` - Regularization type: `"l1"`, `"l2"` or
|
|
52
|
+
/// `"elasticnet"` (in any case; also `"elastic_net"` or
|
|
53
|
+
/// `"elastic-net"`), default (also for `nil`): `"l2"`.
|
|
54
|
+
/// * `elastic_net_l1_ratio:` - L1 ratio of Elastic Net regularization,
|
|
55
|
+
/// between 0 and 1, default (also for `nil`): 0.5; used only with
|
|
56
|
+
/// `"elasticnet"`.
|
|
57
|
+
/// Zero uses only L2; one uses only L1.
|
|
58
|
+
/// * `eta:` - Tolerance of the convergence test, finite and >= 0, default
|
|
59
|
+
/// (also for `nil`): 5e-5: training stops before `max_iter` once the
|
|
60
|
+
/// relative change of the objective stays below it for 3 iterations in
|
|
61
|
+
/// a row; 0 disables the test.
|
|
62
|
+
///
|
|
63
|
+
/// # Returns
|
|
64
|
+
///
|
|
65
|
+
/// `Ok(())` on success; checked before the files, an `ArgumentError` for a
|
|
66
|
+
/// wrong number of arguments, an unknown keyword or an unknown
|
|
67
|
+
/// `regularization`, and a `RangeError` for a `lambda`, `max_iter`,
|
|
68
|
+
/// `max_threads`, `elastic_net_l1_ratio` or `eta` outside its range; then an
|
|
69
|
+
/// `ArgumentError` for a missing input file, or a `RuntimeError` when
|
|
70
|
+
/// training fails.
|
|
71
|
+
fn train(ruby: &Ruby, args: &[Value]) -> Result<(), Error> {
|
|
72
|
+
let args = scan_args::<
|
|
73
|
+
(String, String, String, String, String, String, String),
|
|
74
|
+
(
|
|
75
|
+
Option<Option<f64>>,
|
|
76
|
+
Option<Option<Integer>>,
|
|
77
|
+
Option<Option<Integer>>,
|
|
78
|
+
),
|
|
79
|
+
(),
|
|
80
|
+
(),
|
|
81
|
+
RHash,
|
|
82
|
+
(),
|
|
83
|
+
>(args)?;
|
|
84
|
+
let (seed, corpus, char_def, unk_def, feature_def, rewrite_def, output) = args.required;
|
|
85
|
+
let (lambda, max_iter, max_threads) = args.optional;
|
|
86
|
+
let keywords = get_kwargs::<
|
|
87
|
+
_,
|
|
88
|
+
(),
|
|
89
|
+
(
|
|
90
|
+
Option<Option<String>>,
|
|
91
|
+
Option<Option<f64>>,
|
|
92
|
+
Option<Option<f64>>,
|
|
93
|
+
),
|
|
94
|
+
(),
|
|
95
|
+
>(
|
|
96
|
+
args.keywords,
|
|
97
|
+
&[],
|
|
98
|
+
&["regularization", "elastic_net_l1_ratio", "eta"],
|
|
99
|
+
)?;
|
|
100
|
+
let (regularization, elastic_net_l1_ratio, eta) = keywords.optional;
|
|
101
|
+
|
|
102
|
+
let range_error = |message: String| Error::new(ruby.exception_range_error(), message);
|
|
103
|
+
let regularization = match regularization.flatten() {
|
|
104
|
+
Some(name) => name.parse().map_err(|e: ParseRegularizationTypeError| {
|
|
105
|
+
Error::new(ruby.exception_arg_error(), e.to_string())
|
|
106
|
+
})?,
|
|
107
|
+
None => RegularizationType::default(),
|
|
108
|
+
};
|
|
109
|
+
let lambda_val =
|
|
110
|
+
non_negative_float_argument("lambda", lambda.flatten().unwrap_or(DEFAULT_LAMBDA))
|
|
111
|
+
.map_err(range_error)?;
|
|
112
|
+
let elastic_net_l1_ratio = unit_interval_argument(
|
|
113
|
+
"elastic_net_l1_ratio",
|
|
114
|
+
elastic_net_l1_ratio
|
|
115
|
+
.flatten()
|
|
116
|
+
.unwrap_or(DEFAULT_ELASTIC_NET_L1_RATIO),
|
|
117
|
+
)
|
|
118
|
+
.map_err(range_error)?;
|
|
119
|
+
let eta = non_negative_float_argument("eta", eta.flatten().unwrap_or(DEFAULT_ETA))
|
|
120
|
+
.map_err(range_error)?;
|
|
121
|
+
let max_iter: Option<Positive<u64>> =
|
|
122
|
+
optional_integer_arg(ruby, "max_iter", max_iter.flatten())?;
|
|
123
|
+
let max_threads: Option<Positive<usize>> =
|
|
124
|
+
optional_integer_arg(ruby, "max_threads", max_threads.flatten())?;
|
|
43
125
|
|
|
44
126
|
let seed_path = Path::new(&seed);
|
|
45
127
|
let corpus_path = Path::new(&corpus);
|
|
@@ -74,61 +156,83 @@ fn train(
|
|
|
74
156
|
feature_def_path,
|
|
75
157
|
rewrite_def_path,
|
|
76
158
|
)
|
|
77
|
-
.map_err(|e| to_magnus_error(
|
|
159
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to load trainer configuration: {e}")))?;
|
|
78
160
|
|
|
79
161
|
// Initialize trainer
|
|
80
|
-
let num_threads = max_threads.
|
|
81
|
-
let
|
|
82
|
-
let max_iter_val = max_iter.unwrap_or(100);
|
|
162
|
+
let num_threads = max_threads.map_or_else(num_cpus::get, |Positive(n)| n);
|
|
163
|
+
let max_iter_val = max_iter.map_or(100, |Positive(n)| n);
|
|
83
164
|
|
|
84
165
|
let trainer = Trainer::new(config)
|
|
85
|
-
.map_err(|e| to_magnus_error(
|
|
166
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to initialize trainer: {e}")))?
|
|
86
167
|
.regularization_cost(lambda_val)
|
|
168
|
+
.regularization_type(regularization, elastic_net_l1_ratio)
|
|
87
169
|
.max_iter(max_iter_val)
|
|
170
|
+
.eta(eta)
|
|
88
171
|
.num_threads(num_threads);
|
|
89
172
|
|
|
90
173
|
// Load corpus
|
|
91
174
|
let corpus_file = File::open(corpus_path)
|
|
92
|
-
.map_err(|e| to_magnus_error(
|
|
175
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to open corpus file: {e}")))?;
|
|
93
176
|
let corpus_data = Corpus::from_reader(corpus_file)
|
|
94
|
-
.map_err(|e| to_magnus_error(
|
|
177
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to load corpus: {e}")))?;
|
|
95
178
|
|
|
96
|
-
|
|
179
|
+
console::write_line(
|
|
180
|
+
Stream::Stdout,
|
|
181
|
+
format_args!("Training with {} examples...", corpus_data.len()),
|
|
182
|
+
);
|
|
97
183
|
|
|
98
184
|
// Train model
|
|
99
185
|
let model = trainer
|
|
100
186
|
.train(corpus_data)
|
|
101
|
-
.map_err(|e| to_magnus_error(
|
|
187
|
+
.map_err(|e| to_magnus_error(ruby, format!("Training failed: {e}")))?;
|
|
102
188
|
|
|
103
189
|
// Save model
|
|
104
190
|
if let Some(parent) = output_path.parent() {
|
|
105
191
|
std::fs::create_dir_all(parent).map_err(|e| {
|
|
106
|
-
to_magnus_error(
|
|
192
|
+
to_magnus_error(ruby, format!("Failed to create output directory: {e}"))
|
|
107
193
|
})?;
|
|
108
194
|
}
|
|
109
195
|
|
|
110
196
|
let mut output_file = File::create(output_path)
|
|
111
|
-
.map_err(|e| to_magnus_error(
|
|
197
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to create output file: {e}")))?;
|
|
112
198
|
|
|
113
199
|
model
|
|
114
200
|
.write_model(&mut output_file)
|
|
115
|
-
.map_err(|e| to_magnus_error(
|
|
201
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to write model: {e}")))?;
|
|
116
202
|
|
|
117
|
-
|
|
203
|
+
console::write_line(
|
|
204
|
+
Stream::Stdout,
|
|
205
|
+
format_args!("Model saved to {}", output_path.display()),
|
|
206
|
+
);
|
|
118
207
|
Ok(())
|
|
119
208
|
}
|
|
120
209
|
|
|
121
210
|
/// Export dictionary files from trained model.
|
|
122
211
|
///
|
|
212
|
+
/// Writes the files `lindera export` writes, with the same method
|
|
213
|
+
/// (`SerializableModel::export`): `lex.csv`, `matrix.def`, `unk.def`,
|
|
214
|
+
/// `char.def`, `feature.def`, `rewrite.def`, `left-id.def` and
|
|
215
|
+
/// `right-id.def`, and `metadata.json` when `metadata` is given.
|
|
216
|
+
///
|
|
123
217
|
/// # Arguments
|
|
124
218
|
///
|
|
219
|
+
/// * `ruby` - Ruby runtime handle.
|
|
125
220
|
/// * `model` - Trained model file path (.dat format).
|
|
126
|
-
/// * `output` - Output directory path
|
|
221
|
+
/// * `output` - Output directory path.
|
|
127
222
|
/// * `metadata` - Optional base metadata.json file to update with trained model values.
|
|
128
|
-
|
|
129
|
-
|
|
223
|
+
///
|
|
224
|
+
/// # Returns
|
|
225
|
+
///
|
|
226
|
+
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
227
|
+
fn export(
|
|
228
|
+
ruby: &Ruby,
|
|
229
|
+
model: String,
|
|
230
|
+
output: String,
|
|
231
|
+
metadata: Option<String>,
|
|
232
|
+
) -> Result<(), Error> {
|
|
130
233
|
let model_path = Path::new(&model);
|
|
131
234
|
let output_path = Path::new(&output);
|
|
235
|
+
let metadata_path = metadata.as_deref().map(Path::new);
|
|
132
236
|
|
|
133
237
|
if !model_path.exists() {
|
|
134
238
|
return Err(Error::new(
|
|
@@ -136,131 +240,41 @@ fn export(model: String, output: String, metadata: Option<String>) -> Result<(),
|
|
|
136
240
|
format!("Model file does not exist: {}", model_path.display()),
|
|
137
241
|
));
|
|
138
242
|
}
|
|
243
|
+
if let Some(metadata_path) = metadata_path
|
|
244
|
+
&& !metadata_path.exists()
|
|
245
|
+
{
|
|
246
|
+
return Err(Error::new(
|
|
247
|
+
ruby.exception_arg_error(),
|
|
248
|
+
format!("Metadata file does not exist: {}", metadata_path.display()),
|
|
249
|
+
));
|
|
250
|
+
}
|
|
139
251
|
|
|
140
252
|
// Load trained model
|
|
141
253
|
let model_file = File::open(model_path)
|
|
142
|
-
.map_err(|e| to_magnus_error(
|
|
254
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to open model file: {e}")))?;
|
|
143
255
|
|
|
144
256
|
let serializable_model: SerializableModel = Model::read_model(model_file)
|
|
145
|
-
.map_err(|e| to_magnus_error(
|
|
146
|
-
|
|
147
|
-
// Create output directory
|
|
148
|
-
std::fs::create_dir_all(output_path)
|
|
149
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create output directory: {e}")))?;
|
|
257
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to load model: {e}")))?;
|
|
150
258
|
|
|
151
259
|
// Export dictionary files
|
|
152
|
-
let
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
.write_lexicon(&mut lexicon_file)
|
|
162
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write lexicon: {e}")))?;
|
|
163
|
-
|
|
164
|
-
// Write connection matrix
|
|
165
|
-
let mut connector_file = File::create(&connector_path).map_err(|e| {
|
|
166
|
-
to_magnus_error(
|
|
167
|
-
&ruby,
|
|
168
|
-
format!("Failed to create connection matrix file: {e}"),
|
|
169
|
-
)
|
|
170
|
-
})?;
|
|
171
|
-
serializable_model
|
|
172
|
-
.write_connection_costs(&mut connector_file)
|
|
173
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write connection costs: {e}")))?;
|
|
174
|
-
|
|
175
|
-
// Write unknown word definitions
|
|
176
|
-
let mut unk_file = File::create(&unk_path)
|
|
177
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create unknown word file: {e}")))?;
|
|
178
|
-
serializable_model
|
|
179
|
-
.write_unknown_dictionary(&mut unk_file)
|
|
180
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write unknown dictionary: {e}")))?;
|
|
181
|
-
|
|
182
|
-
// Write character definition file
|
|
183
|
-
let mut char_def_file = File::create(&char_def_path).map_err(|e| {
|
|
184
|
-
to_magnus_error(
|
|
185
|
-
&ruby,
|
|
186
|
-
format!("Failed to create character definition file: {e}"),
|
|
187
|
-
)
|
|
188
|
-
})?;
|
|
189
|
-
|
|
190
|
-
use std::io::Write;
|
|
191
|
-
writeln!(
|
|
192
|
-
char_def_file,
|
|
193
|
-
"# Character definition file generated from trained model"
|
|
194
|
-
)
|
|
195
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
196
|
-
writeln!(char_def_file, "# Format: CATEGORY_NAME invoke group length")
|
|
197
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
198
|
-
writeln!(char_def_file, "DEFAULT 0 1 0")
|
|
199
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
200
|
-
writeln!(char_def_file, "HIRAGANA 1 1 0")
|
|
201
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
202
|
-
writeln!(char_def_file, "KATAKANA 1 1 0")
|
|
203
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
204
|
-
writeln!(char_def_file, "KANJI 0 0 2")
|
|
205
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
206
|
-
writeln!(char_def_file, "ALPHA 1 1 0")
|
|
207
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
208
|
-
writeln!(char_def_file, "NUMERIC 1 1 0")
|
|
209
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
210
|
-
writeln!(char_def_file)
|
|
211
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
212
|
-
|
|
213
|
-
writeln!(char_def_file, "# Character mappings")
|
|
214
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
215
|
-
writeln!(char_def_file, "0x3041..0x3096 HIRAGANA # Hiragana")
|
|
216
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
217
|
-
writeln!(char_def_file, "0x30A1..0x30F6 KATAKANA # Katakana")
|
|
218
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
219
|
-
writeln!(
|
|
220
|
-
char_def_file,
|
|
221
|
-
"0x4E00..0x9FAF KANJI # CJK Unified Ideographs"
|
|
222
|
-
)
|
|
223
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
224
|
-
writeln!(char_def_file, "0x0030..0x0039 NUMERIC # ASCII Digits")
|
|
225
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
226
|
-
writeln!(char_def_file, "0x0041..0x005A ALPHA # ASCII Uppercase")
|
|
227
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
228
|
-
writeln!(char_def_file, "0x0061..0x007A ALPHA # ASCII Lowercase")
|
|
229
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
230
|
-
|
|
231
|
-
let mut files_created = vec![
|
|
232
|
-
lexicon_path.clone(),
|
|
233
|
-
connector_path.clone(),
|
|
234
|
-
unk_path.clone(),
|
|
235
|
-
char_def_path.clone(),
|
|
236
|
-
];
|
|
237
|
-
|
|
238
|
-
// Handle metadata.json update if provided
|
|
239
|
-
if let Some(metadata_str) = metadata {
|
|
240
|
-
let metadata_path = Path::new(&metadata_str);
|
|
241
|
-
if !metadata_path.exists() {
|
|
242
|
-
return Err(Error::new(
|
|
243
|
-
ruby.exception_arg_error(),
|
|
244
|
-
format!("Metadata file does not exist: {}", metadata_path.display()),
|
|
245
|
-
));
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
let output_metadata_path = output_path.join("metadata.json");
|
|
249
|
-
let mut metadata_file = File::create(&output_metadata_path)
|
|
250
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create metadata file: {e}")))?;
|
|
251
|
-
|
|
252
|
-
serializable_model
|
|
253
|
-
.update_metadata_json(metadata_path, &mut metadata_file)
|
|
254
|
-
.map_err(|e| to_magnus_error(&ruby, format!("Failed to update metadata: {e}")))?;
|
|
255
|
-
|
|
256
|
-
files_created.push(output_metadata_path);
|
|
257
|
-
println!("Updated metadata.json with trained model values");
|
|
260
|
+
let files_created = serializable_model
|
|
261
|
+
.export(output_path, metadata_path)
|
|
262
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to export dictionary files: {e}")))?;
|
|
263
|
+
|
|
264
|
+
if metadata_path.is_some() {
|
|
265
|
+
console::write_line(
|
|
266
|
+
Stream::Stdout,
|
|
267
|
+
format_args!("Updated metadata.json with trained model values"),
|
|
268
|
+
);
|
|
258
269
|
}
|
|
259
270
|
|
|
260
|
-
|
|
261
|
-
|
|
271
|
+
console::write_line(
|
|
272
|
+
Stream::Stdout,
|
|
273
|
+
format_args!("Dictionary files exported to: {}", output_path.display()),
|
|
274
|
+
);
|
|
275
|
+
console::write_line(Stream::Stdout, format_args!("Files created:"));
|
|
262
276
|
for file in &files_created {
|
|
263
|
-
|
|
277
|
+
console::write_line(Stream::Stdout, format_args!(" - {}", file.display()));
|
|
264
278
|
}
|
|
265
279
|
|
|
266
280
|
Ok(())
|
|
@@ -277,7 +291,7 @@ fn export(model: String, output: String, metadata: Option<String>) -> Result<(),
|
|
|
277
291
|
///
|
|
278
292
|
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
279
293
|
pub fn define(_ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
280
|
-
module.define_module_function("train", function!(train,
|
|
294
|
+
module.define_module_function("train", function!(train, -1))?;
|
|
281
295
|
module.define_module_function("export", function!(export, 3))?;
|
|
282
296
|
Ok(())
|
|
283
297
|
}
|
data/src/util.rs
CHANGED
|
@@ -1,10 +1,73 @@
|
|
|
1
1
|
//! Utility functions for Ruby-Rust data conversion.
|
|
2
2
|
//!
|
|
3
3
|
//! This module provides helper functions for converting between Ruby objects
|
|
4
|
-
//! and Rust data structures, particularly for working with JSON-like data
|
|
4
|
+
//! and Rust data structures, particularly for working with JSON-like data,
|
|
5
|
+
//! and for checking integer arguments against the range of their setting.
|
|
5
6
|
|
|
7
|
+
use lindera_binding::argument::{
|
|
8
|
+
IntegerArgument, MAX_ARGUMENT_DEPTH, argument_too_deep_message, integer_argument_message,
|
|
9
|
+
};
|
|
6
10
|
use magnus::prelude::*;
|
|
7
|
-
use magnus::{Error, RArray, RHash, Ruby, TryConvert, Value};
|
|
11
|
+
use magnus::{Error, Integer, RArray, RHash, Ruby, TryConvert, Value};
|
|
12
|
+
|
|
13
|
+
/// Converts an `Integer` argument to the type the core stores it as.
|
|
14
|
+
///
|
|
15
|
+
/// A function that declared the core's type (`i16`, `usize`, ...) would have
|
|
16
|
+
/// magnus convert the argument, but magnus's `RangeError` names only the Rust
|
|
17
|
+
/// type (``fixnum too big to convert into `i16` ``), not the parameter or the
|
|
18
|
+
/// range (lindera/lindera#1216). Such a function takes an `Integer` instead,
|
|
19
|
+
/// which magnus converts as it does for the core's type (calling `to_int` on
|
|
20
|
+
/// any other value), and checks the range with this.
|
|
21
|
+
///
|
|
22
|
+
/// # Arguments
|
|
23
|
+
///
|
|
24
|
+
/// * `ruby` - Ruby runtime handle.
|
|
25
|
+
/// * `name` - The parameter's name, as the documentation names it.
|
|
26
|
+
/// * `value` - The value passed, of any size.
|
|
27
|
+
///
|
|
28
|
+
/// # Returns
|
|
29
|
+
///
|
|
30
|
+
/// The value as `T`, or a `RangeError` with the message of
|
|
31
|
+
/// [`integer_argument_message`] when `T` cannot hold the value.
|
|
32
|
+
pub fn integer_arg<T: IntegerArgument>(
|
|
33
|
+
ruby: &Ruby,
|
|
34
|
+
name: &str,
|
|
35
|
+
value: Integer,
|
|
36
|
+
) -> Result<T, Error> {
|
|
37
|
+
// An Integer beyond `i128` is outside every implementing type's range.
|
|
38
|
+
value
|
|
39
|
+
.to_i128()
|
|
40
|
+
.ok()
|
|
41
|
+
.and_then(|wide| T::try_from(wide).ok())
|
|
42
|
+
.ok_or_else(|| {
|
|
43
|
+
Error::new(
|
|
44
|
+
ruby.exception_range_error(),
|
|
45
|
+
integer_argument_message::<T>(name, value),
|
|
46
|
+
)
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/// Converts an optional `Integer` argument with [`integer_arg`].
|
|
51
|
+
///
|
|
52
|
+
/// # Arguments
|
|
53
|
+
///
|
|
54
|
+
/// * `ruby` - Ruby runtime handle.
|
|
55
|
+
/// * `name` - The parameter's name, as the documentation names it.
|
|
56
|
+
/// * `value` - The value passed, `None` for `nil`.
|
|
57
|
+
///
|
|
58
|
+
/// # Returns
|
|
59
|
+
///
|
|
60
|
+
/// `None` for `nil`, the value as `T` otherwise, or the `RangeError` of
|
|
61
|
+
/// [`integer_arg`].
|
|
62
|
+
pub fn optional_integer_arg<T: IntegerArgument>(
|
|
63
|
+
ruby: &Ruby,
|
|
64
|
+
name: &str,
|
|
65
|
+
value: Option<Integer>,
|
|
66
|
+
) -> Result<Option<T>, Error> {
|
|
67
|
+
value
|
|
68
|
+
.map(|value| integer_arg(ruby, name, value))
|
|
69
|
+
.transpose()
|
|
70
|
+
}
|
|
8
71
|
|
|
9
72
|
/// Converts a Ruby value to a serde_json::Value.
|
|
10
73
|
///
|
|
@@ -20,8 +83,26 @@ use magnus::{Error, RArray, RHash, Ruby, TryConvert, Value};
|
|
|
20
83
|
/// # Errors
|
|
21
84
|
///
|
|
22
85
|
/// Returns a `TypeError` if the Ruby value type is not supported, or if it
|
|
23
|
-
/// is a non-finite `Float` (NaN or Infinity), which JSON cannot represent
|
|
86
|
+
/// is a non-finite `Float` (NaN or Infinity), which JSON cannot represent,
|
|
87
|
+
/// and an `ArgumentError` if it nests arrays and hashes more than
|
|
88
|
+
/// [`MAX_ARGUMENT_DEPTH`] levels deep (as one that contains itself does).
|
|
24
89
|
pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value, Error> {
|
|
90
|
+
value_at_depth(ruby, value, 0)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/// Converts a Ruby value that sits inside `depth` arrays and hashes.
|
|
94
|
+
///
|
|
95
|
+
/// # Arguments
|
|
96
|
+
///
|
|
97
|
+
/// * `ruby` - Ruby runtime handle.
|
|
98
|
+
/// * `value` - Ruby value to convert.
|
|
99
|
+
/// * `depth` - The number of arrays and hashes around `value`.
|
|
100
|
+
///
|
|
101
|
+
/// # Returns
|
|
102
|
+
///
|
|
103
|
+
/// A `serde_json::Value` representing the Ruby value, or the errors of
|
|
104
|
+
/// [`rb_value_to_json`].
|
|
105
|
+
fn value_at_depth(ruby: &Ruby, value: Value, depth: usize) -> Result<serde_json::Value, Error> {
|
|
25
106
|
if value.is_nil() {
|
|
26
107
|
Ok(serde_json::Value::Null)
|
|
27
108
|
} else if value.is_kind_of(ruby.class_true_class())
|
|
@@ -68,9 +149,9 @@ pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value,
|
|
|
68
149
|
})?;
|
|
69
150
|
Ok(serde_json::Value::String(s))
|
|
70
151
|
} else if let Ok(arr) = RArray::try_convert(value) {
|
|
71
|
-
|
|
152
|
+
array_at_depth(ruby, arr, depth)
|
|
72
153
|
} else if let Ok(hash) = RHash::try_convert(value) {
|
|
73
|
-
|
|
154
|
+
hash_at_depth(ruby, hash, depth)
|
|
74
155
|
} else {
|
|
75
156
|
Err(Error::new(
|
|
76
157
|
ruby.exception_type_error(),
|
|
@@ -81,20 +162,49 @@ pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value,
|
|
|
81
162
|
}
|
|
82
163
|
}
|
|
83
164
|
|
|
84
|
-
///
|
|
165
|
+
/// Returns the depth of an array or hash that opens inside `depth` others.
|
|
166
|
+
///
|
|
167
|
+
/// # Arguments
|
|
168
|
+
///
|
|
169
|
+
/// * `ruby` - Ruby runtime handle.
|
|
170
|
+
/// * `depth` - The number of arrays and hashes around the new one.
|
|
171
|
+
///
|
|
172
|
+
/// # Returns
|
|
173
|
+
///
|
|
174
|
+
/// The new depth, or an `ArgumentError` when it exceeds
|
|
175
|
+
/// [`MAX_ARGUMENT_DEPTH`].
|
|
176
|
+
fn container_depth(ruby: &Ruby, depth: usize) -> Result<usize, Error> {
|
|
177
|
+
let depth = depth + 1;
|
|
178
|
+
if depth > MAX_ARGUMENT_DEPTH {
|
|
179
|
+
Err(Error::new(
|
|
180
|
+
ruby.exception_arg_error(),
|
|
181
|
+
argument_too_deep_message(),
|
|
182
|
+
))
|
|
183
|
+
} else {
|
|
184
|
+
Ok(depth)
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/// Converts a Ruby array that sits inside `depth` arrays and hashes.
|
|
189
|
+
///
|
|
190
|
+
/// The depth is counted here rather than in [`value_at_depth`] so that an
|
|
191
|
+
/// array obtained through `to_ary` counts too.
|
|
85
192
|
///
|
|
86
193
|
/// # Arguments
|
|
87
194
|
///
|
|
88
195
|
/// * `ruby` - Ruby runtime handle.
|
|
89
196
|
/// * `array` - Ruby array to convert.
|
|
197
|
+
/// * `depth` - The number of arrays and hashes around `array`.
|
|
90
198
|
///
|
|
91
199
|
/// # Returns
|
|
92
200
|
///
|
|
93
|
-
/// A `serde_json::Value` representing the array
|
|
94
|
-
|
|
201
|
+
/// A `serde_json::Value` representing the array, or the errors of
|
|
202
|
+
/// [`rb_value_to_json`].
|
|
203
|
+
fn array_at_depth(ruby: &Ruby, array: RArray, depth: usize) -> Result<serde_json::Value, Error> {
|
|
204
|
+
let depth = container_depth(ruby, depth)?;
|
|
95
205
|
let mut vec = Vec::new();
|
|
96
206
|
for item in array.into_iter() {
|
|
97
|
-
vec.push(
|
|
207
|
+
vec.push(value_at_depth(ruby, item, depth)?);
|
|
98
208
|
}
|
|
99
209
|
Ok(serde_json::Value::Array(vec))
|
|
100
210
|
}
|
|
@@ -108,11 +218,32 @@ fn rb_array_to_json(ruby: &Ruby, array: RArray) -> Result<serde_json::Value, Err
|
|
|
108
218
|
///
|
|
109
219
|
/// # Returns
|
|
110
220
|
///
|
|
111
|
-
/// A `serde_json::Value` representing the hash
|
|
221
|
+
/// A `serde_json::Value` representing the hash, or the errors of
|
|
222
|
+
/// [`rb_value_to_json`]; the hash itself is the first level.
|
|
112
223
|
pub fn rb_hash_to_json(ruby: &Ruby, hash: RHash) -> Result<serde_json::Value, Error> {
|
|
224
|
+
hash_at_depth(ruby, hash, 0)
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/// Converts a Ruby hash that sits inside `depth` arrays and hashes.
|
|
228
|
+
///
|
|
229
|
+
/// The depth is counted here rather than in [`value_at_depth`] so that a
|
|
230
|
+
/// hash obtained through `to_hash` counts too.
|
|
231
|
+
///
|
|
232
|
+
/// # Arguments
|
|
233
|
+
///
|
|
234
|
+
/// * `ruby` - Ruby runtime handle.
|
|
235
|
+
/// * `hash` - Ruby hash to convert.
|
|
236
|
+
/// * `depth` - The number of arrays and hashes around `hash`.
|
|
237
|
+
///
|
|
238
|
+
/// # Returns
|
|
239
|
+
///
|
|
240
|
+
/// A `serde_json::Value` representing the hash, or the errors of
|
|
241
|
+
/// [`rb_value_to_json`].
|
|
242
|
+
fn hash_at_depth(ruby: &Ruby, hash: RHash, depth: usize) -> Result<serde_json::Value, Error> {
|
|
243
|
+
let depth = container_depth(ruby, depth)?;
|
|
113
244
|
let mut map = serde_json::Map::new();
|
|
114
245
|
hash.foreach(|key: String, value: Value| {
|
|
115
|
-
let json_value =
|
|
246
|
+
let json_value = value_at_depth(ruby, value, depth)?;
|
|
116
247
|
map.insert(key, json_value);
|
|
117
248
|
Ok(magnus::r_hash::ForEach::Continue)
|
|
118
249
|
})?;
|