lindera 6.0.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +71 -37
- data/LICENSE +21 -0
- data/README.md +23 -6
- data/extconf.rb +13 -0
- data/src/character_filter.rs +3 -0
- data/src/dictionary.rs +239 -0
- data/src/error.rs +19 -0
- data/src/lib.rs +90 -0
- data/src/metadata.rs +423 -0
- data/src/mode.rs +401 -0
- data/src/schema.rs +568 -0
- data/src/token.rs +188 -0
- data/src/token_filter.rs +3 -0
- data/src/tokenizer.rs +373 -0
- data/src/trainer.rs +283 -0
- data/src/util.rs +168 -0
- metadata +17 -4
- data/ext/lindera_ruby/extconf.rb +0 -8
data/src/trainer.rs
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
//! Training functionality for custom morphological models.
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides functions for training custom morphological analysis models
|
|
4
|
+
//! from annotated corpora.
|
|
5
|
+
|
|
6
|
+
use std::fs::File;
|
|
7
|
+
use std::path::Path;
|
|
8
|
+
|
|
9
|
+
use magnus::{Error, Ruby, function};
|
|
10
|
+
|
|
11
|
+
use lindera::dictionary::trainer::{Corpus, Model, SerializableModel, Trainer, TrainerConfig};
|
|
12
|
+
|
|
13
|
+
use crate::error::to_magnus_error;
|
|
14
|
+
|
|
15
|
+
/// Trains a morphological analysis model from an annotated corpus.
|
|
16
|
+
///
|
|
17
|
+
/// # Arguments
|
|
18
|
+
///
|
|
19
|
+
/// * `seed` - Seed lexicon file path (CSV format).
|
|
20
|
+
/// * `corpus` - Training corpus file path (annotated text).
|
|
21
|
+
/// * `char_def` - Character definition file path (char.def).
|
|
22
|
+
/// * `unk_def` - Unknown word definition file path (unk.def).
|
|
23
|
+
/// * `feature_def` - Feature definition file path (feature.def).
|
|
24
|
+
/// * `rewrite_def` - Rewrite rule definition file path (rewrite.def).
|
|
25
|
+
/// * `output` - Output model file path.
|
|
26
|
+
/// * `lambda` - L1 regularization (0.0-1.0), default: 0.01.
|
|
27
|
+
/// * `max_iter` - Maximum number of iterations, default: 100.
|
|
28
|
+
/// * `max_threads` - Maximum number of threads (nil = auto-detect CPU cores).
|
|
29
|
+
#[allow(clippy::too_many_arguments)]
|
|
30
|
+
fn train(
|
|
31
|
+
seed: String,
|
|
32
|
+
corpus: String,
|
|
33
|
+
char_def: String,
|
|
34
|
+
unk_def: String,
|
|
35
|
+
feature_def: String,
|
|
36
|
+
rewrite_def: String,
|
|
37
|
+
output: String,
|
|
38
|
+
lambda: Option<f64>,
|
|
39
|
+
max_iter: Option<u64>,
|
|
40
|
+
max_threads: Option<usize>,
|
|
41
|
+
) -> Result<(), Error> {
|
|
42
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
43
|
+
|
|
44
|
+
let seed_path = Path::new(&seed);
|
|
45
|
+
let corpus_path = Path::new(&corpus);
|
|
46
|
+
let char_def_path = Path::new(&char_def);
|
|
47
|
+
let unk_def_path = Path::new(&unk_def);
|
|
48
|
+
let feature_def_path = Path::new(&feature_def);
|
|
49
|
+
let rewrite_def_path = Path::new(&rewrite_def);
|
|
50
|
+
let output_path = Path::new(&output);
|
|
51
|
+
|
|
52
|
+
// Validate input files
|
|
53
|
+
for (path, name) in [
|
|
54
|
+
(seed_path, "seed"),
|
|
55
|
+
(corpus_path, "corpus"),
|
|
56
|
+
(char_def_path, "char_def"),
|
|
57
|
+
(unk_def_path, "unk_def"),
|
|
58
|
+
(feature_def_path, "feature_def"),
|
|
59
|
+
(rewrite_def_path, "rewrite_def"),
|
|
60
|
+
] {
|
|
61
|
+
if !path.exists() {
|
|
62
|
+
return Err(Error::new(
|
|
63
|
+
ruby.exception_arg_error(),
|
|
64
|
+
format!("{} file does not exist: {}", name, path.display()),
|
|
65
|
+
));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Load configuration
|
|
70
|
+
let config = TrainerConfig::from_paths(
|
|
71
|
+
seed_path,
|
|
72
|
+
char_def_path,
|
|
73
|
+
unk_def_path,
|
|
74
|
+
feature_def_path,
|
|
75
|
+
rewrite_def_path,
|
|
76
|
+
)
|
|
77
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to load trainer configuration: {e}")))?;
|
|
78
|
+
|
|
79
|
+
// Initialize trainer
|
|
80
|
+
let num_threads = max_threads.unwrap_or_else(num_cpus::get);
|
|
81
|
+
let lambda_val = lambda.unwrap_or(0.01);
|
|
82
|
+
let max_iter_val = max_iter.unwrap_or(100);
|
|
83
|
+
|
|
84
|
+
let trainer = Trainer::new(config)
|
|
85
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to initialize trainer: {e}")))?
|
|
86
|
+
.regularization_cost(lambda_val)
|
|
87
|
+
.max_iter(max_iter_val)
|
|
88
|
+
.num_threads(num_threads);
|
|
89
|
+
|
|
90
|
+
// Load corpus
|
|
91
|
+
let corpus_file = File::open(corpus_path)
|
|
92
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to open corpus file: {e}")))?;
|
|
93
|
+
let corpus_data = Corpus::from_reader(corpus_file)
|
|
94
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to load corpus: {e}")))?;
|
|
95
|
+
|
|
96
|
+
println!("Training with {} examples...", corpus_data.len());
|
|
97
|
+
|
|
98
|
+
// Train model
|
|
99
|
+
let model = trainer
|
|
100
|
+
.train(corpus_data)
|
|
101
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Training failed: {e}")))?;
|
|
102
|
+
|
|
103
|
+
// Save model
|
|
104
|
+
if let Some(parent) = output_path.parent() {
|
|
105
|
+
std::fs::create_dir_all(parent).map_err(|e| {
|
|
106
|
+
to_magnus_error(&ruby, format!("Failed to create output directory: {e}"))
|
|
107
|
+
})?;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
let mut output_file = File::create(output_path)
|
|
111
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create output file: {e}")))?;
|
|
112
|
+
|
|
113
|
+
model
|
|
114
|
+
.write_model(&mut output_file)
|
|
115
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write model: {e}")))?;
|
|
116
|
+
|
|
117
|
+
println!("Model saved to {}", output_path.display());
|
|
118
|
+
Ok(())
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/// Export dictionary files from trained model.
|
|
122
|
+
///
|
|
123
|
+
/// # Arguments
|
|
124
|
+
///
|
|
125
|
+
/// * `model` - Trained model file path (.dat format).
|
|
126
|
+
/// * `output` - Output directory path (creates lex.csv, matrix.def, unk.def, char.def).
|
|
127
|
+
/// * `metadata` - Optional base metadata.json file to update with trained model values.
|
|
128
|
+
fn export(model: String, output: String, metadata: Option<String>) -> Result<(), Error> {
|
|
129
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
130
|
+
let model_path = Path::new(&model);
|
|
131
|
+
let output_path = Path::new(&output);
|
|
132
|
+
|
|
133
|
+
if !model_path.exists() {
|
|
134
|
+
return Err(Error::new(
|
|
135
|
+
ruby.exception_arg_error(),
|
|
136
|
+
format!("Model file does not exist: {}", model_path.display()),
|
|
137
|
+
));
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Load trained model
|
|
141
|
+
let model_file = File::open(model_path)
|
|
142
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to open model file: {e}")))?;
|
|
143
|
+
|
|
144
|
+
let serializable_model: SerializableModel = Model::read_model(model_file)
|
|
145
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to load model: {e}")))?;
|
|
146
|
+
|
|
147
|
+
// Create output directory
|
|
148
|
+
std::fs::create_dir_all(output_path)
|
|
149
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create output directory: {e}")))?;
|
|
150
|
+
|
|
151
|
+
// Export dictionary files
|
|
152
|
+
let lexicon_path = output_path.join("lex.csv");
|
|
153
|
+
let connector_path = output_path.join("matrix.def");
|
|
154
|
+
let unk_path = output_path.join("unk.def");
|
|
155
|
+
let char_def_path = output_path.join("char.def");
|
|
156
|
+
|
|
157
|
+
// Write lexicon file
|
|
158
|
+
let mut lexicon_file = File::create(&lexicon_path)
|
|
159
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create lexicon file: {e}")))?;
|
|
160
|
+
serializable_model
|
|
161
|
+
.write_lexicon(&mut lexicon_file)
|
|
162
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write lexicon: {e}")))?;
|
|
163
|
+
|
|
164
|
+
// Write connection matrix
|
|
165
|
+
let mut connector_file = File::create(&connector_path).map_err(|e| {
|
|
166
|
+
to_magnus_error(
|
|
167
|
+
&ruby,
|
|
168
|
+
format!("Failed to create connection matrix file: {e}"),
|
|
169
|
+
)
|
|
170
|
+
})?;
|
|
171
|
+
serializable_model
|
|
172
|
+
.write_connection_costs(&mut connector_file)
|
|
173
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write connection costs: {e}")))?;
|
|
174
|
+
|
|
175
|
+
// Write unknown word definitions
|
|
176
|
+
let mut unk_file = File::create(&unk_path)
|
|
177
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create unknown word file: {e}")))?;
|
|
178
|
+
serializable_model
|
|
179
|
+
.write_unknown_dictionary(&mut unk_file)
|
|
180
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write unknown dictionary: {e}")))?;
|
|
181
|
+
|
|
182
|
+
// Write character definition file
|
|
183
|
+
let mut char_def_file = File::create(&char_def_path).map_err(|e| {
|
|
184
|
+
to_magnus_error(
|
|
185
|
+
&ruby,
|
|
186
|
+
format!("Failed to create character definition file: {e}"),
|
|
187
|
+
)
|
|
188
|
+
})?;
|
|
189
|
+
|
|
190
|
+
use std::io::Write;
|
|
191
|
+
writeln!(
|
|
192
|
+
char_def_file,
|
|
193
|
+
"# Character definition file generated from trained model"
|
|
194
|
+
)
|
|
195
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
196
|
+
writeln!(char_def_file, "# Format: CATEGORY_NAME invoke group length")
|
|
197
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
198
|
+
writeln!(char_def_file, "DEFAULT 0 1 0")
|
|
199
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
200
|
+
writeln!(char_def_file, "HIRAGANA 1 1 0")
|
|
201
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
202
|
+
writeln!(char_def_file, "KATAKANA 1 1 0")
|
|
203
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
204
|
+
writeln!(char_def_file, "KANJI 0 0 2")
|
|
205
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
206
|
+
writeln!(char_def_file, "ALPHA 1 1 0")
|
|
207
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
208
|
+
writeln!(char_def_file, "NUMERIC 1 1 0")
|
|
209
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
210
|
+
writeln!(char_def_file)
|
|
211
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
212
|
+
|
|
213
|
+
writeln!(char_def_file, "# Character mappings")
|
|
214
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
215
|
+
writeln!(char_def_file, "0x3041..0x3096 HIRAGANA # Hiragana")
|
|
216
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
217
|
+
writeln!(char_def_file, "0x30A1..0x30F6 KATAKANA # Katakana")
|
|
218
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
219
|
+
writeln!(
|
|
220
|
+
char_def_file,
|
|
221
|
+
"0x4E00..0x9FAF KANJI # CJK Unified Ideographs"
|
|
222
|
+
)
|
|
223
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
224
|
+
writeln!(char_def_file, "0x0030..0x0039 NUMERIC # ASCII Digits")
|
|
225
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
226
|
+
writeln!(char_def_file, "0x0041..0x005A ALPHA # ASCII Uppercase")
|
|
227
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
228
|
+
writeln!(char_def_file, "0x0061..0x007A ALPHA # ASCII Lowercase")
|
|
229
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
|
|
230
|
+
|
|
231
|
+
let mut files_created = vec![
|
|
232
|
+
lexicon_path.clone(),
|
|
233
|
+
connector_path.clone(),
|
|
234
|
+
unk_path.clone(),
|
|
235
|
+
char_def_path.clone(),
|
|
236
|
+
];
|
|
237
|
+
|
|
238
|
+
// Handle metadata.json update if provided
|
|
239
|
+
if let Some(metadata_str) = metadata {
|
|
240
|
+
let metadata_path = Path::new(&metadata_str);
|
|
241
|
+
if !metadata_path.exists() {
|
|
242
|
+
return Err(Error::new(
|
|
243
|
+
ruby.exception_arg_error(),
|
|
244
|
+
format!("Metadata file does not exist: {}", metadata_path.display()),
|
|
245
|
+
));
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
let output_metadata_path = output_path.join("metadata.json");
|
|
249
|
+
let mut metadata_file = File::create(&output_metadata_path)
|
|
250
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to create metadata file: {e}")))?;
|
|
251
|
+
|
|
252
|
+
serializable_model
|
|
253
|
+
.update_metadata_json(metadata_path, &mut metadata_file)
|
|
254
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to update metadata: {e}")))?;
|
|
255
|
+
|
|
256
|
+
files_created.push(output_metadata_path);
|
|
257
|
+
println!("Updated metadata.json with trained model values");
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
println!("Dictionary files exported to: {}", output_path.display());
|
|
261
|
+
println!("Files created:");
|
|
262
|
+
for file in &files_created {
|
|
263
|
+
println!(" - {}", file.display());
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
Ok(())
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/// Defines train and export module functions in the given Ruby module.
|
|
270
|
+
///
|
|
271
|
+
/// # Arguments
|
|
272
|
+
///
|
|
273
|
+
/// * `_ruby` - Ruby runtime handle.
|
|
274
|
+
/// * `module` - Parent Ruby module.
|
|
275
|
+
///
|
|
276
|
+
/// # Returns
|
|
277
|
+
///
|
|
278
|
+
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
279
|
+
pub fn define(_ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
280
|
+
module.define_module_function("train", function!(train, 10))?;
|
|
281
|
+
module.define_module_function("export", function!(export, 3))?;
|
|
282
|
+
Ok(())
|
|
283
|
+
}
|
data/src/util.rs
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
//! Utility functions for Ruby-Rust data conversion.
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides helper functions for converting between Ruby objects
|
|
4
|
+
//! and Rust data structures, particularly for working with JSON-like data.
|
|
5
|
+
|
|
6
|
+
use magnus::prelude::*;
|
|
7
|
+
use magnus::{Error, RArray, RHash, Ruby, TryConvert, Value};
|
|
8
|
+
|
|
9
|
+
/// Converts a Ruby value to a serde_json::Value.
|
|
10
|
+
///
|
|
11
|
+
/// # Arguments
|
|
12
|
+
///
|
|
13
|
+
/// * `ruby` - Ruby runtime handle.
|
|
14
|
+
/// * `value` - Ruby value to convert.
|
|
15
|
+
///
|
|
16
|
+
/// # Returns
|
|
17
|
+
///
|
|
18
|
+
/// A `serde_json::Value` representing the Ruby value.
|
|
19
|
+
///
|
|
20
|
+
/// # Errors
|
|
21
|
+
///
|
|
22
|
+
/// Returns a `TypeError` if the Ruby value type is not supported, or if it
|
|
23
|
+
/// is a non-finite `Float` (NaN or Infinity), which JSON cannot represent.
|
|
24
|
+
pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value, Error> {
|
|
25
|
+
if value.is_nil() {
|
|
26
|
+
Ok(serde_json::Value::Null)
|
|
27
|
+
} else if value.is_kind_of(ruby.class_true_class())
|
|
28
|
+
|| value.is_kind_of(ruby.class_false_class())
|
|
29
|
+
{
|
|
30
|
+
let b: bool = TryConvert::try_convert(value).map_err(|e| {
|
|
31
|
+
Error::new(
|
|
32
|
+
ruby.exception_type_error(),
|
|
33
|
+
format!("Failed to convert boolean: {e}"),
|
|
34
|
+
)
|
|
35
|
+
})?;
|
|
36
|
+
Ok(serde_json::Value::Bool(b))
|
|
37
|
+
} else if value.is_kind_of(ruby.class_integer()) {
|
|
38
|
+
let i: i64 = TryConvert::try_convert(value).map_err(|e| {
|
|
39
|
+
Error::new(
|
|
40
|
+
ruby.exception_type_error(),
|
|
41
|
+
format!("Failed to convert integer: {e}"),
|
|
42
|
+
)
|
|
43
|
+
})?;
|
|
44
|
+
Ok(serde_json::Value::from(i))
|
|
45
|
+
} else if value.is_kind_of(ruby.class_float()) {
|
|
46
|
+
let f: f64 = TryConvert::try_convert(value).map_err(|e| {
|
|
47
|
+
Error::new(
|
|
48
|
+
ruby.exception_type_error(),
|
|
49
|
+
format!("Failed to convert float: {e}"),
|
|
50
|
+
)
|
|
51
|
+
})?;
|
|
52
|
+
// JSON has no NaN or Infinity; reject them rather than let them
|
|
53
|
+
// silently turn into `null`, which callers read as "not set".
|
|
54
|
+
serde_json::Number::from_f64(f)
|
|
55
|
+
.map(serde_json::Value::Number)
|
|
56
|
+
.ok_or_else(|| {
|
|
57
|
+
Error::new(
|
|
58
|
+
ruby.exception_type_error(),
|
|
59
|
+
format!("Unsupported non-finite float: {f}"),
|
|
60
|
+
)
|
|
61
|
+
})
|
|
62
|
+
} else if value.is_kind_of(ruby.class_string()) || value.is_kind_of(ruby.class_symbol()) {
|
|
63
|
+
let s: String = TryConvert::try_convert(value).map_err(|e| {
|
|
64
|
+
Error::new(
|
|
65
|
+
ruby.exception_type_error(),
|
|
66
|
+
format!("Failed to convert string: {e}"),
|
|
67
|
+
)
|
|
68
|
+
})?;
|
|
69
|
+
Ok(serde_json::Value::String(s))
|
|
70
|
+
} else if let Ok(arr) = RArray::try_convert(value) {
|
|
71
|
+
rb_array_to_json(ruby, arr)
|
|
72
|
+
} else if let Ok(hash) = RHash::try_convert(value) {
|
|
73
|
+
rb_hash_to_json(ruby, hash)
|
|
74
|
+
} else {
|
|
75
|
+
Err(Error::new(
|
|
76
|
+
ruby.exception_type_error(),
|
|
77
|
+
format!("Unsupported Ruby object type: {}", unsafe {
|
|
78
|
+
value.classname()
|
|
79
|
+
}),
|
|
80
|
+
))
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/// Converts a Ruby array to a serde_json::Value::Array.
|
|
85
|
+
///
|
|
86
|
+
/// # Arguments
|
|
87
|
+
///
|
|
88
|
+
/// * `ruby` - Ruby runtime handle.
|
|
89
|
+
/// * `array` - Ruby array to convert.
|
|
90
|
+
///
|
|
91
|
+
/// # Returns
|
|
92
|
+
///
|
|
93
|
+
/// A `serde_json::Value` representing the array.
|
|
94
|
+
fn rb_array_to_json(ruby: &Ruby, array: RArray) -> Result<serde_json::Value, Error> {
|
|
95
|
+
let mut vec = Vec::new();
|
|
96
|
+
for item in array.into_iter() {
|
|
97
|
+
vec.push(rb_value_to_json(ruby, item)?);
|
|
98
|
+
}
|
|
99
|
+
Ok(serde_json::Value::Array(vec))
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/// Converts a Ruby hash to a serde_json::Value::Object.
|
|
103
|
+
///
|
|
104
|
+
/// # Arguments
|
|
105
|
+
///
|
|
106
|
+
/// * `ruby` - Ruby runtime handle.
|
|
107
|
+
/// * `hash` - Ruby hash to convert.
|
|
108
|
+
///
|
|
109
|
+
/// # Returns
|
|
110
|
+
///
|
|
111
|
+
/// A `serde_json::Value` representing the hash.
|
|
112
|
+
pub fn rb_hash_to_json(ruby: &Ruby, hash: RHash) -> Result<serde_json::Value, Error> {
|
|
113
|
+
let mut map = serde_json::Map::new();
|
|
114
|
+
hash.foreach(|key: String, value: Value| {
|
|
115
|
+
let json_value = rb_value_to_json(ruby, value)?;
|
|
116
|
+
map.insert(key, json_value);
|
|
117
|
+
Ok(magnus::r_hash::ForEach::Continue)
|
|
118
|
+
})?;
|
|
119
|
+
Ok(serde_json::Value::Object(map))
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/// Converts a serde_json::Value to a Ruby value.
|
|
123
|
+
///
|
|
124
|
+
/// # Arguments
|
|
125
|
+
///
|
|
126
|
+
/// * `ruby` - Ruby runtime handle.
|
|
127
|
+
/// * `value` - JSON value to convert.
|
|
128
|
+
///
|
|
129
|
+
/// # Returns
|
|
130
|
+
///
|
|
131
|
+
/// A Ruby `Value` representing the JSON value.
|
|
132
|
+
pub fn json_to_rb_value(ruby: &Ruby, value: &serde_json::Value) -> Result<Value, Error> {
|
|
133
|
+
match value {
|
|
134
|
+
serde_json::Value::Null => Ok(ruby.qnil().as_value()),
|
|
135
|
+
serde_json::Value::Bool(b) => Ok(if *b {
|
|
136
|
+
ruby.qtrue().as_value()
|
|
137
|
+
} else {
|
|
138
|
+
ruby.qfalse().as_value()
|
|
139
|
+
}),
|
|
140
|
+
serde_json::Value::Number(n) => {
|
|
141
|
+
if let Some(i) = n.as_i64() {
|
|
142
|
+
Ok(ruby.integer_from_i64(i).as_value())
|
|
143
|
+
} else if let Some(f) = n.as_f64() {
|
|
144
|
+
Ok(ruby.float_from_f64(f).as_value())
|
|
145
|
+
} else {
|
|
146
|
+
Err(Error::new(
|
|
147
|
+
ruby.exception_type_error(),
|
|
148
|
+
"Unsupported number type",
|
|
149
|
+
))
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
serde_json::Value::String(s) => Ok(ruby.str_new(s).as_value()),
|
|
153
|
+
serde_json::Value::Array(arr) => {
|
|
154
|
+
let rb_arr = ruby.ary_new_capa(arr.len());
|
|
155
|
+
for item in arr {
|
|
156
|
+
rb_arr.push(json_to_rb_value(ruby, item)?)?;
|
|
157
|
+
}
|
|
158
|
+
Ok(rb_arr.as_value())
|
|
159
|
+
}
|
|
160
|
+
serde_json::Value::Object(obj) => {
|
|
161
|
+
let rb_hash = ruby.hash_new();
|
|
162
|
+
for (key, val) in obj {
|
|
163
|
+
rb_hash.aset(ruby.str_new(key), json_to_rb_value(ruby, val)?)?;
|
|
164
|
+
}
|
|
165
|
+
Ok(rb_hash.as_value())
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: lindera
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 6.
|
|
4
|
+
version: 6.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Lindera contributors
|
|
@@ -69,13 +69,26 @@ description: Ruby bindings for Lindera, a morphological analysis library for CJK
|
|
|
69
69
|
(Japanese, Korean, Chinese).
|
|
70
70
|
executables: []
|
|
71
71
|
extensions:
|
|
72
|
-
-
|
|
72
|
+
- extconf.rb
|
|
73
73
|
extra_rdoc_files: []
|
|
74
74
|
files:
|
|
75
75
|
- Cargo.toml
|
|
76
|
+
- LICENSE
|
|
76
77
|
- README.md
|
|
77
|
-
-
|
|
78
|
+
- extconf.rb
|
|
78
79
|
- lib/lindera.rb
|
|
80
|
+
- src/character_filter.rs
|
|
81
|
+
- src/dictionary.rs
|
|
82
|
+
- src/error.rs
|
|
83
|
+
- src/lib.rs
|
|
84
|
+
- src/metadata.rs
|
|
85
|
+
- src/mode.rs
|
|
86
|
+
- src/schema.rs
|
|
87
|
+
- src/token.rs
|
|
88
|
+
- src/token_filter.rs
|
|
89
|
+
- src/tokenizer.rs
|
|
90
|
+
- src/trainer.rs
|
|
91
|
+
- src/util.rs
|
|
79
92
|
homepage: https://github.com/lindera/lindera
|
|
80
93
|
licenses:
|
|
81
94
|
- MIT
|
|
@@ -94,7 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
94
107
|
- !ruby/object:Gem::Version
|
|
95
108
|
version: '0'
|
|
96
109
|
requirements: []
|
|
97
|
-
rubygems_version: 4.0.
|
|
110
|
+
rubygems_version: 4.0.20
|
|
98
111
|
specification_version: 4
|
|
99
112
|
summary: Ruby bindings for Lindera morphological analysis engine
|
|
100
113
|
test_files: []
|
data/ext/lindera_ruby/extconf.rb
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
require "rb_sys/mkmf"
|
|
2
|
-
|
|
3
|
-
create_rust_makefile("lindera/lindera_ruby") do |r|
|
|
4
|
-
# Pass embed features via LINDERA_FEATURES environment variable
|
|
5
|
-
# e.g., LINDERA_FEATURES="embed-ipadic" bundle exec rake compile
|
|
6
|
-
features = ENV.fetch("LINDERA_FEATURES", "").split(",").map(&:strip).reject(&:empty?)
|
|
7
|
-
r.features = features unless features.empty?
|
|
8
|
-
end
|