lindera 6.0.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/trainer.rs ADDED
@@ -0,0 +1,283 @@
1
+ //! Training functionality for custom morphological models.
2
+ //!
3
+ //! This module provides functions for training custom morphological analysis models
4
+ //! from annotated corpora.
5
+
6
+ use std::fs::File;
7
+ use std::path::Path;
8
+
9
+ use magnus::{Error, Ruby, function};
10
+
11
+ use lindera::dictionary::trainer::{Corpus, Model, SerializableModel, Trainer, TrainerConfig};
12
+
13
+ use crate::error::to_magnus_error;
14
+
15
+ /// Trains a morphological analysis model from an annotated corpus.
16
+ ///
17
+ /// # Arguments
18
+ ///
19
+ /// * `seed` - Seed lexicon file path (CSV format).
20
+ /// * `corpus` - Training corpus file path (annotated text).
21
+ /// * `char_def` - Character definition file path (char.def).
22
+ /// * `unk_def` - Unknown word definition file path (unk.def).
23
+ /// * `feature_def` - Feature definition file path (feature.def).
24
+ /// * `rewrite_def` - Rewrite rule definition file path (rewrite.def).
25
+ /// * `output` - Output model file path.
26
+ /// * `lambda` - L1 regularization (0.0-1.0), default: 0.01.
27
+ /// * `max_iter` - Maximum number of iterations, default: 100.
28
+ /// * `max_threads` - Maximum number of threads (nil = auto-detect CPU cores).
29
+ #[allow(clippy::too_many_arguments)]
30
+ fn train(
31
+ seed: String,
32
+ corpus: String,
33
+ char_def: String,
34
+ unk_def: String,
35
+ feature_def: String,
36
+ rewrite_def: String,
37
+ output: String,
38
+ lambda: Option<f64>,
39
+ max_iter: Option<u64>,
40
+ max_threads: Option<usize>,
41
+ ) -> Result<(), Error> {
42
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
43
+
44
+ let seed_path = Path::new(&seed);
45
+ let corpus_path = Path::new(&corpus);
46
+ let char_def_path = Path::new(&char_def);
47
+ let unk_def_path = Path::new(&unk_def);
48
+ let feature_def_path = Path::new(&feature_def);
49
+ let rewrite_def_path = Path::new(&rewrite_def);
50
+ let output_path = Path::new(&output);
51
+
52
+ // Validate input files
53
+ for (path, name) in [
54
+ (seed_path, "seed"),
55
+ (corpus_path, "corpus"),
56
+ (char_def_path, "char_def"),
57
+ (unk_def_path, "unk_def"),
58
+ (feature_def_path, "feature_def"),
59
+ (rewrite_def_path, "rewrite_def"),
60
+ ] {
61
+ if !path.exists() {
62
+ return Err(Error::new(
63
+ ruby.exception_arg_error(),
64
+ format!("{} file does not exist: {}", name, path.display()),
65
+ ));
66
+ }
67
+ }
68
+
69
+ // Load configuration
70
+ let config = TrainerConfig::from_paths(
71
+ seed_path,
72
+ char_def_path,
73
+ unk_def_path,
74
+ feature_def_path,
75
+ rewrite_def_path,
76
+ )
77
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to load trainer configuration: {e}")))?;
78
+
79
+ // Initialize trainer
80
+ let num_threads = max_threads.unwrap_or_else(num_cpus::get);
81
+ let lambda_val = lambda.unwrap_or(0.01);
82
+ let max_iter_val = max_iter.unwrap_or(100);
83
+
84
+ let trainer = Trainer::new(config)
85
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to initialize trainer: {e}")))?
86
+ .regularization_cost(lambda_val)
87
+ .max_iter(max_iter_val)
88
+ .num_threads(num_threads);
89
+
90
+ // Load corpus
91
+ let corpus_file = File::open(corpus_path)
92
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to open corpus file: {e}")))?;
93
+ let corpus_data = Corpus::from_reader(corpus_file)
94
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to load corpus: {e}")))?;
95
+
96
+ println!("Training with {} examples...", corpus_data.len());
97
+
98
+ // Train model
99
+ let model = trainer
100
+ .train(corpus_data)
101
+ .map_err(|e| to_magnus_error(&ruby, format!("Training failed: {e}")))?;
102
+
103
+ // Save model
104
+ if let Some(parent) = output_path.parent() {
105
+ std::fs::create_dir_all(parent).map_err(|e| {
106
+ to_magnus_error(&ruby, format!("Failed to create output directory: {e}"))
107
+ })?;
108
+ }
109
+
110
+ let mut output_file = File::create(output_path)
111
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to create output file: {e}")))?;
112
+
113
+ model
114
+ .write_model(&mut output_file)
115
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write model: {e}")))?;
116
+
117
+ println!("Model saved to {}", output_path.display());
118
+ Ok(())
119
+ }
120
+
121
+ /// Export dictionary files from trained model.
122
+ ///
123
+ /// # Arguments
124
+ ///
125
+ /// * `model` - Trained model file path (.dat format).
126
+ /// * `output` - Output directory path (creates lex.csv, matrix.def, unk.def, char.def).
127
+ /// * `metadata` - Optional base metadata.json file to update with trained model values.
128
+ fn export(model: String, output: String, metadata: Option<String>) -> Result<(), Error> {
129
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
130
+ let model_path = Path::new(&model);
131
+ let output_path = Path::new(&output);
132
+
133
+ if !model_path.exists() {
134
+ return Err(Error::new(
135
+ ruby.exception_arg_error(),
136
+ format!("Model file does not exist: {}", model_path.display()),
137
+ ));
138
+ }
139
+
140
+ // Load trained model
141
+ let model_file = File::open(model_path)
142
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to open model file: {e}")))?;
143
+
144
+ let serializable_model: SerializableModel = Model::read_model(model_file)
145
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to load model: {e}")))?;
146
+
147
+ // Create output directory
148
+ std::fs::create_dir_all(output_path)
149
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to create output directory: {e}")))?;
150
+
151
+ // Export dictionary files
152
+ let lexicon_path = output_path.join("lex.csv");
153
+ let connector_path = output_path.join("matrix.def");
154
+ let unk_path = output_path.join("unk.def");
155
+ let char_def_path = output_path.join("char.def");
156
+
157
+ // Write lexicon file
158
+ let mut lexicon_file = File::create(&lexicon_path)
159
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to create lexicon file: {e}")))?;
160
+ serializable_model
161
+ .write_lexicon(&mut lexicon_file)
162
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write lexicon: {e}")))?;
163
+
164
+ // Write connection matrix
165
+ let mut connector_file = File::create(&connector_path).map_err(|e| {
166
+ to_magnus_error(
167
+ &ruby,
168
+ format!("Failed to create connection matrix file: {e}"),
169
+ )
170
+ })?;
171
+ serializable_model
172
+ .write_connection_costs(&mut connector_file)
173
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write connection costs: {e}")))?;
174
+
175
+ // Write unknown word definitions
176
+ let mut unk_file = File::create(&unk_path)
177
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to create unknown word file: {e}")))?;
178
+ serializable_model
179
+ .write_unknown_dictionary(&mut unk_file)
180
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write unknown dictionary: {e}")))?;
181
+
182
+ // Write character definition file
183
+ let mut char_def_file = File::create(&char_def_path).map_err(|e| {
184
+ to_magnus_error(
185
+ &ruby,
186
+ format!("Failed to create character definition file: {e}"),
187
+ )
188
+ })?;
189
+
190
+ use std::io::Write;
191
+ writeln!(
192
+ char_def_file,
193
+ "# Character definition file generated from trained model"
194
+ )
195
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
196
+ writeln!(char_def_file, "# Format: CATEGORY_NAME invoke group length")
197
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
198
+ writeln!(char_def_file, "DEFAULT 0 1 0")
199
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
200
+ writeln!(char_def_file, "HIRAGANA 1 1 0")
201
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
202
+ writeln!(char_def_file, "KATAKANA 1 1 0")
203
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
204
+ writeln!(char_def_file, "KANJI 0 0 2")
205
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
206
+ writeln!(char_def_file, "ALPHA 1 1 0")
207
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
208
+ writeln!(char_def_file, "NUMERIC 1 1 0")
209
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
210
+ writeln!(char_def_file)
211
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
212
+
213
+ writeln!(char_def_file, "# Character mappings")
214
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
215
+ writeln!(char_def_file, "0x3041..0x3096 HIRAGANA # Hiragana")
216
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
217
+ writeln!(char_def_file, "0x30A1..0x30F6 KATAKANA # Katakana")
218
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
219
+ writeln!(
220
+ char_def_file,
221
+ "0x4E00..0x9FAF KANJI # CJK Unified Ideographs"
222
+ )
223
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
224
+ writeln!(char_def_file, "0x0030..0x0039 NUMERIC # ASCII Digits")
225
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
226
+ writeln!(char_def_file, "0x0041..0x005A ALPHA # ASCII Uppercase")
227
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
228
+ writeln!(char_def_file, "0x0061..0x007A ALPHA # ASCII Lowercase")
229
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to write char.def: {e}")))?;
230
+
231
+ let mut files_created = vec![
232
+ lexicon_path.clone(),
233
+ connector_path.clone(),
234
+ unk_path.clone(),
235
+ char_def_path.clone(),
236
+ ];
237
+
238
+ // Handle metadata.json update if provided
239
+ if let Some(metadata_str) = metadata {
240
+ let metadata_path = Path::new(&metadata_str);
241
+ if !metadata_path.exists() {
242
+ return Err(Error::new(
243
+ ruby.exception_arg_error(),
244
+ format!("Metadata file does not exist: {}", metadata_path.display()),
245
+ ));
246
+ }
247
+
248
+ let output_metadata_path = output_path.join("metadata.json");
249
+ let mut metadata_file = File::create(&output_metadata_path)
250
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to create metadata file: {e}")))?;
251
+
252
+ serializable_model
253
+ .update_metadata_json(metadata_path, &mut metadata_file)
254
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to update metadata: {e}")))?;
255
+
256
+ files_created.push(output_metadata_path);
257
+ println!("Updated metadata.json with trained model values");
258
+ }
259
+
260
+ println!("Dictionary files exported to: {}", output_path.display());
261
+ println!("Files created:");
262
+ for file in &files_created {
263
+ println!(" - {}", file.display());
264
+ }
265
+
266
+ Ok(())
267
+ }
268
+
269
+ /// Defines train and export module functions in the given Ruby module.
270
+ ///
271
+ /// # Arguments
272
+ ///
273
+ /// * `_ruby` - Ruby runtime handle.
274
+ /// * `module` - Parent Ruby module.
275
+ ///
276
+ /// # Returns
277
+ ///
278
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
279
+ pub fn define(_ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
280
+ module.define_module_function("train", function!(train, 10))?;
281
+ module.define_module_function("export", function!(export, 3))?;
282
+ Ok(())
283
+ }
data/src/util.rs ADDED
@@ -0,0 +1,168 @@
1
+ //! Utility functions for Ruby-Rust data conversion.
2
+ //!
3
+ //! This module provides helper functions for converting between Ruby objects
4
+ //! and Rust data structures, particularly for working with JSON-like data.
5
+
6
+ use magnus::prelude::*;
7
+ use magnus::{Error, RArray, RHash, Ruby, TryConvert, Value};
8
+
9
+ /// Converts a Ruby value to a serde_json::Value.
10
+ ///
11
+ /// # Arguments
12
+ ///
13
+ /// * `ruby` - Ruby runtime handle.
14
+ /// * `value` - Ruby value to convert.
15
+ ///
16
+ /// # Returns
17
+ ///
18
+ /// A `serde_json::Value` representing the Ruby value.
19
+ ///
20
+ /// # Errors
21
+ ///
22
+ /// Returns a `TypeError` if the Ruby value type is not supported, or if it
23
+ /// is a non-finite `Float` (NaN or Infinity), which JSON cannot represent.
24
+ pub fn rb_value_to_json(ruby: &Ruby, value: Value) -> Result<serde_json::Value, Error> {
25
+ if value.is_nil() {
26
+ Ok(serde_json::Value::Null)
27
+ } else if value.is_kind_of(ruby.class_true_class())
28
+ || value.is_kind_of(ruby.class_false_class())
29
+ {
30
+ let b: bool = TryConvert::try_convert(value).map_err(|e| {
31
+ Error::new(
32
+ ruby.exception_type_error(),
33
+ format!("Failed to convert boolean: {e}"),
34
+ )
35
+ })?;
36
+ Ok(serde_json::Value::Bool(b))
37
+ } else if value.is_kind_of(ruby.class_integer()) {
38
+ let i: i64 = TryConvert::try_convert(value).map_err(|e| {
39
+ Error::new(
40
+ ruby.exception_type_error(),
41
+ format!("Failed to convert integer: {e}"),
42
+ )
43
+ })?;
44
+ Ok(serde_json::Value::from(i))
45
+ } else if value.is_kind_of(ruby.class_float()) {
46
+ let f: f64 = TryConvert::try_convert(value).map_err(|e| {
47
+ Error::new(
48
+ ruby.exception_type_error(),
49
+ format!("Failed to convert float: {e}"),
50
+ )
51
+ })?;
52
+ // JSON has no NaN or Infinity; reject them rather than let them
53
+ // silently turn into `null`, which callers read as "not set".
54
+ serde_json::Number::from_f64(f)
55
+ .map(serde_json::Value::Number)
56
+ .ok_or_else(|| {
57
+ Error::new(
58
+ ruby.exception_type_error(),
59
+ format!("Unsupported non-finite float: {f}"),
60
+ )
61
+ })
62
+ } else if value.is_kind_of(ruby.class_string()) || value.is_kind_of(ruby.class_symbol()) {
63
+ let s: String = TryConvert::try_convert(value).map_err(|e| {
64
+ Error::new(
65
+ ruby.exception_type_error(),
66
+ format!("Failed to convert string: {e}"),
67
+ )
68
+ })?;
69
+ Ok(serde_json::Value::String(s))
70
+ } else if let Ok(arr) = RArray::try_convert(value) {
71
+ rb_array_to_json(ruby, arr)
72
+ } else if let Ok(hash) = RHash::try_convert(value) {
73
+ rb_hash_to_json(ruby, hash)
74
+ } else {
75
+ Err(Error::new(
76
+ ruby.exception_type_error(),
77
+ format!("Unsupported Ruby object type: {}", unsafe {
78
+ value.classname()
79
+ }),
80
+ ))
81
+ }
82
+ }
83
+
84
+ /// Converts a Ruby array to a serde_json::Value::Array.
85
+ ///
86
+ /// # Arguments
87
+ ///
88
+ /// * `ruby` - Ruby runtime handle.
89
+ /// * `array` - Ruby array to convert.
90
+ ///
91
+ /// # Returns
92
+ ///
93
+ /// A `serde_json::Value` representing the array.
94
+ fn rb_array_to_json(ruby: &Ruby, array: RArray) -> Result<serde_json::Value, Error> {
95
+ let mut vec = Vec::new();
96
+ for item in array.into_iter() {
97
+ vec.push(rb_value_to_json(ruby, item)?);
98
+ }
99
+ Ok(serde_json::Value::Array(vec))
100
+ }
101
+
102
+ /// Converts a Ruby hash to a serde_json::Value::Object.
103
+ ///
104
+ /// # Arguments
105
+ ///
106
+ /// * `ruby` - Ruby runtime handle.
107
+ /// * `hash` - Ruby hash to convert.
108
+ ///
109
+ /// # Returns
110
+ ///
111
+ /// A `serde_json::Value` representing the hash.
112
+ pub fn rb_hash_to_json(ruby: &Ruby, hash: RHash) -> Result<serde_json::Value, Error> {
113
+ let mut map = serde_json::Map::new();
114
+ hash.foreach(|key: String, value: Value| {
115
+ let json_value = rb_value_to_json(ruby, value)?;
116
+ map.insert(key, json_value);
117
+ Ok(magnus::r_hash::ForEach::Continue)
118
+ })?;
119
+ Ok(serde_json::Value::Object(map))
120
+ }
121
+
122
+ /// Converts a serde_json::Value to a Ruby value.
123
+ ///
124
+ /// # Arguments
125
+ ///
126
+ /// * `ruby` - Ruby runtime handle.
127
+ /// * `value` - JSON value to convert.
128
+ ///
129
+ /// # Returns
130
+ ///
131
+ /// A Ruby `Value` representing the JSON value.
132
+ pub fn json_to_rb_value(ruby: &Ruby, value: &serde_json::Value) -> Result<Value, Error> {
133
+ match value {
134
+ serde_json::Value::Null => Ok(ruby.qnil().as_value()),
135
+ serde_json::Value::Bool(b) => Ok(if *b {
136
+ ruby.qtrue().as_value()
137
+ } else {
138
+ ruby.qfalse().as_value()
139
+ }),
140
+ serde_json::Value::Number(n) => {
141
+ if let Some(i) = n.as_i64() {
142
+ Ok(ruby.integer_from_i64(i).as_value())
143
+ } else if let Some(f) = n.as_f64() {
144
+ Ok(ruby.float_from_f64(f).as_value())
145
+ } else {
146
+ Err(Error::new(
147
+ ruby.exception_type_error(),
148
+ "Unsupported number type",
149
+ ))
150
+ }
151
+ }
152
+ serde_json::Value::String(s) => Ok(ruby.str_new(s).as_value()),
153
+ serde_json::Value::Array(arr) => {
154
+ let rb_arr = ruby.ary_new_capa(arr.len());
155
+ for item in arr {
156
+ rb_arr.push(json_to_rb_value(ruby, item)?)?;
157
+ }
158
+ Ok(rb_arr.as_value())
159
+ }
160
+ serde_json::Value::Object(obj) => {
161
+ let rb_hash = ruby.hash_new();
162
+ for (key, val) in obj {
163
+ rb_hash.aset(ruby.str_new(key), json_to_rb_value(ruby, val)?)?;
164
+ }
165
+ Ok(rb_hash.as_value())
166
+ }
167
+ }
168
+ }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: lindera
3
3
  version: !ruby/object:Gem::Version
4
- version: 6.0.0
4
+ version: 6.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Lindera contributors
@@ -69,13 +69,26 @@ description: Ruby bindings for Lindera, a morphological analysis library for CJK
69
69
  (Japanese, Korean, Chinese).
70
70
  executables: []
71
71
  extensions:
72
- - ext/lindera_ruby/extconf.rb
72
+ - extconf.rb
73
73
  extra_rdoc_files: []
74
74
  files:
75
75
  - Cargo.toml
76
+ - LICENSE
76
77
  - README.md
77
- - ext/lindera_ruby/extconf.rb
78
+ - extconf.rb
78
79
  - lib/lindera.rb
80
+ - src/character_filter.rs
81
+ - src/dictionary.rs
82
+ - src/error.rs
83
+ - src/lib.rs
84
+ - src/metadata.rs
85
+ - src/mode.rs
86
+ - src/schema.rs
87
+ - src/token.rs
88
+ - src/token_filter.rs
89
+ - src/tokenizer.rs
90
+ - src/trainer.rs
91
+ - src/util.rs
79
92
  homepage: https://github.com/lindera/lindera
80
93
  licenses:
81
94
  - MIT
@@ -94,7 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
94
107
  - !ruby/object:Gem::Version
95
108
  version: '0'
96
109
  requirements: []
97
- rubygems_version: 4.0.16
110
+ rubygems_version: 4.0.20
98
111
  specification_version: 4
99
112
  summary: Ruby bindings for Lindera morphological analysis engine
100
113
  test_files: []
@@ -1,8 +0,0 @@
1
- require "rb_sys/mkmf"
2
-
3
- create_rust_makefile("lindera/lindera_ruby") do |r|
4
- # Pass embed features via LINDERA_FEATURES environment variable
5
- # e.g., LINDERA_FEATURES="embed-ipadic" bundle exec rake compile
6
- features = ENV.fetch("LINDERA_FEATURES", "").split(",").map(&:strip).reject(&:empty?)
7
- r.features = features unless features.empty?
8
- end