lindera 6.2.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 529da66cbb1d0fd3372e0ab7695c72ac53c55d8a1a411b99d3db0135ac05be56
4
- data.tar.gz: '020678eec7845a566b1572f2a61af447252dc0090a5d9c73f183cddf4e656f46'
3
+ metadata.gz: 57ba8d674304bdfb544e7a350acdfecc032d27fba8d2ec4484dc01e86a270a45
4
+ data.tar.gz: 0fde46037561d56265516e49ebbe0d6f372375e094dd2a5013fba3c88060c60e
5
5
  SHA512:
6
- metadata.gz: a42a702e498f3690ba0ccc3288d09c0d3e5dcc457905d058c101d112cfe24ecd4fda6cedb0e92a675a519880f6d5383efb4b1519fa1111b0b17ecaa5778e6f68
7
- data.tar.gz: 57dcbda79af341e03428c7eb963e407e22bdf78a23687e95be022cdb3b91789c9c1a3b97d01b500ff44e8f700a4ea109b5262866b699d8966ae4f2597546f7d9
6
+ metadata.gz: 4774b1cc1be482185a07ebbceec888a906e152281e573069a1a89964a1012f8bc7bcfbdcb62fca784f747db98a9c1ce12e5d76defff9fe522d6acabd6fa5293d
7
+ data.tar.gz: bf93b38a1c06fafda1c9f087072a775879a8820827f6150ca9adb1b6924ebdfa9e526e3d253cbac90d09e00a143d36d115744a981942c71f7c0231a7d982d2a3
data/Cargo.toml CHANGED
@@ -11,9 +11,9 @@
11
11
 
12
12
  [package]
13
13
  edition = "2024"
14
- rust-version = "1.88"
14
+ rust-version = "1.89"
15
15
  name = "lindera-ruby"
16
- version = "6.2.0"
16
+ version = "7.0.0"
17
17
  build = false
18
18
  publish = false
19
19
  autolib = false
@@ -47,6 +47,8 @@ embed-jieba = ["lindera/embed-jieba"]
47
47
  embed-ko-dic = ["lindera/embed-ko-dic"]
48
48
  embed-sudachidict = ["lindera/embed-sudachidict"]
49
49
  embed-unidic = ["lindera/embed-unidic"]
50
+ embed-unidic-csj = ["lindera/embed-unidic-csj"]
51
+ embed-unidic-cwj = ["lindera/embed-unidic-cwj"]
50
52
  train = ["lindera/train"]
51
53
 
52
54
  [lib]
@@ -58,13 +60,13 @@ crate-type = [
58
60
  path = "src/lib.rs"
59
61
 
60
62
  [dependencies.lindera]
61
- version = "=6.2.0"
63
+ version = "=7.0.0"
62
64
 
63
- [dependencies.lindera-binding-core]
64
- version = "=6.2.0"
65
+ [dependencies.lindera-binding]
66
+ version = "=7.0.0"
65
67
 
66
68
  [dependencies.magnus]
67
- version = "0.8.2"
69
+ version = "0.9.0"
68
70
  features = ["rb-sys"]
69
71
 
70
72
  [dependencies.num_cpus]
data/README.md CHANGED
@@ -16,7 +16,7 @@ lindera-ruby provides a Ruby interface to the Lindera morphological analysis eng
16
16
  ## Requirements
17
17
 
18
18
  - Ruby >= 3.1, with a C compiler and `make` for building native gems
19
- - Rust >= 1.88
19
+ - Rust >= 1.89
20
20
  - libclang (`libclang-dev` on Debian/Ubuntu; the Xcode command line tools on macOS)
21
21
 
22
22
  ## Install
@@ -57,7 +57,7 @@ require "lindera"
57
57
  dictionary = Lindera.load_dictionary("/path/to/ipadic")
58
58
 
59
59
  # Create a tokenizer
60
- tokenizer = Lindera::Tokenizer.new(dictionary, "normal", nil)
60
+ tokenizer = Lindera::Tokenizer.new(dictionary, Lindera::Mode.normal, nil)
61
61
 
62
62
  # Tokenize text
63
63
  tokens = tokenizer.tokenize("関西国際空港")
@@ -87,7 +87,7 @@ JSON.generate(tokens.map(&:to_h))
87
87
  require "lindera"
88
88
 
89
89
  builder = Lindera::TokenizerBuilder.new
90
- builder.set_mode("normal")
90
+ builder.set_mode(Lindera::Mode.normal)
91
91
  builder.set_dictionary("/path/to/ipadic")
92
92
 
93
93
  # Add filters
data/extconf.rb CHANGED
@@ -1,13 +1,15 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # Builds the extension for both `bundle exec rake compile` and `gem install`
2
4
  # (the gemspec lists this file as the extension). Cargo.toml sits next to it in
3
5
  # both layouts, so rb_sys finds the crate without further configuration.
4
- require "mkmf"
5
- require "rb_sys/mkmf"
6
+ require 'mkmf'
7
+ require 'rb_sys/mkmf'
6
8
 
7
- create_rust_makefile("lindera/lindera_ruby") do |r|
9
+ create_rust_makefile('lindera/lindera_ruby') do |r|
8
10
  # Pass embed features via LINDERA_FEATURES environment variable, e.g.
9
11
  # LINDERA_FEATURES="embed-ipadic" bundle exec rake compile
10
12
  # LINDERA_FEATURES="embed-ipadic" gem install lindera
11
- features = ENV.fetch("LINDERA_FEATURES", "").split(",").map(&:strip).reject(&:empty?)
13
+ features = ENV.fetch('LINDERA_FEATURES', '').split(',').map(&:strip).reject(&:empty?)
12
14
  r.features = features unless features.empty?
13
15
  end
data/src/dictionary.rs CHANGED
@@ -6,12 +6,14 @@
6
6
  use std::path::Path;
7
7
 
8
8
  use magnus::prelude::*;
9
- use magnus::{Error, Ruby, function, method};
9
+ use magnus::scan_args::{get_kwargs, scan_args};
10
+ use magnus::{Error, RHash, Ruby, Value, function, method};
10
11
 
11
12
  use lindera::dictionary::{
12
13
  Dictionary, DictionaryBuilder, Metadata, UserDictionary,
13
14
  load_dictionary as lindera_load_dictionary,
14
15
  load_user_dictionary as lindera_load_user_dictionary,
16
+ load_user_dictionary_for as lindera_load_user_dictionary_for,
15
17
  };
16
18
 
17
19
  use crate::error::to_magnus_error;
@@ -136,19 +138,78 @@ fn load_user_dictionary(uri: String, metadata: &RbMetadata) -> Result<RbUserDict
136
138
  .map(|d| RbUserDictionary { inner: d })
137
139
  }
138
140
 
141
+ /// Loads a user dictionary for a loaded system dictionary.
142
+ ///
143
+ /// A `.csv` is built with the system dictionary's metadata and checked
144
+ /// against its connection matrix as it is built, so a row whose context ID
145
+ /// lies outside the matrix fails with an error that names its CSV line and
146
+ /// surface. A `.bin` is loaded as it was built; the `Tokenizer` checks it
147
+ /// when it is built with the system dictionary.
148
+ ///
149
+ /// # Arguments
150
+ ///
151
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
152
+ /// * `uri` - URI or path of the `.csv` or `.bin` user dictionary.
153
+ /// * `dictionary` - The system dictionary the user dictionary is used with.
154
+ ///
155
+ /// # Returns
156
+ ///
157
+ /// A loaded `RbUserDictionary` object.
158
+ fn load_user_dictionary_for(
159
+ ruby: &Ruby,
160
+ uri: String,
161
+ dictionary: &RbDictionary,
162
+ ) -> Result<RbUserDictionary, Error> {
163
+ lindera_load_user_dictionary_for(&uri, &dictionary.inner)
164
+ .map_err(|e| {
165
+ to_magnus_error(
166
+ ruby,
167
+ format!("Failed to load user dictionary from '{uri}': {e}"),
168
+ )
169
+ })
170
+ .map(|d| RbUserDictionary { inner: d })
171
+ }
172
+
139
173
  /// Builds a dictionary from source files.
140
174
  ///
175
+ /// Ruby signature: `Lindera.build_dictionary(input_dir, output_dir, metadata,
176
+ /// context_id_freq: nil)`.
177
+ ///
178
+ /// When the metadata enables `connection_id_mapping`, as the `metadata.json`
179
+ /// of ko-dic and UniDic does, the build relabels the connection matrix's
180
+ /// context IDs by access frequency: by the counts of `context_id_freq`, or,
181
+ /// without it, by the count of dictionary entries. The relabelling changes
182
+ /// the speed of tokenization, never its result.
183
+ ///
141
184
  /// # Arguments
142
185
  ///
143
- /// * `input_dir` - Directory containing dictionary source files.
144
- /// * `output_dir` - Directory where the built dictionary will be saved.
145
- /// * `metadata` - Metadata configuration for the dictionary.
146
- fn build_dictionary(
147
- input_dir: String,
148
- output_dir: String,
149
- metadata: &RbMetadata,
150
- ) -> Result<(), Error> {
151
- let ruby = Ruby::get().expect("Ruby runtime not initialized");
186
+ /// * `ruby` - Ruby runtime handle.
187
+ /// * `args` - The Ruby arguments:
188
+ /// * `input_dir` - Directory containing dictionary source files.
189
+ /// * `output_dir` - Directory where the built dictionary will be saved.
190
+ /// * `metadata` - Metadata configuration for the dictionary.
191
+ /// * `context_id_freq:` - Path of a context-ID frequency histogram, such
192
+ /// as the `context_id_freq.txt` of a dictionary crate, as `lindera
193
+ /// build --context-id-freq` takes it. `nil` or omitted means none. Read
194
+ /// only with `connection_id_mapping`.
195
+ ///
196
+ /// # Returns
197
+ ///
198
+ /// `Ok(())` on success, an `ArgumentError` for a wrong number of arguments,
199
+ /// an unknown keyword or a missing input directory, a `TypeError` for an
200
+ /// argument of the wrong type, or a `RuntimeError` when the build fails,
201
+ /// including when the histogram cannot be read.
202
+ fn build_dictionary(ruby: &Ruby, args: &[Value]) -> Result<(), Error> {
203
+ let args = scan_args::<(String, String, &RbMetadata), (), (), (), RHash, ()>(args)?;
204
+ let (input_dir, output_dir, metadata) = args.required;
205
+ let keywords = get_kwargs::<_, (), (Option<Option<String>>,), ()>(
206
+ args.keywords,
207
+ &[],
208
+ &["context_id_freq"],
209
+ )?;
210
+ let (context_id_freq,) = keywords.optional;
211
+ let context_id_freq = context_id_freq.flatten();
212
+
152
213
  let input_path = Path::new(&input_dir);
153
214
  let output_path = Path::new(&output_dir);
154
215
 
@@ -159,21 +220,31 @@ fn build_dictionary(
159
220
  ));
160
221
  }
161
222
 
162
- let builder = DictionaryBuilder::new(metadata.clone().into());
163
- builder
164
- .build_dictionary(input_path, output_path)
165
- .map_err(|e| to_magnus_error(&ruby, format!("Failed to build dictionary: {e}")))?;
223
+ lindera_binding::dictionary::build_dictionary(
224
+ input_path,
225
+ output_path,
226
+ metadata.clone().into(),
227
+ context_id_freq.as_deref().map(Path::new),
228
+ )
229
+ .map_err(|e| to_magnus_error(ruby, format!("Failed to build dictionary: {e}")))?;
166
230
 
167
231
  Ok(())
168
232
  }
169
233
 
170
234
  /// Builds a user dictionary from a CSV file.
171
235
  ///
236
+ /// The rows' context IDs are not checked here; they are checked when the
237
+ /// user dictionary is combined with a system dictionary. To check them as
238
+ /// the CSV is built, use [`build_user_dictionary_for`] with the system
239
+ /// dictionary.
240
+ ///
172
241
  /// # Arguments
173
242
  ///
174
243
  /// * `_kind` - Dictionary kind (reserved for future use).
175
244
  /// * `input_file` - Path to the CSV file.
176
- /// * `output_dir` - Directory where the built user dictionary will be saved.
245
+ /// * `output_dir` - Path of the file to write the built user dictionary to,
246
+ /// not a directory despite its name. The user dictionary loaders choose
247
+ /// how to load a file by its extension, so end it with `.bin`.
177
248
  /// * `metadata` - Optional metadata configuration.
178
249
  fn build_user_dictionary(
179
250
  _kind: String,
@@ -205,6 +276,41 @@ fn build_user_dictionary(
205
276
  Ok(())
206
277
  }
207
278
 
279
+ /// Builds a user dictionary from a CSV file for a loaded system dictionary.
280
+ ///
281
+ /// The CSV is built with the system dictionary's metadata, and every row's
282
+ /// context IDs are checked against its connection matrix, so a row outside
283
+ /// the matrix fails with an error that names its CSV line and surface, and
284
+ /// no file is written.
285
+ ///
286
+ /// # Arguments
287
+ ///
288
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
289
+ /// * `input_file` - Path to the CSV file.
290
+ /// * `output_file` - Path of the file to write the built user dictionary to.
291
+ /// The user dictionary loaders choose how to load a file by its extension,
292
+ /// so end it with `.bin`.
293
+ /// * `dictionary` - The system dictionary the user dictionary is built for.
294
+ ///
295
+ /// # Returns
296
+ ///
297
+ /// `Ok(())` once the file is written, or a Ruby `RuntimeError` if the CSV
298
+ /// cannot be read or built, a row has a context ID outside the system
299
+ /// dictionary's connection matrix, or the file cannot be written.
300
+ fn build_user_dictionary_for(
301
+ ruby: &Ruby,
302
+ input_file: String,
303
+ output_file: String,
304
+ dictionary: &RbDictionary,
305
+ ) -> Result<(), Error> {
306
+ lindera_binding::dictionary::build_user_dictionary_for(
307
+ Path::new(&input_file),
308
+ Path::new(&output_file),
309
+ &dictionary.inner,
310
+ )
311
+ .map_err(|e| to_magnus_error(ruby, format!("Failed to build user dictionary: {e}")))
312
+ }
313
+
208
314
  /// Defines Dictionary, UserDictionary classes and module functions in the given Ruby module.
209
315
  ///
210
316
  /// # Arguments
@@ -232,8 +338,16 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
232
338
 
233
339
  module.define_module_function("load_dictionary", function!(load_dictionary, 1))?;
234
340
  module.define_module_function("load_user_dictionary", function!(load_user_dictionary, 2))?;
235
- module.define_module_function("build_dictionary", function!(build_dictionary, 3))?;
341
+ module.define_module_function(
342
+ "load_user_dictionary_for",
343
+ function!(load_user_dictionary_for, 2),
344
+ )?;
345
+ module.define_module_function("build_dictionary", function!(build_dictionary, -1))?;
236
346
  module.define_module_function("build_user_dictionary", function!(build_user_dictionary, 4))?;
347
+ module.define_module_function(
348
+ "build_user_dictionary_for",
349
+ function!(build_user_dictionary_for, 3),
350
+ )?;
237
351
 
238
352
  Ok(())
239
353
  }
data/src/metadata.rs CHANGED
@@ -2,25 +2,27 @@
2
2
  //!
3
3
  //! This module provides structures for configuring dictionary metadata, including
4
4
  //! character encodings and schema definitions. The defaults and schema wiring are
5
- //! delegated to [`lindera_binding_core::CoreMetadata`]; this module only adds the
5
+ //! delegated to [`lindera_binding::CoreMetadata`]; this module only adds the
6
6
  //! magnus wrappers.
7
7
 
8
8
  use std::collections::HashMap;
9
9
 
10
10
  use magnus::prelude::*;
11
- use magnus::{Error, Ruby, function, method};
11
+ use magnus::{Error, Integer, Ruby, function, method};
12
12
 
13
13
  use lindera::dictionary::Metadata;
14
- use lindera_binding_core::CoreMetadata;
14
+ use lindera_binding::CoreMetadata;
15
+
16
+ use crate::util::optional_integer_arg;
15
17
 
16
18
  /// Dictionary metadata configuration.
17
19
  ///
18
- /// A thin magnus wrapper over [`lindera_binding_core::CoreMetadata`], which owns
20
+ /// A thin magnus wrapper over [`lindera_binding::CoreMetadata`], which owns
19
21
  /// the default values and the schema wiring.
20
22
  #[magnus::wrap(class = "Lindera::Metadata", free_immediately, size)]
21
23
  #[derive(Debug, Clone)]
22
24
  pub struct RbMetadata {
23
- /// The backing binding-core metadata.
25
+ /// The backing lindera-binding metadata.
24
26
  inner: CoreMetadata,
25
27
  }
26
28
 
@@ -29,38 +31,49 @@ impl RbMetadata {
29
31
  ///
30
32
  /// # Arguments
31
33
  ///
32
- /// All arguments are optional. Default values are used if not provided.
34
+ /// Ruby passes the eight arguments after `ruby` positionally, in this
35
+ /// order; `nil` selects the default.
36
+ ///
37
+ /// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
38
+ /// * `name` - Dictionary name (default: "default").
39
+ /// * `encoding` - Character encoding (default: "UTF-8").
40
+ /// * `default_word_cost` - Default word cost (default: -10000).
41
+ /// * `default_left_context_id` - Default left context ID (default: 1288).
42
+ /// * `default_right_context_id` - Default right context ID (default: 1288).
43
+ /// * `default_field_value` - Default field value (default: "*").
44
+ /// * `flexible_csv` - Allow flexible CSV (default: false).
45
+ /// * `skip_invalid_cost_or_id` - Skip invalid entries (default: false).
33
46
  ///
34
47
  /// # Returns
35
48
  ///
36
- /// A new `RbMetadata` instance.
49
+ /// A new `RbMetadata` instance, or a `RangeError` for a cost outside
50
+ /// -32768 to 32767 or a context ID outside 0 to 65535.
37
51
  #[allow(clippy::too_many_arguments)]
38
52
  fn new(
53
+ ruby: &Ruby,
39
54
  name: Option<String>,
40
55
  encoding: Option<String>,
41
- default_word_cost: Option<i16>,
42
- default_left_context_id: Option<u16>,
43
- default_right_context_id: Option<u16>,
56
+ default_word_cost: Option<Integer>,
57
+ default_left_context_id: Option<Integer>,
58
+ default_right_context_id: Option<Integer>,
44
59
  default_field_value: Option<String>,
45
60
  flexible_csv: Option<bool>,
46
61
  skip_invalid_cost_or_id: Option<bool>,
47
- normalize_details: Option<bool>,
48
- ) -> Self {
49
- RbMetadata {
62
+ ) -> Result<Self, Error> {
63
+ Ok(RbMetadata {
50
64
  inner: CoreMetadata::new(
51
65
  name,
52
66
  encoding,
53
- default_word_cost,
54
- default_left_context_id,
55
- default_right_context_id,
67
+ optional_integer_arg(ruby, "default_word_cost", default_word_cost)?,
68
+ optional_integer_arg(ruby, "default_left_context_id", default_left_context_id)?,
69
+ optional_integer_arg(ruby, "default_right_context_id", default_right_context_id)?,
56
70
  default_field_value,
57
71
  flexible_csv,
58
72
  skip_invalid_cost_or_id,
59
- normalize_details,
60
73
  None,
61
74
  None,
62
75
  ),
63
- }
76
+ })
64
77
  }
65
78
 
66
79
  /// Creates a default metadata instance.
@@ -69,7 +82,9 @@ impl RbMetadata {
69
82
  ///
70
83
  /// A new `RbMetadata` with default values.
71
84
  fn create_default() -> Self {
72
- RbMetadata::new(None, None, None, None, None, None, None, None, None)
85
+ RbMetadata {
86
+ inner: CoreMetadata::create_default(),
87
+ }
73
88
  }
74
89
 
75
90
  /// Loads metadata from a JSON file.
@@ -141,11 +156,6 @@ impl RbMetadata {
141
156
  self.inner.skip_invalid_cost_or_id
142
157
  }
143
158
 
144
- /// Returns whether morphological details should be normalized.
145
- fn normalize_details(&self) -> bool {
146
- self.inner.normalize_details
147
- }
148
-
149
159
  /// Converts the metadata to a Ruby hash.
150
160
  ///
151
161
  /// # Returns
@@ -182,10 +192,6 @@ impl RbMetadata {
182
192
  "skip_invalid_cost_or_id".to_string(),
183
193
  self.inner.skip_invalid_cost_or_id.to_string(),
184
194
  );
185
- dict.insert(
186
- "normalize_details".to_string(),
187
- self.inner.normalize_details.to_string(),
188
- );
189
195
  dict.insert(
190
196
  "dictionary_schema_fields".to_string(),
191
197
  self.inner.dictionary_schema.fields().join(","),
@@ -245,7 +251,7 @@ impl From<Metadata> for RbMetadata {
245
251
  /// `Ok(())` on success, or a Magnus `Error` on failure.
246
252
  pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
247
253
  let metadata_class = module.define_class("Metadata", ruby.class_object())?;
248
- metadata_class.define_singleton_method("new", function!(RbMetadata::new, 9))?;
254
+ metadata_class.define_singleton_method("new", function!(RbMetadata::new, 8))?;
249
255
  metadata_class
250
256
  .define_singleton_method("create_default", function!(RbMetadata::create_default, 0))?;
251
257
  metadata_class
@@ -273,10 +279,6 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
273
279
  "skip_invalid_cost_or_id",
274
280
  method!(RbMetadata::skip_invalid_cost_or_id, 0),
275
281
  )?;
276
- metadata_class.define_method(
277
- "normalize_details",
278
- method!(RbMetadata::normalize_details, 0),
279
- )?;
280
282
  metadata_class.define_method("to_hash", method!(RbMetadata::to_hash, 0))?;
281
283
  metadata_class.define_method("to_h", method!(RbMetadata::to_hash, 0))?;
282
284
  metadata_class.define_method("to_s", method!(RbMetadata::to_s, 0))?;
@@ -288,7 +290,7 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
288
290
  #[cfg(test)]
289
291
  mod tests {
290
292
  use super::*;
291
- use lindera_binding_core::CoreSchema;
293
+ use lindera_binding::CoreSchema;
292
294
 
293
295
  #[test]
294
296
  fn test_rb_metadata_to_lindera_metadata() {
@@ -302,7 +304,6 @@ mod tests {
302
304
  Some("N/A".to_string()),
303
305
  Some(true),
304
306
  Some(true),
305
- Some(true),
306
307
  Some(CoreSchema::new(vec![
307
308
  "surface".to_string(),
308
309
  "cost".to_string(),
@@ -320,7 +321,6 @@ mod tests {
320
321
  assert_eq!(lindera_metadata.default_field_value, "N/A");
321
322
  assert!(lindera_metadata.flexible_csv);
322
323
  assert!(lindera_metadata.skip_invalid_cost_or_id);
323
- assert!(lindera_metadata.normalize_details);
324
324
  assert_eq!(lindera_metadata.dictionary_schema.get_all_fields().len(), 2);
325
325
  assert_eq!(
326
326
  lindera_metadata
@@ -347,7 +347,6 @@ mod tests {
347
347
  "?".to_string(),
348
348
  false,
349
349
  true,
350
- false,
351
350
  dict_schema,
352
351
  user_schema,
353
352
  );
@@ -361,7 +360,6 @@ mod tests {
361
360
  assert_eq!(rb_metadata.default_field_value(), "?");
362
361
  assert!(!rb_metadata.flexible_csv());
363
362
  assert!(rb_metadata.skip_invalid_cost_or_id());
364
- assert!(!rb_metadata.normalize_details());
365
363
  assert_eq!(rb_metadata.inner.dictionary_schema.field_count(), 2);
366
364
  assert_eq!(rb_metadata.inner.user_dictionary_schema.field_count(), 2);
367
365
  }