lindera 6.2.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +8 -6
- data/README.md +3 -3
- data/extconf.rb +6 -4
- data/src/dictionary.rs +130 -16
- data/src/metadata.rs +36 -38
- data/src/mode.rs +212 -154
- data/src/schema.rs +20 -9
- data/src/token.rs +16 -9
- data/src/tokenizer.rs +270 -31
- data/src/trainer.rs +172 -158
- data/src/util.rs +65 -3
- metadata +1 -43
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 57ba8d674304bdfb544e7a350acdfecc032d27fba8d2ec4484dc01e86a270a45
|
|
4
|
+
data.tar.gz: 0fde46037561d56265516e49ebbe0d6f372375e094dd2a5013fba3c88060c60e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4774b1cc1be482185a07ebbceec888a906e152281e573069a1a89964a1012f8bc7bcfbdcb62fca784f747db98a9c1ce12e5d76defff9fe522d6acabd6fa5293d
|
|
7
|
+
data.tar.gz: bf93b38a1c06fafda1c9f087072a775879a8820827f6150ca9adb1b6924ebdfa9e526e3d253cbac90d09e00a143d36d115744a981942c71f7c0231a7d982d2a3
|
data/Cargo.toml
CHANGED
|
@@ -11,9 +11,9 @@
|
|
|
11
11
|
|
|
12
12
|
[package]
|
|
13
13
|
edition = "2024"
|
|
14
|
-
rust-version = "1.
|
|
14
|
+
rust-version = "1.89"
|
|
15
15
|
name = "lindera-ruby"
|
|
16
|
-
version = "
|
|
16
|
+
version = "7.0.0"
|
|
17
17
|
build = false
|
|
18
18
|
publish = false
|
|
19
19
|
autolib = false
|
|
@@ -47,6 +47,8 @@ embed-jieba = ["lindera/embed-jieba"]
|
|
|
47
47
|
embed-ko-dic = ["lindera/embed-ko-dic"]
|
|
48
48
|
embed-sudachidict = ["lindera/embed-sudachidict"]
|
|
49
49
|
embed-unidic = ["lindera/embed-unidic"]
|
|
50
|
+
embed-unidic-csj = ["lindera/embed-unidic-csj"]
|
|
51
|
+
embed-unidic-cwj = ["lindera/embed-unidic-cwj"]
|
|
50
52
|
train = ["lindera/train"]
|
|
51
53
|
|
|
52
54
|
[lib]
|
|
@@ -58,13 +60,13 @@ crate-type = [
|
|
|
58
60
|
path = "src/lib.rs"
|
|
59
61
|
|
|
60
62
|
[dependencies.lindera]
|
|
61
|
-
version = "=
|
|
63
|
+
version = "=7.0.0"
|
|
62
64
|
|
|
63
|
-
[dependencies.lindera-binding
|
|
64
|
-
version = "=
|
|
65
|
+
[dependencies.lindera-binding]
|
|
66
|
+
version = "=7.0.0"
|
|
65
67
|
|
|
66
68
|
[dependencies.magnus]
|
|
67
|
-
version = "0.
|
|
69
|
+
version = "0.9.0"
|
|
68
70
|
features = ["rb-sys"]
|
|
69
71
|
|
|
70
72
|
[dependencies.num_cpus]
|
data/README.md
CHANGED
|
@@ -16,7 +16,7 @@ lindera-ruby provides a Ruby interface to the Lindera morphological analysis eng
|
|
|
16
16
|
## Requirements
|
|
17
17
|
|
|
18
18
|
- Ruby >= 3.1, with a C compiler and `make` for building native gems
|
|
19
|
-
- Rust >= 1.
|
|
19
|
+
- Rust >= 1.89
|
|
20
20
|
- libclang (`libclang-dev` on Debian/Ubuntu; the Xcode command line tools on macOS)
|
|
21
21
|
|
|
22
22
|
## Install
|
|
@@ -57,7 +57,7 @@ require "lindera"
|
|
|
57
57
|
dictionary = Lindera.load_dictionary("/path/to/ipadic")
|
|
58
58
|
|
|
59
59
|
# Create a tokenizer
|
|
60
|
-
tokenizer = Lindera::Tokenizer.new(dictionary,
|
|
60
|
+
tokenizer = Lindera::Tokenizer.new(dictionary, Lindera::Mode.normal, nil)
|
|
61
61
|
|
|
62
62
|
# Tokenize text
|
|
63
63
|
tokens = tokenizer.tokenize("関西国際空港")
|
|
@@ -87,7 +87,7 @@ JSON.generate(tokens.map(&:to_h))
|
|
|
87
87
|
require "lindera"
|
|
88
88
|
|
|
89
89
|
builder = Lindera::TokenizerBuilder.new
|
|
90
|
-
builder.set_mode(
|
|
90
|
+
builder.set_mode(Lindera::Mode.normal)
|
|
91
91
|
builder.set_dictionary("/path/to/ipadic")
|
|
92
92
|
|
|
93
93
|
# Add filters
|
data/extconf.rb
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# Builds the extension for both `bundle exec rake compile` and `gem install`
|
|
2
4
|
# (the gemspec lists this file as the extension). Cargo.toml sits next to it in
|
|
3
5
|
# both layouts, so rb_sys finds the crate without further configuration.
|
|
4
|
-
require
|
|
5
|
-
require
|
|
6
|
+
require 'mkmf'
|
|
7
|
+
require 'rb_sys/mkmf'
|
|
6
8
|
|
|
7
|
-
create_rust_makefile(
|
|
9
|
+
create_rust_makefile('lindera/lindera_ruby') do |r|
|
|
8
10
|
# Pass embed features via LINDERA_FEATURES environment variable, e.g.
|
|
9
11
|
# LINDERA_FEATURES="embed-ipadic" bundle exec rake compile
|
|
10
12
|
# LINDERA_FEATURES="embed-ipadic" gem install lindera
|
|
11
|
-
features = ENV.fetch(
|
|
13
|
+
features = ENV.fetch('LINDERA_FEATURES', '').split(',').map(&:strip).reject(&:empty?)
|
|
12
14
|
r.features = features unless features.empty?
|
|
13
15
|
end
|
data/src/dictionary.rs
CHANGED
|
@@ -6,12 +6,14 @@
|
|
|
6
6
|
use std::path::Path;
|
|
7
7
|
|
|
8
8
|
use magnus::prelude::*;
|
|
9
|
-
use magnus::{
|
|
9
|
+
use magnus::scan_args::{get_kwargs, scan_args};
|
|
10
|
+
use magnus::{Error, RHash, Ruby, Value, function, method};
|
|
10
11
|
|
|
11
12
|
use lindera::dictionary::{
|
|
12
13
|
Dictionary, DictionaryBuilder, Metadata, UserDictionary,
|
|
13
14
|
load_dictionary as lindera_load_dictionary,
|
|
14
15
|
load_user_dictionary as lindera_load_user_dictionary,
|
|
16
|
+
load_user_dictionary_for as lindera_load_user_dictionary_for,
|
|
15
17
|
};
|
|
16
18
|
|
|
17
19
|
use crate::error::to_magnus_error;
|
|
@@ -136,19 +138,78 @@ fn load_user_dictionary(uri: String, metadata: &RbMetadata) -> Result<RbUserDict
|
|
|
136
138
|
.map(|d| RbUserDictionary { inner: d })
|
|
137
139
|
}
|
|
138
140
|
|
|
141
|
+
/// Loads a user dictionary for a loaded system dictionary.
|
|
142
|
+
///
|
|
143
|
+
/// A `.csv` is built with the system dictionary's metadata and checked
|
|
144
|
+
/// against its connection matrix as it is built, so a row whose context ID
|
|
145
|
+
/// lies outside the matrix fails with an error that names its CSV line and
|
|
146
|
+
/// surface. A `.bin` is loaded as it was built; the `Tokenizer` checks it
|
|
147
|
+
/// when it is built with the system dictionary.
|
|
148
|
+
///
|
|
149
|
+
/// # Arguments
|
|
150
|
+
///
|
|
151
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
152
|
+
/// * `uri` - URI or path of the `.csv` or `.bin` user dictionary.
|
|
153
|
+
/// * `dictionary` - The system dictionary the user dictionary is used with.
|
|
154
|
+
///
|
|
155
|
+
/// # Returns
|
|
156
|
+
///
|
|
157
|
+
/// A loaded `RbUserDictionary` object.
|
|
158
|
+
fn load_user_dictionary_for(
|
|
159
|
+
ruby: &Ruby,
|
|
160
|
+
uri: String,
|
|
161
|
+
dictionary: &RbDictionary,
|
|
162
|
+
) -> Result<RbUserDictionary, Error> {
|
|
163
|
+
lindera_load_user_dictionary_for(&uri, &dictionary.inner)
|
|
164
|
+
.map_err(|e| {
|
|
165
|
+
to_magnus_error(
|
|
166
|
+
ruby,
|
|
167
|
+
format!("Failed to load user dictionary from '{uri}': {e}"),
|
|
168
|
+
)
|
|
169
|
+
})
|
|
170
|
+
.map(|d| RbUserDictionary { inner: d })
|
|
171
|
+
}
|
|
172
|
+
|
|
139
173
|
/// Builds a dictionary from source files.
|
|
140
174
|
///
|
|
175
|
+
/// Ruby signature: `Lindera.build_dictionary(input_dir, output_dir, metadata,
|
|
176
|
+
/// context_id_freq: nil)`.
|
|
177
|
+
///
|
|
178
|
+
/// When the metadata enables `connection_id_mapping`, as the `metadata.json`
|
|
179
|
+
/// of ko-dic and UniDic does, the build relabels the connection matrix's
|
|
180
|
+
/// context IDs by access frequency: by the counts of `context_id_freq`, or,
|
|
181
|
+
/// without it, by the count of dictionary entries. The relabelling changes
|
|
182
|
+
/// the speed of tokenization, never its result.
|
|
183
|
+
///
|
|
141
184
|
/// # Arguments
|
|
142
185
|
///
|
|
143
|
-
/// * `
|
|
144
|
-
/// * `
|
|
145
|
-
///
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
186
|
+
/// * `ruby` - Ruby runtime handle.
|
|
187
|
+
/// * `args` - The Ruby arguments:
|
|
188
|
+
/// * `input_dir` - Directory containing dictionary source files.
|
|
189
|
+
/// * `output_dir` - Directory where the built dictionary will be saved.
|
|
190
|
+
/// * `metadata` - Metadata configuration for the dictionary.
|
|
191
|
+
/// * `context_id_freq:` - Path of a context-ID frequency histogram, such
|
|
192
|
+
/// as the `context_id_freq.txt` of a dictionary crate, as `lindera
|
|
193
|
+
/// build --context-id-freq` takes it. `nil` or omitted means none. Read
|
|
194
|
+
/// only with `connection_id_mapping`.
|
|
195
|
+
///
|
|
196
|
+
/// # Returns
|
|
197
|
+
///
|
|
198
|
+
/// `Ok(())` on success, an `ArgumentError` for a wrong number of arguments,
|
|
199
|
+
/// an unknown keyword or a missing input directory, a `TypeError` for an
|
|
200
|
+
/// argument of the wrong type, or a `RuntimeError` when the build fails,
|
|
201
|
+
/// including when the histogram cannot be read.
|
|
202
|
+
fn build_dictionary(ruby: &Ruby, args: &[Value]) -> Result<(), Error> {
|
|
203
|
+
let args = scan_args::<(String, String, &RbMetadata), (), (), (), RHash, ()>(args)?;
|
|
204
|
+
let (input_dir, output_dir, metadata) = args.required;
|
|
205
|
+
let keywords = get_kwargs::<_, (), (Option<Option<String>>,), ()>(
|
|
206
|
+
args.keywords,
|
|
207
|
+
&[],
|
|
208
|
+
&["context_id_freq"],
|
|
209
|
+
)?;
|
|
210
|
+
let (context_id_freq,) = keywords.optional;
|
|
211
|
+
let context_id_freq = context_id_freq.flatten();
|
|
212
|
+
|
|
152
213
|
let input_path = Path::new(&input_dir);
|
|
153
214
|
let output_path = Path::new(&output_dir);
|
|
154
215
|
|
|
@@ -159,21 +220,31 @@ fn build_dictionary(
|
|
|
159
220
|
));
|
|
160
221
|
}
|
|
161
222
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
.
|
|
223
|
+
lindera_binding::dictionary::build_dictionary(
|
|
224
|
+
input_path,
|
|
225
|
+
output_path,
|
|
226
|
+
metadata.clone().into(),
|
|
227
|
+
context_id_freq.as_deref().map(Path::new),
|
|
228
|
+
)
|
|
229
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to build dictionary: {e}")))?;
|
|
166
230
|
|
|
167
231
|
Ok(())
|
|
168
232
|
}
|
|
169
233
|
|
|
170
234
|
/// Builds a user dictionary from a CSV file.
|
|
171
235
|
///
|
|
236
|
+
/// The rows' context IDs are not checked here; they are checked when the
|
|
237
|
+
/// user dictionary is combined with a system dictionary. To check them as
|
|
238
|
+
/// the CSV is built, use [`build_user_dictionary_for`] with the system
|
|
239
|
+
/// dictionary.
|
|
240
|
+
///
|
|
172
241
|
/// # Arguments
|
|
173
242
|
///
|
|
174
243
|
/// * `_kind` - Dictionary kind (reserved for future use).
|
|
175
244
|
/// * `input_file` - Path to the CSV file.
|
|
176
|
-
/// * `output_dir` -
|
|
245
|
+
/// * `output_dir` - Path of the file to write the built user dictionary to,
|
|
246
|
+
/// not a directory despite its name. The user dictionary loaders choose
|
|
247
|
+
/// how to load a file by its extension, so end it with `.bin`.
|
|
177
248
|
/// * `metadata` - Optional metadata configuration.
|
|
178
249
|
fn build_user_dictionary(
|
|
179
250
|
_kind: String,
|
|
@@ -205,6 +276,41 @@ fn build_user_dictionary(
|
|
|
205
276
|
Ok(())
|
|
206
277
|
}
|
|
207
278
|
|
|
279
|
+
/// Builds a user dictionary from a CSV file for a loaded system dictionary.
|
|
280
|
+
///
|
|
281
|
+
/// The CSV is built with the system dictionary's metadata, and every row's
|
|
282
|
+
/// context IDs are checked against its connection matrix, so a row outside
|
|
283
|
+
/// the matrix fails with an error that names its CSV line and surface, and
|
|
284
|
+
/// no file is written.
|
|
285
|
+
///
|
|
286
|
+
/// # Arguments
|
|
287
|
+
///
|
|
288
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
289
|
+
/// * `input_file` - Path to the CSV file.
|
|
290
|
+
/// * `output_file` - Path of the file to write the built user dictionary to.
|
|
291
|
+
/// The user dictionary loaders choose how to load a file by its extension,
|
|
292
|
+
/// so end it with `.bin`.
|
|
293
|
+
/// * `dictionary` - The system dictionary the user dictionary is built for.
|
|
294
|
+
///
|
|
295
|
+
/// # Returns
|
|
296
|
+
///
|
|
297
|
+
/// `Ok(())` once the file is written, or a Ruby `RuntimeError` if the CSV
|
|
298
|
+
/// cannot be read or built, a row has a context ID outside the system
|
|
299
|
+
/// dictionary's connection matrix, or the file cannot be written.
|
|
300
|
+
fn build_user_dictionary_for(
|
|
301
|
+
ruby: &Ruby,
|
|
302
|
+
input_file: String,
|
|
303
|
+
output_file: String,
|
|
304
|
+
dictionary: &RbDictionary,
|
|
305
|
+
) -> Result<(), Error> {
|
|
306
|
+
lindera_binding::dictionary::build_user_dictionary_for(
|
|
307
|
+
Path::new(&input_file),
|
|
308
|
+
Path::new(&output_file),
|
|
309
|
+
&dictionary.inner,
|
|
310
|
+
)
|
|
311
|
+
.map_err(|e| to_magnus_error(ruby, format!("Failed to build user dictionary: {e}")))
|
|
312
|
+
}
|
|
313
|
+
|
|
208
314
|
/// Defines Dictionary, UserDictionary classes and module functions in the given Ruby module.
|
|
209
315
|
///
|
|
210
316
|
/// # Arguments
|
|
@@ -232,8 +338,16 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
|
232
338
|
|
|
233
339
|
module.define_module_function("load_dictionary", function!(load_dictionary, 1))?;
|
|
234
340
|
module.define_module_function("load_user_dictionary", function!(load_user_dictionary, 2))?;
|
|
235
|
-
module.define_module_function(
|
|
341
|
+
module.define_module_function(
|
|
342
|
+
"load_user_dictionary_for",
|
|
343
|
+
function!(load_user_dictionary_for, 2),
|
|
344
|
+
)?;
|
|
345
|
+
module.define_module_function("build_dictionary", function!(build_dictionary, -1))?;
|
|
236
346
|
module.define_module_function("build_user_dictionary", function!(build_user_dictionary, 4))?;
|
|
347
|
+
module.define_module_function(
|
|
348
|
+
"build_user_dictionary_for",
|
|
349
|
+
function!(build_user_dictionary_for, 3),
|
|
350
|
+
)?;
|
|
237
351
|
|
|
238
352
|
Ok(())
|
|
239
353
|
}
|
data/src/metadata.rs
CHANGED
|
@@ -2,25 +2,27 @@
|
|
|
2
2
|
//!
|
|
3
3
|
//! This module provides structures for configuring dictionary metadata, including
|
|
4
4
|
//! character encodings and schema definitions. The defaults and schema wiring are
|
|
5
|
-
//! delegated to [`
|
|
5
|
+
//! delegated to [`lindera_binding::CoreMetadata`]; this module only adds the
|
|
6
6
|
//! magnus wrappers.
|
|
7
7
|
|
|
8
8
|
use std::collections::HashMap;
|
|
9
9
|
|
|
10
10
|
use magnus::prelude::*;
|
|
11
|
-
use magnus::{Error, Ruby, function, method};
|
|
11
|
+
use magnus::{Error, Integer, Ruby, function, method};
|
|
12
12
|
|
|
13
13
|
use lindera::dictionary::Metadata;
|
|
14
|
-
use
|
|
14
|
+
use lindera_binding::CoreMetadata;
|
|
15
|
+
|
|
16
|
+
use crate::util::optional_integer_arg;
|
|
15
17
|
|
|
16
18
|
/// Dictionary metadata configuration.
|
|
17
19
|
///
|
|
18
|
-
/// A thin magnus wrapper over [`
|
|
20
|
+
/// A thin magnus wrapper over [`lindera_binding::CoreMetadata`], which owns
|
|
19
21
|
/// the default values and the schema wiring.
|
|
20
22
|
#[magnus::wrap(class = "Lindera::Metadata", free_immediately, size)]
|
|
21
23
|
#[derive(Debug, Clone)]
|
|
22
24
|
pub struct RbMetadata {
|
|
23
|
-
/// The backing binding
|
|
25
|
+
/// The backing lindera-binding metadata.
|
|
24
26
|
inner: CoreMetadata,
|
|
25
27
|
}
|
|
26
28
|
|
|
@@ -29,38 +31,49 @@ impl RbMetadata {
|
|
|
29
31
|
///
|
|
30
32
|
/// # Arguments
|
|
31
33
|
///
|
|
32
|
-
///
|
|
34
|
+
/// Ruby passes the eight arguments after `ruby` positionally, in this
|
|
35
|
+
/// order; `nil` selects the default.
|
|
36
|
+
///
|
|
37
|
+
/// * `ruby` - Ruby runtime handle (passed by magnus, not by the caller).
|
|
38
|
+
/// * `name` - Dictionary name (default: "default").
|
|
39
|
+
/// * `encoding` - Character encoding (default: "UTF-8").
|
|
40
|
+
/// * `default_word_cost` - Default word cost (default: -10000).
|
|
41
|
+
/// * `default_left_context_id` - Default left context ID (default: 1288).
|
|
42
|
+
/// * `default_right_context_id` - Default right context ID (default: 1288).
|
|
43
|
+
/// * `default_field_value` - Default field value (default: "*").
|
|
44
|
+
/// * `flexible_csv` - Allow flexible CSV (default: false).
|
|
45
|
+
/// * `skip_invalid_cost_or_id` - Skip invalid entries (default: false).
|
|
33
46
|
///
|
|
34
47
|
/// # Returns
|
|
35
48
|
///
|
|
36
|
-
/// A new `RbMetadata` instance
|
|
49
|
+
/// A new `RbMetadata` instance, or a `RangeError` for a cost outside
|
|
50
|
+
/// -32768 to 32767 or a context ID outside 0 to 65535.
|
|
37
51
|
#[allow(clippy::too_many_arguments)]
|
|
38
52
|
fn new(
|
|
53
|
+
ruby: &Ruby,
|
|
39
54
|
name: Option<String>,
|
|
40
55
|
encoding: Option<String>,
|
|
41
|
-
default_word_cost: Option<
|
|
42
|
-
default_left_context_id: Option<
|
|
43
|
-
default_right_context_id: Option<
|
|
56
|
+
default_word_cost: Option<Integer>,
|
|
57
|
+
default_left_context_id: Option<Integer>,
|
|
58
|
+
default_right_context_id: Option<Integer>,
|
|
44
59
|
default_field_value: Option<String>,
|
|
45
60
|
flexible_csv: Option<bool>,
|
|
46
61
|
skip_invalid_cost_or_id: Option<bool>,
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
RbMetadata {
|
|
62
|
+
) -> Result<Self, Error> {
|
|
63
|
+
Ok(RbMetadata {
|
|
50
64
|
inner: CoreMetadata::new(
|
|
51
65
|
name,
|
|
52
66
|
encoding,
|
|
53
|
-
default_word_cost,
|
|
54
|
-
default_left_context_id,
|
|
55
|
-
default_right_context_id,
|
|
67
|
+
optional_integer_arg(ruby, "default_word_cost", default_word_cost)?,
|
|
68
|
+
optional_integer_arg(ruby, "default_left_context_id", default_left_context_id)?,
|
|
69
|
+
optional_integer_arg(ruby, "default_right_context_id", default_right_context_id)?,
|
|
56
70
|
default_field_value,
|
|
57
71
|
flexible_csv,
|
|
58
72
|
skip_invalid_cost_or_id,
|
|
59
|
-
normalize_details,
|
|
60
73
|
None,
|
|
61
74
|
None,
|
|
62
75
|
),
|
|
63
|
-
}
|
|
76
|
+
})
|
|
64
77
|
}
|
|
65
78
|
|
|
66
79
|
/// Creates a default metadata instance.
|
|
@@ -69,7 +82,9 @@ impl RbMetadata {
|
|
|
69
82
|
///
|
|
70
83
|
/// A new `RbMetadata` with default values.
|
|
71
84
|
fn create_default() -> Self {
|
|
72
|
-
RbMetadata
|
|
85
|
+
RbMetadata {
|
|
86
|
+
inner: CoreMetadata::create_default(),
|
|
87
|
+
}
|
|
73
88
|
}
|
|
74
89
|
|
|
75
90
|
/// Loads metadata from a JSON file.
|
|
@@ -141,11 +156,6 @@ impl RbMetadata {
|
|
|
141
156
|
self.inner.skip_invalid_cost_or_id
|
|
142
157
|
}
|
|
143
158
|
|
|
144
|
-
/// Returns whether morphological details should be normalized.
|
|
145
|
-
fn normalize_details(&self) -> bool {
|
|
146
|
-
self.inner.normalize_details
|
|
147
|
-
}
|
|
148
|
-
|
|
149
159
|
/// Converts the metadata to a Ruby hash.
|
|
150
160
|
///
|
|
151
161
|
/// # Returns
|
|
@@ -182,10 +192,6 @@ impl RbMetadata {
|
|
|
182
192
|
"skip_invalid_cost_or_id".to_string(),
|
|
183
193
|
self.inner.skip_invalid_cost_or_id.to_string(),
|
|
184
194
|
);
|
|
185
|
-
dict.insert(
|
|
186
|
-
"normalize_details".to_string(),
|
|
187
|
-
self.inner.normalize_details.to_string(),
|
|
188
|
-
);
|
|
189
195
|
dict.insert(
|
|
190
196
|
"dictionary_schema_fields".to_string(),
|
|
191
197
|
self.inner.dictionary_schema.fields().join(","),
|
|
@@ -245,7 +251,7 @@ impl From<Metadata> for RbMetadata {
|
|
|
245
251
|
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
246
252
|
pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
247
253
|
let metadata_class = module.define_class("Metadata", ruby.class_object())?;
|
|
248
|
-
metadata_class.define_singleton_method("new", function!(RbMetadata::new,
|
|
254
|
+
metadata_class.define_singleton_method("new", function!(RbMetadata::new, 8))?;
|
|
249
255
|
metadata_class
|
|
250
256
|
.define_singleton_method("create_default", function!(RbMetadata::create_default, 0))?;
|
|
251
257
|
metadata_class
|
|
@@ -273,10 +279,6 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
|
273
279
|
"skip_invalid_cost_or_id",
|
|
274
280
|
method!(RbMetadata::skip_invalid_cost_or_id, 0),
|
|
275
281
|
)?;
|
|
276
|
-
metadata_class.define_method(
|
|
277
|
-
"normalize_details",
|
|
278
|
-
method!(RbMetadata::normalize_details, 0),
|
|
279
|
-
)?;
|
|
280
282
|
metadata_class.define_method("to_hash", method!(RbMetadata::to_hash, 0))?;
|
|
281
283
|
metadata_class.define_method("to_h", method!(RbMetadata::to_hash, 0))?;
|
|
282
284
|
metadata_class.define_method("to_s", method!(RbMetadata::to_s, 0))?;
|
|
@@ -288,7 +290,7 @@ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
|
288
290
|
#[cfg(test)]
|
|
289
291
|
mod tests {
|
|
290
292
|
use super::*;
|
|
291
|
-
use
|
|
293
|
+
use lindera_binding::CoreSchema;
|
|
292
294
|
|
|
293
295
|
#[test]
|
|
294
296
|
fn test_rb_metadata_to_lindera_metadata() {
|
|
@@ -302,7 +304,6 @@ mod tests {
|
|
|
302
304
|
Some("N/A".to_string()),
|
|
303
305
|
Some(true),
|
|
304
306
|
Some(true),
|
|
305
|
-
Some(true),
|
|
306
307
|
Some(CoreSchema::new(vec![
|
|
307
308
|
"surface".to_string(),
|
|
308
309
|
"cost".to_string(),
|
|
@@ -320,7 +321,6 @@ mod tests {
|
|
|
320
321
|
assert_eq!(lindera_metadata.default_field_value, "N/A");
|
|
321
322
|
assert!(lindera_metadata.flexible_csv);
|
|
322
323
|
assert!(lindera_metadata.skip_invalid_cost_or_id);
|
|
323
|
-
assert!(lindera_metadata.normalize_details);
|
|
324
324
|
assert_eq!(lindera_metadata.dictionary_schema.get_all_fields().len(), 2);
|
|
325
325
|
assert_eq!(
|
|
326
326
|
lindera_metadata
|
|
@@ -347,7 +347,6 @@ mod tests {
|
|
|
347
347
|
"?".to_string(),
|
|
348
348
|
false,
|
|
349
349
|
true,
|
|
350
|
-
false,
|
|
351
350
|
dict_schema,
|
|
352
351
|
user_schema,
|
|
353
352
|
);
|
|
@@ -361,7 +360,6 @@ mod tests {
|
|
|
361
360
|
assert_eq!(rb_metadata.default_field_value(), "?");
|
|
362
361
|
assert!(!rb_metadata.flexible_csv());
|
|
363
362
|
assert!(rb_metadata.skip_invalid_cost_or_id());
|
|
364
|
-
assert!(!rb_metadata.normalize_details());
|
|
365
363
|
assert_eq!(rb_metadata.inner.dictionary_schema.field_count(), 2);
|
|
366
364
|
assert_eq!(rb_metadata.inner.user_dictionary_schema.field_count(), 2);
|
|
367
365
|
}
|