lindera 5.3.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +72 -32
- data/LICENSE +21 -0
- data/README.md +38 -6
- data/extconf.rb +13 -0
- data/src/character_filter.rs +3 -0
- data/src/dictionary.rs +239 -0
- data/src/error.rs +19 -0
- data/src/lib.rs +90 -0
- data/src/metadata.rs +423 -0
- data/src/mode.rs +401 -0
- data/src/schema.rs +568 -0
- data/src/token.rs +188 -0
- data/src/token_filter.rs +3 -0
- data/src/tokenizer.rs +373 -0
- data/src/trainer.rs +283 -0
- data/src/util.rs +168 -0
- metadata +17 -4
- data/ext/lindera_ruby/extconf.rb +0 -8
data/src/metadata.rs
ADDED
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
//! Dictionary metadata configuration.
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides structures for configuring dictionary metadata, including
|
|
4
|
+
//! character encodings and schema definitions. The defaults and schema wiring are
|
|
5
|
+
//! delegated to [`lindera_binding_core::CoreMetadata`]; this module only adds the
|
|
6
|
+
//! magnus wrappers.
|
|
7
|
+
|
|
8
|
+
use std::collections::HashMap;
|
|
9
|
+
|
|
10
|
+
use magnus::prelude::*;
|
|
11
|
+
use magnus::{Error, Ruby, function, method};
|
|
12
|
+
|
|
13
|
+
use lindera::dictionary::Metadata;
|
|
14
|
+
use lindera_binding_core::CoreMetadata;
|
|
15
|
+
|
|
16
|
+
/// Dictionary metadata configuration.
|
|
17
|
+
///
|
|
18
|
+
/// A thin magnus wrapper over [`lindera_binding_core::CoreMetadata`], which owns
|
|
19
|
+
/// the default values and the schema wiring.
|
|
20
|
+
#[magnus::wrap(class = "Lindera::Metadata", free_immediately, size)]
|
|
21
|
+
#[derive(Debug, Clone)]
|
|
22
|
+
pub struct RbMetadata {
|
|
23
|
+
/// The backing binding-core metadata.
|
|
24
|
+
inner: CoreMetadata,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
impl RbMetadata {
|
|
28
|
+
/// Creates a new `RbMetadata` with optional parameters.
|
|
29
|
+
///
|
|
30
|
+
/// # Arguments
|
|
31
|
+
///
|
|
32
|
+
/// All arguments are optional. Default values are used if not provided.
|
|
33
|
+
///
|
|
34
|
+
/// # Returns
|
|
35
|
+
///
|
|
36
|
+
/// A new `RbMetadata` instance.
|
|
37
|
+
#[allow(clippy::too_many_arguments)]
|
|
38
|
+
fn new(
|
|
39
|
+
name: Option<String>,
|
|
40
|
+
encoding: Option<String>,
|
|
41
|
+
default_word_cost: Option<i16>,
|
|
42
|
+
default_left_context_id: Option<u16>,
|
|
43
|
+
default_right_context_id: Option<u16>,
|
|
44
|
+
default_field_value: Option<String>,
|
|
45
|
+
flexible_csv: Option<bool>,
|
|
46
|
+
skip_invalid_cost_or_id: Option<bool>,
|
|
47
|
+
normalize_details: Option<bool>,
|
|
48
|
+
) -> Self {
|
|
49
|
+
RbMetadata {
|
|
50
|
+
inner: CoreMetadata::new(
|
|
51
|
+
name,
|
|
52
|
+
encoding,
|
|
53
|
+
default_word_cost,
|
|
54
|
+
default_left_context_id,
|
|
55
|
+
default_right_context_id,
|
|
56
|
+
default_field_value,
|
|
57
|
+
flexible_csv,
|
|
58
|
+
skip_invalid_cost_or_id,
|
|
59
|
+
normalize_details,
|
|
60
|
+
None,
|
|
61
|
+
None,
|
|
62
|
+
),
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/// Creates a default metadata instance.
|
|
67
|
+
///
|
|
68
|
+
/// # Returns
|
|
69
|
+
///
|
|
70
|
+
/// A new `RbMetadata` with default values.
|
|
71
|
+
fn create_default() -> Self {
|
|
72
|
+
RbMetadata::new(None, None, None, None, None, None, None, None, None)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/// Loads metadata from a JSON file.
|
|
76
|
+
///
|
|
77
|
+
/// # Arguments
|
|
78
|
+
///
|
|
79
|
+
/// * `path` - Path to the JSON file.
|
|
80
|
+
///
|
|
81
|
+
/// # Returns
|
|
82
|
+
///
|
|
83
|
+
/// A new `RbMetadata` loaded from the file.
|
|
84
|
+
fn from_json_file(path: String) -> Result<Self, Error> {
|
|
85
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
86
|
+
|
|
87
|
+
let json_str = std::fs::read_to_string(&path).map_err(|e| {
|
|
88
|
+
Error::new(
|
|
89
|
+
ruby.exception_io_error(),
|
|
90
|
+
format!("Failed to read file: {e}"),
|
|
91
|
+
)
|
|
92
|
+
})?;
|
|
93
|
+
|
|
94
|
+
let metadata: Metadata = serde_json::from_str(&json_str).map_err(|e| {
|
|
95
|
+
Error::new(
|
|
96
|
+
ruby.exception_arg_error(),
|
|
97
|
+
format!("Failed to parse JSON: {e}"),
|
|
98
|
+
)
|
|
99
|
+
})?;
|
|
100
|
+
|
|
101
|
+
Ok(metadata.into())
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/// Returns the dictionary name.
|
|
105
|
+
fn name(&self) -> String {
|
|
106
|
+
self.inner.name.clone()
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/// Returns the character encoding.
|
|
110
|
+
fn encoding(&self) -> String {
|
|
111
|
+
self.inner.encoding.clone()
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/// Returns the default word cost.
|
|
115
|
+
fn default_word_cost(&self) -> i16 {
|
|
116
|
+
self.inner.default_word_cost
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/// Returns the default left context ID.
|
|
120
|
+
fn default_left_context_id(&self) -> u16 {
|
|
121
|
+
self.inner.default_left_context_id
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/// Returns the default right context ID.
|
|
125
|
+
fn default_right_context_id(&self) -> u16 {
|
|
126
|
+
self.inner.default_right_context_id
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/// Returns the default field value.
|
|
130
|
+
fn default_field_value(&self) -> String {
|
|
131
|
+
self.inner.default_field_value.clone()
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/// Returns whether flexible CSV parsing is enabled.
|
|
135
|
+
fn flexible_csv(&self) -> bool {
|
|
136
|
+
self.inner.flexible_csv
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/// Returns whether invalid cost/ID entries should be skipped.
|
|
140
|
+
fn skip_invalid_cost_or_id(&self) -> bool {
|
|
141
|
+
self.inner.skip_invalid_cost_or_id
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/// Returns whether morphological details should be normalized.
|
|
145
|
+
fn normalize_details(&self) -> bool {
|
|
146
|
+
self.inner.normalize_details
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/// Converts the metadata to a Ruby hash.
|
|
150
|
+
///
|
|
151
|
+
/// # Returns
|
|
152
|
+
///
|
|
153
|
+
/// A HashMap of metadata properties, all as strings. The
|
|
154
|
+
/// `"space_penalty"` entry holds the left-space penalty rules the
|
|
155
|
+
/// dictionary ships as a JSON string, and is present only when it ships
|
|
156
|
+
/// some (ko-dic does).
|
|
157
|
+
fn to_hash(&self) -> HashMap<String, String> {
|
|
158
|
+
let mut dict = HashMap::new();
|
|
159
|
+
dict.insert("name".to_string(), self.inner.name.clone());
|
|
160
|
+
dict.insert("encoding".to_string(), self.inner.encoding.clone());
|
|
161
|
+
dict.insert(
|
|
162
|
+
"default_word_cost".to_string(),
|
|
163
|
+
self.inner.default_word_cost.to_string(),
|
|
164
|
+
);
|
|
165
|
+
dict.insert(
|
|
166
|
+
"default_left_context_id".to_string(),
|
|
167
|
+
self.inner.default_left_context_id.to_string(),
|
|
168
|
+
);
|
|
169
|
+
dict.insert(
|
|
170
|
+
"default_right_context_id".to_string(),
|
|
171
|
+
self.inner.default_right_context_id.to_string(),
|
|
172
|
+
);
|
|
173
|
+
dict.insert(
|
|
174
|
+
"default_field_value".to_string(),
|
|
175
|
+
self.inner.default_field_value.clone(),
|
|
176
|
+
);
|
|
177
|
+
dict.insert(
|
|
178
|
+
"flexible_csv".to_string(),
|
|
179
|
+
self.inner.flexible_csv.to_string(),
|
|
180
|
+
);
|
|
181
|
+
dict.insert(
|
|
182
|
+
"skip_invalid_cost_or_id".to_string(),
|
|
183
|
+
self.inner.skip_invalid_cost_or_id.to_string(),
|
|
184
|
+
);
|
|
185
|
+
dict.insert(
|
|
186
|
+
"normalize_details".to_string(),
|
|
187
|
+
self.inner.normalize_details.to_string(),
|
|
188
|
+
);
|
|
189
|
+
dict.insert(
|
|
190
|
+
"dictionary_schema_fields".to_string(),
|
|
191
|
+
self.inner.dictionary_schema.fields().join(","),
|
|
192
|
+
);
|
|
193
|
+
dict.insert(
|
|
194
|
+
"user_dictionary_schema_fields".to_string(),
|
|
195
|
+
self.inner.user_dictionary_schema.fields().join(","),
|
|
196
|
+
);
|
|
197
|
+
if let Some(space_penalty) = self.inner.space_penalty_json() {
|
|
198
|
+
dict.insert("space_penalty".to_string(), space_penalty);
|
|
199
|
+
}
|
|
200
|
+
dict
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/// Returns the string representation of the metadata.
|
|
204
|
+
fn to_s(&self) -> String {
|
|
205
|
+
format!(
|
|
206
|
+
"Metadata(name='{}', encoding='{}')",
|
|
207
|
+
self.inner.name, self.inner.encoding,
|
|
208
|
+
)
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/// Returns the inspect representation of the metadata.
|
|
212
|
+
fn inspect(&self) -> String {
|
|
213
|
+
format!(
|
|
214
|
+
"#<Lindera::Metadata: name='{}', encoding='{}', schema_fields={}>",
|
|
215
|
+
self.inner.name,
|
|
216
|
+
self.inner.encoding,
|
|
217
|
+
self.inner.dictionary_schema.field_count()
|
|
218
|
+
)
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
impl From<RbMetadata> for Metadata {
|
|
223
|
+
fn from(metadata: RbMetadata) -> Self {
|
|
224
|
+
metadata.inner.into()
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
impl From<Metadata> for RbMetadata {
|
|
229
|
+
fn from(metadata: Metadata) -> Self {
|
|
230
|
+
RbMetadata {
|
|
231
|
+
inner: CoreMetadata::from(metadata),
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/// Defines Metadata class in the given Ruby module.
|
|
237
|
+
///
|
|
238
|
+
/// # Arguments
|
|
239
|
+
///
|
|
240
|
+
/// * `ruby` - Ruby runtime handle.
|
|
241
|
+
/// * `module` - Parent Ruby module.
|
|
242
|
+
///
|
|
243
|
+
/// # Returns
|
|
244
|
+
///
|
|
245
|
+
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
246
|
+
pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
247
|
+
let metadata_class = module.define_class("Metadata", ruby.class_object())?;
|
|
248
|
+
metadata_class.define_singleton_method("new", function!(RbMetadata::new, 9))?;
|
|
249
|
+
metadata_class
|
|
250
|
+
.define_singleton_method("create_default", function!(RbMetadata::create_default, 0))?;
|
|
251
|
+
metadata_class
|
|
252
|
+
.define_singleton_method("from_json_file", function!(RbMetadata::from_json_file, 1))?;
|
|
253
|
+
metadata_class.define_method("name", method!(RbMetadata::name, 0))?;
|
|
254
|
+
metadata_class.define_method("encoding", method!(RbMetadata::encoding, 0))?;
|
|
255
|
+
metadata_class.define_method(
|
|
256
|
+
"default_word_cost",
|
|
257
|
+
method!(RbMetadata::default_word_cost, 0),
|
|
258
|
+
)?;
|
|
259
|
+
metadata_class.define_method(
|
|
260
|
+
"default_left_context_id",
|
|
261
|
+
method!(RbMetadata::default_left_context_id, 0),
|
|
262
|
+
)?;
|
|
263
|
+
metadata_class.define_method(
|
|
264
|
+
"default_right_context_id",
|
|
265
|
+
method!(RbMetadata::default_right_context_id, 0),
|
|
266
|
+
)?;
|
|
267
|
+
metadata_class.define_method(
|
|
268
|
+
"default_field_value",
|
|
269
|
+
method!(RbMetadata::default_field_value, 0),
|
|
270
|
+
)?;
|
|
271
|
+
metadata_class.define_method("flexible_csv", method!(RbMetadata::flexible_csv, 0))?;
|
|
272
|
+
metadata_class.define_method(
|
|
273
|
+
"skip_invalid_cost_or_id",
|
|
274
|
+
method!(RbMetadata::skip_invalid_cost_or_id, 0),
|
|
275
|
+
)?;
|
|
276
|
+
metadata_class.define_method(
|
|
277
|
+
"normalize_details",
|
|
278
|
+
method!(RbMetadata::normalize_details, 0),
|
|
279
|
+
)?;
|
|
280
|
+
metadata_class.define_method("to_hash", method!(RbMetadata::to_hash, 0))?;
|
|
281
|
+
metadata_class.define_method("to_h", method!(RbMetadata::to_hash, 0))?;
|
|
282
|
+
metadata_class.define_method("to_s", method!(RbMetadata::to_s, 0))?;
|
|
283
|
+
metadata_class.define_method("inspect", method!(RbMetadata::inspect, 0))?;
|
|
284
|
+
|
|
285
|
+
Ok(())
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
#[cfg(test)]
|
|
289
|
+
mod tests {
|
|
290
|
+
use super::*;
|
|
291
|
+
use lindera_binding_core::CoreSchema;
|
|
292
|
+
|
|
293
|
+
#[test]
|
|
294
|
+
fn test_rb_metadata_to_lindera_metadata() {
|
|
295
|
+
let rb_metadata = RbMetadata {
|
|
296
|
+
inner: CoreMetadata::new(
|
|
297
|
+
Some("test_dict".to_string()),
|
|
298
|
+
Some("EUC-JP".to_string()),
|
|
299
|
+
Some(-5000),
|
|
300
|
+
Some(100),
|
|
301
|
+
Some(200),
|
|
302
|
+
Some("N/A".to_string()),
|
|
303
|
+
Some(true),
|
|
304
|
+
Some(true),
|
|
305
|
+
Some(true),
|
|
306
|
+
Some(CoreSchema::new(vec![
|
|
307
|
+
"surface".to_string(),
|
|
308
|
+
"cost".to_string(),
|
|
309
|
+
])),
|
|
310
|
+
Some(CoreSchema::new(vec!["surface".to_string()])),
|
|
311
|
+
),
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
let lindera_metadata: Metadata = rb_metadata.into();
|
|
315
|
+
assert_eq!(lindera_metadata.name, "test_dict");
|
|
316
|
+
assert_eq!(lindera_metadata.encoding, "EUC-JP");
|
|
317
|
+
assert_eq!(lindera_metadata.default_word_cost, -5000);
|
|
318
|
+
assert_eq!(lindera_metadata.default_left_context_id, 100);
|
|
319
|
+
assert_eq!(lindera_metadata.default_right_context_id, 200);
|
|
320
|
+
assert_eq!(lindera_metadata.default_field_value, "N/A");
|
|
321
|
+
assert!(lindera_metadata.flexible_csv);
|
|
322
|
+
assert!(lindera_metadata.skip_invalid_cost_or_id);
|
|
323
|
+
assert!(lindera_metadata.normalize_details);
|
|
324
|
+
assert_eq!(lindera_metadata.dictionary_schema.get_all_fields().len(), 2);
|
|
325
|
+
assert_eq!(
|
|
326
|
+
lindera_metadata
|
|
327
|
+
.user_dictionary_schema
|
|
328
|
+
.get_all_fields()
|
|
329
|
+
.len(),
|
|
330
|
+
1
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
#[test]
|
|
335
|
+
fn test_lindera_metadata_to_rb_metadata() {
|
|
336
|
+
let dict_schema =
|
|
337
|
+
lindera::dictionary::Schema::new(vec!["surface".to_string(), "cost".to_string()]);
|
|
338
|
+
let user_schema =
|
|
339
|
+
lindera::dictionary::Schema::new(vec!["surface".to_string(), "reading".to_string()]);
|
|
340
|
+
|
|
341
|
+
let lindera_metadata = Metadata::new(
|
|
342
|
+
"my_dict".to_string(),
|
|
343
|
+
"UTF-8".to_string(),
|
|
344
|
+
-8000,
|
|
345
|
+
500,
|
|
346
|
+
600,
|
|
347
|
+
"?".to_string(),
|
|
348
|
+
false,
|
|
349
|
+
true,
|
|
350
|
+
false,
|
|
351
|
+
dict_schema,
|
|
352
|
+
user_schema,
|
|
353
|
+
);
|
|
354
|
+
|
|
355
|
+
let rb_metadata: RbMetadata = lindera_metadata.into();
|
|
356
|
+
assert_eq!(rb_metadata.name(), "my_dict");
|
|
357
|
+
assert_eq!(rb_metadata.encoding(), "UTF-8");
|
|
358
|
+
assert_eq!(rb_metadata.default_word_cost(), -8000);
|
|
359
|
+
assert_eq!(rb_metadata.default_left_context_id(), 500);
|
|
360
|
+
assert_eq!(rb_metadata.default_right_context_id(), 600);
|
|
361
|
+
assert_eq!(rb_metadata.default_field_value(), "?");
|
|
362
|
+
assert!(!rb_metadata.flexible_csv());
|
|
363
|
+
assert!(rb_metadata.skip_invalid_cost_or_id());
|
|
364
|
+
assert!(!rb_metadata.normalize_details());
|
|
365
|
+
assert_eq!(rb_metadata.inner.dictionary_schema.field_count(), 2);
|
|
366
|
+
assert_eq!(rb_metadata.inner.user_dictionary_schema.field_count(), 2);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
#[test]
|
|
370
|
+
fn test_rb_metadata_defaults() {
|
|
371
|
+
let rb_metadata = RbMetadata::create_default();
|
|
372
|
+
assert_eq!(rb_metadata.name(), "default");
|
|
373
|
+
assert_eq!(rb_metadata.encoding(), "UTF-8");
|
|
374
|
+
assert_eq!(rb_metadata.default_word_cost(), -10000);
|
|
375
|
+
assert_eq!(rb_metadata.default_left_context_id(), 1288);
|
|
376
|
+
assert_eq!(rb_metadata.default_right_context_id(), 1288);
|
|
377
|
+
assert_eq!(rb_metadata.default_field_value(), "*");
|
|
378
|
+
assert!(!rb_metadata.flexible_csv());
|
|
379
|
+
assert_eq!(rb_metadata.inner.dictionary_schema.field_count(), 13);
|
|
380
|
+
assert_eq!(rb_metadata.inner.user_dictionary_schema.field_count(), 3);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
#[test]
|
|
384
|
+
fn test_rb_metadata_roundtrip() {
|
|
385
|
+
let rb_metadata = RbMetadata::create_default();
|
|
386
|
+
let lindera: Metadata = rb_metadata.into();
|
|
387
|
+
let back: RbMetadata = lindera.into();
|
|
388
|
+
assert_eq!(back.name(), "default");
|
|
389
|
+
assert_eq!(back.encoding(), "UTF-8");
|
|
390
|
+
assert_eq!(back.default_word_cost(), -10000);
|
|
391
|
+
assert_eq!(back.default_left_context_id(), 1288);
|
|
392
|
+
assert_eq!(back.default_right_context_id(), 1288);
|
|
393
|
+
assert_eq!(back.default_field_value(), "*");
|
|
394
|
+
assert_eq!(back.inner.dictionary_schema.field_count(), 13);
|
|
395
|
+
assert_eq!(back.inner.user_dictionary_schema.field_count(), 3);
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
#[test]
|
|
399
|
+
fn test_rb_metadata_to_hash_omits_space_penalty_when_none() {
|
|
400
|
+
let hash = RbMetadata::create_default().to_hash();
|
|
401
|
+
assert!(!hash.contains_key("space_penalty"));
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/// ko-dic's `metadata.json` ships left-space penalty rules: `to_hash`
|
|
405
|
+
/// exposes them, and converting back for a dictionary build keeps them.
|
|
406
|
+
#[test]
|
|
407
|
+
fn test_rb_metadata_keeps_ko_dic_space_penalty() {
|
|
408
|
+
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
409
|
+
.join("../lindera-ko-dic/metadata.json");
|
|
410
|
+
let json = std::fs::read_to_string(path).unwrap();
|
|
411
|
+
let metadata: Metadata = serde_json::from_str(&json).unwrap();
|
|
412
|
+
let shipped = metadata.space_penalty.clone().unwrap();
|
|
413
|
+
|
|
414
|
+
let rb_metadata: RbMetadata = metadata.into();
|
|
415
|
+
let hash = rb_metadata.to_hash();
|
|
416
|
+
let space_penalty = hash.get("space_penalty").unwrap();
|
|
417
|
+
assert!(space_penalty.contains("\"rules\""));
|
|
418
|
+
assert!(space_penalty.contains("JKS"));
|
|
419
|
+
|
|
420
|
+
let rebuilt: Metadata = rb_metadata.into();
|
|
421
|
+
assert_eq!(rebuilt.space_penalty, Some(shipped));
|
|
422
|
+
}
|
|
423
|
+
}
|