lindera 5.3.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/src/metadata.rs ADDED
@@ -0,0 +1,423 @@
1
+ //! Dictionary metadata configuration.
2
+ //!
3
+ //! This module provides structures for configuring dictionary metadata, including
4
+ //! character encodings and schema definitions. The defaults and schema wiring are
5
+ //! delegated to [`lindera_binding_core::CoreMetadata`]; this module only adds the
6
+ //! magnus wrappers.
7
+
8
+ use std::collections::HashMap;
9
+
10
+ use magnus::prelude::*;
11
+ use magnus::{Error, Ruby, function, method};
12
+
13
+ use lindera::dictionary::Metadata;
14
+ use lindera_binding_core::CoreMetadata;
15
+
16
+ /// Dictionary metadata configuration.
17
+ ///
18
+ /// A thin magnus wrapper over [`lindera_binding_core::CoreMetadata`], which owns
19
+ /// the default values and the schema wiring.
20
+ #[magnus::wrap(class = "Lindera::Metadata", free_immediately, size)]
21
+ #[derive(Debug, Clone)]
22
+ pub struct RbMetadata {
23
+ /// The backing binding-core metadata.
24
+ inner: CoreMetadata,
25
+ }
26
+
27
+ impl RbMetadata {
28
+ /// Creates a new `RbMetadata` with optional parameters.
29
+ ///
30
+ /// # Arguments
31
+ ///
32
+ /// All arguments are optional. Default values are used if not provided.
33
+ ///
34
+ /// # Returns
35
+ ///
36
+ /// A new `RbMetadata` instance.
37
+ #[allow(clippy::too_many_arguments)]
38
+ fn new(
39
+ name: Option<String>,
40
+ encoding: Option<String>,
41
+ default_word_cost: Option<i16>,
42
+ default_left_context_id: Option<u16>,
43
+ default_right_context_id: Option<u16>,
44
+ default_field_value: Option<String>,
45
+ flexible_csv: Option<bool>,
46
+ skip_invalid_cost_or_id: Option<bool>,
47
+ normalize_details: Option<bool>,
48
+ ) -> Self {
49
+ RbMetadata {
50
+ inner: CoreMetadata::new(
51
+ name,
52
+ encoding,
53
+ default_word_cost,
54
+ default_left_context_id,
55
+ default_right_context_id,
56
+ default_field_value,
57
+ flexible_csv,
58
+ skip_invalid_cost_or_id,
59
+ normalize_details,
60
+ None,
61
+ None,
62
+ ),
63
+ }
64
+ }
65
+
66
+ /// Creates a default metadata instance.
67
+ ///
68
+ /// # Returns
69
+ ///
70
+ /// A new `RbMetadata` with default values.
71
+ fn create_default() -> Self {
72
+ RbMetadata::new(None, None, None, None, None, None, None, None, None)
73
+ }
74
+
75
+ /// Loads metadata from a JSON file.
76
+ ///
77
+ /// # Arguments
78
+ ///
79
+ /// * `path` - Path to the JSON file.
80
+ ///
81
+ /// # Returns
82
+ ///
83
+ /// A new `RbMetadata` loaded from the file.
84
+ fn from_json_file(path: String) -> Result<Self, Error> {
85
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
86
+
87
+ let json_str = std::fs::read_to_string(&path).map_err(|e| {
88
+ Error::new(
89
+ ruby.exception_io_error(),
90
+ format!("Failed to read file: {e}"),
91
+ )
92
+ })?;
93
+
94
+ let metadata: Metadata = serde_json::from_str(&json_str).map_err(|e| {
95
+ Error::new(
96
+ ruby.exception_arg_error(),
97
+ format!("Failed to parse JSON: {e}"),
98
+ )
99
+ })?;
100
+
101
+ Ok(metadata.into())
102
+ }
103
+
104
+ /// Returns the dictionary name.
105
+ fn name(&self) -> String {
106
+ self.inner.name.clone()
107
+ }
108
+
109
+ /// Returns the character encoding.
110
+ fn encoding(&self) -> String {
111
+ self.inner.encoding.clone()
112
+ }
113
+
114
+ /// Returns the default word cost.
115
+ fn default_word_cost(&self) -> i16 {
116
+ self.inner.default_word_cost
117
+ }
118
+
119
+ /// Returns the default left context ID.
120
+ fn default_left_context_id(&self) -> u16 {
121
+ self.inner.default_left_context_id
122
+ }
123
+
124
+ /// Returns the default right context ID.
125
+ fn default_right_context_id(&self) -> u16 {
126
+ self.inner.default_right_context_id
127
+ }
128
+
129
+ /// Returns the default field value.
130
+ fn default_field_value(&self) -> String {
131
+ self.inner.default_field_value.clone()
132
+ }
133
+
134
+ /// Returns whether flexible CSV parsing is enabled.
135
+ fn flexible_csv(&self) -> bool {
136
+ self.inner.flexible_csv
137
+ }
138
+
139
+ /// Returns whether invalid cost/ID entries should be skipped.
140
+ fn skip_invalid_cost_or_id(&self) -> bool {
141
+ self.inner.skip_invalid_cost_or_id
142
+ }
143
+
144
+ /// Returns whether morphological details should be normalized.
145
+ fn normalize_details(&self) -> bool {
146
+ self.inner.normalize_details
147
+ }
148
+
149
+ /// Converts the metadata to a Ruby hash.
150
+ ///
151
+ /// # Returns
152
+ ///
153
+ /// A HashMap of metadata properties, all as strings. The
154
+ /// `"space_penalty"` entry holds the left-space penalty rules the
155
+ /// dictionary ships as a JSON string, and is present only when it ships
156
+ /// some (ko-dic does).
157
+ fn to_hash(&self) -> HashMap<String, String> {
158
+ let mut dict = HashMap::new();
159
+ dict.insert("name".to_string(), self.inner.name.clone());
160
+ dict.insert("encoding".to_string(), self.inner.encoding.clone());
161
+ dict.insert(
162
+ "default_word_cost".to_string(),
163
+ self.inner.default_word_cost.to_string(),
164
+ );
165
+ dict.insert(
166
+ "default_left_context_id".to_string(),
167
+ self.inner.default_left_context_id.to_string(),
168
+ );
169
+ dict.insert(
170
+ "default_right_context_id".to_string(),
171
+ self.inner.default_right_context_id.to_string(),
172
+ );
173
+ dict.insert(
174
+ "default_field_value".to_string(),
175
+ self.inner.default_field_value.clone(),
176
+ );
177
+ dict.insert(
178
+ "flexible_csv".to_string(),
179
+ self.inner.flexible_csv.to_string(),
180
+ );
181
+ dict.insert(
182
+ "skip_invalid_cost_or_id".to_string(),
183
+ self.inner.skip_invalid_cost_or_id.to_string(),
184
+ );
185
+ dict.insert(
186
+ "normalize_details".to_string(),
187
+ self.inner.normalize_details.to_string(),
188
+ );
189
+ dict.insert(
190
+ "dictionary_schema_fields".to_string(),
191
+ self.inner.dictionary_schema.fields().join(","),
192
+ );
193
+ dict.insert(
194
+ "user_dictionary_schema_fields".to_string(),
195
+ self.inner.user_dictionary_schema.fields().join(","),
196
+ );
197
+ if let Some(space_penalty) = self.inner.space_penalty_json() {
198
+ dict.insert("space_penalty".to_string(), space_penalty);
199
+ }
200
+ dict
201
+ }
202
+
203
+ /// Returns the string representation of the metadata.
204
+ fn to_s(&self) -> String {
205
+ format!(
206
+ "Metadata(name='{}', encoding='{}')",
207
+ self.inner.name, self.inner.encoding,
208
+ )
209
+ }
210
+
211
+ /// Returns the inspect representation of the metadata.
212
+ fn inspect(&self) -> String {
213
+ format!(
214
+ "#<Lindera::Metadata: name='{}', encoding='{}', schema_fields={}>",
215
+ self.inner.name,
216
+ self.inner.encoding,
217
+ self.inner.dictionary_schema.field_count()
218
+ )
219
+ }
220
+ }
221
+
222
+ impl From<RbMetadata> for Metadata {
223
+ fn from(metadata: RbMetadata) -> Self {
224
+ metadata.inner.into()
225
+ }
226
+ }
227
+
228
+ impl From<Metadata> for RbMetadata {
229
+ fn from(metadata: Metadata) -> Self {
230
+ RbMetadata {
231
+ inner: CoreMetadata::from(metadata),
232
+ }
233
+ }
234
+ }
235
+
236
+ /// Defines Metadata class in the given Ruby module.
237
+ ///
238
+ /// # Arguments
239
+ ///
240
+ /// * `ruby` - Ruby runtime handle.
241
+ /// * `module` - Parent Ruby module.
242
+ ///
243
+ /// # Returns
244
+ ///
245
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
246
+ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
247
+ let metadata_class = module.define_class("Metadata", ruby.class_object())?;
248
+ metadata_class.define_singleton_method("new", function!(RbMetadata::new, 9))?;
249
+ metadata_class
250
+ .define_singleton_method("create_default", function!(RbMetadata::create_default, 0))?;
251
+ metadata_class
252
+ .define_singleton_method("from_json_file", function!(RbMetadata::from_json_file, 1))?;
253
+ metadata_class.define_method("name", method!(RbMetadata::name, 0))?;
254
+ metadata_class.define_method("encoding", method!(RbMetadata::encoding, 0))?;
255
+ metadata_class.define_method(
256
+ "default_word_cost",
257
+ method!(RbMetadata::default_word_cost, 0),
258
+ )?;
259
+ metadata_class.define_method(
260
+ "default_left_context_id",
261
+ method!(RbMetadata::default_left_context_id, 0),
262
+ )?;
263
+ metadata_class.define_method(
264
+ "default_right_context_id",
265
+ method!(RbMetadata::default_right_context_id, 0),
266
+ )?;
267
+ metadata_class.define_method(
268
+ "default_field_value",
269
+ method!(RbMetadata::default_field_value, 0),
270
+ )?;
271
+ metadata_class.define_method("flexible_csv", method!(RbMetadata::flexible_csv, 0))?;
272
+ metadata_class.define_method(
273
+ "skip_invalid_cost_or_id",
274
+ method!(RbMetadata::skip_invalid_cost_or_id, 0),
275
+ )?;
276
+ metadata_class.define_method(
277
+ "normalize_details",
278
+ method!(RbMetadata::normalize_details, 0),
279
+ )?;
280
+ metadata_class.define_method("to_hash", method!(RbMetadata::to_hash, 0))?;
281
+ metadata_class.define_method("to_h", method!(RbMetadata::to_hash, 0))?;
282
+ metadata_class.define_method("to_s", method!(RbMetadata::to_s, 0))?;
283
+ metadata_class.define_method("inspect", method!(RbMetadata::inspect, 0))?;
284
+
285
+ Ok(())
286
+ }
287
+
288
+ #[cfg(test)]
289
+ mod tests {
290
+ use super::*;
291
+ use lindera_binding_core::CoreSchema;
292
+
293
+ #[test]
294
+ fn test_rb_metadata_to_lindera_metadata() {
295
+ let rb_metadata = RbMetadata {
296
+ inner: CoreMetadata::new(
297
+ Some("test_dict".to_string()),
298
+ Some("EUC-JP".to_string()),
299
+ Some(-5000),
300
+ Some(100),
301
+ Some(200),
302
+ Some("N/A".to_string()),
303
+ Some(true),
304
+ Some(true),
305
+ Some(true),
306
+ Some(CoreSchema::new(vec![
307
+ "surface".to_string(),
308
+ "cost".to_string(),
309
+ ])),
310
+ Some(CoreSchema::new(vec!["surface".to_string()])),
311
+ ),
312
+ };
313
+
314
+ let lindera_metadata: Metadata = rb_metadata.into();
315
+ assert_eq!(lindera_metadata.name, "test_dict");
316
+ assert_eq!(lindera_metadata.encoding, "EUC-JP");
317
+ assert_eq!(lindera_metadata.default_word_cost, -5000);
318
+ assert_eq!(lindera_metadata.default_left_context_id, 100);
319
+ assert_eq!(lindera_metadata.default_right_context_id, 200);
320
+ assert_eq!(lindera_metadata.default_field_value, "N/A");
321
+ assert!(lindera_metadata.flexible_csv);
322
+ assert!(lindera_metadata.skip_invalid_cost_or_id);
323
+ assert!(lindera_metadata.normalize_details);
324
+ assert_eq!(lindera_metadata.dictionary_schema.get_all_fields().len(), 2);
325
+ assert_eq!(
326
+ lindera_metadata
327
+ .user_dictionary_schema
328
+ .get_all_fields()
329
+ .len(),
330
+ 1
331
+ );
332
+ }
333
+
334
+ #[test]
335
+ fn test_lindera_metadata_to_rb_metadata() {
336
+ let dict_schema =
337
+ lindera::dictionary::Schema::new(vec!["surface".to_string(), "cost".to_string()]);
338
+ let user_schema =
339
+ lindera::dictionary::Schema::new(vec!["surface".to_string(), "reading".to_string()]);
340
+
341
+ let lindera_metadata = Metadata::new(
342
+ "my_dict".to_string(),
343
+ "UTF-8".to_string(),
344
+ -8000,
345
+ 500,
346
+ 600,
347
+ "?".to_string(),
348
+ false,
349
+ true,
350
+ false,
351
+ dict_schema,
352
+ user_schema,
353
+ );
354
+
355
+ let rb_metadata: RbMetadata = lindera_metadata.into();
356
+ assert_eq!(rb_metadata.name(), "my_dict");
357
+ assert_eq!(rb_metadata.encoding(), "UTF-8");
358
+ assert_eq!(rb_metadata.default_word_cost(), -8000);
359
+ assert_eq!(rb_metadata.default_left_context_id(), 500);
360
+ assert_eq!(rb_metadata.default_right_context_id(), 600);
361
+ assert_eq!(rb_metadata.default_field_value(), "?");
362
+ assert!(!rb_metadata.flexible_csv());
363
+ assert!(rb_metadata.skip_invalid_cost_or_id());
364
+ assert!(!rb_metadata.normalize_details());
365
+ assert_eq!(rb_metadata.inner.dictionary_schema.field_count(), 2);
366
+ assert_eq!(rb_metadata.inner.user_dictionary_schema.field_count(), 2);
367
+ }
368
+
369
+ #[test]
370
+ fn test_rb_metadata_defaults() {
371
+ let rb_metadata = RbMetadata::create_default();
372
+ assert_eq!(rb_metadata.name(), "default");
373
+ assert_eq!(rb_metadata.encoding(), "UTF-8");
374
+ assert_eq!(rb_metadata.default_word_cost(), -10000);
375
+ assert_eq!(rb_metadata.default_left_context_id(), 1288);
376
+ assert_eq!(rb_metadata.default_right_context_id(), 1288);
377
+ assert_eq!(rb_metadata.default_field_value(), "*");
378
+ assert!(!rb_metadata.flexible_csv());
379
+ assert_eq!(rb_metadata.inner.dictionary_schema.field_count(), 13);
380
+ assert_eq!(rb_metadata.inner.user_dictionary_schema.field_count(), 3);
381
+ }
382
+
383
+ #[test]
384
+ fn test_rb_metadata_roundtrip() {
385
+ let rb_metadata = RbMetadata::create_default();
386
+ let lindera: Metadata = rb_metadata.into();
387
+ let back: RbMetadata = lindera.into();
388
+ assert_eq!(back.name(), "default");
389
+ assert_eq!(back.encoding(), "UTF-8");
390
+ assert_eq!(back.default_word_cost(), -10000);
391
+ assert_eq!(back.default_left_context_id(), 1288);
392
+ assert_eq!(back.default_right_context_id(), 1288);
393
+ assert_eq!(back.default_field_value(), "*");
394
+ assert_eq!(back.inner.dictionary_schema.field_count(), 13);
395
+ assert_eq!(back.inner.user_dictionary_schema.field_count(), 3);
396
+ }
397
+
398
+ #[test]
399
+ fn test_rb_metadata_to_hash_omits_space_penalty_when_none() {
400
+ let hash = RbMetadata::create_default().to_hash();
401
+ assert!(!hash.contains_key("space_penalty"));
402
+ }
403
+
404
+ /// ko-dic's `metadata.json` ships left-space penalty rules: `to_hash`
405
+ /// exposes them, and converting back for a dictionary build keeps them.
406
+ #[test]
407
+ fn test_rb_metadata_keeps_ko_dic_space_penalty() {
408
+ let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
409
+ .join("../lindera-ko-dic/metadata.json");
410
+ let json = std::fs::read_to_string(path).unwrap();
411
+ let metadata: Metadata = serde_json::from_str(&json).unwrap();
412
+ let shipped = metadata.space_penalty.clone().unwrap();
413
+
414
+ let rb_metadata: RbMetadata = metadata.into();
415
+ let hash = rb_metadata.to_hash();
416
+ let space_penalty = hash.get("space_penalty").unwrap();
417
+ assert!(space_penalty.contains("\"rules\""));
418
+ assert!(space_penalty.contains("JKS"));
419
+
420
+ let rebuilt: Metadata = rb_metadata.into();
421
+ assert_eq!(rebuilt.space_penalty, Some(shipped));
422
+ }
423
+ }