lindera 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.toml +71 -37
- data/LICENSE +21 -0
- data/README.md +23 -6
- data/extconf.rb +13 -0
- data/src/character_filter.rs +3 -0
- data/src/dictionary.rs +239 -0
- data/src/error.rs +19 -0
- data/src/lib.rs +90 -0
- data/src/metadata.rs +423 -0
- data/src/mode.rs +401 -0
- data/src/schema.rs +568 -0
- data/src/token.rs +188 -0
- data/src/token_filter.rs +3 -0
- data/src/tokenizer.rs +401 -0
- data/src/trainer.rs +283 -0
- data/src/util.rs +237 -0
- metadata +17 -4
- data/ext/lindera_ruby/extconf.rb +0 -8
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 529da66cbb1d0fd3372e0ab7695c72ac53c55d8a1a411b99d3db0135ac05be56
|
|
4
|
+
data.tar.gz: '020678eec7845a566b1572f2a61af447252dc0090a5d9c73f183cddf4e656f46'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a42a702e498f3690ba0ccc3288d09c0d3e5dcc457905d058c101d112cfe24ecd4fda6cedb0e92a675a519880f6d5383efb4b1519fa1111b0b17ecaa5778e6f68
|
|
7
|
+
data.tar.gz: 57dcbda79af341e03428c7eb963e407e22bdf78a23687e95be022cdb3b91789c9c1a3b97d01b500ff44e8f700a4ea109b5262866b699d8966ae4f2597546f7d9
|
data/Cargo.toml
CHANGED
|
@@ -1,47 +1,81 @@
|
|
|
1
|
+
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
|
2
|
+
#
|
|
3
|
+
# When uploading crates to the registry Cargo will automatically
|
|
4
|
+
# "normalize" Cargo.toml files for maximal compatibility
|
|
5
|
+
# with all versions of Cargo and also rewrite `path` dependencies
|
|
6
|
+
# to registry (e.g., crates.io) dependencies.
|
|
7
|
+
#
|
|
8
|
+
# If you are reading this file be aware that the original Cargo.toml
|
|
9
|
+
# will likely look very different (and much more reasonable).
|
|
10
|
+
# See Cargo.toml.orig for the original contents.
|
|
11
|
+
|
|
1
12
|
[package]
|
|
13
|
+
edition = "2024"
|
|
14
|
+
rust-version = "1.88"
|
|
2
15
|
name = "lindera-ruby"
|
|
3
|
-
|
|
16
|
+
version = "6.2.0"
|
|
17
|
+
build = false
|
|
4
18
|
publish = false
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
19
|
+
autolib = false
|
|
20
|
+
autobins = false
|
|
21
|
+
autoexamples = false
|
|
22
|
+
autotests = false
|
|
23
|
+
autobenches = false
|
|
8
24
|
description = "A Ruby binding for Lindera."
|
|
25
|
+
homepage = "https://github.com/lindera/lindera"
|
|
9
26
|
documentation = "https://docs.rs/lindera-ruby"
|
|
10
|
-
homepage = { workspace = true }
|
|
11
|
-
repository = { workspace = true }
|
|
12
27
|
readme = "README.md"
|
|
13
|
-
keywords = [
|
|
14
|
-
|
|
15
|
-
|
|
28
|
+
keywords = [
|
|
29
|
+
"morphological",
|
|
30
|
+
"analysis",
|
|
31
|
+
"library",
|
|
32
|
+
"ruby",
|
|
33
|
+
"binding",
|
|
34
|
+
]
|
|
35
|
+
categories = ["text-processing"]
|
|
36
|
+
license = "MIT"
|
|
37
|
+
repository = "https://github.com/lindera/lindera"
|
|
38
|
+
resolver = "2"
|
|
39
|
+
|
|
40
|
+
[features]
|
|
41
|
+
default = ["train"]
|
|
42
|
+
embed-cc-cedict = ["lindera/embed-cc-cedict"]
|
|
43
|
+
embed-cjk = ["lindera/embed-cjk"]
|
|
44
|
+
embed-ipadic = ["lindera/embed-ipadic"]
|
|
45
|
+
embed-ipadic-neologd = ["lindera/embed-ipadic-neologd"]
|
|
46
|
+
embed-jieba = ["lindera/embed-jieba"]
|
|
47
|
+
embed-ko-dic = ["lindera/embed-ko-dic"]
|
|
48
|
+
embed-sudachidict = ["lindera/embed-sudachidict"]
|
|
49
|
+
embed-unidic = ["lindera/embed-unidic"]
|
|
50
|
+
train = ["lindera/train"]
|
|
16
51
|
|
|
17
52
|
[lib]
|
|
18
53
|
name = "lindera_ruby"
|
|
19
|
-
crate-type = [
|
|
54
|
+
crate-type = [
|
|
55
|
+
"cdylib",
|
|
56
|
+
"lib",
|
|
57
|
+
]
|
|
58
|
+
path = "src/lib.rs"
|
|
20
59
|
|
|
21
|
-
[
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
[dependencies]
|
|
42
|
-
|
|
43
|
-
lindera-binding-core = { workspace = true }
|
|
44
|
-
magnus = { version = "0.8.2", features = ["rb-sys"] }
|
|
45
|
-
num_cpus = { workspace = true }
|
|
46
|
-
serde = { workspace = true, features = ["derive"] }
|
|
47
|
-
serde_json = { workspace = true }
|
|
60
|
+
[dependencies.lindera]
|
|
61
|
+
version = "=6.2.0"
|
|
62
|
+
|
|
63
|
+
[dependencies.lindera-binding-core]
|
|
64
|
+
version = "=6.2.0"
|
|
65
|
+
|
|
66
|
+
[dependencies.magnus]
|
|
67
|
+
version = "0.8.2"
|
|
68
|
+
features = ["rb-sys"]
|
|
69
|
+
|
|
70
|
+
[dependencies.num_cpus]
|
|
71
|
+
version = "1.17.0"
|
|
72
|
+
|
|
73
|
+
[dependencies.serde]
|
|
74
|
+
version = "1.0.229"
|
|
75
|
+
features = [
|
|
76
|
+
"derive",
|
|
77
|
+
"derive",
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
[dependencies.serde_json]
|
|
81
|
+
version = "1.0.151"
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2019 by the project authors.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
CHANGED
|
@@ -15,15 +15,24 @@ lindera-ruby provides a Ruby interface to the Lindera morphological analysis eng
|
|
|
15
15
|
|
|
16
16
|
## Requirements
|
|
17
17
|
|
|
18
|
-
- Ruby >= 3.1
|
|
19
|
-
- Rust >= 1.
|
|
18
|
+
- Ruby >= 3.1, with a C compiler and `make` for building native gems
|
|
19
|
+
- Rust >= 1.88
|
|
20
|
+
- libclang (`libclang-dev` on Debian/Ubuntu; the Xcode command line tools on macOS)
|
|
20
21
|
|
|
21
|
-
##
|
|
22
|
+
## Install
|
|
22
23
|
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
From v6.1.0 the gem installs from RubyGems. It compiles the native extension
|
|
25
|
+
during installation, which takes a few minutes:
|
|
25
26
|
|
|
26
|
-
|
|
27
|
+
```bash
|
|
28
|
+
gem install lindera
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The gem embeds no dictionary (see [Dictionary](#dictionary)). To embed one,
|
|
32
|
+
pass the Cargo features at install time, e.g.
|
|
33
|
+
`LINDERA_FEATURES="embed-ipadic" gem install lindera`.
|
|
34
|
+
|
|
35
|
+
### Build from Source
|
|
27
36
|
|
|
28
37
|
```bash
|
|
29
38
|
cd lindera-ruby
|
|
@@ -31,6 +40,14 @@ bundle install
|
|
|
31
40
|
bundle exec rake compile
|
|
32
41
|
```
|
|
33
42
|
|
|
43
|
+
`bundle exec rake build` writes the source gem to `pkg/`. Use it instead of
|
|
44
|
+
`gem build`, which cannot package the crate outside the Cargo workspace.
|
|
45
|
+
|
|
46
|
+
## Dictionary
|
|
47
|
+
|
|
48
|
+
Pre-built dictionaries are available from [GitHub Releases](https://github.com/lindera/lindera/releases).
|
|
49
|
+
Download a dictionary archive (e.g. `lindera-ipadic-*.zip`) and extract it to a local path.
|
|
50
|
+
|
|
34
51
|
## Usage
|
|
35
52
|
|
|
36
53
|
```ruby
|
data/extconf.rb
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Builds the extension for both `bundle exec rake compile` and `gem install`
|
|
2
|
+
# (the gemspec lists this file as the extension). Cargo.toml sits next to it in
|
|
3
|
+
# both layouts, so rb_sys finds the crate without further configuration.
|
|
4
|
+
require "mkmf"
|
|
5
|
+
require "rb_sys/mkmf"
|
|
6
|
+
|
|
7
|
+
create_rust_makefile("lindera/lindera_ruby") do |r|
|
|
8
|
+
# Pass embed features via LINDERA_FEATURES environment variable, e.g.
|
|
9
|
+
# LINDERA_FEATURES="embed-ipadic" bundle exec rake compile
|
|
10
|
+
# LINDERA_FEATURES="embed-ipadic" gem install lindera
|
|
11
|
+
features = ENV.fetch("LINDERA_FEATURES", "").split(",").map(&:strip).reject(&:empty?)
|
|
12
|
+
r.features = features unless features.empty?
|
|
13
|
+
end
|
data/src/dictionary.rs
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
//! Dictionary management for morphological analysis.
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides functionality for building, loading, and managing dictionaries
|
|
4
|
+
//! used in morphological analysis.
|
|
5
|
+
|
|
6
|
+
use std::path::Path;
|
|
7
|
+
|
|
8
|
+
use magnus::prelude::*;
|
|
9
|
+
use magnus::{Error, Ruby, function, method};
|
|
10
|
+
|
|
11
|
+
use lindera::dictionary::{
|
|
12
|
+
Dictionary, DictionaryBuilder, Metadata, UserDictionary,
|
|
13
|
+
load_dictionary as lindera_load_dictionary,
|
|
14
|
+
load_user_dictionary as lindera_load_user_dictionary,
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
use crate::error::to_magnus_error;
|
|
18
|
+
use crate::metadata::RbMetadata;
|
|
19
|
+
|
|
20
|
+
/// A morphological analysis dictionary.
|
|
21
|
+
///
|
|
22
|
+
/// Contains the data structures needed for tokenization and morphological analysis.
|
|
23
|
+
#[magnus::wrap(class = "Lindera::Dictionary", free_immediately, size)]
|
|
24
|
+
#[derive(Clone)]
|
|
25
|
+
pub struct RbDictionary {
|
|
26
|
+
/// Inner Lindera dictionary.
|
|
27
|
+
pub inner: Dictionary,
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
impl RbDictionary {
|
|
31
|
+
/// Returns the name of the dictionary metadata.
|
|
32
|
+
///
|
|
33
|
+
/// # Returns
|
|
34
|
+
///
|
|
35
|
+
/// The dictionary metadata name.
|
|
36
|
+
fn metadata_name(&self) -> String {
|
|
37
|
+
self.inner.metadata.name.clone()
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/// Returns the character encoding of the dictionary.
|
|
41
|
+
///
|
|
42
|
+
/// # Returns
|
|
43
|
+
///
|
|
44
|
+
/// The dictionary encoding string.
|
|
45
|
+
fn metadata_encoding(&self) -> String {
|
|
46
|
+
self.inner.metadata.encoding.clone()
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/// Returns the full metadata object.
|
|
50
|
+
///
|
|
51
|
+
/// # Returns
|
|
52
|
+
///
|
|
53
|
+
/// The dictionary metadata.
|
|
54
|
+
fn metadata(&self) -> RbMetadata {
|
|
55
|
+
RbMetadata::from((*self.inner.metadata).clone())
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/// Returns the string representation.
|
|
59
|
+
fn to_s(&self) -> String {
|
|
60
|
+
"Dictionary".to_string()
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/// Returns the inspect representation.
|
|
64
|
+
fn inspect(&self) -> String {
|
|
65
|
+
format!(
|
|
66
|
+
"#<Lindera::Dictionary: name='{}'>",
|
|
67
|
+
self.inner.metadata.name
|
|
68
|
+
)
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/// A user-defined dictionary for custom words.
|
|
73
|
+
///
|
|
74
|
+
/// User dictionaries allow you to add custom words and their morphological features
|
|
75
|
+
/// that are not present in the main dictionary.
|
|
76
|
+
#[magnus::wrap(class = "Lindera::UserDictionary", free_immediately, size)]
|
|
77
|
+
#[derive(Clone)]
|
|
78
|
+
pub struct RbUserDictionary {
|
|
79
|
+
/// Inner Lindera user dictionary.
|
|
80
|
+
pub inner: UserDictionary,
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
impl RbUserDictionary {
|
|
84
|
+
/// Returns the string representation.
|
|
85
|
+
fn to_s(&self) -> String {
|
|
86
|
+
"UserDictionary".to_string()
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/// Returns the inspect representation.
|
|
90
|
+
fn inspect(&self) -> String {
|
|
91
|
+
"UserDictionary()".to_string()
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/// Loads a dictionary from the specified URI.
|
|
96
|
+
///
|
|
97
|
+
/// # Arguments
|
|
98
|
+
///
|
|
99
|
+
/// * `uri` - URI to the dictionary. Can be a file path or embedded dictionary name.
|
|
100
|
+
///
|
|
101
|
+
/// # Returns
|
|
102
|
+
///
|
|
103
|
+
/// A loaded `RbDictionary` object.
|
|
104
|
+
fn load_dictionary(uri: String) -> Result<RbDictionary, Error> {
|
|
105
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
106
|
+
lindera_load_dictionary(&uri)
|
|
107
|
+
.map_err(|e| {
|
|
108
|
+
to_magnus_error(
|
|
109
|
+
&ruby,
|
|
110
|
+
format!("Failed to load dictionary from '{uri}': {e}"),
|
|
111
|
+
)
|
|
112
|
+
})
|
|
113
|
+
.map(|d| RbDictionary { inner: d })
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/// Loads a user dictionary from the specified URI.
|
|
117
|
+
///
|
|
118
|
+
/// # Arguments
|
|
119
|
+
///
|
|
120
|
+
/// * `uri` - URI to the user dictionary directory.
|
|
121
|
+
/// * `metadata` - Metadata configuration for the user dictionary.
|
|
122
|
+
///
|
|
123
|
+
/// # Returns
|
|
124
|
+
///
|
|
125
|
+
/// A loaded `RbUserDictionary` object.
|
|
126
|
+
fn load_user_dictionary(uri: String, metadata: &RbMetadata) -> Result<RbUserDictionary, Error> {
|
|
127
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
128
|
+
let meta: Metadata = metadata.clone().into();
|
|
129
|
+
lindera_load_user_dictionary(&uri, &meta)
|
|
130
|
+
.map_err(|e| {
|
|
131
|
+
to_magnus_error(
|
|
132
|
+
&ruby,
|
|
133
|
+
format!("Failed to load user dictionary from '{uri}': {e}"),
|
|
134
|
+
)
|
|
135
|
+
})
|
|
136
|
+
.map(|d| RbUserDictionary { inner: d })
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/// Builds a dictionary from source files.
|
|
140
|
+
///
|
|
141
|
+
/// # Arguments
|
|
142
|
+
///
|
|
143
|
+
/// * `input_dir` - Directory containing dictionary source files.
|
|
144
|
+
/// * `output_dir` - Directory where the built dictionary will be saved.
|
|
145
|
+
/// * `metadata` - Metadata configuration for the dictionary.
|
|
146
|
+
fn build_dictionary(
|
|
147
|
+
input_dir: String,
|
|
148
|
+
output_dir: String,
|
|
149
|
+
metadata: &RbMetadata,
|
|
150
|
+
) -> Result<(), Error> {
|
|
151
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
152
|
+
let input_path = Path::new(&input_dir);
|
|
153
|
+
let output_path = Path::new(&output_dir);
|
|
154
|
+
|
|
155
|
+
if !input_path.exists() {
|
|
156
|
+
return Err(Error::new(
|
|
157
|
+
ruby.exception_arg_error(),
|
|
158
|
+
format!("Input directory does not exist: {input_dir}"),
|
|
159
|
+
));
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
let builder = DictionaryBuilder::new(metadata.clone().into());
|
|
163
|
+
builder
|
|
164
|
+
.build_dictionary(input_path, output_path)
|
|
165
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to build dictionary: {e}")))?;
|
|
166
|
+
|
|
167
|
+
Ok(())
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/// Builds a user dictionary from a CSV file.
|
|
171
|
+
///
|
|
172
|
+
/// # Arguments
|
|
173
|
+
///
|
|
174
|
+
/// * `_kind` - Dictionary kind (reserved for future use).
|
|
175
|
+
/// * `input_file` - Path to the CSV file.
|
|
176
|
+
/// * `output_dir` - Directory where the built user dictionary will be saved.
|
|
177
|
+
/// * `metadata` - Optional metadata configuration.
|
|
178
|
+
fn build_user_dictionary(
|
|
179
|
+
_kind: String,
|
|
180
|
+
input_file: String,
|
|
181
|
+
output_dir: String,
|
|
182
|
+
metadata: Option<&RbMetadata>,
|
|
183
|
+
) -> Result<(), Error> {
|
|
184
|
+
let ruby = Ruby::get().expect("Ruby runtime not initialized");
|
|
185
|
+
let input_path = Path::new(&input_file);
|
|
186
|
+
let output_path = Path::new(&output_dir);
|
|
187
|
+
|
|
188
|
+
if !input_path.exists() {
|
|
189
|
+
return Err(Error::new(
|
|
190
|
+
ruby.exception_arg_error(),
|
|
191
|
+
format!("Input file does not exist: {input_file}"),
|
|
192
|
+
));
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
let meta = match metadata {
|
|
196
|
+
Some(m) => m.clone().into(),
|
|
197
|
+
None => Metadata::default(),
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
let builder = DictionaryBuilder::new(meta);
|
|
201
|
+
builder
|
|
202
|
+
.build_user_dictionary(input_path, output_path)
|
|
203
|
+
.map_err(|e| to_magnus_error(&ruby, format!("Failed to build user dictionary: {e}")))?;
|
|
204
|
+
|
|
205
|
+
Ok(())
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/// Defines Dictionary, UserDictionary classes and module functions in the given Ruby module.
|
|
209
|
+
///
|
|
210
|
+
/// # Arguments
|
|
211
|
+
///
|
|
212
|
+
/// * `ruby` - Ruby runtime handle.
|
|
213
|
+
/// * `module` - Parent Ruby module.
|
|
214
|
+
///
|
|
215
|
+
/// # Returns
|
|
216
|
+
///
|
|
217
|
+
/// `Ok(())` on success, or a Magnus `Error` on failure.
|
|
218
|
+
pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
|
|
219
|
+
let dict_class = module.define_class("Dictionary", ruby.class_object())?;
|
|
220
|
+
dict_class.define_method("metadata_name", method!(RbDictionary::metadata_name, 0))?;
|
|
221
|
+
dict_class.define_method(
|
|
222
|
+
"metadata_encoding",
|
|
223
|
+
method!(RbDictionary::metadata_encoding, 0),
|
|
224
|
+
)?;
|
|
225
|
+
dict_class.define_method("metadata", method!(RbDictionary::metadata, 0))?;
|
|
226
|
+
dict_class.define_method("to_s", method!(RbDictionary::to_s, 0))?;
|
|
227
|
+
dict_class.define_method("inspect", method!(RbDictionary::inspect, 0))?;
|
|
228
|
+
|
|
229
|
+
let user_dict_class = module.define_class("UserDictionary", ruby.class_object())?;
|
|
230
|
+
user_dict_class.define_method("to_s", method!(RbUserDictionary::to_s, 0))?;
|
|
231
|
+
user_dict_class.define_method("inspect", method!(RbUserDictionary::inspect, 0))?;
|
|
232
|
+
|
|
233
|
+
module.define_module_function("load_dictionary", function!(load_dictionary, 1))?;
|
|
234
|
+
module.define_module_function("load_user_dictionary", function!(load_user_dictionary, 2))?;
|
|
235
|
+
module.define_module_function("build_dictionary", function!(build_dictionary, 3))?;
|
|
236
|
+
module.define_module_function("build_user_dictionary", function!(build_user_dictionary, 4))?;
|
|
237
|
+
|
|
238
|
+
Ok(())
|
|
239
|
+
}
|
data/src/error.rs
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
//! Error types for Lindera operations.
|
|
2
|
+
//!
|
|
3
|
+
//! This module provides error types used throughout the Lindera Ruby bindings.
|
|
4
|
+
|
|
5
|
+
use magnus::{Error, Ruby};
|
|
6
|
+
|
|
7
|
+
/// Converts a Lindera error message into a Magnus Ruby error.
|
|
8
|
+
///
|
|
9
|
+
/// # Arguments
|
|
10
|
+
///
|
|
11
|
+
/// * `ruby` - Ruby runtime handle.
|
|
12
|
+
/// * `msg` - Error message string.
|
|
13
|
+
///
|
|
14
|
+
/// # Returns
|
|
15
|
+
///
|
|
16
|
+
/// A Magnus `Error` wrapping a Ruby `RuntimeError`.
|
|
17
|
+
pub fn to_magnus_error(ruby: &Ruby, msg: String) -> Error {
|
|
18
|
+
Error::new(ruby.exception_runtime_error(), msg)
|
|
19
|
+
}
|
data/src/lib.rs
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
//! # Lindera Ruby Bindings
|
|
2
|
+
//!
|
|
3
|
+
//! Ruby bindings for [Lindera](https://github.com/lindera/lindera), a morphological analysis library for CJK text.
|
|
4
|
+
//!
|
|
5
|
+
//! Lindera provides high-performance tokenization and morphological analysis for:
|
|
6
|
+
//! - Japanese (IPADIC, IPADIC NEologd, UniDic)
|
|
7
|
+
//! - Korean (ko-dic)
|
|
8
|
+
//! - Chinese (CC-CEDICT, Jieba)
|
|
9
|
+
//!
|
|
10
|
+
//! ## Features
|
|
11
|
+
//!
|
|
12
|
+
//! - **Dictionary management**: Build, load, and use custom dictionaries
|
|
13
|
+
//! - **Tokenization**: Multiple tokenization modes (normal, decompose)
|
|
14
|
+
//! - **Filters**: Character and token filtering pipeline
|
|
15
|
+
//! - **Training**: Train custom morphological models (with `train` feature)
|
|
16
|
+
//! - **User dictionaries**: Support for custom user dictionaries
|
|
17
|
+
//!
|
|
18
|
+
//! ## Examples
|
|
19
|
+
//!
|
|
20
|
+
//! ```ruby
|
|
21
|
+
//! require "lindera"
|
|
22
|
+
//!
|
|
23
|
+
//! # Create a tokenizer
|
|
24
|
+
//! builder = Lindera::TokenizerBuilder.new
|
|
25
|
+
//! tokenizer = builder.build
|
|
26
|
+
//!
|
|
27
|
+
//! # Tokenize text
|
|
28
|
+
//! tokens = tokenizer.tokenize("関西国際空港")
|
|
29
|
+
//! tokens.each { |token| puts "#{token.surface}: #{token.details}" }
|
|
30
|
+
//! ```
|
|
31
|
+
|
|
32
|
+
pub mod character_filter;
|
|
33
|
+
pub mod dictionary;
|
|
34
|
+
pub mod error;
|
|
35
|
+
pub mod metadata;
|
|
36
|
+
pub mod mode;
|
|
37
|
+
pub mod schema;
|
|
38
|
+
pub mod token;
|
|
39
|
+
pub mod token_filter;
|
|
40
|
+
pub mod tokenizer;
|
|
41
|
+
pub mod util;
|
|
42
|
+
|
|
43
|
+
#[cfg(feature = "train")]
|
|
44
|
+
pub mod trainer;
|
|
45
|
+
|
|
46
|
+
use magnus::{Error, Ruby, function};
|
|
47
|
+
|
|
48
|
+
/// Returns the version of the lindera-ruby package.
|
|
49
|
+
///
|
|
50
|
+
/// # Returns
|
|
51
|
+
///
|
|
52
|
+
/// A string with the package version.
|
|
53
|
+
fn version() -> String {
|
|
54
|
+
env!("CARGO_PKG_VERSION").to_string()
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/// Ruby extension initialization entry point.
|
|
58
|
+
///
|
|
59
|
+
/// Defines the `Lindera` module and all its classes and functions.
|
|
60
|
+
#[magnus::init]
|
|
61
|
+
fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
62
|
+
let module = ruby.define_module("Lindera")?;
|
|
63
|
+
|
|
64
|
+
// Version
|
|
65
|
+
module.define_module_function("version", function!(version, 0))?;
|
|
66
|
+
|
|
67
|
+
// Schema / FieldDefinition / FieldType
|
|
68
|
+
schema::define(ruby, &module)?;
|
|
69
|
+
|
|
70
|
+
// Metadata
|
|
71
|
+
metadata::define(ruby, &module)?;
|
|
72
|
+
|
|
73
|
+
// Mode / Penalty
|
|
74
|
+
mode::define(ruby, &module)?;
|
|
75
|
+
|
|
76
|
+
// Token
|
|
77
|
+
token::define(ruby, &module)?;
|
|
78
|
+
|
|
79
|
+
// Dictionary / UserDictionary + load/build functions
|
|
80
|
+
dictionary::define(ruby, &module)?;
|
|
81
|
+
|
|
82
|
+
// TokenizerBuilder / Tokenizer
|
|
83
|
+
tokenizer::define(ruby, &module)?;
|
|
84
|
+
|
|
85
|
+
// Trainer (feature = "train")
|
|
86
|
+
#[cfg(feature = "train")]
|
|
87
|
+
trainer::define(ruby, &module)?;
|
|
88
|
+
|
|
89
|
+
Ok(())
|
|
90
|
+
}
|