lindera 5.3.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '00944f67615f31f41975953623102b47755e4cdb8965c3270a881a9e2dfcf5e5'
4
- data.tar.gz: 14e9f373a2e402248ad0cf1c8c731223cc876431979ff5d5db70d88c2865a511
3
+ metadata.gz: 626630dcc1ced79783abf165ad0143ecb945f7ffe96365a09ceeba51a543303c
4
+ data.tar.gz: 03dcddc789f4aca2a0c46a7371b16d1554f4b81c01070fbe2717363146ab7afb
5
5
  SHA512:
6
- metadata.gz: d9a787c1733d3de604920c58ba3d120cdeb44eb3d68826b337985c5beec167189d1fd41c3205685d17653b36c96d691d7abe171861379d617160391a128d3ce7
7
- data.tar.gz: 34b35ab22834a478df2b8ac255f98b7d01a3e430951f38062dfba3c518bad7f515c788031f11308af8be22e8f4aff54fb08c00a63e97a44a221b6dc3290ebd1b
6
+ metadata.gz: fb0a91609265bbe897ce62edc052cea47621f5b5354cc6a6a184dc425f19ba9019c34b5e5af81811e00e96848deb52b59773b4a2f429f230de7a63961db837e2
7
+ data.tar.gz: 49b309e6355434b3ccb54ad7b35777f660c3371c8195c108dc4b0bc07ec3199d0c0d11b3885cc86b97ffec38030272591817828e2fe5daa24721a86de1522613
data/Cargo.toml CHANGED
@@ -1,41 +1,81 @@
1
+ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
2
+ #
3
+ # When uploading crates to the registry Cargo will automatically
4
+ # "normalize" Cargo.toml files for maximal compatibility
5
+ # with all versions of Cargo and also rewrite `path` dependencies
6
+ # to registry (e.g., crates.io) dependencies.
7
+ #
8
+ # If you are reading this file be aware that the original Cargo.toml
9
+ # will likely look very different (and much more reasonable).
10
+ # See Cargo.toml.orig for the original contents.
11
+
1
12
  [package]
13
+ edition = "2024"
14
+ rust-version = "1.88"
2
15
  name = "lindera-ruby"
3
- version = { workspace = true }
4
- edition = { workspace = true }
16
+ version = "6.1.0"
17
+ build = false
18
+ publish = false
19
+ autolib = false
20
+ autobins = false
21
+ autoexamples = false
22
+ autotests = false
23
+ autobenches = false
5
24
  description = "A Ruby binding for Lindera."
25
+ homepage = "https://github.com/lindera/lindera"
6
26
  documentation = "https://docs.rs/lindera-ruby"
7
- homepage = { workspace = true }
8
- repository = { workspace = true }
9
27
  readme = "README.md"
10
- keywords = ["morphological", "analysis", "library", "ruby", "binding"]
11
- categories = { workspace = true }
12
- license = { workspace = true }
28
+ keywords = [
29
+ "morphological",
30
+ "analysis",
31
+ "library",
32
+ "ruby",
33
+ "binding",
34
+ ]
35
+ categories = ["text-processing"]
36
+ license = "MIT"
37
+ repository = "https://github.com/lindera/lindera"
38
+ resolver = "2"
39
+
40
+ [features]
41
+ default = ["train"]
42
+ embed-cc-cedict = ["lindera/embed-cc-cedict"]
43
+ embed-cjk = ["lindera/embed-cjk"]
44
+ embed-ipadic = ["lindera/embed-ipadic"]
45
+ embed-ipadic-neologd = ["lindera/embed-ipadic-neologd"]
46
+ embed-jieba = ["lindera/embed-jieba"]
47
+ embed-ko-dic = ["lindera/embed-ko-dic"]
48
+ embed-sudachidict = ["lindera/embed-sudachidict"]
49
+ embed-unidic = ["lindera/embed-unidic"]
50
+ train = ["lindera/train"]
13
51
 
14
52
  [lib]
15
53
  name = "lindera_ruby"
16
- crate-type = ["cdylib", "lib"]
54
+ crate-type = [
55
+ "cdylib",
56
+ "lib",
57
+ ]
58
+ path = "src/lib.rs"
17
59
 
18
- [features]
19
- embed-ipadic = ["lindera/embed-ipadic"] # Include Japanese dictionary (IPADIC)
20
- embed-ipadic-neologd = [
21
- "lindera/embed-ipadic-neologd",
22
- ] # Include Japanese dictionary (IPADIC NEologd)
23
- embed-unidic = ["lindera/embed-unidic"] # Include Japanese dictionary (UniDic)
24
- embed-ko-dic = ["lindera/embed-ko-dic"] # Include Korean dictionary (ko-dic)
25
- embed-cc-cedict = [
26
- "lindera/embed-cc-cedict",
27
- ] # Include Chinese dictionary (CC-CEDICT)
28
- embed-jieba = ["lindera/embed-jieba"] # Include Chinese dictionary (Jieba)
29
- embed-cjk = [
30
- "lindera/embed-cjk",
31
- ] # Embed CJK dictionaries (IPADIC, ko-dic, Jieba)
32
- train = ["lindera/train"] # Enable training functionality
33
- default = ["train"] # No directories included
34
-
35
- [dependencies]
36
- lindera = { workspace = true }
37
- lindera-binding-core = { workspace = true }
38
- magnus = { version = "0.8.2", features = ["rb-sys"] }
39
- num_cpus = { workspace = true }
40
- serde = { workspace = true, features = ["derive"] }
41
- serde_json = { workspace = true }
60
+ [dependencies.lindera]
61
+ version = "=6.1.0"
62
+
63
+ [dependencies.lindera-binding-core]
64
+ version = "=6.1.0"
65
+
66
+ [dependencies.magnus]
67
+ version = "0.8.2"
68
+ features = ["rb-sys"]
69
+
70
+ [dependencies.num_cpus]
71
+ version = "1.17.0"
72
+
73
+ [dependencies.serde]
74
+ version = "1.0.229"
75
+ features = [
76
+ "derive",
77
+ "derive",
78
+ ]
79
+
80
+ [dependencies.serde_json]
81
+ version = "1.0.151"
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 by the project authors.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md CHANGED
@@ -15,15 +15,24 @@ lindera-ruby provides a Ruby interface to the Lindera morphological analysis eng
15
15
 
16
16
  ## Requirements
17
17
 
18
- - Ruby >= 3.1
19
- - Rust >= 1.85
18
+ - Ruby >= 3.1, with a C compiler and `make` for building native gems
19
+ - Rust >= 1.88
20
+ - libclang (`libclang-dev` on Debian/Ubuntu; the Xcode command line tools on macOS)
20
21
 
21
- ## Dictionary
22
+ ## Install
22
23
 
23
- Pre-built dictionaries are available from [GitHub Releases](https://github.com/lindera/lindera/releases).
24
- Download a dictionary archive (e.g. `lindera-ipadic-*.zip`) and extract it to a local path.
24
+ From v6.1.0 the gem installs from RubyGems. It compiles the native extension
25
+ during installation, which takes a few minutes:
25
26
 
26
- ## Install
27
+ ```bash
28
+ gem install lindera
29
+ ```
30
+
31
+ The gem embeds no dictionary (see [Dictionary](#dictionary)). To embed one,
32
+ pass the Cargo features at install time, e.g.
33
+ `LINDERA_FEATURES="embed-ipadic" gem install lindera`.
34
+
35
+ ### Build from Source
27
36
 
28
37
  ```bash
29
38
  cd lindera-ruby
@@ -31,6 +40,14 @@ bundle install
31
40
  bundle exec rake compile
32
41
  ```
33
42
 
43
+ `bundle exec rake build` writes the source gem to `pkg/`. Use it instead of
44
+ `gem build`, which cannot package the crate outside the Cargo workspace.
45
+
46
+ ## Dictionary
47
+
48
+ Pre-built dictionaries are available from [GitHub Releases](https://github.com/lindera/lindera/releases).
49
+ Download a dictionary archive (e.g. `lindera-ipadic-*.zip`) and extract it to a local path.
50
+
34
51
  ## Usage
35
52
 
36
53
  ```ruby
@@ -49,6 +66,21 @@ tokens.each do |token|
49
66
  end
50
67
  ```
51
68
 
69
+ ### Converting Tokens to Plain Data
70
+
71
+ `to_h` returns the token as a plain Hash keyed by Symbols, keeping each
72
+ field's natural Ruby type, so it serializes without a custom encoder:
73
+
74
+ ```ruby
75
+ require 'json'
76
+
77
+ data = tokens.first.to_h
78
+ # {surface: ..., byte_start: 0, byte_end: 9, position: 0,
79
+ # word_id: 12345, is_unknown: false, details: [...]}
80
+
81
+ JSON.generate(tokens.map(&:to_h))
82
+ ```
83
+
52
84
  ### Using TokenizerBuilder
53
85
 
54
86
  ```ruby
data/extconf.rb ADDED
@@ -0,0 +1,13 @@
1
+ # Builds the extension for both `bundle exec rake compile` and `gem install`
2
+ # (the gemspec lists this file as the extension). Cargo.toml sits next to it in
3
+ # both layouts, so rb_sys finds the crate without further configuration.
4
+ require "mkmf"
5
+ require "rb_sys/mkmf"
6
+
7
+ create_rust_makefile("lindera/lindera_ruby") do |r|
8
+ # Pass embed features via LINDERA_FEATURES environment variable, e.g.
9
+ # LINDERA_FEATURES="embed-ipadic" bundle exec rake compile
10
+ # LINDERA_FEATURES="embed-ipadic" gem install lindera
11
+ features = ENV.fetch("LINDERA_FEATURES", "").split(",").map(&:strip).reject(&:empty?)
12
+ r.features = features unless features.empty?
13
+ end
@@ -0,0 +1,3 @@
1
+ //! Character filters for preprocessing text.
2
+ //!
3
+ //! This module is a placeholder for future character filter implementations.
data/src/dictionary.rs ADDED
@@ -0,0 +1,239 @@
1
+ //! Dictionary management for morphological analysis.
2
+ //!
3
+ //! This module provides functionality for building, loading, and managing dictionaries
4
+ //! used in morphological analysis.
5
+
6
+ use std::path::Path;
7
+
8
+ use magnus::prelude::*;
9
+ use magnus::{Error, Ruby, function, method};
10
+
11
+ use lindera::dictionary::{
12
+ Dictionary, DictionaryBuilder, Metadata, UserDictionary,
13
+ load_dictionary as lindera_load_dictionary,
14
+ load_user_dictionary as lindera_load_user_dictionary,
15
+ };
16
+
17
+ use crate::error::to_magnus_error;
18
+ use crate::metadata::RbMetadata;
19
+
20
+ /// A morphological analysis dictionary.
21
+ ///
22
+ /// Contains the data structures needed for tokenization and morphological analysis.
23
+ #[magnus::wrap(class = "Lindera::Dictionary", free_immediately, size)]
24
+ #[derive(Clone)]
25
+ pub struct RbDictionary {
26
+ /// Inner Lindera dictionary.
27
+ pub inner: Dictionary,
28
+ }
29
+
30
+ impl RbDictionary {
31
+ /// Returns the name of the dictionary metadata.
32
+ ///
33
+ /// # Returns
34
+ ///
35
+ /// The dictionary metadata name.
36
+ fn metadata_name(&self) -> String {
37
+ self.inner.metadata.name.clone()
38
+ }
39
+
40
+ /// Returns the character encoding of the dictionary.
41
+ ///
42
+ /// # Returns
43
+ ///
44
+ /// The dictionary encoding string.
45
+ fn metadata_encoding(&self) -> String {
46
+ self.inner.metadata.encoding.clone()
47
+ }
48
+
49
+ /// Returns the full metadata object.
50
+ ///
51
+ /// # Returns
52
+ ///
53
+ /// The dictionary metadata.
54
+ fn metadata(&self) -> RbMetadata {
55
+ RbMetadata::from((*self.inner.metadata).clone())
56
+ }
57
+
58
+ /// Returns the string representation.
59
+ fn to_s(&self) -> String {
60
+ "Dictionary".to_string()
61
+ }
62
+
63
+ /// Returns the inspect representation.
64
+ fn inspect(&self) -> String {
65
+ format!(
66
+ "#<Lindera::Dictionary: name='{}'>",
67
+ self.inner.metadata.name
68
+ )
69
+ }
70
+ }
71
+
72
+ /// A user-defined dictionary for custom words.
73
+ ///
74
+ /// User dictionaries allow you to add custom words and their morphological features
75
+ /// that are not present in the main dictionary.
76
+ #[magnus::wrap(class = "Lindera::UserDictionary", free_immediately, size)]
77
+ #[derive(Clone)]
78
+ pub struct RbUserDictionary {
79
+ /// Inner Lindera user dictionary.
80
+ pub inner: UserDictionary,
81
+ }
82
+
83
+ impl RbUserDictionary {
84
+ /// Returns the string representation.
85
+ fn to_s(&self) -> String {
86
+ "UserDictionary".to_string()
87
+ }
88
+
89
+ /// Returns the inspect representation.
90
+ fn inspect(&self) -> String {
91
+ "UserDictionary()".to_string()
92
+ }
93
+ }
94
+
95
+ /// Loads a dictionary from the specified URI.
96
+ ///
97
+ /// # Arguments
98
+ ///
99
+ /// * `uri` - URI to the dictionary. Can be a file path or embedded dictionary name.
100
+ ///
101
+ /// # Returns
102
+ ///
103
+ /// A loaded `RbDictionary` object.
104
+ fn load_dictionary(uri: String) -> Result<RbDictionary, Error> {
105
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
106
+ lindera_load_dictionary(&uri)
107
+ .map_err(|e| {
108
+ to_magnus_error(
109
+ &ruby,
110
+ format!("Failed to load dictionary from '{uri}': {e}"),
111
+ )
112
+ })
113
+ .map(|d| RbDictionary { inner: d })
114
+ }
115
+
116
+ /// Loads a user dictionary from the specified URI.
117
+ ///
118
+ /// # Arguments
119
+ ///
120
+ /// * `uri` - URI to the user dictionary directory.
121
+ /// * `metadata` - Metadata configuration for the user dictionary.
122
+ ///
123
+ /// # Returns
124
+ ///
125
+ /// A loaded `RbUserDictionary` object.
126
+ fn load_user_dictionary(uri: String, metadata: &RbMetadata) -> Result<RbUserDictionary, Error> {
127
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
128
+ let meta: Metadata = metadata.clone().into();
129
+ lindera_load_user_dictionary(&uri, &meta)
130
+ .map_err(|e| {
131
+ to_magnus_error(
132
+ &ruby,
133
+ format!("Failed to load user dictionary from '{uri}': {e}"),
134
+ )
135
+ })
136
+ .map(|d| RbUserDictionary { inner: d })
137
+ }
138
+
139
+ /// Builds a dictionary from source files.
140
+ ///
141
+ /// # Arguments
142
+ ///
143
+ /// * `input_dir` - Directory containing dictionary source files.
144
+ /// * `output_dir` - Directory where the built dictionary will be saved.
145
+ /// * `metadata` - Metadata configuration for the dictionary.
146
+ fn build_dictionary(
147
+ input_dir: String,
148
+ output_dir: String,
149
+ metadata: &RbMetadata,
150
+ ) -> Result<(), Error> {
151
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
152
+ let input_path = Path::new(&input_dir);
153
+ let output_path = Path::new(&output_dir);
154
+
155
+ if !input_path.exists() {
156
+ return Err(Error::new(
157
+ ruby.exception_arg_error(),
158
+ format!("Input directory does not exist: {input_dir}"),
159
+ ));
160
+ }
161
+
162
+ let builder = DictionaryBuilder::new(metadata.clone().into());
163
+ builder
164
+ .build_dictionary(input_path, output_path)
165
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to build dictionary: {e}")))?;
166
+
167
+ Ok(())
168
+ }
169
+
170
+ /// Builds a user dictionary from a CSV file.
171
+ ///
172
+ /// # Arguments
173
+ ///
174
+ /// * `_kind` - Dictionary kind (reserved for future use).
175
+ /// * `input_file` - Path to the CSV file.
176
+ /// * `output_dir` - Directory where the built user dictionary will be saved.
177
+ /// * `metadata` - Optional metadata configuration.
178
+ fn build_user_dictionary(
179
+ _kind: String,
180
+ input_file: String,
181
+ output_dir: String,
182
+ metadata: Option<&RbMetadata>,
183
+ ) -> Result<(), Error> {
184
+ let ruby = Ruby::get().expect("Ruby runtime not initialized");
185
+ let input_path = Path::new(&input_file);
186
+ let output_path = Path::new(&output_dir);
187
+
188
+ if !input_path.exists() {
189
+ return Err(Error::new(
190
+ ruby.exception_arg_error(),
191
+ format!("Input file does not exist: {input_file}"),
192
+ ));
193
+ }
194
+
195
+ let meta = match metadata {
196
+ Some(m) => m.clone().into(),
197
+ None => Metadata::default(),
198
+ };
199
+
200
+ let builder = DictionaryBuilder::new(meta);
201
+ builder
202
+ .build_user_dictionary(input_path, output_path)
203
+ .map_err(|e| to_magnus_error(&ruby, format!("Failed to build user dictionary: {e}")))?;
204
+
205
+ Ok(())
206
+ }
207
+
208
+ /// Defines Dictionary, UserDictionary classes and module functions in the given Ruby module.
209
+ ///
210
+ /// # Arguments
211
+ ///
212
+ /// * `ruby` - Ruby runtime handle.
213
+ /// * `module` - Parent Ruby module.
214
+ ///
215
+ /// # Returns
216
+ ///
217
+ /// `Ok(())` on success, or a Magnus `Error` on failure.
218
+ pub fn define(ruby: &Ruby, module: &magnus::RModule) -> Result<(), Error> {
219
+ let dict_class = module.define_class("Dictionary", ruby.class_object())?;
220
+ dict_class.define_method("metadata_name", method!(RbDictionary::metadata_name, 0))?;
221
+ dict_class.define_method(
222
+ "metadata_encoding",
223
+ method!(RbDictionary::metadata_encoding, 0),
224
+ )?;
225
+ dict_class.define_method("metadata", method!(RbDictionary::metadata, 0))?;
226
+ dict_class.define_method("to_s", method!(RbDictionary::to_s, 0))?;
227
+ dict_class.define_method("inspect", method!(RbDictionary::inspect, 0))?;
228
+
229
+ let user_dict_class = module.define_class("UserDictionary", ruby.class_object())?;
230
+ user_dict_class.define_method("to_s", method!(RbUserDictionary::to_s, 0))?;
231
+ user_dict_class.define_method("inspect", method!(RbUserDictionary::inspect, 0))?;
232
+
233
+ module.define_module_function("load_dictionary", function!(load_dictionary, 1))?;
234
+ module.define_module_function("load_user_dictionary", function!(load_user_dictionary, 2))?;
235
+ module.define_module_function("build_dictionary", function!(build_dictionary, 3))?;
236
+ module.define_module_function("build_user_dictionary", function!(build_user_dictionary, 4))?;
237
+
238
+ Ok(())
239
+ }
data/src/error.rs ADDED
@@ -0,0 +1,19 @@
1
+ //! Error types for Lindera operations.
2
+ //!
3
+ //! This module provides error types used throughout the Lindera Ruby bindings.
4
+
5
+ use magnus::{Error, Ruby};
6
+
7
+ /// Converts a Lindera error message into a Magnus Ruby error.
8
+ ///
9
+ /// # Arguments
10
+ ///
11
+ /// * `ruby` - Ruby runtime handle.
12
+ /// * `msg` - Error message string.
13
+ ///
14
+ /// # Returns
15
+ ///
16
+ /// A Magnus `Error` wrapping a Ruby `RuntimeError`.
17
+ pub fn to_magnus_error(ruby: &Ruby, msg: String) -> Error {
18
+ Error::new(ruby.exception_runtime_error(), msg)
19
+ }
data/src/lib.rs ADDED
@@ -0,0 +1,90 @@
1
+ //! # Lindera Ruby Bindings
2
+ //!
3
+ //! Ruby bindings for [Lindera](https://github.com/lindera/lindera), a morphological analysis library for CJK text.
4
+ //!
5
+ //! Lindera provides high-performance tokenization and morphological analysis for:
6
+ //! - Japanese (IPADIC, IPADIC NEologd, UniDic)
7
+ //! - Korean (ko-dic)
8
+ //! - Chinese (CC-CEDICT, Jieba)
9
+ //!
10
+ //! ## Features
11
+ //!
12
+ //! - **Dictionary management**: Build, load, and use custom dictionaries
13
+ //! - **Tokenization**: Multiple tokenization modes (normal, decompose)
14
+ //! - **Filters**: Character and token filtering pipeline
15
+ //! - **Training**: Train custom morphological models (with `train` feature)
16
+ //! - **User dictionaries**: Support for custom user dictionaries
17
+ //!
18
+ //! ## Examples
19
+ //!
20
+ //! ```ruby
21
+ //! require "lindera"
22
+ //!
23
+ //! # Create a tokenizer
24
+ //! builder = Lindera::TokenizerBuilder.new
25
+ //! tokenizer = builder.build
26
+ //!
27
+ //! # Tokenize text
28
+ //! tokens = tokenizer.tokenize("関西国際空港")
29
+ //! tokens.each { |token| puts "#{token.surface}: #{token.details}" }
30
+ //! ```
31
+
32
+ pub mod character_filter;
33
+ pub mod dictionary;
34
+ pub mod error;
35
+ pub mod metadata;
36
+ pub mod mode;
37
+ pub mod schema;
38
+ pub mod token;
39
+ pub mod token_filter;
40
+ pub mod tokenizer;
41
+ pub mod util;
42
+
43
+ #[cfg(feature = "train")]
44
+ pub mod trainer;
45
+
46
+ use magnus::{Error, Ruby, function};
47
+
48
+ /// Returns the version of the lindera-ruby package.
49
+ ///
50
+ /// # Returns
51
+ ///
52
+ /// A string with the package version.
53
+ fn version() -> String {
54
+ env!("CARGO_PKG_VERSION").to_string()
55
+ }
56
+
57
+ /// Ruby extension initialization entry point.
58
+ ///
59
+ /// Defines the `Lindera` module and all its classes and functions.
60
+ #[magnus::init]
61
+ fn init(ruby: &Ruby) -> Result<(), Error> {
62
+ let module = ruby.define_module("Lindera")?;
63
+
64
+ // Version
65
+ module.define_module_function("version", function!(version, 0))?;
66
+
67
+ // Schema / FieldDefinition / FieldType
68
+ schema::define(ruby, &module)?;
69
+
70
+ // Metadata
71
+ metadata::define(ruby, &module)?;
72
+
73
+ // Mode / Penalty
74
+ mode::define(ruby, &module)?;
75
+
76
+ // Token
77
+ token::define(ruby, &module)?;
78
+
79
+ // Dictionary / UserDictionary + load/build functions
80
+ dictionary::define(ruby, &module)?;
81
+
82
+ // TokenizerBuilder / Tokenizer
83
+ tokenizer::define(ruby, &module)?;
84
+
85
+ // Trainer (feature = "train")
86
+ #[cfg(feature = "train")]
87
+ trainer::define(ruby, &module)?;
88
+
89
+ Ok(())
90
+ }