litsea 0.13.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12e321b2284507d1ac91d3a77c6a5a2fa77440da6f655c3591d6535a1226ddbd
-  data.tar.gz: df71d843a1ae5ea741fcbc124d202200ee41f62ef56d340af43c8430cbfb7e0a
+  metadata.gz: 48bde143045210bc8ebc4dacba04155649772c88a97b88d4cbfbbf9d1f4670fa
+  data.tar.gz: d45ddd6b6b9875ab645cd5e20dc539974e189953ecd1c4851c5f90a5c6d4d76c
 SHA512:
-  metadata.gz: 710a2a9a938b22309d001bdbacc36c2035a7c6aaefa364f39fd052ca69c2f3fda7b6b7b8f8d924b723e9dda7b40cb965f981b5625bb02abdd5d4afad0a9b57fa
-  data.tar.gz: 884330a6bbab450ed6f51442d5614edb7640dca5b40b3ed5d8588fe219f9fa6fd51f6ab1770564ae368b6b6d42de9c17b673634a9afec74e91bb7e413b634ef6
+  metadata.gz: 6bf4331b0bd88b94921d492429504761f5ec4a783c020080e9af6012d1e544fb7daae41b38e2580ebe1c3498bf9b49a6c0922c66c1355aef678821b4d2175865
+  data.tar.gz: a8b264e872e9e7a04601dd5f4e964893b0cfc3b2af2880b3328d7ad84c7ece3e499a7c615e972a66f78182f0522b62e5f372c5ce101a233b24f433c5085a883d

data/Cargo.toml

@@ -1,25 +1,58 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
 [package]
+edition = "2024"
+rust-version = "1.87"
 name = "litsea-ruby"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
+version = "0.15.0"
+build = false
+autolib = false
+autobins = false
+autoexamples = false
+autotests = false
+autobenches = false
 description = "Ruby binding for Litsea."
+homepage = "https://github.com/mosuka/litsea"
 documentation = "https://docs.rs/litsea-ruby"
-homepage.workspace = true
-repository.workspace = true
 readme = "README.md"
-keywords = ["word", "segmentation", "nlp", "ruby", "binding"]
-categories.workspace = true
-license.workspace = true
+keywords = [
+    "word",
+    "segmentation",
+    "nlp",
+    "ruby",
+    "binding",
+]
+categories = ["text-processing"]
+license = "MIT"
+repository = "https://github.com/mosuka/litsea"
 
 [lib]
 name = "litsea_ruby"
-crate-type = ["cdylib", "lib"]
+crate-type = [
+    "cdylib",
+    "lib",
+]
+path = "src/lib.rs"
+
+[dependencies.litsea]
+version = "=0.15.0"
+
+[dependencies.litsea-binding-core]
+version = "=0.15.0"
+features = ["remote_model"]
+
+[dependencies.magnus]
+version = "0.9.0"
+features = ["rb-sys"]
 
-[dependencies]
-litsea.workspace = true
-litsea-binding-core = { workspace = true, features = ["remote_model"] }
-magnus = { version = "0.8.2", features = ["rb-sys"] }
-# Used only to link against libruby; the GVL entry point is declared by
-# hand in `gvl.rs` because neither magnus nor rb-sys exposes it.
-rb-sys = "0.9.130"
+[dependencies.rb-sys]
+version = "0.9.130"

data/LICENSE

@@ -0,0 +1,52 @@
+MIT License
+
+Copyright (c) 2025 Minoru OSUKA
+Copyright (c) 2022 ICHINOSE Shogo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+--------------------------------------------------------------------
+
+Copyright (c) 2008, Taku Kudo
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+    * Neither the name of the <ORGANIZATION> nor the names of its
+contributors may be used to endorse or promote products derived from this
+software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -10,7 +10,13 @@ Ruby binding for [Litsea](https://github.com/mosuka/litsea), a compact word segm
 gem install litsea
 ```
 
-The gem compiles the native extension on install, so a Rust toolchain is required. Ruby 3.1 or later.
+The gem compiles the native extension on install, which takes a minute or two. It needs:
+
+- Ruby 3.1 or later, with a C compiler and `make` for building native gems
+- a Rust toolchain, 1.87 or later
+- libclang (`libclang-dev` on Debian and Ubuntu, the Xcode Command Line Tools on macOS)
+
+It compiles against the crates.io releases of `litsea` and `litsea-binding-core` of its own version. The gems released from 0.13.0 until the fix for [#247](https://github.com/mosuka/litsea/issues/247) fail to compile during installation; `gem install litsea` picks the newest release, so this matters only when you pin one of them.
 
 ## Models are not bundled
 
@@ -18,8 +24,8 @@ The gem ships code only. Download a pre-trained model from the [Litsea repositor
 
 | Model | Purpose | Size |
 |-------|---------|------|
-| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | Segmentation | 84 KB – 2.0 MB |
-| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | Segmentation + POS | 3.0 – 8.0 MB |
+| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | Segmentation | 86 KB – 2.0 MB |
+| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | Segmentation + POS | 3.1 – 8.3 MB |
 
 You never have to say which kind you have: the model file identifies itself, and `has_pos?` reports what the loaded model can do.
 
@@ -70,6 +76,28 @@ text.byteslice(token.start, token.end - token.start)   # == token.surface
 
 Calling `segment_with_pos` on a segmentation-only model raises `Litsea::PosUnavailableError`.
 
+### User dictionary
+
+Words passed as `user_words:` are emitted as single tokens wherever they occur, without retraining; every constructor accepts the keyword.
+
+```ruby
+seg = Litsea::Segmenter.open(:japanese, "models/japanese.model", user_words: ["雪だるま", "新幹線"])
+
+seg.segment("雪だるまを作る")
+# => ["雪だるま", "を", "作る"]   (without the dictionary: ["雪", "だる", "ま", "を", "作る"])
+```
+
+Matching is context-free, so a short fragment registered as a word breaks every other word it occurs in. `user_words_gate:` (`0.0`-`1.0`) applies a word only where the model's margin against it is small enough: `0.0` only where the model already agrees with it.
+
+With a POS model, `user_word_tags:` (a Hash of words and UPOS tags, as Strings) registers further words whose tokens `segment_with_pos` gives the registered tag, ahead of the model's own tagging:
+
+```ruby
+tagged = Litsea::Segmenter.open(:japanese, "models/japanese_pos.model", user_word_tags: { "チャットGPT" => "PROPN" })
+
+tagged.segment_with_pos("チャットGPTに聞く").map { |t| "#{t.surface}/#{t.pos}" }
+# => ["チャットGPT/PROPN", "に/ADP", "聞く/VERB"]   (in user_words: instead: "チャットGPT/NOUN")
+```
+
 ### Other model sources
 
 ```ruby
@@ -116,7 +144,7 @@ Every error derives from `Litsea::Error`, so one `rescue` handles them all.
 
 | Error | Raised when |
 |-------|-------------|
-| `Litsea::InvalidArgumentError` | Unknown language name, unknown feature set, reused trainer |
+| `Litsea::InvalidArgumentError` | Unknown language name, unknown feature set, reused trainer, `user_words_gate:` outside `0.0`-`1.0`, a `user_word_tags:` value that is not a UPOS tag |
 | `Litsea::ModelError` | Download failed, or the file is a legacy joint POS model |
 | `Litsea::IoError` | A file could not be read or written |
 | `Litsea::ParseError` | The model or training data is malformed |
@@ -126,10 +154,14 @@ Every error derives from `Litsea::Error`, so one `rescue` handles them all.
 ## Development
 
 ```sh
-make test-litsea-ruby    # cargo test + rake compile + rake test
-make build-litsea-ruby   # release build
+make test-litsea-ruby       # cargo test + rake compile + rake test
+make build-litsea-ruby      # release build
+make package-litsea-ruby    # source gem into pkg/
+make test-litsea-ruby-gem   # gem install that gem in a clean container (needs Docker)
 ```
 
+Build the gem with `make package-litsea-ruby` (`bundle exec rake build`), not `gem build`: this directory's `Cargo.toml` only resolves inside the Cargo workspace, so the task builds the gem from the crate as `cargo package` normalizes it.
+
 The parity tests build the `litsea` CLI and compare the binding's output against it. Note that `bundle` must be available for the active Ruby; with rbenv, `rbenv local 3.4.9` (or any installed 3.1+) is enough.
 
 ## License

data/README_ja.md

@@ -10,7 +10,13 @@
 gem install litsea
 ```
 
-インストール時にネイティブ拡張をコンパイルするため、Rust ツールチェーンが必要です。Ruby 3.1 以降に対応しています。
+インストール時にネイティブ拡張をコンパイルします（1〜2 分かかります）。次のものが必要です。
+
+- Ruby 3.1 以降と、ネイティブ gem のビルドに使う C コンパイラと `make`
+- Rust ツールチェーン 1.87 以降
+- libclang（Debian / Ubuntu は `libclang-dev`、macOS は Xcode Command Line Tools）
+
+gem は crates.io 上の同じバージョンの `litsea` と `litsea-binding-core` に対してコンパイルされます。0.13.0 から [#247](https://github.com/mosuka/litsea/issues/247) の修正までにリリースされた gem は、インストール時のコンパイルに失敗します。`gem install litsea` は最新のリリースを選ぶため、影響があるのはそれらのバージョンを指定した場合だけです。
 
 ## モデルは同梱されません
 
@@ -18,8 +24,8 @@ gem にはコードのみが含まれます。事前学習済みモデルは [Li
 
 | モデル | 用途 | サイズ |
 |-------|------|-------|
-| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | 分割 | 84KB〜2.0MB |
-| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | 分割 + POS | 3.0〜8.0MB |
+| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | 分割 | 86KB〜2.0MB |
+| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | 分割 + POS | 3.1〜8.3MB |
 
 どちらの種別かを指定する必要はありません。モデルファイル自身が種別を持っており、読み込んだモデルで何ができるかは `has_pos?` が示します。
 
@@ -70,6 +76,28 @@ text.byteslice(token.start, token.end - token.start)   # == token.surface
 
 分割専用モデルに対して `segment_with_pos` を呼ぶと `Litsea::PosUnavailableError` が発生します。
 
+### ユーザー辞書
+
+`user_words:` に渡した語は、再学習なしで、現れた場所で必ず 1 トークンとして出力されます。すべてのコンストラクタがこのキーワードを受け取ります。
+
+```ruby
+seg = Litsea::Segmenter.open(:japanese, "models/japanese.model", user_words: ["雪だるま", "新幹線"])
+
+seg.segment("雪だるまを作る")
+# => ["雪だるま", "を", "作る"]   （辞書なしでは ["雪", "だる", "ま", "を", "作る"]）
+```
+
+照合は文脈を見ないため、短い断片を語として登録すると、それを含む他の語をすべて壊します。`user_words_gate:`（`0.0`〜`1.0`）は、モデルの margin が十分小さい場所にだけ語を適用します。`0.0` では、モデルがすでに同意している場所だけに適用します。
+
+POS モデルでは、`user_word_tags:`（語と UPOS タグの Hash。キーと値は String）で、さらに語を登録できます。`segment_with_pos` は、その語のトークンに、モデル自身のタグ付けより優先して、登録したタグを付けます:
+
+```ruby
+tagged = Litsea::Segmenter.open(:japanese, "models/japanese_pos.model", user_word_tags: { "チャットGPT" => "PROPN" })
+
+tagged.segment_with_pos("チャットGPTに聞く").map { |t| "#{t.surface}/#{t.pos}" }
+# => ["チャットGPT/PROPN", "に/ADP", "聞く/VERB"]   （user_words: に入れた場合は "チャットGPT/NOUN"）
+```
+
 ### その他のモデル読み込み方法
 
 ```ruby
@@ -116,7 +144,7 @@ metrics = Litsea::Trainer.new(0.01, 100_000, "features.txt").train("japanese.mod
 
 | エラー | 発生条件 |
 |-------|---------|
-| `Litsea::InvalidArgumentError` | 未知の言語名、未知の feature set、使用済みトレーナ |
+| `Litsea::InvalidArgumentError` | 未知の言語名、未知の feature set、使用済みトレーナ、`0.0`〜`1.0` の外の `user_words_gate:`、UPOS タグでない `user_word_tags:` の値 |
 | `Litsea::ModelError` | ダウンロード失敗、または旧 joint POS モデル |
 | `Litsea::IoError` | ファイルの読み書き失敗 |
 | `Litsea::ParseError` | モデルまたは学習データの形式不正 |
@@ -126,10 +154,14 @@ metrics = Litsea::Trainer.new(0.01, 100_000, "features.txt").train("japanese.mod
 ## 開発
 
 ```sh
-make test-litsea-ruby    # cargo test + rake compile + rake test
-make build-litsea-ruby   # リリースビルド
+make test-litsea-ruby       # cargo test + rake compile + rake test
+make build-litsea-ruby      # リリースビルド
+make package-litsea-ruby    # ソース gem を pkg/ に出力
+make test-litsea-ruby-gem   # その gem をクリーンなコンテナで gem install（Docker が必要）
 ```
 
+gem は `gem build` ではなく `make package-litsea-ruby`（`bundle exec rake build`）でビルドしてください。このディレクトリの `Cargo.toml` は Cargo ワークスペースの中でしか解決できないため、このタスクは `cargo package` が正規化したクレートから gem をビルドします。
+
 パリティテストは `litsea` CLI をビルドし、その出力とバインディングの出力を突き合わせます。なお、有効な Ruby に `bundle` が入っている必要があります（rbenv なら `rbenv local 3.4.9` などで 3.1 以降を選択してください）。
 
 ## ライセンス

data/lib/litsea/version.rb

@@ -3,5 +3,5 @@
 module Litsea
   # Kept in lockstep with the `litsea` crate; `Litsea.version` reports the
   # version the native extension was built from.
-  VERSION = '0.13.0'
+  VERSION = '0.15.0'
 end

data/src/segmenter.rs

@@ -1,21 +1,106 @@
 //! The `Litsea::Segmenter` class.
 
+use std::collections::HashMap;
 use std::path::Path;
 
-use litsea_binding_core::{CoreSegmenter, TokenView};
-use magnus::{Module, Object, RArray, RModule, RString, Ruby, Value, error::Error};
+use litsea::UserDictionary;
+use litsea_binding_core::{CoreSegmenter, TokenView, build_user_dictionary_with_tags};
+use magnus::scan_args::{get_kwargs, scan_args};
+use magnus::{Module, Object, RArray, RModule, RString, Ruby, TryConvert, Value, error::Error};
 
 use crate::error::map_err;
 use crate::gvl::without_gvl;
 use crate::language::language_from_value;
 use crate::token::Token;
 
+/// The keyword arguments every constructor accepts (issues #263, #264): the
+/// optional `user_words:`, `user_words_gate:` and `user_word_tags:`, each of
+/// which may also be passed as `nil`.
+struct UserWordsArgs {
+    /// The words to emit as single tokens.
+    user_words: Option<Vec<String>>,
+    /// The flip-ratio gate in `0.0..=1.0`.
+    user_words_gate: Option<f64>,
+    /// Words to emit as single tokens with the UPOS tag `segment_with_pos`
+    /// gives them.
+    user_word_tags: Option<HashMap<String, String>>,
+}
+
+impl UserWordsArgs {
+    /// Builds the user dictionary these arguments describe.
+    ///
+    /// # Returns
+    /// `None` when no words were given, otherwise the dictionary.
+    ///
+    /// # Errors
+    /// Raises `Litsea::InvalidArgumentError` if a tag is not one of the 17
+    /// UPOS tags, or if the gate is outside `0.0..=1.0`, also when no words
+    /// were given.
+    fn dictionary(self) -> Result<Option<UserDictionary>, Error> {
+        let word_tags: Vec<(String, String)> =
+            self.user_word_tags.unwrap_or_default().into_iter().collect();
+        map_err(build_user_dictionary_with_tags(
+            self.user_words.as_deref().unwrap_or_default(),
+            &word_tags,
+            self.user_words_gate,
+        ))
+    }
+}
+
+/// Splits the arguments of `open`, `from_bytes`, and `from_uri`: the
+/// language and the model source as required positionals, then the
+/// `user_words:` / `user_words_gate:` / `user_word_tags:` keywords.
+///
+/// magnus cannot express optional keyword arguments in a `function!`
+/// arity, so the constructors take their arguments through `scan_args`,
+/// as the trainers do.
+///
+/// # Arguments
+/// * `args` - The raw Ruby arguments.
+///
+/// # Returns
+/// The language value, the model source, and the keyword arguments.
+///
+/// # Errors
+/// Returns a Ruby `ArgumentError` for a wrong argument count or an unknown
+/// keyword, or a `TypeError` if an argument has the wrong type.
+fn scan_constructor_args<T: TryConvert>(
+    args: &[Value],
+) -> Result<(Value, T, UserWordsArgs), Error> {
+    let args = scan_args::<(Value, T), (), (), (), _, ()>(args)?;
+    let (language, source) = args.required;
+    let kwargs = get_kwargs::<
+        _,
+        (),
+        (
+            Option<Option<Vec<String>>>,
+            Option<Option<f64>>,
+            Option<Option<HashMap<String, String>>>,
+        ),
+        (),
+    >(args.keywords, &[], &["user_words", "user_words_gate", "user_word_tags"])?;
+    let (user_words, user_words_gate, user_word_tags) = kwargs.optional;
+    Ok((
+        language,
+        source,
+        UserWordsArgs {
+            user_words: user_words.flatten(),
+            user_words_gate: user_words_gate.flatten(),
+            user_word_tags: user_word_tags.flatten(),
+        },
+    ))
+}
+
 /// A word segmenter, optionally with POS tagging.
 ///
 /// Build one with `Segmenter.open`, `Segmenter.from_bytes`, or
 /// `Segmenter.from_uri`. The kind of model is detected from the file itself,
 /// so `has_pos?` describes what was loaded rather than something the caller
-/// declares.
+/// declares. Every constructor accepts the keyword arguments `user_words:`,
+/// an array of words to emit as single tokens; `user_words_gate:`, a flip
+/// ratio in `0.0..=1.0` that applies a word only where the model's margin
+/// against it is small enough; and `user_word_tags:`, a Hash mapping further
+/// words to the UPOS tag `segment_with_pos` gives them.
 #[magnus::wrap(class = "Litsea::Segmenter", free_immediately, size)]
 pub struct Segmenter {
     /// The wrapped core segmenter.
@@ -26,20 +111,25 @@ impl Segmenter {
     /// Loads a model from a filesystem path.
     ///
     /// # Arguments
-    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
-    /// * `path` - Path to the model file.
+    /// * `args` - `language` (a language name or ISO 639-1 code, as a String
+    ///   or Symbol), `path` (the model file), and the optional
+    ///   `user_words:` / `user_words_gate:` / `user_word_tags:` keywords.
     ///
     /// # Returns
     /// The new segmenter.
     ///
     /// # Errors
-    /// Raises `Litsea::InvalidArgumentError`, `Litsea::IoError`,
+    /// Raises `Litsea::InvalidArgumentError` (also for a tag that is not a
+    /// UPOS tag or a gate outside `0.0..=1.0`), `Litsea::IoError`,
     /// `Litsea::ParseError`, or `Litsea::ModelError`.
-    fn open(language: Value, path: String) -> Result<Self, Error> {
+    fn open(args: &[Value]) -> Result<Self, Error> {
+        let (language, path, user_words): (_, String, _) = scan_constructor_args(args)?;
         let language = language_from_value(language)?;
+        let dictionary = user_words.dictionary()?;
         // Reading and compiling a model is measurable work; let other Ruby
         // threads run while it happens.
-        let inner = without_gvl(|| CoreSegmenter::from_path(language, Path::new(&path)));
+        let inner =
+            without_gvl(|| CoreSegmenter::from_path_with(language, Path::new(&path), dictionary));
         Ok(Self {
             inner: map_err(inner)?,
         })
@@ -51,23 +141,27 @@ impl Segmenter {
     /// (ASCII-8BIT) works as well as one read as UTF-8.
     ///
     /// # Arguments
-    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
-    /// * `data` - The model file contents.
+    /// * `args` - `language` (a language name or ISO 639-1 code, as a String
+    ///   or Symbol), `data` (the model file contents), and the optional
+    ///   `user_words:` / `user_words_gate:` / `user_word_tags:` keywords.
     ///
     /// # Returns
     /// The new segmenter.
     ///
     /// # Errors
-    /// Raises `Litsea::InvalidArgumentError`, `Litsea::ParseError`, or
+    /// Raises `Litsea::InvalidArgumentError` (also for a tag that is not a
+    /// UPOS tag or a gate outside `0.0..=1.0`), `Litsea::ParseError`, or
     /// `Litsea::ModelError`.
-    fn from_bytes(language: Value, data: RString) -> Result<Self, Error> {
+    fn from_bytes(args: &[Value]) -> Result<Self, Error> {
+        let (language, data, user_words): (_, RString, _) = scan_constructor_args(args)?;
         let language = language_from_value(language)?;
+        let dictionary = user_words.dictionary()?;
         // SAFETY: the slice is used only within this call, and nothing here
         // runs Ruby code that could move or free the string in the meantime
-        // (`from_bytes` parses into owned Rust structures).
+        // (`from_bytes_with` parses into owned Rust structures).
         let bytes = unsafe { data.as_slice() };
         Ok(Self {
-            inner: map_err(CoreSegmenter::from_bytes(language, bytes))?,
+            inner: map_err(CoreSegmenter::from_bytes_with(language, bytes, dictionary))?,
         })
     }
 
@@ -77,8 +171,9 @@ impl Segmenter {
     /// The GVL is released while the model is fetched.
     ///
     /// # Arguments
-    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
-    /// * `uri` - The model URI.
+    /// * `args` - `language` (a language name or ISO 639-1 code, as a String
+    ///   or Symbol), `uri` (the model URI), and the optional `user_words:` /
+    ///   `user_words_gate:` / `user_word_tags:` keywords.
     ///
     /// # Returns
     /// The new segmenter.
@@ -86,9 +181,12 @@ impl Segmenter {
     /// # Errors
     /// Raises `Litsea::ModelError` if the download fails, plus the same
     /// errors as `open`.
-    fn from_uri(language: Value, uri: String) -> Result<Self, Error> {
+    fn from_uri(args: &[Value]) -> Result<Self, Error> {
+        let (language, uri, user_words): (_, String, _) = scan_constructor_args(args)?;
         let language = language_from_value(language)?;
-        let inner = without_gvl(|| CoreSegmenter::from_uri_blocking(language, &uri));
+        let dictionary = user_words.dictionary()?;
+        let inner =
+            without_gvl(|| CoreSegmenter::from_uri_blocking_with(language, &uri, dictionary));
         Ok(Self {
             inner: map_err(inner)?,
         })
@@ -224,9 +322,9 @@ fn tokens_to_array(ruby: &Ruby, tokens: Vec<TokenView>) -> Result<RArray, Error>
 /// Returns a Ruby exception if the class cannot be defined.
 pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
     let class = module.define_class("Segmenter", ruby.class_object())?;
-    class.define_singleton_method("open", magnus::function!(Segmenter::open, 2))?;
-    class.define_singleton_method("from_bytes", magnus::function!(Segmenter::from_bytes, 2))?;
-    class.define_singleton_method("from_uri", magnus::function!(Segmenter::from_uri, 2))?;
+    class.define_singleton_method("open", magnus::function!(Segmenter::open, -1))?;
+    class.define_singleton_method("from_bytes", magnus::function!(Segmenter::from_bytes, -1))?;
+    class.define_singleton_method("from_uri", magnus::function!(Segmenter::from_uri, -1))?;
     class.define_method("language", magnus::method!(Segmenter::language, 0))?;
     class.define_method("has_pos?", magnus::method!(Segmenter::has_pos, 0))?;
     class.define_method("segment", magnus::method!(Segmenter::segment, 1))?;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: litsea
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Minoru Osuka
@@ -31,6 +31,7 @@ extensions:
 extra_rdoc_files: []
 files:
 - Cargo.toml
+- LICENSE
 - README.md
 - README_ja.md
 - extconf.rb