litsea 0.14.3 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 849682ce4467eef85788f31a19183ae366bd3c5dd642d3aaa47998f6b1988e1c
-  data.tar.gz: 2458ce9b638527048aa071f83b0743ae6ede8ae45d64c04b67d2fe8fbf33d7c9
+  metadata.gz: 48bde143045210bc8ebc4dacba04155649772c88a97b88d4cbfbbf9d1f4670fa
+  data.tar.gz: d45ddd6b6b9875ab645cd5e20dc539974e189953ecd1c4851c5f90a5c6d4d76c
 SHA512:
-  metadata.gz: 983194c5e2cc5a25b02827085ae9f532ae4d0a238fc32e10107857a3ec20a25ef417019d33e7ce27151d09b050d626280edc5cf037c6769e53a7e37ef0340cc5
-  data.tar.gz: 07e311b43e127e758fbf021d54102114e684e287a56858310473a23f12eb3b0de8aa33117fea49b94897bae809c28682aac707d81567b71038a61b37d2e7a8fe
+  metadata.gz: 6bf4331b0bd88b94921d492429504761f5ec4a783c020080e9af6012d1e544fb7daae41b38e2580ebe1c3498bf9b49a6c0922c66c1355aef678821b4d2175865
+  data.tar.gz: a8b264e872e9e7a04601dd5f4e964893b0cfc3b2af2880b3328d7ad84c7ece3e499a7c615e972a66f78182f0522b62e5f372c5ce101a233b24f433c5085a883d

data/Cargo.toml

@@ -13,7 +13,7 @@
 edition = "2024"
 rust-version = "1.87"
 name = "litsea-ruby"
-version = "0.14.3"
+version = "0.15.0"
 build = false
 autolib = false
 autobins = false
@@ -44,14 +44,14 @@ crate-type = [
 path = "src/lib.rs"
 
 [dependencies.litsea]
-version = "=0.14.3"
+version = "=0.15.0"
 
 [dependencies.litsea-binding-core]
-version = "=0.14.3"
+version = "=0.15.0"
 features = ["remote_model"]
 
 [dependencies.magnus]
-version = "0.8.2"
+version = "0.9.0"
 features = ["rb-sys"]
 
 [dependencies.rb-sys]

data/README.md

@@ -24,8 +24,8 @@ The gem ships code only. Download a pre-trained model from the [Litsea repositor
 
 | Model | Purpose | Size |
 |-------|---------|------|
-| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | Segmentation | 84 KB – 2.0 MB |
-| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | Segmentation + POS | 3.0 – 8.0 MB |
+| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | Segmentation | 86 KB – 2.0 MB |
+| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | Segmentation + POS | 3.1 – 8.3 MB |
 
 You never have to say which kind you have: the model file identifies itself, and `has_pos?` reports what the loaded model can do.
 
@@ -76,6 +76,28 @@ text.byteslice(token.start, token.end - token.start)   # == token.surface
 
 Calling `segment_with_pos` on a segmentation-only model raises `Litsea::PosUnavailableError`.
 
+### User dictionary
+
+Words passed as `user_words:` are emitted as single tokens wherever they occur, without retraining; every constructor accepts the keyword.
+
+```ruby
+seg = Litsea::Segmenter.open(:japanese, "models/japanese.model", user_words: ["雪だるま", "新幹線"])
+
+seg.segment("雪だるまを作る")
+# => ["雪だるま", "を", "作る"]   (without the dictionary: ["雪", "だる", "ま", "を", "作る"])
+```
+
+Matching is context-free, so a short fragment registered as a word breaks every other word it occurs in. `user_words_gate:` (`0.0`-`1.0`) applies a word only where the model's margin against it is small enough: `0.0` only where the model already agrees with it.
+
+With a POS model, `user_word_tags:` (a Hash of words and UPOS tags, as Strings) registers further words whose tokens `segment_with_pos` gives the registered tag, ahead of the model's own tagging:
+
+```ruby
+tagged = Litsea::Segmenter.open(:japanese, "models/japanese_pos.model", user_word_tags: { "チャットGPT" => "PROPN" })
+
+tagged.segment_with_pos("チャットGPTに聞く").map { |t| "#{t.surface}/#{t.pos}" }
+# => ["チャットGPT/PROPN", "に/ADP", "聞く/VERB"]   (in user_words: instead: "チャットGPT/NOUN")
+```
+
 ### Other model sources
 
 ```ruby
@@ -122,7 +144,7 @@ Every error derives from `Litsea::Error`, so one `rescue` handles them all.
 
 | Error | Raised when |
 |-------|-------------|
-| `Litsea::InvalidArgumentError` | Unknown language name, unknown feature set, reused trainer |
+| `Litsea::InvalidArgumentError` | Unknown language name, unknown feature set, reused trainer, `user_words_gate:` outside `0.0`-`1.0`, a `user_word_tags:` value that is not a UPOS tag |
 | `Litsea::ModelError` | Download failed, or the file is a legacy joint POS model |
 | `Litsea::IoError` | A file could not be read or written |
 | `Litsea::ParseError` | The model or training data is malformed |

data/README_ja.md

@@ -24,8 +24,8 @@ gem にはコードのみが含まれます。事前学習済みモデルは [Li
 
 | モデル | 用途 | サイズ |
 |-------|------|-------|
-| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | 分割 | 84KB〜2.0MB |
-| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | 分割 + POS | 3.0〜8.0MB |
+| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | 分割 | 86KB〜2.0MB |
+| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | 分割 + POS | 3.1〜8.3MB |
 
 どちらの種別かを指定する必要はありません。モデルファイル自身が種別を持っており、読み込んだモデルで何ができるかは `has_pos?` が示します。
 
@@ -76,6 +76,28 @@ text.byteslice(token.start, token.end - token.start)   # == token.surface
 
 分割専用モデルに対して `segment_with_pos` を呼ぶと `Litsea::PosUnavailableError` が発生します。
 
+### ユーザー辞書
+
+`user_words:` に渡した語は、再学習なしで、現れた場所で必ず 1 トークンとして出力されます。すべてのコンストラクタがこのキーワードを受け取ります。
+
+```ruby
+seg = Litsea::Segmenter.open(:japanese, "models/japanese.model", user_words: ["雪だるま", "新幹線"])
+
+seg.segment("雪だるまを作る")
+# => ["雪だるま", "を", "作る"]   （辞書なしでは ["雪", "だる", "ま", "を", "作る"]）
+```
+
+照合は文脈を見ないため、短い断片を語として登録すると、それを含む他の語をすべて壊します。`user_words_gate:`（`0.0`〜`1.0`）は、モデルの margin が十分小さい場所にだけ語を適用します。`0.0` では、モデルがすでに同意している場所だけに適用します。
+
+POS モデルでは、`user_word_tags:`（語と UPOS タグの Hash。キーと値は String）で、さらに語を登録できます。`segment_with_pos` は、その語のトークンに、モデル自身のタグ付けより優先して、登録したタグを付けます:
+
+```ruby
+tagged = Litsea::Segmenter.open(:japanese, "models/japanese_pos.model", user_word_tags: { "チャットGPT" => "PROPN" })
+
+tagged.segment_with_pos("チャットGPTに聞く").map { |t| "#{t.surface}/#{t.pos}" }
+# => ["チャットGPT/PROPN", "に/ADP", "聞く/VERB"]   （user_words: に入れた場合は "チャットGPT/NOUN"）
+```
+
 ### その他のモデル読み込み方法
 
 ```ruby
@@ -122,7 +144,7 @@ metrics = Litsea::Trainer.new(0.01, 100_000, "features.txt").train("japanese.mod
 
 | エラー | 発生条件 |
 |-------|---------|
-| `Litsea::InvalidArgumentError` | 未知の言語名、未知の feature set、使用済みトレーナ |
+| `Litsea::InvalidArgumentError` | 未知の言語名、未知の feature set、使用済みトレーナ、`0.0`〜`1.0` の外の `user_words_gate:`、UPOS タグでない `user_word_tags:` の値 |
 | `Litsea::ModelError` | ダウンロード失敗、または旧 joint POS モデル |
 | `Litsea::IoError` | ファイルの読み書き失敗 |
 | `Litsea::ParseError` | モデルまたは学習データの形式不正 |

data/lib/litsea/version.rb

@@ -3,5 +3,5 @@
 module Litsea
   # Kept in lockstep with the `litsea` crate; `Litsea.version` reports the
   # version the native extension was built from.
-  VERSION = '0.14.3'
+  VERSION = '0.15.0'
 end

data/src/segmenter.rs

@@ -1,21 +1,106 @@
 //! The `Litsea::Segmenter` class.
 
+use std::collections::HashMap;
 use std::path::Path;
 
-use litsea_binding_core::{CoreSegmenter, TokenView};
-use magnus::{Module, Object, RArray, RModule, RString, Ruby, Value, error::Error};
+use litsea::UserDictionary;
+use litsea_binding_core::{CoreSegmenter, TokenView, build_user_dictionary_with_tags};
+use magnus::scan_args::{get_kwargs, scan_args};
+use magnus::{Module, Object, RArray, RModule, RString, Ruby, TryConvert, Value, error::Error};
 
 use crate::error::map_err;
 use crate::gvl::without_gvl;
 use crate::language::language_from_value;
 use crate::token::Token;
 
+/// The keyword arguments every constructor accepts (issues #263, #264): the
+/// optional `user_words:`, `user_words_gate:` and `user_word_tags:`, each of
+/// which may also be passed as `nil`.
+struct UserWordsArgs {
+    /// The words to emit as single tokens.
+    user_words: Option<Vec<String>>,
+    /// The flip-ratio gate in `0.0..=1.0`.
+    user_words_gate: Option<f64>,
+    /// Words to emit as single tokens with the UPOS tag `segment_with_pos`
+    /// gives them.
+    user_word_tags: Option<HashMap<String, String>>,
+}
+
+impl UserWordsArgs {
+    /// Builds the user dictionary these arguments describe.
+    ///
+    /// # Returns
+    /// `None` when no words were given, otherwise the dictionary.
+    ///
+    /// # Errors
+    /// Raises `Litsea::InvalidArgumentError` if a tag is not one of the 17
+    /// UPOS tags, or if the gate is outside `0.0..=1.0`, also when no words
+    /// were given.
+    fn dictionary(self) -> Result<Option<UserDictionary>, Error> {
+        let word_tags: Vec<(String, String)> =
+            self.user_word_tags.unwrap_or_default().into_iter().collect();
+        map_err(build_user_dictionary_with_tags(
+            self.user_words.as_deref().unwrap_or_default(),
+            &word_tags,
+            self.user_words_gate,
+        ))
+    }
+}
+
+/// Splits the arguments of `open`, `from_bytes`, and `from_uri`: the
+/// language and the model source as required positionals, then the
+/// `user_words:` / `user_words_gate:` / `user_word_tags:` keywords.
+///
+/// magnus cannot express optional keyword arguments in a `function!`
+/// arity, so the constructors take their arguments through `scan_args`,
+/// as the trainers do.
+///
+/// # Arguments
+/// * `args` - The raw Ruby arguments.
+///
+/// # Returns
+/// The language value, the model source, and the keyword arguments.
+///
+/// # Errors
+/// Returns a Ruby `ArgumentError` for a wrong argument count or an unknown
+/// keyword, or a `TypeError` if an argument has the wrong type.
+fn scan_constructor_args<T: TryConvert>(
+    args: &[Value],
+) -> Result<(Value, T, UserWordsArgs), Error> {
+    let args = scan_args::<(Value, T), (), (), (), _, ()>(args)?;
+    let (language, source) = args.required;
+    let kwargs = get_kwargs::<
+        _,
+        (),
+        (
+            Option<Option<Vec<String>>>,
+            Option<Option<f64>>,
+            Option<Option<HashMap<String, String>>>,
+        ),
+        (),
+    >(args.keywords, &[], &["user_words", "user_words_gate", "user_word_tags"])?;
+    let (user_words, user_words_gate, user_word_tags) = kwargs.optional;
+    Ok((
+        language,
+        source,
+        UserWordsArgs {
+            user_words: user_words.flatten(),
+            user_words_gate: user_words_gate.flatten(),
+            user_word_tags: user_word_tags.flatten(),
+        },
+    ))
+}
+
 /// A word segmenter, optionally with POS tagging.
 ///
 /// Build one with `Segmenter.open`, `Segmenter.from_bytes`, or
 /// `Segmenter.from_uri`. The kind of model is detected from the file itself,
 /// so `has_pos?` describes what was loaded rather than something the caller
-/// declares.
+/// declares. Every constructor accepts the keyword arguments `user_words:`,
+/// an array of words to emit as single tokens; `user_words_gate:`, a flip
+/// ratio in `0.0..=1.0` that applies a word only where the model's margin
+/// against it is small enough; and `user_word_tags:`, a Hash mapping further
+/// words to the UPOS tag `segment_with_pos` gives them.
 #[magnus::wrap(class = "Litsea::Segmenter", free_immediately, size)]
 pub struct Segmenter {
     /// The wrapped core segmenter.
@@ -26,20 +111,25 @@ impl Segmenter {
     /// Loads a model from a filesystem path.
     ///
     /// # Arguments
-    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
-    /// * `path` - Path to the model file.
+    /// * `args` - `language` (a language name or ISO 639-1 code, as a String
+    ///   or Symbol), `path` (the model file), and the optional
+    ///   `user_words:` / `user_words_gate:` / `user_word_tags:` keywords.
     ///
     /// # Returns
     /// The new segmenter.
     ///
     /// # Errors
-    /// Raises `Litsea::InvalidArgumentError`, `Litsea::IoError`,
+    /// Raises `Litsea::InvalidArgumentError` (also for a tag that is not a
+    /// UPOS tag or a gate outside `0.0..=1.0`), `Litsea::IoError`,
     /// `Litsea::ParseError`, or `Litsea::ModelError`.
-    fn open(language: Value, path: String) -> Result<Self, Error> {
+    fn open(args: &[Value]) -> Result<Self, Error> {
+        let (language, path, user_words): (_, String, _) = scan_constructor_args(args)?;
         let language = language_from_value(language)?;
+        let dictionary = user_words.dictionary()?;
         // Reading and compiling a model is measurable work; let other Ruby
         // threads run while it happens.
-        let inner = without_gvl(|| CoreSegmenter::from_path(language, Path::new(&path)));
+        let inner =
+            without_gvl(|| CoreSegmenter::from_path_with(language, Path::new(&path), dictionary));
         Ok(Self {
             inner: map_err(inner)?,
         })
@@ -51,23 +141,27 @@ impl Segmenter {
     /// (ASCII-8BIT) works as well as one read as UTF-8.
     ///
     /// # Arguments
-    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
-    /// * `data` - The model file contents.
+    /// * `args` - `language` (a language name or ISO 639-1 code, as a String
+    ///   or Symbol), `data` (the model file contents), and the optional
+    ///   `user_words:` / `user_words_gate:` / `user_word_tags:` keywords.
     ///
     /// # Returns
     /// The new segmenter.
     ///
     /// # Errors
-    /// Raises `Litsea::InvalidArgumentError`, `Litsea::ParseError`, or
+    /// Raises `Litsea::InvalidArgumentError` (also for a tag that is not a
+    /// UPOS tag or a gate outside `0.0..=1.0`), `Litsea::ParseError`, or
     /// `Litsea::ModelError`.
-    fn from_bytes(language: Value, data: RString) -> Result<Self, Error> {
+    fn from_bytes(args: &[Value]) -> Result<Self, Error> {
+        let (language, data, user_words): (_, RString, _) = scan_constructor_args(args)?;
         let language = language_from_value(language)?;
+        let dictionary = user_words.dictionary()?;
         // SAFETY: the slice is used only within this call, and nothing here
         // runs Ruby code that could move or free the string in the meantime
-        // (`from_bytes` parses into owned Rust structures).
+        // (`from_bytes_with` parses into owned Rust structures).
         let bytes = unsafe { data.as_slice() };
         Ok(Self {
-            inner: map_err(CoreSegmenter::from_bytes(language, bytes))?,
+            inner: map_err(CoreSegmenter::from_bytes_with(language, bytes, dictionary))?,
         })
     }
 
@@ -77,8 +171,9 @@ impl Segmenter {
     /// The GVL is released while the model is fetched.
     ///
     /// # Arguments
-    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
-    /// * `uri` - The model URI.
+    /// * `args` - `language` (a language name or ISO 639-1 code, as a String
+    ///   or Symbol), `uri` (the model URI), and the optional `user_words:` /
+    ///   `user_words_gate:` / `user_word_tags:` keywords.
     ///
     /// # Returns
     /// The new segmenter.
@@ -86,9 +181,12 @@ impl Segmenter {
     /// # Errors
     /// Raises `Litsea::ModelError` if the download fails, plus the same
     /// errors as `open`.
-    fn from_uri(language: Value, uri: String) -> Result<Self, Error> {
+    fn from_uri(args: &[Value]) -> Result<Self, Error> {
+        let (language, uri, user_words): (_, String, _) = scan_constructor_args(args)?;
         let language = language_from_value(language)?;
-        let inner = without_gvl(|| CoreSegmenter::from_uri_blocking(language, &uri));
+        let dictionary = user_words.dictionary()?;
+        let inner =
+            without_gvl(|| CoreSegmenter::from_uri_blocking_with(language, &uri, dictionary));
         Ok(Self {
             inner: map_err(inner)?,
         })
@@ -224,9 +322,9 @@ fn tokens_to_array(ruby: &Ruby, tokens: Vec<TokenView>) -> Result<RArray, Error>
 /// Returns a Ruby exception if the class cannot be defined.
 pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
     let class = module.define_class("Segmenter", ruby.class_object())?;
-    class.define_singleton_method("open", magnus::function!(Segmenter::open, 2))?;
-    class.define_singleton_method("from_bytes", magnus::function!(Segmenter::from_bytes, 2))?;
-    class.define_singleton_method("from_uri", magnus::function!(Segmenter::from_uri, 2))?;
+    class.define_singleton_method("open", magnus::function!(Segmenter::open, -1))?;
+    class.define_singleton_method("from_bytes", magnus::function!(Segmenter::from_bytes, -1))?;
+    class.define_singleton_method("from_uri", magnus::function!(Segmenter::from_uri, -1))?;
     class.define_method("language", magnus::method!(Segmenter::language, 0))?;
     class.define_method("has_pos?", magnus::method!(Segmenter::has_pos, 0))?;
     class.define_method("segment", magnus::method!(Segmenter::segment, 1))?;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: litsea
 version: !ruby/object:Gem::Version
-  version: 0.14.3
+  version: 0.15.0
 platform: ruby
 authors:
 - Minoru Osuka