litsea 0.13.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 12e321b2284507d1ac91d3a77c6a5a2fa77440da6f655c3591d6535a1226ddbd
+  data.tar.gz: df71d843a1ae5ea741fcbc124d202200ee41f62ef56d340af43c8430cbfb7e0a
+SHA512:
+  metadata.gz: 710a2a9a938b22309d001bdbacc36c2035a7c6aaefa364f39fd052ca69c2f3fda7b6b7b8f8d924b723e9dda7b40cb965f981b5625bb02abdd5d4afad0a9b57fa
+  data.tar.gz: 884330a6bbab450ed6f51442d5614edb7640dca5b40b3ed5d8588fe219f9fa6fd51f6ab1770564ae368b6b6d42de9c17b673634a9afec74e91bb7e413b634ef6

data/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "litsea-ruby"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+description = "Ruby binding for Litsea."
+documentation = "https://docs.rs/litsea-ruby"
+homepage.workspace = true
+repository.workspace = true
+readme = "README.md"
+keywords = ["word", "segmentation", "nlp", "ruby", "binding"]
+categories.workspace = true
+license.workspace = true
+
+[lib]
+name = "litsea_ruby"
+crate-type = ["cdylib", "lib"]
+
+[dependencies]
+litsea.workspace = true
+litsea-binding-core = { workspace = true, features = ["remote_model"] }
+magnus = { version = "0.8.2", features = ["rb-sys"] }
+# Used only to link against libruby; the GVL entry point is declared by
+# hand in `gvl.rs` because neither magnus nor rb-sys exposes it.
+rb-sys = "0.9.130"

data/README.md

@@ -0,0 +1,137 @@
+# litsea-ruby
+
+Ruby binding for [Litsea](https://github.com/mosuka/litsea), a compact word segmentation and POS (Part-of-Speech) tagging library for Japanese, Chinese, Korean, and English.
+
+[日本語のREADME](README_ja.md)
+
+## Installation
+
+```sh
+gem install litsea
+```
+
+The gem compiles the native extension on install, so a Rust toolchain is required. Ruby 3.1 or later.
+
+## Models are not bundled
+
+The gem ships code only. Download a pre-trained model from the [Litsea repository](https://github.com/mosuka/litsea/tree/main/models) and point the segmenter at it:
+
+| Model | Purpose | Size |
+|-------|---------|------|
+| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | Segmentation | 84 KB – 2.0 MB |
+| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | Segmentation + POS | 3.0 – 8.0 MB |
+
+You never have to say which kind you have: the model file identifies itself, and `has_pos?` reports what the loaded model can do.
+
+## Usage
+
+### Segmentation
+
+```ruby
+require "litsea"
+
+seg = Litsea::Segmenter.open(:japanese, "models/japanese.model")
+
+seg.segment("これはテストです。")
+# => ["これ", "は", "テスト", "です", "。"]
+
+seg.segment_batch(["これはテストです。", "東京都から神奈川県へ引っ越した"])
+```
+
+The language accepts a Symbol or a String, and the ISO 639-1 code works too: `:ja`, `"japanese"`.
+
+For space-delimited languages the whitespace comes back as its own token, so the tokens always reconstruct the input:
+
+```ruby
+Litsea::Segmenter.open(:korean, "models/korean.model").segment("안녕하세요 반갑습니다")
+# => ["안녕하세요", " ", "반갑습니다"]
+```
+
+### POS tagging
+
+```ruby
+seg = Litsea::Segmenter.open(:japanese, "models/japanese_pos.model")
+
+seg.segment_with_pos("これはテストです。").each do |token|
+  puts "#{token.surface}\t#{token.pos}\t[#{token.start}..#{token.end}]"
+end
+# これ    PRON    [0..6]
+# は      ADP     [6..9]
+# テスト  NOUN    [9..18]
+# です    AUX     [18..24]
+# 。      PUNCT   [24..27]
+```
+
+`start` and `end` are **byte** offsets, so slice with `byteslice` — Ruby's `String#[]` counts characters:
+
+```ruby
+text.byteslice(token.start, token.end - token.start)   # == token.surface
+```
+
+Calling `segment_with_pos` on a segmentation-only model raises `Litsea::PosUnavailableError`.
+
+### Other model sources
+
+```ruby
+Litsea::Segmenter.from_bytes(:korean, File.binread("korean.model"))
+Litsea::Segmenter.from_uri(:chinese, "https://example.com/chinese.model")
+```
+
+### Training
+
+```ruby
+Litsea::Extractor.new(:japanese).extract("corpus.txt", "features.txt")
+
+metrics = Litsea::Trainer.new(0.01, 10_000, "features.txt").train("japanese.model")
+puts format("accuracy: %.2f%%", metrics.accuracy)
+```
+
+Two-stage (segmentation + POS) training:
+
+```ruby
+Litsea::Extractor.new(:japanese).extract_two_stage("corpus_pos.txt", "features", feature_set: "fast")
+
+metrics = Litsea::TwoStageTrainer.new(10, "features").train("japanese_pos.model")
+puts metrics.stage1.accuracy, metrics.stage2.accuracy
+```
+
+A `TwoStageTrainer` can only be used once — training collapses stage 1 into an AdaBoost model, which consumes it. `available?` reports whether it can still run, and a second `train` raises.
+
+### Cancelling a training run
+
+Training releases the GVL, so other Ruby threads keep running — which means one of them can stop a run that is already going:
+
+```ruby
+cancel = Litsea::CancelToken.new
+Thread.new { sleep 60; cancel.cancel }
+
+metrics = Litsea::Trainer.new(0.01, 100_000, "features.txt").train("japanese.model", cancel: cancel)
+```
+
+Cancelling is **not** an error: training stops at its next check point, still writes the partially trained model, and returns its metrics. The binding never installs a signal handler.
+
+## Errors
+
+Every error derives from `Litsea::Error`, so one `rescue` handles them all.
+
+| Error | Raised when |
+|-------|-------------|
+| `Litsea::InvalidArgumentError` | Unknown language name, unknown feature set, reused trainer |
+| `Litsea::ModelError` | Download failed, or the file is a legacy joint POS model |
+| `Litsea::IoError` | A file could not be read or written |
+| `Litsea::ParseError` | The model or training data is malformed |
+| `Litsea::UnsupportedError` | The scheme or operation is unavailable in this build |
+| `Litsea::PosUnavailableError` | POS tagging requested from a segmentation-only model |
+
+## Development
+
+```sh
+make test-litsea-ruby    # cargo test + rake compile + rake test
+make build-litsea-ruby   # release build
+```
+
+The parity tests build the `litsea` CLI and compare the binding's output against it. Note that `bundle` must be available for the active Ruby; with rbenv, `rbenv local 3.4.9` (or any installed 3.1+) is enough.
+
+## License
+
+MIT. See [LICENSE](../LICENSE).

data/README_ja.md

@@ -0,0 +1,137 @@
+# litsea-ruby
+
+[Litsea](https://github.com/mosuka/litsea) の Ruby バインディングです。Litsea は日本語・中国語・韓国語・英語に対応した、コンパクトな単語分割と品詞（POS）タグ付けのライブラリです。
+
+[English README](README.md)
+
+## インストール
+
+```sh
+gem install litsea
+```
+
+インストール時にネイティブ拡張をコンパイルするため、Rust ツールチェーンが必要です。Ruby 3.1 以降に対応しています。
+
+## モデルは同梱されません
+
+gem にはコードのみが含まれます。事前学習済みモデルは [Litsea リポジトリ](https://github.com/mosuka/litsea/tree/main/models)から取得し、パスを指定して読み込んでください。
+
+| モデル | 用途 | サイズ |
+|-------|------|-------|
+| `japanese.model`, `chinese.model`, `korean.model`, `english.model` | 分割 | 84KB〜2.0MB |
+| `japanese_pos.model`, `chinese_pos.model`, `korean_pos.model`, `english_pos.model` | 分割 + POS | 3.0〜8.0MB |
+
+どちらの種別かを指定する必要はありません。モデルファイル自身が種別を持っており、読み込んだモデルで何ができるかは `has_pos?` が示します。
+
+## 使い方
+
+### 分割
+
+```ruby
+require "litsea"
+
+seg = Litsea::Segmenter.open(:japanese, "models/japanese.model")
+
+seg.segment("これはテストです。")
+# => ["これ", "は", "テスト", "です", "。"]
+
+seg.segment_batch(["これはテストです。", "東京都から神奈川県へ引っ越した"])
+```
+
+言語は Symbol でも String でも指定でき、ISO 639-1 コードも使えます（`:ja` / `"japanese"`）。
+
+空白区切りの言語では空白自体が 1 トークンとして返るため、トークンを連結すると常に入力が復元されます。
+
+```ruby
+Litsea::Segmenter.open(:korean, "models/korean.model").segment("안녕하세요 반갑습니다")
+# => ["안녕하세요", " ", "반갑습니다"]
+```
+
+### POS タグ付け
+
+```ruby
+seg = Litsea::Segmenter.open(:japanese, "models/japanese_pos.model")
+
+seg.segment_with_pos("これはテストです。").each do |token|
+  puts "#{token.surface}\t#{token.pos}\t[#{token.start}..#{token.end}]"
+end
+# これ    PRON    [0..6]
+# は      ADP     [6..9]
+# テスト  NOUN    [9..18]
+# です    AUX     [18..24]
+# 。      PUNCT   [24..27]
+```
+
+`start` と `end` は**バイト**オフセットです。Ruby の `String#[]` は文字単位なので、切り出しには `byteslice` を使ってください。
+
+```ruby
+text.byteslice(token.start, token.end - token.start)   # == token.surface
+```
+
+分割専用モデルに対して `segment_with_pos` を呼ぶと `Litsea::PosUnavailableError` が発生します。
+
+### その他のモデル読み込み方法
+
+```ruby
+Litsea::Segmenter.from_bytes(:korean, File.binread("korean.model"))
+Litsea::Segmenter.from_uri(:chinese, "https://example.com/chinese.model")
+```
+
+### 学習
+
+```ruby
+Litsea::Extractor.new(:japanese).extract("corpus.txt", "features.txt")
+
+metrics = Litsea::Trainer.new(0.01, 10_000, "features.txt").train("japanese.model")
+puts format("accuracy: %.2f%%", metrics.accuracy)
+```
+
+二段構成（分割 + POS）の学習:
+
+```ruby
+Litsea::Extractor.new(:japanese).extract_two_stage("corpus_pos.txt", "features", feature_set: "fast")
+
+metrics = Litsea::TwoStageTrainer.new(10, "features").train("japanese_pos.model")
+puts metrics.stage1.accuracy, metrics.stage2.accuracy
+```
+
+`TwoStageTrainer` は 1 度しか使えません。学習時に stage 1 が AdaBoost モデルへ collapse され、トレーナが消費されるためです。再利用可能かどうかは `available?` が示し、2 回目の `train` は例外を発生させます。
+
+### 学習のキャンセル
+
+学習は GVL を解放するため、他の Ruby スレッドが動き続けます。つまり、実行中の学習を別スレッドから停止できます。
+
+```ruby
+cancel = Litsea::CancelToken.new
+Thread.new { sleep 60; cancel.cancel }
+
+metrics = Litsea::Trainer.new(0.01, 100_000, "features.txt").train("japanese.model", cancel: cancel)
+```
+
+キャンセルは**エラーではありません**。次のチェックポイントで停止し、部分的に学習されたモデルを保存してメトリクスを返します。バインディングはシグナルハンドラを登録しません。
+
+## エラー
+
+すべてのエラーは `Litsea::Error` を継承するため、1 つの `rescue` で捕捉できます。
+
+| エラー | 発生条件 |
+|-------|---------|
+| `Litsea::InvalidArgumentError` | 未知の言語名、未知の feature set、使用済みトレーナ |
+| `Litsea::ModelError` | ダウンロード失敗、または旧 joint POS モデル |
+| `Litsea::IoError` | ファイルの読み書き失敗 |
+| `Litsea::ParseError` | モデルまたは学習データの形式不正 |
+| `Litsea::UnsupportedError` | このビルドでは利用できないスキームや操作 |
+| `Litsea::PosUnavailableError` | 分割専用モデルに対する POS タグ付けの要求 |
+
+## 開発
+
+```sh
+make test-litsea-ruby    # cargo test + rake compile + rake test
+make build-litsea-ruby   # リリースビルド
+```
+
+パリティテストは `litsea` CLI をビルドし、その出力とバインディングの出力を突き合わせます。なお、有効な Ruby に `bundle` が入っている必要があります（rbenv なら `rbenv local 3.4.9` などで 3.1 以降を選択してください）。
+
+## ライセンス
+
+MIT。[LICENSE](../LICENSE) を参照してください。

data/extconf.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+require 'rb_sys/mkmf'
+
+# The crate lives at the gem root (it is also a workspace member of the
+# Litsea repository), so no ext_dir override is needed.
+create_rust_makefile('litsea/litsea_ruby')

data/lib/litsea/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Litsea
+  # Kept in lockstep with the `litsea` crate; `Litsea.version` reports the
+  # version the native extension was built from.
+  VERSION = '0.13.0'
+end

data/lib/litsea.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'litsea/version'
+
+# Loads the compiled extension, preferring a version-specific build when
+# rake-compiler produced one.
+begin
+  RUBY_VERSION =~ /(\d+\.\d+)/
+  require "litsea/#{Regexp.last_match(1)}/litsea_ruby"
+rescue LoadError
+  require 'litsea/litsea_ruby'
+end
+
+# Word segmentation and POS tagging.
+#
+# See {Litsea::Segmenter} to get started; every class is defined by the
+# native extension.
+module Litsea
+end

data/src/error.rs

@@ -0,0 +1,120 @@
+//! The Ruby exception hierarchy.
+//!
+//! Mirrors the Python and PHP bindings: one class per [`ErrorKind`], all
+//! below `Litsea::Error`, so `rescue Litsea::Error` catches everything the
+//! binding raises.
+
+use litsea_binding_core::{CoreError, ErrorKind};
+use magnus::{Module, RModule, Ruby, error::Error, exception::ExceptionClass};
+
+/// Holds the exception classes, looked up from the `Litsea` module.
+///
+/// Ruby classes are values, not types, so they are resolved on demand rather
+/// than stored in Rust statics (a `Ruby` handle is only valid on a Ruby
+/// thread).
+struct Exceptions {
+    /// `Litsea::Error`, the base class.
+    base: ExceptionClass,
+    /// `Litsea::InvalidArgumentError`.
+    invalid_argument: ExceptionClass,
+    /// `Litsea::ModelError`.
+    model: ExceptionClass,
+    /// `Litsea::IoError`.
+    io: ExceptionClass,
+    /// `Litsea::ParseError`.
+    parse: ExceptionClass,
+    /// `Litsea::UnsupportedError`.
+    unsupported: ExceptionClass,
+    /// `Litsea::PosUnavailableError`.
+    pos_unavailable: ExceptionClass,
+}
+
+/// Looks up the exception classes defined by [`define_exceptions`].
+///
+/// # Arguments
+/// * `ruby` - The Ruby handle for the current thread.
+///
+/// # Returns
+/// The exception classes, or `None` if the module has not been defined yet.
+fn exceptions(ruby: &Ruby) -> Option<Exceptions> {
+    let module: RModule = ruby.class_object().const_get("Litsea").ok()?;
+    Some(Exceptions {
+        base: module.const_get("Error").ok()?,
+        invalid_argument: module.const_get("InvalidArgumentError").ok()?,
+        model: module.const_get("ModelError").ok()?,
+        io: module.const_get("IoError").ok()?,
+        parse: module.const_get("ParseError").ok()?,
+        unsupported: module.const_get("UnsupportedError").ok()?,
+        pos_unavailable: module.const_get("PosUnavailableError").ok()?,
+    })
+}
+
+/// Defines `Litsea::Error` and its subclasses on the given module.
+///
+/// # Arguments
+/// * `ruby` - The Ruby handle for the current thread.
+/// * `module` - The `Litsea` module to define the classes on.
+///
+/// # Returns
+/// `()` on success.
+///
+/// # Errors
+/// Returns a Ruby exception if a class cannot be defined.
+pub fn define_exceptions(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let base = module.define_error("Error", ruby.exception_standard_error())?;
+    module.define_error("InvalidArgumentError", base)?;
+    module.define_error("ModelError", base)?;
+    module.define_error("IoError", base)?;
+    module.define_error("ParseError", base)?;
+    module.define_error("UnsupportedError", base)?;
+    module.define_error("PosUnavailableError", base)?;
+    Ok(())
+}
+
+/// Converts a [`CoreError`] into the matching Ruby exception.
+///
+/// # Arguments
+/// * `error` - The error to convert.
+///
+/// # Returns
+/// A magnus [`Error`] carrying the class that matches the error's kind;
+/// [`ErrorKind::Runtime`] uses `Litsea::Error` itself.
+pub fn to_ruby_error(error: CoreError) -> Error {
+    let message = error.message().to_string();
+    let Ok(ruby) = Ruby::get() else {
+        // Not on a Ruby thread, which cannot happen for a call that came
+        // from Ruby - but the type system does not know that. The
+        // handle-based replacement for this constructor needs the very
+        // handle we just failed to get, so the deprecated free function is
+        // the only way to build an error here.
+        #[allow(deprecated)]
+        return Error::new(magnus::exception::fatal(), message);
+    };
+
+    let Some(classes) = exceptions(&ruby) else {
+        return Error::new(ruby.exception_runtime_error(), message);
+    };
+
+    let class = match error.kind() {
+        ErrorKind::InvalidArgument => classes.invalid_argument,
+        ErrorKind::Model => classes.model,
+        ErrorKind::Io => classes.io,
+        ErrorKind::Parse => classes.parse,
+        ErrorKind::Unsupported => classes.unsupported,
+        ErrorKind::PosUnavailable => classes.pos_unavailable,
+        ErrorKind::Runtime => classes.base,
+    };
+
+    Error::new(class, message)
+}
+
+/// Maps a core result onto a Ruby result.
+///
+/// # Arguments
+/// * `result` - The result to convert.
+///
+/// # Returns
+/// The original value, or the mapped Ruby exception.
+pub fn map_err<T>(result: Result<T, CoreError>) -> Result<T, Error> {
+    result.map_err(to_ruby_error)
+}

data/src/gvl.rs

@@ -0,0 +1,112 @@
+//! Releasing the GVL around long-running work.
+//!
+//! Ruby is genuinely multi-threaded, so a `train()` that holds the Global VM
+//! Lock blocks every other Ruby thread in the process - including the one
+//! that would cancel it. Releasing the lock is what makes
+//! `Litsea::CancelToken` usable while training runs, exactly as
+//! `Python::detach` does for the Python binding.
+//!
+//! Neither magnus nor `rb-sys` exposes `rb_thread_call_without_gvl`: magnus
+//! lists it among the C functions it does not wrap, and it is declared in
+//! `ruby/thread.h`, which is outside the bindings `rb-sys` generates. The
+//! declaration below is therefore written by hand. The symbol resolves
+//! because the extension links against libruby regardless.
+
+use std::ffi::c_void;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+
+unsafe extern "C" {
+    /// Runs `func` with the GVL released.
+    ///
+    /// `ubf` is the "unblocking function" Ruby calls to interrupt the work;
+    /// passing null selects Ruby's default, which defers interrupts until
+    /// the call returns. That is what we want: the work is pure computation
+    /// with no blocking syscall to interrupt, and it stops on its own when
+    /// the cancellation flag is cleared.
+    fn rb_thread_call_without_gvl(
+        func: unsafe extern "C" fn(*mut c_void) -> *mut c_void,
+        data1: *mut c_void,
+        ubf: *const c_void,
+        data2: *mut c_void,
+    ) -> *mut c_void;
+}
+
+/// State handed to the trampoline and filled in with the closure's result.
+struct Payload<F, R> {
+    /// The closure to run, taken by the trampoline.
+    func: Option<F>,
+    /// Where the result lands; `None` if the closure panicked.
+    result: Option<R>,
+}
+
+/// The `extern "C"` entry point Ruby calls with the GVL released.
+///
+/// # Safety
+/// `data` must be a valid `*mut Payload<F, R>` that outlives the call, which
+/// [`without_gvl`] guarantees by keeping the payload on its own stack frame
+/// for the duration of `rb_thread_call_without_gvl`.
+unsafe extern "C" fn trampoline<F, R>(data: *mut c_void) -> *mut c_void
+where
+    F: FnOnce() -> R,
+{
+    // SAFETY: `without_gvl` passes a pointer to a live `Payload<F, R>` and
+    // does not touch it until this function returns.
+    let payload = unsafe { &mut *(data as *mut Payload<F, R>) };
+
+    if let Some(func) = payload.func.take() {
+        // A panic must not unwind across the C frame Ruby put on the stack:
+        // that is undefined behaviour. Catch it here and report it as a
+        // missing result, which `without_gvl` turns back into a panic on the
+        // Rust side of the boundary.
+        payload.result = catch_unwind(AssertUnwindSafe(func)).ok();
+    }
+
+    std::ptr::null_mut()
+}
+
+/// Runs `func` with the GVL released, so other Ruby threads keep running.
+///
+/// The closure runs on the calling thread, not a new one. It **must not**
+/// touch the Ruby API - doing so without the GVL is undefined behaviour.
+/// Every caller in this crate passes pure Rust work (segmentation or
+/// training) that only reaches `litsea`.
+///
+/// # Arguments
+/// * `func` - The work to run without the GVL.
+///
+/// # Returns
+/// Whatever `func` returns.
+///
+/// # Panics
+/// Re-panics on the Rust side if `func` panicked, after the panic has been
+/// contained inside the C call.
+pub fn without_gvl<F, R>(func: F) -> R
+where
+    F: FnOnce() -> R,
+{
+    let mut payload = Payload {
+        func: Some(func),
+        result: None,
+    };
+
+    // SAFETY: `trampoline::<F, R>` matches the signature Ruby expects, and
+    // `&mut payload` stays valid for the whole call because
+    // `rb_thread_call_without_gvl` returns before this frame is dropped. A
+    // null unblocking function selects Ruby's default deferred-interrupt
+    // behaviour, which is correct for non-blocking computation.
+    unsafe {
+        rb_thread_call_without_gvl(
+            trampoline::<F, R>,
+            std::ptr::addr_of_mut!(payload) as *mut c_void,
+            std::ptr::null(),
+            std::ptr::null_mut(),
+        );
+    }
+
+    match payload.result.take() {
+        Some(result) => result,
+        // The closure panicked; the panic was contained in the trampoline so
+        // it could not unwind through C, and is re-raised here.
+        None => panic!("a Litsea operation panicked while the GVL was released"),
+    }
+}

data/src/language.rs

@@ -0,0 +1,31 @@
+//! Language arguments.
+//!
+//! Ruby callers reach for a Symbol (`:japanese`) as readily as a String, and
+//! magnus converts neither into the other automatically, so both are
+//! accepted here.
+
+use litsea::Language;
+use litsea_binding_core::parse_language;
+use magnus::{Symbol, TryConvert, Value, error::Error};
+
+use crate::error::map_err;
+
+/// Converts a Ruby String or Symbol into a [`Language`].
+///
+/// # Arguments
+/// * `value` - The language name or ISO 639-1 code, as a String or Symbol.
+///
+/// # Returns
+/// The parsed language.
+///
+/// # Errors
+/// Raises `Litsea::InvalidArgumentError` for an unknown language, or a
+/// `TypeError` if the value is neither a String nor a Symbol.
+pub fn language_from_value(value: Value) -> Result<Language, Error> {
+    let name = match Symbol::from_value(value) {
+        Some(symbol) => symbol.name()?.to_string(),
+        None => String::try_convert(value)?,
+    };
+
+    map_err(parse_language(&name))
+}