litsea 0.13.0

data/src/lib.rs

@@ -0,0 +1,84 @@
+//! Ruby binding for Litsea.
+//!
+//! Built with [magnus](https://github.com/matsadler/magnus). Everything
+//! FFI-independent lives in `litsea-binding-core`, so this crate only maps
+//! that surface onto Ruby classes and exceptions.
+//!
+//! Long-running work releases the GVL (see [`gvl`]), so other Ruby threads
+//! keep running during training - which is what lets one of them cancel a
+//! run that is already going.
+//!
+//! ```ruby
+//! require "litsea"
+//!
+//! seg = Litsea::Segmenter.open("japanese", "models/japanese.model")
+//! seg.segment("これはテストです。")
+//! ```
+
+pub mod error;
+pub mod gvl;
+pub mod language;
+pub mod metrics;
+pub mod segmenter;
+pub mod token;
+pub mod trainer;
+
+use magnus::{Ruby, error::Error, function};
+
+/// Returns the version of the underlying `litsea` crate.
+///
+/// # Returns
+/// The version string, for example `"0.12.0"`.
+fn version() -> String {
+    litsea::version().to_string()
+}
+
+/// Returns the names of every supported language.
+///
+/// # Returns
+/// The canonical names, in documentation order.
+fn supported_languages() -> Vec<String> {
+    litsea_binding_core::supported_language_names()
+}
+
+/// Entry point Ruby calls when the extension is required.
+///
+/// # Arguments
+/// * `ruby` - The Ruby handle for the current thread.
+///
+/// # Returns
+/// `()` once every class has been defined.
+///
+/// # Errors
+/// Returns a Ruby exception if a class cannot be defined.
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = ruby.define_module("Litsea")?;
+
+    // Exceptions first: everything else raises through them.
+    error::define_exceptions(ruby, &module)?;
+    token::define(ruby, &module)?;
+    metrics::define(ruby, &module)?;
+    segmenter::define(ruby, &module)?;
+    trainer::define(ruby, &module)?;
+
+    module.define_module_function("version", function!(version, 0))?;
+    module.define_module_function("supported_languages", function!(supported_languages, 0))?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_version_matches_litsea() {
+        assert_eq!(version(), litsea::version());
+    }
+
+    #[test]
+    fn test_supported_languages() {
+        assert_eq!(supported_languages(), vec!["japanese", "chinese", "korean", "english"]);
+    }
+}

data/src/metrics.rs

@@ -0,0 +1,250 @@
+//! Training metric classes.
+
+use litsea::{BinaryMetrics, MulticlassMetrics, TwoStageMetrics};
+use magnus::{Module, RModule, Ruby, error::Error};
+
+/// Metrics from training a binary (segmentation) model.
+///
+/// All percentages are 0-100.
+#[magnus::wrap(class = "Litsea::BinaryMetrics", free_immediately, size)]
+pub struct RbBinaryMetrics {
+    /// The wrapped metrics.
+    inner: BinaryMetrics,
+}
+
+impl RbBinaryMetrics {
+    /// Accuracy, as a percentage.
+    fn accuracy(&self) -> f64 {
+        self.inner.accuracy
+    }
+
+    /// Precision, as a percentage.
+    fn precision(&self) -> f64 {
+        self.inner.precision
+    }
+
+    /// Recall, as a percentage.
+    fn recall(&self) -> f64 {
+        self.inner.recall
+    }
+
+    /// Number of training instances.
+    fn num_instances(&self) -> usize {
+        self.inner.num_instances
+    }
+
+    /// True positives.
+    fn true_positives(&self) -> usize {
+        self.inner.true_positives
+    }
+
+    /// False positives.
+    fn false_positives(&self) -> usize {
+        self.inner.false_positives
+    }
+
+    /// False negatives.
+    fn false_negatives(&self) -> usize {
+        self.inner.false_negatives
+    }
+
+    /// True negatives.
+    fn true_negatives(&self) -> usize {
+        self.inner.true_negatives
+    }
+
+    /// Returns a readable representation.
+    ///
+    /// # Returns
+    /// For example `#<Litsea::BinaryMetrics accuracy=99.12% instances=1234>`.
+    fn inspect(&self) -> String {
+        format!(
+            "#<Litsea::BinaryMetrics accuracy={:.2}% instances={}>",
+            self.inner.accuracy, self.inner.num_instances
+        )
+    }
+}
+
+impl From<BinaryMetrics> for RbBinaryMetrics {
+    /// Wraps `litsea`'s metrics for Ruby.
+    ///
+    /// # Arguments
+    /// * `inner` - The metrics to wrap.
+    ///
+    /// # Returns
+    /// The corresponding [`RbBinaryMetrics`].
+    fn from(inner: BinaryMetrics) -> Self {
+        Self { inner }
+    }
+}
+
+/// Metrics from training a multiclass model.
+///
+/// All percentages are 0-100.
+#[magnus::wrap(class = "Litsea::MulticlassMetrics", free_immediately, size)]
+pub struct RbMulticlassMetrics {
+    /// The wrapped metrics.
+    inner: MulticlassMetrics,
+}
+
+impl RbMulticlassMetrics {
+    /// Accuracy, as a percentage.
+    fn accuracy(&self) -> f64 {
+        self.inner.accuracy
+    }
+
+    /// Macro-averaged precision, as a percentage.
+    fn macro_precision(&self) -> f64 {
+        self.inner.macro_precision
+    }
+
+    /// Macro-averaged recall, as a percentage.
+    fn macro_recall(&self) -> f64 {
+        self.inner.macro_recall
+    }
+
+    /// Number of training instances.
+    fn num_instances(&self) -> usize {
+        self.inner.num_instances
+    }
+
+    /// Gold instances per class, as a Hash keyed by the label name.
+    fn gold_per_class(&self) -> std::collections::HashMap<String, usize> {
+        self.inner.gold_per_class.clone()
+    }
+
+    /// Correct predictions per class.
+    fn correct_per_class(&self) -> std::collections::HashMap<String, usize> {
+        self.inner.correct_per_class.clone()
+    }
+
+    /// Predictions made per class.
+    fn predicted_per_class(&self) -> std::collections::HashMap<String, usize> {
+        self.inner.predicted_per_class.clone()
+    }
+
+    /// Returns a readable representation.
+    ///
+    /// # Returns
+    /// For example `#<Litsea::MulticlassMetrics accuracy=97.30% instances=999>`.
+    fn inspect(&self) -> String {
+        format!(
+            "#<Litsea::MulticlassMetrics accuracy={:.2}% instances={}>",
+            self.inner.accuracy, self.inner.num_instances
+        )
+    }
+}
+
+impl From<MulticlassMetrics> for RbMulticlassMetrics {
+    /// Wraps `litsea`'s metrics for Ruby.
+    ///
+    /// # Arguments
+    /// * `inner` - The metrics to wrap.
+    ///
+    /// # Returns
+    /// The corresponding [`RbMulticlassMetrics`].
+    fn from(inner: MulticlassMetrics) -> Self {
+        Self { inner }
+    }
+}
+
+/// Metrics from training a two-stage model: one set per stage.
+#[magnus::wrap(class = "Litsea::TwoStageMetrics", free_immediately, size)]
+pub struct RbTwoStageMetrics {
+    /// The wrapped metrics.
+    inner: TwoStageMetrics,
+}
+
+impl RbTwoStageMetrics {
+    /// Stage-1 (boundary classifier) metrics.
+    fn stage1(&self) -> RbMulticlassMetrics {
+        RbMulticlassMetrics::from(self.inner.stage1.clone())
+    }
+
+    /// Stage-2 (word tagger) metrics.
+    fn stage2(&self) -> RbMulticlassMetrics {
+        RbMulticlassMetrics::from(self.inner.stage2.clone())
+    }
+
+    /// Returns a readable representation.
+    ///
+    /// # Returns
+    /// For example `#<Litsea::TwoStageMetrics stage1=99.10% stage2=95.40%>`.
+    fn inspect(&self) -> String {
+        format!(
+            "#<Litsea::TwoStageMetrics stage1={:.2}% stage2={:.2}%>",
+            self.inner.stage1.accuracy, self.inner.stage2.accuracy
+        )
+    }
+}
+
+impl From<TwoStageMetrics> for RbTwoStageMetrics {
+    /// Wraps `litsea`'s metrics for Ruby.
+    ///
+    /// # Arguments
+    /// * `inner` - The metrics to wrap.
+    ///
+    /// # Returns
+    /// The corresponding [`RbTwoStageMetrics`].
+    fn from(inner: TwoStageMetrics) -> Self {
+        Self { inner }
+    }
+}
+
+/// Defines the metric classes.
+///
+/// # Arguments
+/// * `ruby` - The Ruby handle for the current thread.
+/// * `module` - The `Litsea` module to define the classes on.
+///
+/// # Returns
+/// `()` on success.
+///
+/// # Errors
+/// Returns a Ruby exception if a class cannot be defined.
+pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let binary = module.define_class("BinaryMetrics", ruby.class_object())?;
+    binary.define_method("accuracy", magnus::method!(RbBinaryMetrics::accuracy, 0))?;
+    binary.define_method("precision", magnus::method!(RbBinaryMetrics::precision, 0))?;
+    binary.define_method("recall", magnus::method!(RbBinaryMetrics::recall, 0))?;
+    binary.define_method("num_instances", magnus::method!(RbBinaryMetrics::num_instances, 0))?;
+    binary.define_method("true_positives", magnus::method!(RbBinaryMetrics::true_positives, 0))?;
+    binary
+        .define_method("false_positives", magnus::method!(RbBinaryMetrics::false_positives, 0))?;
+    binary
+        .define_method("false_negatives", magnus::method!(RbBinaryMetrics::false_negatives, 0))?;
+    binary.define_method("true_negatives", magnus::method!(RbBinaryMetrics::true_negatives, 0))?;
+    binary.define_method("inspect", magnus::method!(RbBinaryMetrics::inspect, 0))?;
+    binary.define_method("to_s", magnus::method!(RbBinaryMetrics::inspect, 0))?;
+
+    let multiclass = module.define_class("MulticlassMetrics", ruby.class_object())?;
+    multiclass.define_method("accuracy", magnus::method!(RbMulticlassMetrics::accuracy, 0))?;
+    multiclass.define_method(
+        "macro_precision",
+        magnus::method!(RbMulticlassMetrics::macro_precision, 0),
+    )?;
+    multiclass
+        .define_method("macro_recall", magnus::method!(RbMulticlassMetrics::macro_recall, 0))?;
+    multiclass
+        .define_method("num_instances", magnus::method!(RbMulticlassMetrics::num_instances, 0))?;
+    multiclass
+        .define_method("gold_per_class", magnus::method!(RbMulticlassMetrics::gold_per_class, 0))?;
+    multiclass.define_method(
+        "correct_per_class",
+        magnus::method!(RbMulticlassMetrics::correct_per_class, 0),
+    )?;
+    multiclass.define_method(
+        "predicted_per_class",
+        magnus::method!(RbMulticlassMetrics::predicted_per_class, 0),
+    )?;
+    multiclass.define_method("inspect", magnus::method!(RbMulticlassMetrics::inspect, 0))?;
+    multiclass.define_method("to_s", magnus::method!(RbMulticlassMetrics::inspect, 0))?;
+
+    let two_stage = module.define_class("TwoStageMetrics", ruby.class_object())?;
+    two_stage.define_method("stage1", magnus::method!(RbTwoStageMetrics::stage1, 0))?;
+    two_stage.define_method("stage2", magnus::method!(RbTwoStageMetrics::stage2, 0))?;
+    two_stage.define_method("inspect", magnus::method!(RbTwoStageMetrics::inspect, 0))?;
+    two_stage.define_method("to_s", magnus::method!(RbTwoStageMetrics::inspect, 0))?;
+
+    Ok(())
+}

data/src/segmenter.rs

@@ -0,0 +1,241 @@
+//! The `Litsea::Segmenter` class.
+
+use std::path::Path;
+
+use litsea_binding_core::{CoreSegmenter, TokenView};
+use magnus::{Module, Object, RArray, RModule, RString, Ruby, Value, error::Error};
+
+use crate::error::map_err;
+use crate::gvl::without_gvl;
+use crate::language::language_from_value;
+use crate::token::Token;
+
+/// A word segmenter, optionally with POS tagging.
+///
+/// Build one with `Segmenter.open`, `Segmenter.from_bytes`, or
+/// `Segmenter.from_uri`. The kind of model is detected from the file itself,
+/// so `has_pos?` describes what was loaded rather than something the caller
+/// declares.
+#[magnus::wrap(class = "Litsea::Segmenter", free_immediately, size)]
+pub struct Segmenter {
+    /// The wrapped core segmenter.
+    inner: CoreSegmenter,
+}
+
+impl Segmenter {
+    /// Loads a model from a filesystem path.
+    ///
+    /// # Arguments
+    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
+    /// * `path` - Path to the model file.
+    ///
+    /// # Returns
+    /// The new segmenter.
+    ///
+    /// # Errors
+    /// Raises `Litsea::InvalidArgumentError`, `Litsea::IoError`,
+    /// `Litsea::ParseError`, or `Litsea::ModelError`.
+    fn open(language: Value, path: String) -> Result<Self, Error> {
+        let language = language_from_value(language)?;
+        // Reading and compiling a model is measurable work; let other Ruby
+        // threads run while it happens.
+        let inner = without_gvl(|| CoreSegmenter::from_path(language, Path::new(&path)));
+        Ok(Self {
+            inner: map_err(inner)?,
+        })
+    }
+
+    /// Loads a model from a raw byte string.
+    ///
+    /// Takes the bytes as they are, so a String read with `File.binread`
+    /// (ASCII-8BIT) works as well as one read as UTF-8.
+    ///
+    /// # Arguments
+    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
+    /// * `data` - The model file contents.
+    ///
+    /// # Returns
+    /// The new segmenter.
+    ///
+    /// # Errors
+    /// Raises `Litsea::InvalidArgumentError`, `Litsea::ParseError`, or
+    /// `Litsea::ModelError`.
+    fn from_bytes(language: Value, data: RString) -> Result<Self, Error> {
+        let language = language_from_value(language)?;
+        // SAFETY: the slice is used only within this call, and nothing here
+        // runs Ruby code that could move or free the string in the meantime
+        // (`from_bytes` parses into owned Rust structures).
+        let bytes = unsafe { data.as_slice() };
+        Ok(Self {
+            inner: map_err(CoreSegmenter::from_bytes(language, bytes))?,
+        })
+    }
+
+    /// Loads a model from a URI.
+    ///
+    /// Accepts a filesystem path, a `file://` path, or an `http(s)://` URL.
+    /// The GVL is released while the model is fetched.
+    ///
+    /// # Arguments
+    /// * `language` - A language name or ISO 639-1 code, as a String or Symbol.
+    /// * `uri` - The model URI.
+    ///
+    /// # Returns
+    /// The new segmenter.
+    ///
+    /// # Errors
+    /// Raises `Litsea::ModelError` if the download fails, plus the same
+    /// errors as `open`.
+    fn from_uri(language: Value, uri: String) -> Result<Self, Error> {
+        let language = language_from_value(language)?;
+        let inner = without_gvl(|| CoreSegmenter::from_uri_blocking(language, &uri));
+        Ok(Self {
+            inner: map_err(inner)?,
+        })
+    }
+
+    /// Returns the language this segmenter was built for.
+    ///
+    /// # Returns
+    /// The canonical language name, for example `"japanese"`.
+    fn language(&self) -> String {
+        self.inner.language().to_string()
+    }
+
+    /// Returns whether this segmenter can tag parts of speech.
+    ///
+    /// # Returns
+    /// `true` when a two-stage POS model was loaded.
+    fn has_pos(&self) -> bool {
+        self.inner.has_pos()
+    }
+
+    /// Splits a sentence into tokens.
+    ///
+    /// # Arguments
+    /// * `text` - The sentence to segment.
+    ///
+    /// # Returns
+    /// The tokens, in order.
+    fn segment(&self, text: String) -> Vec<String> {
+        self.inner.segment(&text)
+    }
+
+    /// Splits several sentences into tokens, releasing the GVL.
+    ///
+    /// # Arguments
+    /// * `texts` - The sentences to segment.
+    ///
+    /// # Returns
+    /// One token array per input sentence, in input order.
+    fn segment_batch(&self, texts: Vec<String>) -> Vec<Vec<String>> {
+        without_gvl(|| self.inner.segment_batch(&texts))
+    }
+
+    /// Splits a sentence into tokens carrying byte offsets.
+    ///
+    /// # Arguments
+    /// * `ruby` - The Ruby handle for the current thread.
+    /// * `text` - The sentence to segment.
+    ///
+    /// # Returns
+    /// The tokens, with `pos` set to `nil`.
+    ///
+    /// # Errors
+    /// Returns a Ruby exception if the result array cannot be built.
+    fn segment_tokens(ruby: &Ruby, rb_self: &Self, text: String) -> Result<RArray, Error> {
+        tokens_to_array(ruby, rb_self.inner.segment_tokens(&text))
+    }
+
+    /// Splits a sentence into tokens and tags each with a UPOS tag.
+    ///
+    /// # Arguments
+    /// * `text` - The sentence to segment and tag.
+    ///
+    /// # Returns
+    /// The tagged tokens, with byte offsets into `text`.
+    ///
+    /// # Errors
+    /// Raises `Litsea::PosUnavailableError` when this segmenter was built
+    /// from a segmentation-only model.
+    fn segment_with_pos(ruby: &Ruby, rb_self: &Self, text: String) -> Result<RArray, Error> {
+        let tokens = map_err(rb_self.inner.segment_with_pos(&text))?;
+        tokens_to_array(ruby, tokens)
+    }
+
+    /// Splits and tags several sentences, releasing the GVL.
+    ///
+    /// # Arguments
+    /// * `texts` - The sentences to segment and tag.
+    ///
+    /// # Returns
+    /// One tagged-token array per input sentence, in input order.
+    ///
+    /// # Errors
+    /// Raises `Litsea::PosUnavailableError` when this segmenter was built
+    /// from a segmentation-only model.
+    fn segment_with_pos_batch(
+        ruby: &Ruby,
+        rb_self: &Self,
+        texts: Vec<String>,
+    ) -> Result<RArray, Error> {
+        let batches = without_gvl(|| rb_self.inner.segment_with_pos_batch(&texts));
+        let outer = ruby.ary_new_capa(texts.len());
+        for tokens in map_err(batches)? {
+            outer.push(tokens_to_array(ruby, tokens)?)?;
+        }
+        Ok(outer)
+    }
+}
+
+/// Builds a Ruby array of [`Token`] objects.
+///
+/// Wrapped types satisfy `IntoValue` but not `IntoValueFromNative`, which is
+/// what `Vec<T>` conversion requires, so the array is built element by
+/// element.
+///
+/// # Arguments
+/// * `ruby` - The Ruby handle for the current thread.
+/// * `tokens` - The token views to wrap.
+///
+/// # Returns
+/// A Ruby array of `Litsea::Token` objects.
+///
+/// # Errors
+/// Returns a Ruby exception if an element cannot be pushed.
+fn tokens_to_array(ruby: &Ruby, tokens: Vec<TokenView>) -> Result<RArray, Error> {
+    let array = ruby.ary_new_capa(tokens.len());
+    for view in tokens {
+        array.push(Token::from(view))?;
+    }
+    Ok(array)
+}
+
+/// Defines `Litsea::Segmenter`.
+///
+/// # Arguments
+/// * `ruby` - The Ruby handle for the current thread.
+/// * `module` - The `Litsea` module to define the class on.
+///
+/// # Returns
+/// `()` on success.
+///
+/// # Errors
+/// Returns a Ruby exception if the class cannot be defined.
+pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let class = module.define_class("Segmenter", ruby.class_object())?;
+    class.define_singleton_method("open", magnus::function!(Segmenter::open, 2))?;
+    class.define_singleton_method("from_bytes", magnus::function!(Segmenter::from_bytes, 2))?;
+    class.define_singleton_method("from_uri", magnus::function!(Segmenter::from_uri, 2))?;
+    class.define_method("language", magnus::method!(Segmenter::language, 0))?;
+    class.define_method("has_pos?", magnus::method!(Segmenter::has_pos, 0))?;
+    class.define_method("segment", magnus::method!(Segmenter::segment, 1))?;
+    class.define_method("segment_batch", magnus::method!(Segmenter::segment_batch, 1))?;
+    class.define_method("segment_tokens", magnus::method!(Segmenter::segment_tokens, 1))?;
+    class.define_method("segment_with_pos", magnus::method!(Segmenter::segment_with_pos, 1))?;
+    class.define_method(
+        "segment_with_pos_batch",
+        magnus::method!(Segmenter::segment_with_pos_batch, 1),
+    )?;
+    Ok(())
+}