lingua_rs 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95237de3af0469a6394db02288b9230472c763dc35cfc08768fc7a0233de6d18
-  data.tar.gz: fe793fbe6f8fa05d408b43579738b1e386db9fc9f93a590700f4726f617f7fa2
+  metadata.gz: e301297292f2de86fd62416f6f06765c7ccd373c81020e2ad365e0c0151a55bf
+  data.tar.gz: fe961910a3e44fa7df10580100b4aecef6f2552c08a7faba278a84eaf7ae89fc
 SHA512:
-  metadata.gz: 9a0fe59c439a1b27c7f0f95b9a3eba1a9c4dabe441ab5a07c0864727d9579b307e0e0c345ff0285b9c11854c7188c3c0726480103ff0390006ef799c3ee5c1bd
-  data.tar.gz: 82c83d17a21affafcc79cf0999658f33c225b5a7f9d532372f00dde0834b5b663cb0a572630042d5d3e725ae05b25189770993a66eebcf257c2b4eec6b5ca5e2
+  metadata.gz: 31e45f89765b5feb782a8b47f9574891c9ce826b5883f840b5f0d06c8715f26c9d9caa999a8e08349f475e307b2ccd28ecb303677ae8e51caa79920d94cc91fc
+  data.tar.gz: 6ace3072e3b7e8e044c55712879ea398dc7f009475cb058444070cd22834bb7f4fa6525ab4a66bab95417d57b1f57b9599cda02dd159be80603280c0646b6414

data/README.md

@@ -1,5 +1,10 @@
 # LinguaRs
 
+[![CI](https://github.com/kochka/lingua_rs/actions/workflows/ci.yml/badge.svg)](https://github.com/kochka/lingua_rs/actions/workflows/ci.yml)
+[![Gem Version](https://badge.fury.io/rb/lingua_rs.svg)](https://badge.fury.io/rb/lingua_rs)
+[![Downloads](https://img.shields.io/gem/dt/lingua_rs)](https://rubygems.org/gems/lingua_rs)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
 A Ruby gem wrapping the [Lingua](https://github.com/pemistahl/lingua-rs) Rust library for language detection.
 
 ## Installation
@@ -104,7 +109,7 @@ detector.detect('Hello world')        # => #<Lingua::Language English>
 
 ```ruby
 detector.confidence('Bonjour le monde', :fr)  # => 0.8217
-Lingua.confidence('Bonjour le monde', 'fr')   # => 0.3794
+Lingua.confidence('Bonjour le monde', 'fr')   # => 0.8217
 
 results = detector.confidence_values('Bonjour le monde')
 results.first.language   # => #<Lingua::Language French>
@@ -113,13 +118,41 @@ results.first.to_s       # => "French (0.82)"
 results.sum(&:confidence) # => 1.0
 ```
 
+### Mixed-language detection
+
+`detect_multiple` identifies multiple languages within a single text and returns an array of `Lingua::Segment` objects. Available on both `Lingua::Detector` and as a module method on `Lingua`.
+
+```ruby
+text = "Parlez-vous français? Ich spreche Französisch nur ein bisschen. A little bit is better than nothing."
+
+segments = Lingua.detect_multiple(text, languages: %w[en fr de])
+segments.each do |s|
+  puts "#{s.language} (#{s.start_index}..#{s.end_index}): #{s.text}"
+end
+# French (0..22): Parlez-vous français?
+# German (23..64): Ich spreche Französisch nur ein bisschen.
+# English (65..101): A little bit is better than nothing.
+
+# With a persistent detector
+detector = Lingua::Detector.new(languages: %w[en fr de])
+detector.detect_multiple(text)
+```
+
 ### `Lingua::Language` methods
 
-`Lingua::Language` objects support equality (`==`) and can be used as Hash keys.
+`Lingua::Language` objects support equality (`==`) and can be used as Hash keys. You can look up a language by name, ISO 639-1 code, or ISO 639-3 code using `[]`:
+
+```ruby
+Lingua::Language['French']  # => #<Lingua::Language French>
+Lingua::Language[:fr]       # => #<Lingua::Language French>
+Lingua::Language['fra']     # => #<Lingua::Language French>
+Lingua::Language['xxx']     # => nil
+```
 
 | Method | Return type | Example |
 |---|---|---|
-| `to_s` | `String` | `'French'` |
+| `name` | `String` | `'French'` |
+| `to_s` | `String` | `'French'` (alias for `name`) |
 | `to_sym` | `Symbol` | `:french` |
 | `to_iso` | `String` | `'fr'` (alias for `to_iso6391`) |
 | `to_iso6391` | `String` | `'fr'` |
@@ -128,6 +161,14 @@ results.sum(&:confidence) # => 1.0
 | `==` | `Boolean` | Compare two languages |
 | `hash` | `Integer` | Hash value (usable as Hash key) |
 
+Class methods:
+
+| Method | Return type | Description |
+|---|---|---|
+| `Lingua::Language.all` | `Array<Lingua::Language>` | All supported languages |
+| `Lingua::Language.names` | `Array<String>` | All language names (e.g. `'French'`) |
+| `Lingua::Language.iso_codes` | `Array<String>` | All ISO 639-1 codes (e.g. `'fr'`) |
+
 ### `Lingua::ConfidenceResult` methods
 
 Returned by `confidence_values`.
@@ -139,10 +180,28 @@ Returned by `confidence_values`.
 | `to_s` | `String` | `'French (0.82)'` |
 | `inspect` | `String` | `'#<Lingua::ConfidenceResult French (0.8217)>'` |
 
+### `Lingua::Segment` methods
+
+Returned by `detect_multiple`.
+
+| Method | Return type | Example |
+|---|---|---|
+| `language` | `Lingua::Language` | `#<Lingua::Language French>` |
+| `start_index` | `Integer` | `0` |
+| `end_index` | `Integer` | `22` |
+| `word_count` | `Integer` | `3` |
+| `text` | `String` | `'Parlez-vous français? '` |
+| `to_s` | `String` | `'French (0-22): Parlez-vous français? '` |
+| `inspect` | `String` | `'#<Lingua::Segment French (0-22) "Parlez-vous français? ">'` |
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake compile` to build the native extension and `rake test` to run the tests.
 
+## Acknowledgements
+
+This gem is built on top of [Lingua](https://github.com/pemistahl/lingua-rs) by [Peter M. Stahl](https://github.com/pemistahl), a highly accurate natural language detection library written in Rust.
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/ext/lingua/src/confidence_result.rs

@@ -1,7 +1,18 @@
 use lingua::Language;
+use magnus::{Error, RModule, Ruby, method, prelude::*};
 
 use crate::language::WrappedLanguage;
 
+pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let class = module.define_class("ConfidenceResult", ruby.class_object())?;
+    class.undef_default_alloc_func();
+    class.define_method("language", method!(ConfidenceResult::language, 0))?;
+    class.define_method("confidence", method!(ConfidenceResult::confidence, 0))?;
+    class.define_method("to_s", method!(ConfidenceResult::to_s, 0))?;
+    class.define_method("inspect", method!(ConfidenceResult::inspect, 0))?;
+    Ok(())
+}
+
 #[magnus::wrap(class = "Lingua::ConfidenceResult")]
 pub struct ConfidenceResult {
     pub language: Language,

data/ext/lingua/src/detector.rs

@@ -1,10 +1,21 @@
 use lingua::{LanguageDetector, LanguageDetectorBuilder};
-use magnus::{Error, RArray, RHash, Ruby};
+use magnus::{Error, RArray, RHash, RModule, Ruby, function, method, prelude::*};
 
 use crate::confidence_result::ConfidenceResult;
+use crate::segment::Segment;
 use crate::helpers::{fetch_option, parse_language, value_to_string};
 use crate::language::WrappedLanguage;
 
+pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let class = module.define_class("Detector", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RubyDetector::new, -1))?;
+    class.define_method("detect", method!(RubyDetector::detect, 1))?;
+    class.define_method("confidence", method!(RubyDetector::confidence, 2))?;
+    class.define_method("confidence_values", method!(RubyDetector::confidence_values, 1))?;
+    class.define_method("detect_multiple", method!(RubyDetector::detect_multiple, 1))?;
+    Ok(())
+}
+
 pub fn compute_confidence(
     detector: &LanguageDetector,
     subject: String,
@@ -66,6 +77,32 @@ impl RubyDetector {
     pub fn confidence_values(&self, subject: String) -> Result<RArray, Error> {
         compute_confidence_values(&self.detector, subject)
     }
+
+    pub fn detect_multiple(&self, subject: String) -> Result<RArray, Error> {
+        compute_detect_multiple(&self.detector, &subject)
+    }
+}
+
+pub fn compute_detect_multiple(
+    detector: &LanguageDetector,
+    subject: &str,
+) -> Result<RArray, Error> {
+    let ruby = Ruby::get().unwrap();
+    let results = detector.detect_multiple_languages_of(subject);
+    let array = ruby.ary_new_capa(results.len());
+    for r in results {
+        let text = subject[r.start_index()..r.end_index()].to_string();
+        let start_index = subject[..r.start_index()].chars().count();
+        let end_index = start_index + text.chars().count();
+        array.push(Segment {
+            language: r.language(),
+            start_index,
+            end_index,
+            word_count: r.word_count(),
+            text,
+        })?;
+    }
+    Ok(array)
 }
 
 pub fn build_detector_from_options(

data/ext/lingua/src/language.rs

@@ -1,11 +1,71 @@
 use lingua::Language;
-use magnus::{Ruby, Symbol};
+use magnus::{Error, RArray, RModule, Ruby, Symbol, function, method, prelude::*};
+
+use crate::helpers::value_to_string;
+
+pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let class = module.define_class("Language", ruby.class_object())?;
+    class.undef_default_alloc_func();
+    class.define_method("name", method!(WrappedLanguage::name, 0))?;
+    class.define_method("to_s", method!(WrappedLanguage::name, 0))?;
+    class.define_method("to_iso6391", method!(WrappedLanguage::to_iso6391, 0))?;
+    class.define_method("to_iso", method!(WrappedLanguage::to_iso6391, 0))?;
+    class.define_method("to_iso6393", method!(WrappedLanguage::to_iso6393, 0))?;
+    class.define_method("to_sym", method!(WrappedLanguage::to_sym, 0))?;
+    class.define_method("inspect", method!(WrappedLanguage::inspect, 0))?;
+    class.define_method("==", method!(WrappedLanguage::eq, 1))?;
+    class.define_method("eql?", method!(WrappedLanguage::eq, 1))?;
+    class.define_method("hash", method!(WrappedLanguage::hash, 0))?;
+    class.define_singleton_method("[]", function!(WrappedLanguage::lookup, 1))?;
+    class.define_singleton_method("all", function!(WrappedLanguage::all, 0))?;
+    class.define_singleton_method("names", function!(WrappedLanguage::names, 0))?;
+    class.define_singleton_method("iso_codes", function!(WrappedLanguage::iso_codes, 0))?;
+    Ok(())
+}
 
 #[magnus::wrap(class = "Lingua::Language")]
 pub struct WrappedLanguage(pub Language);
 
 impl WrappedLanguage {
-    pub fn to_s(&self) -> String {
+    pub fn lookup(value: magnus::Value) -> Result<Option<WrappedLanguage>, Error> {
+        let input = value_to_string(value)?;
+        Ok(crate::helpers::parse_language(&input).map(WrappedLanguage))
+    }
+
+    pub fn all() -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let mut langs: Vec<Language> = Language::all().into_iter().collect();
+        langs.sort_by(|a, b| a.to_string().cmp(&b.to_string()));
+        let array = ruby.ary_new_capa(langs.len());
+        for lang in langs {
+            array.push(WrappedLanguage(lang))?;
+        }
+        Ok(array)
+    }
+
+    pub fn names() -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let mut langs: Vec<Language> = Language::all().into_iter().collect();
+        langs.sort_by(|a, b| a.to_string().cmp(&b.to_string()));
+        let array = ruby.ary_new_capa(langs.len());
+        for lang in langs {
+            array.push(lang.to_string())?;
+        }
+        Ok(array)
+    }
+
+    pub fn iso_codes() -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let mut langs: Vec<Language> = Language::all().into_iter().collect();
+        langs.sort_by(|a, b| a.to_string().cmp(&b.to_string()));
+        let array = ruby.ary_new_capa(langs.len());
+        for lang in langs {
+            array.push(lang.iso_code_639_1().to_string())?;
+        }
+        Ok(array)
+    }
+
+    pub fn name(&self) -> String {
         self.0.to_string()
     }
 

data/ext/lingua/src/lib.rs

@@ -2,11 +2,11 @@ mod confidence_result;
 mod detector;
 mod helpers;
 mod language;
+mod segment;
 
-use magnus::{Error, RArray, RHash, Ruby, function, method, prelude::*};
+use magnus::{Error, RArray, RHash, Ruby, function, prelude::*};
 
-use confidence_result::ConfidenceResult;
-use detector::{RubyDetector, build_detector_from_options, compute_confidence, compute_confidence_values};
+use detector::{build_detector_from_options, compute_confidence, compute_confidence_values, compute_detect_multiple};
 use language::WrappedLanguage;
 
 fn detect(ruby: &Ruby, arguments: RArray) -> Result<Option<WrappedLanguage>, Error> {
@@ -32,42 +32,28 @@ fn confidence_values(ruby: &Ruby, arguments: RArray) -> Result<RArray, Error> {
     compute_confidence_values(&detector, subject)
 }
 
+fn detect_multiple(ruby: &Ruby, arguments: RArray) -> Result<RArray, Error> {
+    let subject = arguments
+        .shift::<String>()
+        .map_err(|_| Error::new(ruby.exception_arg_error(), "expected a string as first argument"))?;
+    let options = arguments.shift::<RHash>().ok();
+    let detector = build_detector_from_options(ruby, options.as_ref())?;
+    compute_detect_multiple(&detector, &subject)
+}
+
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = ruby.define_module("Lingua")?;
 
-    // Lingua::Language
-    let language_class = module.define_class("Language", ruby.class_object())?;
-    language_class.undef_default_alloc_func();
-    language_class.define_method("to_s", method!(WrappedLanguage::to_s, 0))?;
-    language_class.define_method("to_iso6391", method!(WrappedLanguage::to_iso6391, 0))?;
-    language_class.define_method("to_iso", method!(WrappedLanguage::to_iso6391, 0))?;
-    language_class.define_method("to_iso6393", method!(WrappedLanguage::to_iso6393, 0))?;
-    language_class.define_method("to_sym", method!(WrappedLanguage::to_sym, 0))?;
-    language_class.define_method("inspect", method!(WrappedLanguage::inspect, 0))?;
-    language_class.define_method("==", method!(WrappedLanguage::eq, 1))?;
-    language_class.define_method("eql?", method!(WrappedLanguage::eq, 1))?;
-    language_class.define_method("hash", method!(WrappedLanguage::hash, 0))?;
-
-    // Lingua::ConfidenceResult
-    let confidence_class = module.define_class("ConfidenceResult", ruby.class_object())?;
-    confidence_class.undef_default_alloc_func();
-    confidence_class.define_method("language", method!(ConfidenceResult::language, 0))?;
-    confidence_class.define_method("confidence", method!(ConfidenceResult::confidence, 0))?;
-    confidence_class.define_method("to_s", method!(ConfidenceResult::to_s, 0))?;
-    confidence_class.define_method("inspect", method!(ConfidenceResult::inspect, 0))?;
-
-    // Lingua::Detector
-    let detector_class = module.define_class("Detector", ruby.class_object())?;
-    detector_class.define_singleton_method("new", function!(RubyDetector::new, -1))?;
-    detector_class.define_method("detect", method!(RubyDetector::detect, 1))?;
-    detector_class.define_method("confidence", method!(RubyDetector::confidence, 2))?;
-    detector_class.define_method("confidence_values", method!(RubyDetector::confidence_values, 1))?;
+    language::define(ruby, &module)?;
+    confidence_result::define(ruby, &module)?;
+    segment::define(ruby, &module)?;
+    detector::define(ruby, &module)?;
 
-    // Functional API (module methods)
     module.define_singleton_method("detect", function!(detect, -2))?;
     module.define_singleton_method("confidence", function!(confidence, 2))?;
     module.define_singleton_method("confidence_values", function!(confidence_values, -2))?;
+    module.define_singleton_method("detect_multiple", function!(detect_multiple, -2))?;
 
     Ok(())
 }

data/ext/lingua/src/segment.rs

@@ -0,0 +1,59 @@
+use lingua::Language;
+use magnus::{Error, RModule, Ruby, method, prelude::*};
+
+use crate::language::WrappedLanguage;
+
+pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
+    let class = module.define_class("Segment", ruby.class_object())?;
+    class.undef_default_alloc_func();
+    class.define_method("language", method!(Segment::language, 0))?;
+    class.define_method("start_index", method!(Segment::start_index, 0))?;
+    class.define_method("end_index", method!(Segment::end_index, 0))?;
+    class.define_method("word_count", method!(Segment::word_count, 0))?;
+    class.define_method("text", method!(Segment::text, 0))?;
+    class.define_method("to_s", method!(Segment::to_s, 0))?;
+    class.define_method("inspect", method!(Segment::inspect, 0))?;
+    Ok(())
+}
+
+#[magnus::wrap(class = "Lingua::Segment")]
+pub struct Segment {
+    pub language: Language,
+    pub start_index: usize,
+    pub end_index: usize,
+    pub word_count: usize,
+    pub text: String,
+}
+
+impl Segment {
+    pub fn language(&self) -> WrappedLanguage {
+        WrappedLanguage(self.language)
+    }
+
+    pub fn start_index(&self) -> usize {
+        self.start_index
+    }
+
+    pub fn end_index(&self) -> usize {
+        self.end_index
+    }
+
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
+    pub fn text(&self) -> String {
+        self.text.clone()
+    }
+
+    pub fn to_s(&self) -> String {
+        format!("{} ({}-{}): {}", self.language, self.start_index, self.end_index, self.text)
+    }
+
+    pub fn inspect(&self) -> String {
+        format!(
+            "#<Lingua::Segment {} ({}-{}) \"{}\">",
+            self.language, self.start_index, self.end_index, self.text
+        )
+    }
+}

data/lib/lingua/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lingua
-  VERSION = '0.1.0'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lingua_rs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Sébastien Vrillaud
@@ -59,6 +59,7 @@ files:
 - ext/lingua/src/helpers.rs
 - ext/lingua/src/language.rs
 - ext/lingua/src/lib.rs
+- ext/lingua/src/segment.rs
 - lib/lingua.rb
 - lib/lingua/version.rb
 - sig/lingua.rbs
@@ -66,9 +67,10 @@ homepage: https://github.com/kochka/lingua_rs
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/kochka/lingua_rs
   source_code_uri: https://github.com/kochka/lingua_rs
   changelog_uri: https://github.com/kochka/lingua_rs/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/kochka/lingua_rs#readme
+  bug_tracker_uri: https://github.com/kochka/lingua_rs/issues
 rdoc_options: []
 require_paths:
 - lib