lingua_rs 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95237de3af0469a6394db02288b9230472c763dc35cfc08768fc7a0233de6d18
-  data.tar.gz: fe793fbe6f8fa05d408b43579738b1e386db9fc9f93a590700f4726f617f7fa2
+  metadata.gz: c52e545860174c156f8e8276d25762293daa3235766fc88777e5d2b5d53f50f5
+  data.tar.gz: 90bb85fd278795e300b768e3e5079d465f9c5b85a7c3a62f61c4e45745b5ba96
 SHA512:
-  metadata.gz: 9a0fe59c439a1b27c7f0f95b9a3eba1a9c4dabe441ab5a07c0864727d9579b307e0e0c345ff0285b9c11854c7188c3c0726480103ff0390006ef799c3ee5c1bd
-  data.tar.gz: 82c83d17a21affafcc79cf0999658f33c225b5a7f9d532372f00dde0834b5b663cb0a572630042d5d3e725ae05b25189770993a66eebcf257c2b4eec6b5ca5e2
+  metadata.gz: 82cc16c4f8b55d0345f3312675cbf6602365ea514b0f085f12cb9d4c8266601fcdf06d3195a06d74ef64a965ada7d9d79215ee44e70f6ac56e70c4f068cefdcd
+  data.tar.gz: eee0c6da2add0958dd836223f2a5d958e393a83f3b84b240598cf600ff0dbf73c3686e556486ad7b42e44f9781ad4905bcece9a59736e988e9532ec2d4174d4c

data/README.md

@@ -1,5 +1,10 @@
 # LinguaRs
 
+[![CI](https://github.com/kochka/lingua_rs/actions/workflows/ci.yml/badge.svg)](https://github.com/kochka/lingua_rs/actions/workflows/ci.yml)
+[![Gem Version](https://badge.fury.io/rb/lingua_rs.svg)](https://badge.fury.io/rb/lingua_rs)
+[![Downloads](https://img.shields.io/gem/dt/lingua_rs)](https://rubygems.org/gems/lingua_rs)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+
 A Ruby gem wrapping the [Lingua](https://github.com/pemistahl/lingua-rs) Rust library for language detection.
 
 ## Installation
@@ -104,7 +109,7 @@ detector.detect('Hello world')        # => #<Lingua::Language English>
 
 ```ruby
 detector.confidence('Bonjour le monde', :fr)  # => 0.8217
-Lingua.confidence('Bonjour le monde', 'fr')   # => 0.3794
+Lingua.confidence('Bonjour le monde', 'fr')   # => 0.8217
 
 results = detector.confidence_values('Bonjour le monde')
 results.first.language   # => #<Lingua::Language French>
@@ -113,6 +118,26 @@ results.first.to_s       # => "French (0.82)"
 results.sum(&:confidence) # => 1.0
 ```
 
+### Mixed-language detection
+
+`detect_multiple` identifies multiple languages within a single text and returns an array of `Lingua::Segment` objects. Available on both `Lingua::Detector` and as a module method on `Lingua`.
+
+```ruby
+text = "Parlez-vous français? Ich spreche Französisch nur ein bisschen. A little bit is better than nothing."
+
+segments = Lingua.detect_multiple(text, languages: %w[en fr de])
+segments.each do |s|
+  puts "#{s.language} (#{s.start_index}..#{s.end_index}): #{s.text}"
+end
+# French (0..22): Parlez-vous français?
+# German (23..64): Ich spreche Französisch nur ein bisschen.
+# English (65..101): A little bit is better than nothing.
+
+# With a persistent detector
+detector = Lingua::Detector.new(languages: %w[en fr de])
+detector.detect_multiple(text)
+```
+
 ### `Lingua::Language` methods
 
 `Lingua::Language` objects support equality (`==`) and can be used as Hash keys.
@@ -139,10 +164,28 @@ Returned by `confidence_values`.
 | `to_s` | `String` | `'French (0.82)'` |
 | `inspect` | `String` | `'#<Lingua::ConfidenceResult French (0.8217)>'` |
 
+### `Lingua::Segment` methods
+
+Returned by `detect_multiple`.
+
+| Method | Return type | Example |
+|---|---|---|
+| `language` | `Lingua::Language` | `#<Lingua::Language French>` |
+| `start_index` | `Integer` | `0` |
+| `end_index` | `Integer` | `22` |
+| `word_count` | `Integer` | `3` |
+| `text` | `String` | `'Parlez-vous français? '` |
+| `to_s` | `String` | `'French (0-22): Parlez-vous français? '` |
+| `inspect` | `String` | `'#<Lingua::Segment French (0-22) "Parlez-vous français? ">'` |
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake compile` to build the native extension and `rake test` to run the tests.
 
+## Acknowledgements
+
+This gem is built on top of [Lingua](https://github.com/pemistahl/lingua-rs) by [Peter M. Stahl](https://github.com/pemistahl), a highly accurate natural language detection library written in Rust.
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/ext/lingua/src/detector.rs

@@ -2,6 +2,7 @@ use lingua::{LanguageDetector, LanguageDetectorBuilder};
 use magnus::{Error, RArray, RHash, Ruby};
 
 use crate::confidence_result::ConfidenceResult;
+use crate::segment::Segment;
 use crate::helpers::{fetch_option, parse_language, value_to_string};
 use crate::language::WrappedLanguage;
 
@@ -66,6 +67,32 @@ impl RubyDetector {
     pub fn confidence_values(&self, subject: String) -> Result<RArray, Error> {
         compute_confidence_values(&self.detector, subject)
     }
+
+    pub fn detect_multiple(&self, subject: String) -> Result<RArray, Error> {
+        compute_detect_multiple(&self.detector, &subject)
+    }
+}
+
+pub fn compute_detect_multiple(
+    detector: &LanguageDetector,
+    subject: &str,
+) -> Result<RArray, Error> {
+    let ruby = Ruby::get().unwrap();
+    let results = detector.detect_multiple_languages_of(subject);
+    let array = ruby.ary_new_capa(results.len());
+    for r in results {
+        let text = subject[r.start_index()..r.end_index()].to_string();
+        let start_index = subject[..r.start_index()].chars().count();
+        let end_index = start_index + text.chars().count();
+        array.push(Segment {
+            language: r.language(),
+            start_index,
+            end_index,
+            word_count: r.word_count(),
+            text,
+        })?;
+    }
+    Ok(array)
 }
 
 pub fn build_detector_from_options(

data/ext/lingua/src/lib.rs

@@ -1,4 +1,5 @@
 mod confidence_result;
+mod segment;
 mod detector;
 mod helpers;
 mod language;
@@ -6,7 +7,8 @@ mod language;
 use magnus::{Error, RArray, RHash, Ruby, function, method, prelude::*};
 
 use confidence_result::ConfidenceResult;
-use detector::{RubyDetector, build_detector_from_options, compute_confidence, compute_confidence_values};
+use segment::Segment;
+use detector::{RubyDetector, build_detector_from_options, compute_confidence, compute_confidence_values, compute_detect_multiple};
 use language::WrappedLanguage;
 
 fn detect(ruby: &Ruby, arguments: RArray) -> Result<Option<WrappedLanguage>, Error> {
@@ -32,6 +34,15 @@ fn confidence_values(ruby: &Ruby, arguments: RArray) -> Result<RArray, Error> {
     compute_confidence_values(&detector, subject)
 }
 
+fn detect_multiple(ruby: &Ruby, arguments: RArray) -> Result<RArray, Error> {
+    let subject = arguments
+        .shift::<String>()
+        .map_err(|_| Error::new(ruby.exception_arg_error(), "expected a string as first argument"))?;
+    let options = arguments.shift::<RHash>().ok();
+    let detector = build_detector_from_options(ruby, options.as_ref())?;
+    compute_detect_multiple(&detector, &subject)
+}
+
 #[magnus::init]
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = ruby.define_module("Lingua")?;
@@ -57,17 +68,30 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     confidence_class.define_method("to_s", method!(ConfidenceResult::to_s, 0))?;
     confidence_class.define_method("inspect", method!(ConfidenceResult::inspect, 0))?;
 
+    // Lingua::Segment
+    let segment_class = module.define_class("Segment", ruby.class_object())?;
+    segment_class.undef_default_alloc_func();
+    segment_class.define_method("language", method!(Segment::language, 0))?;
+    segment_class.define_method("start_index", method!(Segment::start_index, 0))?;
+    segment_class.define_method("end_index", method!(Segment::end_index, 0))?;
+    segment_class.define_method("word_count", method!(Segment::word_count, 0))?;
+    segment_class.define_method("text", method!(Segment::text, 0))?;
+    segment_class.define_method("to_s", method!(Segment::to_s, 0))?;
+    segment_class.define_method("inspect", method!(Segment::inspect, 0))?;
+
     // Lingua::Detector
     let detector_class = module.define_class("Detector", ruby.class_object())?;
     detector_class.define_singleton_method("new", function!(RubyDetector::new, -1))?;
     detector_class.define_method("detect", method!(RubyDetector::detect, 1))?;
     detector_class.define_method("confidence", method!(RubyDetector::confidence, 2))?;
     detector_class.define_method("confidence_values", method!(RubyDetector::confidence_values, 1))?;
+    detector_class.define_method("detect_multiple", method!(RubyDetector::detect_multiple, 1))?;
 
     // Functional API (module methods)
     module.define_singleton_method("detect", function!(detect, -2))?;
     module.define_singleton_method("confidence", function!(confidence, 2))?;
     module.define_singleton_method("confidence_values", function!(confidence_values, -2))?;
+    module.define_singleton_method("detect_multiple", function!(detect_multiple, -2))?;
 
     Ok(())
 }

data/ext/lingua/src/segment.rs

@@ -0,0 +1,45 @@
+use lingua::Language;
+
+use crate::language::WrappedLanguage;
+
+#[magnus::wrap(class = "Lingua::Segment")]
+pub struct Segment {
+    pub language: Language,
+    pub start_index: usize,
+    pub end_index: usize,
+    pub word_count: usize,
+    pub text: String,
+}
+
+impl Segment {
+    pub fn language(&self) -> WrappedLanguage {
+        WrappedLanguage(self.language)
+    }
+
+    pub fn start_index(&self) -> usize {
+        self.start_index
+    }
+
+    pub fn end_index(&self) -> usize {
+        self.end_index
+    }
+
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
+    pub fn text(&self) -> String {
+        self.text.clone()
+    }
+
+    pub fn to_s(&self) -> String {
+        format!("{} ({}-{}): {}", self.language, self.start_index, self.end_index, self.text)
+    }
+
+    pub fn inspect(&self) -> String {
+        format!(
+            "#<Lingua::Segment {} ({}-{}) \"{}\">",
+            self.language, self.start_index, self.end_index, self.text
+        )
+    }
+}

data/lib/lingua/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lingua
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lingua_rs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sébastien Vrillaud
@@ -59,6 +59,7 @@ files:
 - ext/lingua/src/helpers.rs
 - ext/lingua/src/language.rs
 - ext/lingua/src/lib.rs
+- ext/lingua/src/segment.rs
 - lib/lingua.rb
 - lib/lingua/version.rb
 - sig/lingua.rbs
@@ -66,9 +67,10 @@ homepage: https://github.com/kochka/lingua_rs
 licenses:
 - MIT
 metadata:
-  homepage_uri: https://github.com/kochka/lingua_rs
   source_code_uri: https://github.com/kochka/lingua_rs
   changelog_uri: https://github.com/kochka/lingua_rs/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/kochka/lingua_rs#readme
+  bug_tracker_uri: https://github.com/kochka/lingua_rs/issues
 rdoc_options: []
 require_paths:
 - lib