lingua_rs 0.4.5 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 953c59791dc0b1234e1264f3e75881319edfd752a537a3bb0fd02c9830cb63ea
-  data.tar.gz: 6cf3ce511f0026deb1090c531771a843bb15e0fc4c7637e974f1d50b65330106
+  metadata.gz: 9ec0e0ba5844b9b3e47cd60babbc90ac03791bc83e3327e78b7f2d83ba4a71d8
+  data.tar.gz: 55af5915e63a1b34cbc5c73d2e2c63b6a1ce2030183112c54aad888b299bc889
 SHA512:
-  metadata.gz: 067a63ee83c014a358fa48f0df452c626123d8039862cb9dac57b4e733fb379fc6af97102613bd8c664b3762105123223a488b962896bfc419195d237ff6135f
-  data.tar.gz: 4336e0b72d1dff76ddf28d2721025b4a36a4f6dd031252a09bdfa1eb10fdfc6982dcbc1bbd01d223a84e0f5256c06a98190c4c47a0f154ba06696e4a60abfce7
+  metadata.gz: b6bfc5d1605bf171e7be4dd899e7b82460f4c8b81ab541ff97a18fb59205a752e9cfe12de640c20b6a60a4814bc8e89e884b9d501a2ec3b5a4e4f0b4bc2e094d
+  data.tar.gz: ecf2866f980b9ddd7189d639ea1330a4f47f8f0a528dcb636358c7c6de30add2d8a854cee8252eac36540fc528b1836eedebbd03313eeb1de167cd18c241afbf

data/README.md

@@ -66,8 +66,6 @@ Lingua.detect('Bonjour', languages: %w[en french deu])
 
 ### Options
 
-All options accept both symbol and string keys.
-
 | Option | Type | Description |
 |---|---|---|
 | `languages` | `Array<String>` | Restrict detection to these languages (names, ISO 639-1 or ISO 639-3 codes) |
@@ -132,7 +130,52 @@ detector = Lingua::Detector.new(languages: %w[en fr de])
 detector.detect_multiple(text)
 ```
 
-### `Lingua::Language` methods
+### Parallel batch detection
+
+`Lingua::Detector` provides batch methods that process multiple texts in parallel using [Rayon](https://github.com/rayon-rs/rayon). These are significantly faster than calling single-text methods in a loop.
+
+```ruby
+detector = Lingua::Detector.new(languages: %w[en fr de])
+
+# Detect language of multiple texts
+detector.detect_batch(['Bonjour le monde', 'Hello world', 'Hallo Welt'])
+# => [#<Lingua::Language French>, #<Lingua::Language English>, #<Lingua::Language German>]
+
+# Confidence for a specific language across multiple texts
+detector.confidence_batch(['Bonjour le monde', 'Hello world'], :fr)
+# => [0.8217, 0.0215]
+
+# Confidence values for multiple texts
+detector.confidence_values_batch(['Bonjour le monde', 'Hello world'])
+# => [[#<Lingua::ConfidenceResult French (0.82)>, ...], [#<Lingua::ConfidenceResult English (0.91)>, ...]]
+
+# Mixed-language detection on multiple texts
+detector.detect_multiple_batch(['Bonjour le monde. Hello world.', 'Ich bin müde. Salut tout le monde.'])
+# => [[#<Lingua::Segment French ...>, #<Lingua::Segment English ...>], [#<Lingua::Segment German ...>, #<Lingua::Segment French ...>]]
+```
+
+| Method | Return type | Description |
+|---|---|---|
+| `detect_batch(texts)` | `Array<Language\|nil>` | Detect language of each text |
+| `detect_multiple_batch(texts)` | `Array<Array<Segment>>` | Mixed-language detection for each text |
+| `confidence_values_batch(texts)` | `Array<Array<ConfidenceResult>>` | Confidence values for each text |
+| `confidence_batch(texts, language)` | `Array<Float>` | Confidence for a specific language across texts |
+
+### Error handling
+
+`Lingua::UnknownLanguageError` (subclass of `ArgumentError`) is raised when an unrecognized language name or code is passed:
+
+```ruby
+begin
+  Lingua.detect('Hello', languages: %w[en zzzz])
+rescue Lingua::UnknownLanguageError => e
+  puts e.message # => unknown language: "zzzz"
+end
+```
+
+## API Reference
+
+### `Lingua::Language`
 
 `Lingua::Language` objects support equality (`==`) and can be used as Hash keys. You can look up a language by name, ISO 639-1 code, or ISO 639-3 code using `[]`:
 
@@ -163,7 +206,7 @@ Lingua::Language['xxx']     # => nil
 | `==` | `Boolean` | Compare two languages |
 | `hash` | `Integer` | Hash value (usable as Hash key) |
 
-### `Lingua::ConfidenceResult` methods
+### `Lingua::ConfidenceResult`
 
 Returned by `confidence_values`.
 
@@ -174,7 +217,7 @@ Returned by `confidence_values`.
 | `to_s` | `String` | `'French (0.82)'` |
 | `inspect` | `String` | `'#<Lingua::ConfidenceResult French (0.8217)>'` |
 
-### `Lingua::Segment` methods
+### `Lingua::Segment`
 
 Returned by `detect_multiple`.
 
@@ -188,18 +231,6 @@ Returned by `detect_multiple`.
 | `to_s` | `String` | `'French (0-22): Parlez-vous français? '` |
 | `inspect` | `String` | `'#<Lingua::Segment French (0-22) "Parlez-vous français? ">'` |
 
-### Error handling
-
-`Lingua::UnknownLanguageError` (subclass of `ArgumentError`) is raised when an unrecognized language name or code is passed:
-
-```ruby
-begin
-  Lingua.detect('Hello', languages: %w[en zzzz])
-rescue Lingua::UnknownLanguageError => e
-  puts e.message # => unknown language: "zzzz"
-end
-```
-
 ## Optimization: selecting languages
 
 By default, all 75 languages are compiled into the native extension (~278 MB). If you only need a subset, select languages at build time to reduce binary size and improve detection speed.
@@ -266,6 +297,8 @@ Supported languages (from [Lingua Rust's official list](https://github.com/pemis
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake compile` to build the native extension and `rake test` to run the tests.
 
+To run benchmarks (sequential vs batch performance comparison): `rake bench`
+
 ## Acknowledgements
 
 This gem is built on top of [Lingua](https://github.com/pemistahl/lingua-rs) by [Peter M. Stahl](https://github.com/pemistahl), a highly accurate natural language detection library written in Rust.

data/Rakefile

@@ -19,4 +19,10 @@ RbSys::ExtensionTask.new('lingua', GEMSPEC) do |ext|
   ext.lib_dir = 'lib/lingua'
 end
 
+desc 'Run benchmarks (sequential vs batch)'
+task bench: :compile do
+  ENV['BENCH'] = '1'
+  ruby '-Itest', 'test/test_benchmark.rb'
+end
+
 task default: %i[compile test rubocop]

data/ext/lingua/src/detector.rs

@@ -13,6 +13,10 @@ pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
     class.define_method("confidence", method!(RubyDetector::confidence, 2))?;
     class.define_method("confidence_values", method!(RubyDetector::confidence_values, 1))?;
     class.define_method("detect_multiple", method!(RubyDetector::detect_multiple, 1))?;
+    class.define_method("detect_batch", method!(RubyDetector::detect_batch, 1))?;
+    class.define_method("detect_multiple_batch", method!(RubyDetector::detect_multiple_batch, 1))?;
+    class.define_method("confidence_values_batch", method!(RubyDetector::confidence_values_batch, 1))?;
+    class.define_method("confidence_batch", method!(RubyDetector::confidence_batch, 2))?;
     Ok(())
 }
 
@@ -81,6 +85,73 @@ impl RubyDetector {
     pub fn detect_multiple(&self, subject: String) -> Result<RArray, Error> {
         compute_detect_multiple(&self.detector, &subject)
     }
+
+    pub fn detect_batch(&self, texts: Vec<String>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let results = self.detector.detect_languages_in_parallel_of(&texts);
+        let array = ruby.ary_new_capa(results.len());
+        for result in results {
+            array.push(result.map(WrappedLanguage))?;
+        }
+        Ok(array)
+    }
+
+    pub fn detect_multiple_batch(&self, texts: Vec<String>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let results = self.detector.detect_multiple_languages_in_parallel_of(&texts);
+        let outer = ruby.ary_new_capa(results.len());
+        for (detection_results, subject) in results.iter().zip(texts.iter()) {
+            let inner = ruby.ary_new_capa(detection_results.len());
+            for r in detection_results {
+                let text = subject[r.start_index()..r.end_index()].to_string();
+                let start_index = subject[..r.start_index()].chars().count();
+                let end_index = start_index + text.chars().count();
+                inner.push(Segment {
+                    language: r.language(),
+                    start_index,
+                    end_index,
+                    word_count: r.word_count(),
+                    text,
+                })?;
+            }
+            outer.push(inner)?;
+        }
+        Ok(outer)
+    }
+
+    pub fn confidence_values_batch(&self, texts: Vec<String>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let results = self.detector.compute_language_confidence_values_in_parallel(&texts);
+        let outer = ruby.ary_new_capa(results.len());
+        for values in results {
+            let inner = ruby.ary_new_capa(values.len());
+            for (language, confidence) in values {
+                inner.push(ConfidenceResult {
+                    language,
+                    confidence,
+                })?;
+            }
+            outer.push(inner)?;
+        }
+        Ok(outer)
+    }
+
+    pub fn confidence_batch(&self, texts: Vec<String>, language: magnus::Value) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let language_str = value_to_string(language)?;
+        let lang = parse_language(&language_str).ok_or_else(|| {
+            Error::new(
+                unknown_language_error(&ruby),
+                format!("unknown language: \"{language_str}\""),
+            )
+        })?;
+        let results = self.detector.compute_language_confidence_in_parallel(&texts, lang);
+        let array = ruby.ary_new_capa(results.len());
+        for score in results {
+            array.push(score)?;
+        }
+        Ok(array)
+    }
 }
 
 pub fn compute_detect_multiple(

data/lib/lingua/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lingua
-  VERSION = '0.4.5'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lingua_rs
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.5.0
 platform: ruby
 authors:
 - Sébastien Vrillaud