lingua_rs 0.4.5 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 953c59791dc0b1234e1264f3e75881319edfd752a537a3bb0fd02c9830cb63ea
-  data.tar.gz: 6cf3ce511f0026deb1090c531771a843bb15e0fc4c7637e974f1d50b65330106
+  metadata.gz: 5c97528f5d90b92214436ba9938f56a4ffde0d9997081b896d80eed180ebe59a
+  data.tar.gz: 147e2bca0a928e18b23027f323ac2a297ccbb5c3480e178cc7054f3c1fc230d1
 SHA512:
-  metadata.gz: 067a63ee83c014a358fa48f0df452c626123d8039862cb9dac57b4e733fb379fc6af97102613bd8c664b3762105123223a488b962896bfc419195d237ff6135f
-  data.tar.gz: 4336e0b72d1dff76ddf28d2721025b4a36a4f6dd031252a09bdfa1eb10fdfc6982dcbc1bbd01d223a84e0f5256c06a98190c4c47a0f154ba06696e4a60abfce7
+  metadata.gz: f29859f59b455c0672c8411ab269c78e95b501a350a29dfc5d0966204e593b26aa36268f8c895aeb066962bf7d1d15a75b6ca3bbf576d3074b3b3d34bc393e03
+  data.tar.gz: e5320aba0f339e3d6e8ac92a1fff314a0946dd5713fe5e335f057601a97159ffa031b761eace762859180b786670370d94281903ac1c9d96b60e970e959a31ba

data/README.md

@@ -25,7 +25,7 @@ Then run `bundle install`.
 
 ### Build requirements
 
-A Rust toolchain is required to compile the native extension.
+A Rust toolchain (**1.87+**) is required to compile the native extension.
 
 ## Usage
 
@@ -66,8 +66,6 @@ Lingua.detect('Bonjour', languages: %w[en french deu])
 
 ### Options
 
-All options accept both symbol and string keys.
-
 | Option | Type | Description |
 |---|---|---|
 | `languages` | `Array<String>` | Restrict detection to these languages (names, ISO 639-1 or ISO 639-3 codes) |
@@ -132,7 +130,61 @@ detector = Lingua::Detector.new(languages: %w[en fr de])
 detector.detect_multiple(text)
 ```
 
-### `Lingua::Language` methods
+### Parallel batch detection
+
+`Lingua::Detector` provides batch methods that process multiple texts in parallel using [Rayon](https://github.com/rayon-rs/rayon). These are significantly faster than calling single-text methods in a loop.
+
+```ruby
+detector = Lingua::Detector.new(languages: %w[en fr de])
+
+# Detect language of multiple texts
+detector.detect_batch(['Bonjour le monde', 'Hello world', 'Hallo Welt'])
+# => [#<Lingua::Language French>, #<Lingua::Language English>, #<Lingua::Language German>]
+
+# Confidence for a specific language across multiple texts
+detector.confidence_batch(['Bonjour le monde', 'Hello world'], :fr)
+# => [0.8217, 0.0215]
+
+# Confidence values for multiple texts
+detector.confidence_values_batch(['Bonjour le monde', 'Hello world'])
+# => [[#<Lingua::ConfidenceResult French (0.82)>, ...], [#<Lingua::ConfidenceResult English (0.91)>, ...]]
+
+# Mixed-language detection on multiple texts
+detector.detect_multiple_batch(['Bonjour le monde. Hello world.', 'Ich bin müde. Salut tout le monde.'])
+# => [[#<Lingua::Segment French ...>, #<Lingua::Segment English ...>], [#<Lingua::Segment German ...>, #<Lingua::Segment French ...>]]
+```
+
+| Method | Return type | Description |
+|---|---|---|
+| `detect_batch(texts)` | `Array<Language\|nil>` | Detect language of each text |
+| `confidence_batch(texts, language)` | `Array<Float>` | Confidence for a specific language across texts |
+| `confidence_values_batch(texts)` | `Array<Array<ConfidenceResult>>` | Confidence values for each text |
+| `detect_multiple_batch(texts)` | `Array<Array<Segment>>` | Mixed-language detection for each text |
+
+Results are returned in the same order as the input, so you can pair them with `zip`:
+
+```ruby
+texts = ['Bonjour le monde', 'Hello world', 'Hallo Welt']
+texts.zip(detector.detect_batch(texts)) do |text, language|
+  puts "#{text} => #{language}"
+end
+```
+
+### Error handling
+
+`Lingua::UnknownLanguageError` (subclass of `ArgumentError`) is raised when an unrecognized language name or code is passed:
+
+```ruby
+begin
+  Lingua.detect('Hello', languages: %w[en zzzz])
+rescue Lingua::UnknownLanguageError => e
+  puts e.message # => unknown language: "zzzz"
+end
+```
+
+## API Reference
+
+### `Lingua::Language`
 
 `Lingua::Language` objects support equality (`==`) and can be used as Hash keys. You can look up a language by name, ISO 639-1 code, or ISO 639-3 code using `[]`:
 
@@ -163,7 +215,7 @@ Lingua::Language['xxx']     # => nil
 | `==` | `Boolean` | Compare two languages |
 | `hash` | `Integer` | Hash value (usable as Hash key) |
 
-### `Lingua::ConfidenceResult` methods
+### `Lingua::ConfidenceResult`
 
 Returned by `confidence_values`.
 
@@ -174,7 +226,7 @@ Returned by `confidence_values`.
 | `to_s` | `String` | `'French (0.82)'` |
 | `inspect` | `String` | `'#<Lingua::ConfidenceResult French (0.8217)>'` |
 
-### `Lingua::Segment` methods
+### `Lingua::Segment`
 
 Returned by `detect_multiple`.
 
@@ -188,18 +240,49 @@ Returned by `detect_multiple`.
 | `to_s` | `String` | `'French (0-22): Parlez-vous français? '` |
 | `inspect` | `String` | `'#<Lingua::Segment French (0-22) "Parlez-vous français? ">'` |
 
-### Error handling
+## Command-line tool
 
-`Lingua::UnknownLanguageError` (subclass of `ArgumentError`) is raised when an unrecognized language name or code is passed:
+The gem ships with a `lingua` executable that wraps the library for shell use.
 
-```ruby
-begin
-  Lingua.detect('Hello', languages: %w[en zzzz])
-rescue Lingua::UnknownLanguageError => e
-  puts e.message # => unknown language: "zzzz"
-end
+```bash
+# Text as argument or on stdin
+lingua 'Bonjour le monde'                     # => French
+echo 'Hello world' | lingua                   # => English
+
+# Output format
+lingua -f iso 'Bonjour le monde'              # => fr
+lingua -f iso6393 'Bonjour le monde'          # => fra
+lingua -f json 'Bonjour le monde'             # => {"name":"French","iso":"fr","iso6393":"fra"}
+
+# Confidence values
+lingua -c -l en,fr,de 'Bonjour le monde'
+# French: 0.8217
+# German: 0.1216
+# English: 0.0567
+
+# Mixed-language detection
+lingua -m -l en,fr,de 'Parlez-vous français? Hello world.'
+# French (0-22): Parlez-vous français?
+# English (22-34): Hello world.
+
+# Restrict candidates
+lingua -l en,fr,de 'Bonjour'
+
+# Batch: one text per line (use "-" for stdin)
+lingua --batch texts.txt
+cat texts.txt | lingua --batch -
 ```
 
+| Option | Description |
+|---|---|
+| `-f`, `--format FORMAT` | Output format: `name` (default), `iso`, `iso6393`, `json` |
+| `-c`, `--confidences` | Output confidence for all candidate languages |
+| `-m`, `--multiple` | Detect multiple languages within the text |
+| `-l`, `--languages LIST` | Comma-separated list of languages (names or ISO codes) |
+| `--batch FILE` | Treat each line of FILE (or `-` for stdin) as a separate input |
+| `-v`, `--version` | Show version |
+| `-h`, `--help` | Show help |
+
 ## Optimization: selecting languages
 
 By default, all 75 languages are compiled into the native extension (~278 MB). If you only need a subset, select languages at build time to reduce binary size and improve detection speed.
@@ -266,6 +349,8 @@ Supported languages (from [Lingua Rust's official list](https://github.com/pemis
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake compile` to build the native extension and `rake test` to run the tests.
 
+To run benchmarks (sequential vs batch performance comparison): `rake bench`
+
 ## Acknowledgements
 
 This gem is built on top of [Lingua](https://github.com/pemistahl/lingua-rs) by [Peter M. Stahl](https://github.com/pemistahl), a highly accurate natural language detection library written in Rust.

data/Rakefile

@@ -19,4 +19,10 @@ RbSys::ExtensionTask.new('lingua', GEMSPEC) do |ext|
   ext.lib_dir = 'lib/lingua'
 end
 
+desc 'Run benchmarks (sequential vs batch)'
+task bench: :compile do
+  ENV['BENCH'] = '1'
+  ruby '-Itest', 'test/test_benchmark.rb'
+end
+
 task default: %i[compile test rubocop]

data/exe/lingua

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+require 'json'
+require 'lingua'
+
+def build_parser(options)
+  OptionParser.new do |opts|
+    opts.banner = 'Usage: lingua [options] [TEXT]'
+    opts.separator ''
+    opts.separator 'If TEXT is omitted, input is read from stdin.'
+    opts.separator ''
+    opts.separator 'Options:'
+
+    opts.on('-f', '--format FORMAT', %w[iso iso6393 name json],
+            'Output format: name (default), iso, iso6393, json') { |f| options[:format] = f.to_sym }
+    opts.on('-c', '--confidences',
+            'Output confidence values for all candidate languages') { options[:mode] = :confidences }
+    opts.on('-m', '--multiple',
+            'Detect multiple languages within the text') { options[:mode] = :multiple }
+    opts.on('-l', '--languages LIST',
+            'Restrict detection to comma-separated languages (names or ISO codes)') do |list|
+      options[:languages] = list.split(',').map(&:strip)
+    end
+    opts.on('--batch FILE',
+            'Treat each line of FILE (or "-" for stdin) as a separate input') { |f| options[:batch] = f }
+    opts.on('-h', '--help', 'Show this help') do
+      puts opts
+      exit
+    end
+    opts.on('-v', '--version', 'Show version') do
+      puts Lingua::VERSION
+      exit
+    end
+  end
+end
+
+options = { format: :name, mode: :detect, languages: nil, batch: nil }
+build_parser(options).parse!
+
+def format_language(lang, format)
+  return nil if lang.nil?
+
+  case format
+  when :iso     then lang.to_iso
+  when :iso6393 then lang.to_iso6393
+  when :name    then lang.name
+  when :json    then { name: lang.name, iso: lang.to_iso, iso6393: lang.to_iso6393 }
+  end
+end
+
+def render_detect(lang, format)
+  formatted = format_language(lang, format)
+  if format == :json
+    JSON.generate(formatted)
+  else
+    formatted.to_s
+  end
+end
+
+def render_confidences(results, format)
+  if format == :json
+    JSON.generate(results.map do |r|
+      { language: format_language(r.language, :iso), confidence: r.confidence }
+    end)
+  else
+    results.map { |r| "#{format_language(r.language, format)}: #{format('%.4f', r.confidence)}" }
+           .join("\n")
+  end
+end
+
+def render_multiple(segments, format)
+  if format == :json
+    JSON.generate(segments.map do |s|
+      { language: format_language(s.language, :iso),
+        start_index: s.start_index,
+        end_index: s.end_index,
+        text: s.text }
+    end)
+  else
+    segments.map do |s|
+      "#{format_language(s.language, format)} (#{s.start_index}-#{s.end_index}): #{s.text}"
+    end.join("\n")
+  end
+end
+
+def process(text, detector, options)
+  case options[:mode]
+  when :detect
+    render_detect(detector.detect(text), options[:format])
+  when :confidences
+    render_confidences(detector.confidence_values(text), options[:format])
+  when :multiple
+    render_multiple(detector.detect_multiple(text), options[:format])
+  end
+end
+
+begin
+  detector_options = {}
+  detector_options[:languages] = options[:languages] if options[:languages]
+  detector = Lingua::Detector.new(**detector_options)
+
+  if options[:batch]
+    io = options[:batch] == '-' ? $stdin : File.open(options[:batch], 'r')
+    io.each_line do |line|
+      line = line.chomp
+      next if line.empty?
+
+      puts process(line, detector, options)
+    end
+    io.close unless io == $stdin
+  else
+    text = ARGV.empty? ? $stdin.read : ARGV.join(' ')
+    text = text.strip
+    if text.empty?
+      warn 'lingua: no input text'
+      exit 1
+    end
+    puts process(text, detector, options)
+  end
+rescue Lingua::UnknownLanguageError => e
+  warn "lingua: #{e.message}"
+  exit 1
+end

data/ext/lingua/src/detector.rs

@@ -13,6 +13,10 @@ pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
     class.define_method("confidence", method!(RubyDetector::confidence, 2))?;
     class.define_method("confidence_values", method!(RubyDetector::confidence_values, 1))?;
     class.define_method("detect_multiple", method!(RubyDetector::detect_multiple, 1))?;
+    class.define_method("detect_batch", method!(RubyDetector::detect_batch, 1))?;
+    class.define_method("detect_multiple_batch", method!(RubyDetector::detect_multiple_batch, 1))?;
+    class.define_method("confidence_values_batch", method!(RubyDetector::confidence_values_batch, 1))?;
+    class.define_method("confidence_batch", method!(RubyDetector::confidence_batch, 2))?;
     Ok(())
 }
 
@@ -81,6 +85,73 @@ impl RubyDetector {
     pub fn detect_multiple(&self, subject: String) -> Result<RArray, Error> {
         compute_detect_multiple(&self.detector, &subject)
     }
+
+    pub fn detect_batch(&self, texts: Vec<String>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let results = self.detector.detect_languages_in_parallel_of(&texts);
+        let array = ruby.ary_new_capa(results.len());
+        for result in results {
+            array.push(result.map(WrappedLanguage))?;
+        }
+        Ok(array)
+    }
+
+    pub fn detect_multiple_batch(&self, texts: Vec<String>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let results = self.detector.detect_multiple_languages_in_parallel_of(&texts);
+        let outer = ruby.ary_new_capa(results.len());
+        for (detection_results, subject) in results.iter().zip(texts.iter()) {
+            let inner = ruby.ary_new_capa(detection_results.len());
+            for r in detection_results {
+                let text = subject[r.start_index()..r.end_index()].to_string();
+                let start_index = subject[..r.start_index()].chars().count();
+                let end_index = start_index + text.chars().count();
+                inner.push(Segment {
+                    language: r.language(),
+                    start_index,
+                    end_index,
+                    word_count: r.word_count(),
+                    text,
+                })?;
+            }
+            outer.push(inner)?;
+        }
+        Ok(outer)
+    }
+
+    pub fn confidence_values_batch(&self, texts: Vec<String>) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let results = self.detector.compute_language_confidence_values_in_parallel(&texts);
+        let outer = ruby.ary_new_capa(results.len());
+        for values in results {
+            let inner = ruby.ary_new_capa(values.len());
+            for (language, confidence) in values {
+                inner.push(ConfidenceResult {
+                    language,
+                    confidence,
+                })?;
+            }
+            outer.push(inner)?;
+        }
+        Ok(outer)
+    }
+
+    pub fn confidence_batch(&self, texts: Vec<String>, language: magnus::Value) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let language_str = value_to_string(language)?;
+        let lang = parse_language(&language_str).ok_or_else(|| {
+            Error::new(
+                unknown_language_error(&ruby),
+                format!("unknown language: \"{language_str}\""),
+            )
+        })?;
+        let results = self.detector.compute_language_confidence_in_parallel(&texts, lang);
+        let array = ruby.ary_new_capa(results.len());
+        for score in results {
+            array.push(score)?;
+        }
+        Ok(array)
+    }
 }
 
 pub fn compute_detect_multiple(

data/lib/lingua/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lingua
-  VERSION = '0.4.5'
+  VERSION = '0.6.0'
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: lingua_rs
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.6.0
 platform: ruby
 authors:
 - Sébastien Vrillaud
 bindir: exe
 cert_chain: []
-date: 2026-04-08 00:00:00.000000000 Z
+date: 2026-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -42,7 +42,8 @@ description: Native Ruby bindings for the Lingua Rust library, bringing highly a
   scores and ISO 639-1/639-3 language codes.
 email:
 - kochka@gmail.com
-executables: []
+executables:
+- lingua
 extensions:
 - ext/lingua/extconf.rb
 extra_rdoc_files: []
@@ -50,6 +51,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/lingua
 - ext/lingua/Cargo.toml
 - ext/lingua/extconf.rb
 - ext/lingua/src/confidence_result.rs
@@ -84,7 +86,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: High-accuracy language detection powered by Lingua (Rust).
 test_files: []