lingua_rs 0.3.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e301297292f2de86fd62416f6f06765c7ccd373c81020e2ad365e0c0151a55bf
-  data.tar.gz: fe961910a3e44fa7df10580100b4aecef6f2552c08a7faba278a84eaf7ae89fc
+  metadata.gz: 10f60785757487a1928e8d35dff0a23a5b10143f69cd4c147fe4da13b7362e31
+  data.tar.gz: 587f8b61751bee91d89df4fa2c278163cdd48c601536292f56ed35f4807044fd
 SHA512:
-  metadata.gz: 31e45f89765b5feb782a8b47f9574891c9ce826b5883f840b5f0d06c8715f26c9d9caa999a8e08349f475e307b2ccd28ecb303677ae8e51caa79920d94cc91fc
-  data.tar.gz: 6ace3072e3b7e8e044c55712879ea398dc7f009475cb058444070cd22834bb7f4fa6525ab4a66bab95417d57b1f57b9599cda02dd159be80603280c0646b6414
+  metadata.gz: '03383e1fa0da8d3131f2e8d724b5818697bcddae9604d33a3cfafe7b73b48d32468f2817eaa5b1606f726b47a326b7f2c48f46ef5b3361a605b4f7a7c3b20cc2'
+  data.tar.gz: e347f5ef0485bda583d4df5cfa116699b496184554a2d367615ff3e538e8a8a76769aca20c74a2f67f7fe2caf870d6184d28733e1d040f84d2ca5be8c21ecaf6

data/README.md

@@ -1,9 +1,9 @@
 # LinguaRs
 
 [![CI](https://github.com/kochka/lingua_rs/actions/workflows/ci.yml/badge.svg)](https://github.com/kochka/lingua_rs/actions/workflows/ci.yml)
-[![Gem Version](https://badge.fury.io/rb/lingua_rs.svg)](https://badge.fury.io/rb/lingua_rs)
-[![Downloads](https://img.shields.io/gem/dt/lingua_rs)](https://rubygems.org/gems/lingua_rs)
-[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Gem Version](https://img.shields.io/gem/v/lingua_rs?color=cc342d)](https://rubygems.org/gems/lingua_rs)
+[![Downloads](https://img.shields.io/gem/dt/lingua_rs?color=0969da)](https://rubygems.org/gems/lingua_rs)
+[![License: MIT](https://img.shields.io/badge/License-MIT-6e7781.svg)](https://opensource.org/licenses/MIT)
 
 A Ruby gem wrapping the [Lingua](https://github.com/pemistahl/lingua-rs) Rust library for language detection.
 
@@ -42,19 +42,18 @@ export PATH=/usr/lib/llvm21/bin:$PATH
 require 'lingua'
 
 lang = Lingua.detect('Bonjour le monde')
+lang.french?    # => true
 lang.to_s       # => 'French'
 lang.to_sym     # => :french
 lang.to_iso     # => 'fr'
-lang.to_iso6391 # => 'fr'
 lang.to_iso6393 # => 'fra'
-lang.inspect    # => '#<Lingua::Language French>'
 
 Lingua.detect('') # => nil
 ```
 
 ### Restricting languages
 
-Pass a `languages` option to limit detection to a subset of languages. Values can be full names, ISO 639-1 codes, or ISO 639-3 codes, and can be mixed freely.
+Pass a `languages` option to limit detection to a subset of languages. Values can be full names, ISO 639-1 codes, or ISO 639-3 codes, and can be mixed freely. Restricting the candidate set is also faster, since the detector has fewer languages to compare against.
 
 ```ruby
 # By full name
@@ -154,9 +153,11 @@ Lingua::Language['xxx']     # => nil
 | `name` | `String` | `'French'` |
 | `to_s` | `String` | `'French'` (alias for `name`) |
 | `to_sym` | `Symbol` | `:french` |
+| `iso_code` | `String` | `'fr'` (alias for `to_iso6391`) |
 | `to_iso` | `String` | `'fr'` (alias for `to_iso6391`) |
 | `to_iso6391` | `String` | `'fr'` |
 | `to_iso6393` | `String` | `'fra'` |
+| `french?` | `Boolean` | `true` (works with name, ISO 639-1 or ISO 639-3: `fr?`, `fra?`) |
 | `inspect` | `String` | `'#<Lingua::Language French>'` |
 | `==` | `Boolean` | Compare two languages |
 | `hash` | `Integer` | Hash value (usable as Hash key) |
@@ -194,6 +195,64 @@ Returned by `detect_multiple`.
 | `to_s` | `String` | `'French (0-22): Parlez-vous français? '` |
 | `inspect` | `String` | `'#<Lingua::Segment French (0-22) "Parlez-vous français? ">'` |
 
+### Error handling
+
+`Lingua::UnknownLanguageError` (subclass of `ArgumentError`) is raised when an unrecognized language name or code is passed:
+
+```ruby
+Lingua.detect('Hello', languages: %w[en zzzz])
+# => Lingua::UnknownLanguageError: unknown language: "zzzz"
+
+# Can also be rescued as ArgumentError
+begin
+  Lingua.detect('Hello', languages: %w[zzzz])
+rescue ArgumentError => e
+  puts e.message
+end
+```
+
+## Optimization: selecting languages
+
+By default, all 75 languages are compiled into the native extension (~278 MB). If you only need a subset, set the `LINGUA_LANGUAGES` environment variable before installing to reduce binary size and improve detection speed:
+
+```bash
+LINGUA_LANGUAGES=core bundle install
+```
+
+This compiles only the selected language models (~29 MB for the `core` bundle). You can use individual language names or predefined bundles, and you can mix both in the same build.
+
+This also works with `gem install`, for example:
+
+```bash
+LINGUA_LANGUAGES=french,english,german gem install lingua_rs
+```
+
+Available bundles:
+
+| Bundle | Languages |
+| --- | --- |
+| `core` | `english`, `french`, `german`, `spanish`, `italian`, `portuguese` |
+| `europe_common` | `core` + `dutch`, `polish`, `russian`, `turkish` |
+| `americas` | `english`, `spanish`, `portuguese`, `french` |
+| `mena` | `arabic`, `turkish`, `persian`, `hebrew` |
+| `south_asia` | `hindi`, `urdu`, `bengali`, `tamil`, `telugu`, `punjabi`, `marathi`, `gujarati` |
+| `east_asia` | `chinese`, `japanese`, `korean`, `vietnamese`, `thai` |
+| `africa_common` | `arabic`, `english`, `french`, `swahili`, `somali`, `yoruba`, `zulu` |
+
+Examples:
+
+```bash
+LINGUA_LANGUAGES=core bundle install
+LINGUA_LANGUAGES=europe_common,polish bundle install
+LINGUA_LANGUAGES=east_asia,english bundle install
+```
+
+Language and bundle names must match the Cargo features defined by this gem (lowercase, e.g. `french`, `english`, `german`, `core`, `europe_common`).
+
+Supported languages (from [Lingua Rust's official list](https://github.com/pemistahl/lingua-rs?tab=readme-ov-file#3-which-languages-are-supported)):
+
+`afrikaans`, `albanian`, `arabic`, `armenian`, `azerbaijani`, `basque`, `belarusian`, `bengali`, `bokmal`, `bosnian`, `bulgarian`, `catalan`, `chinese`, `croatian`, `czech`, `danish`, `dutch`, `english`, `esperanto`, `estonian`, `finnish`, `french`, `ganda`, `georgian`, `german`, `greek`, `gujarati`, `hebrew`, `hindi`, `hungarian`, `icelandic`, `indonesian`, `irish`, `italian`, `japanese`, `kazakh`, `korean`, `latin`, `latvian`, `lithuanian`, `macedonian`, `malay`, `maori`, `marathi`, `mongolian`, `nynorsk`, `persian`, `polish`, `portuguese`, `punjabi`, `romanian`, `russian`, `serbian`, `shona`, `slovak`, `slovene`, `somali`, `sotho`, `spanish`, `swahili`, `swedish`, `tagalog`, `tamil`, `telugu`, `thai`, `tsonga`, `tswana`, `turkish`, `ukrainian`, `urdu`, `vietnamese`, `welsh`, `xhosa`, `yoruba`, `zulu`
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake compile` to build the native extension and `rake test` to run the tests.

data/ext/lingua/Cargo.toml

@@ -11,4 +11,187 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.8.2" }
-lingua = "1.8.0"
+lingua = { version = "1.8.0", default-features = false }
+
+[features]
+default = ["full"]
+full = [
+  "europe_common",
+  "americas",
+  "mena",
+  "south_asia",
+  "east_asia",
+  "africa_common",
+  "afrikaans",
+  "albanian",
+  "armenian",
+  "azerbaijani",
+  "basque",
+  "belarusian",
+  "bokmal",
+  "bosnian",
+  "bulgarian",
+  "catalan",
+  "croatian",
+  "czech",
+  "danish",
+  "esperanto",
+  "estonian",
+  "finnish",
+  "ganda",
+  "georgian",
+  "greek",
+  "hungarian",
+  "icelandic",
+  "indonesian",
+  "irish",
+  "kazakh",
+  "latin",
+  "latvian",
+  "lithuanian",
+  "macedonian",
+  "malay",
+  "maori",
+  "mongolian",
+  "nynorsk",
+  "romanian",
+  "serbian",
+  "shona",
+  "slovak",
+  "slovene",
+  "sotho",
+  "swedish",
+  "tagalog",
+  "tsonga",
+  "tswana",
+  "welsh",
+  "xhosa",
+]
+core = [
+  "english",
+  "french",
+  "german",
+  "spanish",
+  "italian",
+  "portuguese",
+]
+europe_common = [
+  "core",
+  "dutch",
+  "polish",
+  "russian",
+  "turkish",
+]
+americas = [
+  "english",
+  "spanish",
+  "portuguese",
+  "french",
+]
+mena = [
+  "arabic",
+  "turkish",
+  "persian",
+  "hebrew",
+]
+south_asia = [
+  "hindi",
+  "urdu",
+  "bengali",
+  "tamil",
+  "telugu",
+  "punjabi",
+  "marathi",
+  "gujarati",
+]
+east_asia = [
+  "chinese",
+  "japanese",
+  "korean",
+  "vietnamese",
+  "thai",
+]
+africa_common = [
+  "arabic",
+  "english",
+  "french",
+  "swahili",
+  "somali",
+  "yoruba",
+  "zulu",
+]
+afrikaans = ["lingua/afrikaans"]
+albanian = ["lingua/albanian"]
+arabic = ["lingua/arabic"]
+armenian = ["lingua/armenian"]
+azerbaijani = ["lingua/azerbaijani"]
+basque = ["lingua/basque"]
+belarusian = ["lingua/belarusian"]
+bengali = ["lingua/bengali"]
+bokmal = ["lingua/bokmal"]
+bosnian = ["lingua/bosnian"]
+bulgarian = ["lingua/bulgarian"]
+catalan = ["lingua/catalan"]
+chinese = ["lingua/chinese"]
+croatian = ["lingua/croatian"]
+czech = ["lingua/czech"]
+danish = ["lingua/danish"]
+dutch = ["lingua/dutch"]
+english = ["lingua/english"]
+esperanto = ["lingua/esperanto"]
+estonian = ["lingua/estonian"]
+finnish = ["lingua/finnish"]
+french = ["lingua/french"]
+ganda = ["lingua/ganda"]
+georgian = ["lingua/georgian"]
+german = ["lingua/german"]
+greek = ["lingua/greek"]
+gujarati = ["lingua/gujarati"]
+hebrew = ["lingua/hebrew"]
+hindi = ["lingua/hindi"]
+hungarian = ["lingua/hungarian"]
+icelandic = ["lingua/icelandic"]
+indonesian = ["lingua/indonesian"]
+irish = ["lingua/irish"]
+italian = ["lingua/italian"]
+japanese = ["lingua/japanese"]
+kazakh = ["lingua/kazakh"]
+korean = ["lingua/korean"]
+latin = ["lingua/latin"]
+latvian = ["lingua/latvian"]
+lithuanian = ["lingua/lithuanian"]
+macedonian = ["lingua/macedonian"]
+malay = ["lingua/malay"]
+maori = ["lingua/maori"]
+marathi = ["lingua/marathi"]
+mongolian = ["lingua/mongolian"]
+nynorsk = ["lingua/nynorsk"]
+persian = ["lingua/persian"]
+polish = ["lingua/polish"]
+portuguese = ["lingua/portuguese"]
+punjabi = ["lingua/punjabi"]
+romanian = ["lingua/romanian"]
+russian = ["lingua/russian"]
+serbian = ["lingua/serbian"]
+shona = ["lingua/shona"]
+slovak = ["lingua/slovak"]
+slovene = ["lingua/slovene"]
+somali = ["lingua/somali"]
+sotho = ["lingua/sotho"]
+spanish = ["lingua/spanish"]
+swahili = ["lingua/swahili"]
+swedish = ["lingua/swedish"]
+tagalog = ["lingua/tagalog"]
+tamil = ["lingua/tamil"]
+telugu = ["lingua/telugu"]
+thai = ["lingua/thai"]
+tsonga = ["lingua/tsonga"]
+tswana = ["lingua/tswana"]
+turkish = ["lingua/turkish"]
+ukrainian = ["lingua/ukrainian"]
+urdu = ["lingua/urdu"]
+vietnamese = ["lingua/vietnamese"]
+welsh = ["lingua/welsh"]
+xhosa = ["lingua/xhosa"]
+yoruba = ["lingua/yoruba"]
+zulu = ["lingua/zulu"]

data/ext/lingua/extconf.rb

@@ -3,4 +3,13 @@
 require 'mkmf'
 require 'rb_sys/mkmf'
 
-create_rust_makefile('lingua/lingua')
+# Allow users to select specific languages to reduce binary size.
+# Example: LINGUA_LANGUAGES=french,english,german bundle install
+languages = ENV.fetch('LINGUA_LANGUAGES', '').split(',').map { |lang| lang.strip.downcase }
+
+create_rust_makefile('lingua/lingua') do |r|
+  unless languages.empty?
+    r.extra_cargo_args << '--no-default-features'
+    r.features = languages
+  end
+end

data/ext/lingua/src/detector.rs

@@ -3,7 +3,7 @@ use magnus::{Error, RArray, RHash, RModule, Ruby, function, method, prelude::*};
 
 use crate::confidence_result::ConfidenceResult;
 use crate::segment::Segment;
-use crate::helpers::{fetch_option, parse_language, value_to_string};
+use crate::helpers::{fetch_option, parse_language, unknown_language_error, value_to_string};
 use crate::language::WrappedLanguage;
 
 pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
@@ -25,7 +25,7 @@ pub fn compute_confidence(
     let language_str = value_to_string(language)?;
     let lang = parse_language(&language_str).ok_or_else(|| {
         Error::new(
-            ruby.exception_arg_error(),
+            unknown_language_error(&ruby),
             format!("unknown language: \"{language_str}\""),
         )
     })?;
@@ -116,12 +116,18 @@ pub fn build_detector_from_options(
         for l in &raw_languages {
             let lang = parse_language(l).ok_or_else(|| {
                 Error::new(
-                    ruby.exception_arg_error(),
+                    unknown_language_error(ruby),
                     format!("unknown language: \"{l}\""),
                 )
             })?;
             languages.push(lang);
         }
+        if languages.is_empty() {
+            return Err(magnus::Error::new(
+                ruby.exception_arg_error(),
+                "languages must contain at least 1 language",
+            ));
+        }
         LanguageDetectorBuilder::from_languages(&languages)
     } else {
         LanguageDetectorBuilder::from_all_languages()
@@ -129,6 +135,12 @@ pub fn build_detector_from_options(
 
     if let Some(opts) = options {
         if let Some(dist) = fetch_option::<f64>(ruby, opts, "minimum_relative_distance") {
+            if !(0.0..=0.99).contains(&dist) {
+                return Err(magnus::Error::new(
+                    ruby.exception_arg_error(),
+                    "minimum_relative_distance must be between 0.0 and 0.99",
+                ));
+            }
             builder.with_minimum_relative_distance(dist);
         }
         if fetch_option::<bool>(ruby, opts, "is_every_language_model_preloaded").unwrap_or(false) {

data/ext/lingua/src/helpers.rs

@@ -1,7 +1,23 @@
 use std::str::FromStr;
+use std::cell::Cell;
 
 use lingua::{IsoCode639_1, IsoCode639_3, Language};
-use magnus::{Error, Ruby, Symbol, prelude::*};
+use magnus::{ExceptionClass, Error, RModule, Ruby, Symbol, prelude::*};
+
+thread_local! {
+    static UNKNOWN_LANGUAGE_ERROR: Cell<Option<ExceptionClass>> = const { Cell::new(None) };
+}
+
+pub fn define_errors(module: &RModule) -> Result<(), Error> {
+    let ruby = Ruby::get().unwrap();
+    let class = module.define_error("UnknownLanguageError", ruby.exception_arg_error())?;
+    UNKNOWN_LANGUAGE_ERROR.set(Some(class));
+    Ok(())
+}
+
+pub fn unknown_language_error(ruby: &Ruby) -> ExceptionClass {
+    UNKNOWN_LANGUAGE_ERROR.get().unwrap_or_else(|| ruby.exception_arg_error())
+}
 
 pub fn parse_language(input: &str) -> Option<Language> {
     Language::from_str(input)

data/ext/lingua/src/language.rs

@@ -10,6 +10,7 @@ pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
     class.define_method("to_s", method!(WrappedLanguage::name, 0))?;
     class.define_method("to_iso6391", method!(WrappedLanguage::to_iso6391, 0))?;
     class.define_method("to_iso", method!(WrappedLanguage::to_iso6391, 0))?;
+    class.define_method("iso_code", method!(WrappedLanguage::to_iso6391, 0))?;
     class.define_method("to_iso6393", method!(WrappedLanguage::to_iso6393, 0))?;
     class.define_method("to_sym", method!(WrappedLanguage::to_sym, 0))?;
     class.define_method("inspect", method!(WrappedLanguage::inspect, 0))?;

data/ext/lingua/src/lib.rs

@@ -45,6 +45,7 @@ fn detect_multiple(ruby: &Ruby, arguments: RArray) -> Result<RArray, Error> {
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = ruby.define_module("Lingua")?;
 
+    helpers::define_errors(&module)?;
     language::define(ruby, &module)?;
     confidence_result::define(ruby, &module)?;
     segment::define(ruby, &module)?;

data/lib/lingua/language.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Lingua
+  # Represents a language returned by Lingua detection and lookup APIs.
+  class Language
+    private
+
+    def respond_to_missing?(method_name, include_private = false)
+      method_name.end_with?('?') || super
+    end
+
+    def method_missing(method_name, *args)
+      if method_name.end_with?('?') && args.empty?
+        match = Lingua::Language[method_name.name.delete_suffix('?')]
+        !match.nil? && self == match
+      else
+        super
+      end
+    end
+  end
+end

data/lib/lingua/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lingua
-  VERSION = '0.3.0'
+  VERSION = '0.4.2'
 end

data/lib/lingua.rb

@@ -2,6 +2,7 @@
 
 require_relative 'lingua/version'
 require_relative 'lingua/lingua'
+require_relative 'lingua/language'
 
 module Lingua
   class Error < StandardError; end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lingua_rs
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Sébastien Vrillaud
@@ -37,17 +37,16 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.3.0
-description: Native Ruby bindings for the Lingua Rust library. Detects languages with
-  confidence scores, ISO 639-1/639-3 support, and configurable accuracy modes.
+description: Native Ruby bindings for the Lingua Rust library, bringing highly accurate
+  language detection for short text and mixed-language text to Ruby, with confidence
+  scores and ISO 639-1/639-3 language codes.
 email:
 - kochka@gmail.com
 executables: []
 extensions:
-- ext/lingua/Cargo.toml
+- ext/lingua/extconf.rb
 extra_rdoc_files: []
 files:
-- Cargo.lock
-- Cargo.toml
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -61,6 +60,7 @@ files:
 - ext/lingua/src/lib.rs
 - ext/lingua/src/segment.rs
 - lib/lingua.rb
+- lib/lingua/language.rb
 - lib/lingua/version.rb
 - sig/lingua.rbs
 homepage: https://github.com/kochka/lingua_rs
@@ -87,5 +87,5 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 rubygems_version: 3.6.9
 specification_version: 4
-summary: Fast language detection for Ruby, powered by Lingua (Rust).
+summary: High-accuracy language detection for Ruby, powered by Lingua (Rust).
 test_files: []