lingua_rs 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e301297292f2de86fd62416f6f06765c7ccd373c81020e2ad365e0c0151a55bf
-  data.tar.gz: fe961910a3e44fa7df10580100b4aecef6f2552c08a7faba278a84eaf7ae89fc
+  metadata.gz: bd0dba860ee235a6acfc46afb3ba851ec08303df9499f403e47624c74b9396e7
+  data.tar.gz: b3d9faca1f0f12fb7abee0643b94f1f494ef02e6b4f2c92846fa88e94d503904
 SHA512:
-  metadata.gz: 31e45f89765b5feb782a8b47f9574891c9ce826b5883f840b5f0d06c8715f26c9d9caa999a8e08349f475e307b2ccd28ecb303677ae8e51caa79920d94cc91fc
-  data.tar.gz: 6ace3072e3b7e8e044c55712879ea398dc7f009475cb058444070cd22834bb7f4fa6525ab4a66bab95417d57b1f57b9599cda02dd159be80603280c0646b6414
+  metadata.gz: 2cae21715d81d650c71df5d55b18fb7b09eb86bcfe1356e5ace3b526a48e9ff31be39d852ee43abe342dffed711c7ea08ad3a9342702960b04a864a674df6068
+  data.tar.gz: 221eb0c62f4b7158d5c95b4d7c28490a2bcabc3f4cffe64914bea62c413c0d35fc8ceab1a6b6dc81a4750e75f8d7deac522bab1d0aa97b5c02cfd7d49e9eab40

data/README.md

@@ -42,19 +42,18 @@ export PATH=/usr/lib/llvm21/bin:$PATH
 require 'lingua'
 
 lang = Lingua.detect('Bonjour le monde')
+lang.french?    # => true
 lang.to_s       # => 'French'
 lang.to_sym     # => :french
 lang.to_iso     # => 'fr'
-lang.to_iso6391 # => 'fr'
 lang.to_iso6393 # => 'fra'
-lang.inspect    # => '#<Lingua::Language French>'
 
 Lingua.detect('') # => nil
 ```
 
 ### Restricting languages
 
-Pass a `languages` option to limit detection to a subset of languages. Values can be full names, ISO 639-1 codes, or ISO 639-3 codes, and can be mixed freely.
+Pass a `languages` option to limit detection to a subset of languages. Values can be full names, ISO 639-1 codes, or ISO 639-3 codes, and can be mixed freely. Restricting the candidate set is also faster, since the detector has fewer languages to compare against.
 
 ```ruby
 # By full name
@@ -154,9 +153,11 @@ Lingua::Language['xxx']     # => nil
 | `name` | `String` | `'French'` |
 | `to_s` | `String` | `'French'` (alias for `name`) |
 | `to_sym` | `Symbol` | `:french` |
+| `iso_code` | `String` | `'fr'` (alias for `to_iso6391`) |
 | `to_iso` | `String` | `'fr'` (alias for `to_iso6391`) |
 | `to_iso6391` | `String` | `'fr'` |
 | `to_iso6393` | `String` | `'fra'` |
+| `french?` | `Boolean` | `true` (works with name, ISO 639-1 or ISO 639-3: `fr?`, `fra?`) |
 | `inspect` | `String` | `'#<Lingua::Language French>'` |
 | `==` | `Boolean` | Compare two languages |
 | `hash` | `Integer` | Hash value (usable as Hash key) |
@@ -194,6 +195,60 @@ Returned by `detect_multiple`.
 | `to_s` | `String` | `'French (0-22): Parlez-vous français? '` |
 | `inspect` | `String` | `'#<Lingua::Segment French (0-22) "Parlez-vous français? ">'` |
 
+### Error handling
+
+`Lingua::UnknownLanguageError` (subclass of `ArgumentError`) is raised when an unrecognized language name or code is passed:
+
+```ruby
+Lingua.detect('Hello', languages: %w[en zzzz])
+# => Lingua::UnknownLanguageError: unknown language: "zzzz"
+
+# Can also be rescued as ArgumentError
+begin
+  Lingua.detect('Hello', languages: %w[zzzz])
+rescue ArgumentError => e
+  puts e.message
+end
+```
+
+## Optimization: selecting languages
+
+By default, all 75 languages are compiled into the native extension (~278 MB). If you only need a subset, set the `LINGUA_LANGUAGES` environment variable before installing to reduce binary size and improve detection speed:
+
+```bash
+LINGUA_LANGUAGES=core bundle install
+```
+
+This compiles only the selected language models (~29 MB for the `core` bundle). You can use individual language names or predefined bundles, and you can mix both in the same build.
+
+This also works with `gem install`, for example:
+
+```bash
+LINGUA_LANGUAGES=french,english,german gem install lingua_rs
+```
+
+Available bundles:
+
+| Bundle | Languages |
+| --- | --- |
+| `core` | `english`, `french`, `german`, `spanish`, `italian`, `portuguese` |
+| `europe_common` | `core` + `dutch`, `polish`, `russian`, `turkish` |
+| `americas` | `english`, `spanish`, `portuguese`, `french` |
+| `mena` | `arabic`, `turkish`, `persian`, `hebrew` |
+| `south_asia` | `hindi`, `urdu`, `bengali`, `tamil`, `telugu`, `punjabi`, `marathi`, `gujarati` |
+| `east_asia` | `chinese`, `japanese`, `korean`, `vietnamese`, `thai` |
+| `africa_common` | `arabic`, `english`, `french`, `swahili`, `somali`, `yoruba`, `zulu` |
+
+Examples:
+
+```bash
+LINGUA_LANGUAGES=core bundle install
+LINGUA_LANGUAGES=europe_common,polish bundle install
+LINGUA_LANGUAGES=east_asia,english bundle install
+```
+
+Language and bundle names must match the Cargo features defined by this gem (lowercase, e.g. `french`, `english`, `german`, `core`, `europe_common`).
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake compile` to build the native extension and `rake test` to run the tests.

data/ext/lingua/Cargo.toml

@@ -11,4 +11,187 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.8.2" }
-lingua = "1.8.0"
+lingua = { version = "1.8.0", default-features = false }
+
+[features]
+default = ["full"]
+full = [
+  "europe_common",
+  "americas",
+  "mena",
+  "south_asia",
+  "east_asia",
+  "africa_common",
+  "afrikaans",
+  "albanian",
+  "armenian",
+  "azerbaijani",
+  "basque",
+  "belarusian",
+  "bokmal",
+  "bosnian",
+  "bulgarian",
+  "catalan",
+  "croatian",
+  "czech",
+  "danish",
+  "esperanto",
+  "estonian",
+  "finnish",
+  "ganda",
+  "georgian",
+  "greek",
+  "hungarian",
+  "icelandic",
+  "indonesian",
+  "irish",
+  "kazakh",
+  "latin",
+  "latvian",
+  "lithuanian",
+  "macedonian",
+  "malay",
+  "maori",
+  "mongolian",
+  "nynorsk",
+  "romanian",
+  "serbian",
+  "shona",
+  "slovak",
+  "slovene",
+  "sotho",
+  "swedish",
+  "tagalog",
+  "tsonga",
+  "tswana",
+  "welsh",
+  "xhosa",
+]
+core = [
+  "english",
+  "french",
+  "german",
+  "spanish",
+  "italian",
+  "portuguese",
+]
+europe_common = [
+  "core",
+  "dutch",
+  "polish",
+  "russian",
+  "turkish",
+]
+americas = [
+  "english",
+  "spanish",
+  "portuguese",
+  "french",
+]
+mena = [
+  "arabic",
+  "turkish",
+  "persian",
+  "hebrew",
+]
+south_asia = [
+  "hindi",
+  "urdu",
+  "bengali",
+  "tamil",
+  "telugu",
+  "punjabi",
+  "marathi",
+  "gujarati",
+]
+east_asia = [
+  "chinese",
+  "japanese",
+  "korean",
+  "vietnamese",
+  "thai",
+]
+africa_common = [
+  "arabic",
+  "english",
+  "french",
+  "swahili",
+  "somali",
+  "yoruba",
+  "zulu",
+]
+afrikaans = ["lingua/afrikaans"]
+albanian = ["lingua/albanian"]
+arabic = ["lingua/arabic"]
+armenian = ["lingua/armenian"]
+azerbaijani = ["lingua/azerbaijani"]
+basque = ["lingua/basque"]
+belarusian = ["lingua/belarusian"]
+bengali = ["lingua/bengali"]
+bokmal = ["lingua/bokmal"]
+bosnian = ["lingua/bosnian"]
+bulgarian = ["lingua/bulgarian"]
+catalan = ["lingua/catalan"]
+chinese = ["lingua/chinese"]
+croatian = ["lingua/croatian"]
+czech = ["lingua/czech"]
+danish = ["lingua/danish"]
+dutch = ["lingua/dutch"]
+english = ["lingua/english"]
+esperanto = ["lingua/esperanto"]
+estonian = ["lingua/estonian"]
+finnish = ["lingua/finnish"]
+french = ["lingua/french"]
+ganda = ["lingua/ganda"]
+georgian = ["lingua/georgian"]
+german = ["lingua/german"]
+greek = ["lingua/greek"]
+gujarati = ["lingua/gujarati"]
+hebrew = ["lingua/hebrew"]
+hindi = ["lingua/hindi"]
+hungarian = ["lingua/hungarian"]
+icelandic = ["lingua/icelandic"]
+indonesian = ["lingua/indonesian"]
+irish = ["lingua/irish"]
+italian = ["lingua/italian"]
+japanese = ["lingua/japanese"]
+kazakh = ["lingua/kazakh"]
+korean = ["lingua/korean"]
+latin = ["lingua/latin"]
+latvian = ["lingua/latvian"]
+lithuanian = ["lingua/lithuanian"]
+macedonian = ["lingua/macedonian"]
+malay = ["lingua/malay"]
+maori = ["lingua/maori"]
+marathi = ["lingua/marathi"]
+mongolian = ["lingua/mongolian"]
+nynorsk = ["lingua/nynorsk"]
+persian = ["lingua/persian"]
+polish = ["lingua/polish"]
+portuguese = ["lingua/portuguese"]
+punjabi = ["lingua/punjabi"]
+romanian = ["lingua/romanian"]
+russian = ["lingua/russian"]
+serbian = ["lingua/serbian"]
+shona = ["lingua/shona"]
+slovak = ["lingua/slovak"]
+slovene = ["lingua/slovene"]
+somali = ["lingua/somali"]
+sotho = ["lingua/sotho"]
+spanish = ["lingua/spanish"]
+swahili = ["lingua/swahili"]
+swedish = ["lingua/swedish"]
+tagalog = ["lingua/tagalog"]
+tamil = ["lingua/tamil"]
+telugu = ["lingua/telugu"]
+thai = ["lingua/thai"]
+tsonga = ["lingua/tsonga"]
+tswana = ["lingua/tswana"]
+turkish = ["lingua/turkish"]
+ukrainian = ["lingua/ukrainian"]
+urdu = ["lingua/urdu"]
+vietnamese = ["lingua/vietnamese"]
+welsh = ["lingua/welsh"]
+xhosa = ["lingua/xhosa"]
+yoruba = ["lingua/yoruba"]
+zulu = ["lingua/zulu"]

data/ext/lingua/extconf.rb

@@ -3,4 +3,13 @@
 require 'mkmf'
 require 'rb_sys/mkmf'
 
-create_rust_makefile('lingua/lingua')
+# Allow users to select specific languages to reduce binary size.
+# Example: LINGUA_LANGUAGES=french,english,german bundle install
+languages = ENV.fetch('LINGUA_LANGUAGES', '').split(',').map { |lang| lang.strip.downcase }
+
+create_rust_makefile('lingua/lingua') do |r|
+  unless languages.empty?
+    r.extra_cargo_args << '--no-default-features'
+    r.features = languages
+  end
+end

data/ext/lingua/src/detector.rs

@@ -3,7 +3,7 @@ use magnus::{Error, RArray, RHash, RModule, Ruby, function, method, prelude::*};
 
 use crate::confidence_result::ConfidenceResult;
 use crate::segment::Segment;
-use crate::helpers::{fetch_option, parse_language, value_to_string};
+use crate::helpers::{fetch_option, parse_language, unknown_language_error, value_to_string};
 use crate::language::WrappedLanguage;
 
 pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
@@ -25,7 +25,7 @@ pub fn compute_confidence(
     let language_str = value_to_string(language)?;
     let lang = parse_language(&language_str).ok_or_else(|| {
         Error::new(
-            ruby.exception_arg_error(),
+            unknown_language_error(&ruby),
             format!("unknown language: \"{language_str}\""),
         )
     })?;
@@ -116,7 +116,7 @@ pub fn build_detector_from_options(
         for l in &raw_languages {
             let lang = parse_language(l).ok_or_else(|| {
                 Error::new(
-                    ruby.exception_arg_error(),
+                    unknown_language_error(ruby),
                     format!("unknown language: \"{l}\""),
                 )
             })?;

data/ext/lingua/src/helpers.rs

@@ -1,7 +1,23 @@
 use std::str::FromStr;
+use std::cell::Cell;
 
 use lingua::{IsoCode639_1, IsoCode639_3, Language};
-use magnus::{Error, Ruby, Symbol, prelude::*};
+use magnus::{ExceptionClass, Error, RModule, Ruby, Symbol, prelude::*};
+
+thread_local! {
+    static UNKNOWN_LANGUAGE_ERROR: Cell<Option<ExceptionClass>> = const { Cell::new(None) };
+}
+
+pub fn define_errors(module: &RModule) -> Result<(), Error> {
+    let ruby = Ruby::get().unwrap();
+    let class = module.define_error("UnknownLanguageError", ruby.exception_arg_error())?;
+    UNKNOWN_LANGUAGE_ERROR.set(Some(class));
+    Ok(())
+}
+
+pub fn unknown_language_error(ruby: &Ruby) -> ExceptionClass {
+    UNKNOWN_LANGUAGE_ERROR.get().unwrap_or_else(|| ruby.exception_arg_error())
+}
 
 pub fn parse_language(input: &str) -> Option<Language> {
     Language::from_str(input)

data/ext/lingua/src/language.rs

@@ -10,6 +10,7 @@ pub fn define(ruby: &Ruby, module: &RModule) -> Result<(), Error> {
     class.define_method("to_s", method!(WrappedLanguage::name, 0))?;
     class.define_method("to_iso6391", method!(WrappedLanguage::to_iso6391, 0))?;
     class.define_method("to_iso", method!(WrappedLanguage::to_iso6391, 0))?;
+    class.define_method("iso_code", method!(WrappedLanguage::to_iso6391, 0))?;
     class.define_method("to_iso6393", method!(WrappedLanguage::to_iso6393, 0))?;
     class.define_method("to_sym", method!(WrappedLanguage::to_sym, 0))?;
     class.define_method("inspect", method!(WrappedLanguage::inspect, 0))?;

data/ext/lingua/src/lib.rs

@@ -45,6 +45,7 @@ fn detect_multiple(ruby: &Ruby, arguments: RArray) -> Result<RArray, Error> {
 fn init(ruby: &Ruby) -> Result<(), Error> {
     let module = ruby.define_module("Lingua")?;
 
+    helpers::define_errors(&module)?;
     language::define(ruby, &module)?;
     confidence_result::define(ruby, &module)?;
     segment::define(ruby, &module)?;

data/lib/lingua/language.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Lingua
+  # Represents a language returned by Lingua detection and lookup APIs.
+  class Language
+    private
+
+    def respond_to_missing?(method_name, include_private = false)
+      method_name.end_with?('?') || super
+    end
+
+    def method_missing(method_name, *args)
+      if method_name.end_with?('?') && args.empty?
+        match = Lingua::Language[method_name.name.delete_suffix('?')]
+        !match.nil? && self == match
+      else
+        super
+      end
+    end
+  end
+end

data/lib/lingua/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lingua
-  VERSION = '0.3.0'
+  VERSION = '0.4.1'
 end

data/lib/lingua.rb

@@ -2,6 +2,7 @@
 
 require_relative 'lingua/version'
 require_relative 'lingua/lingua'
+require_relative 'lingua/language'
 
 module Lingua
   class Error < StandardError; end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lingua_rs
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Sébastien Vrillaud
@@ -43,11 +43,9 @@ email:
 - kochka@gmail.com
 executables: []
 extensions:
-- ext/lingua/Cargo.toml
+- ext/lingua/extconf.rb
 extra_rdoc_files: []
 files:
-- Cargo.lock
-- Cargo.toml
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -61,6 +59,7 @@ files:
 - ext/lingua/src/lib.rs
 - ext/lingua/src/segment.rs
 - lib/lingua.rb
+- lib/lingua/language.rb
 - lib/lingua/version.rb
 - sig/lingua.rbs
 homepage: https://github.com/kochka/lingua_rs