lingua_rs 0.5.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9ec0e0ba5844b9b3e47cd60babbc90ac03791bc83e3327e78b7f2d83ba4a71d8
-  data.tar.gz: 55af5915e63a1b34cbc5c73d2e2c63b6a1ce2030183112c54aad888b299bc889
+  metadata.gz: 94314611395fba47d42021ab91fc1f0db7b7f5311a64796a8c98b36372c5b2b0
+  data.tar.gz: c627fa1a1510945411506662fb22907546f4a5916352680aa09e86f5f93f9d33
 SHA512:
-  metadata.gz: b6bfc5d1605bf171e7be4dd899e7b82460f4c8b81ab541ff97a18fb59205a752e9cfe12de640c20b6a60a4814bc8e89e884b9d501a2ec3b5a4e4f0b4bc2e094d
-  data.tar.gz: ecf2866f980b9ddd7189d639ea1330a4f47f8f0a528dcb636358c7c6de30add2d8a854cee8252eac36540fc528b1836eedebbd03313eeb1de167cd18c241afbf
+  metadata.gz: 202e31c4f0775dfd7f9976720efcde6c0d7af4a3169771d3690f01f5b960a55406e697a06ecf6c48a682ad39f264ccfec2b2dc14bdc7d5a3fc0fcfd5d966c06a
+  data.tar.gz: 618a2269f52c7dd1a0999f93f13e7a58b02aa6e2a584a161e89c914166b7299dd03ef3b2401599997ef91d58ef7c2586daca2968bf125efc0edbf4bd62efb51c

data/README.md

@@ -25,7 +25,7 @@ Then run `bundle install`.
 
 ### Build requirements
 
-A Rust toolchain is required to compile the native extension.
+A Rust toolchain (**1.87+**) is required to compile the native extension.
 
 ## Usage
 
@@ -157,9 +157,18 @@ detector.detect_multiple_batch(['Bonjour le monde. Hello world.', 'Ich bin müde
 | Method | Return type | Description |
 |---|---|---|
 | `detect_batch(texts)` | `Array<Language\|nil>` | Detect language of each text |
-| `detect_multiple_batch(texts)` | `Array<Array<Segment>>` | Mixed-language detection for each text |
-| `confidence_values_batch(texts)` | `Array<Array<ConfidenceResult>>` | Confidence values for each text |
 | `confidence_batch(texts, language)` | `Array<Float>` | Confidence for a specific language across texts |
+| `confidence_values_batch(texts)` | `Array<Array<ConfidenceResult>>` | Confidence values for each text |
+| `detect_multiple_batch(texts)` | `Array<Array<Segment>>` | Mixed-language detection for each text |
+
+Results are returned in the same order as the input, so you can pair them with `zip`:
+
+```ruby
+texts = ['Bonjour le monde', 'Hello world', 'Hallo Welt']
+texts.zip(detector.detect_batch(texts)) do |text, language|
+  puts "#{text} => #{language}"
+end
+```
 
 ### Error handling
 
@@ -231,6 +240,49 @@ Returned by `detect_multiple`.
 | `to_s` | `String` | `'French (0-22): Parlez-vous français? '` |
 | `inspect` | `String` | `'#<Lingua::Segment French (0-22) "Parlez-vous français? ">'` |
 
+## Command-line tool
+
+The gem ships with a `lingua` executable that wraps the library for shell use.
+
+```bash
+# Text as argument or on stdin
+lingua 'Bonjour le monde'                     # => French
+echo 'Hello world' | lingua                   # => English
+
+# Output format
+lingua -f iso 'Bonjour le monde'              # => fr
+lingua -f iso6393 'Bonjour le monde'          # => fra
+lingua -f json 'Bonjour le monde'             # => {"name":"French","iso":"fr","iso6393":"fra"}
+
+# Confidence values
+lingua -c -l en,fr,de 'Bonjour le monde'
+# French: 0.8217
+# German: 0.1216
+# English: 0.0567
+
+# Mixed-language detection
+lingua -m -l en,fr,de 'Parlez-vous français? Hello world.'
+# French (0-22): Parlez-vous français?
+# English (22-34): Hello world.
+
+# Restrict candidates
+lingua -l en,fr,de 'Bonjour'
+
+# Batch: one text per line (use "-" for stdin)
+lingua --batch texts.txt
+cat texts.txt | lingua --batch -
+```
+
+| Option | Description |
+|---|---|
+| `-f`, `--format FORMAT` | Output format: `name` (default), `iso`, `iso6393`, `json` |
+| `-c`, `--confidences` | Output confidence for all candidate languages |
+| `-m`, `--multiple` | Detect multiple languages within the text |
+| `-l`, `--languages LIST` | Comma-separated list of languages (names or ISO codes) |
+| `--batch FILE` | Treat each line of FILE (or `-` for stdin) as a separate input |
+| `-v`, `--version` | Show version |
+| `-h`, `--help` | Show help |
+
 ## Optimization: selecting languages
 
 By default, all 75 languages are compiled into the native extension (~278 MB). If you only need a subset, select languages at build time to reduce binary size and improve detection speed.

data/exe/lingua

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+require 'json'
+require 'lingua'
+
+def build_parser(options)
+  OptionParser.new do |opts|
+    opts.banner = 'Usage: lingua [options] [TEXT]'
+    opts.separator ''
+    opts.separator 'If TEXT is omitted, input is read from stdin.'
+    opts.separator ''
+    opts.separator 'Options:'
+
+    opts.on('-f', '--format FORMAT', %w[iso iso6393 name json],
+            'Output format: name (default), iso, iso6393, json') { |f| options[:format] = f.to_sym }
+    opts.on('-c', '--confidences',
+            'Output confidence values for all candidate languages') { options[:mode] = :confidences }
+    opts.on('-m', '--multiple',
+            'Detect multiple languages within the text') { options[:mode] = :multiple }
+    opts.on('-l', '--languages LIST',
+            'Restrict detection to comma-separated languages (names or ISO codes)') do |list|
+      options[:languages] = list.split(',').map(&:strip)
+    end
+    opts.on('--batch FILE',
+            'Treat each line of FILE (or "-" for stdin) as a separate input') { |f| options[:batch] = f }
+    opts.on('-h', '--help', 'Show this help') do
+      puts opts
+      exit
+    end
+    opts.on('-v', '--version', 'Show version') do
+      puts Lingua::VERSION
+      exit
+    end
+  end
+end
+
+options = { format: :name, mode: :detect, languages: nil, batch: nil }
+build_parser(options).parse!
+
+def format_language(lang, format)
+  return nil if lang.nil?
+
+  case format
+  when :iso     then lang.to_iso
+  when :iso6393 then lang.to_iso6393
+  when :name    then lang.name
+  when :json    then { name: lang.name, iso: lang.to_iso, iso6393: lang.to_iso6393 }
+  end
+end
+
+def render_detect(lang, format)
+  formatted = format_language(lang, format)
+  if format == :json
+    JSON.generate(formatted)
+  else
+    formatted.to_s
+  end
+end
+
+def render_confidences(results, format)
+  if format == :json
+    JSON.generate(results.map do |r|
+      { language: format_language(r.language, :iso), confidence: r.confidence }
+    end)
+  else
+    results.map { |r| "#{format_language(r.language, format)}: #{format('%.4f', r.confidence)}" }
+           .join("\n")
+  end
+end
+
+def render_multiple(segments, format)
+  if format == :json
+    JSON.generate(segments.map do |s|
+      { language: format_language(s.language, :iso),
+        start_index: s.start_index,
+        end_index: s.end_index,
+        text: s.text }
+    end)
+  else
+    segments.map do |s|
+      "#{format_language(s.language, format)} (#{s.start_index}-#{s.end_index}): #{s.text}"
+    end.join("\n")
+  end
+end
+
+def process(text, detector, options)
+  case options[:mode]
+  when :detect
+    render_detect(detector.detect(text), options[:format])
+  when :confidences
+    render_confidences(detector.confidence_values(text), options[:format])
+  when :multiple
+    render_multiple(detector.detect_multiple(text), options[:format])
+  end
+end
+
+begin
+  detector_options = {}
+  detector_options[:languages] = options[:languages] if options[:languages]
+  detector = Lingua::Detector.new(**detector_options)
+
+  if options[:batch]
+    io = options[:batch] == '-' ? $stdin : File.open(options[:batch], 'r')
+    io.each_line do |line|
+      line = line.chomp
+      next if line.empty?
+
+      puts process(line, detector, options)
+    end
+    io.close unless io == $stdin
+  else
+    text = ARGV.empty? ? $stdin.read : ARGV.join(' ')
+    text = text.strip
+    if text.empty?
+      warn 'lingua: no input text'
+      exit 1
+    end
+    puts process(text, detector, options)
+  end
+rescue Lingua::UnknownLanguageError => e
+  warn "lingua: #{e.message}"
+  exit 1
+end

data/ext/lingua/extconf.rb

@@ -11,6 +11,12 @@ selection = with_config('lingua-languages', nil) || ENV.fetch('LINGUA_LANGUAGES'
 languages = selection.to_s.split(',').map { |lang| lang.strip.downcase }.reject(&:empty?)
 
 create_rust_makefile('lingua/lingua') do |r|
+  # Cargo strips debuginfo from release builds by default. With rustc <= 1.98.0,
+  # stripping leaves the LINKEDIT string pool misaligned and dyld on macOS 27+
+  # refuses to load the bundle ("mis-aligned LINKEDIT string pool"), see
+  # rust-lang/rust#157750. Can be removed once rustc >= 1.98.1 is the minimum.
+  r.extra_rustflags << '-Cstrip=none' if RUBY_PLATFORM.include?('darwin')
+
   unless languages.empty?
     r.extra_cargo_args << '--no-default-features'
     r.features = languages

data/lib/lingua/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lingua
-  VERSION = '0.5.0'
+  VERSION = '0.6.1'
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: lingua_rs
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Sébastien Vrillaud
 bindir: exe
 cert_chain: []
-date: 2026-04-08 00:00:00.000000000 Z
+date: 2026-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -42,7 +42,8 @@ description: Native Ruby bindings for the Lingua Rust library, bringing highly a
   scores and ISO 639-1/639-3 language codes.
 email:
 - kochka@gmail.com
-executables: []
+executables:
+- lingua
 extensions:
 - ext/lingua/extconf.rb
 extra_rdoc_files: []
@@ -50,6 +51,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/lingua
 - ext/lingua/Cargo.toml
 - ext/lingua/extconf.rb
 - ext/lingua/src/confidence_result.rs
@@ -84,7 +86,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: High-accuracy language detection powered by Lingua (Rust).
 test_files: []