whatlanguage 1.0.4 → 1.0.5

data/History.txt

@@ -1,3 +1,9 @@
+== 1.0.5 / 2013-10-05
+
+* Many more languages supported
+
+== 1.0.4 / 2013-03-07
+
 == 1.0.1 / 2008-08-22
 
 * Public release

data/README.md

@@ -4,7 +4,7 @@ by Peter Cooper
 
 Text language detection. Quick, fast, memory efficient, and all in pure Ruby. Uses Bloom filters for aforementioned speed and memory benefits.
 
-Works with Dutch, English, Farsi, French, German, Italian, Pinyin, Swedish, Portuguese, Russian and Spanish out of the box.
+Works with Dutch, English, Farsi, French, German, Italian, Pinyin, Swedish, Portuguese, Russian, Arabic, Finnish, Greek, Hebrew, Hungarian, Korean, Norwegian, Polish and Spanish out of the box.
 
 ## Important note
   
@@ -25,7 +25,15 @@ Full Example
     texts << %q{Returns the object in enum with the maximum value.}
     texts << %q{Propose des données au sujet de la langue espagnole.}
     texts << %q{La palabra "mezquita" se usa en español para referirse a todo tipo de edificios dedicados.}
-    
+    texts << %q{اللغة التي هي هذه؟}
+    texts << %q{Mitä kieltä tämä on?}
+    texts << %q{Ποια γλώσσα είναι αυτή;}
+    texts << %q{באיזו שפה זה?}
+    texts << %q{Milyen nyelv ez?}
+    texts << %q{이 어떤 언어인가?}
+    texts << %q{Hvilket språk er dette?}
+    texts << %q{W jakim języku to jest?}
+
     texts.each { |text| puts "#{text[0..18]}... is in #{text.language.to_s.capitalize}" }
 
 Initialize WhatLanguage with all filters
@@ -44,6 +52,10 @@ Convenience method on String
 
     "This is a test".language   # => "English"
 
+Initialize WhatLanguage with certain languages
+
+    wl = WhatLanguage.new(:english, :german, :french)
+
 ## Requirements
 
 None, minor libraries (BloominSimple and BitField) included with this release.
@@ -57,6 +69,10 @@ To test, go into irb, then:
     require 'whatlanguage'
     "Je suis un homme".language
 
+## Credits
+
+Contributions from Konrad Reiche and Salimane Adjao Moustapha appreciated.
+
 ## License
 
 MIT License

data/lib/whatlanguage.rb

@@ -9,7 +9,8 @@ class WhatLanguage
   
   @@data = {}
   
-  def initialize(options = {})
+  def initialize(*selection)
+    @selection = (selection.empty?) ? [:all] : selection
     languages_folder = File.join(File.dirname(__FILE__), "..", "lang")
     Dir.entries(languages_folder).grep(/\.lang/).each do |lang|
       @@data[lang[/\w+/].to_sym] ||= BloominSimple.from_dump(File.new(File.join(languages_folder, lang), 'rb').read, &HASHER)
@@ -21,15 +22,22 @@ class WhatLanguage
   def process_text(text)
     results = Hash.new(0)
     it = 0
-    text.split.collect {|a| a.downcase }.each do |word|
+    text.downcase.split.each do |word|
       it += 1
-      @@data.keys.each do |lang|
+
+      if @selection.include?(:all)
+        languages = @@data.keys
+      else
+        languages = @@data.keys & @selection  # intersection
+      end
+
+      languages.each do |lang|
         results[lang] += 1 if @@data[lang].includes?(word)
       end
       
       # Every now and then check to see if we have a really convincing result.. if so, exit early.
       if it % 4 == 0 && results.size > 1
-        top_results = results.sort_by{|a,b| b}.reverse[0..1]
+        top_results = results.sort_by{|a,b| -b}[0..1]
         
         # Next line may need some tweaking one day..
         break if top_results[0][1] > 4 && ((top_results[0][1] > top_results[1][1] * 2) || (top_results[0][1] - top_results[1][1] > 25))
@@ -55,4 +63,4 @@ class String
   def language
     WhatLanguage.new(:all).language(self)
   end
-end
+end

data/lib/whatlanguage/version.rb

@@ -1,3 +1,3 @@
 class WhatLanguage
-  VERSION = '1.0.4'
+  VERSION = '1.0.5'
 end

data/test/test_whatlanguage.rb

@@ -12,25 +12,73 @@ class TestWhatLanguage < Test::Unit::TestCase
     assert_equal :english, "This is a test".language
   end
 
+  def test_arabic
+    assert_equal :arabic, @wl.language("اللغة التي هي هذه؟")
+  end
+
   def test_dutch
     assert_equal :dutch, @wl.language("Als hadden geweest is, is hebben te laat.")
   end
 
+  def test_farsi
+    assert_equal :farsi, @wl.language("وقتی مادرم به من آموخت که به آواز خواندن.")
+  end
+
+  def test_finnish
+    assert_equal :finnish, @wl.language("Mitä kieltä tämä on?")
+  end
+
   def test_french
     assert_equal :french, @wl.language("Bonjour, je m'appelle Sandrine. Voila ma chatte.")
   end
   
-  def test_spanish
-    assert_equal :spanish, @wl.language("La palabra mezquita se usa en español para referirse a todo tipo de edificios dedicados.")
+  def test_german
+    assert_equal :german, @wl.language("Welche Sprache ist das?")
   end
 
-  def test_swedish
-    assert_equal :swedish, @wl.language("Den spanska räven rev en annan räv alldeles lagom.")
+  def test_greek
+    assert_equal :greek, @wl.language("Ποια γλώσσα είναι αυτή;")
+  end
+
+  def test_hebrew
+    assert_equal :hebrew, @wl.language("באיזו שפה זה?")
+  end
+
+  def test_hungarian
+    assert_equal :hungarian, @wl.language("Milyen nyelv ez?")
+  end
+
+  def test_italian
+    assert_equal :italian, @wl.language("Roma, capitale dell'impero romano, è stata per secoli il centro politico e culturale della civiltà occidentale.")
+  end
+
+  def test_korean
+    assert_equal :korean, @wl.language("이 어떤 언어인가?")
+  end
+
+  def test_norwegian
+    assert_equal :norwegian, @wl.language("Hvilket språk er dette?")
+  end
+
+  def test_polish
+    assert_equal :polish, @wl.language("W jakim języku to jest?")
+  end
+
+  def test_portuguese
+    assert_equal :portuguese, @wl.language("Que linguagem é essa?")
   end
 
   def test_russian
     assert_equal :russian, @wl.language("Все новости в хронологическом порядке")
   end
+
+  def test_spanish
+    assert_equal :spanish, @wl.language("La palabra mezquita se usa en español para referirse a todo tipo de edificios dedicados.")
+  end
+
+  def test_swedish
+    assert_equal :swedish, @wl.language("Vilket språk är detta?")
+  end
   
   def test_nothing
     assert_nil @wl.language("")
@@ -48,4 +96,18 @@ class TestWhatLanguage < Test::Unit::TestCase
     assert_equal :italian, @wl.language("Roma, capitale dell'impero romano, è stata per secoli il centro politico e culturale della civiltà occidentale.")
   end
 
-end
+  def test_language_selection
+    selective_wl = WhatLanguage.new(:german, :english)
+    assert_equal :german, selective_wl.language("der die das")
+  end
+
+  def test_language_selection_empty
+    selective_wl = WhatLanguage.new()
+    assert_equal :russian, selective_wl.language("Все новости в хронологическом порядке")
+  end
+
+  def test_language_selection_mixed
+    selective_wl = WhatLanguage.new(:german, :all, :english)
+    assert_equal :russian, selective_wl.language("Все новости в хронологическом порядке")
+  end
+end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: whatlanguage
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
+  prerelease: 
 platform: ruby
 authors:
 - Peter Cooper
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-07 00:00:00.000000000 Z
+date: 2013-10-05 00:00:00.000000000 Z
 dependencies: []
 description: WhatLanguage rapidly detects the language of a sample of text
 email:
@@ -28,13 +29,21 @@ files:
 - build_lang_from_wordlists.rb
 - copyright-en
 - example.rb
+- lang/arabic.lang
 - lang/dutch.lang
 - lang/english.lang
 - lang/farsi.lang
+- lang/finnish.lang
 - lang/french.lang
 - lang/german.lang
+- lang/greek.lang
+- lang/hebrew.lang
+- lang/hungarian.lang
 - lang/italian.lang
+- lang/korean.lang
+- lang/norwegian.lang
 - lang/pinyin.lang
+- lang/polish.lang
 - lang/portuguese.lang
 - lang/russian.lang
 - lang/spanish.lang
@@ -47,26 +56,27 @@ files:
 - whatlanguage.gemspec
 homepage: https://github.com/peterc/whatlanguage
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 1.8.24
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Natural language detection for text samples
 test_files:
 - test/test_whatlanguage.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: ef04e53215ee669aed4e66e5ec8e66bbcc1f7fa0
-  data.tar.gz: 270302e063e5d85ec875de08ac54100c9db5efb8
-SHA512:
-  metadata.gz: a6be7929232a8b48229cae418de4e4bbb928245f9d4ead0ae38b6286fad8d482e06ec82549781187613799596b3657f99fb442d49348329b3e53ae96130029b0
-  data.tar.gz: 98bea8003cacaf0da3774232f1daf112c11c9b3a826700fd2bb5252e3b9837c012f87f54db17d25c640a45a10e60c5891c8e88b8b0ba7835a6e0389bf2695cce