opener-language-identifier 4.1.0 → 4.2.0

data/lib/opener/language_identifier/detector.rb

@@ -1,41 +1,160 @@
-require 'singleton'
-
-import 'org.vicomtech.opennlp.LanguageDetection.CybozuDetector'
-
 module Opener
   class LanguageIdentifier
     ##
-    # Singleton class wrapped around the Cybozu detector. The Cybozu code uses
-    # the factory pattern and stores a bunch of things on class level. As such
-    # the Cybozu code is *not* thread-safe.
+    # Ruby wrapper around the Cybozu DetectorFactory and Detector classes. This
+    # class automatically handles switching of profiles based on input sizes,
+    # assigning priorities to languages, etc.
     #
     class Detector
-      attr_reader :options
+      attr_reader :profiles_path, :short_profiles_path
+
+      ##
+      # Path to the directory containing the default profiles.
+      #
+      # @return [String]
+      #
+      DEFAULT_PROFILES_PATH = File.expand_path(
+        '../../../../core/target/classes/profiles',
+        __FILE__
+      )
+
+      ##
+      # Path to the directory containing the default short profiles.
+      #
+      # @return [String]
+      #
+      DEFAULT_SHORT_PROFILES_PATH = File.expand_path(
+        '../../../../core/target/classes/short_profiles',
+        __FILE__
+      )
+
+      ##
+      # The amount of characters after which the detector should switch to using
+      # the longer profiles set.
+      #
+      # @return [Fixnum]
+      #
+      SHORT_THRESHOLD = 15
+
+      ##
+      # Prioritize OpeNER languages over the rest. Languages not covered by this
+      # list are automatically given a default priority.
+      #
+      # @return [Hash]
+      #
+      PRIORITIES = {
+        'en' => 1.0,
+        'es' => 0.9,
+        'it' => 0.9,
+        'fr' => 0.9,
+        'de' => 0.9,
+        'nl' => 0.9
+      }
 
-      include Singleton
+      ##
+      # The default priority for non OpeNER languages.
+      #
+      # @return [Float]
+      #
+      DEFAULT_PRIORITY = 0.5
 
-      def initialize(options={})
-        @options = options
-        @detector = CybozuDetector.new(profiles_path)
-        @semaphore = Mutex.new
+      ##
+      # @param [Hash] options
+      #
+      # @option options [String] :profiles_path
+      # @option options [String] :short_profiles_path
+      #
+      def initialize(options = {})
+        options.each do |key, value|
+          instance_variable_set("@#{key}", value) if respond_to?(key)
+        end
+
+        @profiles_path       ||= DEFAULT_PROFILES_PATH
+        @short_profiles_path ||= DEFAULT_SHORT_PROFILES_PATH
       end
 
+      ##
+      # @return [String]
+      #
       def detect(input)
-        @semaphore.synchronize do
-          @detector.detect(input)
-        end
+        return new_detector(input).detect
       end
 
+      ##
+      # @return [Array]
+      #
       def probabilities(input)
-        @semaphore.synchronize do
-          result = @detector.detect_langs(input)
+        return new_detector(input).get_probabilities.to_array
+      end
+
+      ##
+      # Returns a new detector with the profiles set based on the input.
+      #
+      # This method analyses a lowercased version of the input as this yields
+      # better results for short text.
+      #
+      # @param [String] input
+      # @return [CybozuDetector]
+      #
+      def new_detector(input)
+        factory = com.cybozu.labs.langdetect.DetectorFactory.new
+
+        factory.load_profile(determine_profiles(input))
+        factory.set_seed(1)
+
+        priorities = build_priorities(input, factory.langlist)
+        detector   = com.cybozu.labs.langdetect.Detector.new(factory)
+
+        detector.set_prior_map(priorities)
+        detector.append(input.downcase)
+
+        return detector
+      end
+
+      ##
+      # Builds a Java Hash mapping the priorities for all OpeNER and non OpeNER
+      # languages.
+      #
+      # If the input size is smaller than the short profiles threshold non
+      # OpeNER languages are _disabled_. This is to ensure that these languages
+      # are detected properly when analysing only 1-2 words.
+      #
+      # @param [String] input
+      # @param [Array<String>] languages
+      # @return [java.util.HashMap]
+      #
+      def build_priorities(input, languages)
+        priorities = java.util.HashMap.new
+        priority   = short_input?(input) ? 0.0 : DEFAULT_PRIORITY
+
+        PRIORITIES.each do |lang, val|
+          priorities.put(lang, val)
         end
+
+        languages.each do |language|
+          unless priorities.contains_key(language)
+            priorities.put(language, priority)
+          end
+        end
+
+        return priorities
+      end
+
+      ##
+      # @param [String] input
+      # @return [String]
+      #
+      def determine_profiles(input)
+        return short_input?(input) ? short_profiles_path : profiles_path
       end
 
-      def profiles_path
-        default_path = File.expand_path("../../../../core/target/classes/profiles", __FILE__)
-        options.fetch(:profiles_path, default_path)
+      ##
+      # @param [String] input
+      # @return [TrueClass|FalseClass]
+      #
+      def short_input?(input)
+        return input.length <= SHORT_THRESHOLD
       end
-    end
-  end
-end
+    end # Detector
+  end # LanguageIdentifier
+end # Opener

data/lib/opener/language_identifier/version.rb

@@ -1,5 +1,5 @@
 module Opener
   class LanguageIdentifier
-    VERSION = "4.1.0"
+    VERSION = '4.2.0'
   end
 end

data/lib/opener/language_identifier.rb

@@ -1,10 +1,8 @@
-require 'java'
 require 'open3'
 require 'slop'
 require 'builder'
 
-require_relative '../../core/target/LanguageDetection-0.0.1.jar'
-import 'org.vicomtech.opennlp.LanguageDetection.CybozuDetector'
+require_relative '../../core/target/LanguageDetection-1.0.0.jar'
 
 require_relative 'language_identifier/version'
 require_relative 'language_identifier/kaf_builder'
@@ -46,8 +44,8 @@ module Opener
     #  are returned instead of the language/KAF.
     #
     def initialize(options = {})
-      @options = DEFAULT_OPTIONS.merge(options)
-      @detector = Detector.instance
+      @options  = DEFAULT_OPTIONS.merge(options)
+      @detector = Detector.new
     end
 
     ##

data/opener-language-identifier.gemspec

@@ -33,7 +33,6 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'slop', '~> 3.5'
 
   gem.add_development_dependency 'rspec', '~> 3.0'
-  gem.add_development_dependency 'cucumber'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'cliver'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-language-identifier
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-11-25 00:00:00.000000000 Z
+date: 2015-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opener-daemons
@@ -94,20 +94,6 @@ dependencies:
         version: '3.0'
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: cucumber
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
   version_requirements: !ruby/object:Gem::Requirement
@@ -145,9 +131,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- core/target/LanguageDetection-1.0.0.jar
 - core/target/LanguageDetection-0.0.1.jar
-- core/target/classes/org/vicomtech/opennlp/LanguageDetection/CybozuDetector.class
-- core/target/classes/org/vicomtech/opennlp/LanguageDetection/Main.class
 - core/target/classes/profiles/fi
 - core/target/classes/profiles/ta
 - core/target/classes/profiles/mr
@@ -193,6 +178,7 @@ files:
 - core/target/classes/profiles/vi
 - core/target/classes/profiles/et
 - core/target/classes/profiles/hi
+- core/target/classes/profiles/af
 - core/target/classes/profiles/gu
 - core/target/classes/profiles/zh-cn
 - core/target/classes/profiles/mk
@@ -202,15 +188,59 @@ files:
 - core/target/classes/profiles/cs
 - core/target/classes/profiles/bn
 - core/target/classes/profiles/tl
+- core/target/classes/short_profiles/fi
+- core/target/classes/short_profiles/ta
+- core/target/classes/short_profiles/ml
+- core/target/classes/short_profiles/hr
+- core/target/classes/short_profiles/id
+- core/target/classes/short_profiles/es
+- core/target/classes/short_profiles/no
+- core/target/classes/short_profiles/ca
+- core/target/classes/short_profiles/en
+- core/target/classes/short_profiles/ru
+- core/target/classes/short_profiles/te
+- core/target/classes/short_profiles/lt
+- core/target/classes/short_profiles/pa
+- core/target/classes/short_profiles/ja
+- core/target/classes/short_profiles/he
+- core/target/classes/short_profiles/nl
+- core/target/classes/short_profiles/tr
+- core/target/classes/short_profiles/pl
+- core/target/classes/short_profiles/si
+- core/target/classes/short_profiles/fa
+- core/target/classes/short_profiles/de
+- core/target/classes/short_profiles/bg
+- core/target/classes/short_profiles/it
+- core/target/classes/short_profiles/fr
+- core/target/classes/short_profiles/el
+- core/target/classes/short_profiles/pt
+- core/target/classes/short_profiles/uk
+- core/target/classes/short_profiles/da
+- core/target/classes/short_profiles/ar
+- core/target/classes/short_profiles/zh-tw
+- core/target/classes/short_profiles/sq
+- core/target/classes/short_profiles/th
+- core/target/classes/short_profiles/ko
+- core/target/classes/short_profiles/ro
+- core/target/classes/short_profiles/lv
+- core/target/classes/short_profiles/sv
+- core/target/classes/short_profiles/vi
+- core/target/classes/short_profiles/et
+- core/target/classes/short_profiles/hi
+- core/target/classes/short_profiles/gu
+- core/target/classes/short_profiles/zh-cn
+- core/target/classes/short_profiles/mk
+- core/target/classes/short_profiles/ur
+- core/target/classes/short_profiles/hu
+- core/target/classes/short_profiles/cs
+- core/target/classes/short_profiles/bn
+- core/target/classes/short_profiles/tl
 - core/target/classes/com/cybozu/labs/langdetect/ErrorCode.class
-- core/target/classes/com/cybozu/labs/langdetect/GenProfile.class
-- core/target/classes/com/cybozu/labs/langdetect/Command.class
 - core/target/classes/com/cybozu/labs/langdetect/LangDetectException.class
 - core/target/classes/com/cybozu/labs/langdetect/Language.class
 - core/target/classes/com/cybozu/labs/langdetect/Detector.class
 - core/target/classes/com/cybozu/labs/langdetect/af
 - core/target/classes/com/cybozu/labs/langdetect/DetectorFactory.class
-- core/target/classes/com/cybozu/labs/langdetect/util/TagExtractor.class
 - core/target/classes/com/cybozu/labs/langdetect/util/NGram.class
 - core/target/classes/com/cybozu/labs/langdetect/util/LangProfile.class
 - core/target/classes/com/cybozu/labs/langdetect/util/Messages.class