translatomatic 0.1.3 → 0.2.0

data/lib/translatomatic/provider.rb

@@ -0,0 +1,101 @@
+module Translatomatic
+  # Provides methods to access and create instances of
+  # interfaces to translation APIs.
+  module Provider
+    class << self
+      include Translatomatic::Util
+
+      # @return [Class] The provider class corresponding to the given name
+      def find(name)
+        load_providers
+        name && !name.empty? ? const_get(name) : nil
+      end
+
+      # Resolve the given list of provider names to a list of providers.
+      # If the list is empty, return all providers that are configured.
+      # @param list [Array<String>] Provider names or providers
+      # @param options [Hash<String,String>] Provider options
+      # @return [Array<Translatomatic::Provider::Base>] Providers
+      def resolve(list, options = {})
+        list = [list].flatten.compact.collect do |provider|
+          if provider.respond_to?(:translate)
+            provider
+          else
+            klass = find(provider)
+            provider = create_provider(klass, options)
+          end
+          provider
+        end
+
+        # if we didn't resolve to any providers, find all available
+        # providers that work with the given options.
+        list.empty? ? available(options) : list
+      end
+
+      # @return [List<Class>] A list of all provider classes
+      def types
+        load_providers
+        constants.collect { |c| const_get(c) }.select do |klass|
+          klass.is_a?(Class) && klass != Translatomatic::Provider::Base
+        end
+      end
+
+      # @return [List<String>] A list of all providers
+      def names
+        types.collect { |i| i.name.demodulize }
+      end
+
+      # Find all configured providers
+      # @param options [Hash<String,String>] Provider options
+      # @return [Array<#translate>] A list of provider instances
+      def available(options = {})
+        available = types.collect { |klass| create_provider(klass, options) }
+        available.compact
+      end
+
+      # Get errors for the specified provider
+      # @param name [String] Provider name
+      def get_error(name)
+        @provider_errors[name]
+      end
+
+      private
+
+      def test_provider(provider)
+        provider.languages
+      end
+
+      def create_provider(klass, options = {})
+        return nil unless klass
+        klass.new(options)
+      rescue StandardError => e
+        name = klass.name.demodulize
+        log.debug(t('provider.unavailable', name: name))
+        provider_error(name, e)
+        nil
+      end
+
+      def loaded_providers?
+        @loaded_providers
+      end
+
+      def provider_error(name, e)
+        @provider_errors ||= {}
+        @provider_errors[name] = e
+      end
+
+      def load_providers
+        return if loaded_providers?
+        Dir[File.join(__dir__, 'provider/*.rb')].sort.each do |file|
+          begin
+            require file
+          rescue StandardError => e
+            name = File.basename(file, '.rb').classify
+            provider_error(name, e)
+          end
+        end
+        @loaded_providers = true
+      end
+    end
+  end
+end

data/lib/translatomatic/provider/base.rb

@@ -0,0 +1,136 @@
+module Translatomatic
+  module Provider
+    # Base class for interfaces to translation APIs
+    # @abstract
+    class Base
+      include Translatomatic::DefineOptions
+
+      # Listener for translation events
+      attr_accessor :listener
+
+      # @return [boolean] True if a string can have alternative translations
+      def self.supports_alternative_translations?
+        false
+      end
+
+      # @return [boolean] true if this provider supports html5
+      #   <span translate="no"></span> tags.
+      def self.supports_no_translate_html?
+        false
+      end
+
+      def initialize(options = {})
+        @listener = options[:listener]
+      end
+
+      # @return [String] The name of this provider.
+      def name
+        self.class.name.demodulize
+      end
+
+      # @return [String] The name of this provider
+      def to_s
+        name
+      end
+
+      # @return [Array<String>] A list of languages
+      #   supported by this provider.
+      def languages
+        []
+      end
+
+      # Translate strings from one locale to another
+      # @param strings [Array<String,Text>] A list of text/strings to translate.
+      # @param from [String, Translatomatic::Locale] The locale of the
+      #   given strings.
+      # @param to [String, Translatomatic::Locale] The locale to translate to.
+      # @return [Array<Translatomatic::Translation::Result>] Translations
+      def translate(strings, from, to)
+        @updated_listener = false
+        @translations = []
+        @from = from
+        @to = to
+        strings = [strings] unless strings.is_a?(Array)
+        from = build_locale(from)
+        to = build_locale(to)
+        if from.language == to.language
+          strings.each { |i| add_translations(i, i) }
+        else
+          perform_translate(strings, from, to)
+        end
+        @translations
+      end
+
+      private
+
+      include Translatomatic::Util
+
+      TRANSLATION_RETRIES = 3
+
+      # all subclasses must implement this
+      def perform_translate(_strings, _from, _to)
+        raise 'subclass must implement perform_translate'
+      end
+
+      # subclasses that call perform_fetch_translations must implement this
+      def fetch_translations(_string, _from, _to)
+        raise 'subclass must implement fetch_translations'
+      end
+
+      def http_client(*args)
+        @http_client ||= Translatomatic::HTTP::Client.new(*args)
+      end
+
+      # Fetch translations for the given strings, one at a time, by
+      # opening a http connection to the given url and calling
+      # fetch_translation() on each string. Error handling and recovery
+      # is performed by this method.
+      # (subclass must implement fetch_translation if this method is used)
+      def perform_fetch_translations(url, strings, from, to)
+        untranslated = strings.dup
+
+        http_client.start(url) do |_http|
+          until untranslated.empty?
+            # get next string to translate
+            string = untranslated[0]
+            # fetch translation
+            fetch_translations(string, from, to)
+            untranslated.shift
+          end
+        end
+      end
+
+      def add_translations(original, result)
+        # successful translation
+        result = [result] unless result.is_a?(Array)
+        result = convert_to_translations(original, result)
+        @listener.update_progress(1) if @listener
+        @translations += result
+      end
+
+      def convert_to_translations(original, result)
+        result.collect { |i| translation(original, i) }.compact
+      end
+
+      def translation(original, translated)
+        return nil if translated.blank?
+        string1 = Translatomatic::Text[original, @from]
+        string2 = Translatomatic::Text[translated, @to]
+        Translatomatic::Translation::Result.new(string1, string2, name)
+      end
+
+      def batcher(strings, max_count:, max_length:)
+        StringBatcher.new(strings, max_count: max_count, max_length: max_length)
+      end
+
+      def try_hash(hash, *keys)
+        result = hash
+        keys.each do |key|
+          result ||= {}
+          result = result.is_a?(Hash) ? result[key] : nil
+        end
+        result
+      end
+    end
+  end
+end

data/lib/translatomatic/provider/frengly.rb

@@ -0,0 +1,55 @@
+require 'net/http'
+
+module Translatomatic
+  module Provider
+    # Interface to the Frengly translation API
+    # @see http://www.frengly.com/api
+    class Frengly < Base
+      define_option :frengly_api_key, use_env: true,
+                                      desc: t('provider.frengly.api_key')
+      define_option :frengly_email, use_env: true,
+                                    desc: t('provider.email_address')
+      define_option :frengly_password, use_env: true,
+                                       desc: t('provider.password')
+
+      # Create a new Frengly provider instance
+      def initialize(options = {})
+        super(options)
+        @key = options[:frengly_api_key] || ENV['FRENGLY_API_KEY'] # optional
+        @email = options[:frengly_email]
+        @password = options[:frengly_password]
+        raise t('provider.email_required') unless @email
+        raise t('provider.password_required') unless @password
+      end
+
+      # (see Base#languages)
+      def languages
+        %w[en fr de es pt it nl tl fi el iw pl ru sv]
+      end
+
+      private
+
+      URL = 'http://frengly.com/frengly/data/translateREST'.freeze
+
+      def perform_translate(strings, from, to)
+        perform_fetch_translations(URL, strings, from, to)
+      end
+
+      def fetch_translations(string, from, to)
+        body = {
+          src: from.language,
+          dest: to.language,
+          text: string,
+          email: @email,
+          password: @password,
+          premiumkey: @key
+        }.to_json
+
+        # TODO: work out what the response looks like
+        response = http_client.post(URL, body, content_type: 'application/json')
+        data = JSON.parse(response.body)
+        add_translations(string, data['text'])
+      end
+    end
+  end
+end

data/lib/translatomatic/provider/google.rb

@@ -0,0 +1,78 @@
+
+module Translatomatic
+  module Provider
+    # Interface to the Google translation API
+    # @see https://cloud.google.com/translate/
+    class Google < Base
+      define_option :google_api_key,
+                    desc: t('provider.google.api_key'), use_env: true
+      define_option :google_model, enum: %i[base nmt],
+                                   desc: t('provider.google.model'), use_env: true
+
+      # (see Base.supports_no_translate_html?)
+      def self.supports_no_translate_html?
+        true
+      end
+
+      # Create a new Google provider instance
+      def initialize(options = {})
+        super(options)
+        @key = options[:google_api_key] || ENV['GOOGLE_API_KEY']
+        raise t('provider.google.key_required') if @key.nil?
+        @model = options[:google_model]
+      end
+
+      # (see Base#languages)
+      def languages
+        @languages ||= fetch_languages
+      end
+
+      private
+
+      BASE_URL = 'https://translation.googleapis.com'.freeze
+      TRANSLATE_URL = (BASE_URL + '/language/translate/v2').freeze
+      LANGUAGES_URL = (BASE_URL + '/language/translate/v2/languages').freeze
+      # https://cloud.google.com/translate/faq
+      # TODO: limit requested characters per second?
+      LIMIT = [128, 5000].freeze # strings, characters per request
+
+      def perform_translate(strings, from, to)
+        batcher(strings, max_count: LIMIT[0], max_length: LIMIT[1])
+          .each_batch do |texts|
+          perform_translate_texts(texts, from, to)
+        end
+      end
+
+      def perform_translate_texts(texts, from, to)
+        request_body = request_body(texts, from, to)
+        response = http_client.post(TRANSLATE_URL, request_body)
+        body = JSON.parse(response.body)
+        data = body['data'] || {}
+        translations = data['translations'] || []
+        translations = translations.collect { |i| i['translatedText'] }
+        texts.zip(translations).each do |original, translated|
+          add_translations(original, translated)
+        end
+      end
+
+      def request_body(strings, from, to)
+        body = {
+          q: strings,
+          source: from.language,
+          target: to.language,
+          format: 'html', # required for <span translate="no"></span>
+          key: @key
+        }
+        body[:model] = @model if @model
+        body
+      end
+
+      def fetch_languages
+        response = http_client.get(LANGUAGES_URL, key: @key)
+        body = JSON.parse(response.body)
+        languages = try_hash(body, 'data', 'languages') || []
+        languages.collect { |i| i['language'] }
+      end
+    end
+  end
+end

data/lib/translatomatic/provider/google_web.rb

@@ -0,0 +1,50 @@
+module Translatomatic
+  module Provider
+    # Google translation web interface.
+    # supports multiple translations
+    # @see https://translate.google.com.au
+    class GoogleWeb < Base
+      attr_accessor :dt
+
+      # (see Base.supports_alternative_translations?)
+      def self.supports_alternative_translations?
+        true
+      end
+
+      # Create a new GoogleWeb provider instance
+      def initialize(options = {})
+        super(options)
+        require 'google_web_translate'
+        @dt = %w[t at]
+        @debug = options[:debug]
+      end
+
+      # (see Base#languages)
+      def languages
+        api.respond_to?(:languages) ? api.languages : []
+      end
+
+      private
+
+      def api
+        options = { debug: @debug, dt: @dt, http_client: http_client }
+        @api ||= GoogleWebTranslate::API.new(options)
+      end
+
+      def perform_translate(strings, from, to)
+        strings.each do |string|
+          result = api.translate(string, from, to)
+          add_translations(string, translations_from_result(result))
+        end
+      end
+
+      def translations_from_result(result)
+        if result.alternatives.present?
+          result.alternatives
+        else
+          result.translation
+        end
+      end
+    end
+  end
+end

data/lib/translatomatic/provider/microsoft.rb

@@ -0,0 +1,144 @@
+require 'builder'
+
+module Translatomatic
+  module Provider
+    # Interface to the Microsoft translation API
+    # @see https://www.microsoft.com/en-us/translator/translatorapi.aspx
+    # @see http://docs.microsofttranslator.com/text-translate.html
+    class Microsoft < Base
+      define_option :microsoft_api_key,
+                    desc: t('provider.microsoft.api_key'), use_env: true
+
+      # (see Base.supports_alternative_translations?)
+      def self.supports_alternative_translations?
+        true
+      end
+
+      # (see Base.supports_no_translate_html?)
+      def self.supports_no_translate_html?
+        true
+      end
+
+      # Create a new Microsoft provider instance
+      def initialize(options = {})
+        super(options)
+        @key = options[:microsoft_api_key] || ENV['MICROSOFT_API_KEY']
+        raise t('provider.microsoft.key_required') if @key.nil?
+      end
+
+      # (see Base#languages)
+      def languages
+        @languages ||= fetch_languages
+      end
+
+      private
+
+      BASE_URL = 'https://api.microsofttranslator.com/V2/Http.svc'.freeze
+      # this endpoint returns one translation per source text
+      TRANSLATE_URL1 = "#{BASE_URL}/TranslateArray".freeze
+      LIMITS_URL1 = [2000, 10_000].freeze # strings, characters per request
+      # this url returns multiple translations
+      TRANSLATE_URL2 = "#{BASE_URL}/GetTranslationsArray".freeze
+      LIMITS_URL2 = [10, 10_000].freeze # strings, characters per request
+      MAX_TRANSLATIONS = 10 # for URL2
+      LANGUAGES_URL = "#{BASE_URL}/GetLanguagesForTranslate".freeze
+      ARRAYS_NS = 'http://schemas.microsoft.com/2003/10/Serialization/Arrays'.freeze
+      WEB_SERVICE_NS = 'http://schemas.datacontract.org/2004/07/Microsoft.MT.Web.Service.V2'.freeze
+
+      def perform_translate(strings, from, to)
+        # get multiple translations for strings with context, so we
+        # can choose the best translation.
+        strings_with_context = strings.select { |i| context?(i) }
+        strings_without_context = strings.reject { |i| context?(i) }
+
+        fetch_translation_array(strings_with_context, from, to, true)
+        fetch_translation_array(strings_without_context, from, to, false)
+      end
+
+      # fetch translations for given strings
+      def fetch_translation_array(strings, from, to, multiple)
+        limit = multiple ? LIMITS_URL2 : LIMITS_URL1
+        batcher(strings, max_count: limit[0], max_length: limit[1])
+          .each_batch do |texts|
+          translate_texts(texts, from, to, multiple)
+        end
+      end
+
+      def translate_texts(texts, from, to, multiple)
+        url = multiple ? TRANSLATE_URL2 : TRANSLATE_URL1
+        headers = { 'Ocp-Apim-Subscription-Key' => @key }
+        body = build_body_xml(texts, from, to, multiple)
+        response = http_client.post(url, body,
+                                    headers: headers,
+                                    content_type: 'application/xml')
+        add_translations_from_response(response, texts, multiple)
+      end
+
+      def add_translations_from_response(response, texts, multiple)
+        doc = Nokogiri::XML(response.body)
+        if multiple
+          add_translations_from_response_multiple(doc, texts)
+        else
+          results = doc.search('//xmlns:TranslatedText').collect(&:content)
+          texts.zip(results).each do |original, tr|
+            add_translations(original, tr)
+          end
+        end
+      end
+
+      def add_translations_from_response_multiple(doc, texts)
+        # there should be one GetTranslationsResponse for each string
+        responses = doc.search('//xmlns:GetTranslationsResponse')
+        texts.zip(responses).each do |original, tr|
+          results = tr.search('TranslatedText').collect(&:content)
+          add_translations(original, results)
+        end
+      end
+
+      def fetch_languages
+        # this request redirects to a html page
+        headers = { 'Ocp-Apim-Subscription-Key' => @key }
+        query = { 'appid' => '' }
+        response = http_client.get(LANGUAGES_URL, query, headers: headers)
+        log.debug("#{name} response: #{response.body}")
+        doc = Nokogiri::XML(response.body)
+        doc.search('//xmlns:string').collect(&:content)
+      end
+
+      def xml_root(multiple)
+        multiple ? 'GetTranslationsArray' : 'TranslateArray'
+      end
+
+      def build_body_xml(strings, from, to, multiple)
+        root = xml_root(multiple) + 'Request'
+        xml = Builder::XmlMarkup.new
+        xml.tag!(root, 'xmlns:a' => ARRAYS_NS) do
+          xml.AppId
+          xml.From(from)
+          build_options_xml(xml) if multiple
+          build_texts_xml(xml, strings)
+          xml.To(to)
+          xml.MaxTranslations MAX_TRANSLATIONS if multiple
+        end
+      end
+
+      def build_texts_xml(xml, strings)
+        xml.Texts do
+          strings.each do |string|
+            xml.tag!('a:string', string)
+          end
+        end
+      end
+
+      def build_options_xml(xml)
+        xml.tag!('Options', 'xmlns:o' => WEB_SERVICE_NS) do
+          xml.tag!('o:IncludeMultipleMTAlternatives', 'true')
+        end
+      end
+
+      def context?(text)
+        text.is_a?(Translatomatic::Text) && text.context.present?
+      end
+    end
+  end
+end