translatomatic 0.1.3 → 0.2.0

data/lib/translatomatic/translation/fetcher.rb

@@ -0,0 +1,123 @@
+module Translatomatic
+  module Translation
+    # Fetches translations from the database and translation providers
+    class Fetcher
+      def initialize(options = {})
+        ATTRIBUTES.each do |i|
+          instance_variable_set("@#{i}", options[i])
+        end
+      end
+
+      # Fetch a list of translations for all texts given in the constructor
+      # for all providers.
+      # Translations are fetched from the database first, then from providers.
+      # @return [Array<Result>] List of translations
+      def translations
+        collection = Collection.new
+
+        # add all translations from the database to the collection
+        collection.add(find_database_translations(@texts)) if @use_db
+
+        # request translations for all texts that aren't in the database
+        untranslated = untranslated(collection)
+        if untranslated.present?
+          provider_translations = find_provider_translations(untranslated)
+          save_database_translations(provider_translations)
+          collection.add(provider_translations)
+        end
+
+        # puts collection.description
+        collection
+      end
+
+      private
+
+      include Util
+      include Munging
+
+      ATTRIBUTES = %i[provider texts from_locale to_locale
+                      use_db listener].freeze
+
+      # find texts that we do not have translations for
+      # @param collection [Collection] Translation collection
+      # @return [Array<String>] Untranslated texts
+      def untranslated(collection)
+        @texts.reject { |i| collection.translated?(i, @provider.name) }
+      end
+
+      # @return [Array<Result>] translations from the database
+      def find_database_translations(texts)
+        from = db_locale(@from_locale)
+        to = db_locale(@to_locale)
+
+        db_texts = Translatomatic::Model::Text.where(
+          locale: to, provider: @provider.name,
+          from_texts_texts: {
+            locale_id: from,
+            # convert untranslated texts to strings
+            value: texts.collect(&:to_s)
+          }
+        ).joins(:from_text).includes(:from_text)
+
+        texts_to_translations(db_texts, texts)
+      end
+
+      # @return [Array<Result>] translations from provider
+      def find_provider_translations(texts)
+        texts = wrap_notranslate(texts)
+        translations = @provider.translate(
+          texts, @from_locale, @to_locale
+        )
+        # check for valid response from provider and restore variables
+        translations.each do |tr|
+          raise t('provider.invalid_response') unless tr.is_a?(Result)
+        end
+        munge_translation_results(translations)
+      end
+
+      # use the original text from the translation rather than
+      # db_text.from_text.value, as the original string has required
+      # information such as offset and context.
+      def texts_to_translations(db_texts, texts)
+        db_text_map = hashify(db_texts, proc { |i| i.from_text.value })
+        texts.collect do |text|
+          next unless (db_text = db_text_map[text.to_s])
+          @listener.update_progress(1) if @listener
+          provider = db_text.provider
+          translation = build_text(db_text.value, @to_locale)
+          Result.new(text, translation, provider, from_database: true)
+        end.compact
+      end
+
+      def save_database_translations(translations)
+        return unless @use_db
+        ActiveRecord::Base.transaction do
+          from = db_locale(@from_locale)
+          to = db_locale(@to_locale)
+          translations.each do |tr|
+            next if tr.result.nil? # skip invalid translations
+            save_database_translation(from, to, tr)
+          end
+        end
+      end
+
+      def save_database_translation(from_locale, to_locale, translation)
+        original_text = Translatomatic::Model::Text.find_or_create_by!(
+          locale: from_locale,
+          value: translation.original.to_s
+        )
+
+        text = Translatomatic::Model::Text.find_or_create_by!(
+          locale: to_locale, value: translation.result.to_s,
+          from_text: original_text,
+          provider: @provider.name
+        )
+        text
+      end
+
+      def db_locale(locale)
+        Translatomatic::Model::Locale.from_tag(locale)
+      end
+    end
+  end
+end

data/lib/translatomatic/translation/munging.rb

@@ -0,0 +1,112 @@
+module Translatomatic
+  module Translation
+    # Translation result mungification code
+    module Munging
+      private
+
+      # @private
+      NoTranslateTag = Struct.new(:leading_space, :trailing_space)
+
+      def munge_translation_results(translations)
+        if use_notranslate?
+          translations.collect { |tr| remove_notranslate(tr) }
+        else
+          translations.select { |tr| restore_preserved_text(tr) }
+        end
+      end
+
+      def use_notranslate?
+        @provider.class.supports_no_translate_html?
+      end
+
+      # Restore parts of text that should be preserved in the translation.
+      # Used for providers that don't support translate="no" html5 attribute.
+      # This works when the translator preserves parts of the string that
+      # match the preserve_regex, e.g. for variables like '%{name}' if the
+      # translation matches /%{.*}/ then the original text can be restored.
+      # @return [Boolean] True if no errors were encountered
+      def restore_preserved_text(tr)
+        preserve_regex = tr.original.preserve_regex
+        return true unless preserve_regex
+
+        # find parts to preserve in the original string
+        list1 = tr.original.substrings(preserve_regex)
+        # find corresponding parts in the translated string
+        list2 = tr.result.substrings(preserve_regex)
+
+        return false unless list1.length == list2.length
+
+        # we can restore text. sort by largest offset first.
+        conversions = list1.zip(list2).collect.sort_by { |i| -i[0].offset }
+        conversions.each do |v1, v2|
+          tr.result[v2.offset, v2.length] = v1.value
+        end
+        true
+      end
+
+      # @param texts [Array<Text>] Texts to modify
+      # @return [Array<Text>] Texts with sections to preserve wrapped in a
+      #   notranslate directive.
+      def wrap_notranslate(texts)
+        return texts unless use_notranslate? && texts.any?(&:preserve_regex)
+        texts.collect do |text|
+          if text.preserve_regex
+            text.gsub(text.preserve_regex) do |i|
+              '<span translate="no">' + i[0] + '</span>'
+            end
+          else
+            text
+          end
+        end
+      end
+
+      # Update the translation to remove notranslate directives.
+      # Fixes spacing to match the original.
+      # @param tr [Result] Translation result
+      # @return [Result] Updated translation result
+      def remove_notranslate(tr)
+        return tr unless tr.original.preserve_regex
+        original_spacing = find_notranslate_spacing(tr.original)
+        result_spacing = find_notranslate_spacing(tr.result)
+        if original_spacing.length != result_spacing.length
+          # the number of notranslate directives in the result doesn't
+          # match the number in the original. this could mean an invalid
+          # translation?
+          log.debug("possible invalid translation: #{tr.description}")
+          original_spacing = nil # don't attempt to fix spacing.
+        end
+        original = remove_notranslate_text(tr.original)
+        result = remove_notranslate_text(tr.result, original_spacing)
+        Result.new(
+          original, result, tr.provider, from_database: tr.from_database
+        )
+      end
+
+      REGEX1 = %r{<span translate="no">\s*(.*?)\s*</span>}
+      REGEX2 = %r{(\s*)<span translate="no">\s*(.*?)\s*</span>(\s*)}
+
+      # scan text for no translate tags, record leading and trailing space.
+      def find_notranslate_spacing(text)
+        text.scan(REGEX2).collect { |i| [i[0], i[2]] }
+      end
+
+      def remove_notranslate_text(text, fix_spacing = nil)
+        if fix_spacing
+          num = 0 # match number
+          text.gsub(REGEX2) do |i|
+            notranslate_replacement(i[2], fix_spacing, num += 1)
+          end
+        else
+          text.gsub(REGEX1) { |i| i[1] }
+        end
+      end
+
+      def notranslate_replacement(string, fix_spacing, num)
+        spacing = fix_spacing[num - 1]
+        leading = spacing[0]
+        trailing = spacing[1]
+        leading + string + trailing
+      end
+    end
+  end
+end

data/lib/translatomatic/translation/result.rb

@@ -0,0 +1,50 @@
+module Translatomatic
+  module Translation
+    # Data object describing a text translation
+    class Result
+      # @return [Translatomatic::Text] original string
+      attr_reader :original
+
+      # @return [Translatomatic::Text] translated string
+      attr_reader :result
+
+      # @return [Symbol] The name of the provider. Can be nil for translations
+      #   that have been reconstituted from substrings.
+      attr_reader :provider
+
+      # @return [boolean] True if this translation came from the database
+      attr_reader :from_database
+
+      def initialize(original, result, provider, options = {})
+        raise 'original required' unless original.present?
+        raise 'result required' unless result.present?
+        @original = build_text(original)
+        @result = build_text(result)
+        @provider = provider
+        @from_database = options[:from_database]
+      end
+
+      # @return [String] The translated string
+      def to_s
+        result.to_s
+      end
+
+      # @return [String] A description of this translation
+      def description
+        format('%<original>s (%<from_locale>s) -> %<result>s (%<to_locale>s)',
+               original: original.to_s, result: result.to_s,
+               from_locale: original.locale, to_locale: result.locale)
+      end
+
+      private
+
+      def build_text(string)
+        if string.is_a?(Translatomatic::Text)
+          string
+        else
+          Translatomatic::Text.new(string, Locale.default)
+        end
+      end
+    end
+  end
+end

data/lib/translatomatic/translation/sharer.rb

@@ -0,0 +1,32 @@
+module Translatomatic
+  module Translation
+    # Share translations with providers
+    class Sharer
+      def initialize(options = {})
+        @options = options
+      end
+
+      # @param collection [Translatomatic::Translation::Collection]
+      #   Translation collection
+      # @return [void]
+      def share(collection)
+        return if collection.empty?
+
+        tmx = Translatomatic::TMX::Document.from_collection(collection)
+        available = Translatomatic::Provider.available(@options)
+        available.each do |provider|
+          if provider.respond_to?(:upload)
+            log.info(t('sharer.uploading_tmx', name: provider.name))
+            provider.upload(tmx)
+          end
+        end
+
+        ActiveRecord::Base.transaction do
+          db_texts.each do |text|
+            text.update(shared: true) if text.translated?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/translatomatic/translation/stats.rb

@@ -0,0 +1,44 @@
+module Translatomatic
+  module Translation
+    # Translation statistics
+    class Stats
+      include Translatomatic::Util
+
+      # @return [Array<Result>] A list of all translations
+      attr_reader :translations
+
+      # @return [Number] The number of translations that came from the database.
+      attr_reader :from_db
+
+      # @return [Number] The number of translations that came from the provider.
+      attr_reader :from_provider
+
+      # @return [Number] The number of untranslated strings
+      attr_reader :untranslated
+
+      # Combine stats with another object
+      # @param other [Stats] Another stats object
+      # @return [Stats] The result of adding this to other
+      def +(other)
+        raise "expected Stats, got #{other.class}" unless other.is_a?(Stats)
+        Stats.new(translations + other.translations)
+      end
+
+      private
+
+      def initialize(translations = [])
+        @translations = list = translations
+        @from_db = list.count { |i| i.from_database && i.result }
+        @from_provider = list.count { |i| !i.from_database && i.result }
+        @untranslated = list.count { |i| i.result.nil? }
+      end
+
+      def to_s
+        key = 'translator.total_translations'
+        t(key, total: @translations.length,
+               from_db: @from_db, from_provider: @from_provider,
+               untranslated: @untranslated)
+      end
+    end
+  end
+end

data/lib/translatomatic/translator.rb

@@ -1,107 +1,110 @@
-require 'translatomatic/translator/base'
-require 'translatomatic/translator/yandex'
-require 'translatomatic/translator/google'
-require 'translatomatic/translator/microsoft'
-require 'translatomatic/translator/frengly'
-require 'translatomatic/translator/my_memory'
-
-# Provides methods to access and create instances of
-# interfaces to translation APIs.
-module Translatomatic::Translator
-
-  class << self
-    include Translatomatic::Util
-  end
+module Translatomatic
+  # Translates strings from one language to another
+  class Translator
+    attr_reader :stats
 
-  # @return [Class] The translator class corresponding to the given name
-  def self.find(name)
-    name && !name.empty? ? self.const_get(name) : nil
-  end
+    def initialize(options = {})
+      @listener = options[:listener]
+      @providers = resolve_providers(options)
+      raise t('translator.provider_required') if @providers.empty?
+      @providers.each { |i| i.listener = @listener } if @listener
 
-  # Resolve the given list of translator names to a list of translators.
-  # If the list is empty, return all translators that are configured.
-  # @param list [Array<String>] Translator names or translators
-  # @param options [Hash<String,String>] Translator options
-  # @return [Array<Translatomatic::Translator::Base>] Translators
-  def self.resolve(list, options = {})
-    list = [list] unless list.kind_of?(Array)
-    list = list.compact.collect do |translator|
-      if translator.respond_to?(:translate)
-        translator
-      else
-        klass = Translatomatic::Translator.find(translator)
-        translator = klass.new(options)
-      end
-      translator
+      # use database by default if we're connected to a database
+      @use_db = !options[:no_database] && ActiveRecord::Base.connected?
+      log.debug(t('translator.database_disabled')) unless @use_db
+
+      @stats = Translatomatic::Translation::Stats.new
     end
 
-    if list.empty?
-      # find all available translators that work with the given options
-      list = Translatomatic::Translator.available(options)
-      if list.empty?
-        raise t("cli.no_translators")
+    # Translate texts to a target locale
+    # @param texts [Array<Translatomatic::Text>] Texts to translate
+    # @param to_locales [Array<Locale>] Target locale(s)
+    # @return [Array<Translatomatic::Translation>] Translations
+    def translate(texts, to_locales)
+      text_collection = TextCollection.new(texts)
+      to_locales = [to_locales] unless to_locales.is_a?(Array)
+
+      # for each provider
+      #   get translations for all texts from the database
+      #   for texts that are untranslated, call the provider
+      # return translations
+
+      translation_collection = Translation::Collection.new
+      text_collection.each_locale do |from_locale, list|
+        next if list.blank?
+        @providers.each do |provider|
+          to_locales.each do |to_locale|
+            fetcher = Translation::Fetcher.new(
+              provider: provider, texts: list, use_db: @use_db,
+              from_locale: from_locale, to_locale: to_locale,
+              listener: @listener
+            )
+            translations = fetcher.translations
+            translation_collection += translations
+            update_stats(translations)
+          end
+        end
       end
-    end
-    list
-  end
 
-  # @return [List<Class>] A list of all translator classes
-  def self.modules
-    self.constants.collect { |c| self.const_get(c) }.select do |klass|
-      klass.is_a?(Class) && klass != Translatomatic::Translator::Base
+      combine_substrings(translation_collection, text_collection, to_locales)
+      translation_collection
     end
-  end
 
-  # @return [List<String>] A list of all translators
-  def self.names
-    modules.collect { |i| i.name.demodulize }
-  end
+    private
 
-  # Find all configured translators
-  # @param options [Hash<String,String>] Translator options
-  # @return [Array<#translate>] A list of translator instances
-  def self.available(options = {})
-    available = []
-    modules.each do |mod|
-      begin
-        translator = mod.new(options)
-        available << translator
-      rescue Exception
-        log.debug(t("translator.unavailable", name: mod.name.demodulize))
+    include Util
+    include DefineOptions
+
+    define_option :no_database, type: :boolean, default: false,
+                                desc: t('translator.no_database')
+
+    # Combine translations of substrings of the original strings
+    # @param tr_collection [Translatomatic::Translation::Collection]
+    #   Translation collection
+    # @return [void]
+    def combine_substrings(tr_collection, text_collection, to_locales)
+      to_locales.each do |to_locale|
+        text_collection.originals.each do |parent|
+          combine_parent_substrings(tr_collection, parent, to_locale)
+        end
       end
     end
-    available
-  end
 
-  # @return [String] A description of all translators and options
-  def self.list
-    out = t("translator.translators") + "\n"
-    configured_options = {}
-    modules.each do |mod|
-      out += "\n" + mod.name.demodulize + ":\n"
-      opts = mod.options
-      opts.each do |opt|
-        configured_options[opt.name] = config.get(opt.name)
-        optname = opt.name.to_s.gsub("_", "-")
-        out += "  --%-18s  %18s  %10s  %15s\n" % [optname, opt.description,
-          opt.required ? t("translator.required_option") : "",
-          opt.env_name ? "ENV[#{opt.env_name}]" : ""]
+    def combine_parent_substrings(tr_collection, parent, to_locale)
+      # get a list of substring translations for this parent string
+      list = tr_collection.sentences(parent, to_locale)
+      # skip if we have no substrings for this string
+      return if list.blank?
+      list = list.sort_by { |tr| -tr.original.offset }
+
+      translated_parent = build_text(parent.value.dup, to_locale)
+      list.each do |tr|
+        original = tr.original
+        translated = tr.result
+        translated_parent[original.offset, original.length] = translated.to_s
       end
+
+      # add the translation that results from combining the translated
+      # substrings to the translation collection
+      new_translation = translation(parent, translated_parent)
+      tr_collection.add(new_translation)
     end
-    out += "\n"
-    out += t("translator.configured") + "\n"
-    configured = available(configured_options)
-    configured.each do |translator|
-      out += "  " + translator.name + "\n"
+
+    def translation(original, result, provider = nil, options = {})
+      Translatomatic::Translation::Result.new(
+        original, result, provider, options
+      )
     end
-    out += t("translator.no_translators") if configured.empty?
-    out + "\n"
-  end
 
-  private
+    def resolve_providers(options)
+      Translatomatic::Provider.resolve(options[:provider], options)
+    end
 
-  def self.config
-    Translatomatic.config
+    def update_stats(tr_collection)
+      stats = Translatomatic::Translation::Stats.new(
+        tr_collection.translations
+      )
+      @stats += stats
+    end
   end
-
 end