translatomatic 0.1.3 → 0.2.0

data/lib/translatomatic/text_collection.rb

@@ -0,0 +1,66 @@
+module Translatomatic
+  # A collection of texts
+  class TextCollection
+    attr_reader :originals
+
+    def initialize(texts = [])
+      texts = [texts] unless texts.is_a?(Array)
+      texts = texts.select { |i| translatable?(i) }
+      texts = textify(texts) # convert to Text objects
+      @originals = texts
+      @sentences = find_sentences(texts) # convert to sentences
+      contexts = find_contexts(texts)
+      @all_texts = @sentences + contexts
+      group_by_locale(@all_texts)
+    end
+
+    # Iterate over texts in the collection grouped by locale
+    def each_locale
+      @by_locale.each do |locale, list|
+        yield locale, list
+      end
+    end
+
+    # @return [Number] The total number of texts in the collection,
+    #   equal to the number of sentences and context strings.
+    def count
+      @all_texts.length
+    end
+
+    private
+
+    def translatable?(text)
+      # don't translate numbers
+      text && !text.match(/\A\s*\z/) && !text.match(/\A[\d,]+\z/)
+    end
+
+    def textify(texts)
+      texts.collect { |i| build_text(i) }
+    end
+
+    def find_sentences(texts)
+      texts.collect(&:sentences).flatten.uniq
+    end
+
+    def find_contexts(texts)
+      texts.collect { |i| build_text(i.context) }.flatten.uniq.compact
+    end
+
+    def group_by_locale(texts)
+      @by_locale = {}
+      texts.each do |text|
+        list = @by_locale[text.locale] ||= []
+        list << text
+      end
+    end
+
+    def build_text(value, locale = Locale.default)
+      return nil if value.nil?
+      if value.is_a?(Translatomatic::Text)
+        value
+      else
+        Translatomatic::Text.new(value, locale)
+      end
+    end
+  end
+end

data/lib/translatomatic/tmx.rb

@@ -1,6 +1,8 @@
-# Translation Memory Exchange
-# @see https://en.wikipedia.org/wiki/Translation_Memory_eXchange
-module Translatomatic::TMX; end
+module Translatomatic
+  # Translation Memory Exchange
+  # @see https://en.wikipedia.org/wiki/Translation_Memory_eXchange
+  module TMX; end
+end
 
 require 'translatomatic/tmx/translation_unit'
 require 'translatomatic/tmx/document'

data/lib/translatomatic/tmx/document.rb

@@ -1,99 +1,124 @@
-module Translatomatic::TMX
-  # Translation Memory Exchange document
-  class Document
-
-    # Create a new instance
-    # @param units [Array<TranslationUnit>] A list of translation units
-    # @param source_locale [Locale] Source locale
-    # @return [Translatomatic::TMX::Document] a new TMX object
-    def initialize(units, source_locale)
-      units = [units] unless units.kind_of?(Array)
-      @units = units
-      @source_locale = source_locale
-    end
+module Translatomatic
+  module TMX
+    # Translation Memory Exchange document
+    class Document
+      # Create a new instance
+      # @param units [Array<TranslationUnit>] A list of translation units
+      # @param source_locale [Locale] Source locale
+      # @return [Translatomatic::TMX::Document] a new TMX object
+      def initialize(units, source_locale)
+        units = [units] unless units.is_a?(Array)
+        @units = units
+        @source_locale = source_locale
+      end
 
-    # @return [String] An XML string
-    def to_xml(options = {})
-      builder = Nokogiri::XML::Builder.new do |xml|
-        dtd = options[:dtd] || TMX_DTD
-        xml.doc.create_internal_subset('tmx', nil, dtd)
-        xml.tmx(version: "1.4") do
-          xml.header(creationtool: "Translatomatic",
-            creationtoolversion: Translatomatic::VERSION,
-            datatype: "PlainText",
-            segtype: "phrase",  # default segtype
-            adminlang: @source_locale.to_s,
-            srclang: @source_locale.to_s,
-            "o-tmf": DEFAULT_OTMF
-          )
-          xml.body { tmx_body(xml) }
+      # @return [String] An XML string
+      def to_xml(options = {})
+        builder = Nokogiri::XML::Builder.new do |xml|
+          dtd = options[:dtd] || TMX_DTD
+          xml.doc.create_internal_subset('tmx', nil, dtd)
+          xml.tmx(version: '1.4') do
+            xml.header(tmx_header)
+            xml.body { tmx_body(xml) }
+          end
         end
+        builder.to_xml
       end
-      builder.to_xml
-    end
 
-    # Create a TMX document from the given converter
-    # @param texts [Array<Translatomatic::Model::Text>] List of texts
-    # @return [Translatomatic::TMX::Document] TMX document
-    def self.from_texts(texts)
-      # group texts by from_text_id to create units
-      # source_locale: use from_text.locale
-      # origin: use text.translator
-      sources = texts.select { |i| i.from_text.nil? }
-      source_locales = sources.collect { |i| i.locale }.uniq
-      raise t("tmx.multiple_locales") if source_locales.length > 1
-      units = units_from_texts(texts)
-
-      return new(units, source_locales[0])
-    end
+      # Create a TMX document from the given converter
+      # @param texts [Array<Translatomatic::Model::Text>] List of texts
+      # @return [Translatomatic::TMX::Document] TMX document
+      def self.from_texts(texts)
+        # group texts by from_text_id to create units
+        # source_locale: use from_text.locale
+        # origin: use text.provider
+        sources = texts.select { |i| i.from_text.nil? }
+        source_locales = sources.collect(&:locale).uniq
+        raise t('tmx.multiple_locales') if source_locales.length > 1
+        units = units_from_texts(texts)
 
-    def self.valid?(xml)
-      options = Nokogiri::XML::ParseOptions::DTDVALID
-      doc = Nokogiri::XML::Document.parse(xml, nil, nil, options)
-      doc.internal_subset.validate(doc)
-    end
+        new(units, source_locales[0])
+      end
 
-    private
+      # Create a TMX document from a translation collection
+      # @param collection [Translatomatic::Translation::Collection]
+      #   Translation collection
+      # @return [Translatomatic::TMX::Document] TMX document
+      def self.from_collection(collection)
+        originals = collection.translations.collect(&:original)
+        source_locales = originals.collect(&:locale)
+        raise t('tmx.multiple_locales') if source_locales.length > 1
+        units = units_from_collection(collection)
 
-    class << self
-      include Translatomatic::Util
-    end
+        new(units, source_locales[0])
+      end
 
-    TMX_DTD = "http://www.ttt.org/oscarstandards/tmx/tmx14.dtd"
-    DEFAULT_OTMF = "Translatomatic"
+      def self.valid?(xml)
+        options = Nokogiri::XML::ParseOptions::DTDVALID
+        doc = Nokogiri::XML::Document.parse(xml, nil, nil, options)
+        doc.internal_subset.validate(doc)
+      end
 
-    def tmx_body(xml)
-      @units.each do |unit|
-        xml.tu("segtype": unit.strings[0].type) do
-          unit.strings.each do |string|
-            xml.tuv("xml:lang": string.locale.to_s) do
-              xml.seg string.value
-            end
+      private
+
+      class << self
+        include Translatomatic::Util
+
+        private
+
+        # @return [Array<Translatomatic::TMX::TranslationUnit]
+        #   translation unit list
+        def units_from_texts(texts)
+          # group texts by from_text_id
+          texts_by_from_id = {}
+          texts.each do |text|
+            id = text.from_text_id || text.id
+            list = (texts_by_from_id[id] ||= [])
+            list << text
+          end
+
+          # create list of Translation Units
+          texts_by_from_id.values.collect do |list|
+            strings = list.uniq.collect { |i| build_text(i.value, i.locale) }
+            tmx_unit(strings)
           end
         end
-      end
-    end
 
-    # @return [Array<Translatomatic::TMX::TranslationUnit] translation unit list
-    def self.units_from_texts(texts)
-      # group texts by from_text_id
-      texts_by_from_id = {}
-      texts.each do |text|
-        id = text.from_text_id || text.id
-        list = (texts_by_from_id[id] ||= [])
-        list << text
+        # @return [Array<Translatomatic::TMX::TranslationUnit]
+        #   translation unit list
+        def units_from_collection(collection); end
+
+        def tmx_unit(strings)
+          Translatomatic::TMX::TranslationUnit.new(strings)
+        end
       end
 
-      # create list of Translation Units
-      texts_by_from_id.values.collect do |list|
-        strings = list.uniq.collect { |i| string(i.value, i.locale) }
-        tmx_unit(strings)
+      TMX_DTD = 'http://www.ttt.org/oscarstandards/tmx/tmx14.dtd'.freeze
+      DEFAULT_OTMF = 'Translatomatic'.freeze
+
+      def tmx_header
+        {
+          creationtool: 'Translatomatic',
+          creationtoolversion: Translatomatic::VERSION,
+          datatype: 'PlainText',
+          segtype: 'phrase', # default segtype
+          adminlang: @source_locale.to_s,
+          srclang: @source_locale.to_s,
+          'o-tmf' => DEFAULT_OTMF
+        }
       end
-    end
 
-    def self.tmx_unit(strings)
-      Translatomatic::TMX::TranslationUnit.new(strings)
+      def tmx_body(xml)
+        @units.each do |unit|
+          xml.tu('segtype' => unit.strings[0].type) do
+            unit.strings.each do |string|
+              xml.tuv('xml:lang' => string.locale.to_s) do
+                xml.seg string.value
+              end
+            end
+          end
+        end
+      end
     end
-
-  end # class
-end   # module
+  end
+end

data/lib/translatomatic/tmx/translation_unit.rb

@@ -1,23 +1,24 @@
-module Translatomatic::TMX
-  # A TMX Translation Unit.
-  # A translation unit contains a list of strings, and is part of a TMX
-  # document.
-  # @see Translatomatic::TMX::Document
-  class TranslationUnit
+module Translatomatic
+  module TMX
+    # A TMX Translation Unit.
+    # A translation unit contains a list of strings, and is part of a TMX
+    # document.
+    # @see Translatomatic::TMX::Document
+    class TranslationUnit
+      # @return [Array<Translatomatic::Text>] Strings in this translation unit
+      attr_reader :strings
 
-    # @return [Array<Translatomatic::String>] Strings in this translation unit
-    attr_reader :strings
+      # @param strings [Array<Translatomatic::Text>] List of strings
+      def initialize(strings)
+        @strings = strings || []
+      end
 
-    # @param strings [Array<Translatomatic::String>] List of strings
-    def initialize(strings)
-      @strings = strings || []
-    end
-
-    # Test translation unit validity.
-    # A translation unit must contain at least two strings.
-    # @return [boolean] true if this translation unit is valid
-    def valid?
-      @strings.length >= 2
+      # Test translation unit validity.
+      # A translation unit must contain at least two strings.
+      # @return [boolean] true if this translation unit is valid
+      def valid?
+        @strings.length >= 2
+      end
     end
   end
 end

data/lib/translatomatic/translation.rb

@@ -1,30 +1,10 @@
 module Translatomatic
-  # Data object describing a text translation
-  class Translation
-    # @return [Translatomatic::String] original string
-    attr_reader :original
-
-    # @return [Translatomatic::String] translated string
-    attr_accessor :result
-
-    # @return [Symbol] The name of the translator
-    attr_reader :translator
-
-    # @return [boolean] True if this translation came from the database
-    attr_reader :from_database
-
-    def initialize(original, result, translator = nil, from_database = false)
-      @original = original
-      @result = result
-      @translator = translator
-      @from_database = from_database
-    end
-
-    private
-
-    def string(string)
-      string.kind_of?(Translatomatic::String) ? string : Translatomatic::String.new(string)
-    end
-
-  end
+  # Classes for translating strings
+  module Translation; end
 end
+
+require 'translatomatic/translation/collection'
+require 'translatomatic/translation/munging'
+require 'translatomatic/translation/fetcher'
+require 'translatomatic/translation/result'
+require 'translatomatic/translation/stats'

data/lib/translatomatic/translation/collection.rb

@@ -0,0 +1,199 @@
+require 'set'
+
+module Translatomatic
+  module Translation
+    # Stores results of translations.
+    # For each original text, there may be zero or more translations from
+    # one or more providers.
+    class Collection
+      # Create a translation collection
+      def initialize
+        # by_provider[provider] = [Result, ...]
+        @by_provider = {}
+        # by_original[text] = [Result, ...]
+        @by_original = {}
+      end
+
+      # @param string [String,Text] Original string
+      # @return [Array<Result>] All translations for the given string
+      def [](string)
+        @by_original[string.to_s]
+      end
+
+      def empty?
+        @by_original.empty?
+      end
+
+      # @param string [String,Text] Original string
+      # @param locale [Locale] Target locale
+      # @return [Result] The best translation for the given string
+      def get(string, locale)
+        locale = build_locale(locale)
+        list = @by_original[string.to_s] || []
+        list = sort_by_best_match(list)
+        if string.is_a?(Translatomatic::Text) && string.context
+          # string has a context
+          list = sort_by_context_match(list, string.context, locale)
+        end
+        list.find { |i| i.result.locale == locale }
+      end
+
+      # Add a list of translations to the collection
+      # @param translations [Array<Result>] Translation results
+      # @return [void]
+      def add(translations)
+        translations = [translations] unless translations.is_a?(Array)
+        translations.each do |tr|
+          next if tr.result.nil?
+          add_to_list(@by_provider, tr.provider, tr)
+          add_to_list(@by_original, tr.original, tr)
+        end
+      end
+
+      # @return [Number] The number of translations
+      def count
+        translations.length
+      end
+
+      # @return [Boolean] True if there is one or more translations
+      def present?
+        count > 0
+      end
+
+      # Get translations from this collection. If provider is specified,
+      # returns only translations from the given provider, otherwise all
+      # translations are returned.
+      # @param provider [String] Optional name of a provider.
+      # @return [Array<Result>] Translation results
+      def translations(provider = nil)
+        if provider.nil?
+          @by_provider.values.flatten
+        else
+          @by_provider[provider.to_s] || []
+        end
+      end
+
+      # Get a list of the best sentence translations for the given
+      # parent string.
+      # @param parent [Text] Parent text
+      # @param locale [Locale] Target locale
+      # @return [Array<Result>] Substring translation results
+      def sentences(parent, locale)
+        parent.sentences.collect do |sentence|
+          # get translation for sentence
+          translation = get(sentence, locale)
+          # create a new translation with the sentence as the original
+          # string, so that we can rely on the offset value.
+          if translation
+            Result.new(sentence, translation.result, translation.provider,
+                       from_database: translation.from_database)
+          end
+        end.compact
+      end
+
+      # Return a new collection with only the translations that came from
+      # providers (not from the database).
+      # @return [Collection] The collection result
+      def from_providers
+        result = self.class.new
+        provider_translations = translations.reject(&:from_database)
+        result.add(provider_translations)
+        result
+      end
+
+      # @return [Array<Result>] Best translations for each string
+      # @param locale [Locale] Target locale
+      def best_translations(locale)
+        @by_original.keys.collect { |i| get(i, locale) }
+      end
+
+      # @return [Array<String>] A list of providers that translations were
+      #   sourced from.
+      def providers
+        @by_provider.keys
+      end
+
+      # Combine this collection with another
+      # @param other [Collection] Another collection
+      # @return [Collection] The collection result
+      def +(other)
+        result = self.class.new
+        @by_provider.each_value { |i| result.add(i) }
+        other.translations.each { |i| result.add(i) }
+        result
+      end
+
+      # @param string [String,Text] Original string
+      # @param provider [String] Provider name
+      # @return [boolean] True if there is a translation for the given string.
+      def translated?(string, provider)
+        list = @by_provider[provider.to_s] || []
+        list.any? { |tr| tr.original.to_s == string.to_s }
+      end
+
+      # @return [String] String description of all translations
+      def description
+        translations.collect(&:description).join("\n")
+      end
+
+      private
+
+      include Translatomatic::Util
+
+      # sort the list of translations by comparing to the translation for
+      # the string context.
+      # for example:
+      #   context is 'go right' with a translation of 'Geh rechts'.
+      #   translation of 'right' is 'rechts', and 'richtig'.
+      #   translation 'rechts' will be ordered first, as 'rechts' is in
+      #     the translated context string (and is longer than 'recht').
+      def sort_by_context_match(list, context, locale)
+        return list if list.blank?
+        context_results = context_translation_results(list, context, locale)
+        # log.debug("context translations: #{context_results}")
+        # put translations that include the context string(s) first.
+        # also put longer translations that include the context string first.
+        # (fixes matching 'rechts' before 'recht')
+        list.sort_by do |tr|
+          tr_result = tr.result.downcase
+          ctx_match = context_results.any? { |ctx| ctx.include?(tr_result) }
+          ctx_match ? -tr_result.length : 1
+        end
+      end
+
+      # sort the list of translations by finding the most common
+      # translation. if there is an equal number of different translations
+      # use the first most common translation (first provider).
+      def sort_by_best_match(list)
+        by_count = {}
+        list.each do |tr|
+          key = tr.result.downcase
+          by_count[key] = (by_count[key] || 0) + 1
+        end
+
+        # not using sort_by to maintain original sort order
+        # unless count is different.
+        list.sort do |tr1, tr2|
+          by_count[tr1.result.downcase] <=> by_count[tr2.result.downcase]
+        end
+      end
+
+      # @param list [Array<Result>] translations
+      # @param context [Array<String>] context(s)
+      # @return [Array<String>] lowercased context translation strings
+      def context_translation_results(list, context, locale)
+        context_locale = list[0].original.locale
+        context = [context] unless context.is_a?(Array)
+        context.collect do |ctx|
+          context_tr = get(Text[ctx, context_locale], locale)
+          context_tr.result.downcase if context_tr
+        end.compact
+      end
+
+      def add_to_list(hash, key, value)
+        list = hash[key.to_s] ||= []
+        list << value
+      end
+    end
+  end
+end