translatomatic 0.1.1 → 0.1.2

data/lib/translatomatic/converter.rb

@@ -1,171 +1,216 @@
+# The converter ties together functionality from translators,
+# resource files, and the database to convert files from one
+# language to another.
 class Translatomatic::Converter
 
-  class << self
-    attr_reader :options
-    private
-    include Translatomatic::DefineOptions
-  end
-
-  define_options(
-    { name: :translator, type: :string, aliases: "-t",
-      desc: "Translator implementation to use",
-      enum: Translatomatic::Translator.names },
-    { name: :dry_run, type: :boolean, aliases: "-n", desc:
-      "Print actions without performing translations or writing files" },
-    { name: :use_database, type: :boolean, default: true, desc:
-      "Store and retrieve translations in the database" }
-  )
-
-  # @return [Translatomatic::ConverterStats] translation statistics
-  attr_reader :stats
-
-  # @return [Array<Translatomatic::Model::Text>] A list of translations saved to the database
+  # @return [Array<Translatomatic::Model::Text>] A list of translations saved to the database
   attr_reader :db_translations
 
   # Create a converter to translate files
   #
-  # @param options A hash of converter and/or translator options.
+  # @param options [Hash<Symbol,Object>] converter and/or translator options.
   def initialize(options = {})
     @dry_run = options[:dry_run]
-    @translator = options[:translator]
-    @listener = options[:listener]
-
-    # use database by default if we're connected to a database
-    use_db = options.include?(:use_database) ? options[:use_database] : true
-    @use_db = use_db && ActiveRecord::Base.connected?
+    @listener = options[:listener]
+    @translators = Translatomatic::Translator.resolve(options[:translator], options)
+    raise t("converter.translator_required") if @translators.empty?
+    @translators.each { |i| i.listener = @listener } if @listener
 
-    log.debug("database is disabled") unless @use_db
-    if @translator && !@translator.respond_to?(:translate)
-      klass = Translatomatic::Translator.find(@translator)
-      @translator = klass.new(options)
-    end
-    raise "translator required" unless @translator
-    @translator.listener = @listener if @listener
-    @from_db = 0
-    @from_translator = 0
-    @db_translations = []
-  end
+    # use database by default if we're connected to a database
+    use_db = options.include?(:use_database) ? options[:use_database] : true
+    @use_db = use_db && ActiveRecord::Base.connected?
+    log.debug(t("converter.database_disabled")) unless @use_db
+
+    @db_translations = []
+    @translations = {}      # map of original text to Translation
+  end
 
   # @return [Translatomatic::ConverterStats] Translation statistics
   def stats
-    Translatomatic::ConverterStats.new(@from_db, @from_translator)
+    @stats ||= Translatomatic::ConverterStats.new(@translations)
   end
 
-  # Translate contents of source_file to the target locale.
-  # Automatically determines the target filename based on target locale.
+  # Translate properties of source_file to the target locale.
+  # Does not write changes to disk.
   #
-  # @param [String, Translatomatic::ResourceFile] source_file File to translate
-  # @param [String] to_locale The target locale, e.g. "fr"
+  # @param file [String, Translatomatic::ResourceFile] File to translate
+  # @param to_locale [String] The target locale, e.g. "fr"
   # @return [Translatomatic::ResourceFile] The translated resource file
-  def translate(source_file, to_locale)
-    if source_file.kind_of?(Translatomatic::ResourceFile::Base)
-      source = source_file
-    else
-      source = Translatomatic::ResourceFile.load(source_file)
-      raise "unsupported file type #{source_file}" unless source
-    end
-
-    to_locale = parse_locale(to_locale)
-    target = Translatomatic::ResourceFile.load(source.path)
-    target.path = source.locale_path(to_locale)
-    target.locale = to_locale
-    translate_to_target(source, target)
-  end
-
-  # Translates a resource file and writes results to a target resource file
-  #
-  # @param source [Translatomatic::ResourceFile] The source
-  # @param target [Translatomatic::ResourceFile] The file to write
-  # @return [Translatomatic::ResourceFile] The translated resource file
-  def translate_to_target(source, target)
-    # perform translation
-    log.info("translating #{source} to #{target}")
-    properties = translate_properties(source.properties, source.locale, target.locale)
-    target.properties = properties
-    unless @dry_run
-      target.path.parent.mkpath
-      target.save
-    end
-    target
+  def translate(file, to_locale)
+    file = resource_file(file)
+    to_locale = parse_locale(to_locale)
+
+    # do nothing if target language is the same as source language
+    return file if file.locale.language == to_locale.language
+
+    result = Translatomatic::TranslationResult.new(file, to_locale)
+
+    # translate using strings from the database first
+    each_translator(result) { translate_properties_with_db(result) }
+    # send remaining unknown strings to translator
+    each_translator(result) { translate_properties_with_translator(result) }
+
+    log.debug(t("converter.stats", from_db: stats.from_db,
+      from_translator: stats.from_translator,
+      untranslated: result.untranslated.length))
+    @listener.untranslated_texts(result.untranslated) if @listener
+
+    file.properties = result.properties
+    file.locale = to_locale
+    file
   end
-
-  # Translate values in the hash of properties.
-  # Uses existing translations from the database if available.
-  #
-  # @param [Hash] properties Text to translate
-  # @param [String, Locale] from_locale The locale of the given properties
-  # @param [String, Locale] to_locale The target locale for translations
-  # @return [Hash] Translated properties
-  def translate_properties(properties, from_locale, to_locale)
-    from_locale = parse_locale(from_locale)
-    to_locale = parse_locale(to_locale)
-
-    # sanity check
-    return properties if from_locale.language == to_locale.language
-
-    result = Translatomatic::TranslationResult.new(properties,
-      from_locale, to_locale)
 
-    # translate using strings from the database first
-    db_texts = translate_properties_with_db(result)
-
-    # send remaining unknown strings to translator
-    tr_texts = translate_properties_with_translator(result)
-
-    log.debug("translations from db: %d translator: %d untranslated: %d" %
-      [db_texts.length, tr_texts.length, result.untranslated.length])
-    @listener.untranslated_texts(result.untranslated) if @listener
-
-    result.properties
+  # Translates a resource file and writes results to a target resource file.
+  # The path of the target resource file is automatically determined.
+  #
+  # @param source [Translatomatic::ResourceFile] The source
+  # @param to_locale [String] The target locale, e.g. "fr"
+  # @return [Translatomatic::ResourceFile] The translated resource file
+  def translate_to_file(source, to_locale)
+    # Automatically determines the target filename based on target locale.
+    source = resource_file(source)
+    target = Translatomatic::ResourceFile.load(source.path)
+    target.path = source.locale_path(to_locale)
+
+    log.info(t("converter.translating", source: source,
+      source_locale: source.locale, target: target, target_locale: to_locale))
+    translate(target, to_locale)
+    unless @dry_run
+      target.path.parent.mkpath
+      target.save
+    end
+    target
   end
 
-  private
+  private
 
-  include Translatomatic::Util
+  include Translatomatic::Util
+  include Translatomatic::DefineOptions
 
-  # update result with translations from the database.
-  # returns a list of text records from the database.
-  def translate_properties_with_db(result)
-    db_texts = []
-    unless database_disabled?
-      untranslated = result.untranslated.to_a
-      db_texts = find_database_translations(result, untranslated)
+  define_options(
+    { name: :dry_run, type: :boolean, aliases: "-n",
+      desc: t("converter.dry_run")
+    },
+    { name: :use_database, type: :boolean, default: true,
+      desc: t("converter.use_database")
+    }
+  )
 
-      # find strings in untranslated that were matched in the database
-      original_map = {}  # map of original text to translated text from db
-      db_texts.each do |db_text|
-        original_map[db_text.from_text.value] = db_text
+  def each_translator(result)
+    @translators.each do |translator|
+      break if result.untranslated.empty?
+      @current_translator = translator
+      yield
+    end
+  end
+
+  # Attempt to restore interpolated variable names in the translation.
+  # If variable names cannot be restored, sets the translation result to nil.
+  # @param result [Translatomatic::TranslationResult] translation result
+  # @param translation [Translatomatic::Translation] translation
+  # @return [void]
+  def restore_variables(result, translation)
+    file = result.file
+    return unless file.supports_variable_interpolation?
+
+    # find variables in the original string
+    variables = string_variables(translation.original, file.locale, file)
+    # find variables in the translated string
+    translated_variables = string_variables(translation.result, result.to_locale, file)
+
+    if variables.length == translated_variables.length
+      # we can restore variables. sort by largest offset first.
+      # not using translation() method as that adds to @translations hash.
+      conversions = variables.zip(translated_variables).collect {
+         |v1, v2| Translatomatic::Translation.new(v1, v2)
+      }
+      conversions.sort_by! { |t| -t.original.offset }
+      conversions.each do |conversion|
+        v1 = conversion.original
+        v2 = conversion.result
+        translation.result[v2.offset, v2.length] = v1.value
       end
-      matched = untranslated.select { |i| original_map[i.value] }
-      db_texts = db_texts.collect { |i| i.value }
-      result.update_strings(matched, db_texts)
-      @from_db += db_texts.length
-      @listener.translated_texts(db_texts) if @listener
+    else
+      # unable to restore interpolated variable names
+      log.debug("#{@current_translator.name}: unable to restore variables: #{translation.result}")
+      translation.result = nil  # mark result as invalid
     end
-    db_texts
   end
 
-  # update result with translations from the translator.
-  # returns a list of strings from the translator.
-  def translate_properties_with_translator(result)
-    untranslated = result.untranslated.to_a.select { |i| translatable?(i) }
+  def string_variables(value, locale, file)
+    string(value, locale).substrings(file.variable_regex)
+  end
+
+  def resource_file(path)
+    if path.kind_of?(Translatomatic::ResourceFile::Base)
+      path
+    else
+      file = Translatomatic::ResourceFile.load(path)
+      raise t("converter.file_unsupported", file: path) unless file
+      file
+    end
+  end
+
+  # update result with translations from the database.
+  def translate_properties_with_db(result)
+    db_texts = []
+    unless database_disabled?
+      translations = []
+      untranslated = hashify(result.untranslated)
+      db_texts = find_database_translations(result, result.untranslated.to_a)
+      db_texts.each do |db_text|
+        from_text = db_text.from_text.value
+        if untranslated[from_text]
+          translation = translation(untranslated[from_text], db_text.value, true)
+          restore_variables(result, translation)
+          translations << translation
+        end
+      end
+
+      result.update_strings(translations)
+      @listener.translated_texts(db_texts) if @listener
+    end
+    db_texts
+  end
+
+  # update result with translations from the translator.
+  def translate_properties_with_translator(result)
+    untranslated = result.untranslated.to_a.select { |i| translatable?(i) }
     translated = []
-    @from_translator += untranslated.length
     if !untranslated.empty? && !@dry_run
-      translated = @translator.translate(untranslated.collect { |i| i.to_s },
-        result.from_locale, result.to_locale)
-      result.update_strings(untranslated, translated)
-      unless database_disabled?
-        save_database_translations(result, untranslated, translated)
+      untranslated_strings = untranslated.collect { |i| i.to_s }
+      log.debug("translating: #{untranslated_strings}")
+      translated = @current_translator.translate(untranslated_strings,
+        result.from_locale, result.to_locale
+      )
+
+      # create list of translations, filtering out invalid translations
+      translations = []
+      untranslated.zip(translated).each do |from, to|
+        translation = translation(from, to, false)
+        restore_variables(result, translation)
+        translations << translation
       end
-    end
-    translated
+
+      result.update_strings(translations)
+      unless database_disabled?
+        save_database_translations(result, translations)
+      end
+    end
+    translated
   end
 
-  def database_disabled?
-    !@use_db
+  def translation(from, to, from_database = false)
+    translator = @current_translator.name
+    t = Translatomatic::Translation.new(from, to, translator, from_database)
+    @translations[from] = t
+    t
   end
 
+  def database_disabled?
+    !@use_db
+  end
+
   def parse_locale(locale)
     Translatomatic::Locale.parse(locale)
   end
@@ -175,42 +220,44 @@ class Translatomatic::Converter
     !string.empty? && !string.match(/^[\d,]+$/)
   end
 
-  def save_database_translations(result, untranslated, translated)
+  def save_database_translations(result, translations)
     ActiveRecord::Base.transaction do
       from = db_locale(result.from_locale)
       to = db_locale(result.to_locale)
-      untranslated.zip(translated).each do |t1, t2|
-        save_database_translation(from, to, t1, t2)
+      translations.each do |translation|
+        next if translation.result.nil?  # skip invalid translations
+        save_database_translation(from, to, translation)
       end
     end
   end
 
-  def save_database_translation(from_locale, to_locale, t1, t2)
+  def save_database_translation(from_locale, to_locale, translation)
     original_text = Translatomatic::Model::Text.find_or_create_by!(
       locale: from_locale,
-      value: t1.to_s
+      value: translation.original.to_s
     )
 
     text = Translatomatic::Model::Text.find_or_create_by!(
       locale: to_locale,
-      value: t2.to_s,
+      value: translation.result.to_s,
       from_text: original_text,
-      translator: @translator.name
-    )
-    @db_translations += [original_text, text]
+      translator: @current_translator.name
+    )
+    @db_translations += [original_text, text]
     text
   end
 
   def find_database_translations(result, untranslated)
     from = db_locale(result.from_locale)
     to = db_locale(result.to_locale)
-
-    # convert untranslated set to strings
+
     Translatomatic::Model::Text.where({
-      locale: to,
-      from_texts_texts: {
-        locale_id: from,
-        value: untranslated.collect { |i| i.to_s }
+      locale: to,
+      translator: @current_translator.name,
+      from_texts_texts: {
+        locale_id: from,
+        # convert untranslated set to strings
+        value: untranslated.collect { |i| i.to_s }
       }
     }).joins(:from_text)
   end

data/lib/translatomatic/converter_stats.rb

@@ -1,7 +1,8 @@
 # Translation statistics
 class Translatomatic::ConverterStats
+  include Translatomatic::Util
 
-  # @return [Number] The total number of strings translated.
+  # @return [Array<Translatomatic::Translation>] A list of all translations
   attr_reader :translations
 
   # @return [Number] The number of translations that came from the database.
@@ -9,19 +10,22 @@ class Translatomatic::ConverterStats
 
   # @return [Number] The number of translations that came from the translator.
   attr_reader :from_translator
+
+  # @return [Number] The number of untranslated strings
+  attr_reader :untranslated
+
+  private
+
+  def initialize(translations)
+    @translations = translations.values
+    @from_db = @translations.count { |i| i.from_database && i.result }
+    @from_translator = @translations.count { |i| !i.from_database && i.result }
+    @untranslated = @translations.count { |i| i.result == nil }
+  end
 
-  def initialize(from_db, from_translator)
-    @translations = from_db + from_translator
-    @from_db = from_db
-    @from_translator = from_translator
-  end
-
-  def +(other)
-    self.class.new(@from_db + other.from_db, @from_translator + other.from_translator)
-  end
-
-  def to_s
-    "Total translations: #{@translations} " +
-      "(#{@from_db} from database, #{@from_translator} from translator)"
+  def to_s
+    t("converter.total_translations", total: @translations.length,
+      from_db: @from_db, from_translator: @from_translator,
+      untranslated: @untranslated)
   end
 end

data/lib/translatomatic/database.rb

@@ -1,29 +1,23 @@
 require 'active_record'
-
+
+# Database functions
 class Translatomatic::Database
 
-  include Translatomatic::Util
-
   class << self
-    attr_reader :options
-
-    # @param [Hash<Symbol,Object>] options Database options
-    # @return True if we can connect to the database
+    # @param options [Hash<Symbol,Object>] Database options
+    # @return [boolean] True if we can connect to the database
     def enabled?(options = {})
       new(options).connect
     end
-
-    private
-
-    include Translatomatic::DefineOptions
   end
 
   def initialize(options = {})
-    @env = options[:database_env] || DEFAULT_ENV
-    @db_config = database_config(@env, options)
+    @env = options[:database_env] || DEFAULT_ENV
+    @db_config = database_config(@env, options)
     @env_config = @db_config
-    raise "no environment '#{@env}' in #{db_config_path}" unless @env_config[@env]
-    @env_config = @env_config[@env] || {}
+    raise t("database.no_environment",
+      env: @env, file: db_config_path) unless @env_config[@env]
+    @env_config = @env_config[@env] || {}
 
     ActiveRecord::Base.configurations = @db_config
     ActiveRecord::Tasks::DatabaseTasks.env = @env
@@ -70,7 +64,7 @@ class Translatomatic::Database
     return false unless connect
     ActiveRecord::Migrator.migrate(MIGRATIONS_PATH)
     ActiveRecord::Base.clear_cache!
-    log.debug "Database migrated."
+    log.debug t("database.migrated")
   end
 
   # Create the database
@@ -78,10 +72,11 @@ class Translatomatic::Database
   def create
     begin
       ActiveRecord::Tasks::DatabaseTasks.create(@env_config)
-      log.debug "Database created."
+      log.debug t("database.created")
       true
-    rescue LoadError => e
-      log.debug "Database could not be created: " + e.message
+    rescue LoadError => e
+      log.debug t("database.could_not_create")
+      log.error e.message
       false
     end
   end
@@ -91,10 +86,13 @@ class Translatomatic::Database
   def drop
     disconnect
     ActiveRecord::Tasks::DatabaseTasks.drop(@env_config)
-    log.debug "Database deleted."
+    log.debug t("database.deleted")
   end
 
-  private
+  private
+
+  include Translatomatic::Util
+  include Translatomatic::DefineOptions
 
   def sqlite_database_exists?
     @env_config['adapter'] == 'sqlite3' && File.exist?(@env_config['database'])
@@ -113,11 +111,11 @@ class Translatomatic::Database
   DEFAULT_ENV = "production"
 
   define_options(
-    { name: :database_config, description: "Database config file",
+    { name: :database_config, desc: t("database.config_file"),
       default: DEFAULT_CONFIG },
-    { name: :database_env, description: "Database environment",
+    { name: :database_env, desc: t("database.env"),
       default: DEFAULT_ENV })
-
+
   # return path to database config
   def database_config_path(options)
     if options[:database_env] == "test"
@@ -127,19 +125,19 @@ class Translatomatic::Database
     else
       DEFAULT_CONFIG
     end
-  end
-
-  # return database config as a hash
-  def database_config(env, options)
-    if options[:database_config].kind_of?(Hash)
-      return { env => options[:database_config] }
-    end
-
-    db_config_path = database_config_path(options)
-    dbconfig = File.read(db_config_path)
-    dbconfig.gsub!(/\$HOME/, Dir.home)
-    dbconfig.gsub!(/\$GEM_ROOT/, GEM_ROOT)
-    YAML::load(dbconfig) || {}
+  end
+
+  # return database config as a hash
+  def database_config(env, options)
+    if options[:database_config].kind_of?(Hash)
+      return { env => options[:database_config] }
+    end
+
+    db_config_path = database_config_path(options)
+    dbconfig = File.read(db_config_path)
+    dbconfig.gsub!(/\$HOME/, Dir.home)
+    dbconfig.gsub!(/\$GEM_ROOT/, GEM_ROOT)
+    YAML::load(dbconfig) || {}
   end
 
 end

data/lib/translatomatic/escaped_unicode.rb

@@ -41,7 +41,7 @@ module Translatomatic::EscapedUnicode
         )+ |
         [\x80-\xc1\xf5-\xff]       # invalid
         )/nx) { |c|
-          c.size == 1 and raise "Invalid utf8 byte: '#{c}'"
+          c.size == 1 and raise t("unicode.invalid_byte", byte: c)
           s = c.encode("utf-16be", "utf-8").unpack('H*')[0]
           s.force_encoding(::Encoding::ASCII_8BIT)
           s.gsub!(/.{4}/n, '\\\\u\&')

data/lib/translatomatic/extractor/base.rb

@@ -1,4 +1,5 @@
 module Translatomatic::Extractor
+  # Base class for string extraction functionality
   class Base
 
     def initialize(path)
@@ -6,6 +7,7 @@ module Translatomatic::Extractor
       @contents = @path.read
     end
 
+    # @return [Array<String>] All strings found
     def extract
       @contents.scan(/\"(.*?[^\\])"|'(.*?[^\\])'/).flatten.compact
     end

data/lib/translatomatic/extractor/ruby.rb

@@ -1,4 +1,5 @@
 module Translatomatic::Extractor
+  # Class to extract strings from ruby code
   class Ruby < Base
 
   end # class

data/lib/translatomatic/extractor.rb

@@ -1,3 +1,4 @@
+# Classes to extract strings from files
 module Translatomatic::Extractor; end
 
 require 'translatomatic/extractor/base'