translatomatic 0.1.0

data/lib/translatomatic/converter.rb

@@ -0,0 +1,157 @@
+class Translatomatic::Converter
+  include Translatomatic::Util
+
+  class << self
+    attr_reader :options
+    private
+    include Translatomatic::DefineOptions
+  end
+
+  define_options(
+    { name: :translator, type: :string, aliases: "-t",
+      desc: "The translator implementation",
+      enum: Translatomatic::Translator.names },
+    { name: :dry_run, type: :boolean, aliases: "-n", desc:
+      "Print actions without performing translations or writing files" }
+  )
+
+  # @return [Translatomatic::ConverterStats] translation statistics
+  attr_reader :stats
+
+  # Create a converter to translate files
+  #
+  # @param options A hash of converter and/or translator options.
+  def initialize(options = {})
+    @dry_run = options[:dry_run]
+    @translator = options[:translator]
+    if @translator.kind_of?(String) || @translator.kind_of?(Symbol)
+      klass = Translatomatic::Translator.find(@translator)
+      @translator = klass.new(options)
+    end
+    raise "translator required" unless @translator
+    @from_db = 0
+    @from_translator = 0
+  end
+
+  # @return [Translatomatic::ConverterStats] Translation statistics
+  def stats
+    Translatomatic::ConverterStats.new(@from_db, @from_translator)
+  end
+
+  # Translate contents of source_file to the target locale.
+  # Automatically determines the target filename based on target locale.
+  #
+  # @param [String, Translatomatic::ResourceFile] source_file File to translate
+  # @param [String] to_locale The target locale, e.g. "fr"
+  # @return [Translatomatic::ResourceFile] The translated resource file
+  def translate(source_file, to_locale)
+    if source_file.kind_of?(Translatomatic::ResourceFile::Base)
+      source = source_file
+    else
+      source = Translatomatic::ResourceFile.load(source_file)
+      raise "unsupported file type #{source_file}" unless source
+    end
+
+    to_locale = parse_locale(to_locale)
+    target = Translatomatic::ResourceFile.load(source.path)
+    target.path = source.locale_path(to_locale)
+    target.locale = to_locale
+    translate_to_target(source, target)
+  end
+
+  # Translates a resource file and writes results to a target resource file
+  #
+  # @param source [Translatomatic::ResourceFile] The source
+  # @param target [Translatomatic::ResourceFile] The file to write
+  # @return [Translatomatic::ResourceFile] The translated resource file
+  def translate_to_target(source, target)
+    # perform translation
+    log.info "translating #{source} to #{target}"
+    properties = translate_properties(source.properties, source.locale, target.locale)
+    target.properties = properties
+    target.save unless @dry_run
+    target
+  end
+
+  # Translate values in the hash of properties.
+  # Uses existing translations from the database if available.
+  #
+  # @param [Hash] properties Text to translate
+  # @param [String, Locale] from_locale The locale of the given properties
+  # @param [String, Locale] to_locale The target locale for translations
+  # @return [Hash] Translated properties
+  def translate_properties(properties, from_locale, to_locale)
+    from_locale = parse_locale(from_locale)
+    to_locale = parse_locale(to_locale)
+
+    # sanity check
+    return properties if from_locale.language == to_locale.language
+
+    result = Translatomatic::TranslationResult.new(properties,
+      from_locale, to_locale)
+
+    # find translations in database first
+    texts = find_database_translations(result)
+    result.update_db_strings(texts)
+    @from_db += texts.length
+
+    # send remaining unknown strings to translator
+    # (copy untranslated set from result)
+    untranslated = result.untranslated.to_a.select { |i| translatable?(i) }
+    @from_translator += untranslated.length
+    if !untranslated.empty? && !@dry_run
+      translated = @translator.translate(untranslated, from_locale, to_locale)
+      result.update_strings(untranslated, translated)
+      save_database_translations(result, untranslated, translated)
+    end
+
+    log.debug("translations from db: %d translator: %d untranslated: %d" %
+      [texts.length, untranslated.length, result.untranslated.length])
+    result.properties
+  end
+
+  private
+
+  def translatable?(string)
+    # don't translate numbers
+    !string.empty? && !string.match(/^[\d,]+$/)
+  end
+
+  def save_database_translations(result, untranslated, translated)
+    ActiveRecord::Base.transaction do
+      from = db_locale(result.from_locale)
+      to = db_locale(result.to_locale)
+      untranslated.zip(translated).each do |t1, t2|
+        save_database_translation(from, to, t1, t2)
+      end
+    end
+  end
+
+  def save_database_translation(from_locale, to_locale, t1, t2)
+    original_text = Translatomatic::Model::Text.find_or_create_by!(
+      locale: from_locale,
+      value: t1
+    )
+
+    Translatomatic::Model::Text.find_or_create_by!(
+      locale: to_locale,
+      value: t2,
+      from_text: original_text,
+      translator: @translator.class.name.demodulize
+    )
+  end
+
+  def find_database_translations(result)
+    from = db_locale(result.from_locale)
+    to = db_locale(result.to_locale)
+
+    Translatomatic::Model::Text.where({
+      locale: to,
+      from_texts_texts: { locale_id: from, value: result.untranslated.to_a }
+    }).joins(:from_text)
+  end
+
+  def db_locale(locale)
+    Translatomatic::Model::Locale.from_tag(locale)
+  end
+end

data/lib/translatomatic/converter_stats.rb

@@ -0,0 +1,27 @@
+# Translation statistics
+class Translatomatic::ConverterStats
+
+  # @return [Number] The total number of strings translated.
+  attr_reader :translations
+
+  # @return [Number] The number of translations that came from the database.
+  attr_reader :from_db
+
+  # @return [Number] The number of translations that came from the translator.
+  attr_reader :from_translator
+
+  def initialize(from_db, from_translator)
+    @translations = from_db + from_translator
+    @from_db = from_db
+    @from_translator = from_translator
+  end
+
+  def +(other)
+    self.class.new(@from_db + other.from_db, @from_translator + other.from_translator)
+  end
+
+  def to_s
+    "Total translations: #{@translations} " +
+      "(#{@from_db} from database, #{@from_translator} from translator)"
+  end
+end

data/lib/translatomatic/database.rb

@@ -0,0 +1,105 @@
+require 'active_record'
+
+class Translatomatic::Database
+
+  include Translatomatic::Util
+
+  class << self
+    attr_reader :options
+    private
+    include Translatomatic::DefineOptions
+  end
+
+  def initialize(options = {})
+    db_config_path = db_config_path(options)
+    dbconfig = File.read(db_config_path)
+    dbconfig.gsub!(/\$HOME/, Dir.home)
+    dbconfig.gsub!(/\$GEM_ROOT/, GEM_ROOT)
+    @env = options[:database_env] || DEFAULT_ENV
+    @db_config = YAML::load(dbconfig) || {}
+    @env_config = @db_config
+    raise "no environment '#{@env}' in #{db_config_path}" unless @env_config[@env]
+    @env_config = @env_config[@env]
+    ActiveRecord::Base.configurations = @db_config
+    ActiveRecord::Tasks::DatabaseTasks.env = @env
+    ActiveRecord::Tasks::DatabaseTasks.db_dir = DB_PATH
+    ActiveRecord::Tasks::DatabaseTasks.root = DB_PATH
+    ActiveRecord::Tasks::DatabaseTasks.database_configuration = @db_config
+    create unless exists?
+    migrate
+  end
+
+  # Connect to the database
+  # @return [void]
+  def connect
+    ActiveRecord::Base.establish_connection(@env_config)
+  end
+
+  # Disconnect from the database
+  # @return [void]
+  def disconnect
+    ActiveRecord::Base.remove_connection
+  end
+
+  # Test if the database exists
+  # @return [Boolean] true if the database exists
+  def exists?
+    begin
+      connect
+      ActiveRecord::Base.connection.tables
+    rescue
+      return false
+    end
+    true
+  end
+
+  # Run outstanding migrations against the database
+  # @return [void]
+  def migrate
+    connect
+    ActiveRecord::Migrator.migrate(MIGRATIONS_PATH)
+    ActiveRecord::Base.clear_cache!
+    log.debug "Database migrated."
+  end
+
+  # Create the database
+  # @return [void]
+  def create
+    ActiveRecord::Tasks::DatabaseTasks.create(@env_config)
+    log.debug "Database created."
+  end
+
+  # Drop the database
+  # @return [void]
+  def drop
+    disconnect
+    ActiveRecord::Tasks::DatabaseTasks.drop(@env_config)
+    log.debug "Database deleted."
+  end
+
+  private
+
+  DB_PATH = File.join(File.dirname(__FILE__), "..", "..", "db")
+  INTERNAL_DB_CONFIG = File.join(DB_PATH, "database.yml")
+  CUSTOM_DB_CONFIG = File.join(Dir.home, ".translatomatic", "database.yml")
+  DEFAULT_DB_CONFIG = File.exist?(CUSTOM_DB_CONFIG) ? CUSTOM_DB_CONFIG : INTERNAL_DB_CONFIG
+  MIGRATIONS_PATH = File.join(DB_PATH, "migrate")
+  GEM_ROOT = File.join(File.dirname(__FILE__), "..", "..")
+  DEFAULT_ENV = "production"
+
+  define_options(
+    { name: :database_config, description: "Database config file",
+      default: DEFAULT_DB_CONFIG },
+    { name: :database_env, description: "Database environment",
+      default: DEFAULT_ENV })
+
+  def db_config_path(options)
+    if options[:database_env] == "test"
+      INTERNAL_DB_CONFIG  # rspec
+    elsif options[:database_config]
+      return options[:database_config]
+    else
+      DEFAULT_DB_CONFIG
+    end
+  end
+end

data/lib/translatomatic/escaped_unicode.rb

@@ -0,0 +1,90 @@
+# Module to encode and decode unicode chars.
+# This code is highly influced by Florian Frank's JSON gem
+# @see https://github.com/jnbt/java-properties
+# @see https://github.com/flori/json/
+
+module Translatomatic::EscapedUnicode
+
+  # Decodes all unicode chars from escape sequences
+  # @param text [String]
+  # @return [String] The encoded text for chaining
+  def self.unescape(text)
+    string = text.dup
+    string = string.gsub(%r((?:\\[uU](?:[A-Fa-f\d]{4}))+)) do |c|
+      c.downcase!
+      bytes = EMPTY_8BIT_STRING.dup
+      i = 0
+      while c[6 * i] == ?\\ && c[6 * i + 1] == ?u
+        bytes << c[6 * i + 2, 2].to_i(16) << c[6 * i + 4, 2].to_i(16)
+        i += 1
+      end
+      bytes.encode("utf-8", "utf-16be")
+    end
+    string.force_encoding(::Encoding::UTF_8)
+
+    text.replace string
+    text
+  end
+
+  # Decodes all unicode chars into escape sequences
+  # @param text [String]
+  # @return [String] The decoded text for chaining
+  def self.escape(text)
+    string = text.dup
+    string.force_encoding(::Encoding::ASCII_8BIT)
+    string.gsub!(/["\\\x0-\x1f]/n) { |c| MAP[c] || c }
+    string.gsub!(/(
+      (?:
+        [\xc2-\xdf][\x80-\xbf]    |
+        [\xe0-\xef][\x80-\xbf]{2} |
+        [\xf0-\xf4][\x80-\xbf]{3}
+        )+ |
+        [\x80-\xc1\xf5-\xff]       # invalid
+        )/nx) { |c|
+          c.size == 1 and raise "Invalid utf8 byte: '#{c}'"
+          s = c.encode("utf-16be", "utf-8").unpack('H*')[0]
+          s.force_encoding(::Encoding::ASCII_8BIT)
+          s.gsub!(/.{4}/n, '\\\\u\&')
+          s.force_encoding(::Encoding::UTF_8)
+        }
+    string.force_encoding(::Encoding::UTF_8)
+    text.replace string
+    text
+  end
+
+  private
+
+  MAP = {
+    "\x0" => '\u0000',
+    "\x1" => '\u0001',
+    "\x2" => '\u0002',
+    "\x3" => '\u0003',
+    "\x4" => '\u0004',
+    "\x5" => '\u0005',
+    "\x6" => '\u0006',
+    "\x7" => '\u0007',
+    "\xb" => '\u000b',
+    "\xe" => '\u000e',
+    "\xf" => '\u000f',
+    "\x10" => '\u0010',
+    "\x11" => '\u0011',
+    "\x12" => '\u0012',
+    "\x13" => '\u0013',
+    "\x14" => '\u0014',
+    "\x15" => '\u0015',
+    "\x16" => '\u0016',
+    "\x17" => '\u0017',
+    "\x18" => '\u0018',
+    "\x19" => '\u0019',
+    "\x1a" => '\u001a',
+    "\x1b" => '\u001b',
+    "\x1c" => '\u001c',
+    "\x1d" => '\u001d',
+    "\x1e" => '\u001e',
+    "\x1f" => '\u001f',
+  }
+
+  EMPTY_8BIT_STRING = ''
+  EMPTY_8BIT_STRING.force_encoding(::Encoding::ASCII_8BIT)
+
+end

data/lib/translatomatic/model/locale.rb

@@ -0,0 +1,22 @@
+module Translatomatic
+  module Model
+    class Locale < ActiveRecord::Base
+      has_many :texts, class_name: "Translatomatic::Model::Text"
+      validates_presence_of :language
+      validates_uniqueness_of :language, scope: [:script, :region]
+
+      class << self
+        include Translatomatic::Util
+      end
+
+      # create a locale record from an I18n::Locale::Tag object or string
+      def self.from_tag(tag)
+        tag = parse_locale(tag) if tag.kind_of?(String)
+        find_or_create_by!({
+          language: tag.language, script: tag.script, region: tag.region
+        })
+      end
+
+    end
+  end
+end

data/lib/translatomatic/model/text.rb

@@ -0,0 +1,13 @@
+module Translatomatic
+  module Model
+    class Text < ActiveRecord::Base
+      belongs_to :locale, class_name: "Translatomatic::Model::Locale"
+      belongs_to :from_text, class_name: "Translatomatic::Model::Text"
+      has_many :translations, class_name: "Translatomatic::Model::Text",
+        foreign_key: :from_text_id, dependent: :delete_all
+
+      validates_presence_of :value
+      validates_presence_of :locale
+    end
+  end
+end

data/lib/translatomatic/model.rb

@@ -0,0 +1,4 @@
+module Translatomatic::Model; end
+
+require 'translatomatic/model/locale'
+require 'translatomatic/model/text'

data/lib/translatomatic/option.rb

@@ -0,0 +1,24 @@
+module Translatomatic
+  class Option
+    attr_reader :name, :required, :use_env, :description
+
+    def initialize(data = {})
+      @name = data[:name]
+      @required = data[:required]
+      @use_env = data[:use_env]
+      @description = data[:desc]
+      @data = data
+    end
+
+    def to_hash
+      @data
+    end
+  end
+
+  module DefineOptions
+    private
+    def define_options(*options)
+      @options = options.collect { |i| Translatomatic::Option.new(i) }
+    end
+  end
+end

data/lib/translatomatic/resource_file/base.rb

@@ -0,0 +1,137 @@
+# @abstract Subclasses implement different types of resource files
+class Translatomatic::ResourceFile::Base
+
+  attr_accessor :locale
+  attr_accessor :path
+
+  # @return [Hash<String,String>] key -> value properties
+  attr_reader :properties
+
+  # Create a new resource file.
+  # If locale is unspecified, attempts to determine the locale of the file
+  # automatically, and if that fails, uses the default locale.
+  # @param [String] path Path to the file
+  # @param [String] locale Locale of the file contents
+  # @return [Translatomatic::ResourceFile::Base] the resource file.
+  def initialize(path, locale = nil)
+    @path = path.kind_of?(Pathname) ? path : Pathname.new(path)
+    @locale = locale || detect_locale || parse_locale(I18n.default_locale)
+    raise "unable to determine locale" unless @locale && @locale.language
+    @valid = false
+    @properties = {}
+  end
+
+  def format
+    self.class.name.demodulize.downcase.to_sym
+  end
+
+  # Create a path for the current resource file with a given locale
+  # @param [String] locale for the path
+  # @return [Pathname] The path of this resource file modified for the given locale
+  def locale_path(locale)
+    basename = path.sub_ext('').basename.to_s
+
+    extlist = extension_list
+    if extlist.length >= 2 && loc_idx = find_locale(extlist)
+      extlist[loc_idx] = locale.to_s
+    elsif valid_locale?(basename)
+      path.dirname + (locale.to_s + path.extname)
+    else
+      deunderscored = basename.sub(/_.*?$/, '')
+      filename = deunderscored + "_" + locale.to_s + path.extname
+      path.dirname + filename
+    end
+  end
+
+  # Set all properties
+  # @param [Hash<String,String>] properties New properties
+  def properties=(properties)
+    # use set rather that set @properties directly as subclasses override set()
+    properties.each do |key, value|
+      set(key, value)
+    end
+  end
+
+  # Get the value of a property
+  # @param [String] name The name of the property
+  # @return [String] The value of the property
+  def get(name)
+    @properties[name]
+  end
+
+  # Set a property
+  # @param [String] key The name of the property
+  # @param [String] value The new value of the property
+  # @return [String] The new value of the property
+  def set(name, value)
+    @properties[name] = value
+  end
+
+  # Test if the current resource file is valid
+  # @return true if the current file is valid
+  def valid?
+    @valid
+  end
+
+  # Save the resource file.
+  # @param [Pathname] target The destination path
+  # @return [void]
+  def save(target = path)
+    raise "save(path) must be implemented by subclass"
+  end
+
+  # @return [String] String representation of this file
+  def to_s
+    "#{path.basename.to_s} (#{locale})"
+  end
+
+  private
+
+  include Translatomatic::Util
+
+  # detect locale from filename
+  def detect_locale
+    tag = nil
+    basename = path.sub_ext('').basename.to_s
+    directory = path.dirname.basename.to_s
+    extlist = extension_list
+
+    if basename.match(/_([-\w]{2,})$/i)
+      # locale after underscore in filename
+      tag = $1
+    elsif directory.match(/^([-\w]+)\.lproj$/)
+      # xcode localized strings
+      tag = $1
+    elsif extlist.length >= 2 && loc_idx = find_locale(extlist)
+      # multiple parts to extension, e.g. index.html.en
+      tag = extlist[loc_idx]
+    elsif valid_locale?(basename)
+      # try to match on entire basename
+      # (support for rails en.yml)
+      tag = basename
+    end
+
+    tag ? parse_locale(tag, true) : nil
+  end
+
+  # test if the list of strings contains a valid locale
+  # return the index to the locale, or nil if no locales found
+  def find_locale(list)
+    list.find_index { |i| valid_locale?(i) }
+  end
+
+  # ext_sub() only removes the last extension
+  def strip_extensions
+    filename = path.basename.to_s
+    filename.sub!(/\..*$/, '')
+    path.parent + filename
+  end
+
+  # for index.html.de, returns ['html', 'de']
+  def extension_list
+    filename = path.basename.to_s
+    idx = filename.index('.')
+    idx && idx < filename.length - 1 ? filename[idx + 1..-1].split('.') : []
+  end
+
+end

data/lib/translatomatic/resource_file/html.rb

@@ -0,0 +1,33 @@
+module Translatomatic::ResourceFile
+  class HTML < XML
+
+    def self.extensions
+      %w{html htm shtml}
+    end
+
+    # (see Translatomatic::ResourceFile::Base#locale_path)
+    def locale_path(locale)
+      extlist = extension_list
+      if extlist.length >= 2 && loc_idx = find_locale(extlist)
+        # part of the extension is the locale
+        # replace that part with the new locale
+        extlist[loc_idx] = locale.to_s
+        new_extension = extlist.join(".")
+        return strip_extensions.sub_ext("." + new_extension)
+      else
+        # add locale extension
+        ext = path.extname
+        path.sub_ext("#{ext}." + locale.to_s)
+      end
+
+      # fall back to base functionality
+      #super(locale)
+    end
+
+    # (see Translatomatic::ResourceFile::Base#save(path))
+    def save(target = path)
+      target.write(@doc.to_html) if @doc
+    end
+
+  end
+end

data/lib/translatomatic/resource_file/plist.rb

@@ -0,0 +1,29 @@
+module Translatomatic::ResourceFile
+  class Plist < XML
+
+    def self.extensions
+      %w{plist}
+    end
+
+    # (see Translatomatic::ResourceFile::Base#locale_path)
+    # @note localization files in XCode use the following file name
+    #   convention: Project/locale.lproj/filename
+    # @todo refactor this and xcode_strings.rb to use the same code
+    def locale_path(locale)
+      if path.to_s.match(/\/([-\w]+).lproj\/.+.plist$/)
+        # xcode style
+        filename = path.basename
+        path.parent.parent + (locale.to_s + ".lproj") + filename
+      else
+        super(locale)
+      end
+    end
+
+    private
+
+    def text_nodes_xpath
+      '//*[not(self::key)]/text()'
+    end
+
+  end # class
+end   # module