translate-rails3-plus 0.0.1

data/lib/tasks/translate.rake

@@ -0,0 +1,178 @@
+require 'yaml'
+
+class Hash
+  def deep_merge(other)
+    # deep_merge by Stefan Rusterholz, see http://www.ruby-forum.com/topic/142809
+    merger = proc { |key, v1, v2| (Hash === v1 && Hash === v2) ? v1.merge(v2, &merger) : v2 }
+    merge(other, &merger)
+  end
+
+  def set(keys, value)
+    key = keys.shift
+    if keys.empty?
+      self[key] = value
+    else
+      self[key] ||= {}
+      self[key].set keys, value
+    end
+  end
+
+  if ENV['SORT']
+    # copy of ruby's to_yaml method, prepending sort.
+    # before each so we get an ordered yaml file
+    def to_yaml( opts = {} )
+      YAML::quick_emit( self, opts ) do |out|
+        out.map( taguri, to_yaml_style ) do |map|
+          sort.each do |k, v| #<- Adding sort.
+            map.add( k, v )
+          end
+        end
+      end
+    end
+  end
+end
+
+namespace :translate do
+  desc "Show untranslated keys for locale LOCALE"
+  task :untranslated => :environment do
+    from_locale = I18n.default_locale
+    untranslated = Translate::Keys.new.untranslated_keys
+
+    messages = []
+    untranslated.each do |locale, keys|
+      keys.each do |key|
+        from_text = I18n.backend.send(:lookup, from_locale, key)
+        messages << "#{locale}.#{key} (#{from_locale}.#{key}='#{from_text}')"
+      end
+    end
+      
+    if messages.present?
+      messages.each { |m| puts m }
+    else
+      puts "No untranslated keys"
+    end
+  end
+  
+  desc "Show I18n keys that are missing in the config/locales/default_locale.yml YAML file"
+  task :missing => :environment do
+    missing = Translate::Keys.new.missing_keys.inject([]) do |keys, (key, filename)|
+      keys << "#{key} in \t  #{filename} is missing"
+    end
+    puts missing.present? ? missing.join("\n") : "No missing translations in the default locale file"
+  end
+
+  desc "Remove all translation texts that are no longer present in the locale they were translated from"
+  task :remove_obsolete_keys => :environment do
+    I18n.backend.send(:init_translations)
+    master_locale = ENV['LOCALE'] || I18n.default_locale
+    Translate::Keys.translated_locales.each do |locale|
+      texts = {}
+      Translate::Keys.new.i18n_keys(locale).each do |key|
+        if I18n.backend.send(:lookup, master_locale, key).to_s.present?
+          texts[key] = I18n.backend.send(:lookup, locale, key)
+        end
+      end
+      I18n.backend.send(:translations)[locale] = nil # Clear out all current translations
+      I18n.backend.store_translations(locale, Translate::Keys.to_deep_hash(texts))
+      Translate::Storage.new(locale).write_to_file      
+    end
+  end
+
+  desc "Merge I18n keys from log/translations.yml into config/locales/*.yml (for use with the Rails I18n TextMate bundle)"
+  task :merge_keys => :environment do
+    I18n.backend.send(:init_translations)
+    new_translations = YAML::load(IO.read(File.join(Rails.root, "log", "translations.yml")))
+    raise("Can only merge in translations in single locale") if new_translations.keys.size > 1
+    locale = new_translations.keys.first
+
+    overwrites = false
+    Translate::Keys.to_shallow_hash(new_translations[locale]).keys.each do |key|
+      new_text = key.split(".").inject(new_translations[locale]) { |hash, sub_key| hash[sub_key] }
+      existing_text = I18n.backend.send(:lookup, locale.to_sym, key)
+      if existing_text && new_text != existing_text        
+        puts "ERROR: key #{key} already exists with text '#{existing_text.inspect}' and would be overwritten by new text '#{new_text}'. " +
+          "Set environment variable OVERWRITE=1 if you really want to do this."
+        overwrites = true
+      end
+    end
+
+    if !overwrites || ENV['OVERWRITE']
+      I18n.backend.store_translations(locale, new_translations[locale])
+      Translate::Storage.new(locale).write_to_file
+    end
+  end
+  
+  desc "Apply Google translate to auto translate all texts in locale ENV['FROM'] to locale ENV['TO']"
+  task :google => :environment do
+    raise "Please specify FROM and TO locales as environment variables" if ENV['FROM'].blank? || ENV['TO'].blank?
+
+    # Depends on httparty gem
+    # http://www.robbyonrails.com/articles/2009/03/16/httparty-goes-foreign
+    class GoogleApi
+      include HTTParty
+      base_uri 'ajax.googleapis.com'
+      def self.translate(string, to, from)
+        tries = 0
+        begin
+          get("/ajax/services/language/translate",
+            :query => {:langpair => "#{from}|#{to}", :q => string, :v => 1.0},
+            :format => :json)
+        rescue 
+          tries += 1
+          puts("SLEEPING - retrying in 5...")
+          sleep(5)
+          retry if tries < 10
+        end
+      end
+    end
+
+    I18n.backend.send(:init_translations)
+
+    start_at = Time.now
+    translations = {}
+    Translate::Keys.new.i18n_keys(ENV['FROM']).each do |key|
+      from_text = I18n.backend.send(:lookup, ENV['FROM'], key).to_s
+      to_text = I18n.backend.send(:lookup, ENV['TO'], key)
+      if !from_text.blank? && to_text.blank?
+        print "#{key}: '#{from_text[0, 40]}' => "
+        if !translations[from_text]
+          response = GoogleApi.translate(from_text, ENV['TO'], ENV['FROM'])
+          translations[from_text] = response["responseData"] && response["responseData"]["translatedText"]
+        end
+        if !(translation = translations[from_text]).blank?
+          translation.gsub!(/\(\(([a-z_.]+)\)\)/i, '{{\1}}')
+          # Google translate sometimes replaces {{foobar}} with (()) foobar. We skip these
+          if translation !~ /\(\(\)\)/
+            puts "'#{translation[0, 40]}'"
+            I18n.backend.store_translations(ENV['TO'].to_sym, Translate::Keys.to_deep_hash({key => translation}))
+          else
+            puts "SKIPPING since interpolations were messed up: '#{translation[0,40]}'"
+          end
+        else
+          puts "NO TRANSLATION - #{response.inspect}"
+        end
+      end
+    end
+    
+    puts "\nTime elapsed: #{(((Time.now - start_at) / 60) * 10).to_i / 10.to_f} minutes"    
+    Translate::Storage.new(ENV['TO'].to_sym).write_to_file
+  end
+
+  desc "List keys that have changed I18n texts between YAML file ENV['FROM_FILE'] and YAML file ENV['TO_FILE']. Set ENV['VERBOSE'] to see changes"
+  task :changed => :environment do
+    from_hash = Translate::Keys.to_shallow_hash(Translate::File.new(ENV['FROM_FILE']).read)
+    to_hash = Translate::Keys.to_shallow_hash(Translate::File.new(ENV['TO_FILE']).read)
+    from_hash.each do |key, from_value|
+      if (to_value = to_hash[key]) && to_value != from_value
+        key_without_locale = key[/^[^.]+\.(.+)$/, 1]
+        if ENV['VERBOSE']
+          puts "KEY: #{key_without_locale}"
+          puts "FROM VALUE: '#{from_value}'"
+          puts "TO VALUE: '#{to_value}'"
+        else
+          puts key_without_locale
+        end
+      end      
+    end
+  end
+end

data/lib/translate/file.rb

@@ -0,0 +1,35 @@
+require 'fileutils'
+
+class Translate::File
+  attr_accessor :path
+  
+  def initialize(path)
+    self.path = path
+  end
+  
+  def write(keys)
+    FileUtils.mkdir_p File.dirname(path)
+    File.open(path, "w") do |file|
+      file.puts keys_to_yaml(Translate::File.deep_stringify_keys(keys))
+    end    
+  end
+  
+  def read
+    File.exists?(path) ? YAML::load(IO.read(path)) : {}
+  end
+
+  # Stringifying keys for prettier YAML
+  def self.deep_stringify_keys(hash)
+    hash.inject({}) { |result, (key, value)|
+      value = deep_stringify_keys(value) if value.is_a? Hash
+      result[(key.to_s rescue key) || key] = value
+      result
+    }
+  end
+  
+  private
+  def keys_to_yaml(keys)
+    # Using ya2yaml, if available, for UTF8 support
+    keys.respond_to?(:ya2yaml) ? keys.ya2yaml(:escape_as_utf8 => true) : keys.to_yaml
+  end    
+end

data/lib/translate/keys.rb

@@ -0,0 +1,185 @@
+require 'pathname'
+
+class Translate::Keys
+  # Allows keys extracted from lookups in files to be cached
+  def self.files
+    @@files ||= Translate::Keys.new.files
+  end
+
+  # Allows flushing of the files cache
+  def self.files=(files)
+    @@files = files
+  end
+
+  def files
+    @files ||= extract_files
+  end
+  alias_method :to_hash, :files
+
+  def keys
+    files.keys
+  end
+  alias_method :to_a, :keys
+
+  def i18n_keys(locale)
+    I18n.backend.send(:init_translations) unless I18n.backend.initialized?
+    Translate::Keys.to_shallow_hash(I18n.backend.send(:translations)[locale.to_sym]).keys.sort
+  end
+
+  def untranslated_keys
+    Translate::Keys.translated_locales.inject({}) do |missing, locale|
+      missing[locale] = i18n_keys(I18n.default_locale).map do |key|
+        I18n.backend.send(:lookup, locale, key).nil? ? key : nil
+      end.compact
+      missing
+    end
+  end
+
+  def missing_keys
+    locale = I18n.default_locale; yaml_keys = {}
+    yaml_keys = Translate::Storage.file_paths(locale).inject({}) do |keys, path|
+      keys = keys.deep_merge(Translate::File.new(path).read[locale.to_s])
+    end
+    files.reject { |key, file| self.class.contains_key?(yaml_keys, key) }
+  end
+
+  def self.translated_locales
+    I18n.available_locales.reject { |locale| [:root, I18n.default_locale.to_sym].include?(locale) }
+  end
+
+  # Checks if a nested hash contains the keys in dot separated I18n key.
+  #
+  # Example:
+  #
+  # hash = {
+  #   :foo => {
+  #     :bar => {
+  #       :baz => 1
+  #     }
+  #   }
+  # }
+  #
+  # contains_key?("foo", key) # => true
+  # contains_key?("foo.bar", key) # => true
+  # contains_key?("foo.bar.baz", key) # => true
+  # contains_key?("foo.bar.baz.bla", key) # => false
+  #
+  def self.contains_key?(hash, key)
+    keys = key.to_s.split(".")
+    return false if keys.empty?
+    !keys.inject(HashWithIndifferentAccess.new(hash)) do |memo, key|
+      memo.is_a?(Hash) ? memo.try(:[], key) : nil
+    end.nil?
+  end
+
+  # Convert something like:
+  #
+  # {
+  #  :pressrelease => {
+  #    :label => {
+  #      :one => "Pressmeddelande"
+  #    }
+  #   }
+  # }
+  #
+  # to:
+  #
+  #  {'pressrelease.label.one' => "Pressmeddelande"}
+  #
+  def self.to_shallow_hash(hash)
+    hash.inject({}) do |shallow_hash, (key, value)|
+      if value.is_a?(Hash)
+        to_shallow_hash(value).each do |sub_key, sub_value|
+          shallow_hash[[key, sub_key].join(".")] = sub_value
+        end
+      else
+        shallow_hash[key.to_s] = value
+      end
+      shallow_hash
+    end
+  end
+
+  # Convert something like:
+  #
+  #  {'pressrelease.label.one' => "Pressmeddelande"}
+  #
+  # to:
+  #
+  # {
+  #  :pressrelease => {
+  #    :label => {
+  #      :one => "Pressmeddelande"
+  #    }
+  #   }
+  # }
+  def self.to_deep_hash(hash)
+    hash.inject({}) do |deep_hash, (key, value)|
+      keys = key.to_s.split('.').reverse
+      leaf_key = keys.shift
+      key_hash = keys.inject({leaf_key.to_sym => value}) { |hash, key| {key.to_sym => hash} }
+      deep_merge!(deep_hash, key_hash)
+      deep_hash
+    end
+  end
+
+  # deep_merge by Stefan Rusterholz, see http://www.ruby-forum.com/topic/142809
+  def self.deep_merge!(hash1, hash2)
+    merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+    hash1.merge!(hash2, &merger)
+  end
+
+	# Convert something like:
+	#
+	# {'0' => "elem 1", '1' => "elem 2"}
+	#
+	# to:
+	#
+	# ["elem 1", "elem 2"]
+	#
+	def self.arraylize(input_hash)
+		input_hash.inject([]) do |constructed_array, (key, value)|
+			constructed_array << value
+			constructed_array
+		end
+	end
+
+  private
+
+  def extract_files
+    files_to_scan.inject(HashWithIndifferentAccess.new) do |files, file|
+      keys = IO.read(file)
+      if keys.respond_to? "encode"
+        keys = keys.encode("UTF-8").force_encoding("UTF-8")
+      end
+      error_count = 0
+      begin
+        encoded_keys = keys.scan(i18n_lookup_pattern)
+      rescue => e
+        unless error_count > 1
+          if keys.respond_to? 'encode!'
+            keys.encode!('utf-8', 'utf-8', :invalid => :replace)
+          end
+          error_count += 1
+          retry
+        else
+          puts "cannot fix: #{e} on : #{file}"
+        end
+      end
+      encoded_keys.flatten.map(&:to_sym).each do |key|
+        files[key] ||= []
+        path = Pathname.new(File.expand_path(file)).relative_path_from(Pathname.new(Rails.root)).to_s
+        files[key] << path if !files[key].include?(path)
+      end
+      files
+    end
+  end
+
+  def i18n_lookup_pattern
+    /\b(?:I18n\.t|I18n\.translate|t)(?:\s|\():?'([a-z0-9_]+.[a-z0-9_.]+)'\)?/
+  end
+
+  def files_to_scan
+    Dir.glob(File.join(Translate::Storage.root_dir, "{app,config,lib}", "**","*.{rb,erb,rhtml}")) +
+      Dir.glob(File.join(Translate::Storage.root_dir, "public", "javascripts", "**","*.js"))
+  end
+end

data/lib/translate/log.rb

@@ -0,0 +1,35 @@
+class Translate::Log
+  attr_accessor :from_locale, :to_locale, :keys
+  
+  def initialize(from_locale, to_locale, keys)
+    self.from_locale = from_locale
+    self.to_locale = to_locale
+    self.keys = keys
+  end
+  
+  def write_to_file
+    current_texts = File.exists?(file_path) ? file.read : {}
+    current_texts.merge!(from_texts)
+    file.write(current_texts)
+  end
+
+  def read
+    file.read
+  end
+
+  private
+  def file
+    @file ||= Translate::File.new(file_path)    
+  end
+  
+  def from_texts
+    Translate::File.deep_stringify_keys(Translate::Keys.to_deep_hash(keys.inject({}) do |hash, key|
+      hash[key] = I18n.backend.send(:lookup, from_locale, key)
+      hash
+    end))
+  end
+  
+  def file_path
+    File.join(Rails.root, "config", "locales", "log", "from_#{from_locale}_to_#{to_locale}.yml")
+  end
+end

data/lib/translate/routes.rb

@@ -0,0 +1,11 @@
+module Translate
+  class Routes
+    def self.translation_ui(map)
+      map.with_options(:controller => 'translate') do |t|
+        t.translate_list 'translate'
+        t.translate 'translate/translate', :action => 'translate'
+        t.translate_reload 'translate/reload', :action => 'reload'
+      end
+    end
+  end
+end

data/lib/translate/storage.rb

@@ -0,0 +1,28 @@
+class Translate::Storage
+  attr_accessor :locale
+  
+  def initialize(locale)
+    self.locale = locale.to_sym
+  end
+  
+  def write_to_file
+    Translate::File.new(file_path).write(keys)
+  end
+  
+  def self.file_paths(locale)
+    Dir.glob(File.join(root_dir, "config", "locales", "**","#{locale}.yml"))
+  end
+  
+  def self.root_dir
+    Rails.root
+  end
+  
+  private
+  def keys
+    {locale => I18n.backend.send(:translations)[locale]}
+  end
+  
+  def file_path
+    File.join(Translate::Storage.root_dir, "config", "locales", "#{locale}.yml")
+  end
+end

data/lib/translate.rb

@@ -0,0 +1,17 @@
+require 'ya2yaml'
+
+module Translate
+  class Engine < Rails::Engine
+  end if defined?(Rails) && Rails::VERSION::MAJOR == 3
+
+  class << self
+    # For configuring Google Translate API key
+    attr_accessor :api_key
+    # For configuring Bing Application id
+    attr_accessor :app_id
+  end
+end
+
+Dir[File.join(File.dirname(__FILE__), "translate", "*.rb")].each do |file|
+  require file
+end

data/spec/controllers/translate_controller_spec.rb

@@ -0,0 +1,130 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe TranslateController do
+  describe "index" do
+    before(:each) do
+      controller.stub!(:per_page).and_return(1)
+      I18n.backend.stub!(:translations).and_return(i18n_translations)
+      I18n.backend.instance_eval { @initialized = true }
+      keys = mock(:keys)
+      keys.stub!(:i18n_keys).and_return(['vendor.foobar'])
+      Translate::Keys.should_receive(:new).and_return(keys)
+      Translate::Keys.should_receive(:files).and_return(files)
+      I18n.stub!(:available_locales).and_return([:en, :sv])
+      I18n.stub!(:default_locale).and_return(:sv)
+    end
+    
+    it "shows sorted paginated keys from the translate from locale and extracted keys by default" do
+      get_page :index
+      assigns(:from_locale).should == :sv
+      assigns(:to_locale).should == :en
+      assigns(:files).should == files
+      assigns(:keys).sort.should == ['articles.new.page_title', 'home.page_title', 'vendor.foobar']
+      assigns(:paginated_keys).should == ['articles.new.page_title']
+    end
+
+    it "can be paginated with the page param" do
+      get_page :index, :page => 2
+      assigns(:files).should == files
+      assigns(:paginated_keys).should == ['home.page_title']      
+    end
+    
+    it "accepts a key_pattern param with key_type=starts_with" do
+      get_page :index, :key_pattern => 'articles', :key_type => 'starts_with'
+      assigns(:files).should == files
+      assigns(:paginated_keys).should == ['articles.new.page_title']
+      assigns(:total_entries).should == 1
+    end
+
+    it "accepts a key_pattern param with key_type=contains" do
+      get_page :index, :key_pattern => 'page_', :key_type => 'contains'
+      assigns(:files).should == files
+      assigns(:total_entries).should == 2
+      assigns(:paginated_keys).should == ['articles.new.page_title']
+    end
+
+    it "accepts a filter=untranslated param" do
+      get_page :index, :filter => 'untranslated'
+      assigns(:total_entries).should == 2
+      assigns(:paginated_keys).should == ['articles.new.page_title']
+    end
+    
+    it "accepts a filter=translated param" do
+      get_page :index, :filter => 'translated'
+      assigns(:total_entries).should == 1
+      assigns(:paginated_keys).should == ['vendor.foobar']
+    end
+    
+    it "accepts a filter=changed param" do
+      log = mock(:log)
+      old_translations = {:home => {:page_title => "Skapar ny artikel"}}
+      log.should_receive(:read).and_return(Translate::File.deep_stringify_keys(old_translations))
+      Translate::Log.should_receive(:new).with(:sv, :en, {}).and_return(log)      
+      get_page :index, :filter => 'changed'
+      assigns(:total_entries).should == 1
+      assigns(:keys).should == ["home.page_title"]
+    end
+
+    def i18n_translations
+      HashWithIndifferentAccess.new({
+        :en => {
+          :vendor => {
+            :foobar => "Foo Baar"
+          }
+        },
+        :sv => {
+          :articles => {
+            :new => {
+              :page_title => "Skapa ny artikel"
+            }
+          },
+          :home => {
+            :page_title => "Välkommen till I18n"
+          },
+          :vendor => {
+            :foobar => "Fobar"
+          }
+        }
+      })
+    end
+    
+    def files
+      HashWithIndifferentAccess.new({
+        :'home.page_title' => ["app/views/home/index.rhtml"],
+        :'general.back' => ["app/views/articles/new.rhtml", "app/views/categories/new.rhtml"],
+        :'articles.new.page_title' => ["app/views/articles/new.rhtml"]
+      })
+    end
+  end
+  
+  describe "translate" do
+    it "should store translations to I18n backend and then write them to a YAML file" do
+      session[:from_locale] = :sv
+      session[:to_locale] = :en
+      translations = {
+        :articles => {
+          :new => {
+            :title => "New Article"
+          }
+        },
+        :category => "Category"
+      }
+      key_param = {'articles.new.title' => "New Article", "category" => "Category"}
+      I18n.backend.should_receive(:store_translations).with(:en, translations)
+      storage = mock(:storage)
+      storage.should_receive(:write_to_file)
+      Translate::Storage.should_receive(:new).with(:en).and_return(storage)
+      log = mock(:log)
+      log.should_receive(:write_to_file)
+      Translate::Log.should_receive(:new).with(:sv, :en, key_param.keys).and_return(log)
+      post :translate, "key" => key_param
+      response.should be_redirect
+    end
+  end
+  
+  def get_page(*args)
+    get(*args)
+    response.should be_success
+  end
+end

data/spec/file_spec.rb

@@ -0,0 +1,54 @@
+require 'fileutils'
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe Translate::File do
+  describe "write" do
+    before(:each) do
+      @file = Translate::File.new(file_path)
+    end
+
+    after(:each) do
+      FileUtils.rm(file_path)
+    end
+    
+    it "writes all I18n messages for a locale to YAML file" do
+      @file.write(translations)
+      @file.read.should == Translate::File.deep_stringify_keys(translations)
+    end
+
+    def translations
+      {
+        :en => {
+          :article => {
+            :title => "One Article"
+          },
+          :category => "Category"
+        }
+      }
+    end
+  end
+  
+  describe "deep_stringify_keys" do
+    it "should convert all keys in a hash to strings" do
+      Translate::File.deep_stringify_keys({
+        :en => {
+          :article => {
+            :title => "One Article"
+          },
+          :category => "Category"
+        }
+      }).should == {
+        "en" => {
+          "article" => {
+            "title" => "One Article"
+          },
+          "category" => "Category"
+        }
+      }
+    end
+  end
+
+  def file_path
+    File.join(File.dirname(__FILE__), "files", "en.yml")
+  end  
+end

data/spec/files/translate/app/models/article.rb

@@ -0,0 +1,12 @@
+class Article < ActiveRecord::Base
+  def validate
+    # t('li')
+    errors.add_to_base([t(:'article.key1') + "#{t('article.key2')}"])
+    I18n.t 'article.key3'
+    I18n.t 'article.key3'
+    I18n.t :'article.key4'
+    I18n.translate :'article.key5'
+    'bla bla t' + "blubba bla" + ' foobar'
+    'bla bla t ' + "blubba bla" + ' foobar'
+  end
+end

data/spec/files/translate/app/views/category.erb

@@ -0,0 +1 @@
+<%= t(:'category_erb.key1') %>

data/spec/files/translate/app/views/category.html

@@ -0,0 +1 @@
+t(:'category_html.key1')