i18n_googledocs 0.1.4

data/lib/generators/i18n_docs/normalize_generator.rb

@@ -0,0 +1,157 @@
+require 'pathname'
+require 'fileutils'
+
+module I18nDocs
+  module Generators
+    class NormalizeGenerator < Rails::Generators::Base
+      desc "Normalize locale files by converting tabs to spaces"
+
+      class_option  :spaces,  :type => :string, :default => '2', 
+          :desc => "Spaces for each tab"
+
+      class_option  :overwrite,  :type => :boolean, :default => false, 
+          :desc => "Overwrite existing locale files?"
+
+      class_option  :delete,  :type => :boolean, :default => false, 
+          :desc => "Delete old normalized files?"
+
+      class_option  :accept,  :type => :boolean, :default => false, 
+          :desc => "Accept normalized files"
+
+      class_option  :debug,  :type => :boolean, :default => false, 
+          :desc => "Turn debug mode on?"
+
+      argument      :locales, :type => :array,  :default => [], 
+          :desc => "List of locales to normalize"
+
+      def main
+        delete_normalized and return if delete?
+        accept_normalized and return if accept?
+        tabs_to_spaces
+      end
+                    
+      protected
+
+      def delete?
+        options[:delete]
+      end
+
+      def debug?
+        options[:delete]
+      end
+
+      def accept?
+        options[:accept]
+      end
+
+      def delete_normalized
+        locales.each {|locale| delete_for locale }        
+      end
+
+      def delete_for locale = :all
+        path = locale_path(locale)
+        say "Deleting normalized files for: #{locale}"
+        normalized_files(path).each do |file|
+          say "Deleting: #{file}" if debug?
+          File.delete file
+        end       
+      end
+
+      def accept_normalized
+        locales.each {|locale| accept_for locale }        
+      end
+
+      def accept_for locale = :all
+        path = locale_path(locale)
+        say "Accepting normalized files for: #{locale}"
+        normalized_files(path).each do |file|          
+          new_file_name = File.basename(file).gsub /^_/, ''
+          file_path = File.join(File.dirname(file), new_file_name)          
+          FileUtils.mv file, file_path
+
+          say "Accepted for: #{new_file_name}" if debug?
+        end       
+      end
+
+
+      def tabs_to_spaces
+        say "Normalizing tabs for locales: #{locales} - with #{spaces} spaces"
+        locales.empty? ? normalize_for(:all) : for_locales
+      end
+
+      def for_locales
+        locales.each {|locale| normalize_for locale }
+      end        
+
+      def normalize_for locale = :en
+        path = locale_path(locale)
+        replacement = spaces_pr_tab
+        say "Normalizing for: #{locale}" if debug?
+        say "In folder: #{path}" if debug?
+
+        files(path).each do |file|
+          normalize_file_content file
+        end
+        say "Normalize completed", :green        
+      end        
+
+      def locale_path locale
+        (locale != :all) ? File.join(locales_root, locale) : locales_root
+      end
+
+      def normalize_file_content file
+        say "Normalizing file: #{file}" if debug?
+
+        content = File.open(file).read
+        replaced_content = content.gsub /\t/, spaces_pr_tab
+        replaced_content = content.gsub /^---/, ''        
+        replaced_content = content.gsub /^no:/, "'no':"
+
+        File.open(new_file(file), 'w') do |f|
+          f.puts replaced_content
+        end
+      end        
+
+      def new_file file
+        overwrite? ? file : normalized_file_name(file)
+      end
+
+      def normalized_file_name file_name
+        new_file_name = '_' + File.basename(file_name)
+        File.join(File.dirname(file_name), new_file_name)          
+      end
+
+      def overwrite?
+        options[:overwrite]
+      end
+
+      def files path
+        Dir[File.join(path,'*.yml')]
+      end
+
+      def normalized_files path
+        Dir[File.join(path,'_*.yml')]
+      end
+
+      def unnormalized_files path
+        Dir[File.join(path,'[^_]*.yml')]
+      end
+
+      def spaces_pr_tab
+        @spaces ||= (1..spaces).to_a.inject("") {|res, e| res << ' ' }
+      end
+
+      def spaces
+        options[:spaces].to_i
+      end
+
+      def num_spaces
+        spaces.to_i
+      end
+
+      def locales_root
+        Rails.root.join 'config', 'locales'
+      end
+    end
+  end
+end

data/lib/generators/i18n_docs/templates/locale.erb

@@ -0,0 +1,2 @@
+<%= locale %>:
+  <%= key %>: <%= text %>

data/lib/i18n_googledocs.rb

@@ -0,0 +1,51 @@
+# require 'rake'
+
+# load rake tasks in case GEM is included within rails project
+
+require 'faster_csv'
+require 'yaml'
+require 'open-uri'
+require 'localch_i18n/util'
+require 'localch_i18n/missing_keys_finder'
+require 'localch_i18n/csv_to_yaml'
+require 'localch_i18n/translations'
+require 'localch_i18n/translation_file_export'
+require 'localch_i18n/translator'
+
+module I18nDocs
+	class << self
+		attr_accessor :debug
+
+		def debug?
+			@debug
+		end
+
+		def debug!
+			@debug = true
+		end
+
+		def add_locale_paths_for locales
+			locales.each do |locale|
+				path = Rails.root.join('config', 'locales', locale.to_s, '*.yml')
+				puts "Adding locale path: #{path}" if debug?
+			  I18n.load_path += Dir[path]
+			end
+		end
+	end
+end
+
+
+module I18n
+	class << self
+		attr_accessor :google_translation_key
+	end	
+end
+
+if defined?(Rails)
+  class LocalchI18nTask < Rails::Railtie
+    rake_tasks do
+      Dir[File.join(File.dirname(__FILE__),'tasks/*.rake')].each { |f| load f }
+    end
+  end
+end
+

data/lib/localch_i18n/csv_to_yaml.rb

@@ -0,0 +1,68 @@
+module LocalchI18n
+
+  class CsvToYaml
+
+    attr_reader :input_file, :output_file, :locales, :translations
+
+    def initialize(input_file, output_file, locales = [])
+      @input_file = input_file
+      @output_file = File.basename(output_file)
+      @locales = locales.map(&:to_s)
+
+      # init translation hash
+      @translations = {}
+      @locales.each do |locale|
+        @translations[locale] = {}
+      end
+    end
+
+
+    def write_files
+      @locales.each do |locale|
+        output_file_path = defined?(Rails) ? Rails.root.join('config', 'locales', locale, @output_file) : "#{locale}_#{@output_file}"
+        File.open(output_file_path, 'w') do |file|
+          final_translation_hash = {locale => @translations[locale]}
+          file.puts YAML::dump(final_translation_hash)
+        end
+        puts "File '#{@output_file}' for language '#{locale}' written to disc (#{output_file_path})"
+      end
+    end
+
+
+    def process
+      FasterCSV.foreach(@input_file, :headers => true) do |row|
+        process_row(row.to_hash)
+      end
+    end
+
+    def process_row(row_hash)
+      key = row_hash.delete('key')
+
+      key_elements = key.split('.')
+      @locales.each do |locale|
+        raise "Locale missing for key #{key}! (locales in app: #{@locales} / locales in file: #{row_hash.keys.to_s})" if !row_hash.has_key?(locale)
+        store_translation(key_elements, locale, row_hash[locale])
+      end
+    end
+
+
+    def store_translation(keys, locale, value)
+      return nil if value.nil?    # we don't store keys that don't have a valid value
+      # Google Spreadsheet does not export empty strings and therefore we use '_' as a replacement char.
+      value = '' if value == '_'
+
+      tree = keys[0...-1]
+      leaf = keys.last
+      data_hash = tree.inject(@translations[locale]) do |memo, k|
+        if memo.has_key?(k)
+          memo[k]
+        else
+          memo[k] = {}
+        end
+      end
+      data_hash[leaf] = value
+    end
+
+  end
+
+end

data/lib/localch_i18n/missing_keys_finder.rb

@@ -0,0 +1,111 @@
+module LocalchI18n
+  class MissingKeysFinder
+    def initialize(backend)
+      @backend = backend
+      self.load_config
+      self.load_translations
+    end
+
+    # Returns an array with all keys from all locales
+    def all_keys
+      I18n.backend.send(:translations).collect do |check_locale, translations|
+        collect_keys([], translations).sort
+      end.flatten.uniq
+    end
+
+    def find_missing_keys
+      output_available_locales
+      output_unique_key_stats(all_keys)
+
+      missing_keys = {}
+      all_keys.each do |key|
+
+        I18n.available_locales.each do |locale|
+
+          skip = false
+          ls = locale.to_s
+          if !@yaml[ls].nil?
+            @yaml[ls].each do |re|
+              if key.match(re)
+                skip = true
+                break
+              end
+            end
+          end
+
+          if !key_exists?(key, locale) && skip == false
+            if missing_keys[key]
+              missing_keys[key] << locale
+            else
+              missing_keys[key] = [locale]
+            end
+          end
+        end
+      end
+
+      output_missing_keys(missing_keys)
+      return missing_keys
+    end
+
+    def output_available_locales
+      puts "#{I18n.available_locales.size} #{I18n.available_locales.size == 1 ? 'locale' : 'locales'} available: #{I18n.available_locales.join(', ')}"
+    end
+
+    def output_missing_keys(missing_keys)
+      if missing_keys.size > 0
+        puts "#{missing_keys.size} #{missing_keys.size == 1 ? 'key is missing' : 'keys are missing'} from one or more locales:"
+        missing_keys.keys.sort.each do |key|
+          puts "'#{key}': Missing from #{missing_keys[key].collect(&:inspect).join(', ')}"
+        end
+        puts "\nERROR: #{missing_keys.size} #{missing_keys.size == 1 ? 'key is missing' : 'keys are missing'} from one or more locales."
+      else
+        puts "No keys are missing"
+      end
+    end
+
+    def output_unique_key_stats(keys)
+      number_of_keys = keys.size
+      puts "#{number_of_keys} #{number_of_keys == 1 ? 'unique key' : 'unique keys'} found."
+    end
+
+    def collect_keys(scope, translations)
+      full_keys = []
+      translations.to_a.each do |key, translations|
+        new_scope = scope.dup << key
+        if translations.is_a?(Hash)
+          full_keys += collect_keys(new_scope, translations)
+        else
+          full_keys << new_scope.join('.')
+        end
+      end
+      return full_keys
+    end
+
+    # Returns true if key exists in the given locale
+    def key_exists?(key, locale)
+      I18n.locale = locale
+      I18n.translate(key, :raise => true)
+      return true
+    rescue I18n::MissingInterpolationArgument
+      return true
+    rescue I18n::MissingTranslationData
+      return false
+    end
+
+    def load_translations
+      # Make sure we’ve loaded the translations
+      I18n.backend.send(:init_translations)
+    end
+
+    def load_config
+      @yaml = {}
+      begin
+        @yaml = YAML.load_file(File.join(Rails.root, 'config', 'ignore_missing_i18n_keys.yml'))
+      rescue => e
+        STDERR.puts "No ignore_missing_keys.yml config file."
+      end
+
+    end
+
+  end
+end

data/lib/localch_i18n/translation_file_export.rb

@@ -0,0 +1,76 @@
+module LocalchI18n
+  class TranslationFileExport
+    include LocalchI18n::Util
+    
+    attr_accessor :translations
+    attr_reader :main_locale, :current_locale
+    
+    def initialize(source_dir, source_file, output_dir, locales, options = {})
+      @source_dir = source_dir
+      @source_file = source_file
+      @auto_translate = options[:auto_translate]
+
+      @output_file = File.join(output_dir, source_file.gsub('.yml', '.csv'))
+      @locales = locales.map {|l| l.to_s.downcase }
+      
+      @translations = {}
+    end
+    
+    
+    def export
+      load_translations
+      write_to_csv
+    end
+        
+    def write_to_csv
+      @main_locale = main_locale = @locales.include?('en') ? 'en' : @locales.first
+      
+      puts "    #{@source_file}: write CSV to '#{@output_file}' \n\n"
+      
+      FasterCSV.open(@output_file, "wb") do |csv|
+        csv << (["key"] + @locales)
+                
+        if @translations.empty? || !@translations[main_locale] || @translations[main_locale].keys.empty?
+          puts %Q{Translations #{@source_file} for #{main_locale} could not be processed, likely due to a YAML syntax error. 
+Please try again with the --normalize option. 
+
+The problem could also be due to an invalid locale code. Please check the i18n.available_locales setting in config/application.rb} 
+          exit(0)
+        end
+
+        @translations[main_locale].keys.each do |key|
+          values = @locales.map do |locale|
+            @translations[locale][key] if @translations[locale]
+          end
+          csv << values.unshift(key)
+        end
+      end
+      
+    end
+    
+    
+    def load_translations
+      @locales.each do |locale|
+        @current_locale = locale
+        translation_hash = load_language(locale)
+        unless translation_hash.blank?
+          @translations[locale] = flatten_translations_hash(translation_hash)
+        else
+          puts "Error: No translations for locale - #{locale}"
+        end
+      end
+    end
+    
+    def load_language(locale)
+      
+      puts "    #{@source_file}: load translations for '#{locale}'"
+      
+      input_file = File.join(@source_dir, locale, @source_file)
+      
+      # puts "  input file: #{input_file}"
+      load_translations_for input_file, locale
+    end
+    
+  end
+  
+end

data/lib/localch_i18n/translations.rb

@@ -0,0 +1,70 @@
+#
+# Order of method calls
+#  download_files
+#  store_translations
+#  clean_up
+#
+module LocalchI18n
+  class Translations
+    
+    attr_accessor :locales, :tmp_folder, :config_file, :csv_files
+    
+    def initialize(config_file = nil, tmp_folder = nil)
+      @config_file = config_file 
+      @tmp_folder  = tmp_folder
+      
+      @csv_files = {}
+      
+      load_config
+      load_locales
+    end
+    
+    def load_locales
+      @locales = []
+      @locales = I18n.available_locales if defined?(I18n)
+    end
+    
+    def load_config
+      @settings = {}
+      @settings = YAML.load_file(config_file) if File.exists?(config_file)
+    end
+    
+    def download_files
+      files = @settings['files']
+      files.each do |target_file, url|
+        # download file to tmp directory
+        tmp_file = File.basename(target_file).gsub('.yml', '.csv')
+        tmp_file = File.join(@tmp_folder, tmp_file)
+        download(url, tmp_file)
+        @csv_files[target_file] = tmp_file
+      end
+    end
+    
+    def store_translations
+      @csv_files.each do |target_file, csv_file|
+        converter = CsvToYaml.new(csv_file, target_file, @locales)
+        converter.process
+        converter.write_files
+      end
+      
+      @csv_files
+    end
+    
+    def clean_up
+      # remove all tmp files
+      @csv_files.each do |target_file, csv_file|
+        File.unlink(csv_file)
+      end
+    end
+    
+    def download(url, destination_file)
+      puts "Download '#{url}' to '#{destination_file}'"
+      File.open(destination_file, 'w') do |dst|
+        dst.write(open(url).read)
+      end
+    end
+    
+  end
+end
+
+

data/lib/localch_i18n/translator/service.rb

@@ -0,0 +1,32 @@
+module LocalchI18n
+	class Translator
+	  # Subclass this class in order to support another translation service
+	  class Service
+
+	  	# uses to_lang
+	    def initialize options = {}
+	    	configure!
+	    end
+
+	    def translate text, locale
+				text.translate(locale)  
+			end  	
+
+			protected
+
+			def configure!
+	    	load_service!
+				use_key!
+	    end
+
+	    def load_service!
+	    	require 'to_lang'
+	    end
+
+	    def use_key!
+	    	ToLang.start(I18n.google_translation_key)
+	    end
+		end
+	end
+end
+	      

data/lib/localch_i18n/translator.rb

@@ -0,0 +1,61 @@
+require 'localch_i18n/translator/service'
+
+module LocalchI18n
+  # Subclass this class in order to support another translation service
+  class Translator
+    attr_accessor :options 
+
+    def initialize options = {}
+      @options = options
+    end
+
+    # TODO: Refactor to use inject
+    def auto_translate(flat_hash)      
+    	raise '#auto_translate method requires a #current_locale method in the same module' unless respond_to? :current_locale
+
+    	translated_hash = {}
+    	flat_hash.each do |key, text|
+				translated_hash[key] = translate_it text, current_locale
+			end
+			translated_hash
+    end
+
+    def translate_it text, locale
+      text_has_args? ? translate_with_args(text) : text.translate(current_locale)
+    end
+
+    def service= service
+      raise ArgumentError, "Must be a subclass of LocalchI18n::TranslationService, was #{service}" unless service.kind_of?(LocalchI18n::TranslationService)
+      @service = service
+    end
+
+    protected
+
+    def service
+      @service ||= LocalchI18n::TranslationService.new options[:service]
+    end
+
+    # split out args parts and pure text parts
+    # translate non-arg parts and use arg parts "as is", while re-assembling
+    def translate_with_args text, locale      
+      parts = text.split /(%\{\w+\})/
+      parts.inject("") do |res, part|
+        res << var_translate(part, locale)
+      end
+    end
+
+    # translate only non-arg parts
+    def var_translate text, locale
+      is_var?(text) ? text : service.translate(text, locale)
+    end
+
+    # is it a variable part?
+    def is_var? text
+      text =~ /%\{\w+\}/
+    end
+
+    def text_has_args? text
+      text =~ /%\{/
+    end
+  end
+end

data/lib/localch_i18n/util.rb

@@ -0,0 +1,64 @@
+module LocalchI18n
+	module Util
+		def load_translations_for input_file, locale
+			translations = {}
+      translations = YAML.load_file(input_file) if File.exists?(input_file)
+
+      # Hack to fix "bug" when 'no' for Norway encountered. 
+      # Parser turns it into false as the key
+      no = translations[false]
+      translations['no'] = no
+
+      puts "  No translations found!" and return if translations.empty?
+      puts "  Missing or bad translations root key:" and return if !translations[locale]
+      translations[locale]
+    end
+
+    def row_to_hash(key, value)
+    	res = {}
+    	keys = key.split('.')
+    	keys << value
+      h = keys.reverse.inject(res) do |a, n| 
+	      if n != keys.last	
+	      	{ n => a }
+	      else
+	      	n
+	      end
+      end      
+    end    
+
+    # options:
+    #  - parent_key = []
+    #  - auto_translate 
+    def flatten_translations_hash(translations, options = {:parent_key => [] })
+      flat_hash = {}
+      parent_key = options[:parent_key] || []
+      translations.each do |key, t|
+        current_key = parent_key.dup << key
+        if t.is_a?(Hash)
+          # descend
+          options ||= {}
+          options.merge!(:parent_key => current_key)
+          flat_hash.merge!(flatten_translations_hash(t, options))
+        else
+          # leaf -> store as value for key
+          flat_hash[current_key.join('.')] = t
+        end
+      end
+			if options[:auto_translate] || auto_translate?
+      	translator.auto_translate(flat_hash) 
+      else
+      	flat_hash
+      end
+    end
+
+    def auto_translate?
+      false
+    end
+
+    attr_accessor :translator
+    def translator
+      @translator ||= LocalchI18n::Translator.new
+    end
+  end
+end