vocab 0.0.6 → 0.1.0

data/lib/vocab/application.rb

@@ -18,12 +18,16 @@ module Vocab
         options = OpenStruct.new
         parser = OptionParser.new
 
-        parser.banner = 'Usage: vocab [-h] command [platform] [file]'
+        parser.banner = 'Usage: vocab [-h] command [platform] [type] [path]'
         parser.on( '-h', '--help', 'Show this usage message' ) { options.help = true }
         parser.separator ""
         parser.separator "    vocab init"
         parser.separator "    vocab extract rails"
+        parser.separator "    vocab extract rails all"
         parser.separator "    vocab extract android"
+        parser.separator "    vocab clean rails"
+        parser.separator "    vocab convert rails xml2yml <infile>"
+        parser.separator "    vocab convert rails yml2xml <infile>"
         parser.separator "    vocab merge rails"
         parser.separator "    vocab merge android"
         parser.separator "    vocab validate android"
@@ -33,12 +37,29 @@ module Vocab
         commands = parser.parse( ARGV )
         options.command = commands[0]
         options.platform = commands[1]
-        options.path = commands[2]
+        options.type = commands[2]
+        options.path = commands[3]
 
         if( options.command == 'init' )
           Vocab::Settings.create
+        elsif( options.command == 'clean' && options.platform == 'rails' )
+          Cleaner::Rails.clean
+        elsif( options.command == 'clean' && options.platform == 'android' )
+          Cleaner::Android.clean
+        elsif( options.command == 'convert' && options.platform == 'rails' )
+          if options.type == 'xml2yml'
+            Converter::Rails.convert_xml_to_yml( options.path )
+          elsif options.type = 'yml2xml'
+            Converter::Rails.convert_yml_to_xml( options.path )
+          else
+            puts parser.help
+          end
         elsif( options.command == 'extract' && options.platform == 'rails' )
-          Extractor::Rails.extract
+          if options.type == 'all'
+            Extractor::Rails.extract_all
+          else
+            Extractor::Rails.extract
+          end
         elsif( options.command == 'extract' && options.platform == 'android' )
           Extractor::Android.extract
         elsif( options.command == 'merge' && options.platform == 'rails' )

data/lib/vocab/cleaner/android.rb

@@ -0,0 +1,15 @@
+module Vocab
+  module Cleaner
+    class Android < Base
+      class << self
+        def clean_file( file )
+          # TODO: implement
+        end
+
+        def files_to_clean
+          return Dir.glob( "*.full.xml" )
+        end
+      end
+    end
+  end
+end

data/lib/vocab/cleaner/base.rb

@@ -0,0 +1,14 @@
+module Vocab
+  module Cleaner
+    class Base
+      class << self
+        def clean
+          files_to_clean.each do |file|
+            Vocab.ui.say( "Cleaning file: #{file}" )
+            clean_file( file )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vocab/cleaner/rails.rb

@@ -0,0 +1,141 @@
+# Cleans full translation files ending in "full.yml" in the vocab root directory by:
+#   - removing empty keys
+#   - replacing HTML codes with the corresponding UTF-8 characters (e.g. > --> > )
+#   - replacing \x[XX]\x[XX] codes with characters
+#   - removing keys that shouldn't be translated (specified in a blacklist)
+
+module Vocab
+  module Cleaner
+    class Rails < Base
+      FULL_SUFFIX = 'full.yml'
+      DIFF_SUFFIX = 'diff.yml'
+      CLEAN_SUFFIX = 'clean.yml'
+      BLACKLIST = [
+          'devise',
+          'active_record',
+          'activerecord',
+          'number',
+          'datetime'
+        ]
+      WINDOWS_TO_UTF8 = {
+                "\\xC2\\x80" => "\xe2\x82\xac", # EURO SIGN
+                "\\xC2\\x82" => "\xe2\x80\x9a", # SINGLE LOW-9 QUOTATION MARK
+                "\\xC2\\x83" => "\xc6\x92",     # LATIN SMALL LETTER F WITH HOOK
+                "\\xC2\\x84" => "\xe2\x80\x9e", # DOUBLE LOW-9 QUOTATION MARK
+                "\\xC2\\x85" => "\xe2\x80\xa6", # HORIZONTAL ELLIPSIS
+                "\\xC2\\x86" => "\xe2\x80\xa0", # DAGGER
+                "\\xC2\\x87" => "\xe2\x80\xa1", # DOUBLE DAGGER
+                "\\xC2\\x88" => "\xcb\x86",     # MODIFIER LETTER CIRCUMFLEX ACCENT
+                "\\xC2\\x89" => "\xe2\x80\xb0", # PER MILLE SIGN
+                "\\xC2\\x8A" => "\xc5\xa0",     # LATIN CAPITAL LETTER S WITH CARON
+                "\\xC2\\x8B" => "\xe2\x80\xb9", # SINGLE LEFT-POINTING ANGLE QUOTATION
+                "\\xC2\\x8C" => "\xc5\x92",     # LATIN CAPITAL LIGATURE OE
+                "\\xC2\\x8E" => "\xc5\xbd",     # LATIN CAPITAL LETTER Z WITH CARON
+                "\\xC2\\x91" => "\xe2\x80\x98", # LEFT SINGLE QUOTATION MARK
+                "\\xC2\\x92" => "\xe2\x80\x99", # RIGHT SINGLE QUOTATION MARK
+                "\\xC2\\x93" => "\xe2\x80\x9c", # LEFT DOUBLE QUOTATION MARK
+                "\\xC2\\x94" => "\xe2\x80\x9d", # RIGHT DOUBLE QUOTATION MARK
+                "\\xC2\\x95" => "\xe2\x80\xa2", # BULLET
+                "\\xC2\\x96" => "\xe2\x80\x93", # EN DASH
+                "\\xC2\\x97" => "\xe2\x80\x94", # EM DASH
+
+                "\\xC2\\x98" => "\xcb\x9c",     # SMALL TILDE
+                "\\xC2\\x99" => "\xe2\x84\xa2", # TRADE MARK SIGN
+                "\\xC2\\x9A" => "\xc5\xa1",     # LATIN SMALL LETTER S WITH CARON
+                "\\xC2\\x9B" => "\xe2\x80\xba", # SINGLE RIGHT-POINTING ANGLE QUOTATION
+                "\\xC2\\x9C" => "\xc5\x93",     # LATIN SMALL LIGATURE OE
+                "\\xC2\\x9E" => "\xc5\xbe",     # LATIN SMALL LETTER Z WITH CARON
+                "\\xC2\\x9F" => "\xc5\xb8"      # LATIN CAPITAL LETTER Y WITH DIAERESIS
+        }
+
+      class << self
+
+        def clean_file( file )
+          @file = file
+          @locale_name = File.basename( @file, '.yml' )
+          @clean_dir = File.dirname( @file )
+          @clean_name = "#{@clean_dir}/#{@locale_name}.#{CLEAN_SUFFIX}"
+
+          replace_codes
+          clean_yaml
+        end
+
+        def files_to_clean ( dir = Vocab.root )
+          return ( Dir.glob( "#{dir}/*.#{FULL_SUFFIX}" ) + Dir.glob( "#{dir}/*.#{DIFF_SUFFIX}" ) )
+        end
+
+        private
+        def replace_codes
+          translation = File.read( @file )
+
+          cleaned_text = replace_html_codes( translation )
+          cleaned_text = replace_windows_codes( cleaned_text )
+
+          cleaned_file = File.open( @clean_name, 'w' )
+          cleaned_file.puts( cleaned_text )
+          cleaned_file.close
+        end
+
+        def replace_html_codes( translation )
+          entity_matcher = /&.+?;/
+          coder = HTMLEntities.new( :expanded )
+        
+          cleaned_text = translation.gsub( entity_matcher ) do |entity| 
+            entity == "&quot;" ? "\\\"" : coder.decode( entity )
+          end
+
+          return cleaned_text
+        end
+
+        def replace_windows_codes( translation )
+          WINDOWS_TO_UTF8.each { |windows_code,utf8_code| translation.gsub!( windows_code, utf8_code ) }
+          return translation
+        end
+
+        def clean_yaml
+          # TODO: use psych
+          original_engine = YAML::ENGINE.yamler
+          YAML::ENGINE.yamler='syck'
+          translation_hash = YAML.load( File.open( @clean_name, 'r' ))
+
+          cleaned_file = File.open( @clean_name, 'w' )
+          keys =  clean_keys( translation_hash )
+          cleaned_file.puts( replace_hex_codes( keys ) )
+          cleaned_file.close
+
+          ensure
+            YAML::ENGINE.yamler=original_engine
+
+        end
+
+        def replace_hex_codes( keys )
+            # Using ya2yaml, if available, for UTF8 support
+            keys.respond_to?( :ya2yaml ) ? keys.ya2yaml( :escape_as_utf8 => true ) : keys.to_yaml
+        end
+
+        def clean_keys( hash )
+          hash.inject({}) { |result, (key, value)|
+            if valid_key?( key ) 
+              value = clean_keys(value) if value.is_a? Hash
+              unless value.to_s.empty? or value == {}
+                result[(key.to_s rescue key) || key] = value
+              end
+            end
+            result
+          }
+        end
+
+        def valid_key?( key )
+          BLACKLIST.each do |prefix|
+            if key.to_s.include?( prefix )
+              return false
+            end
+          end
+          return true
+        end
+
+
+      end
+    end
+  end
+end

data/lib/vocab/cleaner.rb

@@ -0,0 +1,7 @@
+module Vocab
+  module Cleaner
+    autoload :Base,  'vocab/cleaner/base'
+    autoload :Rails,  'vocab/cleaner/rails'
+    autoload :Android,  'vocab/cleaner/android'
+  end
+end

data/lib/vocab/converter/base.rb

@@ -0,0 +1,17 @@
+module Vocab
+  module Converter
+    class Base
+     class << self
+        def convert_xml_to_yml( path = nil)
+          Vocab.ui.say( "No conversion available" )
+          return nil
+        end
+
+        def convert_yml_to_xml( path = nil)
+          Vocab.ui.say( "No conversion available" )
+          return nil
+        end
+      end
+    end
+  end
+end

data/lib/vocab/converter/rails.rb

@@ -0,0 +1,86 @@
+module Vocab
+  module Converter
+    class Rails < Base
+      SEPARATOR_ESCAPE_CHAR = "\001"
+
+      class << self 
+        def convert_xml_to_yml( file = nil )
+          xml_root = 'hash'
+          keys = xml_to_yml_keys( Hash.from_xml( File.read( file ) )[ xml_root ] )
+          unwound_keys = unwind_keys( keys )
+          out_dir = File.dirname( file )
+          out_file = "#{out_dir}/#{File.basename( file, '.xml' ) + '.yml'}"
+          File.open( out_file, 'w' ) {|f| f.puts( ( keys_to_yaml( unwound_keys ) ) ) }
+        end
+
+        def xml_to_yml_keys( hash )
+          hash.inject( {} ) { |result, ( key, value )|
+            value = xml_to_yml_keys( value ) if value.is_a? Hash
+            yml_key = key.gsub( "-", "_" )
+            result[ yml_key ] = value
+            result
+          }
+        end
+
+        def unwind_keys( hash, separator = "." )
+          result = {}
+          hash.each do |key, value|
+            keys = key.to_s.split( separator )
+            curr = result
+            curr = curr[ keys.shift ] ||= {} while keys.size > 1
+                curr[ keys.shift ] = value
+          end
+
+          return result
+        end
+
+        def wind_keys( hash, separator = nil, subtree = false, prev_key = nil, result = {}, orig_hash = hash )
+          separator ||= I18n.default_separator
+
+          hash.each_pair do |key, value|
+            key = escape_default_separator( key, separator )
+            curr_key = [ prev_key, key ].compact.join( separator ).to_sym
+
+            if value.is_a?( Hash )
+              result[ curr_key ] = value if subtree
+              wind_keys( value, separator, subtree, curr_key, result, orig_hash )
+            else
+              result[ unescape_default_separator( curr_key ) ] = value
+            end
+          end
+
+          return result
+        end
+
+        def keys_to_yaml( keys )
+          # Using ya2yaml, if available, for UTF8 support
+          keys.respond_to?( :ya2yaml ) ? keys.ya2yaml( :escape_as_utf8 => true ) : keys.to_yaml
+        end
+
+        def keys_to_xml( keys )
+          raise "to_xml is broken" unless keys.respond_to?( :to_xml )
+          keys.to_xml
+        end
+      
+        def convert_yml_to_xml( file = nil )
+          xml_root = 'hash'
+          yml = YAML.load_file( file )
+          wound_keys = wind_keys( yml )
+
+          out_dir = File.dirname( file )
+          out_file = "#{out_dir}/#{File.basename( file, '.yml' ) + '.xml'}"
+          File.open( out_file, 'w' ) { |f| f.puts( ( keys_to_xml( wound_keys ) ) ) }
+        end
+
+        def unescape_default_separator( key, separator = nil ) 
+          key.to_s.tr( SEPARATOR_ESCAPE_CHAR, separator || I18n.default_separator ).to_sym
+        end
+
+        def escape_default_separator( key, separator = nil )
+          key.to_s.tr( separator || I18n.default_separator, SEPARATOR_ESCAPE_CHAR )
+        end
+
+      end
+    end
+  end
+end

data/lib/vocab/converter.rb

@@ -0,0 +1,6 @@
+module Vocab
+  module Converter
+    autoload :Base,  'vocab/converter/base'
+    autoload :Rails,  'vocab/converter/rails'
+  end
+end

data/lib/vocab/extractor/rails.rb

@@ -3,20 +3,22 @@ module Vocab
     class Rails < Base
       DIFF = 'en.yml'
       FULL = 'en.full.yml'
+      DIFF_SUFFIX = 'diff.yml'
+      FULL_SUFFIX = 'full.yml'
 
       class << self
-        def write_diff( strings, plurals, path )
-          path ||= "#{Vocab.root}/#{DIFF}"
+        def write_diff( strings, plurals, path, locale = :en )
+          path ||= "#{Vocab.root}/#{locale}.#{DIFF_SUFFIX}"
           write( strings, path )
         end
 
-        def write_full( strings, plurals, path )
-          path ||= "#{Vocab.root}/#{FULL}"
+        def write_full( strings, plurals, path, locale = :en )
+          path ||= "#{Vocab.root}/#{locale}.#{FULL_SUFFIX}"
           write( strings, path )
         end
 
-        def write( translations, path )
-          data = hasherize( translations ).to_yaml
+        def write( translations, path, locale = :en )
+          data = hasherize( translations, locale ).to_yaml
           File.open( path, "w+" ) { |f| f.write( data ) }
           Vocab.ui.say( "Extracted to #{path}" )
         end
@@ -56,16 +58,30 @@ module Vocab
           return {}
         end
 
-        def translations( dir )
+        def extract_all( locales_root = nil, result_dir = nil )
+          locales_root ||= "#{Vocab.root}/config/locales"
+          result_dir ||= Vocab.root
+          
           translator = Vocab::Translator::Rails.new
+          translator.load_dir( locales_root )
+
+          translator.available_locales.each do |locale|
+            strings = translations( locales_root, locale )
+            path = "#{result_dir}/#{locale}.full.yml"
+            write( strings, path, locale )
+          end
+        end
+
+        def translations( dir, locale = :en )
+          translator = Vocab::Translator::Rails.new( locale )
           translator.load_dir( dir )
           return translator.flattened_translations( :prefix => true )
         end
 
-        def hasherize( diff )
-          translator = Vocab::Translator::Rails.new
+        def hasherize( diff, locale = :en )
+          translator = Vocab::Translator::Rails.new( locale )
           diff.each do |key, value|
-            key = key.to_s.gsub!( /^en\./, '' )
+            key = key.to_s.gsub!( /^#{locale.to_s}\./, '' )
             translator.store( key, value )
           end
           return translator.translations( :prefix => true )

data/lib/vocab/merger/android.rb

@@ -1,10 +1,17 @@
 module Vocab
   module Merger
     class Android < Base
+      FORMAT_PATTERN = /%(.+?)\b/ 
+      ARG_PATTERN = /\$(.+?)\b/
 
       def initialize( locales_dir = nil, updates_dir = nil )
         @locales_dir = locales_dir || 'res'
         @updates_dir = updates_dir || 'tmp/translations'
+        @english_path = "#{@locales_dir}/values/strings.xml"
+        if File.exists?( @english_path )
+          @english_strings = english_strings
+          @english_plurals = english_plurals
+        end
       end
 
       def merge_file( path )
@@ -17,17 +24,17 @@ module Vocab
         keys = string_keys
         current = current_strings_for_locale( path )
         updates = update_strings_for_locale( path )
-        return translation_hash( keys, current, updates, path )
+        return translation_hash( keys, current, updates, path, :string_format_changed? )
       end
 
       def plurals( path )
         keys = plural_keys
         current = current_plurals_for_locale( path )
         updates = update_plurals_for_locale( path )
-        return translation_hash( keys, current, updates, path )
+        return translation_hash( keys, current, updates, path, :plural_format_changed? )
       end
 
-      def translation_hash( keys, current, updates, path )
+      def translation_hash( keys, current, updates, path, format_checker = :string_format_changed? )
         translation = {}
         keys.each do |key|
           next if Vocab::Translator::Base.ignore_key?( key )
@@ -35,6 +42,7 @@ module Vocab
           value = updates[ key ] || current[ key ]
           if value
             translation[ key ] = value
+            check_matching_format_strings( key, value, path, format_checker )
           else
             Vocab.ui.warn( "No translation found for key #{key} while merging #{path}" )
           end
@@ -43,6 +51,34 @@ module Vocab
         return translation
       end
 
+      def english_strings
+        return Vocab::Translator::Android.hash_from_xml( @english_path )
+      end
+
+      def english_plurals
+        return Vocab::Translator::Android.plurals_from_xml( @english_path )
+      end
+      
+      def check_matching_format_strings( key, new_value, path, format_checker )
+        send( format_checker, key, new_value, path) 
+      end
+
+      def plural_format_changed?( key, new_value, path )
+        new_value.each do |inner_key,inner_value|
+          if ( @english_plurals[ key ][ inner_key ].to_s.scan( FORMAT_PATTERN ) != inner_value.to_s.scan( FORMAT_PATTERN ) ) ||
+           ( @english_plurals[ key ][ inner_key ].to_s.scan( ARG_PATTERN ) != inner_value.to_s.scan( ARG_PATTERN ) )
+            Vocab.ui.warn( "Format string mismatch for key #{key}, quantity #{inner_key} while merging #{path}. \n English: #{@english_plurals[ key ][ inner_key ]} \n Translation: #{new_value[ inner_key ]}" )
+          end
+        end
+      end
+
+      def string_format_changed?( key, new_value, path )
+        if ( @english_strings[ key ].to_s.scan( FORMAT_PATTERN ) != new_value.to_s.scan( FORMAT_PATTERN ) ) ||
+           ( @english_strings[ key ].to_s.scan( ARG_PATTERN ) != new_value.to_s.scan( ARG_PATTERN ) )
+          Vocab.ui.warn( "Format string mismatch for key #{key} while merging #{path}. \n English: #{@english_strings[ key ]} \n Translation: #{new_value}" )
+        end
+      end
+
       def string_keys
         return Vocab::Translator::Android.string_keys( @locales_dir )
       end

data/lib/vocab/merger/rails.rb

@@ -1,6 +1,7 @@
 module Vocab
   module Merger
     class Rails < Base
+      INTERPOLATION_PATTERN = /%{(.+?)}/
 
       def initialize( locales_dir = nil, updates_dir = nil )
         @locales_dir = locales_dir || 'config/locales'
@@ -13,6 +14,8 @@ module Vocab
 
         # list of keys that need to be in the translated file
         keys = Vocab::Merger::Rails.keys_for_file( locales_path )
+        english = Vocab::Merger::Rails.load_english( locales_path )
+
 
         # existing translations already in the file
         locales_translator = translator( locales_path )
@@ -27,6 +30,7 @@ module Vocab
         updates_translator = translator( update_path )
         updates = updates_translator.flattened_translations
 
+
         # apply updated keys to locales hash
         keys.each do |key|
           next if Vocab::Translator::Base.ignore_key?( key )
@@ -34,6 +38,7 @@ module Vocab
           value = updates[ key ] || locales[ key ]
           if value
             locales_translator.store( key, value )
+            check_matching_interpolations( key, english[ key ], value, locales_path )
           else
             Vocab.ui.warn( "No translation found for key #{key} while merging #{locales_path}" )
           end
@@ -42,6 +47,12 @@ module Vocab
         locales_translator.write_file( locales_path )
       end
 
+      def check_matching_interpolations( key, old_value, new_value, locales_path )
+        if old_value.to_s.scan( INTERPOLATION_PATTERN ) != new_value.to_s.scan( INTERPOLATION_PATTERN )
+          Vocab.ui.warn( "Interpolation mismatch for key #{key} while merging #{locales_path}. \n English: #{old_value} Translation: #{new_value}" )
+        end
+      end
+
       def self.keys_for_file( path )
         en_path = Vocab::Translator::Rails.en_equivalent_path( path )
         translator = Vocab::Translator::Rails.new
@@ -49,6 +60,13 @@ module Vocab
         return translator.flattened_translations.keys
       end
 
+      def self.load_english( path )
+        en_path = Vocab::Translator::Rails.en_equivalent_path( path )
+        translator = Vocab::Translator::Rails.new
+        translator.load_file( en_path )
+        return translator.flattened_translations
+      end
+
       def translatable?( path )
         if File.basename( path ) == 'en.yml'
           Vocab.ui.warn( "can't translate english file #{path}" )

data/lib/vocab/translator/rails.rb

@@ -14,6 +14,10 @@ module Vocab
         @locale = locale
       end
 
+      def available_locales
+        return @backend.available_locales
+      end
+
       def load_dir( dir )
         I18n.load_path = Dir.glob( "#{dir}/**/*.{yml,rb}" )
         load_translations

data/lib/vocab/version.rb

@@ -1,3 +1,3 @@
 module Vocab
-  VERSION = "0.0.6"
+  VERSION = "0.1.0"
 end

data/lib/vocab.rb

@@ -2,10 +2,19 @@ require 'fileutils'
 require 'i18n'
 require 'pathname'
 require 'nokogiri'
+require 'htmlentities'
+require 'yaml'
+require 'ya2yaml'
+require 'active_support/core_ext'
+require 'builder'
+
+ 
 
 require 'vocab/application'
 require 'vocab/extractor'
 require 'vocab/translator'
+require 'vocab/cleaner'
+require 'vocab/converter'
 require 'vocab/merger'
 require 'vocab/settings'
 require 'vocab/ui'

data/spec/cleaner/base_spec.rb

@@ -0,0 +1,19 @@
+require "spec_helper"
+
+describe "Vocab::Cleaner::Base" do
+
+  describe 'clean' do
+
+    it 'cleans a list of files' do
+      files = [ 'es.yml', 'en.yml' ]
+      Vocab::Cleaner::Base.should_receive( :files_to_clean ).and_return( files )
+      
+      files.each do |file|
+        Vocab::Cleaner::Base.should_receive( :clean_file ).with( file )
+        Vocab.ui.should_receive( :say ).with( "Cleaning file: #{file}" )
+      end
+      Vocab::Cleaner::Base.clean
+    end
+
+  end
+end