vocab 0.0.1

data/.gitignore

@@ -0,0 +1,9 @@
+*.gem
+*.tmp
+.bundle
+Gemfile.lock
+pkg/*
+spec/tmp/*
+!spec/tmp/.vocab
+spec/data/android/versions/*.tmp
+tmp/last_translation/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in vocab.gemspec
+gemspec

data/README.md

@@ -0,0 +1,58 @@
+# Description
+
+Command line tool to automate the manipulation of i18n string files in Rails and Android.
+
+Features include:
+
+  * Generate a list all English translations that have been updated or added since the last translation
+  * Integrate a list of newly translated strings into the right place in the project translation file(s)
+  * Validate that all languages have the same keys
+
+# Requirements
+
+  * Must be used in context of git controlled project
+  * Assumes that android string files are called strings.xml
+
+# Installation
+
+Install the gem in your project
+
+    gem install vocab
+
+Set the current git commit to be the starting point for the next translation diff
+
+    vocab init
+
+# Usage
+
+## Extracting Changed and Updated Strings For Translation
+
+    vocab extract android
+
+    or
+
+    vocab extract rails
+
+Note: keys starting with 'debug_' are ignored.  Prepend development-only strings
+with 'debug_' to avoid these keys being sent to translators.
+
+## Merging new translations into project string files
+
+    vocab merge android
+
+    or
+
+    vocab merge rails
+
+## Merging new translations into project string files
+
+    vocab validate android
+
+    or
+
+    vocab validate rails
+
+# TODO
+
+  * Add .processed to each tmp/translation after success
+  * Accept filenames other than strings.xml under tmp/translation/values-xx/yyyy.xml

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+
+gemspec = eval(File.read("vocab.gemspec"))
+
+task :build => "#{gemspec.full_name}.gem"
+
+file "#{gemspec.full_name}.gem" => gemspec.files + ["vocab.gemspec"] do
+  system "gem build vocab.gemspec"
+  system "gem install vocab-#{Vocab::VERSION}.gem"
+end

data/bin/vocab

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'rubygems'
+  gem 'vocab'
+rescue LoadError
+end
+
+require 'vocab'
+
+Vocab::Application.run

data/lib/vocab/application.rb

@@ -0,0 +1,58 @@
+require 'optparse'
+require 'ostruct'
+
+module Vocab
+  class Application
+
+    class << self
+      def run
+        handle_command
+      end
+
+      ##############################
+      # CLI
+      ##############################
+
+      def handle_command
+        options = OpenStruct.new
+        parser = OptionParser.new
+
+        parser.banner = 'Usage: vocab [-h] command [platform] [file]'
+        parser.on( '-h', '--help', 'Show this usage message' ) { options.help = true }
+        parser.separator ""
+        parser.separator "    vocab init"
+        parser.separator "    vocab extract rails"
+        parser.separator "    vocab extract android"
+        parser.separator "    vocab merge rails"
+        parser.separator "    vocab merge android"
+        parser.separator "    vocab validate android"
+        parser.separator "    vocab validate rails"
+        parser.separator ""
+
+        commands = parser.parse( ARGV )
+        options.command = commands[0]
+        options.platform = commands[1]
+        options.path = commands[2]
+
+        if( options.command == 'init' )
+          Vocab::Settings.create
+        elsif( options.command == 'extract' && options.platform == 'rails' )
+          Extractor::Rails.extract
+        elsif( options.command == 'extract' && options.platform == 'android' )
+          Extractor::Android.extract
+        elsif( options.command == 'merge' && options.platform == 'rails' )
+          Merger::Rails.new.merge
+        elsif( options.command == 'merge' && options.platform == 'android' )
+          Merger::Android.new.merge
+        elsif( options.command == 'validate' && options.platform == 'android' )
+          Validator::Android.new.validate
+        elsif( options.command == 'validate' && options.platform == 'rails' )
+          Validator::Rails.new.validate
+        else
+          puts parser.help
+        end
+      end
+
+    end
+  end
+end

data/lib/vocab/extractor/android.rb

@@ -0,0 +1,56 @@
+module Vocab
+  module Extractor
+    class Android < Base
+      class << self
+
+        DIFF = 'strings.diff.xml'
+        FULL = 'strings.full.xml'
+
+        def extract_current( path = nil )
+          path ||= "#{Vocab.root}/res/values/strings.xml"
+          return Vocab::Translator::Android.hash_from_xml( path )
+        end
+
+        def extract_previous( path = nil )
+          path ||= "res/values/strings.xml"
+          sha = Vocab.settings.last_translation
+          xml = previous_file( path, sha )
+
+          tmpfile = "#{Vocab.root}/tmp/last_translation/#{File.basename(path)}"
+          FileUtils.mkdir_p( File.dirname( tmpfile ) )
+          File.open( tmpfile, 'w' ) { |f| f.write( xml ) }
+          return Vocab::Translator::Android.hash_from_xml( tmpfile )
+        end
+
+        def write_diff( diff, path = nil )
+          path ||= "#{Vocab.root}/#{DIFF}"
+          Vocab::Translator::Android.write( diff, path )
+        end
+
+        def write_full( diff, path = nil )
+          path ||= "#{Vocab.root}/#{FULL}"
+          Vocab::Translator::Android.write( diff, path )
+        end
+
+        def examples( locales_dir = nil )
+          locales_dir ||= "#{Vocab.root}/res/values"
+          return Vocab::Translator::Android.locales( locales_dir ).collect do |locale|
+            "tmp/translations/values-#{locale}"
+          end
+        end
+
+        def print_instructions( values = {} )
+          values[ :diff ] = DIFF
+          values[ :full ] = FULL
+          values[ :tree ] = <<-EOS
+tmp/translations/values-es/strings.xml
+tmp/translations/values-zh-rCN/strings.xml
+          EOS
+
+          super( values )
+        end
+
+      end
+    end
+  end
+end

data/lib/vocab/extractor/base.rb

@@ -0,0 +1,92 @@
+module Vocab
+  module Extractor
+    class Base
+      class << self
+        def extract( diff_path = nil, full_path = nil )
+          current = extract_current
+          previous = extract_previous
+          diff = diff( previous, current )
+          write_diff( diff, diff_path )
+          write_full( current, full_path )
+          mkdir_examples
+          print_instructions
+        end
+
+        # make a hash of all the translations that are new or changed in the current yml
+        def diff( previous, current )
+          diff = {}
+          current.each do |key, value|
+            next if Vocab::Translator::Base.ignore_key?( key )
+
+            previous_value = previous[ key ]
+            if( previous_value.nil? || previous_value != value )
+              diff[ key ] = value
+            end
+          end
+          return diff
+        end
+
+        def extract_previous
+          raise "extract_previous not implemented"
+        end
+
+        def extract_current
+          raise "extract_current not implemented"
+        end
+
+        def write_diff( diff, path )
+          raise "write_diff not implemented"
+        end
+
+        def write_full( diff, path )
+          raise "write_full not implemented"
+        end
+
+        def previous_file( path, sha )
+          path = git_path( path )
+          return `cd #{Vocab.root} && git show #{sha}:#{path}`
+        end
+
+        def git_root
+          return `git rev-parse --show-toplevel`.strip
+        end
+
+        def git_path( path )
+          return File.expand_path( path ).gsub( "#{git_root}/", '' )
+        end
+
+        def mkdir_examples
+          examples.each do |example|
+            Vocab.ui.say( "Creating placeholder: #{example}" )
+            FileUtils.mkdir_p( example )
+          end
+        end
+
+        def print_instructions( values = {} )
+instructions = <<-EOS
+
+Extraction completed. To complete the translation:
+
+1. Send the language files to the translators:
+
+#{values[:diff]} for languages that are already in the app
+#{values[:full]} for languages that need a complete translation
+
+2. To integrate new translations:
+
+Place completed translations under tmp/translations, for example:
+
+#{values[:tree]}
+
+3. Merge translations into the project with:
+
+vocab merge rails
+
+EOS
+          Vocab.ui.say( instructions )
+        end
+
+      end
+    end
+  end
+end

data/lib/vocab/extractor/rails.rb

@@ -0,0 +1,81 @@
+module Vocab
+  module Extractor
+    class Rails < Base
+      DIFF = 'en.yml'
+      FULL = 'en.full.yml'
+
+      class << self
+        def write_diff( diff, path )
+          path ||= "#{Vocab.root}/#{DIFF}"
+          write( diff, path )
+        end
+
+        def write_full( full, path )
+          path ||= "#{Vocab.root}/#{FULL}"
+          write( full, path )
+        end
+
+        def write( translations, path )
+          data = hasherize( translations ).to_yaml
+          File.open( path, "w+" ) { |f| f.write( data ) }
+          Vocab.ui.say( "Extracted to #{path}" )
+        end
+
+        def extract_previous( locales_root = nil )
+          locales_root ||= "config/locales"
+          tmpdir = "#{Vocab.root}/tmp/last_translation"
+          FileUtils.rm_rf( "#{tmpdir}/*" )
+
+          sha = Vocab.settings.last_translation
+          translation_files = `git ls-tree --name-only -r #{sha}:#{locales_root}`.split( "\n" )
+          translation_files = translation_files.select { |f| f =~ /en.(yml|rb)$/ }
+          translation_files.each do |path|
+            tmpdir_path = "#{tmpdir}/#{path}"
+            FileUtils.mkdir_p( File.dirname( tmpdir_path ) )
+            File.open( tmpdir_path, "w+" ) do |f|
+              yml = previous_file( "#{locales_root}/#{path}", sha )
+              f.write( yml )
+            end
+          end
+
+          return translations( tmpdir )
+        end
+
+        def extract_current( locales_root = nil )
+          locales_root ||= "#{Vocab.root}/config/locales"
+          return translations( locales_root )
+        end
+
+        def translations( dir )
+          translator = Vocab::Translator::Rails.new
+          translator.load_dir( dir )
+          return translator.flattened_translations( :prefix => true )
+        end
+
+        def hasherize( diff )
+          translator = Vocab::Translator::Rails.new
+          diff.each do |key, value|
+            key = key.to_s.gsub!( /^en\./, '' )
+            translator.store( key, value )
+          end
+          return translator.translations( :prefix => true )
+        end
+
+        def examples
+          return [ "#{Vocab.root}/tmp/translations" ]
+        end
+
+        def print_instructions( values = {} )
+          values[ :diff ] = DIFF
+          values[ :full ] = FULL
+          values[ :tree ] = <<-EOS
+tmp/translations/es.yml
+tmp/translations/zh.yml
+          EOS
+
+          super( values )
+        end
+      end
+    end
+  end
+end

data/lib/vocab/extractor.rb

@@ -0,0 +1,7 @@
+module Vocab
+  module Extractor
+    autoload :Base,  'vocab/extractor/base'
+    autoload :Rails, 'vocab/extractor/rails'
+    autoload :Android, 'vocab/extractor/android'
+  end
+end

data/lib/vocab/merger/android.rb

@@ -0,0 +1,59 @@
+module Vocab
+  module Merger
+    class Android < Base
+
+      def initialize( locales_dir = nil, updates_dir = nil )
+        @locales_dir = locales_dir || 'res'
+        @updates_dir = updates_dir || 'tmp/translations'
+      end
+
+      def merge_file( path )
+        keys = english_keys
+        current = current_for_locale( path )
+        updates = updates_for_locale( path )
+
+        translation = {}
+        keys.each do |key|
+          next if Vocab::Translator::Base.ignore_key?( key )
+
+          value = updates[ key ] || current[ key ]
+          if value
+            translation[ key ] = value
+          else
+            Vocab.ui.warn( "No translation found for key #{key} while merging #{path}" )
+          end
+        end
+
+        Vocab::Translator::Android.write( translation, path )
+      end
+
+      def english_keys
+        return Vocab::Translator::Android.english_keys( @locales_dir )
+      end
+
+      def current_for_locale( path )
+        Vocab::Translator::Android.hash_from_xml( path )
+      end
+
+      def updates_for_locale( path )
+        name = path.gsub( "#{@locales_dir}/", '' )
+        dirname = File.dirname( name )
+        entries = Dir.glob( "#{updates_dir}/#{dirname}/*.xml" )
+        Vocab.ui.warn( "Multiple update files for #{path}: #{entries.join( ',' )}" ) if entries.size > 1
+        update = entries.first
+        Vocab::Translator::Android.hash_from_xml( update )
+      end
+
+      def translation_locales
+        return Vocab::Translator::Android.locales( @updates_dir, false )
+      end
+
+      def files_to_merge
+        return translation_locales.collect do |locale|
+          "#{@locales_dir}/values-#{locale}/strings.xml"
+        end
+      end
+
+    end
+  end
+end

data/lib/vocab/merger/base.rb

@@ -0,0 +1,21 @@
+module Vocab
+  module Merger
+    class Base
+
+      attr_accessor :locales_dir, :updates_dir
+
+      def update_settings
+        sha = Vocab.settings.update_translation
+        Vocab.ui.say( "Updated current translation to #{sha}" )
+      end
+
+      def merge
+        files_to_merge.each do |file|
+          Vocab.ui.say( "Merging file: #{file}" )
+          merge_file( file )
+        end
+        update_settings
+      end
+    end
+  end
+end

data/lib/vocab/merger/rails.rb

@@ -0,0 +1,105 @@
+module Vocab
+  module Merger
+    class Rails < Base
+
+      def initialize( locales_dir = nil, updates_dir = nil )
+        @locales_dir = locales_dir || 'config/locales'
+        @updates_dir = updates_dir || 'tmp/translations'
+      end
+
+      def merge_file( locales_path )
+        return unless translatable?( locales_path )
+        create_if_missing( locales_path )
+
+        # list of keys that need to be in the translated file
+        keys = Vocab::Merger::Rails.keys_for_file( locales_path )
+
+        # existing translations already in the file
+        locales_translator = translator( locales_path )
+        locales = locales_translator.flattened_translations
+
+        # new translations from the translators
+        update_path = "#{@updates_dir}/#{locales_translator.locale}.yml"
+        unless File.exists?( update_path )
+          Vocab.ui.say( "Missing update file: #{update_path} to translate #{locales_path}" )
+          return
+        end
+        updates_translator = translator( update_path )
+        updates = updates_translator.flattened_translations
+
+        # apply updated keys to locales hash
+        keys.each do |key|
+          next if Vocab::Translator::Base.ignore_key?( key )
+
+          value = updates[ key ] || locales[ key ]
+          if value
+            locales_translator.store( key, value )
+          else
+            Vocab.ui.warn( "No translation found for key #{key} while merging #{locales_path}" )
+          end
+        end
+
+        locales_translator.write_file( locales_path )
+      end
+
+      def self.keys_for_file( path )
+        en_path = Vocab::Translator::Rails.en_equivalent_path( path )
+        translator = Vocab::Translator::Rails.new
+        translator.load_file( en_path )
+        return translator.flattened_translations.keys
+      end
+
+      def translatable?( path )
+        if File.basename( path ) == 'en.yml'
+          Vocab.ui.warn( "can't translate english file #{path}" )
+          return false
+        end
+
+        unless File.exists?( Vocab::Translator::Rails.en_equivalent_path( path ) )
+          Vocab.ui.warn( "skipping because no english equivalent for #{path}" )
+          return false
+        end
+
+        if( File.exists?( path ) )
+          extension = File.basename( path, '.yml' )
+          contents = YAML.load_file( path ).keys.first.to_s
+          if( extension != contents )
+            Vocab.ui.warn( "File extension does not match file contents in #{path}" )
+            return false
+          end
+        end
+
+        return true
+      end
+
+      def translator( path )
+        translator = Vocab::Translator::Rails.new
+        translator.load_file( path ) if File.exists?( path )
+        return translator
+      end
+
+      def create_if_missing( path )
+        return if File.exists?( path )
+        locale = File.basename( path, '.yml' )
+        Vocab.ui.say( "Creating file #{path}" )
+        File.open( path, 'w+' ) { |file| file.write( { locale => {} }.to_yaml ) }
+      end
+
+      # TODO cache this so you don't hit the FS so much
+      def translation_locales
+        return Dir.glob( "#{@updates_dir}/*.yml" ).collect { |f| File.basename( f, '.yml' ) }
+      end
+
+      def files_to_merge
+        paths = []
+        Dir.glob( "#{@locales_dir}/**/en.yml" ).each do |en_path|
+          translation_locales.each do |locale|
+            paths << en_path.gsub( /en.yml$/, "#{locale}.yml" )
+          end
+        end
+        return paths
+      end
+
+    end
+  end
+end

data/lib/vocab/merger.rb

@@ -0,0 +1,7 @@
+module Vocab
+  module Merger
+    autoload :Base,  'vocab/merger/base'
+    autoload :Rails, 'vocab/merger/rails'
+    autoload :Android, 'vocab/merger/android'
+  end
+end

data/lib/vocab/settings.rb

@@ -0,0 +1,35 @@
+module Vocab
+  class Settings
+    def initialize( root )
+      @root   = root
+      @local_config = File.exist?( config_file ) ? YAML.load_file( config_file ) : {}
+    end
+
+    def config_file
+      Pathname.new( "#{@root}/.vocab" )
+    end
+
+    def last_translation
+      return @local_config[ 'last_translation' ]
+    end
+
+    def update_translation
+      current_sha = `git rev-parse HEAD`.strip
+      @local_config[ 'last_translation' ] = current_sha
+      write_settings
+      return current_sha
+    end
+
+    def write_settings
+      File.open( config_file, 'w' ) { |f| f.write( @local_config.to_yaml ) }
+      `git add #{config_file}`
+    end
+
+    def self.create
+      Vocab.ui.say( "Writing new .vocab file.  Check this file into your project repo" )
+      settings = Vocab::Settings.new( Dir.pwd )
+      settings.update_translation
+      settings.write_settings
+    end
+  end
+end

data/lib/vocab/translator/android.rb

@@ -0,0 +1,43 @@
+module Vocab
+  module Translator
+    class Android < Base
+
+      def self.hash_from_xml( path )
+        xml = File.open( path ) { |f| f.read }
+        doc = Nokogiri::HTML.fragment( xml ) { |config| config.noblanks }
+        children = doc.search( 'resources/string' )
+        hash = {}
+        children.each { |child| hash[ child['name'] ] = child.text }
+        return hash
+      end
+
+      def self.write( hash, path )
+        builder = Nokogiri::XML::Builder.new do |xml|
+          xml.resources {
+            hash.sort.each do |key, value|
+              xml.string( value, :name => key )
+            end
+          }
+        end
+        File.open( path, 'w' ) { |f| f.write( builder.to_xml( :encoding => 'UTF-8' ) ) }
+      end
+
+      def self.english_keys( locales_dir )
+        path = "#{locales_dir}/values/strings.xml"
+        translations = Vocab::Translator::Android.hash_from_xml( path )
+        return translations.keys
+      end
+
+      def self.locales( dir, strict = true )
+        xml_pattern = strict ? 'strings.xml' : '*'
+
+        locales = []
+        Dir.glob( "#{dir}/values-*/#{xml_pattern}" ).each do |path|
+          locales << $1 if path =~ /values-(.*)\//
+        end
+        return locales
+      end
+
+    end
+  end
+end

data/lib/vocab/translator/base.rb

@@ -0,0 +1,12 @@
+module Vocab
+  module Translator
+    class Base
+
+      def self.ignore_key?( key )
+        return true if key.to_s.start_with?( 'debug_' )
+        return false
+      end
+
+    end
+  end
+end