babelyoda 1.6.0 → 2.0.0

data/lib/babelyoda/strings_parser.rb

@@ -0,0 +1,72 @@
+require_relative 'localization_key'
+require_relative 'localization_value'
+
+module Babelyoda
+  class StringsParser
+    Bit = Struct.new(:token, :value)
+
+    def initialize(lexer, language)
+      @lexer, @language = lexer, language
+    end
+
+    def parse(str, &block)
+      @block = block
+      bitstream = []
+      @lexer.lex(str) do | token, value |
+        bitstream << Bit.new(token, value)
+      end
+      while bitstream.size > 0
+        record = produce(bitstream)
+        @block.call(record) if record
+      end
+    end
+
+    def produce(bs)
+      match_bs(bs, :multiline_comment, :string, :equal_sign, :string, :semicolon) do |bits|
+        localization_key = LocalizationKey.new(cleanup_string(bits[1]), cleanup_comment(bits[0]))
+        localization_value = LocalizationValue.new(@language, cleanup_string(bits[3]))
+        localization_key << localization_value
+        return localization_key
+      end
+      match_bs(bs, :singleline_comment, :string, :equal_sign, :string, :semicolon) do |bits|
+        localization_key = LocalizationKey.new(cleanup_string(bits[1]), cleanup_comment(bits[0]))
+        localization_value = LocalizationValue.new(@language, cleanup_string(bits[3]))
+        localization_key << localization_value
+        return localization_key
+      end
+      match_bs(bs, :string, :equal_sign, :string, :semicolon) do |bits|
+        localization_key = LocalizationKey.new(cleanup_string(bits[0]), nil)
+        localization_value = LocalizationValue.new(@language, cleanup_string(bits[2]))
+        localization_key << localization_value
+        return localization_key
+      end
+      match_bs(bs, :singleline_comment) do |bits|
+        return nil
+      end
+      match_bs(bs, :multiline_comment) do |bits|
+        return nil
+      end
+      raise "Syntax error: #{bs.shift(5).inspect}"
+    end
+
+    def match_bs(bs, *tokens)
+      return unless bs.size >= tokens.size
+      tokens.each_with_index do |token, idx|
+        return unless bs[idx][:token] == token 
+      end
+      yield bs.shift(tokens.size).map { |bit| bit[:value] }
+    end
+
+    def cleanup_comment(str)
+      if str.match(/^\/\/\s*/)
+        str.sub(/^\/\/\s*/, '')
+      else
+        str.sub(/^\/\*\s*/, '').sub(/\s*\*\/$/, '')
+      end
+    end
+
+    def cleanup_string(str)
+      str.sub(/^\"/, '').sub(/\"$/, '')
+    end
+  end
+end

data/lib/babelyoda/tanker.rb

@@ -0,0 +1,190 @@
+require 'builder'
+require 'net/http'
+require 'nokogiri'
+require 'stringio'
+
+require 'babelyoda/specification_loader'
+
+module Babelyoda
+  class Keyset 
+    def to_xml(xml, language = nil)
+      xml.keyset(:id => name) do
+        keys.each_value do |key|
+          xml.key(:id => key.id, :is_plural => 'False') do
+            xml.context(key.context)
+            key.values.each_value do |value|
+              next if language && (value.language.to_s != language.to_s)
+              xml.value(value.text, :language => value.language, :status => value.status)
+            end
+          end
+        end
+      end
+    end
+    
+    def self.parse_xml(node)
+      result = self.new(node[:id])
+      node.css('key').each do |key_node|
+        result.merge_key!(Babelyoda::LocalizationKey.parse_xml(key_node))
+      end
+      result
+    end
+  end
+
+  class LocalizationKey 
+    def self.parse_xml(node)
+      context = node.css('context').first
+      context &&= context.text
+      result = self.new(node[:id], context)
+      node.css('value').each do |value_node|
+        result << Babelyoda::LocalizationValue.parse_xml(value_node)
+      end
+      result
+    end
+  end
+  
+  class LocalizationValue
+    def self.parse_xml(node)
+      self.new(node[:language], node.text, node[:status])
+    end
+  end
+  
+	class Tanker
+		include Babelyoda::SpecificationLoader
+		
+		class FileNameInvalidError < RuntimeError ; end
+
+    attr_accessor :endpoint
+    attr_accessor :token
+    attr_accessor :project_id
+
+    def replace(keyset, language = nil)
+      doc = project_xml do |xml|
+        keyset.to_xml(xml, language)
+      end
+      payload = { 
+        :file => StringIO.new(doc),
+        'project-id' => project_id,
+        'keyset-id' => keyset.name,
+        :format => 'xml'
+      }
+      payload.merge!({:language => language}) if language
+      post('/keysets/replace/', payload)
+    end
+    
+    def list
+      get('/keysets/', { 'project-id' => project_id }).css('keyset').map { |keyset| keyset['id'] }
+    end
+    
+    def create(keyset_name)
+      post('/keysets/create/', { 'project-id' => project_id, 'keyset-id' => keyset_name })
+    end
+    
+    def export(keyset_name = nil, languages = nil, status = nil, safe = false)
+      payload = { 'project-id' => project_id }
+      payload.merge!({ 'keyset-id' => keyset_name }) if keyset_name
+      if languages
+        value = languages
+        value = languages.join(',') if languages.respond_to?(:join)
+        payload.merge!({ 'language' => value })
+      end
+      payload.merge!({ 'status' => status.to_s }) if status
+      payload.merge!({ 'safe' => safe }) if safe
+      get('/projects/export/xml/', payload)
+    end
+    
+    def load_keyset(keyset_name, languages = nil, status = nil, safe = false)
+      doc = export(keyset_name, languages, status, safe)
+      doc.css("keyset[@id='#{keyset_name}']").each do |keyset_node|
+        keyset = Babelyoda::Keyset.parse_xml(keyset_node)
+        return keyset if keyset.name == keyset_name
+      end
+      Babelyoda::Keyset.new(keyset_name)
+    end
+    
+  private
+  
+    MULTIPART_BOUNDARY = '114YANDEXTANKERCLIENTBNDR';
+
+    def multipart_content_type
+      "multipart/form-data; boundary=#{MULTIPART_BOUNDARY}"
+    end
+
+    def method(name)
+      "#{endpoint}#{name}"
+    end
+
+    def project_xml(&block)
+      xml = Builder::XmlMarkup.new
+      xml.instruct!(:xml, :encoding => "UTF-8")
+      xml.tanker do
+        xml.project(:id => project_id) do
+          yield xml
+        end
+      end
+    end
+
+    def multipart_data(payload = {}, boundary = MULTIPART_BOUNDARY)
+      payload.keys.map { |k|
+        "--#{boundary}\r\n" + multipart_field(k, payload[k])
+      }.join('') + "--#{boundary}--\r\n"
+    end
+
+    def multipart_field(k, v)
+      if v.respond_to?(:read)
+        "Content-Disposition: form-data; name=\"#{k}\"; filename=\"#{k}.xml\"\r\n" +
+        "Content-Type: application/octet-stream\r\n" +
+        "Content-Transfer-Encoding: binary\r\n\r\n" +
+        "#{v.read}\r\n"
+      else
+        "Content-Disposition: form-data; name=\"#{k}\"\r\n\r\n" +
+        "#{v}\r\n"
+      end
+    end
+
+    def post(method_name, payload)
+      uri = URI(method(method_name))
+      req = Net::HTTP::Post.new(uri.path)
+      req['AUTHORIZATION'] = token
+      req.content_type = multipart_content_type
+      req.body = multipart_data(payload)
+
+      # puts "POST URI: #{uri}"
+      # puts "POST BODY: #{req.body}"
+
+      res = Net::HTTP.start(uri.host, uri.port) do |http|
+        http.request(req)
+      end
+      
+      case res
+      when Net::HTTPSuccess, Net::HTTPRedirection
+        Nokogiri::XML.parse(res.body)
+      else
+        doc = Nokogiri::XML.parse(res.body)
+        error = doc.css('result error')[0].content
+        raise RuntimeError.new(error)
+      end
+    end
+
+    def get(method_name, payload = nil)
+      uri = URI(method(method_name))
+      uri.query = URI.encode_www_form(payload) if payload
+      req = Net::HTTP::Get.new(uri.request_uri)
+      req['AUTHORIZATION'] = token
+
+      # puts "GET URI: #{uri}"
+
+      res = Net::HTTP.start(uri.host, uri.port) do |http|
+        http.request(req)
+      end
+
+      case res
+      when Net::HTTPSuccess, Net::HTTPRedirection
+        Nokogiri::XML.parse(res.body)
+      else
+        doc = Nokogiri::XML.parse(res.body)
+        error = doc.css('result error')[0].content
+        raise Error.new(error)
+      end
+    end
+	end
+end

data/lib/babelyoda/version.rb

@@ -0,0 +1,3 @@
+module Babelyoda
+  VERSION = "2.0.0"
+end

data/lib/babelyoda/xib.rb

@@ -0,0 +1,94 @@
+require 'fileutils'
+
+require_relative 'file'
+require_relative 'ibtool'
+
+module Babelyoda
+  class Xib
+    attr_reader :filename
+    attr_reader :language
+    
+    def initialize(filename, language)
+      @filename, @language = filename, language
+    end
+    
+    def extractable?(development_language)
+      lproj_part = File.lproj_part(@filename)
+      (!lproj_part.nil?) && lproj_part == "#{development_language}.lproj"
+    end
+    
+    def dirname
+      File.dirname(@filename)
+    end
+    
+    def basename
+      File.basename(File.split(@filename)[1], '.xib')
+    end
+    
+    def resourced?
+      !File.lproj_part(@filename).nil?
+    end
+    
+    def resource!
+      raise "The XIB is already in a resource folder: #{@filename}" unless File.lproj_part(@filename).nil?
+      mv(File.localized(@filename, @language))
+    end
+    
+    def mv(new_filename)
+      FileUtils.mkdir_p(File.dirname(new_filename))
+      FileUtils.mv(@filename, new_filename)
+      @filename = new_filename
+    end
+    
+    def strings?
+      !strings.empty?
+    end
+    
+    def strings
+      Babelyoda::Ibtool.extract_strings(@filename, @language)
+    end
+    
+    def localize(language, scm)
+      puts "Localizing #{filename} => #{File.localized(filename, language)}..."
+      assert_localization_target(language)
+      strings_fn = strings_filename(language)
+      $logger.error "No strings file found: #{strings_fn}" unless File.exist?(strings_fn)
+      Babelyoda::Ibtool.localize(filename, File.localized(filename, language), strings_fn)
+    end
+    
+    def localize_incremental(language, scm)
+      assert_localization_target(language)
+      unless has_incremental_resources?(language, scm)
+        localize(language, scm)
+      else
+        puts "Incrementally localizing #{filename} => #{File.localized(filename, language)}..."
+        strings_fn = strings_filename(language)
+        $logger.error "No strings file found: #{strings_fn}" unless File.exist?(strings_fn)
+        
+        scm.fetch_versions!(filename, File.localized(filename, language)) do |filenames|
+          Babelyoda::Ibtool.localize_incrementally(filename, File.localized(filename, language), strings_fn, filenames[0], filenames[1])
+        end
+      end
+    end
+    
+    def strings_filename(language = nil)
+      language ? File.localized(File.join(dirname, "#{basename}.strings"), language) : File.join(dirname, "#{basename}.strings")
+    end
+    
+    def import_strings(scm)
+      Babelyoda::Ibtool.import_strings(filename, strings_filename)
+    end
+    
+  private
+    
+    def has_incremental_resources?(language, scm)
+      scm.version_exist?(filename) && scm.version_exist?(File.localized(filename, language))
+    end
+    
+    def assert_localization_target(language)
+      raise "Can't localize a XIB file that has not been put into an .lproj folder: #{filename}" unless resourced?
+      raise "Can't localize #{@language} to #{language} for: #{filename}" if @language == language
+    end
+        
+  end
+end

data/lib/babelyoda.rb

@@ -0,0 +1,207 @@
+BABELYODA_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+require 'awesome_print'
+
+require_relative 'babelyoda/genstrings'
+require_relative 'babelyoda/git'
+require_relative 'babelyoda/ibtool'
+require_relative 'babelyoda/keyset'
+require_relative 'babelyoda/localization_key'
+require_relative 'babelyoda/localization_value'
+require_relative 'babelyoda/rake'
+require_relative 'babelyoda/specification'
+require_relative 'babelyoda/tanker'
+require_relative 'babelyoda/xib'
+
+desc "Do a full localization cycle: push new strings, get translations and merge them"
+task :babelyoda => ['babelyoda:push', 'babelyoda:pull'] do
+end
+
+namespace :babelyoda do
+  
+  file 'Babelfile' do
+    Babelyoda::Specification.generate_default_babelfile
+  end
+  
+  desc "Create a basic bootstrap Babelfile"
+  task :init => 'Babelfile' do
+  end
+  
+  Babelyoda::Rake.spec do |spec|
+    
+    desc "Extract strings from sources"
+    task :extract_strings do
+      puts "Extracting strings from sources..."
+      dev_lang = spec.development_language
+
+      spec.scm.transaction("Extract strings from sources") do 
+        Babelyoda::Genstrings.run(spec.source_files, dev_lang) do |keyset|
+          old_strings_filename = strings_filename(keyset.name, dev_lang)
+          old_strings = Babelyoda::Strings.new(old_strings_filename, dev_lang).read
+          old_strings.merge!(keyset)
+          old_strings.save!
+          puts "  #{old_strings_filename}: #{old_strings.keys.size} keys"
+        end
+      end
+    end
+    
+    desc "Extract strings from XIBs"
+    task :extract_xib_strings do
+      puts "Extracting .strings from XIBs..."
+      spec.scm.transaction("Extract strings from XIBs") do 
+        spec.xib_files.each do |xib_filename|
+          xib = Babelyoda::Xib.new(xib_filename, spec.development_language)
+          next unless xib.extractable?(spec.development_language)
+          keyset = xib.strings
+          puts "  #{xib_filename} => #{xib.strings_filename}"
+          Babelyoda::Strings.save_keyset(keyset, xib.strings_filename, spec.development_language)
+        end
+      end
+    end
+    
+    desc "Extracts localizable strings into the corresponding .strings files"
+    task :extract => [:extract_strings, :extract_xib_strings] do
+    end
+    
+    desc "Create remote keysets for local keysets"
+    task :create_keysets => :extract do
+      # Create remote keysets for each local keyset if they don't exist.
+      puts "Creating remote keysets for local keysets..."
+      remote_keyset_names = spec.engine.list
+      spec.strings_files.each do |filename|
+        keyset_name = Babelyoda::Keyset.keyset_name(filename)
+        if remote_keyset_names.include?(keyset_name)
+          puts "  Tanker: An existing keyset found: #{keyset_name}"
+          next 
+        end
+        spec.engine.create(keyset_name)
+        puts "  Tanker: Created NEW keyset: #{keyset_name}"
+      end
+    end
+    
+    desc "Drops remote keys not found in local keysets"
+    task :drop_orphan_keys => :create_keysets do
+      puts "Dropping orphan keys..."
+      spec.strings_files.each do |filename|
+        strings = Babelyoda::Strings.new(filename, spec.development_language).read!
+        puts "  Processing keyset: #{strings.name}"
+        remote_keyset = spec.engine.load_keyset(strings.name)
+        keys_to_drop = []
+        remote_keyset.keys.each_value do |key|
+          unless strings.keys.has_key?(key.id)
+            keys_to_drop << key.id 
+            puts "    Found orphan key: #{key.id}"
+          end
+        end
+        keys_to_drop.each do |key|
+          remote_keyset.keys.delete(key)
+        end
+        spec.engine.replace(remote_keyset)
+        puts "    Dropped keys: #{keys_to_drop.size}"
+      end
+    end
+    
+    desc "Pushes resources to the translators"
+    task :push => :drop_orphan_keys do
+      puts "Pushing local keys to the remote..."
+      spec.strings_files.each do |filename|
+        strings = Babelyoda::Strings.new(filename, spec.development_language).read!
+        puts "  Processing keyset: #{strings.name}"
+        remote_keyset = spec.engine.load_keyset(strings.name, nil, :unapproved)
+        result = remote_keyset.merge!(strings, preserve: true)
+        remote_keyset.ensure_languages!(spec.all_languages)
+        spec.engine.replace(remote_keyset)
+        puts "    New keys: #{result[:new]} Updated keys: #{result[:updated]}"
+      end
+    end
+    
+    desc "Fetches remote strings and merges them down into local .string files"
+    task :fetch_strings do
+      puts "Fetching remote translations..."
+      spec.scm.transaction("Merge in remote translations") do 
+        spec.strings_files.each do |filename|
+          keyset_name = Babelyoda::Keyset.keyset_name(filename)
+          remote_keyset = spec.engine.load_keyset(keyset_name, nil, :unapproved, true)
+          remote_keyset.drop_empty!
+          spec.all_languages.each do |language|
+            keyset_filename = strings_filename(keyset_name, language)
+            Babelyoda::Strings.save_keyset(remote_keyset, keyset_filename, language)
+            puts "  #{keyset_filename}"
+          end
+        end
+      end
+    end
+    
+    desc "Incrementally localizes XIB files"
+    task :localize_xibs do
+      puts "Translating XIB files..."
+      
+      spec.scm.transaction("Localize XIB files") do 
+        spec.xib_files.each do |filename|
+          xib = Babelyoda::Xib.new(filename, spec.development_language)
+
+          xib.import_strings(spec.scm)
+          spec.localization_languages.each do |language|
+            xib.localize_incremental(language, spec.scm)
+          end
+        end
+      end
+      
+      spec.scm.transaction("Update XIB SHA1 version refs") do 
+        spec.xib_files.each do |filename|
+          spec.scm.store_version!(filename)
+          spec.localization_languages.each do |language|
+            spec.scm.store_version!(File.localized(filename, language))
+          end
+        end
+      end
+    end
+
+    desc "Pull remote translations"
+    task :pull => [:fetch_strings, :localize_xibs] do
+    end
+    
+    namespace :remote do
+      
+      desc "List remote keysets"
+      task :list do
+        ap spec.engine.list
+      end
+      
+      desc "Drop remote keysets in KEYSETS"
+      task :drop_keysets do
+        if ENV['KEYSETS']
+          keysets = ENV['KEYSETS'].split(',')
+          if keysets.include?('*')
+            keysets = spec.engine.list
+            puts "Dropping ALL keysets: #{keysets}"
+          else
+            puts "Dropping keysets: #{keysets}"            
+          end
+          keysets.each do |keyset_name|
+            puts "  Dropping: #{keyset_name}"
+            keyset = Babelyoda::Keyset.new(keyset_name)
+            key = Babelyoda::LocalizationKey.new("Dummy", "Dummy")
+            value = Babelyoda::LocalizationValue.new(:en, "Dummy")
+            key << value
+            keyset.merge_key!(key)
+            spec.engine.replace(keyset)
+          end
+          puts "All done!"
+        else
+          puts "Please provide keyset names to drop in the KEYSET environment variable. Separate by commas. Use * for ALL."
+        end
+      end
+
+    end
+  
+  end
+end
+
+def strings_filename(keyset_name, lang)
+  if keyset_name.match(/\//)
+    File.join(File.dirname(keyset_name), "#{lang}.lproj", "#{File.basename(keyset_name)}.strings")
+  else
+    File.join("#{lang}.lproj", "#{keyset_name}.strings")
+  end
+end

data/templates/Babelfile.erb

@@ -0,0 +1,15 @@
+Babelyoda::Specification.new do |s|
+	s.name = 'YOUR PROJECT NAME HERE'
+	s.development_language = :en
+	s.localization_languages = [:ru, :uk, :tr]
+	s.engine = Babelyoda::Tanker.new do |t|
+		t.token = 'FIX: TANKER TOKEN HERE'
+		t.project_id = 'FIX: TANKER PROJECT ID HERE'
+		t.endpoint = 'FIX: TANKER END POINT HERE'
+	end
+	s.scm = Babelyoda::Git.new
+	s.source_files = FileList['Classes/**/*.{m,mm,h}']
+	s.resources_folder = 'Resources'
+	s.xib_files = FileList['{Classes,Resources}/**/en.lproj/*.{xib}']
+	s.strings_files = FileList['{Classes,Resources}/**/en.lproj/*.{strings}']
+end