babelish 0.3.0

data/lib/babelish/csv2base.rb

@@ -0,0 +1,148 @@
+module Babelish
+  class Csv2Base
+    attr_accessor :default_path, :output_file
+    attr_accessor :langs
+    attr_accessor :csv_filename
+    attr_accessor :default_lang
+    attr_accessor :excluded_states, :state_column, :keys_column
+    attr_accessor :languages
+
+    def initialize(filename, langs, args = {})
+      default_args = {
+        :excluded_states => [],
+        :state_column => nil,
+        :keys_column => 0
+      }
+
+      args = default_args.merge!(args)
+
+      @langs = langs
+      if !@langs.is_a?(Hash) || @langs.size == 0
+        raise "wrong format or/and langs parameter" + @langs.inspect
+      end
+
+      @output_file = args[:output_file]
+      @default_path = args[:default_path].to_s
+      @csv_filename = filename
+      @excluded_states = args[:excluded_states]
+      @state_column = args[:state_column]
+      @keys_column = args[:keys_column]
+      @default_lang = args[:default_lang]
+
+      @languages = []
+    end
+
+    def create_file_from_path(file_path)
+      path = File.dirname(file_path)
+      FileUtils.mkdir_p path
+      return File.new(file_path, "w")
+    end
+
+    def language_filepaths(language)
+      #implement in subclass
+      []
+    end
+
+    def process_value(row_value, default_value)
+      value = row_value.nil? ? default_value : row_value
+      value = "" if value.nil?
+      value.gsub!(/\\*\"/, "\\\"") #escape double quotes
+      value.gsub!(/\s*(\n|\\\s*n)\s*/, "\\n") #replace new lines with \n + strip
+      value.gsub!(/%\s+([a-zA-Z@])([^a-zA-Z@]|$)/, "%\\1\\2") #repair string formats ("% d points" etc)
+      value.gsub!(/([^0-9\s\(\{\[^])%/, "\\1 %")
+        value.strip!
+        return value.to_utf8
+      end
+
+      def get_row_format(row_key, row_value)
+        return "\"" + row_key + "\" = \"" + row_value + "\""
+      end
+
+      # Convert csv file to multiple Localizable.strings files for each column
+      def convert(name = @csv_filename)
+        rowIndex     = 0
+        excludedCols = []
+        defaultCol   = 0
+
+        CSV.foreach(name, :quote_char => '"', :col_sep => ',', :row_sep => :auto) do |row|
+
+          if rowIndex == 0
+            #check there's at least two columns
+            return unless row.count > 1
+          else
+            #skip empty lines (or sections)
+            next if row == nil or row[@keys_column].nil?
+          end
+
+          # go through columns
+          row.size.times do |i|
+            next if excludedCols.include? i
+
+            #header
+            if rowIndex == 0
+              # ignore all headers not listed in langs to create files
+              (excludedCols << i and next) unless @langs.has_key?(row[i])
+
+              defaultCol = i if self.default_lang == row[i]
+              language = Language.new(row[i])
+              if @langs[row[i]].is_a?(Array)
+                @langs[row[i]].each do |id|
+                  language.add_language_id(id.to_s)
+                end
+              else
+                language.add_language_id(@langs[row[i]].to_s)
+              end
+              @languages[i] = language
+            elsif !@state_column || (row[@state_column].nil? || row[@state_column] == '' || !@excluded_states.include?(row[@state_column]))
+              # TODO: add option to strip the constant or referenced language
+              key = row[@keys_column].strip
+              value = self.process_value(row[i], row[defaultCol])
+
+              @languages[i].add_content_pair(key, value)
+            end
+          end
+
+          rowIndex += 1
+        end
+
+        write_content
+      end
+
+      def write_content
+        info = "List of created files:\n"
+        count = 0
+        @languages.each do |language|
+          next if language.nil?
+
+          files = []
+          if @output_file
+            files << create_file_from_path(@output_file)
+          else
+            language_filepaths(language).each do |filename|
+              files << create_file_from_path(filename)
+            end
+          end
+          files.each do |file|
+            file.write hash_to_output(language.content)
+            info += "#{file.path.to_s}\n"
+            count += 1
+
+            file.close
+          end
+        end
+
+        info = "Created #{count} files.\n" + info
+        return info
+      end
+
+      def hash_to_output(content = {})
+        output = ''
+        if content && content.size > 0
+          content.each do |key, value|
+            output += get_row_format(key, value)
+          end
+        end
+        return output
+      end
+    end
+  end

data/lib/babelish/csv2json.rb

@@ -0,0 +1,22 @@
+module Babelish
+  require 'json'
+  class CSV2JSON < Csv2Base
+    attr_accessor :file_path
+
+    def initialize(filename, langs, args = {})
+      super(filename, langs, args)
+
+      @file_path = args[:default_path].to_s
+    end
+
+    def language_filepaths(language)
+      require 'pathname'
+      filepath = Pathname.new("#{@file_path}#{language.code}.js")
+      return filepath ? [filepath] : []
+    end
+
+    def hash_to_output(content = {})
+      return content.to_json
+    end
+  end
+end

data/lib/babelish/csv2php.rb

@@ -0,0 +1,24 @@
+module Babelish
+  class CSV2Php < Csv2Base
+    attr_accessor :php_tag
+    attr_accessor :file_path
+
+    def initialize(filename, langs, args = {})
+      super(filename, langs, args)
+
+      @php_tag = args[:php_tag].nil? ? 'lang' : args[:php_tag]
+      @file_path = args[:default_path].to_s
+    end
+
+    def language_filepaths(language)
+      require 'pathname'
+      filepath = Pathname.new(@file_path) + "#{language.code}" + "lang.php"
+      return filepath ? [filepath] : []
+    end
+
+    def get_row_format(row_key, row_value)
+      return "$" + @php_tag + "['#{row_key}'] = \"#{row_value}\";\n"
+    end
+
+  end
+end

data/lib/babelish/csv2strings.rb

@@ -0,0 +1,30 @@
+module Babelish
+  class CSV2Strings < Csv2Base
+    attr_accessor :file_path
+
+    def initialize(filename, langs, args = {})
+      super(filename, langs, args)
+
+      @file_path = args[:default_path].to_s
+    end
+
+    def language_filepaths(language)
+      require 'pathname'
+      filepaths = []
+
+      if language.regions.empty?
+        filepaths << Pathname.new(@file_path) + "#{language.code}.lproj/Localizable.strings"
+      else
+        language.regions.each do |region|
+          filepaths << Pathname.new(@file_path) + "#{language.code}-#{region}.lproj/Localizable.strings"
+        end
+      end
+      filepaths
+    end
+
+    def get_row_format(row_key, row_value)
+      return "\"#{row_key}\" = \"#{row_value}\";\n"
+    end
+
+  end
+end

data/lib/babelish/google_doc.rb

@@ -0,0 +1,36 @@
+module Babelish
+  # Faraday is a dependency of google_drive, this silents the warning
+  # see https://github.com/CocoaPods/CocoaPods/commit/f33f967427b857bf73645fd4d3f19eb05e9be0e0
+  # This is to make sure Faraday doesn't warn the user about the `system_timer` gem missing.
+  old_warn, $-w = $-w, nil
+  begin
+    require "google_drive"
+  ensure
+    $-w = old_warn
+  end
+
+  class GoogleDoc
+    attr_accessor :session
+
+    def authenticate
+      # will try to get token from ~/.ruby_google_drive.token
+      @session = GoogleDrive.saved_session
+    end
+
+    def download(requested_filename, worksheet_index = nil, output_filename = "translations.csv")
+      unless @session
+        authenticate
+      end
+      result = @session.file_by_title(requested_filename)
+      if result.is_a? Array
+        file = result.first
+      else
+        file = result
+      end
+      return nil unless file
+      file.export_as_file(output_filename, "csv", worksheet_index)
+      return output_filename
+    end
+
+  end
+end

data/lib/babelish/json2csv.rb

@@ -0,0 +1,20 @@
+module Babelish
+  require 'json'
+  class JSON2CSV < Base2Csv
+
+    def initialize(args = {:filenames => []})
+      super(args)
+    end
+
+    def load_strings(strings_filename)
+      strings = {}
+      raise Errno::ENOENT unless File.exist?(strings_filename)
+      json_file = File.open(strings_filename, 'r')
+      json_string = json_file.read
+      json_file.close
+      strings = JSON.parse(json_string).to_hash
+      return strings
+    end
+
+  end
+end

data/lib/babelish/language.rb

@@ -0,0 +1,25 @@
+module Babelish
+  class Language
+    attr_accessor :name, :content, :code, :regions
+
+    def initialize(name, content = {})
+      @name = name
+      @content = content
+    end
+
+    def add_content_pair(key, value)
+      @content[key] = value
+    end
+
+    def add_language_id(language_id)
+      code, region = language_id.split('-')
+      @code ||= code
+      @regions ||= []
+      @regions << region if region
+    end
+
+    def region
+      @regions.first
+    end
+  end
+end

data/lib/babelish/php2csv.rb

@@ -0,0 +1,30 @@
+module Babelish
+  class Php2CSV < Base2Csv
+
+    def initialize(args = {:filenames => []})
+      super(args)
+    end
+
+    def load_strings(strings_filename)
+      strings = {}
+      File.open(strings_filename, 'r') do |strings_file|
+        strings_file.read.each_line do |line|
+          parsed_line = parse_dotstrings_line(line)
+          unless parsed_line.nil?
+            converted_line = parse_dotstrings_line(line)
+            strings.merge!(converted_line) unless converted_line.nil?
+          end
+        end
+      end
+      strings
+    end
+
+    def parse_dotstrings_line(line)
+      line.strip!
+      if line[0] != ?# && line[0] != ?=
+        m = line.match(/^[\$].*\[[\"\'](.*)[\"\']\]\s*=\s*[\"\'](.*)[\"\'];/)
+        return {m[1] => m[2]} unless m.nil?
+      end
+    end
+  end
+end

data/lib/babelish/strings2csv.rb

@@ -0,0 +1,47 @@
+module Babelish
+  class Strings2CSV < Base2Csv
+    # default_lang is the the column to refer to if a value is missing
+    # actually default_lang = default_filename
+    attr_accessor :csv_filename, :headers, :filenames, :default_lang
+
+    def initialize(args = {:filenames => []})
+      super(args)
+    end
+
+    def parse_dotstrings_line(line)
+      line.strip!
+      if line[0] != ?# && line[0] != ?=
+        m = line.match(/^[^\"]*\"(.+)\"[^=]+=[^\"]*\"(.*)\";/)
+        return {m[1] => m[2]} unless m.nil?
+      end
+    end
+
+    # Load all strings of a given file
+    def load_strings(strings_filename)
+      strings = {}
+
+      # genstrings uses utf16, so that's what we expect. utf8 should not be impact
+      file = File.open(strings_filename, "r:utf-16:utf-8")
+      begin
+        contents = file.read
+        if RUBY_VERSION == "1.9.2"
+          # fixes conversion, see http://po-ru.com/diary/fixing-invalid-utf-8-in-ruby-revisited/
+          require 'iconv'
+          ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+          contents = ic.iconv(contents + ' ')[0..-2]
+        end
+      rescue Encoding::InvalidByteSequenceError => e
+        # silent error
+        # faults back to utf8
+        contents = File.open(strings_filename, "r:utf-8")
+      end
+      contents.each_line do |line|
+        hash = self.parse_dotstrings_line(line)
+        strings.merge!(hash) if hash
+      end
+
+      strings
+    end
+
+  end
+end

data/lib/babelish/version.rb

@@ -0,0 +1,3 @@
+module Babelish
+  VERSION = '0.3.0'
+end

data/lib/babelish.rb

@@ -0,0 +1,27 @@
+require 'csv'
+
+# Fixes UTF8 issues
+# see http://stackoverflow.com/questions/4583924/string-force-encoding-in-ruby-1-8-7-or-rails-2-x
+class String
+  def to_utf8
+    force_encoding("UTF-8")
+  end
+end
+
+# From CSV
+require "babelish/csv2base"
+require "babelish/csv2strings"
+require "babelish/csv2android"
+require "babelish/csv2php"
+require "babelish/csv2json"
+
+# To CSV
+require "babelish/base2csv"
+require "babelish/strings2csv"
+require "babelish/android2csv"
+require "babelish/php2csv"
+require "babelish/json2csv"
+
+# General
+require "babelish/language"
+require "babelish/google_doc"

data/test/babelish/commands/test_command_android2csv.rb

@@ -0,0 +1,60 @@
+require 'test_helper'
+class TestAndroid2CSVCommand < Test::Unit::TestCase
+  def test_android2csv
+    options = {:filenames => ["test/data/android.xml"]}
+    Commandline.new([], options).android2csv
+
+    assert File.exist?("translations.csv")
+
+    #clean up
+    system("rm -f translations.csv")
+  end
+
+  def test_android2csv_with_dryrun_option
+    options = {:filenames => ["test/data/android.xml"], :dryrun => true}
+    Commandline.new([], options).android2csv
+
+    assert !File.exist?("translations.csv")
+
+    #clean up
+    system("rm -f translations.csv")
+  end
+
+  def test_android2csv_with_output_file
+    options = {:filenames => ["test/data/android.xml"], :csv_filename => "myfile.csv"}
+    # -i, -o
+    Commandline.new([], options).android2csv
+
+    assert File.exist?("myfile.csv")
+
+    #clean up
+    system("rm -f myfile.csv")
+  end
+
+  def test_android2csv_with_headers
+    options = {:filenames => ["test/data/android.xml"], :headers => ["constants", "english"]}
+    # -i, -h
+    Commandline.new([], options).android2csv
+
+    #TODO assertion or move test on at lib level
+
+    #clean up
+    system("rm -f translations.csv")
+  end
+
+  def test_android2csv_with_two_files
+    options = {
+      :filenames => ["test/data/android-en.xml", "test/data/android-fr.xml"],
+      :headers => %w{Constants English French},
+      :csv_filename => "enfr.csv"
+    }
+    # --filenames, --headers, -o
+    Commandline.new([], options).android2csv
+
+    #TODO assertion
+
+    #clean up
+    system("rm -f enfr.csv")
+  end
+
+end

data/test/babelish/commands/test_command_csv2android.rb

@@ -0,0 +1,35 @@
+require 'test_helper'
+class TestCSV2AndroidCommand < Test::Unit::TestCase
+
+  def test_csv2android_with_multiple_2_languages
+    options = {
+    	:filename => "test/data/test_data_multiple_langs.csv",
+    	:langs => {"English" => "en", "French" => "fr"}
+    }
+    Commandline.new([], options).csv2android
+
+    assert File.exist?("./values-en/strings.xml")
+    assert File.exist?("./values-fr/strings.xml")
+
+    #clean up
+    system("rm -rf ./values-en/")
+    system("rm -rf ./values-fr/")
+  end
+
+  def test_csv2android_with_default_path
+    options = {
+    	:filename => "test/data/test_data_multiple_langs.csv",
+    	:langs => {"English" => "en", "French" => "fr"},
+    	:default_path => "mynewlocation"
+    }
+
+    Commandline.new([], options).csv2android
+    # testing
+    assert File.exist?("./mynewlocation/values-en/strings.xml"), "can't find output file for English"
+    assert File.exist?("./mynewlocation/values-fr/strings.xml"), "can't find output file for French"
+
+    #clean up
+    system("rm -rf ./mynewlocation")
+  end
+
+end

data/test/babelish/commands/test_command_csv2strings.rb

@@ -0,0 +1,60 @@
+require 'test_helper'
+class TestCSV2StringsCommand < Test::Unit::TestCase
+
+  def test_csv2strings_with_multiple_2_languages
+    options = {
+    	:filename => "test/data/test_data_multiple_langs.csv",
+    	:langs => {"English" => "en", "French" => "fr"}
+    }
+    Commandline.new([], options).csv2strings
+
+    assert File.exist?("./en.lproj/Localizable.strings")
+    assert File.exist?("./fr.lproj/Localizable.strings")
+
+    #clean up
+    system("rm -rf ./en.lproj/")
+    system("rm -rf ./fr.lproj/")
+  end
+
+  def test_csv2strings_with_default_path
+    options = {
+    	:filename => "test/data/test_data_multiple_langs.csv",
+    	:langs => {"English" => "en", "French" => "fr"},
+    	:default_path => "mynewlocation"
+    }
+    Commandline.new([], options).csv2strings
+
+    # testing
+    assert File.exist?("./mynewlocation/en.lproj/Localizable.strings"), "can't find output file for English"
+    assert File.exist?("./mynewlocation/fr.lproj/Localizable.strings"), "can't find output file for French"
+
+    #clean up
+    system("rm -rf ./mynewlocation")
+  end
+
+  def test_csv2strings_with_fetch_google_doc
+    omit if ENV['TRAVIS']
+    options = {
+      :filename => "my_trads",
+      :langs => {"English" => "en", "French" => "fr"},
+      :fetch => true
+    }
+    assert_nothing_raised do
+      Commandline.new([], options).csv2strings
+    end
+  end
+
+  def test_csv2strings_with_config_file
+    system("cp .babelish.sample .babelish")
+
+    assert_nothing_raised do
+      Commandline.new.csv2strings
+    end
+
+  end
+
+  def teardown
+
+    system("rm -f .babelish")
+  end
+end

data/test/babelish/commands/test_command_strings2csv.rb

@@ -0,0 +1,95 @@
+require 'test_helper'
+class TestStrings2CSVCommand < Test::Unit::TestCase
+
+  def test_strings2csv
+    options = {:filenames => ["test/data/test_data.strings"]}
+    Commandline.new([], options).strings2csv
+
+    assert File.exist?("translations.csv")
+
+    #clean up
+    system("rm -f translations.csv")
+  end
+
+  def test_strings2csv_with_dryrun_option
+    options = {:filenames => ["test/data/test_data.strings"], :dryrun => true}
+    Commandline.new([], options).strings2csv
+
+    assert !File.exist?("translations.csv")
+
+    #clean up
+    system("rm -f translations.csv")
+  end
+
+  def test_strings2csv_with_output_file
+    options = {:filenames => ["test/data/test_data.strings"], :csv_filename => "myfile.csv"}
+    # -i, -o
+    Commandline.new([], options).strings2csv
+
+    assert File.exist?("myfile.csv")
+
+    #clean up
+    system("rm -f myfile.csv")
+  end
+
+  def test_strings2csv_with_headers
+    options = {:filenames => ["test/data/test_data.strings"], :headers => ["constants", "english"]}
+    # -i, -h
+    Commandline.new([], options).strings2csv
+
+    #TODO assertion or move test on at lib level
+
+    #clean up
+    system("rm -f translations.csv")
+  end
+
+  def test_strings2csv_with_two_files
+    options = {
+      :filenames => ["test/data/test_en.strings", "test/data/test_fr.strings"],
+      :headers => %w{Constants English French},
+      :csv_filename => "enfr.csv"
+    }
+    # --filenames, --headers, -o
+    Commandline.new([], options).strings2csv
+
+    #TODO assertion
+
+    #clean up
+    system("rm -f enfr.csv")
+  end
+
+  def test_strings2csv_with_empty_lines
+    options = {
+      :filenames => ["test/data/test_with_nil.strings"],
+      :csv_filename => "test_with_nil.csv"
+    }
+    # -i, -o
+
+
+    assert_nothing_raised do
+      Commandline.new([], options).strings2csv
+    end
+    assert system("diff test_with_nil.csv test/data/test_with_nil.csv"), "no difference on output"
+
+    #clean up
+    system("rm -f test_with_nil.csv")
+  end
+
+  def test_strings2csv_utf16
+    options = {
+      :filenames => ["test/data/test_utf16.strings"],
+      :csv_filename => "test_utf16.csv"
+    }
+    # -i, -o
+
+
+    assert_nothing_raised do
+      Commandline.new([], options).strings2csv
+    end
+
+    #clean up
+    system("rm -f test_utf16.csv")
+
+  end
+
+end