twine 0.7.0 → 0.8.0

data/lib/twine/formatters/gettext.rb

@@ -60,50 +60,43 @@ module Twine
         end
       end
 
-      def write_file(path, lang)
-        default_lang = @strings.language_codes[0]
-        encoding = @options[:output_encoding] || 'UTF-8'
-        File.open(path, "w:#{encoding}") do |f|
-          f.puts "msgid \"\"\nmsgstr \"\"\n\"Language: #{lang}\\n\"\n\"X-Generator: Twine #{Twine::VERSION}\\n\"\n\n"
-          @strings.sections.each do |section|
-            printed_section = false
-            section.rows.each do |row|
-              if row.matches_tags?(@options[:tags], @options[:untagged])
-                if !printed_section
-                  f.puts ''
-                    if section.name && section.name.length > 0
-                      section_name = section.name.gsub('--', '—')
-                      f.puts "# SECTION: #{section_name}"
-                    end
-                  printed_section = true
-                end
-
-                basetrans = row.translated_string_for_lang(default_lang)
-
-                if basetrans
-                  key = row.key
-                  key = key.gsub('"', '\\\\"')
-
-                  comment = row.comment
-                  if comment
-                    comment = comment.gsub('"', '\\\\"')
-                  end
-
-                  if comment && comment.length > 0
-                    f.print "#. \"#{comment}\"\n"
-                  end
-
-                  f.print "msgctxt \"#{key}\"\nmsgid \"#{basetrans}\"\n"
-                  value = row.translated_string_for_lang(lang)
-                  if value
-                    value = value.gsub('"', '\\\\"')
-                  end
-                  f.print "msgstr \"#{value}\"\n\n"
-                end
-              end
-            end
-          end
-        end
+      def format_file(strings, lang)
+        @default_lang = strings.language_codes[0]
+        super
+      end
+
+      def format_header(lang)
+        "msgid \"\"\nmsgstr \"\"\n\"Language: #{lang}\\n\"\n\"X-Generator: Twine #{Twine::VERSION}\\n\"\n"
+      end
+
+      def format_section_header(section)
+        "# SECTION: #{section.name}"
+      end
+
+      def row_pattern
+        "%{comment}%{key}%{base_translation}%{value}"
+      end
+
+      def format_row(row, lang)
+        return nil unless row.translated_string_for_lang(@default_lang)
+
+        super
+      end
+
+      def format_comment(row, lang)
+        "#. \"#{escape_quotes(row.comment)}\"\n" if row.comment
+      end
+
+      def format_key(row, lang)
+        "msgctxt \"#{row.key.dup}\"\n"
+      end
+
+      def format_base_translation(row, lang)
+        "msgid \"#{row.translations[@default_lang]}\"\n"
+      end
+
+      def format_value(row, lang)
+        "msgstr \"#{row.translated_string_for_lang(lang)}\"\n"
       end
     end
   end

data/lib/twine/formatters/jquery.rb

@@ -35,52 +35,50 @@ module Twine
         open(path) do |io|
           json = JSON.load(io)
           json.each do |key, value|
-            value.gsub!("\n","\\n")
             set_translation_for_key(key, lang, value)
           end
         end
       end
 
-      def write_file(path, lang)
-        begin
-          require "json"
-        rescue LoadError
-          raise Twine::Error.new "You must run 'gem install json' in order to read or write jquery-localize files."
-        end
+      def format_file(strings, lang)
+        "{\n#{super}\n}"
+      end
 
-        printed_string = false
-        default_lang = @strings.language_codes[0]
-        encoding = @options[:output_encoding] || 'UTF-8'
-        File.open(path, "w:#{encoding}") do |f|
-          f.print "{"
+      def format_sections(strings, lang)
+        sections = strings.sections.map { |section| format_section(section, lang) }
+        sections.join(",\n\n")
+      end
+
+      def format_section_header(section)
+      end
 
-          @strings.sections.each_with_index do |section, si|
-            printed_section = false
-            section.rows.each_with_index do |row, ri|
-              if row.matches_tags?(@options[:tags], @options[:untagged])
-                if printed_string
-                  f.print ",\n"
-                end
+      def format_section(section, lang)
+        rows = section.rows.dup
 
-                if !printed_section
-                  f.print "\n"
-                  printed_section = true
-                end
+        rows.map! { |row| format_row(row, lang) }
+        rows.compact! # remove nil entries
+        rows.join(",\n")
+      end
 
-                key = row.key
-                key = key.gsub('"', '\\\\"')
+      def key_value_pattern
+        "\"%{key}\":\"%{value}\""
+      end
 
-                value = row.translated_string_for_lang(lang, default_lang)
-                value = value.gsub('"', '\\\\"')
+      def format_key(key)
+        escape_quotes(key)
+      end
 
-                f.print "\"#{key}\":\"#{value}\""
-                printed_string = true
-              end
-            end
-          end
-          f.puts "\n}"
+      def format_value(value)
+        escape_quotes(value)
+      end
 
+      def write_file(path, lang)
+        begin
+          require "json"
+        rescue LoadError
+          raise Twine::Error.new "You must run 'gem install json' in order to read or write jquery-localize files."
         end
+        super
       end
     end
   end

data/lib/twine/formatters/tizen.rb

@@ -5,6 +5,8 @@ require 'rexml/document'
 module Twine
   module Formatters
     class Tizen < Abstract
+      include Twine::Placeholders
+
       FORMAT_NAME = 'tizen'
       EXTENSION = '.xml'
       DEFAULT_FILE_NAME = 'strings.xml'
@@ -90,7 +92,7 @@ module Twine
                   value = CGI.unescapeHTML(value)
                   value.gsub!('\\\'', '\'')
                   value.gsub!('\\"', '"')
-                  value = iosify_substitutions(value)
+                  value = convert_placeholders_from_android_to_twine(value)
                   value.gsub!(/(\\u0020)*|(\\u0020)*\z/) { |spaces| ' ' * (spaces.length / 6) }
                 else
                   value = ""
@@ -111,64 +113,45 @@ module Twine
         end
       end
 
-      def write_file(path, lang)
-        default_lang = nil
-        if DEFAULT_LANG_CODES.has_key?(lang)
-          default_lang = DEFAULT_LANG_CODES[lang]
-        end
-        File.open(path, 'w:UTF-8') do |f|
-          f.puts "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!-- Tizen Strings File -->\n<!-- Generated by Twine #{Twine::VERSION} -->\n<!-- Language: #{lang} -->"
-          f.write '<string_table  Bversion="2.0.0.201311071819" Dversion="20120315">'
-          @strings.sections.each do |section|
-            printed_section = false
-            section.rows.each do |row|
-              if row.matches_tags?(@options[:tags], @options[:untagged])
-                if !printed_section
-                  f.puts ''
-                  if section.name && section.name.length > 0
-                    section_name = section.name.gsub('--', '—')
-                    f.puts "\t<!-- SECTION: #{section_name} -->"
-                  end
-                  printed_section = true
-                end
+      def format_header(lang)
+        "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!-- Tizen Strings File -->\n<!-- Generated by Twine #{Twine::VERSION} -->\n<!-- Language: #{lang} -->"
+      end
 
-                key = row.key
+      def format_sections(strings, lang)
+        result = '<string_table  Bversion="2.0.0.201311071819" Dversion="20120315">'
+        
+        result += super + "\n"
 
-                value = row.translated_string_for_lang(lang, default_lang)
-                if !value && @options[:include_untranslated]
-                  value = row.translated_string_for_lang(@strings.language_codes[0])
-                end
+        result += '</string_table>'
+      end
 
-                if value # if values is nil, there was no appropriate translation, so let Tizen handle the defaulting
-                  value = String.new(value) # use a copy to prevent modifying the original
-
-                  # Tizen enforces the following rules on the values
-                  #  1) apostrophes and quotes must be escaped with a backslash
-                  value.gsub!('\'', '\\\\\'')
-                  value.gsub!('"', '\\\\"')
-                  #  2) HTML escape the string
-                  value = CGI.escapeHTML(value)
-                  #  3) fix substitutions (e.g. %s/%@)
-                  value = androidify_substitutions(value)
-                  #  4) replace beginning and end spaces with \0020. Otherwise Tizen strips them.
-                  value.gsub!(/\A *| *\z/) { |spaces| '\u0020' * spaces.length }
-
-                  comment = row.comment
-                  if comment
-                    comment = comment.gsub('--', '—')
-                  end
-
-                  if comment && comment.length > 0
-                    f.puts "\t<!-- #{comment} -->\n"
-                  end
-                  f.puts "\t<text id=\"IDS_#{key.upcase}\">#{value}</text>"
-                end
-              end
-            end
-          end
+      def format_section_header(section)
+        "\t<!-- SECTION: #{section.name} -->"
+      end
 
-          f.puts '</string_table>'
-        end
+      def format_comment(row, lang)
+        "\t<!-- #{row.comment.gsub('--', '—')} -->\n" if row.comment
+      end
+
+      def key_value_pattern
+        "\t<text id=\"IDS_%{key}\">%{value}</text>"
+      end
+
+      def format_key(key)
+        key.upcase
+      end
+
+      def format_value(value)
+        value = escape_quotes(value)
+        # Tizen enforces the following rules on the values
+        #  1) apostrophes and quotes must be escaped with a backslash
+        value.gsub!("'", "\\\\'")
+        #  2) HTML escape the string
+        value = CGI.escapeHTML(value)
+        #  3) fix substitutions (e.g. %s/%@)
+        value = convert_placeholders_from_twine_to_android(value)
+        #  4) replace beginning and end spaces with \0020. Otherwise Tizen strips them.
+        value.gsub(/\A *| *\z/) { |spaces| '\u0020' * spaces.length }
       end
     end
   end

data/lib/twine/output_processor.rb

@@ -0,0 +1,57 @@
+module Twine
+  module Processors
+
+    class OutputProcessor
+      def initialize(strings, options)
+        @strings = strings
+        @options = options
+      end
+
+      def default_language
+        @options[:developer_language] || @strings.language_codes[0]
+      end
+
+      def fallback_languages(language)
+        fallback_mapping = {
+          'zh-TW' => 'zh-Hant' # if we don't have a zh-TW translation, try zh-Hant before en
+        }
+
+        [fallback_mapping[language], default_language].flatten.compact
+      end
+
+      def process(language)
+        result = StringsFile.new
+
+        result.language_codes.concat @strings.language_codes
+        @strings.sections.each do |section|
+          new_section = StringsSection.new section.name
+
+          section.rows.each do |row|
+            next unless row.matches_tags?(@options[:tags], @options[:untagged])
+
+            value = row.translated_string_for_lang(language)
+
+            next if value && @options[:include] == 'untranslated'
+
+            if value.nil? && @options[:include] != 'translated'
+              value = row.translated_string_for_lang(fallback_languages(language))
+            end
+
+            next unless value
+
+            new_row = row.dup
+            new_row.translations[language] = value
+
+            new_section.rows << new_row
+            result.strings_map[new_row.key] = new_row
+          end
+
+          result.sections << new_section
+        end
+
+        return result
+      end
+    end
+
+  end
+end

data/lib/twine/placeholders.rb

@@ -0,0 +1,54 @@
+module Twine
+  module Placeholders
+    extend self
+
+    PLACEHOLDER_FLAGS_WIDTH_PRECISION_LENGTH = '([-+ 0#])?(\d+|\*)?(\.(\d+|\*))?(hh?|ll?|L|z|j|t)?'
+    PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH = '(\d+\$)?' + PLACEHOLDER_FLAGS_WIDTH_PRECISION_LENGTH
+
+    # http://developer.android.com/guide/topics/resources/string-resource.html#FormattingAndStyling
+    # http://stackoverflow.com/questions/4414389/android-xml-percent-symbol
+    # https://github.com/mobiata/twine/pull/106
+    def convert_placeholders_from_twine_to_android(input)
+      placeholder_types = '[diufFeEgGxXoscpaA]'
+
+      # %@ -> %s
+      value = input.gsub(/(%#{PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH})@/, '\1s')
+
+      placeholder_syntax = PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH + placeholder_types
+      placeholder_regex = /%#{placeholder_syntax}/
+
+      number_of_placeholders = value.scan(placeholder_regex).size
+
+      return value if number_of_placeholders == 0
+
+      # got placeholders -> need to double single percent signs
+      # % -> %% (but %% -> %%, %d -> %d)
+      single_percent_regex = /([^%])(%)(?!(%|#{placeholder_syntax}))/
+      value.gsub! single_percent_regex, '\1%%'
+
+      return value if number_of_placeholders < 2
+
+      # number placeholders
+      non_numbered_placeholder_regex = /%(#{PLACEHOLDER_FLAGS_WIDTH_PRECISION_LENGTH}#{placeholder_types})/
+
+      number_of_non_numbered_placeholders = value.scan(non_numbered_placeholder_regex).size
+
+      return value if number_of_non_numbered_placeholders == 0
+
+      raise Twine::Error.new("The value \"#{input}\" contains numbered and non-numbered placeholders") if number_of_placeholders != number_of_non_numbered_placeholders
+
+      # %d -> %$1d
+      index = 0
+      value.gsub!(non_numbered_placeholder_regex) { "%#{index += 1}$#{$1}" }
+
+      value
+    end
+
+    def convert_placeholders_from_android_to_twine(input)
+      placeholder_regex = /(%#{PLACEHOLDER_PARAMETER_FLAGS_WIDTH_PRECISION_LENGTH})s/
+
+      # %s -> %@
+      input.gsub(placeholder_regex, '\1@')
+    end
+  end
+end

data/lib/twine/runner.rb

@@ -1,30 +1,38 @@
 require 'tmpdir'
+require 'fileutils'
 
 Twine::Plugin.new # Initialize plugins first in Runner.
 
 module Twine
-  VALID_COMMANDS = ['generate-string-file', 'generate-all-string-files', 'consume-string-file', 'consume-all-string-files', 'generate-loc-drop', 'consume-loc-drop', 'validate-strings-file']
-
   class Runner
-    def initialize(args)
-      @options = {}
-      @args = args
-    end
-
     def self.run(args)
-      new(args).run
-    end
+      options = CLI.parse(args)
+      
+      strings = StringsFile.new
+      strings.read options[:strings_file]
+      runner = new(options, strings)
 
-    def run
-      # Parse all CLI arguments.
-      CLI::parse_args(@args, @options)
-      read_strings_data
-      execute_command
+      case options[:command]
+      when 'generate-string-file'
+        runner.generate_string_file
+      when 'generate-all-string-files'
+        runner.generate_all_string_files
+      when 'consume-string-file'
+        runner.consume_string_file
+      when 'consume-all-string-files'
+        runner.consume_all_string_files
+      when 'generate-loc-drop'
+        runner.generate_loc_drop
+      when 'consume-loc-drop'
+        runner.consume_loc_drop
+      when 'validate-strings-file'
+        runner.validate_strings_file
+      end
     end
 
-    def read_strings_data
-      @strings = StringsFile.new
-      @strings.read @options[:strings_file]
+    def initialize(options = {}, strings = StringsFile.new)
+      @options = options
+      @strings = strings
     end
 
     def write_strings_data(path)
@@ -34,44 +42,24 @@ module Twine
       @strings.write(path)
     end
 
-    def execute_command
-      case @options[:command]
-      when 'generate-string-file'
-        generate_string_file
-      when 'generate-all-string-files'
-        generate_all_string_files
-      when 'consume-string-file'
-        consume_string_file
-      when 'consume-all-string-files'
-        consume_all_string_files
-      when 'generate-loc-drop'
-        generate_loc_drop
-      when 'consume-loc-drop'
-        consume_loc_drop
-      when 'validate-strings-file'
-        validate_strings_file
-      end
-    end
-
     def generate_string_file
       lang = nil
-      if @options[:languages]
-        lang = @options[:languages][0]
-      end
+      lang = @options[:languages][0] if @options[:languages]
 
       read_write_string_file(@options[:output_path], false, lang)
     end
 
     def generate_all_string_files
       if !File.directory?(@options[:output_path])
-        raise Twine::Error.new("Directory does not exist: #{@options[:output_path]}")
+        if @options[:create_folders]
+          FileUtils.mkdir_p(@options[:output_path])
+        else
+          raise Twine::Error.new("Directory does not exist: #{@options[:output_path]}")
+        end
       end
 
-      format = @options[:format]
-      if !format
-        format = determine_format_given_directory(@options[:output_path])
-      end
-      if !format
+      format = @options[:format] || determine_format_given_directory(@options[:output_path])
+      unless format
         raise Twine::Error.new "Could not determine format given the contents of #{@options[:output_path]}"
       end
 
@@ -101,7 +89,7 @@ module Twine
           begin
             read_write_string_file(item, true, nil)
           rescue Twine::Error => e
-            STDERR.puts "#{e.message}"
+            Twine::stderr.puts "#{e.message}"
           end
         end
       end
@@ -115,23 +103,15 @@ module Twine
         raise Twine::Error.new("File does not exist: #{path}")
       end
 
-      format = @options[:format]
-      if !format
-        format = determine_format_given_path(path)
-      end
-      if !format
+      format = @options[:format] || determine_format_given_path(path)
+      unless format
         raise Twine::Error.new "Unable to determine format of #{path}"
       end
 
       formatter = formatter_for_format(format)
 
-      if !lang
-        lang = determine_language_given_path(path)
-      end
-      if !lang
-        lang = formatter.determine_language_given_path(path)
-      end
-      if !lang
+      lang = lang || determine_language_given_path(path) || formatter.determine_language_given_path(path)
+      unless lang
         raise Twine::Error.new "Unable to determine language for #{path}"
       end
 
@@ -147,18 +127,14 @@ module Twine
     end
 
     def generate_loc_drop
-      begin
-        require 'zip/zip'
-      rescue LoadError
-        raise Twine::Error.new "You must run 'gem install rubyzip' in order to create or consume localization drops."
-      end
+      require_rubyzip
 
       if File.file?(@options[:output_path])
         File.delete(@options[:output_path])
       end
 
       Dir.mktmpdir do |dir|
-        Zip::ZipFile.open(@options[:output_path], Zip::ZipFile::CREATE) do |zipfile|
+        Zip::File.open(@options[:output_path], Zip::File::CREATE) do |zipfile|
           zipfile.mkdir('Locales')
 
           formatter = formatter_for_format(@options[:format])
@@ -176,18 +152,14 @@ module Twine
     end
 
     def consume_loc_drop
+      require_rubyzip
+
       if !File.file?(@options[:input_path])
         raise Twine::Error.new("File does not exist: #{@options[:input_path]}")
       end
 
-      begin
-        require 'zip/zip'
-      rescue LoadError
-        raise Twine::Error.new "You must run 'gem install rubyzip' in order to create or consume localization drops."
-      end
-
       Dir.mktmpdir do |dir|
-        Zip::ZipFile.open(@options[:input_path]) do |zipfile|
+        Zip::File.open(@options[:input_path]) do |zipfile|
           zipfile.each do |entry|
             if !entry.name.end_with?'/' and !File.basename(entry.name).start_with?'.'
               real_path = File.join(dir, entry.name)
@@ -196,7 +168,7 @@ module Twine
               begin
                 read_write_string_file(real_path, true, nil)
               rescue Twine::Error => e
-                STDERR.puts "#{e.message}"
+                Twine::stderr.puts "#{e.message}"
               end
             end
           end
@@ -212,81 +184,72 @@ module Twine
       all_keys = Set.new
       duplicate_keys = Set.new
       keys_without_tags = Set.new
-      errors = []
+      invalid_keys = Set.new
+      valid_key_regex = /^[A-Za-z0-9_]+$/
 
       @strings.sections.each do |section|
         section.rows.each do |row|
           total_strings += 1
 
-          if all_keys.include? row.key
-            duplicate_keys.add(row.key)
-          else
-            all_keys.add(row.key)
-          end
+          duplicate_keys.add(row.key) if all_keys.include? row.key
+          all_keys.add(row.key)
 
-          if row.tags == nil || row.tags.length == 0
-            keys_without_tags.add(row.key)
-          end
+          keys_without_tags.add(row.key) if row.tags == nil or row.tags.length == 0
+
+          invalid_keys << row.key unless row.key =~ valid_key_regex
         end
       end
 
-      if duplicate_keys.length > 0
-        error_body = duplicate_keys.to_a.join("\n  ")
-        errors << "Found duplicate string key(s):\n  #{error_body}"
+      errors = []
+      join_keys = lambda { |set| set.map { |k| "  " + k }.join("\n") }
+
+      unless duplicate_keys.empty?
+        errors << "Found duplicate string key(s):\n#{join_keys.call(duplicate_keys)}"
       end
 
       if keys_without_tags.length == total_strings
         errors << "None of your strings have tags."
       elsif keys_without_tags.length > 0
-        error_body = keys_without_tags.to_a.join("\n  ")
-        errors << "Found strings(s) without tags:\n  #{error_body}"
+        errors << "Found strings without tags:\n#{join_keys.call(keys_without_tags)}"
       end
 
-      if errors.length > 0
-        raise Twine::Error.new errors.join("\n\n")
+      unless invalid_keys.empty?
+        errors << "Found key(s) with invalid characters:\n#{join_keys.call(invalid_keys)}"
       end
 
-      puts "#{@options[:strings_file]} is valid."
+      raise Twine::Error.new errors.join("\n\n") unless errors.empty?
+
+      Twine::stdout.puts "#{@options[:strings_file]} is valid."
     end
 
     def determine_language_given_path(path)
       code = File.basename(path, File.extname(path))
-      if !@strings.language_codes.include? code
-        code = nil
-      end
-
-      code
+      return code if @strings.language_codes.include? code
     end
 
     def determine_format_given_path(path)
-      ext = File.extname(path)
-      Formatters.formatters.each do |formatter|
-        if formatter::EXTENSION == ext
-          return formatter::FORMAT_NAME
-        end
-      end
-
-      return
+      formatter = Formatters.formatters.find { |f| f::EXTENSION == File.extname(path) }
+      return formatter::FORMAT_NAME if formatter
     end
 
     def determine_format_given_directory(directory)
-      Formatters.formatters.each do |formatter|
-        if formatter.can_handle_directory?(directory)
-          return formatter::FORMAT_NAME
-        end
-      end
-
-      return
+      formatter = Formatters.formatters.find { |f| f.can_handle_directory?(directory) }
+      return formatter::FORMAT_NAME if formatter
     end
 
     def formatter_for_format(format)
-      Formatters.formatters.each do |formatter|
-        if formatter::FORMAT_NAME == format
-          return formatter.new(@strings, @options)
-        end
-      end
+      formatter = Formatters.formatters.find { |f| f::FORMAT_NAME == format }
+      return formatter.new(@strings, @options) if formatter
+    end
+
+    private
 
-      return
+    def require_rubyzip
+      begin
+        require 'zip'
+      rescue LoadError
+        raise Twine::Error.new "You must run 'gem install rubyzip' in order to create or consume localization drops."
+      end
     end
   end
 end