twine 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd42325c94b784158be7dd018af67a02fb3099b0
-  data.tar.gz: 7ee4dae663a07911d402a236fb1f40692b58cc43
+  metadata.gz: 46fb29dd5d1ec899c427901e72e40c081fa8c6c4
+  data.tar.gz: 3faf58c634c4d1903fda10a1ece3cace658dc761
 SHA512:
-  metadata.gz: 54345a2371c8717af7b2285b7fee737b082695501da2e5ecaaee04f76762b6aa2a31eefc0c42c5a95d15ae06af38d1e8b31dfb34b6a20b56c86417044667255e
-  data.tar.gz: 681b338e4b3f9efc9af0659c785b8fcaabb3878e6b3080b8fce7aca0856094e52f0414f25591c27c7d2688c82c45e671ddd6bbb2d25c59afdadb16858005d427
+  metadata.gz: 6ec4b6098a24520cba05e4bb2bd0ea16602c9b52382abfeb45e9bf0308a4b1c03287251160cdec49cb26f44ccd69ff96d17f5dd0c9d56b7612068435c750dfb2
+  data.tar.gz: b1a85e5a3fb56e20a790e7beb8649784f5ffab42f4e8b6db7132680a634275caad8ab698a09369d9b46821c873bf8a51c416f941337d1b101ad079f6bd6d2960

data/README.md

@@ -27,6 +27,10 @@ Twine stores all of its strings in a single file. The format of this file is a s
 
 Each grouping section contains N string definitions. These string definitions start with the string key placed within a single pair of square brackets. This string definition then contains a number of key-value pairs, including a comment, a comma-separated list of tags (which are used by Twine to select a subset of strings), and all of the translations.
 
+### Placeholders
+
+Twine supports [`printf` style placeholders](https://en.wikipedia.org/wiki/Printf_format_string) with one peculiarity: `@` is used for strings instead of `s`. This is because Twine started out as a tool for iOS and OS X projects.
+
 ### Tags
 
 Tags are used by Twine as a way to only work with a subset of your strings at any given point in time. Each string can be assigned zero or more tags which are separated by commas. Tags are optional, though highly recommended. You can get a list of all strings currently missing tags by executing the `validate-strings-file` command.
@@ -35,6 +39,10 @@ Tags are used by Twine as a way to only work with a subset of your strings at an
 
 Whitepace in this file is mostly ignored. If you absolutely need to put spaces at the beginning or end of your translated string, you can wrap the entire string in a pair of `` ` `` characters. If your actual string needs to start *and* end with a grave accent, you can wrap it in another pair of `` ` `` characters. See the example, below.
 
+### References
+
+If you want a key to inherit the values of another key, you can use a reference. Any property not specified for a key will be taken from the reference.
+
 ### Example
 
 	[[General]]
@@ -57,6 +65,9 @@ Whitepace in this file is mostly ignored. If you absolutely need to put spaces a
 			en = The network is currently unavailable.
 			tags = app1
 			comment = An error describing when the device can not connect to the internet.
+		[dismiss_error]
+			ref = yes
+			en = Dismiss
 
 	[[Escaping Example]]
 		[list_item_separator]
@@ -216,6 +227,7 @@ Many thanks to all of the contributors to the Twine project, including:
 * [Kevin Wood](https://github.com/kwood)
 * [Mohammad Hejazi](https://github.com/MohammadHejazi)
 * [Robert Guo](http://www.robertguo.me/)
+* [sebastianludwig](https://github.com/sebastianludwig)
 * [Sergey Pisarchik](https://github.com/SergeyPisarchik)
 * [Shai Shamir](https://github.com/pichirichi)
 

data/lib/twine.rb

@@ -1,10 +1,31 @@
 module Twine
+  @@stdout = STDOUT
+  @@stderr = STDERR
+
+  def self.stdout
+    @@stdout
+  end
+
+  def self.stdout=(out)
+    @@stdout = out
+  end
+
+  def self.stderr
+    @@stderr
+  end
+
+  def self.stderr=(err)
+    @@stderr = err
+  end
+
   class Error < StandardError
   end
 
   require 'twine/plugin'
   require 'twine/cli'
   require 'twine/encoding'
+  require 'twine/output_processor'
+  require 'twine/placeholders'
   require 'twine/formatters'
   require 'twine/runner'
   require 'twine/stringsfile'

data/lib/twine/cli.rb

@@ -1,17 +1,19 @@
 require 'optparse'
 
 module Twine
-  class CLI
-    def initialize(args, options)
-      @options = options
-      @args = args
-    end
-
-    def self.parse_args(args, options)
-      new(args, options).parse_args
-    end
+  module CLI
+    NEEDED_COMMAND_ARGUMENTS = {
+      'generate-string-file' => 3,
+      'generate-all-string-files' => 3,
+      'consume-string-file' => 3,
+      'consume-all-string-files' => 3,
+      'generate-loc-drop' => 3,
+      'consume-loc-drop' => 3,
+      'validate-strings-file' => 2
+    }
 
-    def parse_args
+    def self.parse(args)
+      options = {}
       parser = OptionParser.new do |opts|
         opts.banner = 'Usage: twine COMMAND STRINGS_FILE [INPUT_OR_OUTPUT_PATH] [--lang LANG1,LANG2...] [--tags TAG1,TAG2,TAG3...] [--format FORMAT]'
         opts.separator ''
@@ -27,7 +29,7 @@ module Twine
         opts.separator ''
         opts.separator 'consume-all-string-files -- Slurps all of the strings from a directory into the specified STRINGS_FILE. If you have some files returned to you by your translators you can use this command to incorporate all of their changes. This script will attempt to guess both the language and the format given the filename and extension. For example, "ja.strings" will assume that the file is a Japanese iOS strings file.'
         opts.separator ''
-        opts.separator 'generate-loc-drop -- Generates a zip archive of strings files in any format. The purpose of this command is to create a very simple archive that can be handed off to a translation team. The translation team can unzip the archive, translate all of the strings in the archived files, zip everything back up, and then hand that final archive back to be consumed by the consume-loc-drop command. This command assumes that --include-untranslated has been specified on the command line.'
+        opts.separator 'generate-loc-drop -- Generates a zip archive of strings files in any format. The purpose of this command is to create a very simple archive that can be handed off to a translation team. The translation team can unzip the archive, translate all of the strings in the archived files, zip everything back up, and then hand that final archive back to be consumed by the consume-loc-drop command.'
         opts.separator ''
         opts.separator 'consume-loc-drop -- Consumes an archive of translated files. This archive should be in the same format as the one created by the generate-loc-drop command.'
         opts.separator ''
@@ -36,48 +38,56 @@ module Twine
         opts.separator 'General Options:'
         opts.separator ''
         opts.on('-l', '--lang LANGUAGES', Array, 'The language code(s) to use for the specified action.') do |langs|
-          @options[:languages] = langs
+          options[:languages] = langs
         end
         opts.on('-t', '--tags TAGS', Array, 'The tag(s) to use for the specified action. Only strings with that tag will be processed. Do not specify any tags to match all strings in the strings data file.') do |tags|
-          @options[:tags] = tags
+          options[:tags] = tags
         end
         opts.on('-u', '--untagged', 'If you have specified tags using the --tags flag, then only those tags will be selected. If you also want to select all strings that are untagged, then you can specify this option to do so.') do |u|
-          @options[:untagged] = true
-        end
-        formats = []
-        Formatters.formatters.each do |formatter|
-          formats << formatter::FORMAT_NAME
+          options[:untagged] = true
         end
+        formats = Formatters.formatters.map { |f| f::FORMAT_NAME }
         opts.on('-f', '--format FORMAT', "The file format to read or write (#{formats.join(', ')}). Additional formatters can be placed in the formats/ directory.") do |format|
-          lformat = format.downcase
-          if !formats.include?(lformat)
-            STDERR.puts "Invalid format: #{format}"
+          unless formats.include?(format.downcase)
+            raise Twine::Error.new "Invalid format: #{format}"
           end
-          @options[:format] = lformat
+          options[:format] = format.downcase
         end
         opts.on('-a', '--consume-all', 'Normally, when consuming a string file, Twine will ignore any string keys that do not exist in your master file.') do |a|
-          @options[:consume_all] = true
+          options[:consume_all] = true
+        end
+        opts.on('-i', '--include SET', "This flag will determine which strings are included when generating strings files. It's possible values:",
+                                        "  all: All strings both translated and untranslated for the specified language are included. This is the default value.",
+                                        "  translated: Only translated strings are included.",
+                                        "  untranslated: Only untranslated strings are included.") do |set|
+          unless ['all', 'translated', 'untranslated'].include?(set.downcase)
+            raise Twine::Error.new "Invalid include flag: #{set}"
+          end
+          options[:include] = set.downcase
         end
-        opts.on('-s', '--include-untranslated', 'This flag will cause any Android string files that are generated to include strings that have not yet been translated for the current language.') do |s|
-          @options[:include_untranslated] = true
+        unless options[:include]
+          options[:include] = 'all' 
         end
         opts.on('-o', '--output-file OUTPUT_FILE', 'Write the new strings database to this file instead of replacing the original file. This flag is only useful when running the consume-string-file or consume-loc-drop commands.') do |o|
-          @options[:output_path] = o
+          options[:output_path] = o
         end
         opts.on('-n', '--file-name FILE_NAME', 'When running the generate-all-string-files command, this flag may be used to overwrite the default file name of the format.') do |n|
-          @options[:file_name] = n
+          options[:file_name] = n
+        end
+        opts.on('-r', '--create-folders', "When running the generate-all-string-files command, this flag may be used to create output folders for all languages, if they don't exist yet. As a result all languages will be exported, not only the ones where an output folder already exists.") do |r|
+          options[:create_folders] = true
         end
-        opts.on('-d', '--developer-language LANG', 'When writing the strings data file, set the specified language as the "developer language". In practice, this just means that this language will appear first in the strings data file.') do |d|
-          @options[:developer_language] = d
+        opts.on('-d', '--developer-language LANG', 'When writing the strings data file, set the specified language as the "developer language". In practice, this just means that this language will appear first in the strings data file. When generating files this language will be used as default language and its translations will be used if a key is not localized for the output language.') do |d|
+          options[:developer_language] = d
         end
         opts.on('-c', '--consume-comments', 'Normally, when consuming a string file, Twine will ignore all comments in the file. With this flag set, any comments encountered will be read and parsed into the strings data file. This is especially useful when creating your first strings data file from an existing project.') do |c|
-          @options[:consume_comments] = true
+          options[:consume_comments] = true
         end
-        opts.on('-e', '--encoding ENCODING', 'Twine defaults to encoding all output files in UTF-8. This flag will tell Twine to use an alternate encoding for these files. For example, you could use this to write Apple .strings files in UTF-16. This flag currently only works with Apple .strings files and is currently only supported in Ruby 1.9.3 or greater.') do |e|
-          if !"".respond_to?(:encode)
+        opts.on('-e', '--encoding ENCODING', 'Twine defaults to encoding all output files in UTF-8. This flag will tell Twine to use an alternate encoding for these files. For example, you could use this to write Apple .strings files in UTF-16. This flag is currently only supported in Ruby 1.9.3 or greater.') do |e|
+          unless "".respond_to? :encode
             raise Twine::Error.new "The --encoding flag is only supported on Ruby 1.9.3 or greater."
           end
-          @options[:output_encoding] = e
+          options[:output_encoding] = e
         end
         opts.on('-h', '--help', 'Show this message.') do |h|
           puts opts.help
@@ -98,89 +108,56 @@ module Twine
         opts.separator '> twine consume-loc-drop strings.txt LocDrop5.zip'
         opts.separator '> twine validate-strings-file strings.txt'
       end
-      parser.parse! @args
+      parser.parse! args
 
-      if @args.length == 0
+      if args.length == 0
         puts parser.help
         exit
       end
 
-      @options[:command] = @args[0]
-
-      if !VALID_COMMANDS.include? @options[:command]
-        raise Twine::Error.new "Invalid command: #{@options[:command]}"
+      number_of_needed_arguments = NEEDED_COMMAND_ARGUMENTS[args[0]]
+      unless number_of_needed_arguments
+        raise Twine::Error.new "Invalid command: #{args[0]}"
       end
+      options[:command] = args[0]
 
-      if @args.length == 1
+      if args.length < 2
         raise Twine::Error.new 'You must specify your strings file.'
       end
+      options[:strings_file] = args[1]
 
-      @options[:strings_file] = @args[1]
+      if args.length < number_of_needed_arguments
+        raise Twine::Error.new 'Not enough arguments.'
+      elsif args.length > number_of_needed_arguments
+        raise Twine::Error.new "Unknown argument: #{args[number_of_needed_arguments]}"
+      end
 
-      case @options[:command]
+      case options[:command]
       when 'generate-string-file'
-        if @args.length == 3
-          @options[:output_path] = @args[2]
-        elsif @args.length > 3
-          raise Twine::Error.new "Unknown argument: #{@args[3]}"
-        else
-          raise Twine::Error.new 'Not enough arguments.'
-        end
-        if @options[:languages] and @options[:languages].length > 1
+        options[:output_path] = args[2]
+        if options[:languages] and options[:languages].length > 1
           raise Twine::Error.new 'Please only specify a single language for the generate-string-file command.'
         end
       when 'generate-all-string-files'
-        if ARGV.length == 3
-          @options[:output_path] = @args[2]
-        elsif @args.length > 3
-          raise Twine::Error.new "Unknown argument: #{@args[3]}"
-        else
-          raise Twine::Error.new 'Not enough arguments.'
-        end
+        options[:output_path] = args[2]
       when 'consume-string-file'
-        if @args.length == 3
-          @options[:input_path] = @args[2]
-        elsif @args.length > 3
-          raise Twine::Error.new "Unknown argument: #{@args[3]}"
-        else
-          raise Twine::Error.new 'Not enough arguments.'
-        end
-        if @options[:languages] and @options[:languages].length > 1
+        options[:input_path] = args[2]
+        if options[:languages] and options[:languages].length > 1
           raise Twine::Error.new 'Please only specify a single language for the consume-string-file command.'
         end
       when 'consume-all-string-files'
-        if @args.length == 3
-          @options[:input_path] = @args[2]
-        elsif @args.length > 3
-          raise Twine::Error.new "Unknown argument: #{@args[3]}"
-        else
-          raise Twine::Error.new 'Not enough arguments.'
-        end
+        options[:input_path] = args[2]
       when 'generate-loc-drop'
-        @options[:include_untranslated] = true
-        if @args.length == 3
-          @options[:output_path] = @args[2]
-        elsif @args.length > 3
-          raise Twine::Error.new "Unknown argument: #{@args[3]}"
-        else
-          raise Twine::Error.new 'Not enough arguments.'
-        end
-        if !@options[:format]
+        options[:output_path] = args[2]
+        if !options[:format]
           raise Twine::Error.new 'You must specify a format.'
         end
       when 'consume-loc-drop'
-        if @args.length == 3
-          @options[:input_path] = @args[2]
-        elsif @args.length > 3
-          raise Twine::Error.new "Unknown argument: #{@args[3]}"
-        else
-          raise Twine::Error.new 'Not enough arguments.'
-        end
+        options[:input_path] = args[2]
       when 'validate-strings-file'
-        if @args.length > 2
-          raise Twine::Error.new "Unknown argument: #{@args[2]}"
-        end
       end
+
+      return options
     end
   end
 end

data/lib/twine/formatters/abstract.rb

@@ -1,8 +1,10 @@
+require 'fileutils'
+
 module Twine
   module Formatters
     class Abstract
-      attr_accessor :strings
-      attr_accessor :options
+      attr_reader :strings
+      attr_reader :options
 
       def self.can_handle_directory?(path)
         return false
@@ -11,70 +13,23 @@ module Twine
       def initialize(strings, options)
         @strings = strings
         @options = options
+        @output_processor = Processors::OutputProcessor.new @strings, @options
       end
 
-      def iosify_substitutions(str)
-        # use "@" instead of "s" for substituting strings
-        str.gsub!(/%([0-9\$]*)s/, '%\1@')
-        return str
-      end
-
-      def androidify_substitutions(str)
-        # 1) use "s" instead of "@" for substituting strings
-        str.gsub!(/%([0-9\$]*)@/, '%\1s')
-
-        # 1a) escape strings that begin with a lone "@"
-        str.sub!(/^@ /, '\\@ ')
-
-        # 2) if there is more than one substitution in a string, make sure they are numbered
-        substituteCount = 0
-        startFound = false
-        str.each_char do |c|
-          if startFound
-            if c == "%"
-              # ignore as this is a literal %
-            elsif c.match(/\d/)
-              # leave the string alone if it already has numbered substitutions
-              return str
-            else
-              substituteCount += 1
-            end
-            startFound = false
-          elsif c == "%"
-            startFound = true
-          end
-        end
-
-        if substituteCount > 1
-          currentSub = 1
-          startFound = false
-          newstr = ""
-          str.each_char do |c|
-            if startFound
-              if !(c == "%")
-                newstr = newstr + "#{currentSub}$"
-                currentSub += 1
-              end
-              startFound = false
-            elsif c == "%"
-              startFound = true
-            end
-            newstr = newstr + c
-          end
-          return newstr
-        else
-          return str
-        end
-      end
-      
       def set_translation_for_key(key, lang, value)
+        value = value.gsub("\n", "\\n")
+
         if @strings.strings_map.include?(key)
-          @strings.strings_map[key].translations[lang] = value
+          row = @strings.strings_map[key]
+          reference = @strings.strings_map[row.reference_key] if row.reference_key
+
+          if !reference or value != reference.translations[lang]
+            row.translations[lang] = value
+          end
         elsif @options[:consume_all]
-          STDERR.puts "Adding new string '#{key}' to strings data file."
-          arr = @strings.sections.select { |s| s.name == 'Uncategorized' }
-          current_section = arr ? arr[0] : nil
-          if !current_section
+          Twine::stderr.puts "Adding new string '#{key}' to strings data file."
+          current_section = @strings.sections.find { |s| s.name == 'Uncategorized' }
+          unless current_section
             current_section = StringsSection.new('Uncategorized')
             @strings.sections.insert(0, current_section)
           end
@@ -82,13 +37,13 @@ module Twine
           current_section.rows << current_row
           
           if @options[:tags] && @options[:tags].length > 0
-              current_row.tags = @options[:tags]            
+            current_row.tags = @options[:tags]            
           end
           
           @strings.strings_map[key] = current_row
           @strings.strings_map[key].translations[lang] = value
         else
-          STDERR.puts "Warning: '#{key}' not found in strings data file."
+          Twine::stderr.puts "Warning: '#{key}' not found in strings data file."
         end
         if !@strings.language_codes.include?(lang)
           @strings.add_language_code(lang)
@@ -96,8 +51,16 @@ module Twine
       end
 
       def set_comment_for_key(key, comment)
+        return unless @options[:consume_comments]
+        
         if @strings.strings_map.include?(key)
-          @strings.strings_map[key].comment = comment
+          row = @strings.strings_map[key]
+          
+          reference = @strings.strings_map[row.reference_key] if row.reference_key
+
+          if !reference or comment != reference.raw_comment
+            row.comment = comment
+          end
         end
       end
 
@@ -109,38 +72,126 @@ module Twine
         raise NotImplementedError.new("You must implement determine_language_given_path in your formatter class.")
       end
 
+      def output_path_for_language(lang)
+        lang
+      end
+
       def read_file(path, lang)
         raise NotImplementedError.new("You must implement read_file in your formatter class.")
       end
 
-      def write_file(path, lang)
-        raise NotImplementedError.new("You must implement write_file in your formatter class.")
+      def format_file(strings, lang)
+        header = format_header(lang)
+        result = ""
+        result += header + "\n" if header
+        result += format_sections(strings, lang)
       end
 
-      def write_all_files(path)
-        if !File.directory?(path)
-          raise Twine::Error.new("Directory does not exist: #{path}")
+      def format_header(lang)
+      end
+
+      def format_sections(strings, lang)
+        sections = strings.sections.map { |section| format_section(section, lang) }
+        sections.join("\n")
+      end
+
+      def format_section_header(section)
+      end
+
+      def format_section(section, lang)
+        rows = section.rows.dup
+
+        result = ""
+        unless rows.empty?
+          if section.name && section.name.length > 0
+            section_header = format_section_header(section)
+            result += "\n#{section_header}" if section_header
+          end
         end
 
+        rows.map! { |row| format_row(row, lang) }
+        rows.compact! # remove nil entries
+        rows.map! { |row| "\n#{row}" }  # prepend newline
+        result += rows.join
+      end
+
+      def row_pattern
+        "%{comment}%{key_value}"
+      end
+
+      def format_row(row, lang)
+        return nil unless row.translated_string_for_lang(lang)
+
+        result = row_pattern.scan(/%\{([a-z_]+)\}/).flatten
+        result.map! { |element| send("format_#{element}".to_sym, row, lang) }
+        result.flatten.join
+      end
+
+      def format_comment(row, lang)
+      end
+
+      def format_key_value(row, lang)
+        value = row.translated_string_for_lang(lang)
+        key_value_pattern % { key: format_key(row.key.dup), value: format_value(value.dup) }
+      end
+
+      def key_value_pattern
+        raise NotImplementedError.new("You must implement key_value_pattern in your formatter class.")
+      end
+
+      def format_key(key)
+        key
+      end
+
+      def format_value(value)
+        value
+      end
+
+      def escape_quotes(text)
+        text.gsub('"', '\\\\"')
+      end
+
+      def write_file(path, lang)
+        encoding = @options[:output_encoding] || 'UTF-8'
+
+        processed_strings = @output_processor.process(lang)
+
+        File.open(path, "w:#{encoding}") do |f|
+          f.puts format_file(processed_strings, lang)
+        end
+      end
+
+      def write_all_files(path)
         file_name = @options[:file_name] || default_file_name
-        langs_written = []
-        Dir.foreach(path) do |item|
-          if item == "." or item == ".."
-            next
+        if @options[:create_folders]
+          @strings.language_codes.each do |lang|
+            output_path = File.join(path, output_path_for_language(lang))
+
+            FileUtils.mkdir_p(output_path)
+
+            write_file(File.join(output_path, file_name), lang)
           end
-          item = File.join(path, item)
-          if File.directory?(item)
+        else
+          language_written = false
+          Dir.foreach(path) do |item|
+            next if item == "." or item == ".."
+
+            item = File.join(path, item)
+            next unless File.directory?(item)
+
             lang = determine_language_given_path(item)
-            if lang
-              write_file(File.join(item, file_name), lang)
-              langs_written << lang
-            end
+            next unless lang
+
+            write_file(File.join(item, file_name), lang)
+            language_written = true
+          end
+
+          if !language_written
+            raise Twine::Error.new("Failed to generate any files: No languages found at #{path}")
           end
-        end
-        if langs_written.empty?
-          raise Twine::Error.new("Failed to generate any files: No languages found at #{path}")
         end
       end
+
     end
   end
 end