yury-twine 0.9.1

data/lib/twine/runner.rb

@@ -0,0 +1,332 @@
+require 'tmpdir'
+require 'fileutils'
+
+Twine::Plugin.new # Initialize plugins first in Runner.
+
+module Twine
+  class Runner
+    def self.run(args)
+      options = CLI.parse(args)
+
+      twine_file = TwineFile.new
+      twine_file.read options[:twine_file]
+      runner = new(options, twine_file)
+
+      case options[:command]
+      when 'generate-localization-file'
+        runner.generate_localization_file
+      when 'generate-all-localization-files'
+        runner.generate_all_localization_files
+      when 'consume-localization-file'
+        runner.consume_localization_file
+      when 'consume-all-localization-files'
+        runner.consume_all_localization_files
+      when 'generate-loc-drop'
+        runner.generate_loc_drop
+      when 'consume-loc-drop'
+        runner.consume_loc_drop
+      when 'validate-twine-file'
+        runner.validate_twine_file
+      end
+    end
+
+    def initialize(options = {}, twine_file = TwineFile.new)
+      @options = options
+      @twine_file = twine_file
+    end
+
+    def write_twine_data(path)
+      if @options[:developer_language]
+        @twine_file.set_developer_language_code(@options[:developer_language])
+      end
+      @twine_file.write(path)
+    end
+
+    def generate_localization_file
+      validate_twine_file if @options[:validate]
+
+      lang = nil
+      lang = @options[:languages][0] if @options[:languages]
+
+      formatter, lang = prepare_read_write(@options[:output_path], lang)
+      output = formatter.format_file(lang)
+
+      raise Twine::Error.new "Nothing to generate! The resulting file would not contain any translations." unless output
+
+      IO.write(@options[:output_path], output, encoding: encoding)
+    end
+
+    def generate_all_localization_files
+      validate_twine_file if @options[:validate]
+
+      if !File.directory?(@options[:output_path])
+        if @options[:create_folders]
+          FileUtils.mkdir_p(@options[:output_path])
+        else
+          raise Twine::Error.new("Directory does not exist: #{@options[:output_path]}")
+        end
+      end
+
+      formatter_for_directory = find_formatter { |f| f.can_handle_directory?(@options[:output_path]) }
+      formatter = formatter_for_format(@options[:format]) || formatter_for_directory
+
+      unless formatter
+        raise Twine::Error.new "Could not determine format given the contents of #{@options[:output_path]}"
+      end
+
+      file_name = @options[:file_name] || formatter.default_file_name
+      if @options[:create_folders]
+        @twine_file.language_codes.each do |lang|
+          output_path = File.join(@options[:output_path], formatter.output_path_for_language(lang))
+
+          FileUtils.mkdir_p(output_path)
+
+          file_path = File.join(output_path, file_name)
+
+          output = formatter.format_file(lang)
+          unless output
+            Twine::stderr.puts "Skipping file at path #{file_path} since it would not contain any translations."
+            next
+          end
+
+          IO.write(file_path, output, encoding: encoding)
+        end
+      else
+        language_found = false
+        Dir.foreach(@options[:output_path]) do |item|
+          next if item == "." or item == ".."
+
+          output_path = File.join(@options[:output_path], item)
+          next unless File.directory?(output_path)
+
+          lang = formatter.determine_language_given_path(output_path)
+          next unless lang
+
+          language_found = true
+
+          file_path = File.join(output_path, file_name)
+          output = formatter.format_file(lang)
+          unless output
+            Twine::stderr.puts "Skipping file at path #{file_path} since it would not contain any translations."
+            next
+          end
+
+          IO.write(file_path, output, encoding: encoding)
+        end
+
+        unless language_found
+          raise Twine::Error.new("Failed to generate any files: No languages found at #{@options[:output_path]}")
+        end
+      end
+
+    end
+
+    def consume_localization_file
+      lang = nil
+      if @options[:languages]
+        lang = @options[:languages][0]
+      end
+
+      read_localization_file(@options[:input_path], lang)
+      output_path = @options[:output_path] || @options[:twine_file]
+      write_twine_data(output_path)
+    end
+
+    def consume_all_localization_files
+      if !File.directory?(@options[:input_path])
+        raise Twine::Error.new("Directory does not exist: #{@options[:output_path]}")
+      end
+
+      Dir.glob(File.join(@options[:input_path], "**/*")) do |item|
+        if File.file?(item)
+          begin
+            read_localization_file(item)
+          rescue Twine::Error => e
+            Twine::stderr.puts "#{e.message}"
+          end
+        end
+      end
+
+      output_path = @options[:output_path] || @options[:twine_file]
+      write_twine_data(output_path)
+    end
+
+    def generate_loc_drop
+      validate_twine_file if @options[:validate]
+
+      require_rubyzip
+
+      if File.file?(@options[:output_path])
+        File.delete(@options[:output_path])
+      end
+
+      Dir.mktmpdir do |temp_dir|
+        Zip::File.open(@options[:output_path], Zip::File::CREATE) do |zipfile|
+          zipfile.mkdir('Locales')
+
+          formatter = formatter_for_format(@options[:format])
+          @twine_file.language_codes.each do |lang|
+            if @options[:languages] == nil || @options[:languages].length == 0 || @options[:languages].include?(lang)
+              file_name = lang + formatter.extension
+              temp_path = File.join(temp_dir, file_name)
+              zip_path = File.join('Locales', file_name)
+
+              output = formatter.format_file(lang)
+              unless output
+                Twine::stderr.puts "Skipping file #{file_name} since it would not contain any translations."
+                next
+              end
+
+              IO.write(temp_path, output, encoding: encoding)
+              zipfile.add(zip_path, temp_path)
+            end
+          end
+        end
+      end
+    end
+
+    def consume_loc_drop
+      require_rubyzip
+
+      if !File.file?(@options[:input_path])
+        raise Twine::Error.new("File does not exist: #{@options[:input_path]}")
+      end
+
+      Dir.mktmpdir do |temp_dir|
+        Zip::File.open(@options[:input_path]) do |zipfile|
+          zipfile.each do |entry|
+            next if entry.name.end_with? '/' or File.basename(entry.name).start_with? '.'
+
+            real_path = File.join(temp_dir, entry.name)
+            FileUtils.mkdir_p(File.dirname(real_path))
+            zipfile.extract(entry.name, real_path)
+            begin
+              read_localization_file(real_path)
+            rescue Twine::Error => e
+              Twine::stderr.puts "#{e.message}"
+            end
+          end
+        end
+      end
+
+      output_path = @options[:output_path] || @options[:twine_file]
+      write_twine_data(output_path)
+    end
+
+    def validate_twine_file
+      total_definitions = 0
+      all_keys = Set.new
+      duplicate_keys = Set.new
+      keys_without_tags = Set.new
+      invalid_keys = Set.new
+      valid_key_regex = /^[A-Za-z0-9_]+$/
+
+      @twine_file.sections.each do |section|
+        section.definitions.each do |definition|
+          total_definitions += 1
+
+          duplicate_keys.add(definition.key) if all_keys.include? definition.key
+          all_keys.add(definition.key)
+
+          keys_without_tags.add(definition.key) if definition.tags == nil or definition.tags.length == 0
+
+          invalid_keys << definition.key unless definition.key =~ valid_key_regex
+        end
+      end
+
+      errors = []
+      join_keys = lambda { |set| set.map { |k| "  " + k }.join("\n") }
+
+      unless duplicate_keys.empty?
+        errors << "Found duplicate key(s):\n#{join_keys.call(duplicate_keys)}"
+      end
+
+      if @options[:pedantic]
+        if keys_without_tags.length == total_definitions
+          errors << "None of your definitions have tags."
+        elsif keys_without_tags.length > 0
+          errors << "Found definitions without tags:\n#{join_keys.call(keys_without_tags)}"
+        end
+      end
+
+      unless invalid_keys.empty?
+        errors << "Found key(s) with invalid characters:\n#{join_keys.call(invalid_keys)}"
+      end
+
+      raise Twine::Error.new errors.join("\n\n") unless errors.empty?
+
+      Twine::stdout.puts "#{@options[:twine_file]} is valid."
+    end
+
+    private
+
+    def encoding
+      @options[:output_encoding] || 'UTF-8'
+    end
+
+    def require_rubyzip
+      begin
+        require 'zip'
+      rescue LoadError
+        raise Twine::Error.new "You must run 'gem install rubyzip' in order to create or consume localization drops."
+      end
+    end
+
+    def determine_language_given_path(path)
+      code = File.basename(path, File.extname(path))
+      return code if @twine_file.language_codes.include? code
+    end
+
+    def formatter_for_format(format)
+      find_formatter { |f| f.format_name == format }
+    end
+
+    def find_formatter(&block)
+      formatter = Formatters.formatters.find &block
+      return nil unless formatter
+      formatter.twine_file = @twine_file
+      formatter.options = @options
+      formatter
+    end
+
+    def read_localization_file(path, lang = nil)
+      unless File.file?(path)
+        raise Twine::Error.new("File does not exist: #{path}")
+      end
+
+      formatter, lang = prepare_read_write(path, lang)
+
+      encoding = @options[:encoding] || Twine::Encoding.encoding_for_path(path)
+
+      IO.open(IO.sysopen(path, 'rb'), 'rb', external_encoding: encoding, internal_encoding: 'UTF-8') do |io|
+        io.read(2) if Twine::Encoding.has_bom?(path)
+        formatter.read(io, lang)
+      end
+    end
+
+    def prepare_read_write(path, lang)
+      formatter_for_path = find_formatter { |f| f.extension == File.extname(path) }
+      formatter = formatter_for_format(@options[:format]) || formatter_for_path
+
+      unless formatter
+        if !path.include?("Localizable.stringsdict")
+          raise Twine::Error.new "Unable to determine format of #{path}"
+        else
+          raise Twine::Error.new "Ignoring for now #{path}"
+        end
+      end
+
+      if formatter.can_handle_file?(path)
+        lang = lang || determine_language_given_path(path) || formatter.determine_language_given_path(path)
+        unless lang
+          raise Twine::Error.new "Unable to determine language for #{path}"
+        end
+
+        @twine_file.language_codes << lang unless @twine_file.language_codes.include? lang
+      else
+        raise Twine::Error.new "Ignoring #{path}"
+      end
+      return formatter, lang
+    end
+  end
+end

data/lib/twine/twine_file.rb

@@ -0,0 +1,266 @@
+module Twine
+  class TwineDefinition
+    attr_reader :key
+    attr_accessor :comment
+    attr_accessor :ios_comment
+    attr_accessor :tags
+    attr_reader :translations
+    attr_accessor :reference
+    attr_accessor :reference_key
+
+    def initialize(key)
+      @key = key
+      @comment = nil
+      @ios_comment = nil
+      @tags = nil
+      @translations = {}
+    end
+
+    def comment
+      raw_comment || (reference.comment if reference)
+    end
+
+    def raw_comment
+      @comment
+    end
+
+    def ios_comment
+      raw_ios_comment || (reference.ios_comment if reference)
+    end
+
+    def raw_ios_comment
+      @ios_comment
+    end
+
+    # [['tag1', 'tag2'], ['~tag3']] == (tag1 OR tag2) AND (!tag3)
+    def matches_tags?(tags, include_untagged)
+      if tags == nil || tags.empty?  # The user did not specify any tags. Everything passes.
+        return true
+      elsif @tags == nil  # This definition has no tags -> check reference (if any)
+        return reference ? reference.matches_tags?(tags, include_untagged) : include_untagged
+      elsif @tags.empty?
+        return include_untagged
+      else
+        return tags.all? do |set|
+          regular_tags, negated_tags = set.partition { |tag| tag[0] != '~' }
+          negated_tags.map! { |tag| tag[1..-1] }
+          matches_regular_tags = (!regular_tags.empty? && !(regular_tags & @tags).empty?)
+          matches_negated_tags = (!negated_tags.empty? && (negated_tags & @tags).empty?)
+          matches_regular_tags or matches_negated_tags
+        end
+      end
+
+      return false
+    end
+
+    def translation_for_lang(lang)
+      translation = [lang].flatten.map { |l| @translations[l] }.first
+
+      translation = reference.translation_for_lang(lang) if translation.nil? && reference
+
+      return translation
+    end
+
+    # Twine adds a copy of the dev language's translation if there is no definition provided for the language,
+    # which is useful in the main Localizable.strings file because iOS doesn't auto use the Base language's
+    # value, but we don't want that behaviour in our plurals
+    def translation_for_lang_or_nil(lang, dev_lang)
+      translation = [lang].flatten.map { |l| @translations[l] }.first
+
+      # translation never comes back as nil because Twine fills with the dev_lang string
+      if lang != dev_lang
+        [lang].flatten.map do |l|
+          if @translations[l] == @translations[dev_lang]
+            return nil
+          end
+        end
+      end
+
+      return translation
+    end
+  end
+
+  class TwineSection
+    attr_reader :name
+    attr_reader :definitions
+
+    def initialize(name)
+      @name = name
+      @definitions = []
+    end
+
+    def is_uncategorized
+      return @name == 'Uncategorized'
+    end
+  end
+
+  class TwineFile
+    attr_reader :sections
+    attr_reader :definitions_by_key
+    attr_reader :language_codes
+
+    private
+
+    def match_key(text)
+      match = /^\[(.+)\]$/.match(text)
+      return match[1] if match
+    end
+
+    public
+
+    def initialize
+      @sections = []
+      @definitions_by_key = {}
+      @language_codes = []
+    end
+
+    def add_language_code(code)
+      if @language_codes.length == 0
+        @language_codes << code
+      elsif !@language_codes.include?(code)
+        dev_lang = @language_codes[0]
+        @language_codes << code
+        @language_codes.delete(dev_lang)
+        @language_codes.sort!
+        @language_codes.insert(0, dev_lang)
+      end
+    end
+
+    def set_developer_language_code(code)
+      @language_codes.delete(code)
+      @language_codes.insert(0, code)
+    end
+
+    def read(path)
+      if !File.file?(path)
+        file = File.new(path, 'w:UTF-8')
+        file.close
+      end
+
+      File.open(path, 'r:UTF-8') do |f|
+        line_num = 0
+        current_section = nil
+        current_definition = nil
+        while line = f.gets
+          parsed = false
+          line.strip!
+          line_num += 1
+
+          if line.length == 0
+            next
+          end
+
+          if line.length > 4 && line[0, 2] == '[['
+            match = /^\[\[(.+)\]\]$/.match(line)
+            if match
+              current_section = TwineSection.new(match[1])
+              @sections << current_section
+              parsed = true
+            end
+          elsif line.length > 2 && line[0, 1] == '['
+            key = match_key(line)
+            if key
+              current_definition = TwineDefinition.new(key)
+              @definitions_by_key[current_definition.key] = current_definition
+              if !current_section
+                current_section = TwineSection.new('')
+                @sections << current_section
+              end
+              current_section.definitions << current_definition
+              parsed = true
+            end
+          else
+            match = /^([^=]+)=(.*)$/.match(line)
+            if match
+              key = match[1].strip
+              value = match[2].strip
+
+              value = value[1..-2] if value[0] == '`' && value[-1] == '`'
+              case key
+              when 'ios'
+                current_definition.ios_comment = value
+              when 'comment'
+                current_definition.comment = value
+              when 'tags'
+                current_definition.tags = value.split(',')
+              when 'ref'
+                current_definition.reference_key = value if value
+              else
+                if !@language_codes.include? key
+                  add_language_code(key)
+                end
+                current_definition.translations[key] = value
+              end
+              parsed = true
+            end
+          end
+
+          if !parsed
+            raise Twine::Error.new("Unable to parse line #{line_num} of #{path}: #{line}")
+          end
+        end
+      end
+
+      # resolve_references
+      @definitions_by_key.each do |key, definition|
+        next unless definition.reference_key
+        definition.reference = @definitions_by_key[definition.reference_key]
+      end
+    end
+
+    def write(path)
+      dev_lang = @language_codes[0]
+
+      File.open(path, 'w:UTF-8') do |f|
+        @sections.each do |section|
+          if f.pos > 0
+            f.puts ''
+          end
+
+          f.puts "[[#{section.name}]]"
+
+          section.definitions.each do |definition|
+            f.puts "\t[#{definition.key}]"
+
+            value = write_value(definition, dev_lang, f)
+            if !value && !definition.reference_key
+              puts "Warning: #{definition.key} does not exist in developer language '#{dev_lang}'"
+            end
+
+            if definition.reference_key
+              f.puts "\t\tref = #{definition.reference_key}"
+            end
+            if definition.tags && definition.tags.length > 0
+              tag_str = definition.tags.join(',')
+              f.puts "\t\ttags = #{tag_str}"
+            end
+            if definition.raw_comment and definition.raw_comment.length > 0
+              f.puts "\t\tcomment = #{definition.raw_comment}"
+            end
+            if definition.raw_ios_comment and definition.raw_ios_comment.length > 0
+              f.puts "\t\tios = #{definition.raw_ios_comment}"
+            end
+            @language_codes[1..-1].each do |lang|
+              write_value(definition, lang, f)
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def write_value(definition, language, file)
+      value = definition.translations[language]
+      return nil unless value
+
+      if value[0] == ' ' || value[-1] == ' ' || (value[0] == '`' && value[-1] == '`')
+        value = '`' + value + '`'
+      end
+
+      file.puts "\t\t#{language} = #{value}"
+      return value
+    end
+
+  end
+end