i18n-translators-tools 0.1

data/i18n-translators-tools.gemspec

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+require 'rubygems'
+require 'find'
+
+spec = Gem::Specification.new do |s|
+  s.platform = Gem::Platform::RUBY
+  s.summary = "I18n transation utility which helps to manage files with locales."
+  s.email = "pejuko@gmail.com"
+  s.authors = ["Petr Kovar"]
+  s.name = 'i18n-translators-tools'
+  s.version = '0.1'
+  s.date = '2010-07-16'
+  s.requirements << 'i18n' << 'ya2yaml'
+  s.require_path = 'lib'
+  s.files = ["bin/i18n-translate", "test/tc_translate.rb", "test/locale/src/default.yml", "test/locale/src/cze.yml", "test/locale/src/cze.rb", "test/locale/src/cze.po", "lib/i18n-translate.rb", "lib/i18n/translate.rb", "lib/i18n/processor.rb", "lib/i18n/processor/yaml.rb", "lib/i18n/processor/ruby.rb", "lib/i18n/processor/gettext.rb", "lib/i18n/backend/translate.rb", "README.md", "i18n-translators-tools.gemspec", "Rakefile"]
+  s.executables = ["i18n-translate"]
+  s.description = <<EOF
+This package brings you useful utility which can help you to handle locale files
+and translations in your Ruby projects. Read README.md file and run i18n-translate
+without parameters for more information.
+EOF
+end
+

data/lib/i18n-translate.rb

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+require 'i18n'
+
+dir = File.expand_path(File.dirname(__FILE__))
+$:.unshift(dir) unless $:.include?(dir)
+
+require 'i18n/backend/translate'
+require 'i18n/translate'
+

data/lib/i18n/backend/translate.rb

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+module I18n
+
+  module Backend
+    # It is highly recommended to use Translator wit Fallback plugin
+    #
+    #   I18n::Backend::Simple.send(:include, I18n::Backend::Translator)
+    #   I18n::Backend::Simple.send(:include, I18n::Backend::Fallbacks)
+    #
+    # notice that Translator have to be included BEFORE Fallback otherwise
+    # the fallback will get Hash (even with empty translation) and won't work.
+    module Translate
+
+      # wrapper which can work with both format
+      # the simple and the Translator's
+      def translate(locale, key, options = {})
+        result = super(locale, key, options)
+        return result unless result.kind_of?(Hash)
+        return nil unless result[:t]
+
+        tr = result[:t]
+        values = options.except(*I18n::Backend::Base::RESERVED_KEYS)
+
+        tr = resolve(locale, key, tr, options)
+        tr = interpolate(locale, tr, values) if values
+
+        tr
+      end
+
+    end # module Backend::Translator
+  end # module Backend
+
+end
+

data/lib/i18n/processor.rb

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+module I18n::Translate
+
+  module Processor
+    @processors = []
+
+    class << self
+      attr_reader :processors
+    end
+
+    def self.<<(processor)
+      @processors << processor
+    end
+
+    def self.read(fname, tr)
+      processor = find_processor(fname)
+      raise "Unknown file format" unless processor
+      worker = processor.new(fname, tr)
+      worker.read
+    end
+
+    def self.write(fname, data, tr)
+      processor = find_processor(fname)
+      raise "Unknown file format `#{fname}'" unless processor
+      worker = processor.new(fname, tr)
+      worker.write(data)
+    end
+
+    def self.find_processor(fname)
+      @processors.each do |processor|
+        return processor if processor.can_handle?(fname)
+      end
+      nil
+    end
+
+
+    class Template
+      FORMAT = []
+
+      def self.inherited(processor)
+        Processor << processor
+      end
+
+      attr_reader :filename, :translate
+
+      def initialize(fname, tr)
+        @filename = fname
+        @translate = tr
+      end
+
+      def read
+        data = File.open(@filename, mode("r")){ |f| f.read }
+        import(data)
+      end
+
+      def write(data)
+        File.open(@filename, mode("w")) {|f| f << export(data)}
+      end
+
+      def self.can_handle?(fname)
+        fname =~ %r{\.([^\.]+)$}
+        self::FORMAT.include?($1)
+      end
+
+    protected
+
+      def import(data)
+        data
+      end
+
+      def export(data)
+        data
+      end
+
+      def mode(m)
+        mode = m.dup
+        mode << ":" << @translate.options[:encoding] if defined?(Encoding)
+        mode
+      end
+    end
+  end
+
+end
+
+
+require 'i18n/processor/yaml'
+require 'i18n/processor/ruby'
+require 'i18n/processor/gettext'
+

data/lib/i18n/processor/gettext.rb

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+module I18n::Translate::Processor
+
+  class Gettext < Template
+    FORMAT = ['po']
+
+  protected
+
+    def import(data)
+      hash = {}
+
+      entry = {}
+      key = nil
+      last = nil
+      data.each_line do |line|
+        # empty line starts new entry
+        if line =~ %r{^\s*$}
+          if not entry.empty? and key
+            I18n::Translate.set(key, entry, hash, @translate.options[:separator])
+          end
+          key = nil
+          last = nil
+          entry = {}
+          next
+        end
+        
+        case line
+
+        # translator's comment
+        when %r{^# (.*)$}
+          entry["comment"] = $1.to_s.strip
+
+        # translator's comment
+        when %r{^#: (.*)$}
+          entry["line"] = $1.to_s.strip
+
+        # flag
+        when %r{^#, (.*)$}
+          flags = $1.split(",").compact.map{|x| x.strip}
+          fuzzy = flags.delete("fuzzy")
+          unless fuzzy
+            entry["flag"] = "ok"
+          else
+            flags.delete_if{|x| not I18n::Translate::FLAGS.include?(x)}
+            entry["flag"] = flags.first unless flags.empty?
+          end
+
+        # old default
+        when %r{^#\| msgid (.*)$}
+          entry["old"] = $1.to_s.strip
+
+        # key
+        when %r{^msgctxt "(.*)"$}
+          key = $1.to_s.strip
+          last = "key"
+
+        # default
+        when %r{^msgid "(.*)"$}
+          last = "default"
+          entry[last] = $1.to_s.strip
+
+        # translation
+        when %r{^msgstr "(.*)"$}
+          last = "t"
+          entry[last] = $1.to_s.strip
+
+        # string continuation
+        when %r{^"(.*)"$}
+          if last == "key"
+            key = "#{key}#{$1.to_s.strip}"
+          elsif last
+            entry[last] = "#{entry[last]}#{$1.to_s.strip}"
+          end
+        end
+      end
+
+      # last line at end of file
+      if not entry.empty? and key
+        I18n::Translate.set(key, entry, hash, @translate.options[:separator])
+      end
+
+      {@translate.lang => hash}
+    end
+
+
+    # this export ignores data
+    def export(data)
+      str = ""
+      keys = I18n::Translate.hash_to_keys(@translate.default).sort
+
+      keys.each do |key|
+        entry = [""]
+        value = @translate.find(key, @translate.target)
+        next unless value
+
+        if value.kind_of?(String)
+          entry << %~msgctxt #{key.inspect}~
+          entry << %~msgid #{@translate.find(key, @translate.default).to_s.inspect}~
+          entry << %~msgstr #{value.to_s.inspect}~
+        else
+          entry << %~#  #{value["comment"]}~ unless value["comment"].to_s.empty?
+          entry << %~#: #{value["line"]}~ unless value["line"].to_s.empty?
+          flags = []
+          flags << "fuzzy" if value["fuzzy"]
+          flags << value["flag"] unless value["flag"].to_s.strip.empty?
+          entry << %~#, #{flags.join(", ")}~ unless flags.empty?
+          entry << %~#| msgid #{value["old"]}~ unless value["old"].to_s.empty?
+          entry << %~msgctxt #{key.inspect}~
+          entry << %~msgid #{value["default"].to_s.inspect}~
+          entry << %~msgstr #{value["t"].to_s.inspect}~
+        end
+
+        entry << ""
+
+        str << entry.join("\n")
+      end
+
+      str
+    end
+
+  end
+
+end

data/lib/i18n/processor/ruby.rb

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+module I18n::Translate::Processor
+
+  class Ruby < Template
+    FORMAT = ['rb']
+
+  protected
+
+    def import(data)
+      eval(data)
+    end
+
+    # serialize hash to string
+    def export(data, indent=0)
+      str = "{\n"
+  
+      data.keys.sort{|k1,k2| k1.to_s <=> k2.to_s}.each_with_index do |k, i|
+        str << ("  " * (indent+1))
+        str << "#{k.inspect} => "
+        if data[k].kind_of?(Hash)
+          str << export(data[k], indent+1)
+        else
+          str << data[k].inspect
+        end
+        str << "," if i < (data.keys.size - 1)
+        str << "\n"
+      end
+  
+      str << ("  " * (indent))
+      str << "}"
+  
+      str
+    end
+
+  end
+
+end

data/lib/i18n/processor/yaml.rb

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+require 'yaml'
+require 'ya2yaml'
+
+module I18n::Translate::Processor
+  class YAML < Template
+    FORMAT = ['yml', 'yaml']
+
+  protected
+
+    def import(data)
+      ::YAML.load(data)
+    end
+
+    def export(data)
+      data.ya2yaml
+    end
+
+  end
+end

data/lib/i18n/translate.rb

@@ -0,0 +1,413 @@
+# -*- coding: utf-8 -*-
+# vi: fenc=utf-8:expandtab:ts=2:sw=2:sts=2
+# 
+# @author: Petr Kovar <pejuko@gmail.com>
+
+require 'fileutils'
+require 'find'
+
+require 'i18n/processor'
+
+# I18n::Translate introduces new format for translations. To make
+# I18n.t work properly you need to include Translator's backend:
+#
+#   I18n::Backend::Simple.send(:include, I18n::Backend::Translate)
+#   I18n::Backend::Simple.send(:include, I18n::Backend::Fallbacks)
+#
+# notice that Translator have to be included BEFORE Fallback otherwise
+# the fallback will get Hash (even with empty translation) and won't work.
+#
+# It is hightly recommended to use Fallback backend together with Translate.
+# If you have experienced nil or empty translations this can fix the problem.
+#
+# Format of entry:
+#
+#   old: old default string
+#   default: new default string
+#   comment: translator's comments
+#   t: translation
+#   line: the lines, where is this key used # not implemented yet
+#   flag: ok || incomplete || changed || untranslated
+#   fuzzy: true # exists only where flag != ok (nice to have when you want
+#                 edit files manualy)
+#
+# This format is for leaves in the tree hiearchy for plurals it should look like
+#
+#  key:
+#    one:
+#      old:
+#      default:
+#      t:
+#      ...
+#    other:
+#      old:
+#      default:
+#      t:
+#      ...
+#
+module I18n::Translate
+
+  FLAGS = %w(ok incomplete changed untranslated)
+  FORMATS = %w(yml rb po)       # the first one is preferred if :format => auto
+
+  # returns flat array of all keys e.g. ["system.message.ok", "system.message.error", ...]
+  def self.hash_to_keys(hash, separator=".", prefix="")
+    res = []
+    hash.keys.each do |key|
+      str = prefix.empty? ? key : "#{prefix}#{separator}#{key}"
+      if hash[key].kind_of?(Hash)
+        str = hash_to_keys( hash[key], separator, str )
+      end
+      res << str
+    end
+    res.flatten
+  end
+
+  # returns what is stored under key
+  def self.find(key, hash, separator=".")
+    h = hash
+    path = key.to_s.split(separator)
+    path.each do |key|
+      h = h[key]
+      return nil unless h
+    end
+    h
+  end
+
+  def self.set(key, value, hash, separator=".")
+    h = hash
+    path = key.to_s.split(separator)
+    path[0..-2].each do |chunk|
+      h[chunk] ||= {}
+      h = h[chunk]
+    end
+    unless value
+      h[path[-1]] = nil
+    else
+      h[path[-1]] = value
+    end
+  end
+
+  # scans :locale_dir for files with valid formats and returns
+  # list of files with locales. If block is given then
+  # it creates Translate object for each entry and pass it to the block
+  def self.scan(opts=Translate::DEFAULT_OPTIONS, &block)
+    o = Translate::DEFAULT_OPTIONS.merge(opts)
+    o[:exclude] ||= []
+
+    entries = []
+    if o[:deep] == true
+      Find.find(o[:locale_dir]) {|e| entries << e}
+    else
+      entries = Dir[File.join(o[:locale_dir], "*")]
+    end
+
+    locales = []
+    entries.each do |entry|
+      locale, format = Translate.valid_file?(entry, o[:format])
+      if (not format) or (locale == o[:default])
+        puts "#{entry}...skipping" if o[:verbose]
+        next unless format
+        next if locale == o[:default]
+      end
+
+      exclude = false
+      o[:exclude].each do |ex|
+        if entry =~ %r|#{ex}|
+          exclude = true
+          break
+        end
+      end
+      puts "#{entry}...excluded" if exclude and o[:verbose]
+      next if exclude
+
+      locales << entry 
+      dir = File.dirname(entry)
+
+      if block
+        yield Translate.new(locale, o.merge({:format => format, :locale_dir => dir, :default => o[:default]}))
+      end
+
+    end
+
+    locales
+  end
+
+
+
+  # it breaks proc and lambdas objects
+  class Translate
+    DEFAULT_OPTIONS = {
+      :separator => ".",       # default key separator e.g. "model.article.message.not.found"
+      :locale_dir => "locale", # where to search for files
+      :default => "default",   # default name for file containing default app's key => string
+      :force_encoding => true, # in ruby 1.9 forces string encoding
+      :encoding => "utf-8",    # encoding name to be forced to
+      :format => "auto"        # auto, rb, yml
+    }
+
+    attr_reader :default, :target, :merge, :options, :lang, :default_file, :lang_file
+
+    # loads default and lang files
+    def initialize(lang, opts={})
+      @lang = lang.to_s
+      raise "Empty locale" if @lang.empty?
+      @options = DEFAULT_OPTIONS.merge(opts)
+      @options[:default_format] ||= @options[:format]
+
+      @default, @default_file = load_locale( @options[:default], @options[:default_format] )
+      @target, @lang_file = load_locale( @lang )
+
+      merge!
+    end
+
+    # check if the file has supported format
+    def self.valid_file?(fname, format=Translate::DEFAULT_OPTIONS[:format])
+      pattern = ".*?"
+      pattern = format if format != "auto"
+      fname =~ /\/?([^\/]*?)\.(#{pattern})$/
+      locale, format = $1, $2
+      if I18n::Translate::FORMATS.include?($2)
+        return [locale, format]
+      end
+      nil
+    end
+
+    # will merge only one key and returns hash
+    #   {
+    #     :key => 'key',
+    #     :default => '',              # value set in default file
+    #     :old_default => '',          # value set as old in target file
+    #                                    (value from default file from last translation
+    #                                     if the field has changed)
+    #     :old_t => '',                # if flag == 'changed' then old_t = t and t = ''
+    #     :t => '',                    # value set in target file
+    #     :line => 'some/file.rb: 44', # name of source file and number of line
+    #                                    (a copy in target file from default file)
+    #                                    !!! line is unused for now
+    #     :comment => ''               # a comment added by a translator
+    #     :flag => ok || incomplete || changed || untranslated    # set by merging tool except incomplete
+    #                                                               which is set by translator
+    #   }
+    def [](key)
+      d = I18n::Translate.find(key, @default, @options[:separator])
+      raise "Translate#[key]: wrong key '#{key}'" unless d
+
+      entry = {"key" => key, "default" => d}
+      
+      # translation doesn't exist
+      trg = I18n::Translate.find(key, @target, @options[:separator])
+      if (not trg) or
+         (trg.kind_of?(String) and trg.strip.empty?) or
+         (trg.kind_of?(Hash) and trg["t"].to_s.strip.empty?)
+        entry["old_default"] = ""
+        entry["old_t"] = ""
+        entry["t"] = ""
+        entry["comment"] = trg.kind_of?(Hash) ? trg["comment"].to_s.strip : ""
+        entry["flag"] = "untranslated"
+        return entry
+      end
+
+      # default has changed => new translation is probably required
+      if trg.kind_of?(Hash)
+        entry["old_t"] = trg["t"].to_s.strip
+        entry["t"] = ""
+        entry["comment"] = trg["comment"].to_s.strip
+        entry["flag"] = "changed"
+
+        if d != trg["default"]
+          entry["old_default"] = trg["default"].to_s.strip
+          return entry
+        elsif not trg["old"].to_s.strip.empty?
+          entry["old_default"] =  trg["old"].to_s.strip
+          return entry
+        end
+      end
+
+      # nothing has changed 
+      entry["old_default"] = trg.kind_of?(Hash) ? trg["old"].to_s.strip : ""
+      entry["old_t"] = ""
+      entry["t"] = trg.kind_of?(Hash) ? trg["t"].to_s.strip : trg.to_s.strip
+      entry["comment"] = trg.kind_of?(Hash) ? trg["comment"].to_s.strip : ""
+      entry["flag"] = (trg.kind_of?(Hash) and trg["flag"]) ? trg["flag"].to_s.strip : "ok"
+
+      entry
+    end
+
+    # wrapper for I18n::Translate.find with presets options
+    def find(key, hash=@translate, separator=@options[:separator])
+      I18n::Translate.find(key, hash, separator)
+    end
+
+    # will create path in @target for 'key' and set the 'value'
+    def []=(key, value)
+      I18n::Translate.set(key, value, @target, @options[:separator])
+    end
+
+    # merge merged and edited hash into @target
+    # translation can be hash or array
+    # * array format is the same as self.merge is
+    #   [ {key => , t =>, ...}, {key =>, ...}, ... ]
+    # * hash format is supposed to be the format obtained from web form
+    #   {:key => {t =>, ...}, :key => {...}, ...}
+    def assign(translation)
+      translation.each do |transl|
+        key, values = nil
+        if transl.kind_of?(Hash)
+          # merge format: [{key => , t =>, ...}, ...]
+          key, values = transl["key"], transl
+        elsif transl.kind_of?(Array)
+          # web format: {key => {t => }, ...}
+          key, values = transl
+        end
+
+        old_t = values["old_t"].to_s.strip
+        new_t = values["t"].to_s.strip
+        default = values["default"].to_s.strip
+        old_default = values["old_default"].to_s.strip
+        flag = values["flag"].to_s.strip
+        comment = values["comment"].to_s.strip
+
+        if old_t.respond_to?("force_encoding") and @options[:force_encoding]
+          enc = @options[:encoding]
+          old_t.force_encoding(enc)
+          new_t.force_encoding(enc)
+          default.force_encoding(enc)
+          old_default.force_encoding(enc)
+          flag.force_encoding(enc)
+          comment.force_encoding(enc)
+        end
+
+        trg = {
+          "comment" => comment,
+          "flag" => flag
+        }
+
+        if flag == "ok"
+          trg["t"] = new_t.empty? ? old_t : new_t
+          trg["default"] = default
+          trg["old"] = "" 
+        else
+          trg["t"] = new_t.empty? ? old_t : new_t
+          trg["default"] = default
+          trg["old"] = old_default
+        end
+
+        # make fallback work
+        trg["t"] = nil if trg["t"].empty?
+
+        # say that this entry is not completed yet
+        # useful if you edit files in text editor and serching for next one
+        trg["fuzzy"] = true if flag != "ok"
+
+        self[key] = trg
+      end
+    end
+
+    # re-read @target data from the disk and create @merge
+    def reload!
+      @target, @lang_file  = load_locale( @lang )
+      merge!
+    end
+
+    # merge @default and @target into list @merge
+    def merge!
+      @merge = merge_locale
+    end
+
+    # export @target to file
+    def export!
+      save_locale(@lang)
+    end
+
+    # throw away translators metadata and convert
+    # hash to default I18n format
+    def strip!
+      keys = I18n::Translate.hash_to_keys(@default, @options[:separator])
+      keys.each do |key|
+        entry = I18n::Translate.find(key, @target, @options[:separator])
+        raise "Translate#[key]: wrong key '#{key}'" unless entry
+        next unless entry.kind_of?(Hash)
+        self[key] = entry["t"]
+      end
+  
+      self
+    end
+
+    # returns statistics hash
+    # {:total => N, :ok => N, :changed => N, :incomplete => N, :untranslated => N, :fuzzy => N, :progress => N}
+    def stat
+      stat = {
+        :total => @merge.size,
+        :ok => @merge.select{|e| e["flag"] == "ok"}.size,
+        :changed => @merge.select{|e| e["flag"] == "changed"}.size,
+        :incomplete => @merge.select{|e| e["flag"] == "incomplete"}.size,
+        :untranslated => @merge.select{|e| e["flag"] == "untranslated"}.size,
+        :fuzzy => @merge.select{|e| e["flag"] != "ok"}.size
+      }
+      stat[:progress] = (stat[:ok].to_f / stat[:total].to_f) * 100
+      stat
+    end
+
+    def to_yaml
+      trg = {@lang => @target}
+      #YAML.dump(trg)
+      trg.ya2yaml
+    end
+
+    def to_rb
+      trg = {@lang => @target}
+      trg.to_rb
+    end
+
+  protected
+
+    # returns first file for @lang.type
+    def file_name(lang, type=@options[:format])
+      fname = "#{@options[:locale_dir]}/#{lang}.#{type}"
+      if type == "auto"
+        pattern = "#{@options[:locale_dir]}/#{lang}.*"
+        fname = Dir[pattern].select{|x| Translate.valid_file?(x)}.first
+      end
+      fname = "#{@options[:locale_dir]}/#{lang}.#{FORMATS.first}" unless fname
+      fname
+    end
+
+    # loads locales from .rb or .yml file
+    def load_locale(lang, type=@options[:format])
+      fname = file_name(lang, type)
+
+      if File.exists?(fname)
+        return [Processor.read(fname, self)[lang], fname]
+      else
+        STDERR << "Warning: I18n::Translate#load_locale: file `#{fname}' does NOT exists. Creating empty locale.\n"
+      end
+
+      [{}, fname]
+    end
+
+    # save to the first file found as lang.*
+    # detects .rb and .yml
+    def save_locale(lang)
+      fname = file_name(lang)
+      backup(fname)
+      Processor.write(fname, {@lang => @target}, self)
+    end
+
+    # backup file if file exists
+    def backup(fname)
+      FileUtils.cp(fname, "#{fname}.bak") if File.exists?(fname)
+    end
+
+    # creates array of hashes as specified in self[] function
+    def merge_locale
+      keys = I18n::Translate.hash_to_keys(@default, @options[:separator])
+      keys.sort!
+      keys.inject([]) do |sum, key|
+        sum << self[key]
+      end
+    end
+
+
+  end # class Translate
+end # module DML