fair-gettext 2.0.1 → 2.0.2

data/lib/gettext/tools/parser/erb.rb

@@ -0,0 +1,52 @@
+=begin
+  parser/erb.rb - parser for ERB
+
+  Copyright (C) 2005-2009  Masao Mutoh
+ 
+  You may redistribute it and/or modify it under the same
+  license terms as Ruby or LGPL.
+=end
+
+require 'erb'
+require 'gettext/tools/parser/ruby'
+
+module GetText
+  module ErbParser
+    extend self
+
+    @config = {
+      :extnames => ['.rhtml', '.erb']
+    }
+
+    # Sets some preferences to parse ERB files.
+    # * config: a Hash of the config. It can takes some values below:
+    #   * :extnames: An Array of target files extension. Default is [".rhtml"].
+    def init(config)
+      config.each{|k, v|
+	@config[k] = v
+      }
+    end
+
+    def parse(file, targets = []) # :nodoc:
+      src = ERB.new(IO.readlines(file).join).src
+      # Remove magic comment prepended by erb in Ruby 1.9.
+      src.sub!(/\A#.*?coding[:=].*?\n/, '') if src.respond_to?(:encode)
+      erb = src.split(/$/)
+      RubyParser.parse_lines(file, erb, targets)
+    end
+
+    def target?(file) # :nodoc:
+      @config[:extnames].each do |v|
+	return true if File.extname(file) == v
+      end
+      false
+    end
+  end
+end
+
+if __FILE__ == $0
+  # ex) ruby glade.rhtml foo.rhtml  bar.rhtml
+  ARGV.each do |file|
+    p GetText::ErbParser.parse(file)
+  end
+end

data/lib/gettext/tools/parser/glade.rb

@@ -0,0 +1,98 @@
+=begin
+  parser/glade.rb - parser for Glade-2
+
+  Copyright (C) 2004,2005  Masao Mutoh
+ 
+  You may redistribute it and/or modify it under the same
+  license terms as Ruby or LGPL.
+=end
+
+require 'cgi'
+require 'gettext'
+
+module GetText
+  module GladeParser
+    extend GetText
+    extend self
+    
+    bindtextdomain("rgettext")
+
+    TARGET1 = /<property.*translatable="yes">(.*)/
+    TARGET2 = /(.*)<\/property>/
+
+    def parse(file, targets = []) # :nodoc: 
+      lines = IO.readlines(file)
+      parse_lines(file, lines, targets)
+    end
+
+    #from ary of lines.
+    def parse_lines(file, lines, targets) # :nodoc:
+      cnt = 0
+      target = false
+      line_no = 0
+      val = nil
+      
+      loop do 
+        line = lines.shift
+        break unless line
+        
+        cnt += 1
+        if TARGET1 =~ line
+          line_no = cnt
+          val = $1 + "\n"
+          target = true
+          if TARGET2 =~ $1
+            val = $1
+            add_target(val, file, line_no, targets)
+            val = nil
+            target = false
+          end
+        elsif target
+          if TARGET2 =~ line
+            val << $1
+            add_target(val, file, line_no, targets)
+            val = nil
+            target = false
+          else
+            val << line
+          end
+        end
+      end
+      targets
+    end
+
+    XML_RE = /<\?xml/ 
+    GLADE_RE = /glade-2.0.dtd/
+ 
+    def target?(file) # :nodoc:
+      data = IO.readlines(file)
+      if XML_RE =~ data[0] and GLADE_RE =~ data[1]
+	true
+      else
+	if File.extname(file) == '.glade'
+	  raise _("`%{file}' is not glade-2.0 format.") % {:file => file}
+	end
+	false
+      end
+    end
+
+    def add_target(val, file, line_no, targets) # :nodoc:
+      return unless val.size > 0
+      assoc_data = targets.assoc(val)
+      val = CGI.unescapeHTML(val)
+      if assoc_data 
+        targets[targets.index(assoc_data)] = assoc_data << "#{file}:#{line_no}"
+      else
+        targets << [val.gsub(/\n/, '\n'), "#{file}:#{line_no}"]
+      end
+      targets
+    end
+  end
+end
+
+if __FILE__ == $0
+  # ex) ruby glade.rb foo.glade  bar.glade
+  ARGV.each do |file|
+    p GetText::GladeParser.parse(file)
+  end
+end

data/lib/gettext/tools/parser/ruby.rb

@@ -0,0 +1,226 @@
+#!/usr/bin/ruby
+=begin
+  parser/ruby.rb - parser for ruby script
+
+  Copyright (C) 2003-2009  Masao Mutoh
+  Copyright (C) 2005       speakillof
+  Copyright (C) 2001,2002  Yasushi Shoji, Masao Mutoh
+ 
+  You may redistribute it and/or modify it under the same
+  license terms as Ruby or LGPL.
+
+=end
+
+require 'irb/ruby-lex.rb'
+require 'stringio'
+require 'gettext/tools/pomessage'
+
+module GetText
+  class RubyLexX < RubyLex  # :nodoc: all
+    # Parser#parse resemlbes RubyLex#lex
+    def parse
+      until (  (tk = token).kind_of?(RubyToken::TkEND_OF_SCRIPT) && !@continue or tk.nil?  )
+        s = get_readed
+        if RubyToken::TkSTRING === tk
+          def tk.value
+            @value 
+          end
+          
+          def tk.value=(s)
+            @value = s
+          end
+          
+          if @here_header
+            s = s.sub(/\A.*?\n/, '').sub(/^.*\n\Z/, '')
+          else
+            begin
+              s = eval(s)
+            rescue Exception
+              # Do nothing.
+            end
+          end
+          
+          tk.value = s
+        end
+        
+        if $DEBUG
+          if tk.is_a? TkSTRING
+            $stderr.puts("#{tk}: #{tk.value}")
+          elsif tk.is_a? TkIDENTIFIER
+            $stderr.puts("#{tk}: #{tk.name}")
+          else
+            $stderr.puts(tk)
+          end
+        end
+        
+        yield tk
+      end
+      return nil
+    end
+
+    # Original parser does not keep the content of the comments,
+    # so monkey patching this with new token type and extended
+    # identify_comment implementation
+    RubyToken.def_token :TkCOMMENT_WITH_CONTENT, TkVal
+
+    def identify_comment
+      @ltype = "#"
+      get_readed # skip the hash sign itself
+
+      while ch = getc
+        if ch == "\n"
+          @ltype = nil
+          ungetc
+          break
+        end
+      end
+      return Token(TkCOMMENT_WITH_CONTENT, get_readed)
+    end
+
+  end
+
+  # Extends PoMessage for RubyParser.
+  # Implements a sort of state machine to assist the parser.
+  module PoMessageForRubyParser
+    # Supports parsing by setting attributes by and by.
+    def set_current_attribute(str)
+      param = @param_type[@param_number]
+      raise ParseError, 'no more string parameters expected' unless param
+      set_value(param, str)
+    end
+
+    def init_param
+      @param_number = 0
+      self
+    end
+
+    def advance_to_next_attribute
+      @param_number += 1
+    end    
+  end
+  class PoMessage
+    include PoMessageForRubyParser
+    alias :initialize_old :initialize
+    def initialize(type)
+      initialize_old(type)
+      init_param
+    end
+  end
+
+  module RubyParser
+    extend self
+    
+    ID = ['gettext', '_', 'N_', 'sgettext', 's_']
+    PLURAL_ID = ['ngettext', 'n_', 'Nn_', 'ns_', 'nsgettext']
+    MSGCTXT_ID = ['pgettext', 'p_']
+    MSGCTXT_PLURAL_ID = ['npgettext', 'np_']
+
+    # (Since 2.1.0) the 2nd parameter is deprecated
+    # (and ignored here). 
+    # And You don't need to keep the pomessages as unique.
+
+    def parse(path, deprecated = [])  # :nodoc:
+      lines = IO.readlines(path)
+      parse_lines(path, lines, deprecated)
+    end
+
+    def parse_lines(path, lines, deprecated = [])  # :nodoc:
+      pomessages = deprecated
+      file = StringIO.new(lines.join + "\n")
+      rl = RubyLexX.new
+      rl.set_input(file)
+      rl.skip_space = true
+      #rl.readed_auto_clean_up = true
+
+      pomessage = nil
+      line_no = nil
+      last_comment = ''
+      reset_comment = false
+      rl.parse do |tk|
+        begin
+          case tk
+          when RubyToken::TkIDENTIFIER, RubyToken::TkCONSTANT
+            store_pomessage(pomessages, pomessage, path, line_no, last_comment)
+            if ID.include?(tk.name)
+              pomessage = PoMessage.new(:normal)
+            elsif PLURAL_ID.include?(tk.name)
+              pomessage = PoMessage.new(:plural)
+            elsif MSGCTXT_ID.include?(tk.name)
+              pomessage = PoMessage.new(:msgctxt)
+            elsif MSGCTXT_PLURAL_ID.include?(tk.name)
+              pomessage = PoMessage.new(:msgctxt_plural)
+            else
+              pomessage = nil
+            end
+            line_no = tk.line_no.to_s
+          when RubyToken::TkSTRING
+            pomessage.set_current_attribute tk.value if pomessage
+          when RubyToken::TkPLUS, RubyToken::TkNL
+            #do nothing
+          when RubyToken::TkCOMMA
+            pomessage.advance_to_next_attribute if pomessage
+          else
+            if store_pomessage(pomessages, pomessage, path, line_no, last_comment)
+              pomessage = nil
+            end
+          end
+        rescue
+          $stderr.print "\n\nError"
+          $stderr.print " parsing #{path}:#{tk.line_no}\n\t #{lines[tk.line_no - 1]}" if tk
+          $stderr.print "\n #{$!.inspect} in\n"
+          $stderr.print $!.backtrace.join("\n")
+          $stderr.print "\n"
+          exit 1
+        end
+
+        case tk 
+        when RubyToken::TkCOMMENT_WITH_CONTENT
+          last_comment = "" if reset_comment
+          if last_comment.empty?
+            # new comment from programmer to translator?
+            comment1 = tk.value.lstrip
+            if comment1 =~ /^TRANSLATORS\:/
+              last_comment = $'
+            end
+          else
+            last_comment += "\n" 
+            last_comment += tk.value
+          end
+          reset_comment = false
+        when RubyToken::TkNL
+        else
+          reset_comment = true
+        end
+      end
+      pomessages
+    end
+
+    def target?(file)  # :nodoc:
+      true # always true, as the default parser.
+    end
+
+    private
+    def store_pomessage(pomessages, pomessage, file_name, line_no, last_comment) #:nodoc:
+      if pomessage && pomessage.msgid
+        pomessage.sources << file_name + ":" + line_no
+        pomessage.add_comment(last_comment) unless last_comment.empty?
+        pomessages << pomessage
+        true
+      else
+        false
+      end
+    end
+  end 
+end
+
+if __FILE__ == $0
+  require 'pp'
+  ARGV.each do |path|
+    pp GetText::RubyParser.parse(path)
+  end
+  
+  #rl = GetText::RubyLexX.new; rl.set_input(ARGF)  
+  #rl.parse do |tk|
+  #p tk
+  #end  
+end

data/lib/gettext/tools/pomessage.rb

@@ -0,0 +1,197 @@
+module GetText
+  class ParseError < StandardError
+  end
+
+  # Contains data related to the expression or sentence that
+  # is to be translated.
+  class PoMessage
+    PARAMS = {
+      :normal => [:msgid],
+      :plural => [:msgid, :msgid_plural],
+      :msgctxt => [:msgctxt, :msgid],
+      :msgctxt_plural => [:msgctxt, :msgid, :msgid_plural]
+    }
+
+    @@max_line_length = 70
+
+    # Sets the max line length.
+    def self.max_line_length=(len)
+      @@max_line_length = len
+    end
+
+    # Gets the max line length.
+    def self.max_line_length
+      @@max_line_length
+    end
+
+    # Required
+    attr_accessor :type          # :normal, :plural, :msgctxt, :msgctxt_plural 
+    attr_accessor :msgid
+    # Options
+    attr_accessor :msgid_plural
+    attr_accessor :msgctxt
+    attr_accessor :sources    # ["file1:line1", "file2:line2", ...]
+    attr_accessor :comment
+
+    # Create the object. +type+ should be :normal, :plural, :msgctxt or :msgctxt_plural. 
+    def initialize(type)
+      @type = type
+      @sources = []
+      @param_type = PARAMS[@type]
+    end
+
+    # Support for extracted comments. Explanation s.
+    # http://www.gnu.org/software/gettext/manual/gettext.html#Names
+    def add_comment(new_comment)
+      if (new_comment and ! new_comment.empty?)
+        @comment ||= ""
+        @comment += new_comment
+      end
+      to_s
+    end
+
+    # Returns a parameter representation suitable for po-files
+    # and other purposes.
+    def escaped(param_name)
+      orig = self.send param_name
+      orig.gsub(/"/, '\"').gsub(/\r/, '')
+    end
+
+    # Checks if the other translation target is mergeable with
+    # the current one. Relevant are msgid and translation context (msgctxt).
+    def ==(other)
+      other.msgid == self.msgid && other.msgctxt == self.msgctxt
+    end
+
+    # Merges two translation targets with the same msgid and returns the merged
+    # result. If one is declared as plural and the other not, then the one
+    # with the plural wins.
+    def merge(other)
+      return self unless other
+      raise ParseError, "Translation targets do not match: \n" \
+      "  self: #{self.inspect}\n  other: '#{other.inspect}'" unless self == other
+      if other.msgid_plural && !self.msgid_plural
+        res = other
+        unless (res.sources.include? self.sources[0])
+          res.sources += self.sources
+          res.add_comment(self.comment)
+        end
+      else
+        res = self
+        unless (res.sources.include? other.sources[0])
+          res.sources += other.sources
+          res.add_comment(other.comment)
+        end
+      end
+      res
+    end
+
+    # Output the po message for the po-file.
+    def to_po_str
+      raise "msgid is nil." unless @msgid
+      raise "sources is nil." unless @sources
+
+      str = ""
+      # extracted comments
+      if comment
+        comment.split("\n").each do |comment_line|
+          str << "\n#. #{comment_line.strip}"
+        end
+      end
+
+      # references
+      curr_pos = @@max_line_length
+      sources.each do |e|
+        if curr_pos + e.size > @@max_line_length
+          str << "\n#:"
+          curr_pos = 3
+        else
+          curr_pos += (e.size + 1)
+        end
+        str << " " << e
+      end
+
+      # msgctxt, msgid, msgstr
+      str << "\nmsgctxt \"" << msgctxt << "\"" if msgctxt?
+      str << "\nmsgid \"" << escaped(:msgid) << "\"\n"
+      if plural?
+        str << "msgid_plural \"" << escaped(:msgid_plural) << "\"\n"
+        str << "msgstr[0] \"\"\n"
+        str << "msgstr[1] \"\"\n"
+      else
+        str << "msgstr \"\"\n"
+      end
+      str
+    end
+
+    # Returns true if the type is kind of msgctxt.
+    # And if this is a kind of msgctxt and msgctxt property 
+    # is nil, then raise an RuntimeException.
+    def msgctxt?
+      if [:msgctxt, :msgctxt_plural].include? @type
+        raise "This PoMessage is a kind of msgctxt but the msgctxt property is nil. msgid: #{msgid}" unless @msgctxt
+        true
+      end
+    end
+
+    # Returns true if the type is kind of plural.
+    # And if this is a kind of plural and msgid_plural property 
+    # is nil, then raise an RuntimeException.
+    def plural?
+      if [:plural, :msgctxt_plural].include? @type
+        raise "This PoMessage is a kind of plural but the msgid_plural property is nil. msgid: #{msgid}" unless @msgid_plural
+        true
+      end
+    end
+
+    private
+
+    # sets or extends the value of a translation target params like msgid,
+    # msgctxt etc.
+    #   param is symbol with the name of param
+    #   value - new value
+    def set_value(param, value)
+      send "#{param}=", (send(param) || '') + value.gsub(/\n/, '\n')
+    end
+
+    public
+    # For backward comatibility. This doesn't support "comment".
+    # ary = [msgid1, "file1:line1", "file2:line"]
+    def self.new_from_ary(ary)
+      ary = ary.dup
+      msgid = ary.shift
+      sources = ary
+      type = :normal
+      msgctxt = nil
+      msgid_plural = nil
+      
+      if msgid.include? "\004"
+        msgctxt, msgid = msgid.split(/\004/)
+        type = :msgctxt
+      end
+      if msgid.include? "\000"
+        ids = msgid.split(/\000/)
+        msgid = ids[0]
+        msgid_plural = ids[1]
+        if type == :msgctxt
+          type = :msgctxt_plural
+        else
+          type = :plural
+        end
+      end
+      ret = self.new(type)
+      ret.msgid = msgid
+      ret.sources = sources
+      ret.msgctxt = msgctxt
+      ret.msgid_plural = msgid_plural
+      ret
+    end
+
+    def [](number)
+      param = @param_type[number]
+      raise ParseError, 'no more string parameters expected' unless param
+      send param
+    end
+  end
+  
+end