gettext 2.0.4 → 2.1.0

data/lib/gettext/tools/parser/ruby.rb

@@ -0,0 +1,226 @@
+#!/usr/bin/ruby
+=begin
+  parser/ruby.rb - parser for ruby script
+
+  Copyright (C) 2003-2009  Masao Mutoh
+  Copyright (C) 2005       speakillof
+  Copyright (C) 2001,2002  Yasushi Shoji, Masao Mutoh
+ 
+  You may redistribute it and/or modify it under the same
+  license terms as Ruby or LGPL.
+
+=end
+
+require 'irb/ruby-lex.rb'
+require 'stringio'
+require 'gettext/tools/pomessage'
+
+module GetText
+  class RubyLexX < RubyLex  # :nodoc: all
+    # Parser#parse resemlbes RubyLex#lex
+    def parse
+      until (  (tk = token).kind_of?(RubyToken::TkEND_OF_SCRIPT) && !@continue or tk.nil?  )
+        s = get_readed
+        if RubyToken::TkSTRING === tk
+          def tk.value
+            @value 
+          end
+          
+          def tk.value=(s)
+            @value = s
+          end
+          
+          if @here_header
+            s = s.sub(/\A.*?\n/, '').sub(/^.*\n\Z/, '')
+          else
+            begin
+              s = eval(s)
+            rescue Exception
+              # Do nothing.
+            end
+          end
+          
+          tk.value = s
+        end
+        
+        if $DEBUG
+          if tk.is_a? TkSTRING
+            $stderr.puts("#{tk}: #{tk.value}")
+          elsif tk.is_a? TkIDENTIFIER
+            $stderr.puts("#{tk}: #{tk.name}")
+          else
+            $stderr.puts(tk)
+          end
+        end
+        
+        yield tk
+      end
+      return nil
+    end
+
+    # Original parser does not keep the content of the comments,
+    # so monkey patching this with new token type and extended
+    # identify_comment implementation
+    RubyToken.def_token :TkCOMMENT_WITH_CONTENT, TkVal
+
+    def identify_comment
+      @ltype = "#"
+      get_readed # skip the hash sign itself
+
+      while ch = getc
+        if ch == "\n"
+          @ltype = nil
+          ungetc
+          break
+        end
+      end
+      return Token(TkCOMMENT_WITH_CONTENT, get_readed)
+    end
+
+  end
+
+  # Extends PoMessage for RubyParser.
+  # Implements a sort of state machine to assist the parser.
+  module PoMessageForRubyParser
+    # Supports parsing by setting attributes by and by.
+    def set_current_attribute(str)
+      param = @param_type[@param_number]
+      raise ParseError, 'no more string parameters expected' unless param
+      set_value(param, str)
+    end
+
+    def init_param
+      @param_number = 0
+      self
+    end
+
+    def advance_to_next_attribute
+      @param_number += 1
+    end    
+  end
+  class PoMessage
+    include PoMessageForRubyParser
+    alias :initialize_old :initialize
+    def initialize(type)
+      initialize_old(type)
+      init_param
+    end
+  end
+
+  module RubyParser
+    extend self
+    
+    ID = ['gettext', '_', 'N_', 'sgettext', 's_']
+    PLURAL_ID = ['ngettext', 'n_', 'Nn_', 'ns_', 'nsgettext']
+    MSGCTXT_ID = ['pgettext', 'p_']
+    MSGCTXT_PLURAL_ID = ['npgettext', 'np_']
+
+    # (Since 2.1.0) the 2nd parameter is deprecated
+    # (and ignored here). 
+    # And You don't need to keep the pomessages as unique.
+
+    def parse(path, deprecated = [])  # :nodoc:
+      lines = IO.readlines(path)
+      parse_lines(path, lines, deprecated)
+    end
+
+    def parse_lines(path, lines, deprecated = [])  # :nodoc:
+      pomessages = deprecated
+      file = StringIO.new(lines.join + "\n")
+      rl = RubyLexX.new
+      rl.set_input(file)
+      rl.skip_space = true
+      #rl.readed_auto_clean_up = true
+
+      pomessage = nil
+      line_no = nil
+      last_comment = ''
+      reset_comment = false
+      rl.parse do |tk|
+        begin
+          case tk
+          when RubyToken::TkIDENTIFIER, RubyToken::TkCONSTANT
+            store_pomessage(pomessages, pomessage, path, line_no, last_comment)
+            if ID.include?(tk.name)
+              pomessage = PoMessage.new(:normal)
+            elsif PLURAL_ID.include?(tk.name)
+              pomessage = PoMessage.new(:plural)
+            elsif MSGCTXT_ID.include?(tk.name)
+              pomessage = PoMessage.new(:msgctxt)
+            elsif MSGCTXT_PLURAL_ID.include?(tk.name)
+              pomessage = PoMessage.new(:msgctxt_plural)
+            else
+              pomessage = nil
+            end
+            line_no = tk.line_no.to_s
+          when RubyToken::TkSTRING
+            pomessage.set_current_attribute tk.value if pomessage
+          when RubyToken::TkPLUS, RubyToken::TkNL
+            #do nothing
+          when RubyToken::TkCOMMA
+            pomessage.advance_to_next_attribute if pomessage
+          else
+            if store_pomessage(pomessages, pomessage, path, line_no, last_comment)
+              pomessage = nil
+            end
+          end
+        rescue
+          $stderr.print "\n\nError"
+          $stderr.print " parsing #{path}:#{tk.line_no}\n\t #{lines[tk.line_no - 1]}" if tk
+          $stderr.print "\n #{$!.inspect} in\n"
+          $stderr.print $!.backtrace.join("\n")
+          $stderr.print "\n"
+          exit 1
+        end
+
+        case tk 
+        when RubyToken::TkCOMMENT_WITH_CONTENT
+          last_comment = "" if reset_comment
+          if last_comment.empty?
+            # new comment from programmer to translator?
+            comment1 = tk.value.lstrip
+            if comment1 =~ /^TRANSLATORS\:/
+              last_comment = $'
+            end
+          else
+            last_comment += "\n" 
+            last_comment += tk.value
+          end
+          reset_comment = false
+        when RubyToken::TkNL
+        else
+          reset_comment = true
+        end
+      end
+      pomessages
+    end
+
+    def target?(file)  # :nodoc:
+      true # always true, as the default parser.
+    end
+
+    private
+    def store_pomessage(pomessages, pomessage, file_name, line_no, last_comment) #:nodoc:
+      if pomessage && pomessage.msgid
+        pomessage.sources << file_name + ":" + line_no
+        pomessage.add_comment(last_comment) unless last_comment.empty?
+        pomessages << pomessage
+        true
+      else
+        false
+      end
+    end
+  end 
+end
+
+if __FILE__ == $0
+  require 'pp'
+  ARGV.each do |path|
+    pp GetText::RubyParser.parse(path)
+  end
+  
+  #rl = GetText::RubyLexX.new; rl.set_input(ARGF)  
+  #rl.parse do |tk|
+  #p tk
+  #end  
+end

data/lib/gettext/tools/pomessage.rb

@@ -0,0 +1,197 @@
+module GetText
+  class ParseError < StandardError
+  end
+
+  # Contains data related to the expression or sentence that
+  # is to be translated.
+  class PoMessage
+    PARAMS = {
+      :normal => [:msgid],
+      :plural => [:msgid, :msgid_plural],
+      :msgctxt => [:msgctxt, :msgid],
+      :msgctxt_plural => [:msgctxt, :msgid, :msgid_plural]
+    }
+
+    @@max_line_length = 70
+
+    # Sets the max line length.
+    def self.max_line_length=(len)
+      @@max_line_length = len
+    end
+
+    # Gets the max line length.
+    def self.max_line_length
+      @@max_line_length
+    end
+
+    # Required
+    attr_accessor :type          # :normal, :plural, :msgctxt, :msgctxt_plural 
+    attr_accessor :msgid
+    # Options
+    attr_accessor :msgid_plural
+    attr_accessor :msgctxt
+    attr_accessor :sources    # ["file1:line1", "file2:line2", ...]
+    attr_accessor :comment
+
+    # Create the object. +type+ should be :normal, :plural, :msgctxt or :msgctxt_plural. 
+    def initialize(type)
+      @type = type
+      @sources = []
+      @param_type = PARAMS[@type]
+    end
+
+    # Support for extracted comments. Explanation s.
+    # http://www.gnu.org/software/gettext/manual/gettext.html#Names
+    def add_comment(new_comment)
+      if (new_comment and ! new_comment.empty?)
+        @comment ||= ""
+        @comment += new_comment
+      end
+      to_s
+    end
+
+    # Returns a parameter representation suitable for po-files
+    # and other purposes.
+    def escaped(param_name)
+      orig = self.send param_name
+      orig.gsub(/"/, '\"').gsub(/\r/, '')
+    end
+
+    # Checks if the other translation target is mergeable with
+    # the current one. Relevant are msgid and translation context (msgctxt).
+    def ==(other)
+      other.msgid == self.msgid && other.msgctxt == self.msgctxt
+    end
+
+    # Merges two translation targets with the same msgid and returns the merged
+    # result. If one is declared as plural and the other not, then the one
+    # with the plural wins.
+    def merge(other)
+      return self unless other
+      raise ParseError, "Translation targets do not match: \n" \
+      "  self: #{self.inspect}\n  other: '#{other.inspect}'" unless self == other
+      if other.msgid_plural && !self.msgid_plural
+        res = other
+        unless (res.sources.include? self.sources[0])
+          res.sources += self.sources
+          res.add_comment(self.comment)
+        end
+      else
+        res = self
+        unless (res.sources.include? other.sources[0])
+          res.sources += other.sources
+          res.add_comment(other.comment)
+        end
+      end
+      res
+    end
+
+    # Output the po message for the po-file.
+    def to_po_str
+      raise "msgid is nil." unless @msgid
+      raise "sources is nil." unless @sources
+
+      str = ""
+      # extracted comments
+      if comment
+        comment.split("\n").each do |comment_line|
+          str << "\n#. #{comment_line.strip}"
+        end
+      end
+
+      # references
+      curr_pos = @@max_line_length
+      sources.each do |e|
+        if curr_pos + e.size > @@max_line_length
+          str << "\n#:"
+          curr_pos = 3
+        else
+          curr_pos += (e.size + 1)
+        end
+        str << " " << e
+      end
+
+      # msgctxt, msgid, msgstr
+      str << "\nmsgctxt \"" << msgctxt << "\"" if msgctxt?
+      str << "\nmsgid \"" << escaped(:msgid) << "\"\n"
+      if plural?
+        str << "msgid_plural \"" << escaped(:msgid_plural) << "\"\n"
+        str << "msgstr[0] \"\"\n"
+        str << "msgstr[1] \"\"\n"
+      else
+        str << "msgstr \"\"\n"
+      end
+      str
+    end
+
+    # Returns true if the type is kind of msgctxt.
+    # And if this is a kind of msgctxt and msgctxt property 
+    # is nil, then raise an RuntimeException.
+    def msgctxt?
+      if [:msgctxt, :msgctxt_plural].include? @type
+        raise "This PoMessage is a kind of msgctxt but the msgctxt property is nil. msgid: #{msgid}" unless @msgctxt
+        true
+      end
+    end
+
+    # Returns true if the type is kind of plural.
+    # And if this is a kind of plural and msgid_plural property 
+    # is nil, then raise an RuntimeException.
+    def plural?
+      if [:plural, :msgctxt_plural].include? @type
+        raise "This PoMessage is a kind of plural but the msgid_plural property is nil. msgid: #{msgid}" unless @msgid_plural
+        true
+      end
+    end
+
+    private
+
+    # sets or extends the value of a translation target params like msgid,
+    # msgctxt etc.
+    #   param is symbol with the name of param
+    #   value - new value
+    def set_value(param, value)
+      send "#{param}=", (send(param) || '') + value.gsub(/\n/, '\n')
+    end
+
+    public
+    # For backward comatibility. This doesn't support "comment".
+    # ary = [msgid1, "file1:line1", "file2:line"]
+    def self.new_from_ary(ary)
+      ary = ary.dup
+      msgid = ary.shift
+      sources = ary
+      type = :normal
+      msgctxt = nil
+      msgid_plural = nil
+      
+      if msgid.include? "\004"
+        msgctxt, msgid = msgid.split(/\004/)
+        type = :msgctxt
+      end
+      if msgid.include? "\000"
+        ids = msgid.split(/\000/)
+        msgid = ids[0]
+        msgid_plural = ids[1]
+        if type == :msgctxt
+          type = :msgctxt_plural
+        else
+          type = :plural
+        end
+      end
+      ret = self.new(type)
+      ret.msgid = msgid
+      ret.sources = sources
+      ret.msgctxt = msgctxt
+      ret.msgid_plural = msgid_plural
+      ret
+    end
+
+    def [](number)
+      param = @param_type[number]
+      raise ParseError, 'no more string parameters expected' unless param
+      send param
+    end
+  end
+  
+end

data/lib/gettext/{poparser.rb → tools/poparser.rb}

@@ -9,18 +9,15 @@
 
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by racc 1.4.5
-# from racc grammer file "src/poparser.ry".
+# This file is automatically generated by Racc 1.4.6
+# from Racc grammer file "".
 #
 
-require 'racc/parser'
-
-
+require 'racc/parser.rb'
 module GetText
-
   class PoParser < Racc::Parser
 
-module_eval <<'..end src/poparser.ry modeval..ida5d3d657e2', 'src/poparser.ry', 108
+module_eval(<<'...end poparser.ry/module_eval...', 'poparser.ry', 108)
   include GetText
   GetText.bindtextdomain("rgettext")
 
@@ -117,6 +114,7 @@ module_eval <<'..end src/poparser.ry modeval..ida5d3d657e2', 'src/poparser.ry',
       encoding = detect_file_encoding(po_file)
       args << "r:#{encoding}"
     end
+    @po_file = po_file
     parse(File.open(*args) {|io| io.read }, data, ignore_fuzzy)
   end
 
@@ -129,31 +127,8 @@ module_eval <<'..end src/poparser.ry modeval..ida5d3d657e2', 'src/poparser.ry',
     Encoding.default_external
   end
   private :detect_file_encoding
-..end src/poparser.ry modeval..ida5d3d657e2
-
-##### racc 1.4.5 generates ###
-
-racc_reduce_table = [
- 0, 0, :racc_error,
- 0, 10, :_reduce_none,
- 2, 10, :_reduce_none,
- 2, 10, :_reduce_none,
- 2, 10, :_reduce_none,
- 2, 12, :_reduce_5,
- 1, 13, :_reduce_none,
- 1, 13, :_reduce_none,
- 4, 15, :_reduce_8,
- 5, 16, :_reduce_9,
- 2, 17, :_reduce_10,
- 1, 17, :_reduce_none,
- 3, 18, :_reduce_12,
- 1, 11, :_reduce_13,
- 2, 14, :_reduce_14,
- 1, 14, :_reduce_15 ]
-
-racc_reduce_n = 16
-
-racc_shift_n = 26
+...end poparser.ry/module_eval...
+##### State transition tables begin ###
 
 racc_action_table = [
      3,    13,     5,     7,     9,    15,    16,    17,    20,    17,
@@ -187,89 +162,112 @@ racc_goto_pointer = [
 racc_goto_default = [
    nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil ]
 
-racc_token_table = {
- false => 0,
- Object.new => 1,
- :COMMENT => 2,
- :MSGID => 3,
- :MSGCTXT => 4,
- :MSGID_PLURAL => 5,
- :MSGSTR => 6,
- :STRING => 7,
- :PLURAL_NUM => 8 }
+racc_reduce_table = [
+  0, 0, :racc_error,
+  0, 10, :_reduce_none,
+  2, 10, :_reduce_none,
+  2, 10, :_reduce_none,
+  2, 10, :_reduce_none,
+  2, 12, :_reduce_5,
+  1, 13, :_reduce_none,
+  1, 13, :_reduce_none,
+  4, 15, :_reduce_8,
+  5, 16, :_reduce_9,
+  2, 17, :_reduce_10,
+  1, 17, :_reduce_none,
+  3, 18, :_reduce_12,
+  1, 11, :_reduce_13,
+  2, 14, :_reduce_14,
+  1, 14, :_reduce_15 ]
 
-racc_use_result_var = true
+racc_reduce_n = 16
+
+racc_shift_n = 26
+
+racc_token_table = {
+  false => 0,
+  :error => 1,
+  :COMMENT => 2,
+  :MSGID => 3,
+  :MSGCTXT => 4,
+  :MSGID_PLURAL => 5,
+  :MSGSTR => 6,
+  :STRING => 7,
+  :PLURAL_NUM => 8 }
 
 racc_nt_base = 9
 
+racc_use_result_var = true
+
 Racc_arg = [
- racc_action_table,
- racc_action_check,
- racc_action_default,
- racc_action_pointer,
- racc_goto_table,
- racc_goto_check,
- racc_goto_default,
- racc_goto_pointer,
- racc_nt_base,
- racc_reduce_table,
- racc_token_table,
- racc_shift_n,
- racc_reduce_n,
- racc_use_result_var ]
+  racc_action_table,
+  racc_action_check,
+  racc_action_default,
+  racc_action_pointer,
+  racc_goto_table,
+  racc_goto_check,
+  racc_goto_default,
+  racc_goto_pointer,
+  racc_nt_base,
+  racc_reduce_table,
+  racc_token_table,
+  racc_shift_n,
+  racc_reduce_n,
+  racc_use_result_var ]
 
 Racc_token_to_s_table = [
-'$end',
-'error',
-'COMMENT',
-'MSGID',
-'MSGCTXT',
-'MSGID_PLURAL',
-'MSGSTR',
-'STRING',
-'PLURAL_NUM',
-'$start',
-'msgfmt',
-'comment',
-'msgctxt',
-'message',
-'string_list',
-'single_message',
-'plural_message',
-'msgstr_plural',
-'msgstr_plural_line']
+  "$end",
+  "error",
+  "COMMENT",
+  "MSGID",
+  "MSGCTXT",
+  "MSGID_PLURAL",
+  "MSGSTR",
+  "STRING",
+  "PLURAL_NUM",
+  "$start",
+  "msgfmt",
+  "comment",
+  "msgctxt",
+  "message",
+  "string_list",
+  "single_message",
+  "plural_message",
+  "msgstr_plural",
+  "msgstr_plural_line" ]
 
 Racc_debug_parser = true
 
-##### racc system variables end #####
+##### State transition tables end #####
 
- # reduce 0 omitted
+# reduce 0 omitted
 
- # reduce 1 omitted
+# reduce 1 omitted
 
- # reduce 2 omitted
+# reduce 2 omitted
 
- # reduce 3 omitted
+# reduce 3 omitted
 
- # reduce 4 omitted
+# reduce 4 omitted
 
-module_eval <<'.,.,', 'src/poparser.ry', 25
-  def _reduce_5( val, _values, result )
-    @msgctxt = unescape(val[1]) + "\004"
-   result
+module_eval(<<'.,.,', 'poparser.ry', 23)
+  def _reduce_5(val, _values, result)
+        @msgctxt = unescape(val[1]) + "\004"
+  
+    result
   end
 .,.,
 
- # reduce 6 omitted
+# reduce 6 omitted
 
- # reduce 7 omitted
+# reduce 7 omitted
 
-module_eval <<'.,.,', 'src/poparser.ry', 48
-  def _reduce_8( val, _values, result )
-    if @fuzzy and $ignore_fuzzy 
+module_eval(<<'.,.,', 'poparser.ry', 35)
+  def _reduce_8(val, _values, result)
+        if @fuzzy and $ignore_fuzzy 
       if val[1] != ""
         $stderr.print _("Warning: fuzzy message was ignored.\n")
-        $stderr.print "         msgid '#{val[1]}'\n"
+        $stderr.print "  #{@po_file}: msgid '#{val[1]}'\n"
       else
         on_message('', unescape(val[3]))
       end
@@ -278,13 +276,14 @@ module_eval <<'.,.,', 'src/poparser.ry', 48
       on_message(@msgctxt + unescape(val[1]), unescape(val[3]))
     end
     result = ""
-   result
+  
+    result
   end
 .,.,
 
-module_eval <<'.,.,', 'src/poparser.ry', 65
-  def _reduce_9( val, _values, result )
-    if @fuzzy and $ignore_fuzzy
+module_eval(<<'.,.,', 'poparser.ry', 52)
+  def _reduce_9(val, _values, result)
+        if @fuzzy and $ignore_fuzzy
       if val[1] != ""
         $stderr.print _("Warning: fuzzy message was ignored.\n")
         $stderr.print "msgid = '#{val[1]}\n"
@@ -296,55 +295,62 @@ module_eval <<'.,.,', 'src/poparser.ry', 65
       on_message(@msgctxt + unescape(val[1]) + "\000" + unescape(val[3]), unescape(val[4]))
     end
     result = ""
-   result
+  
+    result
   end
 .,.,
 
-module_eval <<'.,.,', 'src/poparser.ry', 76
-  def _reduce_10( val, _values, result )
-    if val[0].size > 0
+module_eval(<<'.,.,', 'poparser.ry', 70)
+  def _reduce_10(val, _values, result)
+        if val[0].size > 0
       result = val[0] + "\000" + val[1]
     else
       result = ""
     end
-   result
+  
+    result
   end
 .,.,
 
- # reduce 11 omitted
+# reduce 11 omitted
 
-module_eval <<'.,.,', 'src/poparser.ry', 84
-  def _reduce_12( val, _values, result )
-    result = val[2]
-   result
+module_eval(<<'.,.,', 'poparser.ry', 82)
+  def _reduce_12(val, _values, result)
+        result = val[2]
+  
+    result
   end
 .,.,
 
-module_eval <<'.,.,', 'src/poparser.ry', 91
-  def _reduce_13( val, _values, result )
-    on_comment(val[0])
-   result
+module_eval(<<'.,.,', 'poparser.ry', 89)
+  def _reduce_13(val, _values, result)
+        on_comment(val[0])
+  
+    result
   end
 .,.,
 
-module_eval <<'.,.,', 'src/poparser.ry', 99
-  def _reduce_14( val, _values, result )
-    result = val.delete_if{|item| item == ""}.join
-   result
+module_eval(<<'.,.,', 'poparser.ry', 97)
+  def _reduce_14(val, _values, result)
+        result = val.delete_if{|item| item == ""}.join
+  
+    result
   end
 .,.,
 
-module_eval <<'.,.,', 'src/poparser.ry', 103
-  def _reduce_15( val, _values, result )
-    result = val[0]
-   result
+module_eval(<<'.,.,', 'poparser.ry', 101)
+  def _reduce_15(val, _values, result)
+        result = val[0]
+  
+    result
   end
 .,.,
 
- def _reduce_none( val, _values, result )
-  result
- end
+def _reduce_none(val, _values, result)
+  val[0]
+end
 
   end   # class PoParser
+  end   # module GetText
+
 
-end   # module GetText