gettext 3.2.9 → 3.3.0

data/Rakefile

@@ -49,7 +49,8 @@ end
 
 po_parser_ry_path = "src/po_parser.ry"
 file po_parser_rb_path => po_parser_ry_path do
-  racc = File.join(Gem.bindir, "racc")
+  racc_spec = Gem::Specification.find_by_name("racc")
+  racc = File.join(racc_spec.bin_dir, racc_spec.executable)
   tempfile = Tempfile.new("gettext-po-parser")
   ruby(racc, "-g", po_parser_ry_path, "-o", tempfile.path)
 

data/doc/text/news.md

@@ -1,5 +1,48 @@
 # News
 
+## 3.3.0: 2020-01-08 {#version-3-3-0}
+
+### Improvements
+
+  * Fixed README markup.
+    [GitHub#57][Patch by Alexander Paukste]
+
+  * Suppressed warnings.
+    [GitHub#58][Patch by 284km]
+
+  * Improved README.
+    [GitHub#62][Patch by Robert Graff]
+
+  * Added support for finding `racc` of Ruby 2.7.
+    [GitHub#65][Patch by KITAITI Makoto]
+
+  * Added support for Ruby 2.7.
+    [GitHub#64][Reported by Anatol Pomozov]
+
+  * Dropped support for Ruby 2.4.
+
+### Fixes
+
+  * Fixed a bug that `n_` may return nil.
+    [GitHub#60][Patch by Michaël Hoste]
+
+  * Fixed a sort by msgid bug.
+    [GitHub#61][Patch by Robert Graff]
+
+### Thanks
+
+  * Alexander Paukste
+
+  * 284km
+
+  * Michaël Hoste
+
+  * Robert Graff
+
+  * KITAITI Makoto
+
+  * Anatol Pomozov
+
 ## 3.2.9: 2018-03-05 {#version-3-2-9}
 
 ### Fixes

data/gettext.gemspec

@@ -15,8 +15,7 @@ So you can use GNU gettext tools for maintaining.
   EOD
   s.authors = ["Kouhei Sutou", "Masao Mutoh"]
   s.email = ["kou@clear-code.com", "mutomasa at gmail.com"]
-  s.homepage = "http://ruby-gettext.github.com/"
-  s.rubyforge_project = "gettext"
+  s.homepage = "https://ruby-gettext.github.io/"
   s.require_paths = ["lib"]
   Dir.chdir(base_dir) do
     s.files = Dir.glob("{locale,bin,data,doc/text,lib,po,samples,src,test}/**/*")
@@ -28,6 +27,8 @@ So you can use GNU gettext tools for maintaining.
     s.test_files = Dir.glob("test/test_*.rb")
   end
 
+  s.required_ruby_version = ">= 2.5.0"
+
   s.add_runtime_dependency("locale", ">= 2.0.5")
   s.add_runtime_dependency("text", ">= 1.3.0")
   s.add_development_dependency("rake")

data/lib/gettext/locale_path.rb

@@ -85,7 +85,11 @@ module GetText
         this_path_rules = rule % {:lang => "([^\/]+)", :name => name}
         Dir.glob(rule % {:lang => "*", :name => name}).each do |path|
           if /#{this_path_rules}/ =~ path
-            @locale_paths[$1] = path.untaint unless @locale_paths[$1]
+            locale_path = $1
+            unless @locale_paths[locale_path]
+              path.untaint if RUBY_VERSION < "2.7"
+              @locale_paths[locale_path] = path
+            end
           end
         end
       end

data/lib/gettext/po.rb

@@ -222,11 +222,11 @@ module GetText
     def sort(entries)
       case @order
       when :reference, :references # :references is deprecated.
-        sorted_entries = sort_by_reference(entries)
+        sort_by_reference(entries)
       when :msgid
-        sorted_entries = sort_by_msgid(entries)
+        sort_by_msgid(entries)
       else
-        sorted_entries = entries.to_a
+        entries.to_a
       end
     end
 
@@ -268,7 +268,7 @@ module GetText
     def sort_by_msgid(entries)
       entries.sort_by do |msgid_entry|
         # msgid_entry = [[msgctxt, msgid], POEntry]
-        msgid_entry[0]
+        msgid_entry[0][1]
       end
     end
   end

data/lib/gettext/po_entry.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2012-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2012-2019  Sutou Kouhei <kou@clear-code.com>
 # Copyright (C) 2010  masone (Christian Felder) <ema@rh-productions.ch>
 # Copyright (C) 2009  Masao Mutoh
 #
@@ -210,13 +208,25 @@ module GetText
       true
     end
 
-    def [](number)
-      param = @param_type[number]
-      raise ParseError, 'no more string parameters expected' unless param
-      send param
+    def [](number_or_param)
+      __send__(resolve_param(number_or_param))
+    end
+
+    def []=(number_or_param, value)
+      __send__("#{resolve_param(number_or_param)}=", value)
     end
 
     private
+    def resolve_param(number_or_param)
+      case number_or_param
+      when Integer
+        param = @param_type[number_or_param]
+        raise ParseError, 'no more string parameters expected' unless param
+        param
+      else
+        number_or_param
+      end
+    end
 
     # sets or extends the value of a translation target params like msgid,
     # msgctxt etc.

data/lib/gettext/text_domain.rb

@@ -124,7 +124,7 @@ module GetText
         # [[msgstr[0], msgstr[1], msgstr[2],...], cond]
         mo = @mofiles[lang.to_s]
         cond = (mo and mo != :empty) ? mo.plural_as_proc : DEFAULT_PLURAL_CALC
-        ret = [msg.split("\000"), cond]
+        ret = [msg.split("\000", -1), cond]
       else
         ret = [[msg], DEFAULT_SINGLE_CALC]
       end

data/lib/gettext/tools/parser/ruby.rb

@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
 =begin
   parser/ruby.rb - parser for ruby script
 
-  Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
+  Copyright (C) 2013-2019  Sutou Kouhei <kou@clear-code.com>
   Copyright (C) 2003-2009  Masao Mutoh
   Copyright (C) 2005       speakillof
   Copyright (C) 2001,2002  Yasushi Shoji, Masao Mutoh
@@ -12,162 +11,271 @@
 
 =end
 
-require "irb/ruby-lex"
+require "ripper"
 require "stringio"
-require "gettext/po_entry"
 
-require "ripper"
+require "gettext/po_entry"
 
 module GetText
-  class RubyLexX < RubyLex  # :nodoc: all
-    class StringExtractor < Ripper::Filter
+  class RubyParser
+    class POExtractor < Ripper::Filter
+      ID = ["gettext", "_", "N_", "sgettext", "s_"]
+      PLURAL_ID = ["ngettext", "n_", "Nn_", "ns_", "nsgettext"]
+      MSGCTXT_ID = ["pgettext", "p_"]
+      MSGCTXT_PLURAL_ID = ["npgettext", "np_"]
+
+      attr_accessor :use_comment
+      attr_accessor :comment_tag
       def initialize(*args)
-        super
+        super(*args)
+        @in_block_arguments = false
+        @ignore_next_comma = false
+        @context_stack = []
+        @need_definition_name = false
+        @current_po_entry = nil
+        @current_po_entry_nth_attribute = 0
+        @use_comment = false
+        @comment_tag = nil
+        @last_comment = ""
+        @reset_comment = false
+        @embed_expression_level = 0
         @string_mark_stack = []
+        @string_stack = []
       end
 
-      def on_default(event, token, output)
-        case event
-        when :on_tstring_content
-          if @string_mark_stack.last == "\""
-            output << token.gsub(/\\./) do |data|
-              case data
-              when "\\n"
-                "\n"
-              when "\\t"
-                "\t"
-              when "\\\\"
-                "\\"
-              when "\\\""
-                "\""
-              when "\\\#"
-                "#"
-              else
-                data
-              end
-            end
-          else
-            output << token.gsub(/\\./) do |data|
-              case data
-              when "\\\\"
-                "\\"
-              when "\\'"
-                "'"
-              else
-                data
-              end
-            end
-          end
-        when :on_tstring_beg
-          unless @string_mark_stack.empty?
-            output << token
-          end
-          @string_mark_stack << token
-        when :on_tstring_end
-          @string_mark_stack.pop
-          unless @string_mark_stack.empty?
-            output << token
+      def process_on_op(token, po)
+        @in_block_arguments = !@in_block_arguments if token == "|"
+        po
+      end
+
+      def process_on_kw(token, po)
+        store_po_entry(po)
+        case token
+        when "begin", "case", "do", "for"
+          @context_stack.push(token)
+        when "class", "def", "module"
+          @context_stack.push(token)
+        when "if", "unless", "until", "while"
+          # postfix case
+          unless state.allbits?(Ripper::EXPR_LABEL)
+            @context_stack.push(token)
           end
+        when "end"
+          @context_stack.pop
+        end
+        po
+      end
+
+      def process_on_ident(token, po)
+        if @embed_expression_level > 0
+          @string_stack.last << token
+          return po
+        end
+
+        store_po_entry(po)
+
+        return po if @in_block_arguments
+        return po if state.allbits?(Ripper::EXPR_ENDFN)
+
+        case token
+        when *ID
+          @current_po_entry = POEntry.new(:normal)
+        when *PLURAL_ID
+          @current_po_entry = POEntry.new(:plural)
+        when *MSGCTXT_ID
+          @current_po_entry = POEntry.new(:msgctxt)
+        when *MSGCTXT_PLURAL_ID
+          @current_po_entry = POEntry.new(:msgctxt_plural)
+        end
+        if @current_po_entry
+          @current_po_entry.add_comment(@last_comment) unless @last_comment.empty?
+          @last_comment = ""
+          @current_po_entry.references << "#{filename}:#{lineno}"
+          @current_po_entry_nth_attribute = 0
+        end
+        po
+      end
+
+      def process_on_const(token, po)
+        case token
+        when "N_"," Nn_"
+          # TODO: Check the next token is :on_lparen
+          process_on_ident(token, po)
         else
-          unless @string_mark_stack.empty?
-            output << token.to_s
-          end
+          po
         end
-        output
       end
-    end
 
-    # Parser#parse resemlbes RubyLex#lex
-    def parse
-      until (  (tk = token).kind_of?(RubyToken::TkEND_OF_SCRIPT) && !@continue or tk.nil?  )
-        s = get_readed
-        if RubyToken::TkSTRING === tk or RubyToken::TkDSTRING === tk
-          def tk.value
-            @value
+      def process_on_comment(token, po)
+        @last_comment = "" if @reset_comment
+        @reset_comment = false
+        if @last_comment.empty?
+          content = token.gsub(/\A#\s*/, "").chomp
+          if comment_to_be_extracted?(content)
+            @last_comment << content
           end
+        else
+          content = token.gsub(/\A#/, "").chomp
+          @last_comment << "\n"
+          @last_comment << content
+        end
+        po
+      end
 
-          def tk.value=(s)
-            @value = s
-          end
+      def process_on_sp(token, po)
+        po
+      end
 
-          if @here_header
-            s = s.sub(/\A.*?\n/, "").sub(/^.*\n\Z/, "")
-          else
-            s = StringExtractor.new(s).parse("")
+      def process_on_tstring_beg(token, po)
+        @string_mark_stack << token
+        @string_stack << ""
+        po
+      end
+
+      def process_on_tstring_content(token, po)
+        if @string_mark_stack.last == "\""
+          @string_stack.last << token.gsub(/\\./) do |data|
+            case data
+            when "\\n"
+              "\n"
+            when "\\t"
+              "\t"
+            when "\\\\"
+              "\\"
+            when "\\\""
+              "\""
+            when "\\\#"
+              "#"
+            else
+              data
+            end
+          end
+        else
+          @string_stack.last << token.gsub(/\\./) do |data|
+            case data
+            when "\\\\"
+              "\\"
+            when "\\'"
+              "'"
+            else
+              data
+            end
           end
+        end
+        po
+      end
 
-          tk.value = s
+      def process_on_tstring_end(token, po)
+        @ignore_next_comma = false
+        @string_mark_stack.pop
+        last_string = @string_stack.pop
+        if @current_po_entry and last_string
+          @current_po_entry[@current_po_entry_nth_attribute] =
+            (@current_po_entry[@current_po_entry_nth_attribute] || "") +
+            last_string
         end
+        po
+      end
 
-        if $DEBUG
-          if tk.is_a? TkSTRING or tk.is_a? TkDSTRING
-            $stderr.puts("#{tk}: #{tk.value}")
-          elsif tk.is_a? TkIDENTIFIER
-            $stderr.puts("#{tk}: #{tk.name}")
-          else
-            $stderr.puts(tk)
-          end
+      def process_on_heredoc_beg(token, po)
+        if token.end_with?("'")
+          @string_mark_stack << "'"
+        else
+          @string_mark_stack << "\""
         end
+        @string_stack << ""
+        po
+      end
 
-        yield tk
+      def process_on_heredoc_end(token, po)
+        process_on_tstring_end(token, po)
+      end
+
+      def process_on_regexp_beg(token, po)
+        @string_mark_stack << "\""
+        @string_stack << ""
+        po
       end
-      return nil
-    end
 
-    # Original parser does not keep the content of the comments,
-    # so monkey patching this with new token type and extended
-    # identify_comment implementation
-    RubyToken.def_token :TkCOMMENT_WITH_CONTENT, TkVal
+      def process_on_embexpr_beg(token, po)
+        @embed_expression_level += 1
+        @string_stack.last << token
+        po
+      end
 
-    def identify_comment
-      @ltype = "#"
-      get_readed # skip the hash sign itself
+      def process_on_embexpr_end(token, po)
+        @embed_expression_level -= 1
+        @string_stack.last << token
+        po
+      end
 
-      while ch = getc
-        if ch == "\n"
-          @ltype = nil
-          ungetc
-          break
+      def process_on_regexp_end(token, po)
+        @string_mark_stack.pop
+        @string_stack.pop
+        po
+      end
+
+      def process_on_int(token, po)
+        @ignore_next_comma = true
+        po
+      end
+
+      def process_on_comma(token, po)
+        unless @ignore_next_comma
+          if @current_po_entry
+            @current_po_entry_nth_attribute += 1
+          end
         end
+        po
       end
-      return Token(TkCOMMENT_WITH_CONTENT, get_readed)
-    end
 
-  end
+      def process_on_rparen(token, po)
+        store_po_entry(po)
+        po
+      end
 
-  # Extends POEntry for RubyParser.
-  # Implements a sort of state machine to assist the parser.
-  module POEntryForRubyParser
-    # Supports parsing by setting attributes by and by.
-    def set_current_attribute(str)
-      param = @param_type[@param_number]
-      raise ParseError, "no more string parameters expected" unless param
-      set_value(param, str)
-    end
+      def process_on_nl(token, po)
+        @reset_comment = true
+        po
+      end
 
-    def init_param
-      @param_number = 0
-      self
-    end
+      def on_default(event, token, po)
+        trace(event, token) do
+          process_method = "process_#{event}"
+          if respond_to?(process_method)
+            __send__(process_method, token, po)
+          else
+            po
+          end
+        end
+      end
 
-    def advance_to_next_attribute
-      @param_number += 1
-    end
-  end
-  class POEntry
-    include POEntryForRubyParser
-    alias :initialize_old :initialize
-    def initialize(type)
-      initialize_old(type)
-      init_param
-    end
-  end
+      private
+      @@debug = ENV["GETTEXT_RUBY_PARSER_DEBUG"]
+      def debug?
+        @@debug
+      end
 
-  class RubyParser
-    ID = ["gettext", "_", "N_", "sgettext", "s_"]
-    PLURAL_ID = ["ngettext", "n_", "Nn_", "ns_", "nsgettext"]
-    MSGCTXT_ID = ["pgettext", "p_"]
-    MSGCTXT_PLURAL_ID = ["npgettext", "np_"]
+      def trace(event_name, token)
+        pp [event_name, token, state, @context_stack.last] if debug?
+        yield
+      end
+
+      def store_po_entry(po)
+        return if @current_po_entry.nil?
+        po << @current_po_entry if @current_po_entry.msgid
+        @current_po_entry = nil
+        @current_po_entry_nth_attribute = 0
+      end
+
+      def comment_to_be_extracted?(comment)
+        return false unless @use_comment
+
+        return true if @comment_tag.nil?
+
+        comment.start_with?(@comment_tag)
+      end
+    end
 
     class << self
       def target?(file)  # :nodoc:
@@ -282,106 +390,12 @@ module GetText
     end
 
     def parse_source(source)
-      po = []
-      file = StringIO.new(source)
-      rl = RubyLexX.new
-      rl.set_input(file)
-      rl.skip_space = true
-      #rl.readed_auto_clean_up = true
-
-      po_entry = nil
-      line_no = nil
-      last_comment = ""
-      reset_comment = false
-      ignore_next_comma = false
-      rl.parse do |tk|
-        begin
-          ignore_current_comma = ignore_next_comma
-          ignore_next_comma = false
-          case tk
-          when RubyToken::TkIDENTIFIER, RubyToken::TkCONSTANT
-            if store_po_entry(po, po_entry, line_no, last_comment)
-              last_comment = ""
-            end
-            if ID.include?(tk.name)
-              po_entry = POEntry.new(:normal)
-            elsif PLURAL_ID.include?(tk.name)
-              po_entry = POEntry.new(:plural)
-            elsif MSGCTXT_ID.include?(tk.name)
-              po_entry = POEntry.new(:msgctxt)
-            elsif MSGCTXT_PLURAL_ID.include?(tk.name)
-              po_entry = POEntry.new(:msgctxt_plural)
-            else
-              po_entry = nil
-            end
-            line_no = tk.line_no.to_s
-          when RubyToken::TkBITOR
-            po_entry = nil
-          when RubyToken::TkSTRING, RubyToken::TkDSTRING
-            po_entry.set_current_attribute tk.value if po_entry
-          when RubyToken::TkPLUS, RubyToken::TkNL
-            #do nothing
-          when RubyToken::TkINTEGER
-            ignore_next_comma = true
-          when RubyToken::TkCOMMA
-            unless ignore_current_comma
-              po_entry.advance_to_next_attribute if po_entry
-            end
-          else
-            if store_po_entry(po, po_entry, line_no, last_comment)
-              po_entry = nil
-              last_comment = ""
-            end
-          end
-        rescue
-          $stderr.print "\n\nError"
-          $stderr.print " parsing #{@path}:#{tk.line_no}\n\t #{source.lines.to_a[tk.line_no - 1]}" if tk
-          $stderr.print "\n #{$!.inspect} in\n"
-          $stderr.print $!.backtrace.join("\n")
-          $stderr.print "\n"
-          exit 1
-        end
-
-        case tk
-        when RubyToken::TkCOMMENT_WITH_CONTENT
-          last_comment = "" if reset_comment
-          if last_comment.empty?
-            comment1 = tk.value.lstrip
-            if comment_to_be_extracted?(comment1)
-              last_comment += comment1
-            end
-          else
-            last_comment += "\n"
-            last_comment += tk.value
-          end
-          reset_comment = false
-        when RubyToken::TkNL
-        else
-          reset_comment = true
-        end
-      end
-      po
-    end
-
-    private
-    def store_po_entry(po, po_entry, line_no, last_comment) #:nodoc:
-      if po_entry && po_entry.msgid
-        po_entry.references << @path + ":" + line_no
-        po_entry.add_comment(last_comment) unless last_comment.empty?
-        po << po_entry
-        true
-      else
-        false
+      extractor = POExtractor.new(source, @path)
+      if @options.key?(:comment_tag)
+        extractor.use_comment = true
+        extractor.comment_tag = @options[:comment_tag]
       end
-    end
-
-    def comment_to_be_extracted?(comment)
-      return false unless @options.has_key?(:comment_tag)
-
-      tag = @options[:comment_tag]
-      return true if tag.nil?
-
-      /\A#{Regexp.escape(tag)}/ === comment
+      extractor.parse([])
     end
   end
 end