parser 2.1.0.pre1 → 2.1.0.pre2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85ff1f420d063d29c4d333d54a120569fd0429e9
-  data.tar.gz: f7fbd9186d41a3e8ca2561cbb590f8bde8067a6e
+  metadata.gz: ada647b43495037cf17844d58832f74d63348d88
+  data.tar.gz: 4c4a750b4e696fb5f31f8b6c46718f2d71731651
 SHA512:
-  metadata.gz: 40f6351913140cd1b73bf9b2d2ada48dd56ce29dad4e7c08bb347762b5d36a9bac95aa5b2b59c9f46155d35fa2db7e6e3400da72ceabe6d847381d932886e030
-  data.tar.gz: d1f1581bffe9ebecbd6fa02eaac2125db6ce329203d2d28061361588101b8442ace659f1b53e9a09189a6430506bc3662f323d7bebadef3cf41c2b98eb43634c
+  metadata.gz: 69ae3a6286559fa7942af81b631cc1389cb0075f5c683cc460a46f1d3a29a1527f174604dd220c687c4b660e1e4bb409b49ce401f8113d4b5be5481486a3fe12
+  data.tar.gz: 94b9c811b9d55d468ecc7de739228fa4c6622eb46bbfb7bdf3b4ef637f566fd2804e459354059b92ae6fb96fa0d2b2d9ffe9980bbf2a35236ac1d86687da985e

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 Changelog
 =========
 
+v2.1.0.pre2 (2013-11-25)
+------------------------
+
+API modifications:
+ * Parser::Diagnostic: expose reason symbolically (closes #115, #116). (Ian MacLeod)
+
+vir (2013-11-13)
+----------------
+
+API modifications:
+ * lexer.rl: coerce literals to UTF-8 in ASCII-encoded files if they contain \uXXXX (Peter Zotov)
+
 v2.1.0.pre1 (2013-11-12)
 ------------------------
 

data/lib/parser.rb

@@ -16,6 +16,7 @@ end
 #
 module Parser
   require 'parser/version'
+  require 'parser/messages'
 
   module AST
     require 'parser/ast/node'
@@ -64,58 +65,6 @@ module Parser
 
   require 'parser/rewriter'
 
-  ERRORS = {
-    # Lexer errors
-    :unicode_point_too_large => 'invalid Unicode codepoint (too large)',
-    :invalid_escape          => 'invalid escape character syntax',
-    :incomplete_escape       => 'incomplete character syntax',
-    :invalid_hex_escape      => 'invalid hex escape',
-    :invalid_unicode_escape  => 'invalid Unicode escape',
-    :unterminated_unicode    => 'unterminated Unicode escape',
-    :escape_eof              => 'escape sequence meets end of file',
-    :string_eof              => 'unterminated string meets end of file',
-    :regexp_options          => 'unknown regexp options: %{options}',
-    :cvar_name               => "`%{name}' is not allowed as a class variable name",
-    :ivar_name               => "`%{name}' is not allowed as an instance variable name",
-    :trailing_in_number      => "trailing `%{character}' in number",
-    :empty_numeric           => 'numeric literal without digits',
-    :invalid_octal           => 'invalid octal digit',
-    :no_dot_digit_literal    => 'no .<digit> floating literal anymore; put 0 before dot',
-    :bare_backslash          => 'bare backslash only allowed before newline',
-    :unexpected              => "unexpected `%{character}'",
-    :embedded_document       => 'embedded document meets end of file (and they embark on a romantic journey)',
-
-    # Lexer warnings
-    :invalid_escape_use      => 'invalid character syntax; use ?%{escape}',
-    :ambiguous_literal       => 'ambiguous first argument; parenthesize arguments or add whitespace to the right',
-    :ambiguous_prefix        => "`%{prefix}' interpreted as argument prefix",
-
-    # Parser errors
-    :nth_ref_alias           => 'cannot define an alias for a back-reference variable',
-    :begin_in_method         => 'BEGIN in method',
-    :backref_assignment      => 'cannot assign to a back-reference variable',
-    :invalid_assignment      => 'cannot assign to a keyword',
-    :module_name_const       => 'class or module name must be a constant literal',
-    :unexpected_token        => 'unexpected token %{token}',
-    :argument_const          => 'formal argument cannot be a constant',
-    :argument_ivar           => 'formal argument cannot be an instance variable',
-    :argument_gvar           => 'formal argument cannot be a global variable',
-    :argument_cvar           => 'formal argument cannot be a class variable',
-    :duplicate_argument      => 'duplicate argument name',
-    :empty_symbol            => 'empty symbol literal',
-    :odd_hash                => 'odd number of entries for a hash',
-    :singleton_literal       => 'cannot define a singleton method for a literal',
-    :dynamic_const           => 'dynamic constant assignment',
-    :module_in_def           => 'module definition in method body',
-    :class_in_def            => 'class definition in method body',
-    :unexpected_percent_str  => '%{type}: unknown type of percent-literal',
-    :block_and_blockarg      => 'both block argument and literal block are passed',
-    :masgn_as_condition      => 'multiple assignment in conditional context',
-
-    # Parser warnings
-    :useless_else            => 'else without rescue is useless',
-  }.freeze
-
   ##
   # Verify that the current Ruby implementation supports Encoding.
   # @raise [RuntimeError]

data/lib/parser/base.rb

@@ -235,11 +235,11 @@ module Parser
       when /^[a-z_]/
         # OK
       when /^[A-Z]/
-        diagnostic :error, :argument_const, name_t
+        diagnostic :error, :argument_const, nil, name_t
       end
     end
 
-    def diagnostic(level, kind, location_t, highlights_ts=[])
+    def diagnostic(level, reason, arguments, location_t, highlights_ts=[])
       _, location = location_t
 
       highlights = highlights_ts.map do |token|
@@ -247,9 +247,8 @@ module Parser
         range
       end
 
-      message = ERRORS[kind]
       @diagnostics.process(
-          Diagnostic.new(level, message, location, highlights))
+          Diagnostic.new(level, reason, arguments, location, highlights))
 
       if level == :error
         yyerror
@@ -260,9 +259,8 @@ module Parser
       token_name = token_to_str(error_token_id)
       _, location = error_value
 
-      message = ERRORS[:unexpected_token] % { :token => token_name }
-      @diagnostics.process(
-          Diagnostic.new(:error, message, location))
+      @diagnostics.process(Diagnostic.new(
+          :error, :unexpected_token, { :token => token_name }, location))
     end
   end
 

data/lib/parser/builders/default.rb

@@ -146,7 +146,7 @@ module Parser
         n(:sym, [ str.children.first.to_sym ],
           collection_map(begin_t, str.loc.expression, end_t))
       elsif @parser.version == 18 && parts.empty?
-        diagnostic(:error, ERRORS[:empty_symbol], loc(begin_t).join(loc(end_t)))
+        diagnostic :error, :empty_symbol, nil, loc(begin_t).join(loc(end_t))
       else
         n(:dsym, [ *parts ],
           collection_map(begin_t, parts, end_t))
@@ -233,8 +233,7 @@ module Parser
 
     def pair_list_18(list)
       if list.size % 2 != 0
-        message = ERRORS[:odd_hash]
-        diagnostic :error, message, list.last.loc.expression
+        diagnostic :error, :odd_hash, nil, list.last.loc.expression
       else
         list.
           each_slice(2).map do |key, value|
@@ -390,8 +389,7 @@ module Parser
 
       when :const
         if @parser.in_def?
-          message = ERRORS[:dynamic_const]
-          diagnostic :error, message, node.loc.expression
+          diagnostic :error, :dynamic_const, nil, node.loc.expression
         end
 
         node.updated(:casgn)
@@ -404,12 +402,10 @@ module Parser
 
       when :nil, :self, :true, :false,
            :__FILE__, :__LINE__, :__ENCODING__
-        message = ERRORS[:invalid_assignment]
-        diagnostic :error, message, node.loc.expression
+        diagnostic :error, :invalid_assignment, nil, node.loc.expression
 
       when :back_ref, :nth_ref
-        message = ERRORS[:backref_assignment]
-        diagnostic :error, message, node.loc.expression
+        diagnostic :error, :backref_assignment, nil, node.loc.expression
       end
     end
 
@@ -442,8 +438,7 @@ module Parser
         end
 
       when :back_ref, :nth_ref
-        message = ERRORS[:backref_assignment]
-        diagnostic :error, message, lhs.loc.expression
+        diagnostic :error, :backref_assignment, nil, lhs.loc.expression
       end
     end
 
@@ -497,8 +492,7 @@ module Parser
       when :int, :str, :dstr, :sym, :dsym,
            :regexp, :array, :hash
 
-        message = ERRORS[:singleton_literal]
-        diagnostic :error, message, definee.loc.expression
+        diagnostic :error, :singleton_literal, nil, definee.loc.expression
 
       else
         n(:defs, [ definee, value(name_t).to_sym, args, body ],
@@ -634,8 +628,7 @@ module Parser
       last_arg = call_args.last
 
       if last_arg && last_arg.type == :block_pass
-        diagnostic :error, ERRORS[:block_and_blockarg],
-                   last_arg.loc.expression
+        diagnostic :error, :block_and_blockarg, nil, last_arg.loc.expression
       end
 
       n(:block, [ method_call, args, body ],
@@ -935,8 +928,7 @@ module Parser
     def check_condition(cond)
       case cond.type
       when :masgn
-        diagnostic :error, ERRORS[:masgn_as_condition],
-                   cond.loc.expression
+        diagnostic :error, :masgn_as_condition, nil, cond.loc.expression
 
       when :begin
         if cond.children.count == 1
@@ -988,7 +980,7 @@ module Parser
           if that_arg.nil?
             map[this_name] = this_arg
           elsif arg_name_collides?(this_name, that_name)
-            diagnostic :error, ERRORS[:duplicate_argument],
+            diagnostic :error, :duplicate_argument, nil,
                        this_arg.loc.name, [ that_arg.loc.name ]
           end
 
@@ -1341,9 +1333,9 @@ module Parser
       token[1] if token && token[0]
     end
 
-    def diagnostic(type, message, location, highlights=[])
+    def diagnostic(type, reason, arguments, location, highlights=[])
       @parser.diagnostics.process(
-          Diagnostic.new(type, message, location, highlights))
+          Diagnostic.new(type, reason, arguments, location, highlights))
 
       if type == :error
         @parser.send :yyerror

data/lib/parser/diagnostic.rb

@@ -7,6 +7,14 @@ module Parser
   #  @see LEVELS
   #  @return [Symbol] diagnostic level
   #
+  # @!attribute [r] reason
+  #  @see Parser::MESSAGES
+  #  @return [Symbol] reason for error
+  #
+  # @!attribute [r] arguments
+  #  @see Parser::MESSAGES
+  #  @return [Symbol] extended arguments that describe the error
+  #
   # @!attribute [r] message
   #  @return [String] error message
   #
@@ -26,30 +34,40 @@ module Parser
     #
     LEVELS = [:note, :warning, :error, :fatal].freeze
 
-    attr_reader :level, :message
+    attr_reader :level, :reason, :arguments
     attr_reader :location, :highlights
 
     ##
     # @param [Symbol] level
-    # @param [String] message
+    # @param [Symbol] reason
+    # @param [Hash] arguments
     # @param [Parser::Source::Range] location
     # @param [Array<Parser::Source::Range>] highlights
     #
-    def initialize(level, message, location, highlights=[])
+    def initialize(level, reason, arguments, location, highlights=[])
       unless LEVELS.include?(level)
         raise ArgumentError,
               "Diagnostic#level must be one of #{LEVELS.join(', ')}; " \
               "#{level.inspect} provided."
       end
+      raise 'Expected a location' unless location
 
       @level       = level
-      @message     = message.to_s.dup.freeze
+      @reason      = reason
+      @arguments   = (arguments || {}).dup.freeze
       @location    = location
       @highlights  = highlights.dup.freeze
 
       freeze
     end
 
+    ##
+    # @return [String] the rendered message.
+    #
+    def message
+      MESSAGES[@reason] % @arguments
+    end
+
     ##
     # Renders the diagnostic message as a clang-like diagnostic.
     #
@@ -76,7 +94,7 @@ module Parser
       highlight_line[range] = '^' * @location.size
 
       [
-        "#{@location.to_s}: #{@level}: #{@message}",
+        "#{@location.to_s}: #{@level}: #{message}",
         source_line,
         highlight_line,
       ]

data/lib/parser/diagnostic/engine.rb

@@ -13,9 +13,10 @@ module Parser
   #  end
   #
   #  engine     = Parser::Diagnostic::Engine.new(consumer)
-  #  diagnostic = Parser::Diagnostic.new(:warning, 'warning!', buffer, 1..2)
+  #  diagnostic = Parser::Diagnostic.new(
+  #      :warning, :unexpected_token, { :token => 'abc' }, buffer, 1..2)
   #
-  #  engine.process(diagnostic) # => "warning!"
+  #  engine.process(diagnostic) # => "unexpected token abc"
   #
   # @api public
   #

data/lib/parser/lexer.rl

@@ -348,9 +348,9 @@ class Parser::Lexer
     nil
   end
 
-  def diagnostic(type, message, location=range, highlights=[])
+  def diagnostic(type, reason, arguments=nil, location=range, highlights=[])
     @diagnostics.process(
-        Parser::Diagnostic.new(type, message, location, highlights))
+        Parser::Diagnostic.new(type, reason, arguments, location, highlights))
   end
 
   #
@@ -622,7 +622,7 @@ class Parser::Lexer
 
       if codepoint >= 0x110000
         @escape = lambda do
-          diagnostic :error, Parser::ERRORS[:unicode_point_too_large],
+          diagnostic :error, :unicode_point_too_large, nil,
                      range(codepoint_s, codepoint_s + codepoint_str.length)
         end
 
@@ -644,7 +644,7 @@ class Parser::Lexer
 
   action invalid_complex_escape {
     @escape = lambda do
-      diagnostic :error, Parser::ERRORS[:invalid_escape]
+      diagnostic :error, :invalid_escape
     end
   }
 
@@ -684,7 +684,7 @@ class Parser::Lexer
     | 'x' ( c_any - xdigit )
       % {
         @escape = lambda do
-          diagnostic :error, Parser::ERRORS[:invalid_hex_escape],
+          diagnostic :error, :invalid_hex_escape, nil,
                      range(@escape_s - 1, p + 2)
         end
       }
@@ -699,7 +699,7 @@ class Parser::Lexer
           )
       % {
         @escape = lambda do
-          diagnostic :error, Parser::ERRORS[:invalid_unicode_escape],
+          diagnostic :error, :invalid_unicode_escape, nil,
                      range(@escape_s - 1, p)
         end
       }
@@ -713,7 +713,7 @@ class Parser::Lexer
         | xdigit{7,}
         ) % {
           @escape = lambda do
-            diagnostic :fatal, Parser::ERRORS[:unterminated_unicode],
+            diagnostic :fatal, :unterminated_unicode, nil,
                        range(p - 1, p)
           end
         }
@@ -741,8 +741,7 @@ class Parser::Lexer
     | ( c_any - [0-7xuCMc] ) %unescape_char
 
     | c_eof % {
-      diagnostic :fatal, Parser::ERRORS[:escape_eof],
-                 range(p - 1, p)
+      diagnostic :fatal, :escape_eof, nil, range(p - 1, p)
     }
   );
 
@@ -856,7 +855,7 @@ class Parser::Lexer
   # has to handle such case specially.
   action extend_string_eol {
     if @te == pe
-      diagnostic :fatal, Parser::ERRORS[:string_eof],
+      diagnostic :fatal, :string_eof, nil,
                  range(literal.str_s, literal.str_s + 1)
     end
 
@@ -1031,8 +1030,8 @@ class Parser::Lexer
       => {
         unknown_options = tok.scan(/[^imxouesn]/)
         if unknown_options.any?
-          message = Parser::ERRORS[:regexp_options] % { :options => unknown_options.join }
-          diagnostic :error, message
+          diagnostic :error, :regexp_options,
+                     { :options => unknown_options.join }
         end
 
         emit(:tREGEXP_OPT)
@@ -1183,8 +1182,7 @@ class Parser::Lexer
       class_var_v
       => {
         if tok =~ /^@@[0-9]/
-          message = Parser::ERRORS[:cvar_name] % { :name => tok }
-          diagnostic :error, message
+          diagnostic :error, :cvar_name, { :name => tok }
         end
 
         emit(:tCVAR)
@@ -1194,8 +1192,7 @@ class Parser::Lexer
       instance_var_v
       => {
         if tok =~ /^@[0-9]/
-          message = Parser::ERRORS[:ivar_name] % { :name => tok }
-          diagnostic :error, message
+          diagnostic :error, :ivar_name, { :name => tok }
         end
 
         emit(:tIVAR)
@@ -1369,8 +1366,7 @@ class Parser::Lexer
       # Ambiguous regexp literal.
       w_space+ '/'
       => {
-        diagnostic :warning, Parser::ERRORS[:ambiguous_literal],
-                   range(@te - 1, @te)
+        diagnostic :warning, :ambiguous_literal, nil, range(@te - 1, @te)
 
         fhold; fgoto expr_beg;
       };
@@ -1379,8 +1375,7 @@ class Parser::Lexer
       # Ambiguous splat, kwsplat or block-pass.
       w_space+ %{ tm = p } ( '+' | '-' | '*' | '&' | '**' )
       => {
-        message = Parser::ERRORS[:ambiguous_prefix] % { :prefix => tok(tm, @te) }
-        diagnostic :warning, message,
+        diagnostic :warning, :ambiguous_prefix, { :prefix => tok(tm, @te) },
                    range(tm, @te)
 
         p = tm - 1
@@ -1581,8 +1576,7 @@ class Parser::Lexer
 
       '%' c_eof
       => {
-        diagnostic :fatal, Parser::ERRORS[:string_eof],
-                   range(@ts, @ts + 1)
+        diagnostic :fatal, :string_eof, nil, range(@ts, @ts + 1)
       };
 
       # Heredoc start.
@@ -1660,8 +1654,7 @@ class Parser::Lexer
       => {
         escape = { " "  => '\s', "\r" => '\r', "\n" => '\n', "\t" => '\t',
                    "\v" => '\v', "\f" => '\f' }[tok[1]]
-        message = Parser::ERRORS[:invalid_escape_use] % { :escape => escape }
-        diagnostic :warning, message, range
+        diagnostic :warning, :invalid_escape_use, { :escape => escape }, range
 
         p = @ts - 1
         fgoto expr_end;
@@ -1669,8 +1662,7 @@ class Parser::Lexer
 
       '?' c_eof
       => {
-        diagnostic :fatal, Parser::ERRORS[:incomplete_escape],
-                   range(@ts, @ts + 1)
+        diagnostic :fatal, :incomplete_escape, nil, range(@ts, @ts + 1)
       };
 
       # f ?aa : b: Disambiguate with a character literal.
@@ -1914,16 +1906,16 @@ class Parser::Lexer
         digits = tok(@num_digits_s, @num_suffix_s)
 
         if digits.end_with? '_'
-          diagnostic :error, Parser::ERRORS[:trailing_in_number] % { :character => '_' },
+          diagnostic :error, :trailing_in_number, { :character => '_' },
                      range(@te - 1, @te)
         elsif digits.empty? && @num_base == 8 && version?(18)
           # 1.8 did not raise an error on 0o.
           digits = "0"
         elsif digits.empty?
-          diagnostic :error, Parser::ERRORS[:empty_numeric]
+          diagnostic :error, :empty_numeric
         elsif @num_base == 8 && (invalid_idx = digits.index(/[89]/))
           invalid_s = @num_digits_s + invalid_idx
-          diagnostic :error, Parser::ERRORS[:invalid_octal],
+          diagnostic :error, :invalid_octal, nil,
                      range(invalid_s, invalid_s + 1)
         end
 
@@ -1938,14 +1930,14 @@ class Parser::Lexer
 
       flo_frac flo_pow?
       => {
-        diagnostic :error, Parser::ERRORS[:no_dot_digit_literal]
+        diagnostic :error, :no_dot_digit_literal
       };
 
       flo_int [eE]
       => {
         if version?(18, 19, 20)
           diagnostic :error,
-                     Parser::ERRORS[:trailing_in_number] % { :character => tok(@te - 1, @te) },
+                     :trailing_in_number, { :character => tok(@te - 1, @te) },
                      range(@te - 1, @te)
         else
           emit(:tINTEGER, tok(@ts, @te - 1).to_i)
@@ -1957,7 +1949,7 @@ class Parser::Lexer
       => {
         if version?(18, 19, 20)
           diagnostic :error,
-                     Parser::ERRORS[:trailing_in_number] % { :character => tok(@te - 1, @te) },
+                     :trailing_in_number, { :character => tok(@te - 1, @te) },
                      range(@te - 1, @te)
         else
           emit(:tFLOAT, tok(@ts, @te - 1).to_f)
@@ -2078,15 +2070,13 @@ class Parser::Lexer
            fnext expr_value; fbreak; };
 
       '\\' c_line {
-        diagnostic :error, Parser::ERRORS[:bare_backslash],
-                   range(@ts, @ts + 1)
+        diagnostic :error, :bare_backslash, nil, range(@ts, @ts + 1)
         fhold;
       };
 
       c_any
       => {
-        message = Parser::ERRORS[:unexpected] % { :character => tok.inspect[1..-2] }
-        diagnostic :fatal, message
+        diagnostic :fatal, :unexpected, { :character => tok.inspect[1..-2] }
       };
 
       c_eof => do_eof;
@@ -2120,7 +2110,7 @@ class Parser::Lexer
 
       c_line* zlen
       => {
-        diagnostic :fatal, Parser::ERRORS[:embedded_document],
+        diagnostic :fatal, :embedded_document, nil,
                    range(@eq_begin_s, @eq_begin_s + '=begin'.length)
       };
   *|;