parser 2.0.0.pre2 → 2.0.0.pre3

data/lib/parser/lexer/explanation.rb

@@ -35,7 +35,7 @@ module Parser
     def decorate(range, token, info)
       from, to = range.begin.column, range.end.column
 
-      line = range.source_line
+      line = range.source_line + '   '
       line[from...to] = "\e[4m#{line[from...to]}\e[0m"
 
       tail_len   = to - from - 1

data/lib/parser/lexer/literal.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 module Parser
 
   class Lexer::Literal
@@ -36,6 +38,11 @@ module Parser
       @lexer       = lexer
       @nesting     = 1
 
+      # DELIMITERS and TYPES are hashes with keys encoded in binary.
+      # Coerce incoming data to the same encoding.
+      str_type  = coerce_encoding(str_type)
+      delimiter = coerce_encoding(delimiter)
+
       unless TYPES.include?(str_type)
         message = ERRORS[:unexpected_percent_str] % { :type => str_type }
         lexer.send(:diagnostic, :error, message, @lexer.send(:range, str_s, str_s + 2))
@@ -46,11 +53,6 @@ module Parser
       # Start of the string type specifier.
       @str_s       = str_s
 
-      # Data buffer.
-      @buffer      = ''
-      # Start of the current chunk in data buffer.
-      @buffer_s    = nil
-
       @start_tok, @interpolate = TYPES[str_type]
       @start_delim = DELIMITERS.include?(delimiter) ? delimiter : nil
       @end_delim   = DELIMITERS.fetch(delimiter, delimiter)
@@ -73,6 +75,8 @@ module Parser
         @str_type << delimiter
       end
 
+      clear_buffer
+
       emit_start_tok unless @monolithic
     end
 
@@ -98,6 +102,8 @@ module Parser
     end
 
     def munge_escape?(character)
+      character = coerce_encoding(character)
+
       if words? && character =~ /[ \t\v\r\f\n]/
         true
       else
@@ -105,15 +111,9 @@ module Parser
       end
     end
 
-    def delimiter?(delimiter)
-      if @indent
-        @end_delim == delimiter.lstrip
-      else
-        @end_delim == delimiter
-      end
-    end
-
     def nest_and_try_closing(delimiter, ts, te)
+      delimiter = coerce_encoding(delimiter)
+
       if @start_delim && @start_delim == delimiter
         @nesting += 1
       elsif delimiter?(delimiter)
@@ -157,7 +157,11 @@ module Parser
 
       @buffer_e = te
 
-      @buffer << string
+      if defined?(Encoding)
+        @buffer << string.encode(@lexer.encoding)
+      else
+        @buffer << string
+      end
     end
 
     def flush_string
@@ -169,9 +173,7 @@ module Parser
       unless @buffer.empty?
         emit(:tSTRING_CONTENT, @buffer, @buffer_s, @buffer_e)
 
-        @buffer   = ''
-        @buffer_s = nil
-        @buffer_e = nil
+        clear_buffer
         extend_content
       end
     end
@@ -192,6 +194,36 @@ module Parser
 
     protected
 
+    def delimiter?(delimiter)
+      if @indent
+        @end_delim == delimiter.lstrip
+      else
+        @end_delim == delimiter
+      end
+    end
+
+    def coerce_encoding(string)
+      if defined?(Encoding)
+        string.encode(Encoding::UTF_8,
+                      :invalid => :replace, :undef => :replace)
+      else
+        string
+      end
+    end
+
+    def clear_buffer
+      @buffer = ''
+
+      # Prime the buffer with lexer encoding; otherwise,
+      # concatenation will produce varying results.
+      if defined?(Encoding)
+        @buffer.force_encoding(@lexer.encoding)
+      end
+
+      @buffer_s = nil
+      @buffer_e = nil
+    end
+
     def emit_start_tok
       str_e = @heredoc_e || @str_s + @str_type.length
       emit(@start_tok, @str_type, @str_s, str_e)

data/lib/parser/rewriter.rb

@@ -40,6 +40,8 @@ module Parser
   # Keep in mind that {Parser::Rewriter} does not take care of indentation when
   # inserting/replacing code so you'll have to do this yourself.
   #
+  # @api public
+  #
   class Rewriter < Parser::AST::Processor
     ##
     # Rewrites the AST/source buffer and returns a String containing the new

data/lib/parser/ruby18.y

@@ -418,19 +418,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -461,19 +453,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -1340,7 +1324,7 @@ rule
                     }
                 | tPIPE block_var tPIPE
                     {
-                      result = @builder.args(val[0], val[1], val[2])
+                      result = @builder.args(val[0], val[1], val[2], false)
                     }
 
         do_block: kDO_BLOCK

data/lib/parser/ruby19.y

@@ -473,19 +473,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -520,19 +512,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -1425,9 +1409,14 @@ rule
                     }
                 | f_bad_arg
 
-          lambda: f_larglist lambda_body
+          lambda:   {
+                      @static_env.extend_dynamic
+                    }
+                  f_larglist lambda_body
                     {
-                      result = [ val[0], val[1] ]
+                      result = [ val[1], val[2] ]
+
+                      @static_env.unextend
                     }
 
      f_larglist: tLPAREN2 f_args opt_bv_decl rparen

data/lib/parser/ruby20.y

@@ -481,19 +481,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -528,19 +520,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -668,21 +652,13 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT tOP_ASGN arg
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2], [ val[3] ])
-                      end
-
-                      const  = @builder.assignable(
+                      const  = @builder.const_op_assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                       result = @builder.op_assign(const, val[3], val[4])
                     }
                 | tCOLON3 tCONSTANT tOP_ASGN arg
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1], [ val[2] ])
-                      end
-
-                      const  = @builder.assignable(
+                      const  = @builder.const_op_assignable(
                                   @builder.const_global(val[0], val[1]))
                       result = @builder.op_assign(const, val[2], val[3])
                     }
@@ -1486,9 +1462,14 @@ opt_block_args_tail:
                     }
                 | f_bad_arg
 
-          lambda: f_larglist lambda_body
+          lambda:   {
+                      @static_env.extend_dynamic
+                    }
+                  f_larglist lambda_body
                     {
-                      result = [ val[0], val[1] ]
+                      result = [ val[1], val[2] ]
+
+                      @static_env.unextend
                     }
 
      f_larglist: tLPAREN2 f_args opt_bv_decl tRPAREN

data/lib/parser/ruby21.y

@@ -116,11 +116,7 @@ rule
    stmt_or_begin: stmt
                 | klBEGIN tLCURLY top_compstmt tRCURLY
                     {
-                      if in_def?
-                        diagnostic(:error, :begin_in_method, val[0])
-                      end
-
-                      result = @builder.preexe(val[0], val[1], val[2], val[3])
+                      diagnostic(:error, :begin_in_method, val[0])
                     }
 
             stmt: kALIAS fitem
@@ -476,19 +472,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -523,19 +511,11 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                     }
                 | tCOLON3 tCONSTANT
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1])
-                      end
-
                       result = @builder.assignable(
                                   @builder.const_global(val[0], val[1]))
                     }
@@ -663,21 +643,13 @@ rule
                     }
                 | primary_value tCOLON2 tCONSTANT tOP_ASGN arg
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[2], [ val[3] ])
-                      end
-
-                      const  = @builder.assignable(
+                      const  = @builder.const_op_assignable(
                                   @builder.const_fetch(val[0], val[1], val[2]))
                       result = @builder.op_assign(const, val[3], val[4])
                     }
                 | tCOLON3 tCONSTANT tOP_ASGN arg
                     {
-                      if in_def?
-                        diagnostic(:error, :dynamic_const, val[1], [ val[2] ])
-                      end
-
-                      const  = @builder.assignable(
+                      const  = @builder.const_op_assignable(
                                   @builder.const_global(val[0], val[1]))
                       result = @builder.op_assign(const, val[2], val[3])
                     }
@@ -1487,9 +1459,14 @@ opt_block_args_tail:
                     }
                 | f_bad_arg
 
-          lambda: f_larglist lambda_body
+          lambda:   {
+                      @static_env.extend_dynamic
+                    }
+                  f_larglist lambda_body
                     {
-                      result = [ val[0], val[1] ]
+                      result = [ val[1], val[2] ]
+
+                      @static_env.unextend
                     }
 
      f_larglist: tLPAREN2 f_args opt_bv_decl tRPAREN
@@ -2336,5 +2313,5 @@ warn "warning: Ruby 2.1 is not released yet and parser support may be incomplete
   end
 
   def default_encoding
-    Encoding::BINARY
+    Encoding::UTF_8
   end

data/lib/parser/runner.rb

@@ -155,7 +155,12 @@ module Parser
         source.force_encoding(@parser.default_encoding)
 
         buffer = Parser::Source::Buffer.new(filename)
-        buffer.source = source
+
+        if @parser.class.name == 'Parser::Ruby18'
+          buffer.raw_source = source
+        else
+          buffer.source     = source
+        end
 
         process_buffer(buffer)
       end

data/lib/parser/source/buffer.rb

@@ -3,20 +3,32 @@
 module Parser
   module Source
 
+    ##
+    # @api public
+    #
     class Buffer
       attr_reader :name, :first_line
 
+      ENCODING_RE =
+        /\#.*coding\s*[:=]\s*
+          (
+            # Special-case: there's a UTF8-MAC encoding.
+            (utf8-mac)
+          |
+            # Chew the suffix; it's there for emacs compat.
+            ([A-Za-z0-9_-]+?)(-unix|-dos|-mac)
+          |
+            ([A-Za-z0-9_-]+)
+          )
+        /x
+
       def self.recognize_encoding(string)
         return if string.empty?
 
         # extract the first two lines in an efficient way
-        string =~ /(.*)\n?(.*\n)?/
+        string =~ /\A(.*)\n?(.*\n)?/
         first_line, second_line = $1, $2
 
-        [first_line, second_line].each do |line|
-          line.force_encoding(Encoding::ASCII_8BIT) if line
-        end
-
         if first_line =~ /\A\xef\xbb\xbf/ # BOM
           return Encoding::UTF_8
         elsif first_line[0, 2] == '#!'
@@ -25,8 +37,8 @@ module Parser
           encoding_line = first_line
         end
 
-        if encoding_line =~ /^#.*coding\s*[:=]\s*([A-Za-z0-9_-]+)/
-          Encoding.find($1)
+        if (result = ENCODING_RE.match(encoding_line))
+          Encoding.find(result[2] || result[3] || result[5])
         else
           nil
         end
@@ -37,15 +49,16 @@ module Parser
       # string.
       #
       def self.reencode_string(string)
-        encoding = recognize_encoding(string)
+        original_encoding = string.encoding
+        detected_encoding = recognize_encoding(string.force_encoding(Encoding::BINARY))
 
-        if encoding.nil?
+        if detected_encoding.nil?
+          string.force_encoding(original_encoding)
+        elsif detected_encoding == Encoding::BINARY
           string
-        elsif encoding == Encoding::BINARY
-          string.force_encoding(Encoding::BINARY)
         else
           string.
-            force_encoding(encoding).
+            force_encoding(detected_encoding).
             encode(Encoding::UTF_8)
         end
       end
@@ -76,16 +89,20 @@ module Parser
       end
 
       def source=(source)
-        if @source
-          raise ArgumentError, 'Source::Buffer is immutable'
-        end
-
-        if source.respond_to? :encoding
+        if defined?(Encoding)
           source = source.dup if source.frozen?
           source = self.class.reencode_string(source)
         end
 
-        @source = source.freeze
+        self.raw_source = source
+      end
+
+      def raw_source=(source)
+        if @source
+          raise ArgumentError, 'Source::Buffer is immutable'
+        end
+
+        @source = source.gsub(/\r\n/, "\n").freeze
       end
 
       def decompose_position(position)
@@ -94,12 +111,18 @@ module Parser
         [ @first_line + line_no, position - line_begin ]
       end
 
-      def source_line(line)
+      def source_line(lineno)
         unless @lines
-          @lines = @source.lines.map(&:chomp)
+          @lines = @source.lines.to_a
+          @lines.each { |line| line.gsub!(/\n$/, '') }
+
+          # Lexer has an "infinite stream of EOF symbols" after the
+          # actual EOF, so in some cases (e.g. EOF token of ruby-parse -E)
+          # tokens will refer to one line past EOF.
+          @lines << ""
         end
 
-        @lines[line - @first_line].dup
+        @lines[lineno - @first_line].dup
       end
 
       private