parser 0.9.alpha1 → 0.9.0

data/lib/parser/source/buffer.rb

@@ -0,0 +1,78 @@
+module Parser
+  module Source
+
+    class Buffer
+      attr_reader :name, :first_line
+
+      def initialize(name, first_line = 1)
+        @name       = name
+        @first_line = first_line
+        @source     = nil
+      end
+
+      def read
+        self.source = File.read(@name)
+
+        self
+      end
+
+      def source
+        if @source.nil?
+          raise RuntimeError, 'Cannot extract source from uninitialized Source::Buffer'
+        end
+
+        @source
+      end
+
+      def source=(source)
+        @source = source.freeze
+
+        freeze
+      end
+
+      def decompose_position(position)
+        line       = line_for(position)
+        line_begin = line_begin_positions[line]
+
+        [ @first_line + line, position - line_begin ]
+      end
+
+      def source_line(line)
+        mapped_line = line - @first_line
+
+        # Consider improving this naïve implementation.
+        source_line = source.lines.drop(mapped_line).first
+
+        # Line endings will be commonly present for all lines
+        # except the last one. It does not make sense to keep them.
+        if source_line.end_with? "\n"
+          source_line.chomp
+        else
+          source_line
+        end
+      end
+
+      private
+
+      def line_begin_positions
+        # TODO: Optimize this.
+        [0] + source.
+          each_char.
+          with_index.
+          select do |char, index|
+            char == "\n"
+          end.map do |char, index|
+            index + 1
+          end
+      end
+
+      def line_for(position)
+        # TODO: Optimize this.
+        line_begin_positions.rindex do |line_beg|
+          line_beg <= position
+        end
+      end
+    end
+
+  end
+end

data/lib/parser/source/map.rb

@@ -0,0 +1,20 @@
+module Parser
+  module Source
+
+    # General idea for Map subclasses: only store what's
+    # absolutely necessary; don't duplicate the info contained in
+    # ASTs; if it can be extracted from source given only the other
+    # stored information, don't store it.
+    #
+    class Map
+      attr_reader :expression
+
+      def initialize(expression)
+        @expression = expression
+
+        freeze
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/operator.rb

@@ -0,0 +1,15 @@
+module Parser
+  module Source
+
+    class Map::Operator < Map
+      attr_reader :operator
+
+      def initialize(operator, expression)
+        @operator = operator
+
+        super(expression)
+      end
+    end
+
+  end
+end

data/lib/parser/source/map/variable_assignment.rb

@@ -0,0 +1,15 @@
+module Parser
+  module Source
+
+    class Map::VariableAssignment < Map::Operator
+      attr_reader :name
+
+      def initialize(name, operator, expression)
+        @name = name
+
+        super(operator, expression)
+      end
+    end
+
+  end
+end

data/lib/parser/source/range.rb

@@ -0,0 +1,66 @@
+module Parser
+  module Source
+
+    class Range
+      attr_reader :source_buffer
+      attr_reader :begin, :end
+
+      def initialize(source_buffer, begin_, end_)
+        @source_buffer = source_buffer
+        @begin, @end = begin_, end_
+
+        freeze
+      end
+
+      def size
+        @end - @begin + 1
+      end
+
+      def line
+        line, _ = @source_buffer.decompose_position(@begin)
+
+        line
+      end
+
+      def begin_column
+        _, column = @source_buffer.decompose_position(@begin)
+
+        column
+      end
+
+      def end_column
+        _, column = @source_buffer.decompose_position(@end)
+
+        column
+      end
+
+      def column_range
+        begin_column..end_column
+      end
+
+      def source_line
+        @source_buffer.source_line(line)
+      end
+
+      def to_s
+        line, column = @source_buffer.decompose_position(@begin)
+        [@source_buffer.name, line, column + 1].join(':')
+      end
+
+      def join(other)
+        if other.source_buffer == @source_buffer
+          Range.new(@source_buffer,
+              [@begin, other.begin].min,
+              [@end, other.end].max)
+        else
+          raise ArgumentError, "Cannot join SourceRanges for different SourceFiles"
+        end
+      end
+
+      def inspect
+        "#<Source::Range #{@source_buffer.name} #{@begin}..#{@end}>"
+      end
+    end
+
+  end
+end

data/lib/parser/static_environment.rb

@@ -1,22 +1,24 @@
-require 'set'
-
 module Parser
 
   class StaticEnvironment
     def initialize
+      reset
+    end
+
+    def reset
       @variables = Set[]
       @stack     = []
     end
 
     def extend_static
-      @stack.push @variables
+      @stack.push(@variables)
       @variables = Set[]
 
       self
     end
 
     def extend_dynamic
-      @stack.push @variables
+      @stack.push(@variables)
       @variables = @variables.dup
 
       self
@@ -24,14 +26,18 @@ module Parser
 
     def unextend
       @variables = @stack.pop
+
+      self
     end
 
     def declare(name)
-      @variables.add name
+      @variables.add(name.to_sym)
+
+      self
     end
 
     def declared?(name)
-      @variables.include? name
+      @variables.include?(name.to_sym)
     end
   end
 

data/parser.gemspec

@@ -1,25 +1,35 @@
-# coding: utf-8
+# encoding: utf-8
 
 Gem::Specification.new do |spec|
-  spec.name          = "parser"
-  spec.version       = "0.9.alpha1"
-  spec.authors       = ["Peter Zotov"]
-  spec.email         = ["whitequark@whitequark.org"]
-  spec.description   = %q{A Ruby parser.}
+  spec.name          = 'parser'
+  spec.version       = '0.9.0'
+  spec.authors       = ['Peter Zotov']
+  spec.email         = ['whitequark@whitequark.org']
+  spec.description   = %q{A Ruby parser written in pure Ruby.}
   spec.summary       = spec.description
-  spec.homepage      = "http://github.com/whitequark/parser"
-  spec.license       = "MIT"
+  spec.homepage      = 'http://github.com/whitequark/parser'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files`.split($/) + %w(
                           lib/parser/lexer.rb
+                          lib/parser/ruby18.rb
                        )
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.executables   = %w()
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
   spec.required_ruby_version = '>= 1.9'
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake",    "~> 10.0"
-  spec.add_development_dependency "racc"
+  spec.add_dependency             'ast',       '~> 1.0'
+
+  spec.add_development_dependency 'bundler',   '~> 1.2'
+  spec.add_development_dependency 'rake',      '~> 0.9'
+  spec.add_development_dependency 'racc'
+
+  spec.add_development_dependency 'minitest',  '~> 4.7.0'
+  spec.add_development_dependency 'simplecov', '~> 0.7'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'json_pure' # for coveralls on 1.9.2
+
+  spec.add_development_dependency 'simplecov-sublime-ruby-coverage'
 end

data/test/helper.rb

@@ -0,0 +1,45 @@
+require 'tempfile'
+
+require 'simplecov'
+require 'coveralls'
+
+if SimpleCov.usable?
+  if defined?(TracePoint)
+    require_relative 'racc_coverage_helper'
+
+    RaccCoverage.start(%w(ruby18.y ruby19.y),
+                       File.expand_path('../../lib/parser', __FILE__))
+
+    # Report results faster.
+    at_exit { RaccCoverage.stop }
+  end
+
+  require 'simplecov-sublime-ruby-coverage'
+
+  SimpleCov.start do
+    self.formatter = SimpleCov::Formatter::MultiFormatter[
+      SimpleCov::Formatter::HTMLFormatter,
+      SimpleCov::Formatter::SublimeRubyCoverageFormatter,
+      Coveralls::SimpleCov::Formatter
+    ]
+
+    add_group "Grammars" do |source_file|
+      source_file.filename =~ %r{\.y$}
+    end
+
+    # Exclude the testsuite itself.
+    add_filter "/test/"
+
+    # Exclude generated files.
+    add_filter do |source_file|
+      source_file.filename =~ %r{/lib/parser/(lexer|ruby\d+)\.rb$}
+    end
+  end
+end
+
+# minitest/autorun must go after SimpleCov to preserve
+# correct order of at_exit hooks.
+require 'minitest/autorun'
+
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+require 'parser'

data/test/parse_helper.rb

@@ -0,0 +1,204 @@
+require 'parser/all'
+
+module ParseHelper
+  include AST::Sexp
+
+  ALL_VERSIONS = %w(1.8 1.9)
+
+  def setup
+    @diagnostics = []
+
+    super if defined?(super)
+  end
+
+  def parser_for_ruby_version(version)
+    case version
+    when '1.8'; parser = Parser::Ruby18.new
+    when '1.9'; parser = Parser::Ruby19.new
+    # when '2.0'; parser = Parser::Ruby20.new # not yet
+    else raise "Unrecognized Ruby version #{version}"
+    end
+
+    parser.diagnostics.consumer = lambda do |diagnostic|
+      @diagnostics << diagnostic
+    end
+
+    parser
+  end
+
+  def with_versions(code, versions)
+    versions.each do |version|
+      @diagnostics.clear
+
+      parser = parser_for_ruby_version(version)
+      yield version, parser
+    end
+  end
+
+  def assert_source_range(begin_pos, end_pos, range, version, what)
+    assert range.is_a?(Parser::Source::Range),
+           "(#{version}) is_a?(Source::Range) for #{what}"
+
+    assert_equal begin_pos, range.begin,
+                 "(#{version}) begin of #{what}"
+
+    assert_equal end_pos, range.end,
+                 "(#{version}) end of #{what}"
+  end
+
+  # Use like this:
+  # ```
+  # assert_parses(
+  #   s(:send, s(:lit, 10), :+, s(:lit, 20))
+  #   %q{10 + 20},
+  #   %q{~~~~~~~ expression
+  #     |   ^ operator
+  #     |     ~~ expression (lit)
+  #     },
+  #     %w(1.8 1.9) # optional
+  # )
+  # ```
+  def assert_parses(ast, code, source_maps='', versions=ALL_VERSIONS)
+    with_versions(code, versions) do |version, parser|
+      source_file = Parser::Source::Buffer.new('(assert_parses)')
+      source_file.source = code
+
+      parsed_ast = parser.parse(source_file)
+
+      assert_equal ast, parsed_ast,
+                   "(#{version}) AST equality"
+
+      parse_source_map_descriptions(source_maps) \
+          do |begin_pos, end_pos, map_field, ast_path, line|
+
+        astlet = traverse_ast(parsed_ast, ast_path)
+
+        if astlet.nil?
+          # This is a testsuite bug.
+          raise "No entity with AST path #{ast_path} in #{parsed_ast.inspect}"
+        end
+
+        next # TODO skip location checking
+
+        assert astlet.source_map.respond_to?(map_field),
+               "(#{version}) source_map.respond_to?(#{map_field.inspect}) for:\n#{parsed_ast.inspect}"
+
+        range = astlet.source_map.send(map_field)
+
+        assert_source_range(begin_pos, end_pos, range, version, line.inspect)
+      end
+    end
+  end
+
+  # Use like this:
+  # ```
+  # assert_diagnoses(
+  #   [:warning, :ambiguous_prefix, { prefix: '*' }],
+  #   %q{foo *bar},
+  #   %q{    ^ location
+  #     |     ~~~ highlights (0)})
+  # ```
+  def assert_diagnoses(diagnostic, code, source_maps='', versions=ALL_VERSIONS)
+    with_versions(code, versions) do |version, parser|
+      source_file = Parser::Source::Buffer.new('(assert_diagnoses)')
+      source_file.source = code
+
+      begin
+        parser = parser.parse(source_file)
+      rescue Parser::SyntaxError
+        # do nothing; the diagnostic was reported
+      end
+
+      # Remove this `if' when no diagnostics fail to render.
+      if @diagnostics.count != 1
+        assert_equal 1, @diagnostics.count,
+                     "(#{version}) emits a single diagnostic, not\n" \
+                     "#{@diagnostics.map(&:render).join("\n")}"
+      end
+
+      emitted_diagnostic = @diagnostics.first
+
+      level, kind, substitutions = diagnostic
+      message = Parser::ERRORS[kind] % substitutions
+
+      assert_equal level, emitted_diagnostic.level
+      assert_equal message, emitted_diagnostic.message
+
+      parse_source_map_descriptions(source_maps) \
+          do |begin_pos, end_pos, map_field, ast_path, line|
+
+        next # TODO skip location checking
+
+        case map_field
+        when 'location'
+          assert_source_range begin_pos, end_pos,
+                              emitted_diagnostic.location,
+                              version, "location"
+
+        when 'highlights'
+          index = ast_path.first.to_i
+
+          assert_source_range begin_pos, end_pos,
+                              emitted_diagnostic.highlights[index],
+                              version, "#{index}th highlight"
+
+        else
+          raise "Unknown diagnostic range #{map_field}"
+        end
+      end
+    end
+  end
+
+  SOURCE_MAP_DESCRIPTION_RE =
+      /(?x)
+       ^(?# $1 skip)            ^(\s*)
+        (?# $2 highlight)        ([~\^]+)
+                                 \s+
+        (?# $3 source_map_field) ([a-z]+)
+        (?# $5 ast_path)         (\s+\(([a-z_.\/0-9]+)\))?
+                                $/
+
+  def parse_source_map_descriptions(descriptions)
+    unless block_given?
+      return to_enum(:parse_source_map_descriptions, descriptions)
+    end
+
+    descriptions.each_line do |line|
+      # Remove leading "     |", if it exists.
+      line = line.sub(/^\s*\|/, '').rstrip
+
+      next if line.empty?
+
+      if (match = SOURCE_MAP_DESCRIPTION_RE.match(line))
+        begin_pos        = match[1].length
+        end_pos          = begin_pos + match[2].length - 1
+        source_map_field = match[3]
+
+        if match[5]
+          ast_path = match[5].split('.')
+        else
+          ast_path = []
+        end
+
+        yield begin_pos, end_pos, source_map_field, ast_path, line
+      else
+        raise "Cannot parse source map description line: #{line.inspect}."
+      end
+    end
+  end
+
+  def traverse_ast(ast, path)
+    path.inject(ast) do |astlet, path_component|
+      # Split "dstr/2" to :dstr and 1
+      type_str, index_str = path_component.split('/')
+      type, index = type_str.to_sym, index_str.to_i - 1
+
+      matching_children = \
+        astlet.children.select do |child|
+          AST::Node === child && child.type == type
+        end
+
+      matching_children[index]
+    end
+  end
+end