parser 0.9.alpha → 0.9.alpha1

data/lib/parser/lexer_literal.rb

@@ -0,0 +1,175 @@
+module Parser
+
+  class LexerLiteral
+    DELIMITERS = { '(' => ')', '[' => ']', '{' => '}', '<' => '>' }
+    MONOLITHIC = { :tSTRING_BEG => :tSTRING, :tSYMBEG => :tSYMBOL }
+
+    TYPES = {
+    # type      start token    interpolate?
+      "'"  => [ :tSTRING_BEG,  false ],
+      '%q' => [ :tSTRING_BEG,  false ],
+      '"'  => [ :tSTRING_BEG,  true  ],
+      '%'  => [ :tSTRING_BEG,  true  ],
+      '%Q' => [ :tSTRING_BEG,  true  ],
+
+      '%w' => [ :tQWORDS_BEG,  false ],
+      '%W' => [ :tWORDS_BEG,   true  ],
+
+      ":'" => [ :tSYMBEG,      false ],
+      '%s' => [ :tSYMBEG,      false ],
+      ':"' => [ :tSYMBEG,      true  ],
+
+      '/'  => [ :tREGEXP_BEG,  true  ],
+      '%r' => [ :tREGEXP_BEG,  true  ],
+
+      '%x' => [ :tXSTRING_BEG, true  ],
+      '`'  => [ :tXSTRING_BEG, true  ],
+    }
+
+    attr_reader   :heredoc_e
+    attr_accessor :saved_herebody_s
+
+    def initialize(lexer, str_type, delimiter, str_s, heredoc_e = nil, indent = false)
+      @lexer       = lexer
+      @nesting     = 1
+
+      unless TYPES.include? str_type
+        lexer.send :error, "#{str_type}: unknown type of %string"
+      end
+
+      # String type. For :'foo', it is :'
+      @str_type    = str_type
+      # Start of the string type specifier.
+      @str_s       = str_s
+
+      # Data buffer.
+      @buffer      = ""
+      # Start of the current chunk in data buffer.
+      @buffer_s    = nil
+
+      @start_tok, @interpolate = TYPES[str_type]
+      @start_delim = DELIMITERS.include?(delimiter) ? delimiter : nil
+      @end_delim   = DELIMITERS.fetch(delimiter, delimiter)
+
+      @heredoc_e   = heredoc_e
+      @indent      = indent
+
+      @interp_braces = 0
+
+      # Monolithic strings are glued into a single token, e.g.
+      # tSTRING_BEG tSTRING_CONTENT tSTRING_END -> tSTRING.
+      @monolithic  = (
+          [:tSTRING_BEG, :tSYMBEG].include?(type) &&
+          !heredoc?
+      )
+
+      emit_start_tok unless @monolithic
+    end
+
+    def interpolate?
+      @interpolate
+    end
+
+    def words?
+      type == :tWORDS_BEG || type == :tQWORDS_BEG
+    end
+
+    def regexp?
+      type == :tREGEXP_BEG
+    end
+
+    def heredoc?
+      !!@heredoc_e
+    end
+
+    def type
+      @start_tok
+    end
+
+    def munge_escape?(character)
+      if words? && character =~ /[ \t\v\r\f\n]/
+        true
+      else
+        ['\\', @start_delim, @end_delim].include? character
+      end
+    end
+
+    def delimiter?(delimiter)
+      if @indent
+        @end_delim == delimiter.lstrip
+      else
+        @end_delim == delimiter
+      end
+    end
+
+    def nest_and_try_closing(delimiter, ts, te)
+      if @start_delim && @start_delim == delimiter
+        @nesting += 1
+      elsif delimiter?(delimiter)
+        @nesting -= 1
+      end
+
+      # Finalize if last matching delimiter is closed.
+      if @nesting == 0
+        # Emit the string as a single token if it's applicable.
+        if @monolithic
+          @lexer.emit MONOLITHIC[@start_tok], @buffer, @str_s, te
+        else
+          # If this is a heredoc, @buffer contains the sentinel now.
+          # Just throw it out. Lexer flushes the heredoc after each
+          # non-heredoc-terminating \n anyway, so no data will be lost.
+          flush_string unless heredoc?
+
+          @lexer.emit :tSTRING_END, @end_delim, ts, te
+        end
+      end
+    end
+
+    def start_interp_brace
+      @interp_braces += 1
+    end
+
+    def end_interp_brace_and_try_closing
+      @interp_braces -= 1
+
+      (@interp_braces == 0)
+    end
+
+    def extend_string(string, ts, te)
+      if @buffer_s.nil?
+        @buffer_s = ts
+      end
+
+      @buffer_e = te
+
+      @buffer << string
+    end
+
+    def flush_string
+      if @monolithic
+        emit_start_tok
+        @monolithic = false
+      end
+
+      unless @buffer.empty?
+        @lexer.emit :tSTRING_CONTENT, @buffer, @buffer_s, @buffer_e
+
+        if words?
+          @lexer.emit :tSPACE, nil, @buffer_e, @buffer_e + 1
+        end
+
+        @buffer   = ""
+        @buffer_s = nil
+        @buffer_e = nil
+      end
+    end
+
+    protected
+
+    def emit_start_tok
+      str_e = @heredoc_e || @str_s + @str_type.length
+      @lexer.emit @start_tok, @str_type, @str_s, str_e
+    end
+  end
+
+end

data/lib/parser/static_environment.rb

@@ -0,0 +1,38 @@
+require 'set'
+
+module Parser
+
+  class StaticEnvironment
+    def initialize
+      @variables = Set[]
+      @stack     = []
+    end
+
+    def extend_static
+      @stack.push @variables
+      @variables = Set[]
+
+      self
+    end
+
+    def extend_dynamic
+      @stack.push @variables
+      @variables = @variables.dup
+
+      self
+    end
+
+    def unextend
+      @variables = @stack.pop
+    end
+
+    def declare(name)
+      @variables.add name
+    end
+
+    def declared?(name)
+      @variables.include? name
+    end
+  end
+
+end

data/lib/parser/syntax_error.rb

@@ -0,0 +1,3 @@
+module Parser
+  class SyntaxError < StandardError; end
+end

data/parser.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "parser"
+  spec.version       = "0.9.alpha1"
+  spec.authors       = ["Peter Zotov"]
+  spec.email         = ["whitequark@whitequark.org"]
+  spec.description   = %q{A Ruby parser.}
+  spec.summary       = spec.description
+  spec.homepage      = "http://github.com/whitequark/parser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/) + %w(
+                          lib/parser/lexer.rb
+                       )
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 1.9'
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake",    "~> 10.0"
+  spec.add_development_dependency "racc"
+end

data/test/{test_ruby_lexer.rb → test_lexer.rb}

@@ -1,19 +1,65 @@
-#!/usr/bin/env ruby
 # encoding: ascii-8bit
 
 require 'minitest/autorun'
-require 'ruby_lexer'
-require 'ruby_parser_extras'
+require 'parser/lexer'
 
-class TestRubyLexer < MiniTest::Unit::TestCase
-  alias :deny :refute
+class TestLexer < MiniTest::Unit::TestCase
+  def setup_lexer version
+    @lex = Parser::Lexer.new(version)
+  end
 
   def setup
     setup_lexer 18
   end
 
-  def setup_lexer version
-    @lex = RubyLexer.new(version)
+  def util_bad_token s, *args
+    assert_raises Parser::SyntaxError do
+      util_lex_token s, *args
+    end
+  end
+
+  def util_escape expected, input
+    @lex.reset
+    @lex.source = "%Q[\\#{input}]"
+
+    lex_token, lex_value = @lex.advance
+
+    if lex_value.respond_to?(:force_encoding)
+      lex_value.force_encoding('ASCII-8BIT')
+    end
+
+    assert_equal [:tSTRING, expected],
+                 [lex_token, lex_value],
+                 @lex.source
+  end
+
+  def util_escape_bad input
+    assert_raises Parser::SyntaxError do
+      @lex.state = :expr_beg
+      util_lex_token "%Q[\\#{input}]"
+    end
+  end
+
+  def util_lex_fname name, type, end_state = :expr_end
+    util_lex_token("def #{name} ", :kDEF, "def", type, name)
+
+    assert_equal end_state, @lex.state
+  end
+
+  def util_lex_token input, *args
+    @lex.reset(false)
+    @lex.source = input
+
+    until args.empty? do
+      token, value = args.shift(2)
+
+      lex_token, lex_value = @lex.advance
+      assert lex_token, "no more tokens"
+      assert_equal [token, value], [lex_token, lex_value], input
+    end
+
+    lex_token, lex_value = @lex.advance
+    refute lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}"
   end
 
   def test_advance
@@ -23,7 +69,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     assert token # blah
 
     token, = @lex.advance
-    deny   token # nada
+    refute token # nada
   end
 
   def test_read_escape
@@ -367,9 +413,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   end
 
   def test_yylex_cvar_bad
-    assert_raises RubyParser::SyntaxError do
-      util_lex_token "@@1"
-    end
+    util_bad_token "@@1"
   end
 
   def test_yylex_div
@@ -403,15 +447,16 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :kEND, "end")
   end
 
-  def test_yylex_do_cond
-    @lex.cond.push true
+  # TODO
+  # def test_yylex_do_cond
+  #   @lex.cond.push true
 
-    util_lex_token("x do 42 end",
-                   :tIDENTIFIER, "x",
-                   :kDO_COND, "do",
-                   :tINTEGER, 42,
-                   :kEND, "end")
-  end
+  #   util_lex_token("x do 42 end",
+  #                  :tIDENTIFIER, "x",
+  #                  :kDO_COND, "do",
+  #                  :tINTEGER, 42,
+  #                  :kEND, "end")
+  # end
 
   def test_yylex_dot
     util_lex_token ".", :tDOT, "."
@@ -868,25 +913,25 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   end
 
   def test_yylex_question_eh_a__18
-    @lex = RubyLexer.new 18
+    setup_lexer 18
 
     util_lex_token "?a", :tINTEGER, 97
   end
 
   def test_yylex_question_eh_a__19
-    @lex = RubyLexer.new 19
+    setup_lexer 19
 
     util_lex_token '?a', :tSTRING, "a"
   end
 
   def test_yylex_question_eh_escape_M_escape_C__18
-    @lex = RubyLexer.new 18
+    setup_lexer 18
 
     util_lex_token '?\M-\C-a', :tINTEGER, 129
   end
 
   def test_yylex_question_eh_escape_M_escape_C__19
-    @lex = RubyLexer.new 19
+    setup_lexer 19
 
     util_lex_token '?\M-\C-a', :tSTRING, "\M-\C-a"
   end
@@ -1214,13 +1259,13 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   end
 
   def test_yylex_question__18
-    @lex = RubyLexer.new 18
+    setup_lexer 18
 
     util_lex_token "?*", :tINTEGER, 42
   end
 
   def test_yylex_question__19
-    @lex = RubyLexer.new 19
+    setup_lexer 19
 
     util_lex_token "?*", :tSTRING, "*"
   end
@@ -1239,7 +1284,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   end
 
   def test_yylex_question_ws_backslashed__18
-    @lex = RubyLexer.new 18
+    setup_lexer 18
 
     @lex.state = :expr_beg
     util_lex_token "?\\ ", :tINTEGER, 32
@@ -1256,7 +1301,7 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   end
 
   def test_yylex_question_ws_backslashed__19
-    @lex = RubyLexer.new 19
+    setup_lexer 19
 
     @lex.state = :expr_beg
     util_lex_token "?\\ ", :tSTRING, " "
@@ -1887,8 +1932,9 @@ class TestRubyLexer < MiniTest::Unit::TestCase
 
   def test_yylex_underscore_end
     @lex.source = "__END__\n"
+
     tok, = @lex.advance
-    deny tok
+    refute tok
   end
 
   def test_yylex_uplus
@@ -1916,8 +1962,9 @@ class TestRubyLexer < MiniTest::Unit::TestCase
   end
 
   def test_yylex_static_env
-    env = RubyParserStuff::Environment.new
-    env[:a] = :lvar
+    env = Parser::StaticEnvironment.new
+    env.declare :a
+
     @lex.static_env = env
 
     util_lex_token("a [42]",
@@ -1926,103 +1973,4 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tINTEGER,    42,
                    :tRBRACK,     "]")
   end
-
-  def test_zbug_float_in_decl
-    util_lex_token("def initialize(u = ",
-                   :kDEF, "def",
-                   :tIDENTIFIER, "initialize",
-                   :tLPAREN2, "(",
-                   :tIDENTIFIER, "u",
-                   :tEQL, "=")
-
-    assert_equal :expr_beg, @lex.state
-
-    util_lex_token("0.0, s = 0.0",
-                   :tFLOAT, 0.0,
-                   :tCOMMA, ',',
-                   :tIDENTIFIER, "s",
-                   :tEQL, "=",
-                   :tFLOAT, 0.0)
-  end
-
-  def test_zbug_id_equals
-    util_lex_token("a =",
-                   :tIDENTIFIER, "a",
-                   :tEQL, "=")
-
-    assert_equal :expr_beg, @lex.state
-
-    util_lex_token("0.0",
-                   :tFLOAT, 0.0)
-  end
-
-  def test_zbug_no_spaces_in_decl
-    util_lex_token("def initialize(u=",
-                   :kDEF, "def",
-                   :tIDENTIFIER, "initialize",
-                   :tLPAREN2, "(",
-                   :tIDENTIFIER, "u",
-                   :tEQL, "=")
-
-    assert_equal :expr_beg, @lex.state
-
-    util_lex_token("0.0,s=0.0",
-                   :tFLOAT, 0.0,
-                   :tCOMMA, ",",
-                   :tIDENTIFIER, "s",
-                   :tEQL, "=",
-                   :tFLOAT, 0.0)
-  end
-
-  ############################################################
-
-  def util_bad_token s, *args
-    assert_raises RubyParser::SyntaxError do
-      util_lex_token s, *args
-    end
-  end
-
-  def util_escape expected, input
-    @lex.reset
-    @lex.source = "%Q[\\#{input}]"
-
-    lex_token, lex_value = @lex.advance
-
-    if lex_value.respond_to?(:force_encoding)
-      lex_value.force_encoding('ASCII-8BIT')
-    end
-
-    assert_equal [:tSTRING, expected],
-                 [lex_token, lex_value],
-                 @lex.source
-  end
-
-  def util_escape_bad input
-    assert_raises RubyParser::SyntaxError do
-      @lex.state = :expr_beg
-      util_lex_token "%Q[\\#{input}]"
-    end
-  end
-
-  def util_lex_fname name, type, end_state = :expr_end
-    util_lex_token("def #{name} ", :kDEF, "def", type, name)
-
-    assert_equal end_state, @lex.state
-  end
-
-  def util_lex_token input, *args
-    @lex.reset(false)
-    @lex.source = input
-
-    until args.empty? do
-      token, value = args.shift(2)
-
-      lex_token, lex_value = @lex.advance
-      assert lex_token, "no more tokens"
-      assert_equal [token, value], [lex_token, lex_value], input
-    end
-
-    lex_token, lex_value = @lex.advance
-    deny lex_token, "must be empty, but had #{[lex_token, lex_value].inspect}"
-  end
 end