parser 1.4.2 → 2.0.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d62cbf813cac258da20e47d807e808d4a5d86d98
-  data.tar.gz: e708512bed38b4f38419882d82847f2272c11604
+  metadata.gz: 461f75016915d79b41bf1d98c2ffd979b85b92a8
+  data.tar.gz: e4e95ba6d0d7a54da4d29f10d103f9a9541a1102
 SHA512:
-  metadata.gz: 250e93427ef7b9907fe8157f1ec494bb8a2e0d589760c137ffb553a81a1056ed079237d3cbe7d9ded2e15ff95a4e27d4bc6e74f19fede3050def8b06b37a9da3
-  data.tar.gz: 05778b96f9bbe5d31ec2e5a071e09b4a9e6336ea06f447b80eb04dd8ed1ae53d45638f0653f3e594abdb35a9bc22d07e68de770e94c0ab190ea5af25022343ef
+  metadata.gz: af23698e24fc19b7c12cb521d21657be6a1aa5b7771562b3643e8be35082765fe02e3e81a8801f8667f28adb0114f3353b512ac9121eb673b68836c05286bfde
+  data.tar.gz: 82bc2d8abbf80d1c46860ae02ebea4dfe284799b88e3a9163b1121993158952eeff5a7c969b27d9d3e2281441eefd056c60c4f55eb66dd199d0324c73dbf6b8c

data/.rubocop.yml

@@ -0,0 +1,10 @@
+AllCops:
+  Excludes:
+    - lib/parser/lexer.rb
+    - lib/parser/ruby18.rb
+    - lib/parser/ruby19.rb
+    - lib/parser/ruby20.rb
+    - lib/parser/ruby21.rb
+
+HashSyntax:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,167 @@
+Changelog
+=========
+
+2.0.0.beta1 (2013-05-25)
+------------------------
+
+API modifications:
+ * Completely rewrite whitespace handling in lexer (fixes #36). (Peter Zotov)
+ * Rename Parser::AST::Node#source_map to #location, #src to #loc (closes #40). (Peter Zotov)
+ * Rename Parser::Source::Range#to_source to #source (refs #40). (Peter Zotov)
+ * Rename (cdecl) node to (casgn), remove (cvdecl) nodes (fixes #26). (Peter Zotov)
+
+Features implemented:
+ * Add Source::Comment.associate for mapping comments back to nodes (fixes #31). (Peter Zotov)
+ * Return AST and comments from Parser::Base#parse_with_comments. (Peter Zotov)
+ * Return comments from Parser::Base#tokenize (fixes #46). (Peter Zotov)
+ * Add tokenizer, Parser::Base#tokenize (refs #46). (Peter Zotov)
+ * lexer.rl: better location reporting for invalid unicode codepoints (fixes #38). (Peter Zotov)
+ * lexer.rl: better location reporting for unterminated =begin (fixes #37). (Peter Zotov)
+ * Better location reporting for hashes with labels. (Peter Zotov)
+ * Add `dot' source map to (send) nodes (fixes #34). (Peter Zotov)
+ * Significantly improve performance of Source::Buffer (fixes #28). (Peter Zotov)
+
+Bugs fixed:
+ * lexer.rl: fix lexing label at line_begin "foo:bar" (fixes #48). (Peter Zotov)
+ * lexer.rl: "Option /^I/" is a method call (fixes #32). (Peter Zotov)
+ * Don't allow destructive mutation of line cache in Source::Buffer. (Peter Zotov)
+ * Fix quantifier in magic encoding parser (refs #33). (Peter Zotov)
+ * Better handling of magic encoding comment edge cases (fixes #33). (Peter Zotov)
+
+v1.3.2 (2013-05-13)
+-------------------
+
+Features implemented:
+ * lexer.rl: disallow "$-" (dollar, dash, no character) special. (Peter Zotov)
+
+Bugs fixed:
+ * Source::Range: fix #to_source for multiline ranges. (Peter Zotov)
+ * builders/default: source map for class/module name (fixes #24). (Peter Zotov)
+
+v1.3.1 (2013-05-09)
+-------------------
+
+Bugs fixed:
+ * ruby{19,20,21}.y: "def foo\n=begin\n=end\nend" (fixes #22). (Peter Zotov)
+ * lexer.rl: "rescue::Exception" (fixes #23). (Peter Zotov)
+
+v1.3.0 (2013-04-26)
+-------------------
+
+Bugs fixed:
+ * lexer.rl: "alias foo bar \n alias bar baz". (Peter Zotov)
+
+v1.2.0 (2013-04-25)
+-------------------
+
+Bugs fixed:
+ * lexer.rl: lex "def String.foo; end" correctly (fixes #16). (Peter Zotov)
+ * lexer.rl: reject "1end", "1.1end". (Peter Zotov)
+
+v1.1.0 (2013-04-18)
+-------------------
+
+API modifications:
+ * ruby19.y, ruby20.y, ruby21.y: check for Encoding support (fixes #9). (Peter Zotov)
+
+Features implemented:
+ * builders/default: ignore duplicate _ args (>=1.9), _.* args (>1.9) (fixes #5). (Peter Zotov)
+ * builders/default: detect duplicate argument names (refs #5). (Peter Zotov)
+ * lexer.rl: "def foo bar: 1; end" (for ruby 2.1) (fixes #15). (Peter Zotov)
+ * ruby21.y: required keyword arguments. (Peter Zotov)
+
+Bugs fixed:
+ * ruby20.y, ruby21.y: "foo::A += 1" and friends (scoped constant op-asgn). (Peter Zotov)
+
+v1.0.1 (2013-04-18)
+-------------------
+
+Bugs fixed:
+ * builders/default: %Q{#{1}} and friends (fixes #14). (Peter Zotov)
+
+v1.0.0 (2013-04-17)
+-------------------
+
+Features implemented:
+ * ruby20.y: "meth 1 do end.fun(bar) {}" and friends. (Peter Zotov)
+ * ruby20.y: keyword arguments. (Peter Zotov)
+ * ruby20.y: { **kwsplat }. (Peter Zotov)
+
+v0.9.2 (2013-04-16)
+-------------------
+
+Features implemented:
+ * lexer.rl: "-> (a) {}". (Peter Zotov)
+ * builders/default: treat &&/|| lhs/rhs as conditional context. (Peter Zotov)
+ * ruby20.y: "class Foo \< a:b; end". (Peter Zotov)
+ * lexer.rl: "class \<\< a:b". (Peter Zotov)
+ * ruby19.y, ruby20.y: "f { || a:b }". (Peter Zotov)
+ * ruby19.y, ruby20.y: "def foo() a:b end", "def foo\n a:b end". (Peter Zotov)
+ * lexer.rl: %i/%I. (Peter Zotov)
+ * lexer.rl: warn at "foo **bar". (Peter Zotov)
+ * lexer.rl: ** at expr_beg is tDSTAR. (Peter Zotov)
+ * ruby20.y: "f {|;\nvar\n|}". (Peter Zotov)
+ * ruby20.y: "p () {}". (Peter Zotov)
+ * ruby20.y: "p begin 1.times do 1 end end". (Peter Zotov)
+ * ruby20.y: better error message for BEGIN{} in a method body. (Peter Zotov)
+
+Bugs fixed:
+ * lexer.rl, ruby18.y, ruby19.y, ruby20.y: "%W[#{a}#@b foo #{c}]". (Peter Zotov)
+ * lexer.rl: parse "foo=1; foo / bar #/" as method call on 1.8, division on 1.9. (Peter Zotov)
+ * ruby18.y, ruby19.y: BEGIN{} does not introduce a scope. (Peter Zotov)
+ * lexer.rl: improve whitespace handling. (Peter Zotov)
+
+v0.9.0 (2013-04-15)
+-------------------
+
+API modifications:
+ * runtime compatibility with 1.8.7. (Peter Zotov)
+
+Features implemented:
+ * builders/default: check for multiple assignment in conditions (fixes #4). (Peter Zotov)
+ * builders/default: check if actual block and blockarg are passed (fixes #6). (Peter Zotov)
+ * ruby19.y: "foo::A += m foo". (Peter Zotov)
+ * ruby18.y, ruby19.y: "rescue without else is useless" warning. (Peter Zotov)
+ * ruby19.y: 99.16% coverage, 100% sans error recovery. (Peter Zotov)
+ * ruby19.y: mlhs arguments "def foo((a, *, p)) end". (Peter Zotov)
+ * ruby19.y: "fun (1) {}" and friends. (Peter Zotov)
+ * ruby19.y: mlhs post variables "a, *b, c = ...". (Peter Zotov)
+ * builders/default: @@a |= 1; def f; @@a |= 1; end. (Peter Zotov)
+ * ruby18.y: fun (&foo). (Peter Zotov)
+ * ruby18.y: block formal arguments. 99.33% coverage. (Peter Zotov)
+ * ruby18.y: fun(meth 1 do end); fun(1, meth 1 do end). (Peter Zotov)
+ * ruby18.y: "meth 1 do end.fun(bar)" and friends. (Peter Zotov)
+ * ruby18.y: foo () {}; a.foo () {}; a::foo () {}. (Peter Zotov)
+ * ruby18.y: various call argument combinations. (Peter Zotov)
+ * ruby18.y: foo (1, 2); foo (). (Peter Zotov)
+ * ruby18.y: foo (1).to_i. (Peter Zotov)
+ * ruby18.y: fun{}; fun(){}; fun(1){}; fun do end. (Peter Zotov)
+ * ruby18.y: foo.fun bar. (Peter Zotov)
+ * lexer.rl, ruby18.y: add support for cond/cmdarg stack states. (Peter Zotov)
+ * ruby18.y: rescue. (Peter Zotov)
+ * ruby18.y: begin end while|until (tests only). (Peter Zotov)
+ * ruby18.y: case. (Peter Zotov)
+ * ruby18.y: foo[m bar]. (Peter Zotov)
+ * ruby18.y: for..in. (Peter Zotov)
+
+Bugs fixed:
+ * lexer.rl: handle : at expr_beg as a symbol, at expr_end as tCOLON. (Peter Zotov)
+ * lexer.rl: handle "rescue #foo\nbar". (Peter Zotov)
+ * lexer.rl: handle "foo.#bar\nbaz". (Peter Zotov)
+ * lexer.rl: fix location info for symbols. (Peter Zotov)
+ * lexer.rl: handle \<backslash>\<nl> at expr_beg. (Peter Zotov)
+ * lexer.rl: emit tCONSTANT/tIDENTIFIER/tFID in expr_dot. (Peter Zotov)
+ * lexer.rl: correctly disambiguate "x ::Foo" as tIDENT, tCOLON3, ... (Peter Zotov)
+ * lexer.rl: correctly disambiguate ident!= as tIDENTIFIER, tNEQ. (Peter Zotov)
+ * lexer.rl: correctly report the %r%% tREGEXP_BEG value as %r%. (Peter Zotov)
+ * ruby19.y: emit correct error on "nil = 1" and friends. (Peter Zotov)
+ * ruby19.y: 1.9 permits empty symbol literals. (Peter Zotov)
+ * ruby18.y: foo(&bar). (Peter Zotov)
+ * lexer.rl: don't lookahead two tokens on "func %{str} do". (Peter Zotov)
+ * lexer.rl: fix lexing of non-interp heredoc with trailing backslash. (Peter Zotov)
+ * lexer.rl: fix erroneous number and =begin lookahead in expr_beg. (Peter Zotov)
+ * lexer.rl: fix stack corruption. (Peter Zotov)
+ * lexer.rl: /= at expr_beg. (Peter Zotov)
+ * lexer.rl: class\<\<self. (Peter Zotov)
+ * fix lexing comments at expr_beg "{#1\n}". (Peter Zotov)
+

data/CONTRIBUTING.md

@@ -0,0 +1,10 @@
+Contributing to Parser
+----------------------
+
+Parser uses [Semantic Versioning](http://semver.org). Additionally, Parser employs a script to extract information from VCS (git) log and form a Changelog file. Thus, each commit which affects the public API in any way must be marked with one of the following sigils, or characters at the beginning of line:
+
+ * `-` for bugfixes. For example: `- lexer.rl: fixed lexing of "alias $foo $bar".`
+ * `+` for features. For example: `+ Implemented Parser::Rewriter, a module for non-intrusive rewriting of source code.`
+ * `*` for miscellaneous changes. For example: `* Converted measurement units from metric to imperial.`
+
+These categories map nicely to semantic versioning: `-` bugfixes increment patchlevel, `+` features increment minor version, `*` API changes increment major version.

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in parser.gemspec
 gemspec
+
+group :development do
+  gem 'rubocop', :platform => [:ruby_19, :ruby_20]
+end

data/README.md

@@ -1,32 +1,51 @@
 # Parser
 
+[![Gem Version](https://badge.fury.io/rb/parser.png)](http://badge.fury.io/rb/parser)
 [![Build Status](https://travis-ci.org/whitequark/parser.png?branch=master)](https://travis-ci.org/whitequark/parser)
 [![Code Climate](https://codeclimate.com/github/whitequark/parser.png)](https://codeclimate.com/github/whitequark/parser)
 [![Coverage Status](https://coveralls.io/repos/whitequark/parser/badge.png?branch=master)](https://coveralls.io/r/whitequark/parser)
 
-_Parser_ is a production-ready Ruby parser written in pure Ruby. It performs on par or better than Ripper, Melbourne, JRubyParser or ruby_parser.
+_Parser_ is a production-ready Ruby parser written in pure Ruby. It
+performs on par or better than Ripper, Melbourne, JRubyParser or
+ruby_parser.
 
 ## Installation
 
-    $ gem install parser
+~~~
+$ gem install parser
+~~~
 
 ## Usage
 
 Parse a chunk of code:
 
 ~~~ ruby
-require 'parser/ruby20'
+require 'parser/current'
 
-p Parser::Ruby20.parse("2 + 2")
+p Parser::CurrentRuby.parse("2 + 2")
 # (send
 #   (int 2) :+
 #   (int 2))
 ~~~
 
+Access the AST's source map:
+
+~~~ ruby
+p Parser::CurrentRuby.parse("2 + 2").src
+# #<Parser::Source::Map::Send:0x007fe0ca8a69b8
+#   @begin=nil,
+#   @end=nil,
+#   @expression=#<Source::Range (string) 0...5>,
+#   @selector=#<Source::Range (string) 2...3>>
+
+p Parser::CurrentRuby.parse("2 + 2").src.selector.to_source
+# "+"
+~~~
+
 Parse a chunk of code and display all diagnostics:
 
 ~~~ ruby
-parser = Parser::Ruby20.new
+parser = Parser::CurrentRuby.new
 parser.diagnostics.consumer = lambda do |diag|
   puts diag.render
 end
@@ -45,6 +64,36 @@ p parser.parse(buffer)
 
 If you reuse the same parser object for multiple `#parse` runs, you need to `#reset` it.
 
+You can also use the `ruby-parse` utility (it's bundled with the gem) to play with Parser:
+
+~~~
+$ ruby-parse -L -e "2+2"
+(send
+  (int 2) :+
+  (int 2))
+2+2
+ ~ selector
+~~~ expression
+(int 2)
+2+2
+~ expression
+(int 2)
+2+2
+
+$ ruby-parse -E -e "2+2"
+2+2
+^ tINTEGER 2                                    expr_end     [0 <= cond] [0 <= cmdarg]
+2+2
+ ^ tPLUS "+"                                    expr_beg     [0 <= cond] [0 <= cmdarg]
+2+2
+  ^ tINTEGER 2                                  expr_end     [0 <= cond] [0 <= cmdarg]
+2+2
+  ^ false "$eof"                                expr_end     [0 <= cond] [0 <= cmdarg]
+(send
+  (int 2) :+
+  (int 2))
+~~~
+
 ## Features
 
  * Precise source location reporting.
@@ -54,7 +103,7 @@ If you reuse the same parser object for multiple `#parse` runs, you need to `#re
  * Parsing error recovery.
  * Improved [clang-like][] diagnostic messages with location information.
  * Written in pure Ruby, runs on MRI 1.8.7 or >=1.9.2, JRuby and Rubinius in 1.8 and 1.9 mode.
- * Single runtime dependency: the [ast][] gem.
+ * Only two runtime dependencies: the gems [ast][] and [slop][].
  * RubyParser compatibility (WIP, no, not really yet).
  * [Insane][insane-lexer] Ruby lexer rewritten from scratch in Ragel.
  * 100% test coverage for Bison grammars (except error recovery).
@@ -62,6 +111,7 @@ If you reuse the same parser object for multiple `#parse` runs, you need to `#re
 
   [clang-like]: http://clang.llvm.org/diagnostics.html
   [ast]: http://rubygems.org/gems/ast
+  [slop]: http://rubygems.org/gems/slop
   [insane-lexer]: http://whitequark.org/blog/2013/04/01/ruby-hacking-guide-ch-11-finite-state-lexer/
 
 ## Contributors

data/Rakefile

@@ -1,15 +1,17 @@
+# encoding:utf-8
+
 require 'bundler/gem_tasks'
 require 'rake/testtask'
 require 'rake/clean'
 
-task :default => [:generate, :test]
+task :default => [:test]
 
 Rake::TestTask.new do |t|
   t.libs       = %w(test/ lib/)
   t.test_files = FileList["test/**/test_*.rb"]
 end
 
-task :build => :generate_release
+task :build => [:generate_release, :changelog]
 
 GENERATED_FILES = %w(lib/parser/lexer.rb
                      lib/parser/ruby18.rb
@@ -40,11 +42,71 @@ task :clean_env do
   ENV.delete 'RACC_DEBUG'
 end
 
-desc 'Generates YARD documentation'
+desc 'Generate YARD documentation'
 task :yard => :generate do
   sh('yard doc')
 end
 
+desc 'Generate Changelog'
+task :changelog do
+  fs     = "\u{fffd}"
+  format = "%d#{fs}%s#{fs}%an#{fs}%ai"
+
+  # Format: version => { commit-class => changes }
+  changelog = Hash.new do |hash, version|
+    hash[version] = Hash.new do |hash, klass|
+      hash[klass] = []
+    end
+  end
+
+  IO.popen("git log --pretty='#{format}' HEAD", 'r') do |io|
+    current_version = nil
+
+    io.each_line do |line|
+      version, message, author, date = line.
+            match(/^(?: \((.*)\))?#{fs}(.*)#{fs}(.*)#{fs}(.*)$/o).captures
+      date = Date.parse(date)
+
+      current_version = "#{$1} (#{date})" if version =~ /(v\d+.\d+.\d+)/
+      current_version = "#{Parser::VERSION} (#{date})" if version =~ /HEAD/
+
+      next if current_version.nil? || message !~ /^[+*-]/
+
+      changelog[current_version][message[0]] << "#{message[1..-1]} (#{author})"
+    end
+  end
+
+  commit_classes = {
+    '*' => 'API modifications:',
+    '+' => 'Features implemented:',
+    '-' => 'Bugs fixed:',
+  }
+
+  File.open('CHANGELOG.md', 'w') do |io|
+    io.puts 'Changelog'
+    io.puts '========='
+    io.puts
+
+    changelog.each do |version, commits|
+      io.puts version
+      io.puts '-' * version.length
+      io.puts
+
+      commit_classes.each do |sigil, description|
+        next unless commits[sigil].any?
+
+        io.puts description
+        commits[sigil].each do |commit|
+          io.puts " * #{commit.gsub('<', '\<').lstrip}"
+        end
+        io.puts
+      end
+    end
+  end
+
+  sh('git commit CHANGELOG.md -m "Update changelog."')
+end
+
 rule '.rb' => '.rl' do |t|
   sh "ragel -R #{t.source} -o #{t.name}"
 end
@@ -59,3 +121,4 @@ rule '.rb' => '.y' do |t|
   sh "racc", *opts
 end
 
+task :test => [:generate]

data/doc/AST_FORMAT.md

@@ -392,19 +392,6 @@ Format:
 
 ### To class variable
 
-#### Inside a class scope
-
-Format:
-
-~~~
-(cvdecl :@@foo (lvar :bar))
-"@@foo = bar"
-       ^ operator
- ~~~~~~~~~~~ expression
-~~~
-
-#### Inside a method scope
-
 Format:
 
 ~~~
@@ -432,7 +419,7 @@ Format:
 Format:
 
 ~~~
-(cdecl (cbase) :Foo (int 1))
+(casgn (cbase) :Foo (int 1))
 "::Foo = 1"
    ~~~ name
        ~ operator
@@ -444,7 +431,7 @@ Format:
 Format:
 
 ~~~
-(cdecl (lvar :a) :Foo (int 1))
+(casgn (lvar :a) :Foo (int 1))
 "a::Foo = 1"
     ~~~ name
         ~ operator
@@ -456,7 +443,7 @@ Format:
 Format:
 
 ~~~
-(cdecl nil :Foo (int 1))
+(casgn nil :Foo (int 1))
 "Foo = 1"
  ~~~ name
      ~ operator