rb-ruby_parser 2.0.4.1

data/.autotest

@@ -0,0 +1,43 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+require 'autotest/rcov' if ENV['RCOV']
+
+Autotest.add_hook :initialize do |at|
+  at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
+  at.libs << ":../../ParseTree/dev/test:../../sexp_processor/dev/lib"
+  at.add_exception 'unit'
+  at.add_exception 'coverage'
+  at.add_exception 'coverage.info'
+  at.add_exception '.diff'
+
+  at.libs << ':../../minitest/dev/lib'
+  at.testlib = "minitest/autorun"
+  at.unit_diff = "unit_diff -u -b"
+
+  at.add_mapping(/^lib\/.*\.y$/) do |f, _|
+    at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%
+  end
+
+  at.add_mapping(/pt_testcase.rb/) do |f, _|
+    at.files_matching(/test_.*rb$/)
+  end
+
+  %w(TestEnvironment TestStackState).each do |klass|
+    at.extra_class_map[klass] = "test/test_ruby_parser_extras.rb"
+  end
+
+  %w(TestRubyParser TestParseTree).each do |klass| # HACK
+    at.extra_class_map[klass] = "test/test_ruby_parser.rb"
+  end
+end
+
+Autotest.add_hook :run_command do |at|
+  system "rake parser"
+end
+
+class Autotest
+  def ruby
+    File.expand_path "~/.multiruby/install/1.9.0-0/bin/ruby"
+  end
+end if ENV['ONENINE']

data/.gitignore

@@ -0,0 +1,2 @@
+lib/ruby_parser.output
+lib/ruby_parser.rb

data/History.txt

@@ -0,0 +1,176 @@
+=== 2.0.4 / 2009-10-31
+
+* Forked from seattlerb/ruby_parser
+
+* 2 minor enhancement
+  
+  * gemspec, name of gem changed pending gemcutter namespacing
+  * Jeweler config in git
+
+* 1 bug fix:
+
+  * Allow text after =end of block comment
+
+=== 2.0.4 / 2009-08-18
+
+* 1 minor enhancement:
+
+  * Changed requires around to be more accurate.
+
+* 4 bug fixes:
+
+  * Fixed .autotest for minitest
+  * Fixed emacs escape lexing bug: "\C-\\" (maglev/gemstone)
+  * Fixed octal lexing edgecases. (maglev/gemstone)
+  * Fixed regexp lexing edgecases. (maglev/gemstone)
+
+=== 2.0.3 / 2009-06-23
+
+* 4 minor enhancements:
+
+  * Removed dead code handling d/regex match node cruft.
+  * Switched to minitest
+  * Updated .autotest and rakefile wrt rcov for new hoe capabilities
+  * Updated hoe for new capabilities
+
+* 4 bug fixes:
+
+  * Environment#all now deals with strange edge cases in RAD.
+  * Fixed packaging/compilation issue.
+  * Minor 1.9 fixes
+  * hoe -> flay -> rubyparser rakefile circularity fixed
+
+=== 2.0.2 / 2009-01-20
+
+* 2 minor enhancements:
+
+  * Added gauntlet_rubyparser plugin. YAY for easy massive bug-hunting.
+  * Promoted Sexp's file/line/comments to sexp_processor.
+
+* 4 bug fixes:
+
+  * Fixed and improved the readme
+  * Fixed lexing heredoc newlines.
+  * Fixed line numbers on defns.
+  * Fixed rdoc generation bug pointed out by hugh sasse (who rocks)
+
+=== 2.0.1 / 2008-11-04
+
+* 2 minor enhancements:
+
+  * Updated for changes to splat node in many contexts.
+  * Made PT a developer dep
+
+=== 2.0.0 / 2008-10-22
+
+* 1 major enhancement
+
+  * Brought on the AWESOME! 4x faster! no known lexing/parsing bugs!
+
+* 71 minor enhancements
+
+  * 1.9: Added Fixnum#ord.
+  * 1.9: Added missing Regexp constants and did it so it'd work on 1.9.
+  * Added #store_comment and #comments
+  * Added StringScanner #begin_of_line?
+  * Added a bunch of tests for regexp escape chars, #parse_string, #read_escape, ? numbers, ? whitespace.
+  * Added a hack for rubinius' r2l eval bug.
+  * Added a new token type tSTRING that bypasses tSTRING_BEG/END entirely. Only does non-interpolated strings and then falls back to the old way. MUCH cleaner tho.
+  * Added bin/ruby_parse
+  * Added compare rule to Rakefile.
+  * Added coverage files/dirs to clean rule.
+  * Added file and line numbers to all sexp nodes. Column/ranges to come.
+  * Added lex_state change for lvars at the end of yylex.
+  * Added lexed comments to defn/defs/class/module nodes.
+  * Added stats gathering for yylex. Reordered yylex for avg data
+  * Added tSYMBOL token type and parser rule to speed up symbol lexing.
+  * Added tally output for getch, unread, and unread_many.
+  * Added tests for ambigous uminus/uplus, backtick in cmdarg, square and curly brackets, numeric gvars, eos edge cases, string quoting %<> and %%%.
+  * All cases throughout yylex now return directly if they match, no passthroughs.
+  * All lexer cases now slurp entire token in one swoop.
+  * All zarrays are now just empty arrays.
+  * Changed s(:block_arg, :blah) to :"&blah" in args sexp.
+  * Cleaned up lexer error handling. Now just raises all over.
+  * Cleaned up read_escape and regx_options
+  * Cleaned up tokadd_string (for some definition of cleaned).
+  * Converted single quoted strings to new tSTRING token type.
+  * Coverage is currently 94.4% on lexer.
+  * Done what I can to clean up heredoc lexing... still sucks.
+  * Flattened resbodies in rescue node. Fixed .autotest file.
+  * Folded lex_keywords back in now that it screams.
+  * Found very last instanceof ILiteralNode in the code. haha!
+  * Got the tests subclassing PTTC and cleaned up a lot. YAY
+  * Handle yield(*ary) properly
+  * MASSIVELY cleaned out =begin/=end comment processor.
+  * Massive overhaul on Keyword class. All hail the mighty Hash!
+  * Massively cleaned up ident= edge cases and fixed a stupid bug from jruby.
+  * Merged @/@@ scanner together, going to try to do the same everywhere.
+  * Refactored fix_arg_lex_state, common across the lexer.
+  * Refactored new_fcall into new_call.
+  * Refactored some code to get better profile numbers.
+  * Refactored some more #fix_arg_lex_state.
+  * Refactored tail of yylex into its own method.
+  * Removed Module#kill
+  * Removed Token, replaced with Sexp.
+  * Removed all parse_number and parse_quote tests.
+  * Removed argspush, argscat. YAY!
+  * Removed as many token_buffer.split(//)'s as possible. 1 to go.
+  * Removed begins from compstmts
+  * Removed buffer arg for tokadd_string.
+  * Removed crufty (?) solo '@' token... wtf was that anyhow?
+  * Removed most jruby/stringio cruft from StringScanner.
+  * Removed one unread_many... 2 to go. They're harder.
+  * Removed store_comment, now done directly.
+  * Removed token_buffer. Now I just use token ivar.
+  * Removed use of s() from lexer. Changed the way line numbers are gathered.
+  * Renamed *qwords to *awords.
+  * Renamed StringScanner to RPStringScanner (a subclass) to fix namespace trashing.
+  * Renamed parse to process and aliased to parse.
+  * Renamed token_buffer to string_buffer since that arcane shit still needs it.
+  * Resolved the rest of the lexing issues I brought up w/ ruby-core.
+  * Revamped tokadd_escape.
+  * Rewrote Keyword and KWtable.
+  * Rewrote RubyLexer using StringScanner.
+  * Rewrote tokadd_escape. 79 lines down to 21.
+  * Split out lib/ruby_parser_extras.rb so lexer is standalone.
+  * Started to clean up the parser and make it as skinny as possible
+  * Stripped out as much code as possible.
+  * Stripped yylex of some dead code.
+  * Switched from StringIO to StringScanner.
+  * Updated rakefile for new hoe.
+  * Uses pure ruby racc if ENV['PURE_RUBY'], otherwise use c.
+  * Wrote a ton of lexer tests. Coverage is as close to 100% as possible.
+  * Wrote args to clean up the big nasty args processing grammar section.
+  * lex_strterm is now a plain array, removed RubyLexer#s(...).
+  * yield and super now flatten args.
+
+* 21+ bug fixes:
+
+  * I'm sure this list is missing a lot:
+  * Fixed 2 bugs both involving attrasgn (and ilk) esp when lhs is an array.
+  * Fixed a bug in the lexer for strings with single digit hex escapes.
+  * Fixed a bug parsing: a (args) { expr }... the space caused a different route to be followed and all hell broke loose.
+  * Fixed a bug with x\n=beginvar not putting begin back.
+  * Fixed attrasgn to have arglists, not arrays.
+  * Fixed bug in defn/defs with block fixing.
+  * Fixed class/module's name slot if colon2/3.
+  * Fixed dstr with empty interpolation body.
+  * Fixed for 1.9 string/char changes.
+  * Fixed lexer BS wrt determining token type of words.
+  * Fixed lexer BS wrt pass through values and lexing words. SO STUPID.
+  * Fixed lexing of floats.
+  * Fixed lexing of identifiers followed by equals. I hope.
+  * Fixed masgn with splat on lhs
+  * Fixed new_super to deal with block_pass correctly.
+  * Fixed parser's treatment of :colon2 and :colon3.
+  * Fixed regexp scanning of escaped numbers, ANY number is valid, not just octs.
+  * Fixed string scanning of escaped octs, allowing 1-3 chars.
+  * Fixed unescape for \n
+  * Fixed: omg this is stupid. '()' was returning bare nil
+  * Fixed: remove_begin now goes to the end, not sure why it didn't before.
+
+=== 1.0.0 / 2007-12-20
+
+* 1 major enhancement
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,13 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/ruby_parse
+lib/gauntlet_rubyparser.rb
+lib/ruby_lexer.rb
+lib/ruby_parser.y
+lib/ruby_parser_extras.rb
+test/test_ruby_lexer.rb
+test/test_ruby_parser.rb
+test/test_ruby_parser_extras.rb

data/README.txt

@@ -0,0 +1,86 @@
+= ruby_parser
+
+* http://parsetree.rubyforge.org/
+
+== DESCRIPTION:
+
+ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
+racc--which does by default use a C extension). RP's output is
+the same as ParseTree's output: s-expressions using ruby's arrays and
+base types.
+
+As an example:
+
+  def conditional1(arg1)
+    if arg1 == 0 then
+      return 1
+    end
+    return 0
+  end
+
+becomes:
+
+  s(:defn, :conditional1,
+   s(:args, :arg1),
+   s(:scope,
+    s(:block,
+     s(:if,
+      s(:call, s(:lvar, :arg1), :==, s(:arglist, s(:lit, 0))),
+      s(:return, s(:lit, 1)),
+      nil),
+     s(:return, s(:lit, 0)))))
+
+== FEATURES/PROBLEMS:
+
+* Pure ruby, no compiles.
+* Includes preceding comment data for defn/defs/class/module nodes!
+* Incredibly simple interface.
+* Output is 100% equivalent to ParseTree.
+  * Can utilize PT's SexpProcessor and UnifiedRuby for language processing.
+* Known Issue: Speed is now pretty good, but can always improve:
+  * RP parses a corpus of 3702 files in 125s (avg 108 Kb/s)
+  * MRI+PT parsed the same in 67.38s (avg 200.89 Kb/s)
+* Known Issue: Code is much better, but still has a long way to go.
+* Known Issue: Totally awesome.
+* Known Issue: line number values can be slightly off. Parsing LR sucks.
+
+== SYNOPSIS:
+
+  RubyParser.new.parse "1+1"
+  # => s(:call, s(:lit, 1), :+, s(:array, s(:lit, 1)))
+
+== REQUIREMENTS:
+
+* ruby. woot.
+* sexp_processor for Sexp and SexpProcessor classes.
+* ParseTree for testing.
+* racc full package for parser development (compiling .y to .rb).
+
+== INSTALL:
+
+* sudo gem install ruby_parser
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007-2008 Ryan Davis
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,144 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.plugin :seattlerb
+
+Hoe.add_include_dirs("../../ParseTree/dev/test",
+                     "../../RubyInline/dev/lib",
+                     "../../sexp_processor/dev/lib")
+
+hoe = Hoe.spec 'ruby_parser' do
+  developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
+
+  self.rubyforge_name = 'parsetree'
+
+  extra_dev_deps << ['ParseTree',      '~> 3.0']
+  extra_deps     << ['sexp_processor', '~> 3.0']
+end
+
+hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
+
+[:default, :multi, :test].each do |t|
+  task t => :parser
+end
+
+path = "pkg/ruby_parser-#{hoe.version}"
+task path => :parser do
+  Dir.chdir path do
+    sh "rake parser"
+  end
+end
+
+desc "build the parser"
+task :parser => ["lib/ruby_parser.rb"]
+
+rule '.rb' => '.y' do |t|
+  # -v = verbose
+  # -t = debugging parser ~4% reduction in speed -- keep for now
+  # -l = no-line-convert
+  sh "racc -v -t -l -o #{t.name} #{t.source}"
+end
+
+task :clean do
+  rm_rf(Dir["**/*~"] +
+        Dir["**/*.diff"] +
+        Dir["coverage.info"] +
+        Dir["coverage"] +
+        Dir["lib/ruby_parser.rb"] +
+        Dir["lib/*.output"])
+end
+
+def next_num(glob)
+  num = Dir[glob].max[/\d+/].to_i + 1
+end
+
+desc "Compares PT to RP and deletes all files that match"
+task :compare do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    system "./cmp.rb -q #{file} && rm #{file}"
+  end
+  system 'find -d unit -type d -empty -exec rmdir {} \;'
+end
+
+desc "Compares PT to RP and stops on first failure"
+task :find_bug do
+  files = Dir["unit/**/*.rb"]
+  puts "Parsing #{files.size} files"
+  files.each do |file|
+    puts file
+    sh "./cmp.rb -q #{file}"
+  end
+end
+
+task :sort do
+  sh 'grepsort "^ +def" lib/ruby_lexer.rb'
+  sh 'grepsort "^ +def (test|util)" test/test_ruby_lexer.rb'
+end
+
+task :loc do
+  loc1  = `wc -l ../1.0.0/lib/ruby_lexer.rb`[/\d+/]
+  flog1 = `flog -s ../1.0.0/lib/ruby_lexer.rb`[/\d+\.\d+/]
+  loc2  = `cat lib/ruby_lexer.rb lib/ruby_parser_extras.rb | wc -l`[/\d+/]
+  flog2 = `flog -s lib/ruby_lexer.rb lib/ruby_parser_extras.rb`[/\d+\.\d+/]
+
+  loc1, loc2, flog1, flog2 = loc1.to_i, loc2.to_i, flog1.to_f, flog2.to_f
+
+  puts "1.0.0: loc = #{loc1} flog = #{flog1}"
+  puts "dev  : loc = #{loc2} flog = #{flog2}"
+  puts "delta: loc = #{loc2-loc1} flog = #{flog2-flog1}"
+end
+
+desc "Validate against all normal files in unit dir"
+task :validate do
+  sh "./cmp.rb unit/*.rb"
+end
+
+def run_and_log cmd, prefix
+  files = ENV['FILES'] || 'unit/*.rb'
+  p, x = prefix, "txt"
+  n = Dir["#{p}.*.#{x}"].map { |s| s[/\d+/].to_i }.max + 1 rescue 1
+  f = "#{p}.#{n}.#{x}"
+
+  sh "#{cmd} #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g #{files} &> #{f}"
+
+  puts File.read(f)
+end
+
+desc "Benchmark against all normal files in unit dir"
+task :benchmark do
+  run_and_log "ruby", "benchmark"
+end
+
+desc "Profile against all normal files in unit dir"
+task :profile do
+  run_and_log "zenprofile", "profile"
+end
+
+desc "what was that command again?"
+task :huh? do
+  puts "ruby #{Hoe::RUBY_FLAGS} bin/ruby_parse -q -g ..."
+end
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "rb-ruby_parser"
+    gem.summary = %Q{Fork of ParseTree/ruby_parser}
+    gem.description = %Q{This version respects text after block comment =end}
+    gem.email = "ray.baxter@gmail.com"
+    gem.homepage = "http://github.com/raybaxter/rb-ruby_parser"
+    gem.authors = ["Ray Baxter"]
+    gem.add_development_dependency "minitest", ">= 0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+# vim: syntax=Ruby