rubylexer 0.6.2 → 0.7.0

data/test/data/rescue.rb

@@ -0,0 +1,35 @@
+begin
+  1 rescue 2
+ rescue; 2
+end
+
+def foo
+
+  1 rescue 2
+ rescue; 2
+
+end
+
+
+class Foo
+
+  1 rescue 2
+ rescue; 2
+
+end
+
+
+module Moo
+
+  1 rescue 2
+ rescue; 2
+
+end
+
+
+class <<Foo
+
+  1 rescue 2
+ rescue; 2
+
+end

data/test/data/s.rb

@@ -0,0 +1,186 @@
+#string content tokens move around in this file; its not really a problem
+
+if false 
+      t=0
+      t ?'t':0
+      "#{t ?'t':0}"
+
+	e.hour %= 12;	( downcase == 'p')
+	e.hour %= 12;	begin downcase == 'p'; end
+	e.hour %= 12;	if downcase == 'p'; end
+	e.hour %= 12;	(downcase)
+	e.hour %= 12;	if downcase; end
+
+      proc do <<EOD
+          [#{dqname}]
+EOD
+      end
+      
+      proc do <<-EOD
+          [#{dqname}]
+        EOD
+      end
+      
+      proc do|x| <<-EOD
+          [#{dqname}]
+        EOD
+      end
+      
+      proc do|| <<-EOD
+          [#{dqname}]
+        EOD
+      end
+      
+      proc { <<-EOD
+          [#{dqname}]
+        EOD
+      }
+
+      proc {|x| <<-EOD
+          [#{dqname}]
+        EOD
+      }
+
+      p <<-EOD
+          [#{dqname}]
+        EOD
+
+
+
+
+      proc do "
+          [#{dqname}]
+          "
+      end
+      
+      proc do `
+          [#{dqname}]
+          `
+      end
+      
+      proc do /
+          [#{dqname}]
+          /
+      end
+      
+      proc do '
+          [#{dqname}]
+          '
+      end
+      
+      proc do %Q{
+          [#{dqname}]
+          }
+      end
+      
+      proc do %{
+          [#{dqname}]
+          }
+      end
+      
+end      
+
+def SelfReferencing 
+#old name alias
+end
+
+def SelfReferencing #old name alias
+end
+
+def SelfReferencing#old name alias
+end
+
+def SelfReferencing;end#old name alias
+
+p ?a
+p :a
+p :@a
+p :@@a
+p :$a
+p :A
+p :+
+p :`#`
+p ? p : 0
+
+if false
+O=0
+false ? 2:O
+false ? 2: O
+false ? 2 :O
+false ? 2 : O
+
+eof??n
+eof? ?n
+
+eof??nil:true
+eof? ?nil:true
+eof?? nil:true
+eof? ? nil:true
+
+eof??nil:true
+eof??nil :true
+eof??nil: true
+eof??nil : true
+
+eof!?nil:true
+eof! ?nil:true
+eof!? nil:true
+eof! ? nil:true
+
+eof!?nil:true
+eof!?nil :true
+eof!?nil: true
+eof!?nil : true
+
+end
+
+p <<'FOO'.gsub(" ","\n")
+BAR#{BAZ}
+FOO
+
+p "\
+#{foo}"
+
+p <<end
+#{compile_body}\
+#{outvar}
+end
+
+
+
+
+p "#{<<foobar3}"
+bim
+baz
+bof
+foobar3
+
+p "#{<<foobar2
+bim
+baz
+bof
+foobar2
+}"
+
+
+p <<one ; p "#{<<two}"
+1111111111111111
+fdsgdfgdsfffff
+one
+2222222222222222
+sdfasdfasdfads
+two
+
+p "#{<<foobar1.each('|'){|s| '\nthort: '+s}
+jbvd|g4543ghb|!@G$dfsd|fafr|e
+|s4e5rrwware|BBBBB|*&^(*&^>"PMK:njs;d|
+
+foobar1
+}"
+
+
+p <<""
+foo
+
+
+bar

data/test/data/strinc.rb

@@ -0,0 +1,2 @@
+"fooo#{33}fsfd"
+

data/test/data/untermed_here.rb.broken

@@ -0,0 +1,2 @@
+<<here
+blah, blah blah

data/test/data/untermed_string.rb.broken

@@ -0,0 +1 @@
+"blah, blah

data/testing.txt

@@ -1,8 +1,10 @@
 
 Running the tests:
-the simplest thing to do is run testcode/locatetest. this will use locate to find as much ruby
-code on your system and test each specimen to see if it can be tokenized correctly (by feeding it 
-to testcode/rubylexervsruby.rb, the operation of which is outlined below under 'testing strategy').
+the simplest thing to do is run "ruby -Ilib test/code/locatetest.rb". 
+this will use locate to find as much ruby code on your system and test 
+each specimen to see if it can be tokenized correctly (by feeding it to 
+testcode/rubylexervsruby.rb, the operation of which is outlined below 
+under 'testing strategy').
 
 Interpreting the output of rubylexervsruby.rb (and locatetest):
 in rubylexervsruby, i've tried to follow the philosophy that the test program
@@ -127,4 +129,4 @@ thinks should be 2 tokens. and in fact, this is the case with strings: ruby divi
 string into string open, string body, and string close tokens with option interpolations,
 whereas rubylexer has just a single string token (with subtokens, if interpolations are
 present.) this difference in handling accounts in part for rubylexer's inability
-to correctly lex certain very complicated strings.
+to correctly lex certain very complicated strings.

metadata

@@ -1,76 +1,180 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.8.10
+rubygems_version: 0.9.0
 specification_version: 1
 name: rubylexer
 version: !ruby/object:Gem::Version 
-  version: 0.6.2
-date: 2005-06-03
-summary: lexer of ruby in ruby
+  version: 0.7.0
+date: 2008-02-17 00:00:00 +00:00
+summary: RubyLexer is a lexer library for Ruby, written in Ruby.
 require_paths: 
-  - "."
-email: 
-homepage: 
+- lib
+email: rubylexer-owner @at@ inforadical .dot. net
+homepage: http://rubylexer.rubyforge.org/
 rubyforge_project: rubylexer
-description: 
+description: RubyLexer is a lexer library for Ruby, written in Ruby. Rubylexer is meant as a lexer for Ruby that's complete and correct; all legal Ruby  code should be lexed correctly by RubyLexer as well. Just enough parsing  capability is included to give RubyLexer enough context to tokenize correctly in all cases. (This turned out to be more parsing than I had thought or  wanted to take on at first.) RubyLexer handles the hard things like  complicated strings, the ambiguous nature of some punctuation characters and  keywords in ruby, and distinguishing methods and local variables.
 autorequire: 
 default_executable: 
-bindir: bin
-has_rdoc: false
+bindir: ""
+has_rdoc: true
 required_ruby_version: !ruby/object:Gem::Version::Requirement 
   requirements: 
-    - 
-      - ">"
-      - !ruby/object:Gem::Version 
-        version: 0.0.0
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
   version: 
 platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
 authors: 
-  - Caleb Clausen
+- Caleb Clausen
 files: 
-  - rubylexer.rb
-  - assert.rb
-  - context.rb
-  - rlold.rb
-  - rulexer.rb
-  - tokenprinter.rb
-  - charhandler.rb
-  - io.each_til_charset.rb
-  - rubycode.rb
-  - symboltable.rb
-  - charset.rb
-  - require.rb
-  - token.rb
-  - COPYING
-  - README
-  - Rantfile
-  - howtouse.txt
-  - testing.txt
-  - testresults/placeholder
-  - testcode/deletewarns.rb
-  - testcode/dumptokens.rb
-  - testcode/locatetest
-  - testcode/rubylexervsruby.rb
-  - testcode/rubylexervsruby.sh
-  - testcode/tokentest.rb
-  - testcode/torment
-  - testdata/1.rb.broken
-  - testdata/23.rb
-  - testdata/g.rb
-  - testdata/newsyntax.rb
-  - testdata/noeolatend.rb
-  - testdata/p.rb
-  - testdata/pleac.rb.broken
-  - testdata/pre.rb
-  - testdata/pre.unix.rb
-  - testdata/regtest.rb
-  - testdata/tokentest.assert.rb.can
-  - testdata/untitled1.rb
-  - testdata/w.rb
-  - testdata/wsdlDriver.rb
-test_files: []
-rdoc_options: []
-extra_rdoc_files: []
+- COPYING
+- README.txt
+- Manifest.txt
+- Rakefile
+- howtouse.txt
+- History.txt
+- testing.txt
+- lib/rubylexer/rubycode.rb
+- lib/rubylexer/context.rb
+- lib/rubylexer/token.rb
+- lib/rubylexer/0.6.rb
+- lib/rubylexer/0.6.2.rb
+- lib/rubylexer/0.7.0.rb
+- lib/rubylexer/version.rb
+- lib/rubylexer/rulexer.rb
+- lib/rubylexer/tokenprinter.rb
+- lib/rubylexer/charset.rb
+- lib/rubylexer/symboltable.rb
+- lib/rubylexer/charhandler.rb
+- lib/assert.rb
+- lib/rubylexer.rb
+- test/data/gemlist.txt
+- test/data/blockassigntest.rb
+- test/data/for.rb
+- test/data/chunky_bacon.rb
+- test/data/and.rb
+- test/data/pre.unix.rb
+- test/data/untermed_string.rb.broken
+- test/data/__end__2.rb
+- test/data/w.rb
+- test/data/if.rb
+- test/data/pre.rb
+- test/data/jarh.rb
+- test/data/regtest.rb
+- test/data/chunky_bacon4.rb
+- test/data/__end__.rb
+- test/data/strinc.rb
+- test/data/lbrace.rb
+- test/data/p.rb
+- test/data/chunky.plain.rb
+- test/data/noeolatend.rb
+- test/data/g.rb
+- test/data/23.rb
+- test/data/lbrack.rb
+- test/data/untitled1.rb
+- test/data/rescue.rb
+- test/data/tokentest.assert.rb.can
+- test/data/pleac.rb.broken
+- test/data/heart.rb
+- test/data/s.rb
+- test/data/wsdlDriver.rb
+- test/data/p-op.rb
+- test/data/1.rb.broken
+- test/data/untermed_here.rb.broken
+- test/data/newsyntax.rb
+- test/data/chunky_bacon3.rb
+- test/data/chunky_bacon2.rb
+- test/data/format.rb
+- test/code/locatetest.rb
+- test/code/rubylexervsruby.rb
+- test/code/dl_all_gems.rb
+- test/code/unpack_all_gems.rb
+- test/code/tokentest.rb
+- test/code/dumptokens.rb
+- test/code/torment
+- test/code/locatetest
+- test/code/deletewarns.rb
+test_files: 
+- test/code/locatetest.rb
+- test/code/rubylexervsruby.rb
+- test/code/dl_all_gems.rb
+- test/code/unpack_all_gems.rb
+- test/code/tokentest.rb
+- test/code/dumptokens.rb
+- test/code/torment
+- test/code/locatetest
+- test/code/deletewarns.rb
+- test/data/blockassigntest.rb
+- test/data/for.rb
+- test/data/chunky_bacon.rb
+- test/data/and.rb
+- test/data/pre.unix.rb
+- test/data/untermed_string.rb.broken
+- test/data/__end__2.rb
+- test/data/w.rb
+- test/data/if.rb
+- test/data/pre.rb
+- test/data/jarh.rb
+- test/data/regtest.rb
+- test/data/chunky_bacon4.rb
+- test/data/__end__.rb
+- test/data/strinc.rb
+- test/data/lbrace.rb
+- test/data/p.rb
+- test/data/chunky.plain.rb
+- test/data/noeolatend.rb
+- test/data/g.rb
+- test/data/23.rb
+- test/data/lbrack.rb
+- test/data/untitled1.rb
+- test/data/rescue.rb
+- test/data/tokentest.assert.rb.can
+- test/data/pleac.rb.broken
+- test/data/heart.rb
+- test/data/s.rb
+- test/data/wsdlDriver.rb
+- test/data/p-op.rb
+- test/data/1.rb.broken
+- test/data/untermed_here.rb.broken
+- test/data/newsyntax.rb
+- test/data/chunky_bacon3.rb
+- test/data/chunky_bacon2.rb
+- test/data/format.rb
+- test/results/
+rdoc_options: 
+- --main
+- README.txt
+extra_rdoc_files: 
+- README.txt
+- Manifest.txt
+- howtouse.txt
+- History.txt
+- testing.txt
+- test/data/gemlist.txt
 executables: []
+
 extensions: []
+
 requirements: []
-dependencies: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sequence
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.5.0
+    version: