rubylexer 0.7.1 → 0.7.2

data/lib/rubylexer/context.rb

@@ -272,6 +272,12 @@ module NestedContexts
   class ExpectDoOrNlContext < NestedContext
   end
 
+  class ExpectThenOrNlContext < NestedContext
+    def initialize starter, linenum
+      super starter, "then", linenum
+    end
+  end
+
   class TernaryContext < NestedContext
     def initialize(linenum)
       dflt_initialize('?',':',linenum)

data/lib/rubylexer/rulexer.rb

@@ -255,7 +255,7 @@ if FASTER_STRING_ESCAPES
         special=/\\./m
       else
         crunch=/#(?=[^{$@])/
-        escaped=/\\([^xcCM0-7]|(c|[CM].)([^\\]|(?=\\))|x.[0-9a-fA-F]?|[0-7]{1,3})/m
+        escaped=/\\(?>[^xcCM0-7]|(?>c|[CM].)(?>[^\\]|(?=\\))|(?>x.[0-9a-fA-F]?)|(?>[0-7]{1,3}))/m
         special=
           case delimiter
           when '\\': crunch
@@ -265,7 +265,7 @@ if FASTER_STRING_ESCAPES
         special_char<< maybe_crunch="#"
       end
       normal="[^#{Regexp.quote '\\'+special_char}]"
-      interior=INTERIOR_REX_CACHE[special_char]||=/#{normal}*(#{special}+#{normal}*)*/
+      interior=INTERIOR_REX_CACHE[special_char]||=/(?>#{normal}*)(?>((?>#{special}+)(?>#{normal}*))*)/
 
       #backslash is just scanned thru, not interpreted
       #... that will change token format
@@ -651,6 +651,7 @@ end
       klass= RubyLexer===self ? self.class : RubyLexer
       rl=klass.new(@filename,@file,@linenum,offset_adjust())
       rl.extend RecursiveRubyLexer
+      rl.enable_macros! if @enable_macro
 #      rl.offset_adjust_set! offset_adjust()
       assert offset_adjust()==rl.offset_adjust()
 

data/lib/rubylexer/token.rb

@@ -73,7 +73,6 @@ class KeywordToken < WToken   #also some operators
 
   #-----------------------------------
   def has_end!
-    assert self===RubyLexer::BEGINWORDS
     @has_end=true
   end
 
@@ -82,7 +81,7 @@ class KeywordToken < WToken   #also some operators
     self===RubyLexer::BEGINWORDS and @has_end||=nil
   end
 
-  attr_accessor :comma_type
+  attr_accessor :comma_type, :ternary
 
   def has_no_block!
      @has_no_block=true
@@ -537,9 +536,10 @@ class KwParamListEndToken  < ZwToken
 end
 
 #-------------------------
-class EndDefHeaderToken < ZwToken
+class EndHeaderToken < ZwToken
   def as; ";" end
 end
+EndDefHeaderToken=EndHeaderToken
 
 #-------------------------
 class EscNlToken < IgnoreToken

data/lib/rubylexer/version.rb

@@ -1,3 +1,3 @@
 class RubyLexer
-  VERSION='0.7.1'
+  VERSION='0.7.2'
 end

data/rubylexer.vpj

@@ -48,37 +48,10 @@
 			</Target>
 		</Menu>
 	</Config>
-	<Files>
+	<CustomFolders>
 		<Folder
 			Name="Source Files"
 			Filters="*.c;*.C;*.cc;*.cpp;*.cp;*.cxx;*.prg;*.pas;*.dpr;*.asm;*.s;*.bas;*.java;*.cs;*.sc;*.e;*.cob;*.html;*.rc;*.tcl;*.py;*.pl;*.rb">
-			<F N="lib/rubylexer/0.6.2.rb"/>
-			<F N="lib/rubylexer/0.6.rb"/>
-			<F N="lib/rubylexer/0.7.0.rb"/>
-			<F N="lib/rubylexer/0.7.1.rb"/>
-			<F N="test/code/all_the_gems.rb"/>
-			<F N="test/code/all_the_raas.rb"/>
-			<F N="test/code/all_the_rubies.rb"/>
-			<F N="lib/assert.rb"/>
-			<F N="lib/rubylexer/charhandler.rb"/>
-			<F N="lib/rubylexer/charset.rb"/>
-			<F N="lib/rubylexer/context.rb"/>
-			<F N="test/code/deletewarns.rb"/>
-			<F N="test/code/dumptokens.rb"/>
-			<F N="test/code/isolate_error.rb"/>
-			<F N="test/code/locatetest.rb"/>
-			<F N="test/code/regression.rb"/>
-			<F N="lib/rubylexer/rubycode.rb"/>
-			<F N="lib/rubylexer.rb"/>
-			<F N="test/code/rubylexervsruby.rb"/>
-			<F N="lib/rubylexer/rulexer.rb"/>
-			<F N="lib/rubylexer/symboltable.rb"/>
-			<F N="test/code/tarball.rb"/>
-			<F N="test/code/testcases.rb"/>
-			<F N="lib/rubylexer/token.rb"/>
-			<F N="lib/rubylexer/tokenprinter.rb"/>
-			<F N="test/code/tokentest.rb"/>
-			<F N="lib/rubylexer/version.rb"/>
 		</Folder>
 		<Folder
 			Name="Header Files"
@@ -92,7 +65,44 @@
 		<Folder
 			Name="Other Files"
 			Filters="">
-			<F N="testing.txt"/>
 		</Folder>
+	</CustomFolders>
+	<Files AutoFolders="DirectoryView">
+		<Folder Name="lib">
+			<Folder Name="rubylexer">
+				<F N="lib/rubylexer/0.6.2.rb"/>
+				<F N="lib/rubylexer/0.6.rb"/>
+				<F N="lib/rubylexer/0.7.0.rb"/>
+				<F N="lib/rubylexer/0.7.1.rb"/>
+				<F N="lib/rubylexer/charhandler.rb"/>
+				<F N="lib/rubylexer/charset.rb"/>
+				<F N="lib/rubylexer/context.rb"/>
+				<F N="lib/rubylexer/rubycode.rb"/>
+				<F N="lib/rubylexer/rulexer.rb"/>
+				<F N="lib/rubylexer/symboltable.rb"/>
+				<F N="lib/rubylexer/token.rb"/>
+				<F N="lib/rubylexer/tokenprinter.rb"/>
+				<F N="lib/rubylexer/version.rb"/>
+			</Folder>
+			<F N="lib/assert.rb"/>
+			<F N="lib/rubylexer.rb"/>
+		</Folder>
+		<Folder Name="test">
+			<Folder Name="code">
+				<F N="test/code/all_the_gems.rb"/>
+				<F N="test/code/all_the_raas.rb"/>
+				<F N="test/code/all_the_rubies.rb"/>
+				<F N="test/code/deletewarns.rb"/>
+				<F N="test/code/dumptokens.rb"/>
+				<F N="test/code/isolate_error.rb"/>
+				<F N="test/code/locatetest.rb"/>
+				<F N="test/code/regression.rb"/>
+				<F N="test/code/rubylexervsruby.rb"/>
+				<F N="test/code/tarball.rb"/>
+				<F N="test/code/testcases.rb"/>
+				<F N="test/code/tokentest.rb"/>
+			</Folder>
+		</Folder>
+		<F N="testing.txt"/>
 	</Files>
 </Project>

data/test/code/dumptokens.rb

@@ -35,12 +35,13 @@ class Token
 end
 end
 
-silent=file=nil
+macros=silent=file=nil
 
 #allow -e
 opts=GetoptLong.new(
   ["--eval", "-e", GetoptLong::REQUIRED_ARGUMENT],
-  ["--silent", "-s", GetoptLong::NO_ARGUMENT]
+  ["--silent", "-s", GetoptLong::NO_ARGUMENT],
+  ["--macro", "-m", GetoptLong::NO_ARGUMENT]
 )
 opts.each{|opt,arg|
   case opt
@@ -49,6 +50,8 @@ opts.each{|opt,arg|
     name='-e'
   when '--silent'
     silent=true
+  when '--macro'
+    macros=true
   end
 }
      
@@ -61,6 +64,7 @@ file||=if name=ARGV.first
   end
 
 lexer=RubyLexer.new(name, file) 
+lexer.enable_macros! if macros
 if silent
   until RubyLexer::EoiToken===(tok=lexer.get1token)
   end

data/test/code/regression.rb

@@ -52,7 +52,6 @@ class<<RubyLexerVsRuby
 =end
 end
 
-if __FILE__==$0
   require 'test/unit'
   
   class LexerTests<Test::Unit::TestCase
@@ -106,4 +105,3 @@ if __FILE__==$0
 #    puts test_code+illegal_test_code
     eval test_code+illegal_test_code
   end
-end

data/test/data/oneliners.rb

@@ -43,7 +43,7 @@ p %\hah, #{backslash} as string delimiter\
 def foo(bar=5,tt=6) end
 wwww,eeee=1,2
 x{a.b,c.d=1,2}
-x{|a.b,c.d|}
+x{proc{|a.b,c.d|}}
 
 p % foo
 p % foo

metadata

@@ -1,33 +1,49 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.4
-specification_version: 1
 name: rubylexer
 version: !ruby/object:Gem::Version 
-  version: 0.7.1
-date: 2008-09-05 00:00:00 +01:00
-summary: RubyLexer is a lexer library for Ruby, written in Ruby.
-require_paths: 
-- lib
-email: rubylexer-owner @at@ inforadical .dot. net
-homepage: http://rubylexer.rubyforge.org/
-rubyforge_project: rubylexer
-description: RubyLexer is a lexer library for Ruby, written in Ruby. Rubylexer is meant as a lexer for Ruby that's complete and correct; all legal Ruby  code should be lexed correctly by RubyLexer as well. Just enough parsing  capability is included to give RubyLexer enough context to tokenize correctly in all cases. (This turned out to be more parsing than I had thought or  wanted to take on at first.) RubyLexer handles the hard things like  complicated strings, the ambiguous nature of some punctuation characters and  keywords in ruby, and distinguishing methods and local variables.
-autorequire: 
-default_executable: 
-bindir: ""
-has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
-  version: 
+  version: 0.7.2
 platform: ruby
-signing_key: 
-cert_chain: 
-post_install_message: 
 authors: 
 - Caleb Clausen
+autorequire: 
+bindir: ""
+cert_chain: []
+
+date: 2008-10-13 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sequence
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.2.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.8.0
+    version: 
+description: RubyLexer is a lexer library for Ruby, written in Ruby. Rubylexer is meant as a lexer for Ruby that's complete and correct; all legal Ruby  code should be lexed correctly by RubyLexer as well. Just enough parsing  capability is included to give RubyLexer enough context to tokenize correctly in all cases. (This turned out to be more parsing than I had thought or  wanted to take on at first.) RubyLexer handles the hard things like  complicated strings, the ambiguous nature of some punctuation characters and  keywords in ruby, and distinguishing methods and local variables.
+email: rubylexer-owner @at@ inforadical .dot. net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.txt
+- Manifest.txt
+- howtouse.txt
+- History.txt
+- testing.txt
 files: 
 - COPYING
 - README.txt
@@ -146,133 +162,32 @@ files:
 - test/data/whatnot.rb
 - test/data/ws_strdelim.rb
 - test/test.sh
-test_files: 
-- test/code/regression.rb
-- test/code/isolate_error.rb
-- test/code/tarball.rb
-- test/code/testcases.rb
-- test/code/locatetest.rb
-- test/code/CVS
-- test/code/all_the_rubies.rb
-- test/code/rubylexervsruby.rb
-- test/code/strgen.rb
-- test/code/lexloop
-- test/code/tokentest.rb
-- test/code/errscan
-- test/code/dumptokens.rb
-- test/code/all_the_raas.rb
-- test/code/torment
-- test/code/locatetest
-- test/code/deletewarns.rb
-- test/code/all_the_gems.rb
-- test/data/here_escnl.rb
-- test/data/migration.rb
-- test/data/tkweird.rb
-- test/data/heremonsters.rb
-- test/data/make_ws_strdelim.rb
-- test/data/heremonsters_dos.rb.broken
-- test/data/blockassigntest.rb
-- test/data/for.rb
-- test/data/__eof6.rb
-- test/data/multilinestring.rb
-- test/data/chunky_bacon.rb
-- test/data/and.rb
-- test/data/oneliners.rb
-- test/data/here_squote.rb
-- test/data/pre.unix.rb
-- test/data/hd0.rb
-- test/data/modl.rb
-- test/data/untermed_string.rb.broken
-- test/data/__end__2.rb
-- test/data/w.rb
-- test/data/stuff3.rb
-- test/data/if.rb
-- test/data/heremonsters_dos.rb
-- test/data/whatnot.rb
-- test/data/pre.rb
-- test/data/jarh.rb
-- test/data/regtest.rb
-- test/data/hdr.rb
-- test/data/chunky_bacon4.rb
-- test/data/__end__.rb
-- test/data/stanzas.rb
-- test/data/modl_fails.rb
-- test/data/stuff4.rb
-- test/data/strinc.rb
-- test/data/lbrace.rb
-- test/data/stuff.rb
-- test/data/p.rb
-- test/data/chunky.plain.rb
-- test/data/noeolatend.rb
-- test/data/__eof5.rb
-- test/data/g.rb
-- test/data/23.rb
-- test/data/illegal_stanzas.rb
-- test/data/here_escnl_dos.rb
-- test/data/lbrack.rb
-- test/data/ws_strdelim.rb
-- test/data/untitled1.rb
-- test/data/rescue.rb
-- test/data/unending_stuff.rb
-- test/data/tokentest.assert.rb.can
-- test/data/maven2_builer_test.rb
-- test/data/simple.rb
-- test/data/heremonsters.rb.broken.save
-- test/data/modl_dos.rb
-- test/data/pleac.rb.broken
-- test/data/heart.rb
-- test/data/s.rb
-- test/data/__eof2.rb
-- test/data/hdr_dos2.rb
-- test/data/wsdlDriver.rb
-- test/data/cvtesc.rb
-- test/data/heremonsters.rb.broken
-- test/data/p-op.rb
-- test/data/illegal_oneliners.rb
-- test/data/hdempty.rb
-- test/data/hdr_dos.rb
-- test/data/1.rb.broken
-- test/data/untermed_here.rb.broken
-- test/data/newsyntax.rb
-- test/data/stuff2.rb
-- test/data/chunky_bacon3.rb
-- test/data/chunky_bacon2.rb
-- test/data/strdelim_crlf.rb
-- test/data/simple_dos.rb
-- test/data/hdateof.rb
-- test/data/format.rb
-- test/results/
+has_rdoc: true
+homepage: http://rubylexer.rubyforge.org/
+post_install_message: 
 rdoc_options: 
 - --main
 - README.txt
-extra_rdoc_files: 
-- README.txt
-- Manifest.txt
-- howtouse.txt
-- History.txt
-- testing.txt
-executables: []
-
-extensions: []
-
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
 requirements: []
 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: sequence
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.2.0
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.5.1
-    version: 
+rubyforge_project: rubylexer
+rubygems_version: 1.3.0
+signing_key: 
+specification_version: 2
+summary: RubyLexer is a lexer library for Ruby, written in Ruby.
+test_files: 
+- test/code/regression.rb