rubylexer 0.6.2

data/tokenprinter.rb

@@ -0,0 +1,152 @@
+=begin copyright
+    rubylexer - a ruby lexer written in ruby
+    Copyright (C) 2004,2005  Caleb Clausen
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+=end
+
+
+
+require "assert"
+
+
+
+
+#-------------------------------
+class SimpleTokenPrinter
+
+   def initialize(*bogus)
+      @tokens=@srclines=0
+   end
+
+   TOKENSPERLINE=8
+   TOKENSMAGICMAP="\n"+' '*(TOKENSPERLINE-1)
+
+   def pprint(tok) print(sprint(tok)) end
+
+   def sprint(tok)
+      case tok
+      when Newline then "#{@lastfal if ((@srclines+=1)%16==0)} #; "
+      when FileAndLineToken then @lastfal=tok;''
+      when IgnoreToken then ''  #skip comments&whitespace
+      else tok.to_s + TOKENSMAGICMAP[ (@tokens+=1) % TOKENSPERLINE, 1 ]
+      end
+   end
+end
+
+   class EscNl; def ws_munge(tp)
+      tp.lasttok=self
+      return to_s
+   end end
+   class FileAndLineToken; def ws_munge(tp)
+      tp.lasttok=self
+      tp.lastfal=self
+      return ''
+   end end
+   class Newline; def ws_munge(tp)
+      tp.lasttok=self
+      return"#{tp.lastfal_every_10_lines}\n"
+   end end
+   class IgnoreToken; def ws_munge(tp)
+         #tp.latestline+= to_s.scan("\n").size
+         tp.lasttok=self
+         unless tp.inws
+           tp.inws=true
+           return ' '
+         end
+         return ''
+   end end
+   class OutlinedHereBodyToken; def ws_munge(tp)
+      nil
+   end end
+   class ZwToken; def ws_munge(tp)
+      case tp.showzw 
+      when 2: explicit_form_all
+      when 1: explicit_form
+      when 0: nil
+      else raise 'unknown showzw'
+      end
+   end end
+   class Token; def ws_munge(tp)
+      nil
+   end end
+
+#-------------------------------
+class KeepWsTokenPrinter
+   attr_accessor :lasttok, :lastfal, :inws #,:latestline
+   attr :showzw
+   ACCUMSIZE=50
+
+
+   def initialize(sep='',line=1,showzw=0)
+      @sep=sep
+      @inws=false
+      @lasttok=''
+      #@latestline=line
+      @lastprintline=0
+      @accum=[]
+      @showzw=showzw
+   end
+
+   def pprint(tok)
+
+      @accum<<aprint(tok).to_s
+      if @accum.size>ACCUMSIZE or EoiToken===tok
+         print(@accum)
+         @accum=[]
+      end
+   end
+
+   def aprint(tok)
+      result=tok.ws_munge(self) and return result
+
+      #insert extra ws unless an ambiguous op immediately follows
+      #id or num, in which case ws would change the meaning
+      result=if (ZwToken===tok or NoWsToken===@lasttok)
+        tok.to_s
+      else
+        [@sep.dup,tok.to_s]
+      end
+
+      @lasttok=tok
+      @inws=false
+
+      return result
+   end
+
+   alias sprint aprint
+
+   def lastfal_every_10_lines
+      if(@lastprintline+=1) > 10
+         @lastprintline=0
+         %Q[ #@lastfal]
+      end
+   end
+
+
+   #insert extra ws unless an ambiguous op immediately follows
+   #id or num, in which case ws would change the meaning
+   def ws_would_change_meaning?(tok,lasttok)   #yukk -- is it used?
+      tok.kind_of?(String) and
+      /^[%(]$/===tok and #is ambiguous operator?
+          lasttok and
+          (lasttok.kind_of?(Numeric) or
+           (lasttok.kind_of?(String) and
+            /^[$@a-zA-Z_]/===@lasttok)) #lasttok is id or num?
+   end
+end
+
+#-------------------------------
+

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.10
+specification_version: 1
+name: rubylexer
+version: !ruby/object:Gem::Version 
+  version: 0.6.2
+date: 2005-06-03
+summary: lexer of ruby in ruby
+require_paths: 
+  - "."
+email: 
+homepage: 
+rubyforge_project: rubylexer
+description: 
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: false
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+  - Caleb Clausen
+files: 
+  - rubylexer.rb
+  - assert.rb
+  - context.rb
+  - rlold.rb
+  - rulexer.rb
+  - tokenprinter.rb
+  - charhandler.rb
+  - io.each_til_charset.rb
+  - rubycode.rb
+  - symboltable.rb
+  - charset.rb
+  - require.rb
+  - token.rb
+  - COPYING
+  - README
+  - Rantfile
+  - howtouse.txt
+  - testing.txt
+  - testresults/placeholder
+  - testcode/deletewarns.rb
+  - testcode/dumptokens.rb
+  - testcode/locatetest
+  - testcode/rubylexervsruby.rb
+  - testcode/rubylexervsruby.sh
+  - testcode/tokentest.rb
+  - testcode/torment
+  - testdata/1.rb.broken
+  - testdata/23.rb
+  - testdata/g.rb
+  - testdata/newsyntax.rb
+  - testdata/noeolatend.rb
+  - testdata/p.rb
+  - testdata/pleac.rb.broken
+  - testdata/pre.rb
+  - testdata/pre.unix.rb
+  - testdata/regtest.rb
+  - testdata/tokentest.assert.rb.can
+  - testdata/untitled1.rb
+  - testdata/w.rb
+  - testdata/wsdlDriver.rb
+test_files: []
+rdoc_options: []
+extra_rdoc_files: []
+executables: []
+extensions: []
+requirements: []
+dependencies: []