rubylexer 0.7.0 → 0.7.1

data/test/data/strdelim_crlf.rb

@@ -0,0 +1,6 @@
+%
+sdfsdsdfsdfsd
+
+
+
+

data/test/data/stuff.rb

@@ -0,0 +1,6 @@
+p <<stuff+'foobar'.tr('j-l','d-f')\
++"more stuff"
+12345678
+the quick brown fox jumped over the lazy dog
+stuff
+

data/test/data/stuff2.rb

@@ -0,0 +1,5 @@
+p <<stuff+'foobar'.tr('j-l','d-f')\
++"more stuff"
+12345678
+the quick brown fox jumped over the lazy dog
+stuff

data/test/data/stuff3.rb

@@ -0,0 +1,6 @@
+p <<stuff+'foobar'.tr('j-l','d-f')\
++"more stuff"
+12345678
+the quick brown fox jumped over the lazy dog
+stuff
++"even more"

data/test/data/stuff4.rb

@@ -0,0 +1,6 @@
+p <<stuff+'foobar'.tr('j-l','d-f')\
++"more stuff"
+12345678
+the quick brown fox jumped over the lazy dog
+stuff
++"even more"

data/test/data/tkweird.rb

@@ -0,0 +1,20 @@
+  WIDGET_DESTROY_HOOK = '<WIDGET_DESTROY_HOOK>'
+  INTERP._invoke_without_enc('event', 'add', 
+                             "<#{WIDGET_DESTROY_HOOK}>", '<Destroy>')
+  INTERP._invoke_without_enc('bind', 'all', "<#{WIDGET_DESTROY_HOOK}>",
+                             install_cmd(proc{|path|
+                                unless TkCore::INTERP.deleted?
+                                  begin
+                                    if (widget=TkCore::INTERP.tk_windows[path])
+                                      if widget.respond_to?(:__destroy_hook__)
+                                        widget.__destroy_hook__
+                                      end
+                                    end
+                                  rescue Exception=>e
+                                      p e if $DEBUG
+                                  end
+                                end
+                             }) << ' %W')
+
+
+

data/test/data/unending_stuff.rb

@@ -0,0 +1,5 @@
+p <<stuff+'foobar'.tr('j-l','d-f')\
++"more stuff"
+12345678
+the quick brown fox jumped over the lazy dog
+stuff

data/test/data/whatnot.rb

@@ -0,0 +1,8 @@
+aaa=<<whatnot; p "#{'uh,yeah'
+gonna take it down, to the nitty-grit
+gonna tell you mother-fuckers why you ain't shit
+cause suckers like you just make me strong
+you been pumpin' that bullshit all day long
+whatnot
+}"
+p aaa

data/test/test.sh

@@ -0,0 +1,239 @@
+#!/bin/sh
+#    rubylexer - a ruby lexer written in ruby
+#    Copyright (C) 2004,2005  Caleb Clausen
+#
+#    This library is free software; you can redistribute it and/or
+#    modify it under the terms of the GNU Lesser General Public
+#    License as published by the Free Software Foundation; either
+#    version 2.1 of the License, or (at your option) any later version.
+#
+#    This library is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#    Lesser General Public License for more details.
+#
+#    You should have received a copy of the GNU Lesser General Public
+#    License along with this library; if not, write to the Free Software
+#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+
+#export DEBUG=-rdebug
+
+
+#export RUBYLEXERVSRUBY=testcode/rubylexervsruby.sh
+test $RUBY || export RUBY=ruby
+export RUBYLEXERVSRUBY="$RUBY testcode/rubylexervsruby.rb"
+
+#ruma driver tests
+#(disabled for now)
+#./rumadriver.rb testdata/foobar.rm rulexer.rb
+#exit
+
+$RUBY -d $DEBUG testcode/tokentest.rb assert.rb | diff -ub testdata/tokentest.assert.rb.can -
+#$RUBY -d $DEBUG testcode/tokentest.rb testdata/a | diff -ub testdata/tokentest.a.can -
+$RUBYLEXERVSRUBY testdata/p.rb
+$RUBYLEXERVSRUBY testdata/w.rb
+$RUBYLEXERVSRUBY testdata/g.rb
+
+
+$RUBYLEXERVSRUBY testdata/regtest.rb
+
+$RUBYLEXERVSRUBY   testdata/untitled1.rb
+
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/htree/template.rb #small diff, no semantic change
+#set -e
+
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_libtcltk-ruby1.8_exmaples_sample_sample2.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_image_size.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_rexml_dtd_entitydecl.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_rexml_dtd_notationdecl.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_rexml_xpath.rb
+#exit
+#nil char at eof:
+$RUBYLEXERVSRUBY /usr/share/freeride/freebase/lib/freebase/databus.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_fox_gui/editpane.rb
+
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_tools_fox_source_browser/basic_source_browser.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_tk.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_cgi.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_fox_gui/appframe.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_ruby1.8-examples_examples_test.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_tools_source_parser/basic_parser.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_commands/core_commands.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_fox_gui/fxscintilla/scintilla.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/rdoc/parsers/parse_c.rb
+$RUBYLEXERVSRUBY /usr/share/doc/rdtool/examples/rdswap.rb.gz
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_cgi.rb
+$RUBYLEXERVSRUBY domain.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_optparse.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_resolv.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_resolv.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_racc_output.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_commands/key_manager.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/progressbar.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/yaml/rubytypes.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_fox_gui/fxscintilla/ruby_colourize.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_fox_gui/outputpane.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/yaml/basenode.rb
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_fox_gui/fxscintilla/ruby_autoindent.rb
+$RUBYLEXERVSRUBY /usr/share/doc/libxtemplate-ruby1.8/examples/sample33.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/rmail/header.rb
+$RUBYLEXERVSRUBY /usr/share/doc/libxtemplate-ruby1.8/examples/sample34.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/runit/testsuite.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/zip/zip.rb
+$RUBYLEXERVSRUBY /usr/lib/ruby/1.8/html/template.rb
+
+#exit
+
+
+
+
+
+
+
+
+
+
+
+
+
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_xmlrpc_utils.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_rubyunit.rb
+
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_parsedate.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_date_format.rb
+
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_tkcanvas.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_tkdialog.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_net_imap.rb
+
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_libtk-ruby1.8_exmaples_tkmultilistbox.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_ruby-examples_examples_mine.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_ruby-examples_examples_observ.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_ruby1.8-examples_examples_mine.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_ruby1.8-examples_examples_observ.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_aswiki_util.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_libopengl-ruby_examples_clip.rb
+
+
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_amrita_parts.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_cgi_session.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_complex.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_csv.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_libglade-ruby_examples_test.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_amrita_ams.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_amrita_format.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_amrita_node.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_amrita_tag.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_amstd_version.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_webrick_httputils.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_net_imap.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_resolv-replace.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_irb_ruby-lex.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_date.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_racc_raccp.rb
+
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_rd_rdinlineparser.tab.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_rd_rdblockparser.tab.rb
+
+
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/texturesurf.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libncurses-ruby1.8/examples/tclock.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtkglext1-ruby/examples/simple-mixed.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/surface.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/tess.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/lines.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libamrita-ruby1.6/examples/bbs/bbs.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/menus.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/feedback.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/wrap.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/main.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/pickdepth.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/tree_store.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/teaambient.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libart2-ruby/examples/testlibart2.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgnomecanvas2-ruby/examples/canvas-primitives.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/images.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libamrita-ruby1.6/examples/bbs/test/testbbs.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgnomecanvas2-ruby/examples/canvas-curve.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/texgen.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/stock_browser.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libncurses-ruby1.8/examples/example.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/pixbufs.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/list_store.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgda2-ruby/examples/sample.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libamrita-ruby1.6/examples/bbs/model.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/teapots.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgstreamer0.6-ruby/examples/media-type2.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgnome2-ruby/examples/test-gnome/app-helper.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/testgtk/spinbutton.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/select.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/quadric.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/drawingarea.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/accpersp.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/dof.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/testgtk/cursors.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/accanti.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/testgtk/dnd.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/plane.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libnet-acl-ruby1.6/examples/acltest.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/aapoly.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/changedisplay.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgnome2-ruby/examples/gnome-config.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/jitter.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/image.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/font.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtkglext1-ruby/examples/gtkglut.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgstreamer0.6-ruby/examples/gst-inspect.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/fog.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtkglext1-ruby/examples/font.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/alpha3D.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/alpha.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/testgtk/testgtk.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgnomecanvas2-ruby/examples/canvas-arrowhead.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/aargb.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtkglext1-ruby/examples/simple.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgnome2-ruby/examples/gnome-app-helper.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/gtk-demo/textview.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/stroke.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/testgtk/progressbar.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/colormat.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtkglext1-ruby/examples/share-lists.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgtk2-ruby/examples/testgtk/liststore.rb.gz
+
+$RUBYLEXERVSRUBY /usr/share/freeride/plugins/rubyide_fox_gui/fxscintilla/ruby_properties.rb
+$RUBYLEXERVSRUBY /usr/share/doc/rubymagick/examples/demo.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libopengl-ruby/examples/texbind.rb.gz
+$RUBYLEXERVSRUBY /usr/share/doc/libgnome2-ruby/examples/test-gnome/font-picker.rb.gz
+
+echo 'skipping some cmds:'
+echo 'these generate split-line warnings (low prio issue for now)'
+cat <<end
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_optparse.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_libxmlrpc-ruby1.6_examples_perf_async.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_libdrb-ruby1.6_examples_dlogd.rb
+$RUBYLEXERVSRUBY testdata/_usr_bin_dpkg-checkdeps.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_libgtk-ruby_examples_testgtk_ctree.rb
+$RUBYLEXERVSRUBY testdata/_usr_share_doc_ruby-examples_examples_cal.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.8_rd_rd2man-lib.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_xmltree.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_parsearg.rb
+$RUBYLEXERVSRUBY testdata/_usr_lib_ruby_1.6_tracer.rb
+end
+
+#oops, looks like this one's missing an end
+#$RUBYLEXERVSRUBY /usr/share/freeride/freebase/plugins/raa_xmlrpc4r/rpcd.rb
+
+#not legal in ruby 1.8
+#$RUBYLEXERVSRUBY /usr/share/doc/libtcltk-ruby1.6/exmaples/sample/sample1.rb
+
+#these 3 aren't legal ruby:
+#$RUBYLEXERVSRUBY /usr/share/doc/libdb4.1-ruby1.8/bdb.rb.gz
+#$RUBYLEXERVSRUBY /usr/share/doc/libdb2-ruby1.8/bdb.rb.gz
+#$RUBYLEXERVSRUBY /usr/share/doc/libdb3-ruby1.8/bdb.rb.gz
+
+#not even vaguely legal ruby:
+#$RUBYLEXERVSRUBY /usr/share/doc/libdebconf-client-ruby/examples/test.rb.templates.ja
+#$RUBYLEXERVSRUBY /usr/share/doc/libdebconf-client-ruby/examples/test.rb.templates.in
+#$RUBYLEXERVSRUBY /usr/share/doc/libdebconf-client-ruby/examples/test.rb.templates
+

data/testing.txt

@@ -1,69 +1,62 @@
-
 Running the tests:
-the simplest thing to do is run "ruby -Ilib test/code/locatetest.rb". 
-this will use locate to find as much ruby code on your system and test 
+The simplest thing to do is run "ruby -Ilib test/code/regression.rb". This 
+tests against a list of known ruby expressions. It will take several minutes 
+to run. Currently, there are 5 failures.
+
+If you're ambitious, try this command: "ruby -Ilib test/code/locatetest.rb". 
+This will use locate to find as much ruby code on your system and test 
 each specimen to see if it can be tokenized correctly (by feeding it to 
 testcode/rubylexervsruby.rb, the operation of which is outlined below 
 under 'testing strategy').
 
 Interpreting the output of rubylexervsruby.rb (and locatetest):
-in rubylexervsruby, i've tried to follow the philosophy that the test program
-doesn't print anything unless there's an error. perhaps i haven't followed
+In rubylexervsruby, I've tried to follow the philosophy that the test program
+doesn't print anything unless there's an error. Perhaps I haven't followed
 this far enough; every run of rubylexervsruby produces a little output, and 
 sometimes a run will produce output that doesn't actually indicate a problem, 
-or only a low-priority problem. (since locatetest, torment, and test all run
-rubylexervsruby over and over, they all produce lots of (mostly harmless) 
-output. sorry.)
+or only a low-priority problem. (Since locatetest runs rubylexervsruby over 
+and over, it produces lots of (mostly harmless) output. Sorry.)
 
-the following types of output should be ignored:
+The following types of output should be ignored:
 
 diff file or chunk headers
 
 lines that look like this:
   executing: ruby testcode/tokentest.rb ...     #normal, 1 for every file
-or this:
-  warning moved from 24 to 22: ambiguous first argument; put parentheses or even spaces
 or this:
   Created warning(s) in new file, line 85: useless use of <=> in void context
 or this:
   Removed warning(s) from old file (?!), line 85: useless use of <=> in void context
-indicate that a warning was added deleted, or moved. ultimately, these should
+indicate that a warning was added or deleted. Ultimately, these should
 go away, but right now it's a low-priority issue.
 
-if you ever see ruby stack dump in rubylexervsruby output, that's certainly
-an error (if the input ruby code is valid). 
-
-something that looks like a unidiff chunk body (not header) may indicate 
-an error as well. the problem is that sometimes those morpheaous warnings 
-sneak through my filter (which is supposed to condense them into a single 
-line like those above), so you will see diff chunks where the only real
-difference is a warning. here are some examples of the kind of diff chunks 
-that should NOT cause alarm:
-
---:89: warning: useless use of <=> in void context
---:92: warning: useless use of <=> in void context
-+-:90: warning: useless use of <=> in void context
- Stack now 0 2 62 300 5 110 365 544
-
- Shifting token tIDENTIFIER, Entering state 34
--Reading a token: -:318: warning: ambiguous first argument; put parentheses or even spaces
--Next token is token tINTEGER ()
-+Reading a token: Next token is token tINTEGER ()
- Reducing stack by rule 476 (line 2382), tIDENTIFIER -> operation
-
-if you look closely, (and are experienced in reading unidiff output), you'll
-see that the only difference is a warning. to understand more about how the
-unidiff output is created, see the section on testing strategy below.
-
-htree/template.rb:
-testing this file prints a small unidiff chunk. analysis indicates that the
-problem is because ruby's lexer generates an extra (empty) string content 
-token at this point, which mine omits. there's no actual semantic difference
-between the two tokenizations, so there's nothing to be concerned about. in
-a future release, when my lexer supports the notion of string contents and
-string delimiters as separate token types, i'll try to emulate ruby more 
-closely. the same case is replicated in p.rb.
-(in other words, ignore the error in this file and the identical one in p.rb.)
+If you ever see ruby stack dump in rubylexervsruby output, that's certainly
+an error. 
+
+Something that looks like a unidiff chunk body (not header) may indicate 
+an error as well. To understand more about how the unidiff output is 
+created, see the section on testing strategy below.
+
+htree/template.rb should be ok now.
+
+currently, lots of warnings are printed about token offsets being off by 1,
+particularly the AssignmentRhsListToken. This is a problem, but for now I'm
+ignoring it. 
+
+Diff chunks like this indicate a minor problem with the placement of (empty)
+string fragments. Ignore it for now:
+
+@@ -13,2 +13,3 @@
+ Shifting token tSTRING_BEG ()
++Shifting token tSTRING_CONTENT ()
+ Shifting token tSTRING_DBEG ()
+@@ -19,2 +20,2 @@
+ Shifting token '\n' ()
+@@ -13,2 +13,3 @@
+ Shifting token tSTRING_BEG ()
++Shifting token tSTRING_CONTENT ()
+ Shifting token tSTRING_DBEG ()
+
 
 
 if you find any output that doesn't look like one of the above exceptions, 
@@ -76,10 +69,6 @@ rpcd.rb from freeride -- missing an end
 sample1.rb from 1.6 version of tcltk -- not legal in ruby 1.8
 bdb.rb from libdb2, 3, and 4 -- not how you declare [] method in ruby
 
-testdata/p.rb (my menagerie of weird test cases) is one of the worst 
-offenders; it prints lots of output when tested, but all of the problems
-are harmless or minor.
-
 only the 10 first lines of each failing file are printed. the rest, as well
 as other intermediate files are kept in the testresults directory. the test
 output files are named *.prs.diff. beware: this directory is never cleaned,

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.0
+rubygems_version: 0.9.4
 specification_version: 1
 name: rubylexer
 version: !ruby/object:Gem::Version 
-  version: 0.7.0
-date: 2008-02-17 00:00:00 +00:00
+  version: 0.7.1
+date: 2008-09-05 00:00:00 +01:00
 summary: RubyLexer is a lexer library for Ruby, written in Ruby.
 require_paths: 
 - lib
@@ -50,7 +50,6 @@ files:
 - lib/rubylexer/charhandler.rb
 - lib/assert.rb
 - lib/rubylexer.rb
-- test/data/gemlist.txt
 - test/data/blockassigntest.rb
 - test/data/for.rb
 - test/data/chunky_bacon.rb
@@ -89,58 +88,158 @@ files:
 - test/data/format.rb
 - test/code/locatetest.rb
 - test/code/rubylexervsruby.rb
-- test/code/dl_all_gems.rb
-- test/code/unpack_all_gems.rb
 - test/code/tokentest.rb
 - test/code/dumptokens.rb
 - test/code/torment
 - test/code/locatetest
 - test/code/deletewarns.rb
+- lib/rubylexer/0.7.1.rb
+- rubylexer.vpj
+- test/code/all_the_gems.rb
+- test/code/all_the_raas.rb
+- test/code/all_the_rubies.rb
+- test/code/errscan
+- test/code/isolate_error.rb
+- test/code/lexloop
+- test/code/regression.rb
+- test/code/strgen.rb
+- test/code/tarball.rb
+- test/code/testcases.rb
+- test/data/cvtesc.rb
+- test/data/__eof2.rb
+- test/data/__eof5.rb
+- test/data/__eof6.rb
+- test/data/hd0.rb
+- test/data/hdateof.rb
+- test/data/hdempty.rb
+- test/data/hdr_dos2.rb
+- test/data/hdr_dos.rb
+- test/data/hdr.rb
+- test/data/here_escnl_dos.rb
+- test/data/here_escnl.rb
+- test/data/heremonsters_dos.rb
+- test/data/heremonsters_dos.rb.broken
+- test/data/heremonsters.rb
+- test/data/heremonsters.rb.broken
+- test/data/heremonsters.rb.broken.save
+- test/data/here_squote.rb
+- test/data/illegal_oneliners.rb
+- test/data/illegal_stanzas.rb
+- test/data/make_ws_strdelim.rb
+- test/data/maven2_builer_test.rb
+- test/data/migration.rb
+- test/data/modl_dos.rb
+- test/data/modl_fails.rb
+- test/data/modl.rb
+- test/data/multilinestring.rb
+- test/data/oneliners.rb
+- test/data/simple_dos.rb
+- test/data/simple.rb
+- test/data/stanzas.rb
+- test/data/strdelim_crlf.rb
+- test/data/stuff2.rb
+- test/data/stuff3.rb
+- test/data/stuff4.rb
+- test/data/stuff.rb
+- test/data/tkweird.rb
+- test/data/unending_stuff.rb
+- test/data/whatnot.rb
+- test/data/ws_strdelim.rb
+- test/test.sh
 test_files: 
+- test/code/regression.rb
+- test/code/isolate_error.rb
+- test/code/tarball.rb
+- test/code/testcases.rb
 - test/code/locatetest.rb
+- test/code/CVS
+- test/code/all_the_rubies.rb
 - test/code/rubylexervsruby.rb
-- test/code/dl_all_gems.rb
-- test/code/unpack_all_gems.rb
+- test/code/strgen.rb
+- test/code/lexloop
 - test/code/tokentest.rb
+- test/code/errscan
 - test/code/dumptokens.rb
+- test/code/all_the_raas.rb
 - test/code/torment
 - test/code/locatetest
 - test/code/deletewarns.rb
+- test/code/all_the_gems.rb
+- test/data/here_escnl.rb
+- test/data/migration.rb
+- test/data/tkweird.rb
+- test/data/heremonsters.rb
+- test/data/make_ws_strdelim.rb
+- test/data/heremonsters_dos.rb.broken
 - test/data/blockassigntest.rb
 - test/data/for.rb
+- test/data/__eof6.rb
+- test/data/multilinestring.rb
 - test/data/chunky_bacon.rb
 - test/data/and.rb
+- test/data/oneliners.rb
+- test/data/here_squote.rb
 - test/data/pre.unix.rb
+- test/data/hd0.rb
+- test/data/modl.rb
 - test/data/untermed_string.rb.broken
 - test/data/__end__2.rb
 - test/data/w.rb
+- test/data/stuff3.rb
 - test/data/if.rb
+- test/data/heremonsters_dos.rb
+- test/data/whatnot.rb
 - test/data/pre.rb
 - test/data/jarh.rb
 - test/data/regtest.rb
+- test/data/hdr.rb
 - test/data/chunky_bacon4.rb
 - test/data/__end__.rb
+- test/data/stanzas.rb
+- test/data/modl_fails.rb
+- test/data/stuff4.rb
 - test/data/strinc.rb
 - test/data/lbrace.rb
+- test/data/stuff.rb
 - test/data/p.rb
 - test/data/chunky.plain.rb
 - test/data/noeolatend.rb
+- test/data/__eof5.rb
 - test/data/g.rb
 - test/data/23.rb
+- test/data/illegal_stanzas.rb
+- test/data/here_escnl_dos.rb
 - test/data/lbrack.rb
+- test/data/ws_strdelim.rb
 - test/data/untitled1.rb
 - test/data/rescue.rb
+- test/data/unending_stuff.rb
 - test/data/tokentest.assert.rb.can
+- test/data/maven2_builer_test.rb
+- test/data/simple.rb
+- test/data/heremonsters.rb.broken.save
+- test/data/modl_dos.rb
 - test/data/pleac.rb.broken
 - test/data/heart.rb
 - test/data/s.rb
+- test/data/__eof2.rb
+- test/data/hdr_dos2.rb
 - test/data/wsdlDriver.rb
+- test/data/cvtesc.rb
+- test/data/heremonsters.rb.broken
 - test/data/p-op.rb
+- test/data/illegal_oneliners.rb
+- test/data/hdempty.rb
+- test/data/hdr_dos.rb
 - test/data/1.rb.broken
 - test/data/untermed_here.rb.broken
 - test/data/newsyntax.rb
+- test/data/stuff2.rb
 - test/data/chunky_bacon3.rb
 - test/data/chunky_bacon2.rb
+- test/data/strdelim_crlf.rb
+- test/data/simple_dos.rb
+- test/data/hdateof.rb
 - test/data/format.rb
 - test/results/
 rdoc_options: 
@@ -152,7 +251,6 @@ extra_rdoc_files:
 - howtouse.txt
 - History.txt
 - testing.txt
-- test/data/gemlist.txt
 executables: []
 
 extensions: []
@@ -165,9 +263,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Version::Requirement 
     requirements: 
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.0.0
+        version: 0.2.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -176,5 +274,5 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.5.0
+        version: 1.5.1
     version: