rexical 1.0.5 → 1.0.8

data/README.rdoc

@@ -1,30 +1,33 @@
 = Rexical
 
-* http://github.com/tenderlove/rexical/tree/master
+home :: http://github.com/sparklemotion/rexical/tree/master
 
 == DESCRIPTION
 
-Rexical is a lexical scanner generator.
-It is written in Ruby itself, and generates Ruby program.
-It is designed for use with Racc.
+Rexical is a lexical scanner generator that is used with
+Racc to generate Ruby programs.
+Rexical is written in Ruby.
 
 
 == SYNOPSIS
 
-Here is a sample lexical definition:
+Several examples of Rexical grammar files are provided in
+the sample directory.
+
+Here is an example of a lexical definition:
 
   class Sample
   macro
     BLANK         [\ \t]+
   rule
-    BLANK         # no action
+    {BLANK}       # no action
     \d+           { [:digit, text.to_i] }
     \w+           { [:word, text] }
     \n
     .             { [text, text] }
   end
 
-Here is the command line usage:
+Here are examples of the command line usage:
 
   $ rex  sample1.rex  --stub
   $ ruby sample1.rex.rb  sample1.c
@@ -36,6 +39,9 @@ Here is the command line usage:
   $ rex  calc3.rex
   $ ruby calc3.tab.rb
 
+The description files for lexical analysis in the sample directory
+are the files ending with the .rex extension.
+
 == REQUIREMENTS
 
 *  ruby version 1.8.x or later.
@@ -50,3 +56,5 @@ Rexical is distributed under the terms of the GNU Lesser General
 Public License version 2. Note that you do NOT need to follow
 LGPL for your own parser (Rexical outputs). You can provide those
 files under any licenses you want.
+
+See COPYING for more details.

data/Rakefile

@@ -1,29 +1,3 @@
-# -*- ruby -*-
+require 'minitest/test_task'
 
-require 'rubygems'
-require 'hoe'
-
-Hoe.plugin :debugging
-Hoe.plugin :git
-Hoe.plugins.delete :rubyforge
-
-Hoe.spec 'rexical' do
-  self.readme_file  = 'README.rdoc'
-  self.history_file = 'CHANGELOG.rdoc'
-  developer('Aaron Patterson', 'aaronp@rubyforge.org')
-  self.rubyforge_name = 'ruby-rex'
-  self.extra_rdoc_files = FileList['*.rdoc']
-end
-
-namespace :gem do
-  namespace :spec do
-    task :dev do
-      File.open("#{HOE.name}.gemspec", 'w') do |f|
-        HOE.spec.version = "#{HOE.version}.#{Time.now.strftime("%Y%m%d%H%M%S")}"
-        f.write(HOE.spec.to_ruby)
-      end
-    end
-  end
-end
-
-# vim: syntax=Ruby
+Minitest::TestTask.create

data/lib/rexical/generator.rb

@@ -11,13 +11,9 @@
 require 'strscan'
 module Rexical
 
-## ---------------------------------------------------------------------
   class ParseError < StandardError ; end
 
-## ---------------------------------------------------------------------
   class Generator
-
-## ---------------------------------------------------------------------
     attr_accessor :grammar_file
     attr_accessor :grammar_lines
     attr_accessor :scanner_file
@@ -30,7 +26,6 @@ module Rexical
     attr_accessor :independent
     attr_accessor :debug
 
-## ---------------------------------------------------------------------
     def initialize(opts)
       @lineno  =  0
       @macro  =  {}
@@ -43,22 +38,18 @@ module Rexical
       @opt  =  opts
     end
 
-## ---------------------------------------------------------------------
     def add_header( st )
       @scanner_header  +=  "#{st}\n"
     end
 
-## ---------------------------------------------------------------------
     def add_footer( st )
       @scanner_footer  +=  "#{st}\n"
     end
 
-## ---------------------------------------------------------------------
     def add_inner( st )
       @scanner_inner  +=  "#{st}\n"
     end
 
-## ---------------------------------------------------------------------
     def add_option( st )
       opts = st.split
       opts.each do |opt|
@@ -69,11 +60,12 @@ module Rexical
           @opt['--stub'] = true
         when /independent/i
           @opt['--independent'] = true
+        when /matcheos/i
+          @opt['--matcheos'] = true
         end
       end
     end
 
-## ---------------------------------------------------------------------
     def add_macro( st )
       ss  =  StringScanner.new(st)
       ss.scan(/\s+/)
@@ -105,7 +97,6 @@ module Rexical
       raise ParseError, "parse error in add_macro:'#{st}'"
     end
 
-## ---------------------------------------------------------------------
     def add_rule( rule_state, rule_expr, rule_action=nil )
       st = rule_expr.dup
       @macro.each_pair do |k, e|
@@ -218,16 +209,11 @@ module Rexical
               end
 
             end # case state3
-
           end # case state2
-
         end # case state1
-
       end # while
-
     end
 
-## ---------------------------------------------------------------------
     def parse_rule(st)
       st.strip!
       return  if st.size == 0 or st[0,1] == '#'
@@ -240,7 +226,6 @@ module Rexical
       [rule_state, rule_expr, ss.post_match]
     end
 
-## ---------------------------------------------------------------------
     def parse_action(st, lastmodes=[])
       modes  =  lastmodes
       mode  =  lastmodes[-1]
@@ -293,89 +278,83 @@ module Rexical
       return  modes
     end
 
-## ---------------------------------------------------------------------
-
-REX_HEADER = <<-REX_EOT
-#--
-# DO NOT MODIFY!!!!
-# This file is automatically generated by rex %s
-# from lexical definition file "%s".
-#++
-
-REX_EOT
-
-REX_UTIL = <<-REX_EOT
-  require 'strscan'
+    REX_HEADER = <<-REX_EOT.gsub(/^ {6}/, '')
+      #--
+      # DO NOT MODIFY!!!!
+      # This file is automatically generated by rex %s
+      # from lexical definition file "%s".
+      #++
 
-  class ScanError < StandardError ; end
+    REX_EOT
 
-  attr_reader   :lineno
-  attr_reader   :filename
-  attr_accessor :state
+    REX_UTIL = <<-REX_EOT
+      require 'strscan'
 
-  def scan_setup(str)
-    @ss = StringScanner.new(str)
-    @lineno =  1
-    @state  = nil
-  end
+      class ScanError < StandardError ; end
 
-  def action
-    yield
-  end
+      attr_reader   :lineno
+      attr_reader   :filename
+      attr_accessor :state
 
-  def scan_str(str)
-    scan_setup(str)
-    do_parse
-  end
-  alias :scan :scan_str
+      def scan_setup(str)
+        @ss = StringScanner.new(str)
+        @lineno =  1
+        @state  = nil
+      end
 
-  def load_file( filename )
-    @filename = filename
-    open(filename, "r") do |f|
-      scan_setup(f.read)
-    end
-  end
+      def action
+        yield
+      end
 
-  def scan_file( filename )
-    load_file(filename)
-    do_parse
-  end
+      def scan_str(str)
+        scan_setup(str)
+        do_parse
+      end
+      alias :scan :scan_str
 
-REX_EOT
+      def load_file( filename )
+        @filename = filename
+        File.open(filename, "r") do |f|
+          scan_setup(f.read)
+        end
+      end
 
-REX_STUB = <<-REX_EOT
+      def scan_file( filename )
+        load_file(filename)
+        do_parse
+      end
 
-if __FILE__ == $0
-  exit  if ARGV.size != 1
-  filename = ARGV.shift
-  rex = %s.new
-  begin
-    rex.load_file  filename
-    while  token = rex.next_token
-      p token
-    end
-  rescue
-    $stderr.printf  %s, rex.filename, rex.lineno, $!.message
-  end
-end
-REX_EOT
+    REX_EOT
 
-## ---------------------------------------------------------------------
+    REX_STUB = <<-REX_EOT
 
+      if __FILE__ == $0
+        exit  if ARGV.size != 1
+        filename = ARGV.shift
+        rex = %s.new
+        begin
+          rex.load_file  filename
+          while  token = rex.next_token
+            p token
+          end
+        rescue
+          $stderr.printf  %s, rex.filename, rex.lineno, $!.message
+        end
+      end
+    REX_EOT
 
     def scanner_io
       unless  scanner_file = @opt['--output-file']
         scanner_file  =  grammar_file + ".rb"
       end
-      f = File.open(scanner_file, 'w')
+      File.open(scanner_file, 'wb')
     end
     private :scanner_io
 
     def write_scanner f = scanner_io
-      ## scan flag
       flag = ""
       flag += "i"  if @opt['--ignorecase']
-      ## header
+
       f.printf REX_HEADER, Rexical::VERSION, grammar_file
 
       unless @opt['--independent']
@@ -391,120 +370,127 @@ REX_EOT
         f.puts "class #{@class_name} < Racc::Parser"
       end
 
-      ## utility method
       f.print REX_UTIL
 
-      ## scanner method
+      eos_check = @opt["--matcheos"] ? "" : "return if @ss.eos?"
 
+      ## scanner method
       f.print <<-REX_EOT
 
-  def next_token
-    return if @ss.eos?
-    
-    # skips empty actions
-    until token = _next_token or @ss.eos?; end
-    token
-  end
-
-  def _next_token
-    text = @ss.peek(1)
-    @lineno  +=  1  if text == "\\n"
-    token = case @state
-    REX_EOT
+        def next_token
+          #{eos_check}
 
-    exclusive_states.each do |es|
-      f.printf <<-REX_EOT
-    when #{es ? es.to_s : "nil"}
-      case
+          # skips empty actions
+          until token = _next_token or @ss.eos?; end
+          token
+        end
+
+        def _next_token
+          text = @ss.peek(1)
+          @lineno  +=  1  if text == "\\n"
+          token = case @state
       REX_EOT
-      rules.each do |rule|
-        exclusive_state, start_state, rule_expr, rule_action = *rule
-        if es == exclusive_state
 
-          if rule_action
-            if start_state
-              f.print <<-REX_EOT
-      when((state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag})))
-         action #{rule_action}
+      exclusive_states.each do |es|
+        if es.nil?
+          f.printf <<-REX_EOT
+            when #{(["nil"] + rules.collect{ |rule| rule[1].nil? ? "nil" : rule[1] }).uniq.join(', ')}
+          REX_EOT
+        else
+          f.printf <<-REX_EOT
+            when #{es}
+          REX_EOT
+        end
+        f.printf <<-REX_EOT
+          case
+        REX_EOT
 
-              REX_EOT
-            else
-              f.print <<-REX_EOT
-      when (text = @ss.scan(/#{rule_expr}/#{flag}))
-         action #{rule_action}
+        rules.each do |rule|
+          exclusive_state, start_state, rule_expr, rule_action = *rule
+          if es == exclusive_state
 
-              REX_EOT
-            end
-          else
-            if start_state
-              f.print <<-REX_EOT
-      when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
-        ;
+            if rule_action
+              if start_state
+                f.print <<-REX_EOT
+                  when((state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag})))
+                     action #{rule_action}
+
+                REX_EOT
+              else
+                f.print <<-REX_EOT
+                  when (text = @ss.scan(/#{rule_expr}/#{flag}))
+                     action #{rule_action}
 
-              REX_EOT
+                REX_EOT
+              end
             else
-              f.print <<-REX_EOT
-      when (text = @ss.scan(/#{rule_expr}/#{flag}))
-        ;
+              if start_state
+                f.print <<-REX_EOT
+                  when (@state == #{start_state}) && (text = @ss.scan(/#{rule_expr}/#{flag}))
+                    ;
+
+                REX_EOT
+              else
+                f.print <<-REX_EOT
+                  when (text = @ss.scan(/#{rule_expr}/#{flag}))
+                    ;
 
-              REX_EOT
+                REX_EOT
+              end
             end
-          end
 
+          end
+        end # rules.each
+
+        if @opt["--matcheos"]
+          eos_check = <<-REX_EOT
+            when @@ss.scan(/$/)
+               ;
+          REX_EOT
+        else
+          eos_check = ""
         end
-      end
-      f.print <<-REX_EOT
-      else
-        text = @ss.string[@ss.pos .. -1]
-        raise  ScanError, "can not match: '" + text + "'"
-      end  # if
 
-      REX_EOT
-    end
-    f.print <<-REX_EOT
-    else
-      raise  ScanError, "undefined state: '" + state.to_s + "'"
-    end  # case state
-      REX_EOT
-    if @opt['--debug']
+        f.print <<-REX_EOT
+          #{eos_check}
+          else
+            text = @ss.string[@ss.pos .. -1]
+            raise  ScanError, "can not match: '" + text + "'"
+          end  # if
+
+        REX_EOT
+      end # exclusive_states.each
+
       f.print <<-REX_EOT
-    p token
+        else
+          raise  ScanError, "undefined state: '" + state.to_s + "'"
+        end  # case state
       REX_EOT
-    end
-    f.print <<-REX_EOT
-    token
-  end  # def _next_token
+
+      if @opt['--debug']
+        f.print <<-REX_EOT
+          p token
+        REX_EOT
+      end
+
+      f.print <<-REX_EOT
+          token
+        end  # def _next_token
 
       REX_EOT
 
-      ## inner method
       @scanner_inner.each_line do |s|
         f.print s
       end
       f.puts "end # class"
 
-      ## footer
       @scanner_footer.each_line do |s|
         f.print s
-      end # case
+      end
 
-      ## stub main
       f.printf REX_STUB, @class_name, '"%s:%d:%s\n"'  if @opt['--stub']
       f.close
 
-    end ## def
-  end ## class
-end ## module
-
-
-## ---------------------------------------------------------------------
-## test
-
-if __FILE__ == $0
-  rex = Rexical::Generator.new(nil)
-  rex.grammar_file = "sample.rex"
-  rex.read_grammar
-  rex.parse
-  rex.write_scanner
-end
-
+    end ## def write_scanner
+  end ## class Generator
+end ## module Rexical

data/lib/rexical/rexcmd.rb

@@ -22,6 +22,7 @@ o -s --stub             - append stub code for debug
 o -i --ignorecase       - ignore char case
 o -C --check-only       - syntax check only
 o -  --independent      - independent mode
+o -  --matcheos         - allow match against end of string
 o -d --debug            - print debug information
 o -h --help             - print this message and quit
 o -  --version          - print version and quit
@@ -65,7 +66,8 @@ EOT
     @cmd  =  File.basename($0, ".rb")
     tmp  =  OPTIONS.lines.collect do |line|
         next if /\A\s*\z/ === line
-        disp, sopt, lopt, takearg, doc  =  line.strip.split(/\s+/, 5)
+        # disp, sopt, lopt, takearg, doc
+        _, sopt, lopt, takearg, _  =  line.strip.split(/\s+/, 5)
         a  =  []
         a.push lopt    unless lopt == '-'
         a.push sopt    unless sopt == '-'
@@ -83,7 +85,7 @@ EOT
             "#{@cmd}: #{name} given twice" if @opt.key? name
         @opt[name]  =  arg.empty? ? true : arg
       end
-    rescue GetoptLong::AmbigousOption, GetoptLong::InvalidOption,
+    rescue GetoptLong::AmbiguousOption, GetoptLong::InvalidOption,
            GetoptLong::MissingArgument, GetoptLong::NeedlessArgument
       usage $!.message
     end

data/lib/rexical/version.rb

@@ -0,0 +1,5 @@
+module Rexical
+  VERSION = "1.0.8"
+  Copyright = "Copyright (c) 2005-2006 ARIMA Yasuhiro"
+  Mailto = "arima.yasuhiro@nifty.com"
+end

data/lib/rexical.rb

@@ -1,8 +1,3 @@
-require 'rexical/generator'
-require 'rexical/rexcmd'
-
-module Rexical
-  VERSION = "1.0.5"
-  Copyright = 'Copyright (c) 2005-2006 ARIMA Yasuhiro'
-  Mailto    = 'arima.yasuhiro@nifty.com'
-end
+require_relative "rexical/generator"
+require_relative "rexical/rexcmd"
+require_relative "rexical/version"

data/sample/calc3.tab.rb

@@ -1,7 +1,7 @@
 #
 # DO NOT MODIFY!!!!
 # This file is automatically generated by racc 1.4.4
-# from racc grammer file "calc3.racc".
+# from racc grammar file "calc3.racc".
 #
 
 require 'racc/parser'

data/sample/error1.rex

@@ -7,7 +7,7 @@ class Error1
 macro
   BLANK         [\ \t]+
 rule
-  BLANK         # no action
+  {BLANK}       # no action
   \d+           { [:digit, text.to_i] }
   \w+           { [:word, text] }
   \n

data/sample/error2.rex

@@ -7,9 +7,9 @@ class Error2
 macro
   BLANK         [\ \t]+
 rule
-  BLANK         # no action
+  {BLANK}       # no action
   \d+           { [:digit, text.to_i] }
   \w+           { [:word, text] }
   \n
-  .             { state = :NONDEF ; [text, text] }
+  .             { self.state = :NONDEF ; [text, text] }
 end

data/sample/sample.rex

@@ -7,7 +7,7 @@ class Sample
 macro
   BLANK         [\ \t]+
 rule
-  BLANK         # no action
+  {BLANK}       # no action
   \d+           { [:digit, text.to_i] }
   \w+           { [:word, text] }
   \n