rexical 1.0.5 → 1.0.7

data/README.rdoc

@@ -4,27 +4,30 @@
 
 == DESCRIPTION
 
-Rexical is a lexical scanner generator.
-It is written in Ruby itself, and generates Ruby program.
-It is designed for use with Racc.
+Rexical is a lexical scanner generator that is used with
+Racc to generate Ruby programs.
+Rexical is written in Ruby.
 
 
 == SYNOPSIS
 
-Here is a sample lexical definition:
+Several examples of Rexical grammar files are provided in
+the sample directory.
+
+Here is an example of a lexical definition:
 
   class Sample
   macro
     BLANK         [\ \t]+
   rule
-    BLANK         # no action
+    {BLANK}       # no action
     \d+           { [:digit, text.to_i] }
     \w+           { [:word, text] }
     \n
     .             { [text, text] }
   end
 
-Here is the command line usage:
+Here are examples of the command line usage:
 
   $ rex  sample1.rex  --stub
   $ ruby sample1.rex.rb  sample1.c
@@ -36,6 +39,9 @@ Here is the command line usage:
   $ rex  calc3.rex
   $ ruby calc3.tab.rb
 
+The description files for lexical analysis in the sample directory
+are the files ending with the .rex extension.
+
 == REQUIREMENTS
 
 *  ruby version 1.8.x or later.
@@ -50,3 +56,4 @@ Rexical is distributed under the terms of the GNU Lesser General
 Public License version 2. Note that you do NOT need to follow
 LGPL for your own parser (Rexical outputs). You can provide those
 files under any licenses you want.
+

data/Rakefile

@@ -11,8 +11,10 @@ Hoe.spec 'rexical' do
   self.readme_file  = 'README.rdoc'
   self.history_file = 'CHANGELOG.rdoc'
   developer('Aaron Patterson', 'aaronp@rubyforge.org')
-  self.rubyforge_name = 'ruby-rex'
   self.extra_rdoc_files = FileList['*.rdoc']
+  self.extra_dev_deps += [
+    ["rubocop", "~> 0.74.0"]
+  ]
 end
 
 namespace :gem do

data/lib/rexical.rb

@@ -1,8 +1,8 @@
-require 'rexical/generator'
-require 'rexical/rexcmd'
+require "rexical/generator"
+require "rexical/rexcmd"
 
 module Rexical
-  VERSION = "1.0.5"
-  Copyright = 'Copyright (c) 2005-2006 ARIMA Yasuhiro'
-  Mailto    = 'arima.yasuhiro@nifty.com'
+  VERSION = "1.0.7"
+  Copyright = "Copyright (c) 2005-2006 ARIMA Yasuhiro"
+  Mailto = "arima.yasuhiro@nifty.com"
 end

data/lib/rexical/generator.rb

@@ -11,13 +11,9 @@
 require 'strscan'
 module Rexical
 
-## ---------------------------------------------------------------------
   class ParseError < StandardError ; end
 
-## ---------------------------------------------------------------------
   class Generator
-
-## ---------------------------------------------------------------------
     attr_accessor :grammar_file
     attr_accessor :grammar_lines
     attr_accessor :scanner_file
@@ -30,7 +26,6 @@ module Rexical
     attr_accessor :independent
     attr_accessor :debug
 
-## ---------------------------------------------------------------------
     def initialize(opts)
       @lineno  =  0
       @macro  =  {}
@@ -43,22 +38,18 @@ module Rexical
       @opt  =  opts
     end
 
-## ---------------------------------------------------------------------
     def add_header( st )
       @scanner_header  +=  "#{st}\n"
     end
 
-## ---------------------------------------------------------------------
     def add_footer( st )
       @scanner_footer  +=  "#{st}\n"
     end
 
-## ---------------------------------------------------------------------
     def add_inner( st )
       @scanner_inner  +=  "#{st}\n"
     end
 
-## ---------------------------------------------------------------------
     def add_option( st )
       opts = st.split
       opts.each do |opt|
@@ -69,11 +60,12 @@ module Rexical
           @opt['--stub'] = true
         when /independent/i
           @opt['--independent'] = true
+        when /matcheos/i
+          @opt['--matcheos'] = true
         end
       end
     end
 
-## ---------------------------------------------------------------------
     def add_macro( st )
       ss  =  StringScanner.new(st)
       ss.scan(/\s+/)
@@ -105,7 +97,6 @@ module Rexical
       raise ParseError, "parse error in add_macro:'#{st}'"
     end
 
-## ---------------------------------------------------------------------
     def add_rule( rule_state, rule_expr, rule_action=nil )
       st = rule_expr.dup
       @macro.each_pair do |k, e|
@@ -218,16 +209,11 @@ module Rexical
               end
 
             end # case state3
-
           end # case state2
-
         end # case state1
-
       end # while
-
     end
 
-## ---------------------------------------------------------------------
     def parse_rule(st)
       st.strip!
       return  if st.size == 0 or st[0,1] == '#'
@@ -240,7 +226,6 @@ module Rexical
       [rule_state, rule_expr, ss.post_match]
     end
 
-## ---------------------------------------------------------------------
     def parse_action(st, lastmodes=[])
       modes  =  lastmodes
       mode  =  lastmodes[-1]
@@ -293,89 +278,83 @@ module Rexical
       return  modes
     end
 
-## ---------------------------------------------------------------------
-
-REX_HEADER = <<-REX_EOT
-#--
-# DO NOT MODIFY!!!!
-# This file is automatically generated by rex %s
-# from lexical definition file "%s".
-#++
-
-REX_EOT
-
-REX_UTIL = <<-REX_EOT
-  require 'strscan'
+    REX_HEADER = <<-REX_EOT.gsub(/^ {6}/, '')
+      #--
+      # DO NOT MODIFY!!!!
+      # This file is automatically generated by rex %s
+      # from lexical definition file "%s".
+      #++
 
-  class ScanError < StandardError ; end
+    REX_EOT
 
-  attr_reader   :lineno
-  attr_reader   :filename
-  attr_accessor :state
+    REX_UTIL = <<-REX_EOT
+      require 'strscan'
 
-  def scan_setup(str)
-    @ss = StringScanner.new(str)
-    @lineno =  1
-    @state  = nil
-  end
+      class ScanError < StandardError ; end
 
-  def action
-    yield
-  end
+      attr_reader   :lineno
+      attr_reader   :filename
+      attr_accessor :state
 
-  def scan_str(str)
-    scan_setup(str)
-    do_parse
-  end
-  alias :scan :scan_str
+      def scan_setup(str)
+        @ss = StringScanner.new(str)
+        @lineno =  1
+        @state  = nil
+      end
 
-  def load_file( filename )
-    @filename = filename
-    open(filename, "r") do |f|
-      scan_setup(f.read)
-    end
-  end
+      def action
+        yield
+      end
 
-  def scan_file( filename )
-    load_file(filename)
-    do_parse
-  end
+      def scan_str(str)
+        scan_setup(str)
+        do_parse
+      end
+      alias :scan :scan_str
 
-REX_EOT
+      def load_file( filename )
+        @filename = filename
+        File.open(filename, "r") do |f|
+          scan_setup(f.read)
+        end
+      end
 
-REX_STUB = <<-REX_EOT
+      def scan_file( filename )
+        load_file(filename)
+        do_parse
+      end
 
-if __FILE__ == $0
-  exit  if ARGV.size != 1
-  filename = ARGV.shift
-  rex = %s.new
-  begin
-    rex.load_file  filename
-    while  token = rex.next_token
-      p token
-    end
-  rescue
-    $stderr.printf  %s, rex.filename, rex.lineno, $!.message
-  end
-end
-REX_EOT
+    REX_EOT
 
-## ---------------------------------------------------------------------
+    REX_STUB = <<-REX_EOT
 
+      if __FILE__ == $0
+        exit  if ARGV.size != 1
+        filename = ARGV.shift
+        rex = %s.new
+        begin
+          rex.load_file  filename
+          while  token = rex.next_token
+            p token
+          end
+        rescue
+          $stderr.printf  %s, rex.filename, rex.lineno, $!.message
+        end
+      end
+    REX_EOT
 
     def scanner_io
       unless  scanner_file = @opt['--output-file']
         scanner_file  =  grammar_file + ".rb"
       end
-      f = File.open(scanner_file, 'w')
+      File.open(scanner_file, 'wb')
     end
     private :scanner_io
 
     def write_scanner f = scanner_io
-      ## scan flag
       flag = ""
       flag += "i"  if @opt['--ignorecase']
-      ## header
+
       f.printf REX_HEADER, Rexical::VERSION, grammar_file
 
       unless @opt['--independent']
@@ -391,120 +370,127 @@ REX_EOT
         f.puts "class #{@class_name} < Racc::Parser"
       end
 
-      ## utility method
       f.print REX_UTIL
 
-      ## scanner method
+      eos_check = @opt["--matcheos"] ? "" : "return if @ss.eos?"
 
+      ## scanner method
       f.print <<-REX_EOT
 
-  def next_token
-    return if @ss.eos?
-    
-    # skips empty actions
-    until token = _next_token or @ss.eos?; end
-    token
-  end
-
-  def _next_token
-    text = @ss.peek(1)
-    @lineno  +=  1  if text == "\\n"
-    token = case @state
-    REX_EOT
+        def next_token
+          #{eos_check}
 
-    exclusive_states.each do |es|
-      f.printf <<-REX_EOT
-    when #{es ? es.to_s : "nil"}
-      case
+          # skips empty actions
+          until token = _next_token or @ss.eos?; end
+          token
+        end
+
+        def _next_token
+          text = @ss.peek(1)
+          @lineno  +=  1  if text == "\\n"
+          token = case @state
       REX_EOT
-      rules.each do |rule|
-        exclusive_state, start_state, rule_expr, rule_action = *rule
-        if es == exclusive_state
 
-          if rule_action
-            if start_state
-              f.print <<-REX_EOT
-      when((state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag})))
-         action #{rule_action}
+      exclusive_states.each do |es|
+        if es.nil?
+          f.printf <<-REX_EOT
+            when #{(["nil"] + rules.collect{ |rule| rule[1].nil? ? "nil" : rule[1] }).uniq.join(', ')}
+          REX_EOT
+        else
+          f.printf <<-REX_EOT
+            when #{es}
+          REX_EOT
+        end
+        f.printf <<-REX_EOT
+          case
+        REX_EOT
 
-              REX_EOT
-            else
-              f.print <<-REX_EOT
-      when (text = @ss.scan(/#{rule_expr}/#{flag}))
-         action #{rule_action}
+        rules.each do |rule|
+          exclusive_state, start_state, rule_expr, rule_action = *rule
+          if es == exclusive_state
 
-              REX_EOT
-            end
-          else
-            if start_state
-              f.print <<-REX_EOT
-      when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
-        ;
+            if rule_action
+              if start_state
+                f.print <<-REX_EOT
+                  when((state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag})))
+                     action #{rule_action}
+
+                REX_EOT
+              else
+                f.print <<-REX_EOT
+                  when (text = @ss.scan(/#{rule_expr}/#{flag}))
+                     action #{rule_action}
 
-              REX_EOT
+                REX_EOT
+              end
             else
-              f.print <<-REX_EOT
-      when (text = @ss.scan(/#{rule_expr}/#{flag}))
-        ;
+              if start_state
+                f.print <<-REX_EOT
+                  when (@state == #{start_state}) && (text = @ss.scan(/#{rule_expr}/#{flag}))
+                    ;
+
+                REX_EOT
+              else
+                f.print <<-REX_EOT
+                  when (text = @ss.scan(/#{rule_expr}/#{flag}))
+                    ;
 
-              REX_EOT
+                REX_EOT
+              end
             end
-          end
 
+          end
+        end # rules.each
+
+        if @opt["--matcheos"]
+          eos_check = <<-REX_EOT
+            when @@ss.scan(/$/)
+               ;
+          REX_EOT
+        else
+          eos_check = ""
         end
-      end
-      f.print <<-REX_EOT
-      else
-        text = @ss.string[@ss.pos .. -1]
-        raise  ScanError, "can not match: '" + text + "'"
-      end  # if
 
-      REX_EOT
-    end
-    f.print <<-REX_EOT
-    else
-      raise  ScanError, "undefined state: '" + state.to_s + "'"
-    end  # case state
-      REX_EOT
-    if @opt['--debug']
+        f.print <<-REX_EOT
+          #{eos_check}
+          else
+            text = @ss.string[@ss.pos .. -1]
+            raise  ScanError, "can not match: '" + text + "'"
+          end  # if
+
+        REX_EOT
+      end # exclusive_states.each
+
       f.print <<-REX_EOT
-    p token
+        else
+          raise  ScanError, "undefined state: '" + state.to_s + "'"
+        end  # case state
       REX_EOT
-    end
-    f.print <<-REX_EOT
-    token
-  end  # def _next_token
+
+      if @opt['--debug']
+        f.print <<-REX_EOT
+          p token
+        REX_EOT
+      end
+
+      f.print <<-REX_EOT
+          token
+        end  # def _next_token
 
       REX_EOT
 
-      ## inner method
       @scanner_inner.each_line do |s|
         f.print s
       end
       f.puts "end # class"
 
-      ## footer
       @scanner_footer.each_line do |s|
         f.print s
-      end # case
+      end
 
-      ## stub main
       f.printf REX_STUB, @class_name, '"%s:%d:%s\n"'  if @opt['--stub']
       f.close
 
-    end ## def
-  end ## class
-end ## module
-
-
-## ---------------------------------------------------------------------
-## test
-
-if __FILE__ == $0
-  rex = Rexical::Generator.new(nil)
-  rex.grammar_file = "sample.rex"
-  rex.read_grammar
-  rex.parse
-  rex.write_scanner
-end
-
+    end ## def write_scanner
+  end ## class Generator
+end ## module Rexical