rexical 1.0.5 → 1.0.8

data/sample/sample.rex.rb

@@ -1,100 +1,92 @@
-#
-# DO NOT MODIFY!!!!
-# This file is automatically generated by rex 1.0.0
-# from lexical definition file "sample.rex".
-#
-
-require 'racc/parser'
-#
-# sample.rex
-# lexical definition sample for rex
-#
-
-class Sample < Racc::Parser
-  require 'strscan'
-
-  class ScanError < StandardError ; end
-
-  attr_reader :lineno
-  attr_reader :filename
-
-  def scan_setup ; end
-
-  def action &block
-    yield
-  end
-
-  def scan_str( str )
-    scan_evaluate  str
-    do_parse
-  end
-
-  def load_file( filename )
-    @filename = filename
-    open(filename, "r") do |f|
-      scan_evaluate  f.read
-    end
-  end
-
-  def scan_file( filename )
-    load_file  filename
-    do_parse
-  end
-
-  def next_token
-    @rex_tokens.shift
-  end
-
-  def scan_evaluate( str )
-    scan_setup
-    @rex_tokens = []
-    @lineno  =  1
-    ss = StringScanner.new(str)
-    state = nil
-    until ss.eos?
-      text = ss.peek(1)
-      @lineno  +=  1  if text == "\n"
-      case state
-      when nil
-        case
-        when (text = ss.scan(/BLANK/))
-          ;
-
-        when (text = ss.scan(/\d+/))
-           @rex_tokens.push action { [:digit, text.to_i] }
-
-        when (text = ss.scan(/\w+/))
-           @rex_tokens.push action { [:word, text] }
-
-        when (text = ss.scan(/\n/))
-          ;
-
-        when (text = ss.scan(/./))
-           @rex_tokens.push action { [text, text] }
-
-        else
-          text = ss.string[ss.pos .. -1]
-          raise  ScanError, "can not match: '" + text + "'"
-        end  # if
-
-      else
-        raise  ScanError, "undefined state: '" + state.to_s + "'"
-      end  # case state
-    end  # until ss
-  end  # def scan_evaluate
-
-end # class
-
-if __FILE__ == $0
-  exit  if ARGV.size != 1
-  filename = ARGV.shift
-  rex = Sample.new
-  begin
-    rex.load_file  filename
-    while  token = rex.next_token
-      p token
-    end
-  rescue
-    $stderr.printf  "%s:%d:%s\n", rex.filename, rex.lineno, $!.message
-  end
-end
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.5
+# from lexical definition file "sample/sample.rex".
+#++
+
+require 'racc/parser'
+#
+# sample.rex
+# lexical definition sample for rex
+#
+
+class Sample < Racc::Parser
+      require 'strscan'
+
+      class ScanError < StandardError ; end
+
+      attr_reader   :lineno
+      attr_reader   :filename
+      attr_accessor :state
+
+      def scan_setup(str)
+        @ss = StringScanner.new(str)
+        @lineno =  1
+        @state  = nil
+      end
+
+      def action
+        yield
+      end
+
+      def scan_str(str)
+        scan_setup(str)
+        do_parse
+      end
+      alias :scan :scan_str
+
+      def load_file( filename )
+        @filename = filename
+        open(filename, "r") do |f|
+          scan_setup(f.read)
+        end
+      end
+
+      def scan_file( filename )
+        load_file(filename)
+        do_parse
+      end
+
+
+        def next_token
+          return if @ss.eos?
+
+          # skips empty actions
+          until token = _next_token or @ss.eos?; end
+          token
+        end
+
+        def _next_token
+          text = @ss.peek(1)
+          @lineno  +=  1  if text == "\n"
+          token = case @state
+            when nil
+          case
+                  when (text = @ss.scan(/[ \t]+/))
+                    ;
+
+                  when (text = @ss.scan(/\d+/))
+                     action { [:digit, text.to_i] }
+
+                  when (text = @ss.scan(/\w+/))
+                     action { [:word, text] }
+
+                  when (text = @ss.scan(/\n/))
+                    ;
+
+                  when (text = @ss.scan(/./))
+                     action { [text, text] }
+
+          
+          else
+            text = @ss.string[@ss.pos .. -1]
+            raise  ScanError, "can not match: '" + text + "'"
+          end  # if
+
+        else
+          raise  ScanError, "undefined state: '" + state.to_s + "'"
+        end  # case state
+          token
+        end  # def _next_token
+
+end # class

data/sample/sample1.rex

@@ -19,11 +19,11 @@ rule
 # [:state]  pattern  [actions]
 
 # remark
-                {REM_IN}        { state = :REMS; [:rem_in, text] }
-  :REMS         {REM_OUT}       { state = nil;   [:rem_out, text] }
+                {REM_IN}        { self.state = :REMS; [:rem_in, text] }
+  :REMS         {REM_OUT}       { self.state = nil;   [:rem_out, text] }
   :REMS         .*(?={REM_OUT}) {                [:remark, text] }
-                {REM}           { state = :REM;  [:rem_in, text] }
-  :REM          \n              { state = nil;   [:rem_out, text] }
+                {REM}           { self.state = :REM;  [:rem_in, text] }
+  :REM          \n              { self.state = nil;   [:rem_out, text] }
   :REM          .*(?=$)         {                [:remark, text] }
 
 # literal

data/sample/sample2.rex

@@ -16,8 +16,8 @@ macro
   REMARK        \'              # '
 
 rule
-                {REMARK}        { state = :REM;  [:rem_in, text] } # '
-  :REM          \n              { state = nil;   [:rem_out, text] }
+                {REMARK}        { self.state = :REM;  [:rem_in, text] } # '
+  :REM          \n              { self.state = nil;   [:rem_out, text] }
   :REM          .*(?=$)         {                [:remark, text] }
 
                 \"[^"]*\"       { [:string, text] } # "

data/sample/xhtmlparser.rex

@@ -28,34 +28,34 @@ macro
 rule
 
 # [:state]  pattern  [actions]
-                {XTAG_IN}               { state = :TAG; [:xtag_in, text] }
-                {ETAG_IN}               { state = :TAG; [:etag_in, text] }
-                {TAG_IN}                { state = :TAG; [:tag_in, text] }
-  :TAG          {EXT}                   { state = :EXT; [:ext, text] }
+                {XTAG_IN}               { self.state = :TAG; [:xtag_in, text] }
+                {ETAG_IN}               { self.state = :TAG; [:etag_in, text] }
+                {TAG_IN}                { self.state = :TAG; [:tag_in, text] }
+  :TAG          {EXT}                   { self.state = :EXT; [:ext, text] }
 
-  :EXT          {REM}                   { state = :REM; [:rem_in, text] }
-  :EXT          {XTAG_OUT}              { state = nil;  [:xtag_out, text] }
-  :EXT          {TAG_OUT}               { state = nil;  [:tag_out, text] }
+  :EXT          {REM}                   { self.state = :REM; [:rem_in, text] }
+  :EXT          {XTAG_OUT}              { self.state = nil;  [:xtag_out, text] }
+  :EXT          {TAG_OUT}               { self.state = nil;  [:tag_out, text] }
   :EXT          .+(?={REM})             {               [:exttext, text] }
   :EXT          .+(?={TAG_OUT})         {               [:exttext, text] }
   :EXT          .+(?=$)                 {               [:exttext, text] }
   :EXT          \n
 
-  :REM          {REM}                   { state = :EXT; [:rem_out, text] }
+  :REM          {REM}                   { self.state = :EXT; [:rem_out, text] }
   :REM          .+(?={REM})             {               [:remtext, text] }
   :REM          .+(?=$)                 {               [:remtext, text] }
   :REM          \n
 
   :TAG          {BLANK}
-  :TAG          {XTAG_OUT}              { state = nil;  [:xtag_out, text] }
-  :TAG          {ETAG_OUT}              { state = nil;  [:etag_out, text] }
-  :TAG          {TAG_OUT}               { state = nil;  [:tag_out, text] }
+  :TAG          {XTAG_OUT}              { self.state = nil;  [:xtag_out, text] }
+  :TAG          {ETAG_OUT}              { self.state = nil;  [:etag_out, text] }
+  :TAG          {TAG_OUT}               { self.state = nil;  [:tag_out, text] }
   :TAG          {EQUAL}                 {               [:equal, text] }
-  :TAG          {Q1}                    { state = :Q1;  [:quote1, text] } # '
-  :Q1           {Q1}                    { state = :TAG; [:quote1, text] } # '
+  :TAG          {Q1}                    { self.state = :Q1;  [:quote1, text] } # '
+  :Q1           {Q1}                    { self.state = :TAG; [:quote1, text] } # '
   :Q1           [^{Q1}]+(?={Q1})        {               [:value, text] }  # '
-  :TAG          {Q2}                    { state = :Q2;  [:quote2, text] } # "
-  :Q2           {Q2}                    { state = :TAG; [:quote2, text] } # "
+  :TAG          {Q2}                    { self.state = :Q2;  [:quote2, text] } # "
+  :Q2           {Q2}                    { self.state = :TAG; [:quote2, text] } # "
   :Q2           [^{Q2}]+(?={Q2})        {               [:value, text] }  # "
 
   :TAG          [\w\-]+(?={EQUAL})      {               [:attr, text] }

data/test/assets/test.rex

@@ -5,8 +5,12 @@ class C < SomethingElse
 macro
   w         [\s\r\n\f]*
 
+rule
+
 # [:state]  pattern  [actions]
 
             {w}~={w}         { [:INCLUDES, text] }
 
 end
+end
+end

data/test/rex-20060125.rb

@@ -91,7 +91,7 @@ def get_options
           "#{$cmd}: #{name} given twice" if opt.key? name
       opt[name]  =  arg.empty? ? true : arg
     end
-  rescue GetoptLong::AmbigousOption, GetoptLong::InvalidOption,
+  rescue GetoptLong::AmbiguousOption, GetoptLong::InvalidOption,
          GetoptLong::MissingArgument, GetoptLong::NeedlessArgument
     usage 1, $!.message
   end

data/test/rex-20060511.rb

@@ -89,7 +89,7 @@ class RexRunner
             "#{@cmd}: #{name} given twice" if @opt.key? name
         @opt[name]  =  arg.empty? ? true : arg
       end
-    rescue GetoptLong::AmbigousOption, GetoptLong::InvalidOption,
+    rescue GetoptLong::AmbiguousOption, GetoptLong::InvalidOption,
            GetoptLong::MissingArgument, GetoptLong::NeedlessArgument
       usage $!.message
     end

data/test/test_generator.rb

@@ -1,9 +1,11 @@
-require 'test/unit'
+gem "minitest"
+require 'minitest/autorun'
 require 'tempfile'
 require 'rexical'
 require 'stringio'
+require 'open3'
 
-class TestGenerator < Test::Unit::TestCase
+class TestGenerator < Minitest::Test
   def test_header_is_written_after_module
     rex = Rexical::Generator.new(
       "--independent" => true
@@ -25,6 +27,29 @@ class TestGenerator < Test::Unit::TestCase
     assert_equal '#++', comments.last
   end
 
+  def test_rubocop_security
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_file = File.join File.dirname(__FILE__), 'assets', 'test.rex'
+    rex.read_grammar
+    rex.parse
+
+    output = Tempfile.new(["rex_output", ".rb"])
+    begin
+      rex.write_scanner output
+      output.close
+
+      stdin, stdoe, wait_thr = Open3.popen2e "rubocop --only Security #{output.path}"
+      if ! wait_thr.value.success?
+        fail stdoe.read
+      end
+    ensure
+      output.close
+      output.unlink
+    end
+  end
+
   def test_read_non_existent_file
     rex = Rexical::Generator.new(nil)
     rex.grammar_file = 'non_existent_file'
@@ -71,6 +96,22 @@ end
     assert_match 'Calculator < Foo::Bar', source
   end
 
+  def test_stateful_lexer
+    m = build_lexer %q{
+class Foo
+rule
+          \d      { @state = :digit; [:foo, text] }
+  :digit  \w      { @state = nil; [:w, text] }
+end
+    }
+    scanner = m::Foo.new
+    scanner.scan_setup('1w1')
+    assert_tokens [
+      [:foo, '1'],
+      [:w, 'w'],
+      [:foo, '1']], scanner
+  end
+
   def test_simple_scanner
     m = build_lexer %q{
 class Calculator
@@ -185,7 +226,7 @@ end
     calc2.scan_setup('ababa')
 
     # Doesn't lex all 'a's
-    assert_raise(lexer::Calculator::ScanError) { tokens(calc1) }
+    assert_raises(lexer::Calculator::ScanError) { tokens(calc1) }
 
     # Does lex alternating 'a's and 'b's
     calc2.scan_setup('ababa')
@@ -219,6 +260,21 @@ end
     calc.state = nil
     assert_equal [:A, 'a'], calc.next_token
   end
+  def test_match_eos
+    lexer = build_lexer %q{
+class Calculator
+option
+matcheos
+rule
+      a        { [:A, text] }
+      $        { [:EOF, ""] }
+:B    b        { [:B, text] }
+     }
+     calc = lexer::Calculator.new
+     calc.scan_setup("a")
+     assert_equal [:A, 'a'], calc.next_token
+     assert_equal [:EOF, ""], calc.next_token
+  end
 
   def parse_lexer(str)
     rex = Rexical::Generator.new("--independent" => true)

metadata

@@ -1,55 +1,43 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rexical
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 0
-  - 5
-  version: 1.0.5
+version: !ruby/object:Gem::Version
+  version: 1.0.8
 platform: ruby
-authors: 
+authors:
 - Aaron Patterson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2010-12-08 00:00:00 -08:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
+date: 2024-05-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: getoptlong
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 6
-        - 2
-        version: 2.6.2
-  type: :development
-  version_requirements: *id001
-description: |-
-  Rexical is a lexical scanner generator.
-  It is written in Ruby itself, and generates Ruby program.
-  It is designed for use with Racc.
-email: 
-- aaronp@rubyforge.org
-executables: 
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Rexical is a lexical scanner generator that is used with Racc to generate
+  Ruby programs. Rexical is written in Ruby.
+email:
+executables:
 - rex
 extensions: []
-
-extra_rdoc_files: 
-- Manifest.txt
+extra_rdoc_files:
 - CHANGELOG.rdoc
 - DOCUMENTATION.en.rdoc
 - DOCUMENTATION.ja.rdoc
 - README.rdoc
-files: 
+files:
 - CHANGELOG.rdoc
+- COPYING
 - DOCUMENTATION.en.rdoc
 - DOCUMENTATION.ja.rdoc
 - Manifest.txt
@@ -60,6 +48,7 @@ files:
 - lib/rexical.rb
 - lib/rexical/generator.rb
 - lib/rexical/rexcmd.rb
+- lib/rexical/version.rb
 - sample/a.cmd
 - sample/b.cmd
 - sample/c.cmd
@@ -85,38 +74,30 @@ files:
 - test/rex-20060125.rb
 - test/rex-20060511.rb
 - test/test_generator.rb
-has_rdoc: true
-homepage: http://github.com/tenderlove/rexical/tree/master
-licenses: []
-
-post_install_message: 
-rdoc_options: 
-- --main
+homepage: http://github.com/sparklemotion/rexical
+licenses:
+- LGPL-2.1-only
+metadata: {}
+post_install_message:
+rdoc_options:
+- "--main"
 - README.rdoc
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: ruby-rex
-rubygems_version: 1.3.7
-signing_key: 
-specification_version: 3
-summary: Rexical is a lexical scanner generator
-test_files: 
-- test/test_generator.rb
+rubygems_version: 3.5.10
+signing_key:
+specification_version: 4
+summary: Rexical is a lexical scanner generator that is used with Racc to generate
+  Ruby programs
+test_files: []