rexical 1.0.5 → 1.0.7

data/lib/rexical/rexcmd.rb

@@ -22,6 +22,7 @@ o -s --stub             - append stub code for debug
 o -i --ignorecase       - ignore char case
 o -C --check-only       - syntax check only
 o -  --independent      - independent mode
+o -  --matcheos         - allow match against end of string
 o -d --debug            - print debug information
 o -h --help             - print this message and quit
 o -  --version          - print version and quit
@@ -65,7 +66,8 @@ EOT
     @cmd  =  File.basename($0, ".rb")
     tmp  =  OPTIONS.lines.collect do |line|
         next if /\A\s*\z/ === line
-        disp, sopt, lopt, takearg, doc  =  line.strip.split(/\s+/, 5)
+        # disp, sopt, lopt, takearg, doc
+        _, sopt, lopt, takearg, _  =  line.strip.split(/\s+/, 5)
         a  =  []
         a.push lopt    unless lopt == '-'
         a.push sopt    unless sopt == '-'

data/sample/calc3.tab.rb

@@ -1,7 +1,7 @@
 #
 # DO NOT MODIFY!!!!
 # This file is automatically generated by racc 1.4.4
-# from racc grammer file "calc3.racc".
+# from racc grammar file "calc3.racc".
 #
 
 require 'racc/parser'

data/sample/error1.rex

@@ -7,7 +7,7 @@ class Error1
 macro
   BLANK         [\ \t]+
 rule
-  BLANK         # no action
+  {BLANK}       # no action
   \d+           { [:digit, text.to_i] }
   \w+           { [:word, text] }
   \n

data/sample/error2.rex

@@ -7,7 +7,7 @@ class Error2
 macro
   BLANK         [\ \t]+
 rule
-  BLANK         # no action
+  {BLANK}       # no action
   \d+           { [:digit, text.to_i] }
   \w+           { [:word, text] }
   \n

data/sample/sample.rex

@@ -7,7 +7,7 @@ class Sample
 macro
   BLANK         [\ \t]+
 rule
-  BLANK         # no action
+  {BLANK}       # no action
   \d+           { [:digit, text.to_i] }
   \w+           { [:word, text] }
   \n

data/sample/sample.rex.rb

@@ -1,100 +1,92 @@
-#
-# DO NOT MODIFY!!!!
-# This file is automatically generated by rex 1.0.0
-# from lexical definition file "sample.rex".
-#
-
-require 'racc/parser'
-#
-# sample.rex
-# lexical definition sample for rex
-#
-
-class Sample < Racc::Parser
-  require 'strscan'
-
-  class ScanError < StandardError ; end
-
-  attr_reader :lineno
-  attr_reader :filename
-
-  def scan_setup ; end
-
-  def action &block
-    yield
-  end
-
-  def scan_str( str )
-    scan_evaluate  str
-    do_parse
-  end
-
-  def load_file( filename )
-    @filename = filename
-    open(filename, "r") do |f|
-      scan_evaluate  f.read
-    end
-  end
-
-  def scan_file( filename )
-    load_file  filename
-    do_parse
-  end
-
-  def next_token
-    @rex_tokens.shift
-  end
-
-  def scan_evaluate( str )
-    scan_setup
-    @rex_tokens = []
-    @lineno  =  1
-    ss = StringScanner.new(str)
-    state = nil
-    until ss.eos?
-      text = ss.peek(1)
-      @lineno  +=  1  if text == "\n"
-      case state
-      when nil
-        case
-        when (text = ss.scan(/BLANK/))
-          ;
-
-        when (text = ss.scan(/\d+/))
-           @rex_tokens.push action { [:digit, text.to_i] }
-
-        when (text = ss.scan(/\w+/))
-           @rex_tokens.push action { [:word, text] }
-
-        when (text = ss.scan(/\n/))
-          ;
-
-        when (text = ss.scan(/./))
-           @rex_tokens.push action { [text, text] }
-
-        else
-          text = ss.string[ss.pos .. -1]
-          raise  ScanError, "can not match: '" + text + "'"
-        end  # if
-
-      else
-        raise  ScanError, "undefined state: '" + state.to_s + "'"
-      end  # case state
-    end  # until ss
-  end  # def scan_evaluate
-
-end # class
-
-if __FILE__ == $0
-  exit  if ARGV.size != 1
-  filename = ARGV.shift
-  rex = Sample.new
-  begin
-    rex.load_file  filename
-    while  token = rex.next_token
-      p token
-    end
-  rescue
-    $stderr.printf  "%s:%d:%s\n", rex.filename, rex.lineno, $!.message
-  end
-end
+#--
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.5
+# from lexical definition file "sample/sample.rex".
+#++
+
+require 'racc/parser'
+#
+# sample.rex
+# lexical definition sample for rex
+#
+
+class Sample < Racc::Parser
+      require 'strscan'
+
+      class ScanError < StandardError ; end
+
+      attr_reader   :lineno
+      attr_reader   :filename
+      attr_accessor :state
+
+      def scan_setup(str)
+        @ss = StringScanner.new(str)
+        @lineno =  1
+        @state  = nil
+      end
+
+      def action
+        yield
+      end
+
+      def scan_str(str)
+        scan_setup(str)
+        do_parse
+      end
+      alias :scan :scan_str
+
+      def load_file( filename )
+        @filename = filename
+        open(filename, "r") do |f|
+          scan_setup(f.read)
+        end
+      end
+
+      def scan_file( filename )
+        load_file(filename)
+        do_parse
+      end
+
+
+        def next_token
+          return if @ss.eos?
+
+          # skips empty actions
+          until token = _next_token or @ss.eos?; end
+          token
+        end
+
+        def _next_token
+          text = @ss.peek(1)
+          @lineno  +=  1  if text == "\n"
+          token = case @state
+            when nil
+          case
+                  when (text = @ss.scan(/[ \t]+/))
+                    ;
+
+                  when (text = @ss.scan(/\d+/))
+                     action { [:digit, text.to_i] }
+
+                  when (text = @ss.scan(/\w+/))
+                     action { [:word, text] }
+
+                  when (text = @ss.scan(/\n/))
+                    ;
+
+                  when (text = @ss.scan(/./))
+                     action { [text, text] }
+
+          
+          else
+            text = @ss.string[@ss.pos .. -1]
+            raise  ScanError, "can not match: '" + text + "'"
+          end  # if
+
+        else
+          raise  ScanError, "undefined state: '" + state.to_s + "'"
+        end  # case state
+          token
+        end  # def _next_token
+
+end # class

data/test/assets/test.rex

@@ -5,8 +5,12 @@ class C < SomethingElse
 macro
   w         [\s\r\n\f]*
 
+rule
+
 # [:state]  pattern  [actions]
 
             {w}~={w}         { [:INCLUDES, text] }
 
 end
+end
+end

data/test/test_generator.rb

@@ -1,9 +1,11 @@
-require 'test/unit'
+gem "minitest"
+require 'minitest/autorun'
 require 'tempfile'
 require 'rexical'
 require 'stringio'
+require 'open3'
 
-class TestGenerator < Test::Unit::TestCase
+class TestGenerator < Minitest::Test
   def test_header_is_written_after_module
     rex = Rexical::Generator.new(
       "--independent" => true
@@ -25,6 +27,29 @@ class TestGenerator < Test::Unit::TestCase
     assert_equal '#++', comments.last
   end
 
+  def test_rubocop_security
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_file = File.join File.dirname(__FILE__), 'assets', 'test.rex'
+    rex.read_grammar
+    rex.parse
+
+    output = Tempfile.new(["rex_output", ".rb"])
+    begin
+      rex.write_scanner output
+      output.close
+
+      stdin, stdoe, wait_thr = Open3.popen2e "rubocop --only Security #{output.path}"
+      if ! wait_thr.value.success?
+        fail stdoe.read
+      end
+    ensure
+      output.close
+      output.unlink
+    end
+  end
+
   def test_read_non_existent_file
     rex = Rexical::Generator.new(nil)
     rex.grammar_file = 'non_existent_file'
@@ -71,6 +96,22 @@ end
     assert_match 'Calculator < Foo::Bar', source
   end
 
+  def test_stateful_lexer
+    m = build_lexer %q{
+class Foo
+rule
+          \d      { @state = :digit; [:foo, text] }
+  :digit  \w      { @state = nil; [:w, text] }
+end
+    }
+    scanner = m::Foo.new
+    scanner.scan_setup('1w1')
+    assert_tokens [
+      [:foo, '1'],
+      [:w, 'w'],
+      [:foo, '1']], scanner
+  end
+
   def test_simple_scanner
     m = build_lexer %q{
 class Calculator
@@ -185,7 +226,7 @@ end
     calc2.scan_setup('ababa')
 
     # Doesn't lex all 'a's
-    assert_raise(lexer::Calculator::ScanError) { tokens(calc1) }
+    assert_raises(lexer::Calculator::ScanError) { tokens(calc1) }
 
     # Does lex alternating 'a's and 'b's
     calc2.scan_setup('ababa')
@@ -219,6 +260,21 @@ end
     calc.state = nil
     assert_equal [:A, 'a'], calc.next_token
   end
+  def test_match_eos
+    lexer = build_lexer %q{
+class Calculator
+option
+matcheos
+rule
+      a        { [:A, text] }
+      $        { [:EOF, ""] }
+:B    b        { [:B, text] }
+     }
+     calc = lexer::Calculator.new
+     calc.scan_setup("a")
+     assert_equal [:A, 'a'], calc.next_token
+     assert_equal [:EOF, ""], calc.next_token
+  end
 
   def parse_lexer(str)
     rex = Rexical::Generator.new("--independent" => true)

metadata

@@ -1,54 +1,79 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rexical
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 0
-  - 5
-  version: 1.0.5
+version: !ruby/object:Gem::Version
+  version: 1.0.7
 platform: ruby
-authors: 
+authors:
 - Aaron Patterson
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-12-08 00:00:00 -08:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
+date: 2019-08-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.74.0
+  type: :development
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.74.0
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 6
-        - 2
-        version: 2.6.2
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :development
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.18'
 description: |-
-  Rexical is a lexical scanner generator.
-  It is written in Ruby itself, and generates Ruby program.
-  It is designed for use with Racc.
-email: 
+  Rexical is a lexical scanner generator that is used with
+  Racc to generate Ruby programs.
+  Rexical is written in Ruby.
+email:
 - aaronp@rubyforge.org
-executables: 
+executables:
 - rex
 extensions: []
-
-extra_rdoc_files: 
-- Manifest.txt
+extra_rdoc_files:
 - CHANGELOG.rdoc
 - DOCUMENTATION.en.rdoc
 - DOCUMENTATION.ja.rdoc
+- Manifest.txt
 - README.rdoc
-files: 
+files:
 - CHANGELOG.rdoc
 - DOCUMENTATION.en.rdoc
 - DOCUMENTATION.ja.rdoc
@@ -85,38 +110,30 @@ files:
 - test/rex-20060125.rb
 - test/rex-20060511.rb
 - test/test_generator.rb
-has_rdoc: true
 homepage: http://github.com/tenderlove/rexical/tree/master
-licenses: []
-
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
-rdoc_options: 
-- --main
+rdoc_options:
+- "--main"
 - README.rdoc
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: ruby-rex
-rubygems_version: 1.3.7
+rubygems_version: 3.0.3
 signing_key: 
-specification_version: 3
-summary: Rexical is a lexical scanner generator
-test_files: 
-- test/test_generator.rb
+specification_version: 4
+summary: Rexical is a lexical scanner generator that is used with Racc to generate
+  Ruby programs
+test_files: []