rexical 1.0.3 → 1.0.4

data/CHANGELOG.rdoc

@@ -1,3 +1,9 @@
+=== 1.0.4
+
+* Bug fixes
+
+  * Generated tokenizer only tokenizes on pulls
+
 === 1.0.3
 
 * Bug fixes

data/Manifest.txt

@@ -8,7 +8,6 @@ Rakefile
 bin/rex
 lib/rexical.rb
 lib/rexical/generator.rb
-lib/rexical/info.rb
 lib/rexical/rexcmd.rb
 sample/a.cmd
 sample/b.cmd

data/Rakefile

@@ -3,15 +3,12 @@
 require 'rubygems'
 require 'hoe'
 
-$LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
-require 'rexical'
-
-HOE = Hoe.new('rexical', Rexical::VERSION) do |p|
-  p.readme_file  = 'README.rdoc'
-  p.history_file = 'CHANGELOG.rdoc'
-  p.developer('Aaron Patterson', 'aaronp@rubyforge.org')
-  p.rubyforge_name = 'ruby-rex'
-  p.extra_rdoc_files = FileList['*.rdoc']
+Hoe.spec 'rexical' do
+  self.readme_file  = 'README.rdoc'
+  self.history_file = 'CHANGELOG.rdoc'
+  developer('Aaron Patterson', 'aaronp@rubyforge.org')
+  self.rubyforge_name = 'ruby-rex'
+  self.extra_rdoc_files = FileList['*.rdoc']
 end
 
 namespace :gem do

data/lib/rexical.rb

@@ -1,3 +1,8 @@
 require 'rexical/generator'
-require 'rexical/info'
 require 'rexical/rexcmd'
+
+module Rexical
+  VERSION = "1.0.4"
+  Copyright = 'Copyright (c) 2005-2006 ARIMA Yasuhiro'
+  Mailto    = 'arima.yasuhiro@nifty.com'
+end

data/lib/rexical/generator.rb

@@ -95,6 +95,7 @@ module Rexical
         end
       end
       expr = st[0,ndx]
+      expr.gsub!('\ ', ' ')
       key  =  '{' + key + '}'
       @macro.each_pair do |k, e|
         expr.gsub!(k) { |m| e }
@@ -314,47 +315,39 @@ REX_UTIL = <<-REX_EOT
 
   class ScanError < StandardError ; end
 
-  attr_reader :lineno
-  attr_reader :filename
+  attr_reader   :lineno
+  attr_reader   :filename
+  attr_accessor :state
 
-  def scan_setup ; end
+  def scan_setup(str)
+    @ss = StringScanner.new(str)
+    @lineno =  1
+    @state  = nil
+  end
 
-  def action &block
+  def action(&block)
     yield
   end
 
-  def scan_str( str )
-    scan_evaluate  str
+  def scan_str(str)
+    scan_setup(str)
     do_parse
   end
 
   def load_file( filename )
     @filename = filename
     open(filename, "r") do |f|
-      scan_evaluate  f.read
+      scan_setup(f.read)
     end
   end
 
   def scan_file( filename )
-    load_file  filename
+    load_file(filename)
     do_parse
   end
 
 REX_EOT
 
-REX_NEXTTOKEN_DEBUG = <<-REX_EOT
-  def next_token
-    p token = @rex_tokens.shift
-    token
-  end
-REX_EOT
-
-REX_NEXTTOKEN = <<-REX_EOT
-  def next_token
-    @rex_tokens.shift
-  end
-REX_EOT
-
 REX_STUB = <<-REX_EOT
 
 if __FILE__ == $0
@@ -409,83 +402,80 @@ REX_EOT
       ## utility method
       f.print REX_UTIL
 
-      if @opt['--debug']
-        f.print REX_NEXTTOKEN_DEBUG
-      else
-        f.print REX_NEXTTOKEN
-      end
-
       ## scanner method
 
       f.print <<-REX_EOT
 
-  def scan_evaluate( str )
-    scan_setup
-    @rex_tokens = []
-    @lineno  =  1
-    ss = StringScanner.new(str)
-    state = nil
-    until ss.eos?
-      text = ss.peek(1)
-      @lineno  +=  1  if text == "\\n"
-      case state
+  def next_token
+    return if @ss.eos?
+
+    text = @ss.peek(1)
+    @lineno  +=  1  if text == "\\n"
+    token = case @state
+    REX_EOT
+
+    exclusive_states.each do |es|
+      f.printf <<-REX_EOT
+    when #{es ? es.to_s : "nil"}
+      case
       REX_EOT
+      rules.each do |rule|
+        exclusive_state, start_state, rule_expr, rule_action = *rule
+        if es == exclusive_state
 
-      exclusive_states.each do |es|
-        f.printf <<-REX_EOT
-      when #{es ? es.to_s : "nil"}
-        case
-        REX_EOT
-        rules.each do |rule|
-          exclusive_state, start_state, rule_expr, rule_action = *rule
-          if es == exclusive_state
-
-            if rule_action
-              if start_state
-                f.print <<-REX_EOT
-        when (state == #{start_state}) and (text = ss.scan(/#{rule_expr}/#{flag}))
-           @rex_tokens.push action #{rule_action}
-
-                REX_EOT
-              else
-                f.print <<-REX_EOT
-        when (text = ss.scan(/#{rule_expr}/#{flag}))
-           @rex_tokens.push action #{rule_action}
+          if rule_action
+            if start_state
+              f.print <<-REX_EOT
+      when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
+         action #{rule_action}
 
-                REX_EOT
-              end
+              REX_EOT
             else
-              if start_state
-                f.print <<-REX_EOT
-        when (state == #{start_state}) and (text = ss.scan(/#{rule_expr}/#{flag}))
-          ;
+              f.print <<-REX_EOT
+      when (text = @ss.scan(/#{rule_expr}/#{flag}))
+         action #{rule_action}
 
-                REX_EOT
-              else
-                f.print <<-REX_EOT
-        when (text = ss.scan(/#{rule_expr}/#{flag}))
-          ;
-
-                REX_EOT
-              end
+              REX_EOT
             end
+          else
+            if start_state
+              f.print <<-REX_EOT
+      when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
+        ;
 
+              REX_EOT
+            else
+              f.print <<-REX_EOT
+      when (text = @ss.scan(/#{rule_expr}/#{flag}))
+        ;
+
+              REX_EOT
+            end
           end
-        end
-        f.print <<-REX_EOT
-        else
-          text = ss.string[ss.pos .. -1]
-          raise  ScanError, "can not match: '" + text + "'"
-        end  # if
 
-        REX_EOT
+        end
       end
       f.print <<-REX_EOT
       else
-        raise  ScanError, "undefined state: '" + state.to_s + "'"
-      end  # case state
-    end  # until ss
-  end  # def scan_evaluate
+        text = @ss.string[@ss.pos .. -1]
+        raise  ScanError, "can not match: '" + text + "'"
+      end  # if
+
+      REX_EOT
+    end
+    f.print <<-REX_EOT
+    else
+      raise  ScanError, "undefined state: '" + state.to_s + "'"
+    end  # case state
+      REX_EOT
+    if @opt['--debug']
+      f.print <<-REX_EOT
+    p token
+      REX_EOT
+    end
+    f.print <<-REX_EOT
+    token
+  end  # def next_token
 
       REX_EOT
 

data/test/test_generator.rb

@@ -34,10 +34,7 @@ class TestGenerator < Test::Unit::TestCase
   end
 
   def test_scanner_inherits
-    rex = Rexical::Generator.new(
-      "--independent" => true
-    )
-    rex.grammar_lines = StringScanner.new %q{
+    source = parse_lexer %q{
 class Calculator < Bar
 rule
   \d+       { [:NUMBER, text.to_i] }
@@ -45,18 +42,11 @@ rule
 end
     }
 
-    rex.parse
-
-    output = StringIO.new
-    rex.write_scanner output
-    assert_match 'Calculator < Bar', output.string
+    assert_match 'Calculator < Bar', source
   end
 
   def test_scanner_inherits_many_levels
-    rex = Rexical::Generator.new(
-      "--independent" => true
-    )
-    rex.grammar_lines = StringScanner.new %q{
+    source = parse_lexer %q{
 class Calculator < Foo::Bar
 rule
   \d+       { [:NUMBER, text.to_i] }
@@ -64,18 +54,11 @@ rule
 end
     }
 
-    rex.parse
-
-    output = StringIO.new
-    rex.write_scanner output
-    assert_match 'Calculator < Foo::Bar', output.string
+    assert_match 'Calculator < Foo::Bar', source
   end
 
   def test_simple_scanner
-    rex = Rexical::Generator.new(
-      "--independent" => true
-    )
-    rex.grammar_lines = StringScanner.new %q{
+    m = build_lexer %q{
 class Calculator
 rule
   \d+       { [:NUMBER, text.to_i] }
@@ -83,15 +66,8 @@ rule
 end
     }
 
-    rex.parse
-
-    output = StringIO.new
-    rex.write_scanner output
-
-    m = Module.new
-    m.module_eval output.string
     calc = m::Calculator.new
-    calc.scan_evaluate('1 2 10')
+    calc.scan_setup('1 2 10')
 
     assert_tokens [[:NUMBER, 1],
                   [:S, ' '],
@@ -100,11 +76,21 @@ end
                   [:NUMBER, 10]], calc
   end
 
+  def test_parses_macros_with_escapes
+    source = parse_lexer %q{
+class Foo
+macro
+  w  [\ \t]+
+rule
+  {w}  { [:SPACE, text] }
+end
+    }
+
+    assert source.index('@ss.scan(/[ \t]+/))')
+  end
+
   def test_simple_scanner_with_macros
-    rex = Rexical::Generator.new(
-      "--independent" => true
-    )
-    rex.grammar_lines = StringScanner.new %q{
+    m = build_lexer %q{
 class Calculator
 macro
   digit     \d+
@@ -114,15 +100,8 @@ rule
 end
     }
 
-    rex.parse
-
-    output = StringIO.new
-    rex.write_scanner output
-
-    m = Module.new
-    m.module_eval output.string
     calc = m::Calculator.new
-    calc.scan_evaluate('1 2 10')
+    calc.scan_setup('1 2 10')
 
     assert_tokens [[:NUMBER, 1],
                   [:S, ' '],
@@ -132,10 +111,7 @@ end
   end
 
   def test_nested_macros
-    rex = Rexical::Generator.new(
-      "--independent" => true
-    )
-    rex.grammar_lines = StringScanner.new %q{
+    source = parse_lexer %q{
 class Calculator
 macro
   nonascii  [^\0-\177]
@@ -145,18 +121,11 @@ rule
 end
     }
 
-    rex.parse
-
-    output = StringIO.new
-    rex.write_scanner output
-    assert_match '"[^\0-\177]*"', output.string
+    assert_match '"[^\0-\177]*"', source
   end
 
   def test_more_nested_macros
-    rex = Rexical::Generator.new(
-      "--independent" => true
-    )
-    rex.grammar_lines = StringScanner.new %q{
+    source = parse_lexer %q{
 class Calculator
 macro
   nonascii  [^\0-\177]
@@ -167,18 +136,85 @@ rule
 end
     }
 
+    assert_match '"[^\0-\177]*"', source
+  end
+
+  def test_changing_state_during_lexing
+    lexer = build_lexer %q{
+class Calculator
+rule
+       a       { self.state = :B  ; [:A, text] }
+  :B   b       { self.state = nil ; [:B, text] }
+end
+    }
+
+    calc1 = lexer::Calculator.new
+    calc2 = lexer::Calculator.new
+    calc1.scan_setup('aaaaa')
+    calc2.scan_setup('ababa')
+
+    # Doesn't lex all 'a's
+    assert_raise(lexer::Calculator::ScanError) { tokens(calc1) }
+
+    # Does lex alternating 'a's and 'b's
+    calc2.scan_setup('ababa')
+
+    assert_tokens [[:A, 'a'],
+                   [:B, 'b'],
+                   [:A, 'a'],
+                   [:B, 'b'],
+                   [:A, 'a']], calc2
+  end
+
+  def test_changing_state_is_possible_between_next_token_calls
+    lexer = build_lexer %q{
+class Calculator
+rule
+       a       { [:A, text] }
+  :B   b       { [:B, text] }
+end
+    }
+
+    calc = lexer::Calculator.new
+    calc.scan_setup('ababa')
+
+    assert_equal [:A, 'a'], calc.next_token
+    calc.state = :B
+    assert_equal [:B, 'b'], calc.next_token
+    calc.state = nil
+    assert_equal [:A, 'a'], calc.next_token
+    calc.state = :B
+    assert_equal [:B, 'b'], calc.next_token
+    calc.state = nil
+    assert_equal [:A, 'a'], calc.next_token
+  end
+
+  def parse_lexer(str)
+    rex = Rexical::Generator.new("--independent" => true)
+    out = StringIO.new
+
+    rex.grammar_lines = StringScanner.new(str)
     rex.parse
+    rex.write_scanner(out)
 
-    output = StringIO.new
-    rex.write_scanner output
-    assert_match '"[^\0-\177]*"', output.string
+    out.string
   end
 
-  def assert_tokens expected, scanner
+  def build_lexer(str)
+    mod = Module.new
+    mod.module_eval(parse_lexer(str))
+    mod
+  end
+
+  def tokens(scanner)
     tokens = []
     while token = scanner.next_token
       tokens << token
     end
-    assert_equal expected, tokens
+    tokens
+  end
+
+  def assert_tokens(expected, scanner)
+    assert_equal expected, tokens(scanner)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: rexical
 version: !ruby/object:Gem::Version 
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors: 
 - Aaron Patterson
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-07 00:00:00 -07:00
+date: 2009-06-30 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.12.1
+        version: 2.3.1
     version: 
 description: |-
   Rexical is a lexical scanner generator.
@@ -49,7 +49,6 @@ files:
 - bin/rex
 - lib/rexical.rb
 - lib/rexical/generator.rb
-- lib/rexical/info.rb
 - lib/rexical/rexcmd.rb
 - sample/a.cmd
 - sample/b.cmd
@@ -101,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: ruby-rex
-rubygems_version: 1.3.3
+rubygems_version: 1.3.4
 signing_key: 
 specification_version: 3
 summary: Rexical is a lexical scanner generator

data/lib/rexical/info.rb

@@ -1,16 +0,0 @@
-#
-# info.rb
-#
-#   Copyright (c) 2005-2006 ARIMA Yasuhiro <arima.yasuhiro@nifty.com>
-#
-#   This program is free software.
-#   You can distribute/modify this program under the terms of
-#   the GNU Lesser General Public License version 2 or later.
-#
-
-module Rexical
-  VERSION   = '1.0.3'
-  Copyright = 'Copyright (c) 2005-2006 ARIMA Yasuhiro'
-  Mailto    = 'arima.yasuhiro@nifty.com'
-end
-