rexical 1.0.3

data/test/assets/test.rex

@@ -0,0 +1,12 @@
+module A
+module B
+class C < SomethingElse
+
+macro
+  w         [\s\r\n\f]*
+
+# [:state]  pattern  [actions]
+
+            {w}~={w}         { [:INCLUDES, text] }
+
+end

data/test/rex-20060125.rb

@@ -0,0 +1,152 @@
+#!C:/Program Files/ruby-1.8/bin/ruby
+#
+# rex
+#
+#   Copyright (c) 2005 ARIMA Yasuhiro <arima.yasuhiro@nifty.com>
+#
+#   This program is free software.
+#   You can distribute/modify this program under the terms of
+#   the GNU LGPL, Lesser General Public License version 2.1.
+#   For details of LGPL, see the file "COPYING".
+#
+
+## ---------------------------------------------------------------------
+
+REX_OPTIONS  =  <<-EOT
+
+o -o --output-file <outfile>  file name of output [<filename>.rb]
+o -s --stub             - append stub code for debug
+o -i --ignorecase       - ignore char case
+o -C --check-only       - syntax check only
+o -  --independent      - independent mode
+o -d --debug            - print debug information
+o -h --help             - print this message and quit
+o -  --version          - print version and quit
+o -  --copyright        - print copyright and quit
+
+EOT
+
+## ---------------------------------------------------------------------
+
+require 'getoptlong'
+require 'rex/generator'
+require 'rex/info'
+
+## ---------------------------------------------------------------------
+
+=begin
+class Rex
+  def initialize
+  end
+end
+=end
+
+def main
+  $cmd  =  File.basename($0, ".rb")
+  opt  =  get_options
+  filename  =  ARGV[0]
+
+  rex  =  Rex::Generator.new(opt)
+  begin
+    rex.grammar_file  =  filename
+    rex.read_grammar
+    rex.parse
+    if opt['--check-only']
+      $stderr.puts "syntax ok"
+      return  0
+    end
+    rex.write_scanner
+
+  rescue Rex::ParseError, Errno::ENOENT
+    msg  =  $!.to_s
+    unless /\A\d/ === msg
+      msg[0,0]  =  ' '
+    end
+    $stderr.puts "#{$cmd}:#{rex.grammar_file}:#{rex.lineno}:#{msg}"
+    return  1
+
+  end
+  return  0
+end
+
+## ---------------------------------------------------------------------
+def get_options
+  tmp  =  REX_OPTIONS.collect do |line|
+      next if /\A\s*\z/ === line
+      disp, sopt, lopt, takearg, doc  =  line.strip.split(/\s+/, 5)
+      a  =  []
+      a.push lopt    unless lopt == '-'
+      a.push sopt    unless sopt == '-'
+      a.push takearg == '-' ?
+             GetoptLong::NO_ARGUMENT : GetoptLong::REQUIRED_ARGUMENT
+      a
+  end
+  getopt  =  GetoptLong.new(*tmp.compact)
+  getopt.quiet  =  true
+
+  opt  =  {}
+  begin
+    getopt.each do |name, arg|
+      raise GetoptLong::InvalidOption,
+          "#{$cmd}: #{name} given twice" if opt.key? name
+      opt[name]  =  arg.empty? ? true : arg
+    end
+  rescue GetoptLong::AmbigousOption, GetoptLong::InvalidOption,
+         GetoptLong::MissingArgument, GetoptLong::NeedlessArgument
+    usage 1, $!.message
+  end
+
+  usage    if opt['--help']
+
+  if opt['--version']
+    puts "#{$cmd} version #{Rex::Version}"
+    exit 0
+  end
+  if opt['--copyright']
+    puts "#{$cmd} version #{Rex::Version}"
+    puts "#{Rex::Copyright} <#{Rex::Mailto}>"
+    exit 0
+  end
+
+  usage(1, 'no grammar file given')    if ARGV.empty?
+  usage(1, 'too many grammar files given')    if ARGV.size > 1
+
+  opt
+end
+
+## ---------------------------------------------------------------------
+
+def usage(status=0, msg=nil )
+  f  =  (status == 0 ? $stdout : $stderr)
+  f.puts "#{$cmd}: #{msg}"  if msg
+  f.print <<-EOT
+Usage: #{$cmd} [options] <grammar file>
+Options:
+  EOT
+
+  REX_OPTIONS.each do |line|
+    next if line.strip.empty?
+    if /\A\s*\z/ === line
+      f.puts
+      next
+    end
+
+    disp, sopt, lopt, takearg, doc  =  line.strip.split(/\s+/, 5)
+    if disp == 'o'
+      sopt  =  nil if sopt == '-'
+      lopt  =  nil if lopt == '-'
+      opt  =  [sopt, lopt].compact.join(',')
+
+      takearg  =  nil if takearg == '-'
+      opt  =  [opt, takearg].compact.join(' ')
+
+      f.printf "%-27s %s\n", opt, doc
+    end
+  end
+
+  exit status
+end
+
+## ---------------------------------------------------------------------
+
+main

data/test/rex-20060511.rb

@@ -0,0 +1,143 @@
+#!/usr/local/bin/ruby
+#
+# rex
+#
+#   Copyright (c) 2005-2006 ARIMA Yasuhiro <arima.yasuhiro@nifty.com>
+#
+#   This program is free software.
+#   You can distribute/modify this program under the terms of
+#   the GNU LGPL, Lesser General Public License version 2.1.
+#   For details of LGPL, see the file "COPYING".
+#
+
+## ---------------------------------------------------------------------
+
+REX_OPTIONS  =  <<-EOT
+
+o -o --output-file <outfile>  file name of output [<filename>.rb]
+o -s --stub             - append stub code for debug
+o -i --ignorecase       - ignore char case
+o -C --check-only       - syntax check only
+o -  --independent      - independent mode
+o -d --debug            - print debug information
+o -h --help             - print this message and quit
+o -  --version          - print version and quit
+o -  --copyright        - print copyright and quit
+
+EOT
+
+## ---------------------------------------------------------------------
+
+require 'getoptlong'
+require 'rex/generator'
+require 'rex/info'
+
+## ---------------------------------------------------------------------
+
+class RexRunner
+  def run
+    @status  =  1
+    usage 'no grammar file given'    if ARGV.empty?
+    usage 'too many grammar files given'    if ARGV.size > 1
+    filename  =  ARGV[0]
+
+    rex  =  Rex::Generator.new(@opt)
+    begin
+      rex.grammar_file  =  filename
+      rex.read_grammar
+      rex.parse
+      if @opt['--check-only']
+        $stderr.puts "syntax ok"
+        return  0
+      end
+      rex.write_scanner
+      @status  =  0
+
+    rescue Rex::ParseError, Errno::ENOENT
+      msg  =  $!.to_s
+      unless /\A\d/ === msg
+        msg[0,0]  =  ' '
+      end
+      $stderr.puts "#{@cmd}:#{rex.grammar_file}:#{rex.lineno}:#{msg}"
+
+    ensure
+      exit @status
+
+    end
+  end
+
+  def initialize
+    @status  =  2
+    @cmd  =  File.basename($0, ".rb")
+    tmp  =  REX_OPTIONS.collect do |line|
+        next if /\A\s*\z/ === line
+        disp, sopt, lopt, takearg, doc  =  line.strip.split(/\s+/, 5)
+        a  =  []
+        a.push lopt    unless lopt == '-'
+        a.push sopt    unless sopt == '-'
+        a.push takearg == '-' ?
+               GetoptLong::NO_ARGUMENT : GetoptLong::REQUIRED_ARGUMENT
+        a
+    end
+    getopt  =  GetoptLong.new(*tmp.compact)
+    getopt.quiet  =  true
+
+    @opt  =  {}
+    begin
+      getopt.each do |name, arg|
+        raise GetoptLong::InvalidOption,
+            "#{@cmd}: #{name} given twice" if @opt.key? name
+        @opt[name]  =  arg.empty? ? true : arg
+      end
+    rescue GetoptLong::AmbigousOption, GetoptLong::InvalidOption,
+           GetoptLong::MissingArgument, GetoptLong::NeedlessArgument
+      usage $!.message
+    end
+
+    usage    if @opt['--help']
+
+    if @opt['--version']
+      puts "#{@cmd} version #{Rex::Version}"
+      exit 0
+    end
+    if @opt['--copyright']
+      puts "#{@cmd} version #{Rex::Version}"
+      puts "#{Rex::Copyright} <#{Rex::Mailto}>"
+      exit 0
+    end
+  end
+
+  def usage( msg=nil )
+    f  =  $stderr
+    f.puts "#{@cmd}: #{msg}"  if msg
+    f.print <<-EOT
+Usage: #{@cmd} [options] <grammar file>
+Options:
+    EOT
+
+    REX_OPTIONS.each do |line|
+      next if line.strip.empty?
+      if /\A\s*\z/ === line
+        f.puts
+        next
+      end
+
+      disp, sopt, lopt, takearg, doc  =  line.strip.split(/\s+/, 5)
+      if disp == 'o'
+        sopt  =  nil if sopt == '-'
+        lopt  =  nil if lopt == '-'
+        opt  =  [sopt, lopt].compact.join(',')
+
+        takearg  =  nil if takearg == '-'
+        opt  =  [opt, takearg].compact.join(' ')
+
+        f.printf "%-27s %s\n", opt, doc
+      end
+    end
+
+    exit @status
+  end
+end
+
+RexRunner.new.run
+

data/test/test_generator.rb

@@ -0,0 +1,184 @@
+require 'test/unit'
+require 'tempfile'
+require 'rexical'
+require 'stringio'
+
+class TestGenerator < Test::Unit::TestCase
+  def test_header_is_written_after_module
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_file = File.join File.dirname(__FILE__), 'assets', 'test.rex'
+    rex.read_grammar
+    rex.parse
+
+    output = StringIO.new
+    rex.write_scanner output
+
+    comments = []
+    output.string.split(/[\n]/).each do |line|
+      comments << line.chomp if line =~ /^#/
+    end
+
+    assert_match 'DO NOT MODIFY', comments.join
+    assert_equal '#--', comments.first
+    assert_equal '#++', comments.last
+  end
+
+  def test_read_non_existent_file
+    rex = Rexical::Generator.new(nil)
+    rex.grammar_file = 'non_existent_file'
+    assert_raises Errno::ENOENT do
+      rex.read_grammar
+    end
+  end
+
+  def test_scanner_inherits
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_lines = StringScanner.new %q{
+class Calculator < Bar
+rule
+  \d+       { [:NUMBER, text.to_i] }
+  \s+       { [:S, text] }
+end
+    }
+
+    rex.parse
+
+    output = StringIO.new
+    rex.write_scanner output
+    assert_match 'Calculator < Bar', output.string
+  end
+
+  def test_scanner_inherits_many_levels
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_lines = StringScanner.new %q{
+class Calculator < Foo::Bar
+rule
+  \d+       { [:NUMBER, text.to_i] }
+  \s+       { [:S, text] }
+end
+    }
+
+    rex.parse
+
+    output = StringIO.new
+    rex.write_scanner output
+    assert_match 'Calculator < Foo::Bar', output.string
+  end
+
+  def test_simple_scanner
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_lines = StringScanner.new %q{
+class Calculator
+rule
+  \d+       { [:NUMBER, text.to_i] }
+  \s+       { [:S, text] }
+end
+    }
+
+    rex.parse
+
+    output = StringIO.new
+    rex.write_scanner output
+
+    m = Module.new
+    m.module_eval output.string
+    calc = m::Calculator.new
+    calc.scan_evaluate('1 2 10')
+
+    assert_tokens [[:NUMBER, 1],
+                  [:S, ' '],
+                  [:NUMBER, 2],
+                  [:S, ' '],
+                  [:NUMBER, 10]], calc
+  end
+
+  def test_simple_scanner_with_macros
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_lines = StringScanner.new %q{
+class Calculator
+macro
+  digit     \d+
+rule
+  {digit}       { [:NUMBER, text.to_i] }
+  \s+       { [:S, text] }
+end
+    }
+
+    rex.parse
+
+    output = StringIO.new
+    rex.write_scanner output
+
+    m = Module.new
+    m.module_eval output.string
+    calc = m::Calculator.new
+    calc.scan_evaluate('1 2 10')
+
+    assert_tokens [[:NUMBER, 1],
+                  [:S, ' '],
+                  [:NUMBER, 2],
+                  [:S, ' '],
+                  [:NUMBER, 10]], calc
+  end
+
+  def test_nested_macros
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_lines = StringScanner.new %q{
+class Calculator
+macro
+  nonascii  [^\0-\177]
+  string    "{nonascii}*"
+rule
+  {string}       { [:STRING, text] }
+end
+    }
+
+    rex.parse
+
+    output = StringIO.new
+    rex.write_scanner output
+    assert_match '"[^\0-\177]*"', output.string
+  end
+
+  def test_more_nested_macros
+    rex = Rexical::Generator.new(
+      "--independent" => true
+    )
+    rex.grammar_lines = StringScanner.new %q{
+class Calculator
+macro
+  nonascii  [^\0-\177]
+  sing      {nonascii}*
+  string    "{sing}"
+rule
+  {string}       { [:STRING, text] }
+end
+    }
+
+    rex.parse
+
+    output = StringIO.new
+    rex.write_scanner output
+    assert_match '"[^\0-\177]*"', output.string
+  end
+
+  def assert_tokens expected, scanner
+    tokens = []
+    while token = scanner.next_token
+      tokens << token
+    end
+    assert_equal expected, tokens
+  end
+end