rex 1.0.2 → 2.0.2

data/README.ja

@@ -1,73 +0,0 @@
-Rex README
-===========
-
-  Rex �� Ruby �Τ���Υ�����ʥ����ͥ졼���Ǥ���
-  lex �� Ruby �Ǥ��������ޤ���
-  Racc �ȤȤ�˻Ȥ��褦���߷פ���Ƥ��ޤ���
-
-
-ɬ�״Ķ�
---------
-
-  *  ruby 1.8 �ʹ�
-
-
-���󥹥ȡ���
-------------
-
-  �ѥå������Υȥåץǥ��쥯�ȥ�Ǽ��Τ褦�����Ϥ��Ƥ���������
-  ($ ���̾�桼����# �ϥ롼�ȤΥץ���ץȤǤ�)
-
-      $ ruby setup.rb config
-      $ ruby setup.rb setup
-     ($ su)
-      # ruby setup.rb install
-
-  ������̾�Υѥ��� Racc �����󥹥ȡ��뤵��ޤ�����ʬ�ι���
-  �ʥǥ��쥯�ȥ�˥��󥹥ȡ��뤷�����Ȥ��ϡ�setup.rb config ��
-  �Ƽ索�ץ�����Ĥ��Ƽ¹Ԥ��Ƥ������������ץ����Υꥹ�Ȥ�
-
-      $ ruby setup.rb --help
-
-  �Ǹ����ޤ���
-
-
-�ƥ���
-------
-
-  sample/ �ʲ��ˤ����Ĥ� Rex ��ʸˡ�ե�����Υ���ץ뤬�Ѱ�
-  ���Ƥ���ޤ����ʲ���¹Ԥ��Ƥ���������
-
-      $ rex  sample1.rex  --stub
-      $ ruby sample1.rex.rb  sample1.c
-
-      $ rex  sample2.rex  --stub
-      $ ruby sample2.rex.rb  sample2.bas
-
-      $ racc calc3.racc
-      $ rex  calc3.rex
-      $ ruby calc3.tab.rb
-
-  Rex �ξܤ���ʸˡ�� doc/ �ǥ��쥯�ȥ�ʲ��򸫤Ƥ���������
-  �ޤ�������� sample/ �ǥ��쥯�ȥ�ʲ��򸫤Ƥ���������
-
-
-�饤����
-----------
-
-  �饤���󥹤� GNU Lesser General Public License (LGPL) version 2
-  �Ǥ����������桼�����񤤤���§�ե�����䡢Racc ������������������
-  Ruby ������ץȤϤ����оݳ��Ǥ��������ʥ饤���󥹤����ۤ��Ƥ���������
-
-
-�Х��ʤ�
---------
-
-  Rex ��ȤäƤ��ƥХ��餷�����ݤ����������顢�����Υ��ɥ쥹�ޤ�
-  �᡼��򤯤�������
-  ���ΤȤ��ϤǤ�������Х���Ƹ��Ǥ���ʸˡ�ե�������դ��Ƥ���������
-
-
-                                                   ARIMA Yasuhiro
-                                         arima.yasuhiro@nifty.com
-                            http://raa.ruby-lang.org/project/rex/

data/README.rdoc

@@ -1,53 +0,0 @@
-= Rex
-
-* http://rex.rubyforge.org
-* http://github.com/aaronp/rex/tree/master
-
-== DESCRIPTION
-
-Rex is a lexical scanner generator.
-It is written in Ruby itself, and generates Ruby program.
-It is designed for use with Racc.
-
-
-== SYNOPSIS
-
-Here is a sample lexical definition:
-
-  class Sample
-  macro
-    BLANK         [\ \t]+
-  rule
-    BLANK         # no action
-    \d+           { [:digit, text.to_i] }
-    \w+           { [:word, text] }
-    \n
-    .             { [text, text] }
-  end
-
-Here is the command line usage:
-
-  $ rex  sample1.rex  --stub
-  $ ruby sample1.rex.rb  sample1.c
-  
-  $ rex  sample2.rex  --stub
-  $ ruby sample2.rex.rb  sample2.bas
-  
-  $ racc calc3.racc
-  $ rex  calc3.rex
-  $ ruby calc3.tab.rb
-
-== REQUIREMENTS
-
-*  ruby version 1.8.x or later.
-
-== INSTALL
-
-* sudo gem install rex
-
-== LICENSE
-
-Rex is distributed under the terms of the GNU Lesser General
-Public License version 2. Note that you do NOT need to follow
-LGPL for your own parser (Rex outputs). You can provide those
-files under any licenses you want.

data/Rakefile

@@ -1,28 +0,0 @@
-# -*- ruby -*-
-
-require 'rubygems'
-require 'hoe'
-
-$LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
-require 'rex'
-
-HOE = Hoe.new('rex', Rex::VERSION) do |p|
-  p.readme_file  = 'README.rdoc'
-  p.history_file = 'CHANGELOG.rdoc'
-  p.developer('Aaron Patterson', 'aaronp@rubyforge.org')
-  p.extra_rdoc_files = FileList['*.rdoc']
-  p.rubyforge_name = 'ruby-rex'
-end
-
-namespace :gem do
-  namespace :spec do
-    task :dev do
-      File.open("#{HOE.name}.gemspec", 'w') do |f|
-        HOE.spec.version = "#{HOE.version}.#{Time.now.strftime("%Y%m%d%H%M%S")}"
-        f.write(HOE.spec.to_ruby)
-      end
-    end
-  end
-end
-
-# vim: syntax=Ruby

data/bin/rex

@@ -1,18 +0,0 @@
-#!/usr/bin/env ruby
-#
-# rex
-#
-#   Copyright (c) 2005-2006 ARIMA Yasuhiro <arima.yasuhiro@nifty.com>
-#
-#   This program is free software.
-#   You can distribute/modify this program under the terms of
-#   the GNU LGPL, Lesser General Public License version 2.1.
-#   For details of LGPL, see the file "COPYING".
-#
-
-## ---------------------------------------------------------------------
-
-require 'rubygems'
-require 'rex'
-
-Rex::Cmd.new.run

data/lib/rex/generator.rb

@@ -1,523 +0,0 @@
-#
-# generator.rb
-#
-#   Copyright (c) 2005-2006 ARIMA Yasuhiro <arima.yasuhiro@nifty.com>
-#
-#   This program is free software.
-#   You can distribute/modify this program under the terms of
-#   the GNU Lesser General Public License version 2 or later.
-#
-
-require 'strscan'
-module Rex
-
-## ---------------------------------------------------------------------
-  class ParseError < StandardError ; end
-
-## ---------------------------------------------------------------------
-  class Generator
-
-## ---------------------------------------------------------------------
-    attr_accessor :grammar_file
-    attr_accessor :grammar_lines
-    attr_accessor :scanner_file
-    attr_accessor :module_name
-    attr_accessor :class_name
-    attr_accessor :lineno
-    attr_accessor :rules
-    attr_accessor :exclusive_states
-    attr_accessor :ignorecase
-    attr_accessor :independent
-    attr_accessor :debug
-
-## ---------------------------------------------------------------------
-    def initialize(opts)
-      @lineno  =  0
-      @macro  =  {}
-      @rules  =  []
-      @exclusive_states = [nil]
-      @grammar_lines  =  nil
-      @scanner_header  =  ""
-      @scanner_footer  =  ""
-      @scanner_inner  =  ""
-      @opt  =  opts
-    end
-
-## ---------------------------------------------------------------------
-    def add_header( st )
-      @scanner_header  +=  "#{st}\n"
-    end
-
-## ---------------------------------------------------------------------
-    def add_footer( st )
-      @scanner_footer  +=  "#{st}\n"
-    end
-
-## ---------------------------------------------------------------------
-    def add_inner( st )
-      @scanner_inner  +=  "#{st}\n"
-    end
-
-## ---------------------------------------------------------------------
-    def add_option( st )
-      opts = st.split
-      opts.each do |opt|
-        case opt
-        when /ignorecase/i
-          @opt['--ignorecase'] = true
-        when /stub/i
-          @opt['--stub'] = true
-        when /independent/i
-          @opt['--independent'] = true
-        end
-      end
-    end
-
-## ---------------------------------------------------------------------
-    def add_macro( st )
-      ss  =  StringScanner.new(st)
-      ss.scan(/\s+/)
-      key = ss.scan(/\S+/)
-      ss.scan(/\s+/)
-      st = ss.post_match
-      len  =  st.size
-      ndx  =  0
-      while ndx <= len
-        c  =  st[ndx,1]
-        ndx  +=  1
-        case  c
-        when '\\'
-          ndx  +=  1
-          next
-        when '#', ' '
-          ndx  -=  1
-          break
-        end
-      end
-      expr = st[0,ndx]
-      key  =  '{' + key + '}'
-      @macro.each_pair do |k, e|
-        expr.gsub!(k) { |m| e }
-      end
-      @macro[key]  =  expr
-    rescue
-      raise ParseError, "parse error in add_macro:'#{st}'"
-    end
-
-## ---------------------------------------------------------------------
-    def add_rule( rule_state, rule_expr, rule_action=nil )
-      st = rule_expr.dup
-      @macro.each_pair do |k, e|
-        rule_expr.gsub!(k) { |m| e }
-      end
-      if rule_state.to_s[1,1] =~ /[A-Z]/
-        @exclusive_states << rule_state  unless @exclusive_states.include?(rule_state)
-        exclusive_state = rule_state
-        start_state = nil
-      else
-        exclusive_state = nil
-        start_state = rule_state
-      end
-      rule = [exclusive_state, start_state, rule_expr, rule_action]
-      @rules << rule
-    rescue
-      raise ParseError, "parse error in add_rule:'#{st}'"
-    end
-
-    def read_grammar
-      @grammar_lines = StringScanner.new File.read(grammar_file)
-    end
-
-    def next_line
-      @lineno += 1
-      @grammar_lines.scan_until(/\n/).chomp
-    rescue
-      nil
-    end
-
-    def parse
-      state1  =  :HEAD
-      state2  =  nil
-      state3  =  nil
-      lastmodes  =  []
-      while st = next_line
-        case state1
-        when :FOOT
-          add_footer  st
-
-        when :HEAD
-          ss  =  StringScanner.new(st)
-          if ss.scan(/class/)
-            state1  =  :CLASS
-            st  =  ss.post_match.strip
-            if st =~ /^(\S+)::(\S+)/
-              @module_name  =  $1
-              @class_name  =  $2
-            else
-              @module_name  =  nil
-              @class_name  =  st
-            end
-          else
-            add_header  st
-          end
-
-        when :CLASS
-          s = st.strip
-          next  if s.size == 0 or s[0,1] == '#'
-
-          ss  =  StringScanner.new(st)
-          if ss.scan(/option.*$/)
-            state2 = :OPTION
-            next
-          end
-          if ss.scan(/inner.*$/)
-            state2 = :INNER
-            next
-          end
-          if ss.scan(/macro.*$/)
-            state2 = :MACRO
-            next
-          end
-          if ss.scan(/rule.*$/)
-            state2 = :RULE
-            next
-          end
-          if ss.scan(/end.*$/)
-            state1 = :FOOT
-            next
-          end
-
-          case state2
-          when :OPTION
-            add_option  st
-
-          when :INNER
-            add_inner  st
-
-          when :MACRO
-            add_macro  st
-
-          when :RULE
-            case state3
-            when nil
-              rule_state, rule_expr, rule_action  =  parse_rule(st)
-              if rule_action =~ /\s*\{/
-                lastmodes = parse_action(rule_action, lastmodes)
-                if lastmodes.empty?
-                  add_rule  rule_state, rule_expr, rule_action
-                else
-                  state3  =  :CONT
-                  rule_action  +=  "\n"
-                end
-              else
-                add_rule  rule_state, rule_expr
-              end
-
-            when :CONT
-              rule_action  +=  "#{st}\n"
-              lastmodes = parse_action(st, lastmodes)
-              if lastmodes.empty?
-                state3  =  nil
-                add_rule  rule_state, rule_expr, rule_action
-              else
-              end
-
-            end # case state3
-
-          end # case state2
-
-        end # case state1
-
-      end # while
-
-    end
-
-## ---------------------------------------------------------------------
-    def parse_rule(st)
-      st.strip!
-      return  if st.size == 0 or st[0,1] == '#'
-      ss  =  StringScanner.new(st)
-      ss.scan(/\s+/)
-      rule_state  =  ss.scan(/\:\S+/)
-      ss.scan(/\s+/)
-      rule_expr  =  ss.scan(/\S+/)
-      ss.scan(/\s+/)
-      [rule_state, rule_expr, ss.post_match]
-    end
-
-## ---------------------------------------------------------------------
-    def parse_action(st, lastmodes=[])
-      modes  =  lastmodes
-      mode  =  lastmodes[-1]
-      ss  =  StringScanner.new(st)
-      until ss.eos?
-        c  =  ss.scan(/./)
-        case  c
-        when '#'
-          if (mode == :brace) or (mode == nil)
-            #p [c, mode, modes]
-            return  modes
-          end
-        when '{'
-          if (mode == :brace) or (mode == nil)
-            mode = :brace
-            modes.push  mode
-          end
-        when '}'
-          if (mode == :brace)
-            modes.pop
-            mode = modes[0]
-          end
-        when "'"
-          if (mode == :brace)
-            mode = :quote
-            modes.push  mode
-          elsif (mode == :quote)
-            modes.pop
-            mode = modes[0]
-          end
-        when '"'
-          if (mode == :brace)
-            mode = :doublequote
-            modes.push  mode
-          elsif (mode == :doublequote)
-            modes.pop
-            mode = modes[0]
-          end
-        when '`'
-          if (mode == :brace)
-            mode = :backquote
-            modes.push  mode
-          elsif (mode == :backquote)
-            modes.pop
-            mode = modes[0]
-          end
-        end
-      end
-      #p [c, mode, modes]
-      return  modes
-    end
-
-## ---------------------------------------------------------------------
-
-REX_HEADER = <<-REX_EOT
-#--
-# DO NOT MODIFY!!!!
-# This file is automatically generated by rex %s
-# from lexical definition file "%s".
-#++
-
-REX_EOT
-
-REX_UTIL = <<-REX_EOT
-  require 'strscan'
-
-  class ScanError < StandardError ; end
-
-  attr_reader :lineno
-  attr_reader :filename
-
-  def scan_setup ; end
-
-  def action &block
-    yield
-  end
-
-  def scan_str( str )
-    scan_evaluate  str
-    do_parse
-  end
-
-  def load_file( filename )
-    @filename = filename
-    open(filename, "r") do |f|
-      scan_evaluate  f.read
-    end
-  end
-
-  def scan_file( filename )
-    load_file  filename
-    do_parse
-  end
-
-REX_EOT
-
-REX_NEXTTOKEN_DEBUG = <<-REX_EOT
-  def next_token
-    p token = @rex_tokens.shift
-    token
-  end
-REX_EOT
-
-REX_NEXTTOKEN = <<-REX_EOT
-  def next_token
-    @rex_tokens.shift
-  end
-REX_EOT
-
-REX_STUB = <<-REX_EOT
-
-if __FILE__ == $0
-  exit  if ARGV.size != 1
-  filename = ARGV.shift
-  rex = %s.new
-  begin
-    rex.load_file  filename
-    while  token = rex.next_token
-      p token
-    end
-  rescue
-    $stderr.printf  %s, rex.filename, rex.lineno, $!.message
-  end
-end
-REX_EOT
-
-## ---------------------------------------------------------------------
-
-
-    def scanner_io
-      unless  scanner_file = @opt['--output-file']
-        scanner_file  =  grammar_file + ".rb"
-      end
-      f = File.open(scanner_file, 'w')
-    end
-    private :scanner_io
-
-    def write_scanner f = scanner_io
-      ## scan flag
-      flag = ""
-      flag += "i"  if @opt['--ignorecase']
-      ## header
-      f.printf REX_HEADER, Rex::VERSION, grammar_file
-
-      unless @opt['--independent']
-        f.printf "require 'racc/parser'\n"
-      end
-
-      @scanner_header.each_line do |s|
-        f.print s
-      end
-      if @module_name
-        f.puts "module #{@module_name}"
-      end
-      if @opt['--independent']
-        f.puts "class #{@class_name}"
-      else
-        f.puts "class #{@class_name} < Racc::Parser"
-      end
-
-      ## utility method
-      f.print REX_UTIL
-
-      if @opt['--debug']
-        f.print REX_NEXTTOKEN_DEBUG
-      else
-        f.print REX_NEXTTOKEN
-      end
-
-      ## scanner method
-
-      f.print <<-REX_EOT
-
-  def scan_evaluate( str )
-    scan_setup
-    @rex_tokens = []
-    @lineno  =  1
-    ss = StringScanner.new(str)
-    state = nil
-    until ss.eos?
-      text = ss.peek(1)
-      @lineno  +=  1  if text == "\\n"
-      case state
-      REX_EOT
-
-      exclusive_states.each do |es|
-        f.printf <<-REX_EOT
-      when #{es ? es.to_s : "nil"}
-        case
-        REX_EOT
-        rules.each do |rule|
-          exclusive_state, start_state, rule_expr, rule_action = *rule
-          if es == exclusive_state
-
-            if rule_action
-              if start_state
-                f.print <<-REX_EOT
-        when (state == #{start_state}) and (text = ss.scan(/#{rule_expr}/#{flag}))
-           @rex_tokens.push action #{rule_action}
-
-                REX_EOT
-              else
-                f.print <<-REX_EOT
-        when (text = ss.scan(/#{rule_expr}/#{flag}))
-           @rex_tokens.push action #{rule_action}
-
-                REX_EOT
-              end
-            else
-              if start_state
-                f.print <<-REX_EOT
-        when (state == #{start_state}) and (text = ss.scan(/#{rule_expr}/#{flag}))
-          ;
-
-                REX_EOT
-              else
-                f.print <<-REX_EOT
-        when (text = ss.scan(/#{rule_expr}/#{flag}))
-          ;
-
-                REX_EOT
-              end
-            end
-
-          end
-        end
-        f.print <<-REX_EOT
-        else
-          text = ss.string[ss.pos .. -1]
-          raise  ScanError, "can not match: '" + text + "'"
-        end  # if
-
-        REX_EOT
-      end
-      f.print <<-REX_EOT
-      else
-        raise  ScanError, "undefined state: '" + state.to_s + "'"
-      end  # case state
-    end  # until ss
-  end  # def scan_evaluate
-
-      REX_EOT
-
-      ## inner method
-      @scanner_inner.each_line do |s|
-        f.print s
-      end
-      f.puts "end # class"
-      f.puts "end # module"  if @module_name
-
-      ## footer
-      @scanner_footer.each_line do |s|
-        f.print s
-      end # case
-
-      ## stub main
-      f.printf REX_STUB, @class_name, '"%s:%d:%s\n"'  if @opt['--stub']
-      f.close
-
-    end ## def
-  end ## class
-end ## module
-
-
-## ---------------------------------------------------------------------
-## test
-
-if __FILE__ == $0
-  rex = Rex::Generator.new(nil)
-  rex.grammar_file = "sample.rex"
-  rex.read_grammar
-  rex.parse
-  rex.write_scanner
-end
-