RubyGems - rexical - Versions diffs - 1.0.3 - Mend

rexical 1.0.3

Potentially problematic release.

This version of rexical might be problematic. Click here for more details.

Files changed (38) hide show

@@ -0,0 +1,15 @@
+#
+# eooro1.rex
+# lexical definition sample for rex
+#
+class Error1
+macro
+  BLANK         [\ \t]+
+rule
+  BLANK         # no action
+  \d+           { [:digit, text.to_i] }
+  \w+           { [:word, text] }
+  \n
+#  .             { [text, text] }
+end

data/sample/error2.rex ADDED

@@ -0,0 +1,15 @@
+#
+# error2.rex
+# lexical definition sample for rex
+#
+class Error2
+macro
+  BLANK         [\ \t]+
+rule
+  BLANK         # no action
+  \d+           { [:digit, text.to_i] }
+  \w+           { [:word, text] }
+  \n
+  .             { state = :NONDEF ; [text, text] }
+end

data/sample/sample.html ADDED

@@ -0,0 +1,32 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+  "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+  <head>
+    <!-- Comment -->
+    <title> Title </title>
+    <meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">
+    <meta name="robots" content="noindex,nofollow">
+    <link rel="stylesheet" type="text/css" href="sample.css">
+  </head>
+  <script language="JavaScript">
+  <!--
+  function foo(msg) {
+    ndx = Math.floor(Math.random() * msg.length);
+    return msg[ndx];
+  }
+  //-->
+  </script>
+  <body>
+    <div id="div1" class="div">
+      <p id="p1" class="p">
+        HTML
+        4.01
+      </p>
+    </div>
+    <form action="sample.cgi" method="post">
+      <p>
+        <input type="text" name="t1" id="t1" value="TEXT1">
+      </p>
+    </form>
+  </body>
+</html>

data/sample/sample.rex ADDED

@@ -0,0 +1,15 @@
+#
+# sample.rex
+# lexical definition sample for rex
+#
+class Sample
+macro
+  BLANK         [\ \t]+
+rule
+  BLANK         # no action
+  \d+           { [:digit, text.to_i] }
+  \w+           { [:word, text] }
+  \n
+  .             { [text, text] }
+end

data/sample/sample.rex.rb ADDED

@@ -0,0 +1,100 @@
+#
+# DO NOT MODIFY!!!!
+# This file is automatically generated by rex 1.0.0
+# from lexical definition file "sample.rex".
+#
+require 'racc/parser'
+#
+# sample.rex
+# lexical definition sample for rex
+#
+class Sample < Racc::Parser
+  require 'strscan'
+  class ScanError < StandardError ; end
+  attr_reader :lineno
+  attr_reader :filename
+  def scan_setup ; end
+  def action &block
+    yield
+  end
+  def scan_str( str )
+    scan_evaluate  str
+    do_parse
+  end
+  def load_file( filename )
+    @filename = filename
+    open(filename, "r") do |f|
+      scan_evaluate  f.read
+    end
+  end
+  def scan_file( filename )
+    load_file  filename
+    do_parse
+  end
+  def next_token
+    @rex_tokens.shift
+  end
+  def scan_evaluate( str )
+    scan_setup
+    @rex_tokens = []
+    @lineno  =  1
+    ss = StringScanner.new(str)
+    state = nil
+    until ss.eos?
+      text = ss.peek(1)
+      @lineno  +=  1  if text == "\n"
+      case state
+      when nil
+        case
+        when (text = ss.scan(/BLANK/))
+          ;
+        when (text = ss.scan(/\d+/))
+           @rex_tokens.push action { [:digit, text.to_i] }
+        when (text = ss.scan(/\w+/))
+           @rex_tokens.push action { [:word, text] }
+        when (text = ss.scan(/\n/))
+          ;
+        when (text = ss.scan(/./))
+           @rex_tokens.push action { [text, text] }
+        else
+          text = ss.string[ss.pos .. -1]
+          raise  ScanError, "can not match: '" + text + "'"
+        end  # if
+      else
+        raise  ScanError, "undefined state: '" + state.to_s + "'"
+      end  # case state
+    end  # until ss
+  end  # def scan_evaluate
+end # class
+if __FILE__ == $0
+  exit  if ARGV.size != 1
+  filename = ARGV.shift
+  rex = Sample.new
+  begin
+    rex.load_file  filename
+    while  token = rex.next_token
+      p token
+    end
+  rescue
+    $stderr.printf  "%s:%d:%s\n", rex.filename, rex.lineno, $!.message
+  end
+end

data/sample/sample.xhtml ADDED

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="Shift_JIS"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+  "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <!-- Comment -->
+    <title>Title</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS" />
+    <meta name="robots" content="noindex,nofollow" />
+    <link rel="stylesheet" type="text/css" href="sample.css" />
+  </head>
+  <script language="JavaScript">
+  <!--
+  function foo(msg) {
+    ndx = Math.floor(Math.random() * msg.length);
+    return msg[ndx];
+  }
+  //-->
+  </script>
+  <body>
+    <div id="div1" class="div">
+      <p id="p1" class="p">
+        XHTML 1.1
+      </p>
+    </div>
+    <form action="sample.cgi" method="post">
+      <p>
+        <input type="text" name="t1" id="t1" value="TEXT1" />
+      </p>
+    </form>
+  </body>
+</html>

data/sample/sample1.c ADDED

@@ -0,0 +1,9 @@
+int main(int argc, char **argv)
+{
+  /* block remark */
+  int i = 100;
+  // inline remark
+  printf("hello, world\n");
+}

data/sample/sample1.rex ADDED

@@ -0,0 +1,43 @@
+#
+# sample1.rex
+# lexical definition sample for rex
+#
+# usage
+#  rex  sample1.rex  --stub
+#  ruby sample1.rex.rb  sample1.c
+#
+class Sample1
+macro
+  BLANK         \s+
+  REM_IN        \/\*
+  REM_OUT       \*\/
+  REM           \/\/
+rule
+# [:state]  pattern  [actions]
+# remark
+                {REM_IN}        { state = :REMS; [:rem_in, text] }
+  :REMS         {REM_OUT}       { state = nil;   [:rem_out, text] }
+  :REMS         .*(?={REM_OUT}) {                [:remark, text] }
+                {REM}           { state = :REM;  [:rem_in, text] }
+  :REM          \n              { state = nil;   [:rem_out, text] }
+  :REM          .*(?=$)         {                [:remark, text] }
+# literal
+                \"[^"]*\"       { [:string, text] } # "
+                \'[^']\'        { [:character, text] } # '
+# skip
+                {BLANK}         # no action
+# numeric
+                \d+             { [:digit, text.to_i] }
+# identifier
+                \w+             { [:word, text] }
+                .               { [text, text] }
+end

data/sample/sample2.bas ADDED

@@ -0,0 +1,4 @@
+' inline remark
+i = 100
+input st
+print "hello, world"

data/sample/sample2.rex ADDED

@@ -0,0 +1,33 @@
+#
+# sample2.rex
+# lexical definition sample for rex
+#
+# usage
+#  rex  sample2.rex  --stub
+#  ruby sample2.rex.rb  sample2.bas
+#
+class Sample2
+option
+  ignorecase
+macro
+  BLANK         \s+
+  REMARK        \'              # '
+rule
+                {REMARK}        { state = :REM;  [:rem_in, text] } # '
+  :REM          \n              { state = nil;   [:rem_out, text] }
+  :REM          .*(?=$)         {                [:remark, text] }
+                \"[^"]*\"       { [:string, text] } # "
+                {BLANK}         # no action
+                INPUT           { [:input, text] }
+                PRINT           { [:print, text] }
+                \d+             { [:digit, text.to_i] }
+                \w+             { [:word, text] }
+                .               { [text, text] }
+end

data/sample/simple.html ADDED

@@ -0,0 +1,7 @@
+<html>
+  <body>
+    <p>
+      Hello World.
+    </p>
+  </body>
+</html>

data/sample/simple.xhtml ADDED

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="Shift_JIS"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+  "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <body>
+    <p>
+      XHTML 1.1
+    </p>
+  </body>
+</html>

data/sample/xhtmlparser.racc ADDED

@@ -0,0 +1,66 @@
+#
+# xml parser
+#
+class XHTMLParser
+rule
+  target        : /* none */
+                | xml_doc
+  xml_doc       : xml_header  extra  xml_body
+                | xml_header  xml_body
+                | xml_body
+  xml_header    : xtag_in  element  attributes  xtag_out
+  xml_body      : tag_from  contents  tag_to
+  tag_from      : tag_in  element  attributes  tag_out
+  tag_empty     : tag_in  element  attributes  etag_out
+  tag_to        : etag_in  element  tag_out
+  attributes    : /* none */
+                | attributes  attribute
+  attribute     : attr  equal  quoted
+  quoted        : quote1  value  quote1
+                | quote2  value  quote2
+  contents      : /* none */
+                | contents  content
+  content       : text
+                | extra
+                | tag_from  contents  tag_to
+                | tag_empty
+  extra         : tag_in  ext  extra_texts  tag_out
+  extra_texts   : /* none */
+                | extra_texts  rem_in  remtexts  rem_out
+                | extra_texts  exttext
+  remtexts      : remtext
+                | remtexts  remtext
+end
+---- header ----
+#
+# generated by racc
+#
+require 'xhtmlparser.rex'
+---- inner ----
+---- footer ----
+exit  if ARGV.size == 0
+filename = ARGV.shift
+htmlparser = XHTMLParser.new
+htmlparser.scan_file  filename

data/sample/xhtmlparser.rex ADDED

@@ -0,0 +1,72 @@
+#
+# xhtmlparser.rex
+# lexical scanner definition for rex
+#
+# usage
+#   rex  xhtmlparser.rex  --stub
+#   ruby xhtmlparser.rex.rb  sample.xhtml
+#
+class XHTMLParser
+option
+  ignorecase
+macro
+  BLANK         \s+
+  TAG_IN        \<
+  TAG_OUT       \>
+  ETAG_IN       \<\/
+  ETAG_OUT      \/\>
+  XTAG_IN       \<\?
+  XTAG_OUT      \?\>
+  EXT           \!
+  REM           \-\-
+  EQUAL         \=
+  Q1            \'
+  Q2            \"
+rule
+# [:state]  pattern  [actions]
+                {XTAG_IN}               { state = :TAG; [:xtag_in, text] }
+                {ETAG_IN}               { state = :TAG; [:etag_in, text] }
+                {TAG_IN}                { state = :TAG; [:tag_in, text] }
+  :TAG          {EXT}                   { state = :EXT; [:ext, text] }
+  :EXT          {REM}                   { state = :REM; [:rem_in, text] }
+  :EXT          {XTAG_OUT}              { state = nil;  [:xtag_out, text] }
+  :EXT          {TAG_OUT}               { state = nil;  [:tag_out, text] }
+  :EXT          .+(?={REM})             {               [:exttext, text] }
+  :EXT          .+(?={TAG_OUT})         {               [:exttext, text] }
+  :EXT          .+(?=$)                 {               [:exttext, text] }
+  :EXT          \n
+  :REM          {REM}                   { state = :EXT; [:rem_out, text] }
+  :REM          .+(?={REM})             {               [:remtext, text] }
+  :REM          .+(?=$)                 {               [:remtext, text] }
+  :REM          \n
+  :TAG          {BLANK}
+  :TAG          {XTAG_OUT}              { state = nil;  [:xtag_out, text] }
+  :TAG          {ETAG_OUT}              { state = nil;  [:etag_out, text] }
+  :TAG          {TAG_OUT}               { state = nil;  [:tag_out, text] }
+  :TAG          {EQUAL}                 {               [:equal, text] }
+  :TAG          {Q1}                    { state = :Q1;  [:quote1, text] } # '
+  :Q1           {Q1}                    { state = :TAG; [:quote1, text] } # '
+  :Q1           [^{Q1}]+(?={Q1})        {               [:value, text] }  # '
+  :TAG          {Q2}                    { state = :Q2;  [:quote2, text] } # "
+  :Q2           {Q2}                    { state = :TAG; [:quote2, text] } # "
+  :Q2           [^{Q2}]+(?={Q2})        {               [:value, text] }  # "
+  :TAG          [\w\-]+(?={EQUAL})      {               [:attr, text] }
+  :TAG          [\w\-]+                 {               [:element, text] }
+                \s+(?=\S)
+                .*\S(?=\s*{ETAG_IN})    {               [:text, text] }
+                .*\S(?=\s*{TAG_IN})     {               [:text, text] }
+                .*\S(?=\s*$)            {               [:text, text] }
+                \s+(?=$)
+inner
+end