rex 0.0.4 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.rdoc +7 -0
- data/DOCUMENTATION.en.rdoc +215 -0
- data/DOCUMENTATION.ja.rdoc +205 -0
- data/Manifest.txt +37 -0
- data/README.ja +73 -0
- data/README.rdoc +53 -0
- data/Rakefile +28 -0
- data/bin/rex +18 -0
- data/lib/rex.rb +3 -23
- data/lib/rex/generator.rb +523 -0
- data/lib/rex/info.rb +16 -0
- data/lib/rex/rexcmd.rb +136 -0
- data/sample/a.cmd +1 -0
- data/sample/b.cmd +1 -0
- data/sample/c.cmd +4 -0
- data/sample/calc3.racc +47 -0
- data/sample/calc3.rex +15 -0
- data/sample/calc3.rex.rb +94 -0
- data/sample/calc3.tab.rb +188 -0
- data/sample/error1.rex +15 -0
- data/sample/error2.rex +15 -0
- data/sample/sample.html +32 -0
- data/sample/sample.rex +15 -0
- data/sample/sample.rex.rb +100 -0
- data/sample/sample.xhtml +32 -0
- data/sample/sample1.c +9 -0
- data/sample/sample1.rex +43 -0
- data/sample/sample2.bas +4 -0
- data/sample/sample2.rex +33 -0
- data/sample/simple.html +7 -0
- data/sample/simple.xhtml +10 -0
- data/sample/xhtmlparser.racc +66 -0
- data/sample/xhtmlparser.rex +72 -0
- data/test/assets/test.rex +12 -0
- data/test/rex-20060125.rb +152 -0
- data/test/rex-20060511.rb +143 -0
- data/test/test_generator.rb +184 -0
- metadata +74 -57
- data/README +0 -50
- data/lib/rex/array.rb +0 -13
- data/lib/rex/kernel.rb +0 -11
- data/lib/rex/modules/boolean.rb +0 -14
- data/lib/rex/modules/caller_name.rb +0 -15
- data/lib/rex/modules/ordinal.rb +0 -35
- data/lib/rex/modules/propercase.rb +0 -15
- data/lib/rex/modules/roman.rb +0 -41
- data/lib/rex/modules/scrub.rb +0 -66
- data/lib/rex/modules/stacktrace.rb +0 -31
- data/lib/rex/modules/swap.rb +0 -13
- data/lib/rex/modules/to_proc.rb +0 -13
- data/lib/rex/modules/tuple.rb +0 -14
- data/lib/rex/modules/wrap.rb +0 -15
- data/lib/rex/numeric.rb +0 -19
- data/lib/rex/string.rb +0 -21
- data/lib/rex/symbol.rb +0 -13
- data/test/boolean.rb +0 -64
- data/test/caller_name.rb +0 -26
- data/test/ordinal.rb +0 -609
- data/test/propercase.rb +0 -40
- data/test/roman.rb +0 -335
- data/test/scrub.rb +0 -27
- data/test/stacktrace.rb +0 -24
- data/test/swap.rb +0 -44
- data/test/to_proc.rb +0 -24
- data/test/tuple.rb +0 -26
- data/test/wrap.rb +0 -44
data/sample/error1.rex
ADDED
data/sample/error2.rex
ADDED
data/sample/sample.html
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
|
|
2
|
+
"http://www.w3.org/TR/html4/loose.dtd">
|
|
3
|
+
<html>
|
|
4
|
+
<head>
|
|
5
|
+
<!-- Comment -->
|
|
6
|
+
<title> Title </title>
|
|
7
|
+
<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS">
|
|
8
|
+
<meta name="robots" content="noindex,nofollow">
|
|
9
|
+
<link rel="stylesheet" type="text/css" href="sample.css">
|
|
10
|
+
</head>
|
|
11
|
+
<script language="JavaScript">
|
|
12
|
+
<!--
|
|
13
|
+
function foo(msg) {
|
|
14
|
+
ndx = Math.floor(Math.random() * msg.length);
|
|
15
|
+
return msg[ndx];
|
|
16
|
+
}
|
|
17
|
+
//-->
|
|
18
|
+
</script>
|
|
19
|
+
<body>
|
|
20
|
+
<div id="div1" class="div">
|
|
21
|
+
<p id="p1" class="p">
|
|
22
|
+
HTML
|
|
23
|
+
4.01
|
|
24
|
+
</p>
|
|
25
|
+
</div>
|
|
26
|
+
<form action="sample.cgi" method="post">
|
|
27
|
+
<p>
|
|
28
|
+
<input type="text" name="t1" id="t1" value="TEXT1">
|
|
29
|
+
</p>
|
|
30
|
+
</form>
|
|
31
|
+
</body>
|
|
32
|
+
</html>
|
data/sample/sample.rex
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
#
|
|
2
|
+
# DO NOT MODIFY!!!!
|
|
3
|
+
# This file is automatically generated by rex 1.0.0
|
|
4
|
+
# from lexical definition file "sample.rex".
|
|
5
|
+
#
|
|
6
|
+
|
|
7
|
+
require 'racc/parser'
|
|
8
|
+
#
|
|
9
|
+
# sample.rex
|
|
10
|
+
# lexical definition sample for rex
|
|
11
|
+
#
|
|
12
|
+
|
|
13
|
+
class Sample < Racc::Parser
|
|
14
|
+
require 'strscan'
|
|
15
|
+
|
|
16
|
+
class ScanError < StandardError ; end
|
|
17
|
+
|
|
18
|
+
attr_reader :lineno
|
|
19
|
+
attr_reader :filename
|
|
20
|
+
|
|
21
|
+
def scan_setup ; end
|
|
22
|
+
|
|
23
|
+
def action &block
|
|
24
|
+
yield
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def scan_str( str )
|
|
28
|
+
scan_evaluate str
|
|
29
|
+
do_parse
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def load_file( filename )
|
|
33
|
+
@filename = filename
|
|
34
|
+
open(filename, "r") do |f|
|
|
35
|
+
scan_evaluate f.read
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def scan_file( filename )
|
|
40
|
+
load_file filename
|
|
41
|
+
do_parse
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def next_token
|
|
45
|
+
@rex_tokens.shift
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def scan_evaluate( str )
|
|
49
|
+
scan_setup
|
|
50
|
+
@rex_tokens = []
|
|
51
|
+
@lineno = 1
|
|
52
|
+
ss = StringScanner.new(str)
|
|
53
|
+
state = nil
|
|
54
|
+
until ss.eos?
|
|
55
|
+
text = ss.peek(1)
|
|
56
|
+
@lineno += 1 if text == "\n"
|
|
57
|
+
case state
|
|
58
|
+
when nil
|
|
59
|
+
case
|
|
60
|
+
when (text = ss.scan(/BLANK/))
|
|
61
|
+
;
|
|
62
|
+
|
|
63
|
+
when (text = ss.scan(/\d+/))
|
|
64
|
+
@rex_tokens.push action { [:digit, text.to_i] }
|
|
65
|
+
|
|
66
|
+
when (text = ss.scan(/\w+/))
|
|
67
|
+
@rex_tokens.push action { [:word, text] }
|
|
68
|
+
|
|
69
|
+
when (text = ss.scan(/\n/))
|
|
70
|
+
;
|
|
71
|
+
|
|
72
|
+
when (text = ss.scan(/./))
|
|
73
|
+
@rex_tokens.push action { [text, text] }
|
|
74
|
+
|
|
75
|
+
else
|
|
76
|
+
text = ss.string[ss.pos .. -1]
|
|
77
|
+
raise ScanError, "can not match: '" + text + "'"
|
|
78
|
+
end # if
|
|
79
|
+
|
|
80
|
+
else
|
|
81
|
+
raise ScanError, "undefined state: '" + state.to_s + "'"
|
|
82
|
+
end # case state
|
|
83
|
+
end # until ss
|
|
84
|
+
end # def scan_evaluate
|
|
85
|
+
|
|
86
|
+
end # class
|
|
87
|
+
|
|
88
|
+
if __FILE__ == $0
|
|
89
|
+
exit if ARGV.size != 1
|
|
90
|
+
filename = ARGV.shift
|
|
91
|
+
rex = Sample.new
|
|
92
|
+
begin
|
|
93
|
+
rex.load_file filename
|
|
94
|
+
while token = rex.next_token
|
|
95
|
+
p token
|
|
96
|
+
end
|
|
97
|
+
rescue
|
|
98
|
+
$stderr.printf "%s:%d:%s\n", rex.filename, rex.lineno, $!.message
|
|
99
|
+
end
|
|
100
|
+
end
|
data/sample/sample.xhtml
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="Shift_JIS"?>
|
|
2
|
+
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
|
|
3
|
+
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
|
4
|
+
<html xmlns="http://www.w3.org/1999/xhtml">
|
|
5
|
+
<head>
|
|
6
|
+
<!-- Comment -->
|
|
7
|
+
<title>Title</title>
|
|
8
|
+
<meta http-equiv="Content-Type" content="text/html; charset=Shift_JIS" />
|
|
9
|
+
<meta name="robots" content="noindex,nofollow" />
|
|
10
|
+
<link rel="stylesheet" type="text/css" href="sample.css" />
|
|
11
|
+
</head>
|
|
12
|
+
<script language="JavaScript">
|
|
13
|
+
<!--
|
|
14
|
+
function foo(msg) {
|
|
15
|
+
ndx = Math.floor(Math.random() * msg.length);
|
|
16
|
+
return msg[ndx];
|
|
17
|
+
}
|
|
18
|
+
//-->
|
|
19
|
+
</script>
|
|
20
|
+
<body>
|
|
21
|
+
<div id="div1" class="div">
|
|
22
|
+
<p id="p1" class="p">
|
|
23
|
+
XHTML 1.1
|
|
24
|
+
</p>
|
|
25
|
+
</div>
|
|
26
|
+
<form action="sample.cgi" method="post">
|
|
27
|
+
<p>
|
|
28
|
+
<input type="text" name="t1" id="t1" value="TEXT1" />
|
|
29
|
+
</p>
|
|
30
|
+
</form>
|
|
31
|
+
</body>
|
|
32
|
+
</html>
|
data/sample/sample1.c
ADDED
data/sample/sample1.rex
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
#
|
|
2
|
+
# sample1.rex
|
|
3
|
+
# lexical definition sample for rex
|
|
4
|
+
#
|
|
5
|
+
# usage
|
|
6
|
+
# rex sample1.rex --stub
|
|
7
|
+
# ruby sample1.rex.rb sample1.c
|
|
8
|
+
#
|
|
9
|
+
|
|
10
|
+
class Sample1
|
|
11
|
+
macro
|
|
12
|
+
BLANK \s+
|
|
13
|
+
REM_IN \/\*
|
|
14
|
+
REM_OUT \*\/
|
|
15
|
+
REM \/\/
|
|
16
|
+
|
|
17
|
+
rule
|
|
18
|
+
|
|
19
|
+
# [:state] pattern [actions]
|
|
20
|
+
|
|
21
|
+
# remark
|
|
22
|
+
{REM_IN} { state = :REMS; [:rem_in, text] }
|
|
23
|
+
:REMS {REM_OUT} { state = nil; [:rem_out, text] }
|
|
24
|
+
:REMS .*(?={REM_OUT}) { [:remark, text] }
|
|
25
|
+
{REM} { state = :REM; [:rem_in, text] }
|
|
26
|
+
:REM \n { state = nil; [:rem_out, text] }
|
|
27
|
+
:REM .*(?=$) { [:remark, text] }
|
|
28
|
+
|
|
29
|
+
# literal
|
|
30
|
+
\"[^"]*\" { [:string, text] } # "
|
|
31
|
+
\'[^']\' { [:character, text] } # '
|
|
32
|
+
|
|
33
|
+
# skip
|
|
34
|
+
{BLANK} # no action
|
|
35
|
+
|
|
36
|
+
# numeric
|
|
37
|
+
\d+ { [:digit, text.to_i] }
|
|
38
|
+
|
|
39
|
+
# identifier
|
|
40
|
+
\w+ { [:word, text] }
|
|
41
|
+
. { [text, text] }
|
|
42
|
+
|
|
43
|
+
end
|
data/sample/sample2.bas
ADDED
data/sample/sample2.rex
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
#
|
|
2
|
+
# sample2.rex
|
|
3
|
+
# lexical definition sample for rex
|
|
4
|
+
#
|
|
5
|
+
# usage
|
|
6
|
+
# rex sample2.rex --stub
|
|
7
|
+
# ruby sample2.rex.rb sample2.bas
|
|
8
|
+
#
|
|
9
|
+
|
|
10
|
+
class Sample2
|
|
11
|
+
option
|
|
12
|
+
ignorecase
|
|
13
|
+
|
|
14
|
+
macro
|
|
15
|
+
BLANK \s+
|
|
16
|
+
REMARK \' # '
|
|
17
|
+
|
|
18
|
+
rule
|
|
19
|
+
{REMARK} { state = :REM; [:rem_in, text] } # '
|
|
20
|
+
:REM \n { state = nil; [:rem_out, text] }
|
|
21
|
+
:REM .*(?=$) { [:remark, text] }
|
|
22
|
+
|
|
23
|
+
\"[^"]*\" { [:string, text] } # "
|
|
24
|
+
|
|
25
|
+
{BLANK} # no action
|
|
26
|
+
|
|
27
|
+
INPUT { [:input, text] }
|
|
28
|
+
PRINT { [:print, text] }
|
|
29
|
+
|
|
30
|
+
\d+ { [:digit, text.to_i] }
|
|
31
|
+
\w+ { [:word, text] }
|
|
32
|
+
. { [text, text] }
|
|
33
|
+
end
|
data/sample/simple.html
ADDED
data/sample/simple.xhtml
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
#
|
|
2
|
+
# xml parser
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
class XHTMLParser
|
|
6
|
+
|
|
7
|
+
rule
|
|
8
|
+
target : /* none */
|
|
9
|
+
| xml_doc
|
|
10
|
+
|
|
11
|
+
xml_doc : xml_header extra xml_body
|
|
12
|
+
| xml_header xml_body
|
|
13
|
+
| xml_body
|
|
14
|
+
|
|
15
|
+
xml_header : xtag_in element attributes xtag_out
|
|
16
|
+
|
|
17
|
+
xml_body : tag_from contents tag_to
|
|
18
|
+
|
|
19
|
+
tag_from : tag_in element attributes tag_out
|
|
20
|
+
|
|
21
|
+
tag_empty : tag_in element attributes etag_out
|
|
22
|
+
|
|
23
|
+
tag_to : etag_in element tag_out
|
|
24
|
+
|
|
25
|
+
attributes : /* none */
|
|
26
|
+
| attributes attribute
|
|
27
|
+
|
|
28
|
+
attribute : attr equal quoted
|
|
29
|
+
|
|
30
|
+
quoted : quote1 value quote1
|
|
31
|
+
| quote2 value quote2
|
|
32
|
+
|
|
33
|
+
contents : /* none */
|
|
34
|
+
| contents content
|
|
35
|
+
|
|
36
|
+
content : text
|
|
37
|
+
| extra
|
|
38
|
+
| tag_from contents tag_to
|
|
39
|
+
| tag_empty
|
|
40
|
+
|
|
41
|
+
extra : tag_in ext extra_texts tag_out
|
|
42
|
+
|
|
43
|
+
extra_texts : /* none */
|
|
44
|
+
| extra_texts rem_in remtexts rem_out
|
|
45
|
+
| extra_texts exttext
|
|
46
|
+
|
|
47
|
+
remtexts : remtext
|
|
48
|
+
| remtexts remtext
|
|
49
|
+
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
---- header ----
|
|
53
|
+
#
|
|
54
|
+
# generated by racc
|
|
55
|
+
#
|
|
56
|
+
require 'xhtmlparser.rex'
|
|
57
|
+
---- inner ----
|
|
58
|
+
|
|
59
|
+
---- footer ----
|
|
60
|
+
|
|
61
|
+
exit if ARGV.size == 0
|
|
62
|
+
filename = ARGV.shift
|
|
63
|
+
|
|
64
|
+
htmlparser = XHTMLParser.new
|
|
65
|
+
htmlparser.scan_file filename
|
|
66
|
+
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#
|
|
2
|
+
# xhtmlparser.rex
|
|
3
|
+
# lexical scanner definition for rex
|
|
4
|
+
#
|
|
5
|
+
# usage
|
|
6
|
+
# rex xhtmlparser.rex --stub
|
|
7
|
+
# ruby xhtmlparser.rex.rb sample.xhtml
|
|
8
|
+
#
|
|
9
|
+
|
|
10
|
+
class XHTMLParser
|
|
11
|
+
option
|
|
12
|
+
ignorecase
|
|
13
|
+
|
|
14
|
+
macro
|
|
15
|
+
BLANK \s+
|
|
16
|
+
TAG_IN \<
|
|
17
|
+
TAG_OUT \>
|
|
18
|
+
ETAG_IN \<\/
|
|
19
|
+
ETAG_OUT \/\>
|
|
20
|
+
XTAG_IN \<\?
|
|
21
|
+
XTAG_OUT \?\>
|
|
22
|
+
EXT \!
|
|
23
|
+
REM \-\-
|
|
24
|
+
EQUAL \=
|
|
25
|
+
Q1 \'
|
|
26
|
+
Q2 \"
|
|
27
|
+
|
|
28
|
+
rule
|
|
29
|
+
|
|
30
|
+
# [:state] pattern [actions]
|
|
31
|
+
{XTAG_IN} { state = :TAG; [:xtag_in, text] }
|
|
32
|
+
{ETAG_IN} { state = :TAG; [:etag_in, text] }
|
|
33
|
+
{TAG_IN} { state = :TAG; [:tag_in, text] }
|
|
34
|
+
:TAG {EXT} { state = :EXT; [:ext, text] }
|
|
35
|
+
|
|
36
|
+
:EXT {REM} { state = :REM; [:rem_in, text] }
|
|
37
|
+
:EXT {XTAG_OUT} { state = nil; [:xtag_out, text] }
|
|
38
|
+
:EXT {TAG_OUT} { state = nil; [:tag_out, text] }
|
|
39
|
+
:EXT .+(?={REM}) { [:exttext, text] }
|
|
40
|
+
:EXT .+(?={TAG_OUT}) { [:exttext, text] }
|
|
41
|
+
:EXT .+(?=$) { [:exttext, text] }
|
|
42
|
+
:EXT \n
|
|
43
|
+
|
|
44
|
+
:REM {REM} { state = :EXT; [:rem_out, text] }
|
|
45
|
+
:REM .+(?={REM}) { [:remtext, text] }
|
|
46
|
+
:REM .+(?=$) { [:remtext, text] }
|
|
47
|
+
:REM \n
|
|
48
|
+
|
|
49
|
+
:TAG {BLANK}
|
|
50
|
+
:TAG {XTAG_OUT} { state = nil; [:xtag_out, text] }
|
|
51
|
+
:TAG {ETAG_OUT} { state = nil; [:etag_out, text] }
|
|
52
|
+
:TAG {TAG_OUT} { state = nil; [:tag_out, text] }
|
|
53
|
+
:TAG {EQUAL} { [:equal, text] }
|
|
54
|
+
:TAG {Q1} { state = :Q1; [:quote1, text] } # '
|
|
55
|
+
:Q1 {Q1} { state = :TAG; [:quote1, text] } # '
|
|
56
|
+
:Q1 [^{Q1}]+(?={Q1}) { [:value, text] } # '
|
|
57
|
+
:TAG {Q2} { state = :Q2; [:quote2, text] } # "
|
|
58
|
+
:Q2 {Q2} { state = :TAG; [:quote2, text] } # "
|
|
59
|
+
:Q2 [^{Q2}]+(?={Q2}) { [:value, text] } # "
|
|
60
|
+
|
|
61
|
+
:TAG [\w\-]+(?={EQUAL}) { [:attr, text] }
|
|
62
|
+
:TAG [\w\-]+ { [:element, text] }
|
|
63
|
+
|
|
64
|
+
\s+(?=\S)
|
|
65
|
+
.*\S(?=\s*{ETAG_IN}) { [:text, text] }
|
|
66
|
+
.*\S(?=\s*{TAG_IN}) { [:text, text] }
|
|
67
|
+
.*\S(?=\s*$) { [:text, text] }
|
|
68
|
+
\s+(?=$)
|
|
69
|
+
|
|
70
|
+
inner
|
|
71
|
+
|
|
72
|
+
end
|