rex 0.0.4 → 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (66) hide show
  1. data/CHANGELOG.rdoc +7 -0
  2. data/DOCUMENTATION.en.rdoc +215 -0
  3. data/DOCUMENTATION.ja.rdoc +205 -0
  4. data/Manifest.txt +37 -0
  5. data/README.ja +73 -0
  6. data/README.rdoc +53 -0
  7. data/Rakefile +28 -0
  8. data/bin/rex +18 -0
  9. data/lib/rex.rb +3 -23
  10. data/lib/rex/generator.rb +523 -0
  11. data/lib/rex/info.rb +16 -0
  12. data/lib/rex/rexcmd.rb +136 -0
  13. data/sample/a.cmd +1 -0
  14. data/sample/b.cmd +1 -0
  15. data/sample/c.cmd +4 -0
  16. data/sample/calc3.racc +47 -0
  17. data/sample/calc3.rex +15 -0
  18. data/sample/calc3.rex.rb +94 -0
  19. data/sample/calc3.tab.rb +188 -0
  20. data/sample/error1.rex +15 -0
  21. data/sample/error2.rex +15 -0
  22. data/sample/sample.html +32 -0
  23. data/sample/sample.rex +15 -0
  24. data/sample/sample.rex.rb +100 -0
  25. data/sample/sample.xhtml +32 -0
  26. data/sample/sample1.c +9 -0
  27. data/sample/sample1.rex +43 -0
  28. data/sample/sample2.bas +4 -0
  29. data/sample/sample2.rex +33 -0
  30. data/sample/simple.html +7 -0
  31. data/sample/simple.xhtml +10 -0
  32. data/sample/xhtmlparser.racc +66 -0
  33. data/sample/xhtmlparser.rex +72 -0
  34. data/test/assets/test.rex +12 -0
  35. data/test/rex-20060125.rb +152 -0
  36. data/test/rex-20060511.rb +143 -0
  37. data/test/test_generator.rb +184 -0
  38. metadata +74 -57
  39. data/README +0 -50
  40. data/lib/rex/array.rb +0 -13
  41. data/lib/rex/kernel.rb +0 -11
  42. data/lib/rex/modules/boolean.rb +0 -14
  43. data/lib/rex/modules/caller_name.rb +0 -15
  44. data/lib/rex/modules/ordinal.rb +0 -35
  45. data/lib/rex/modules/propercase.rb +0 -15
  46. data/lib/rex/modules/roman.rb +0 -41
  47. data/lib/rex/modules/scrub.rb +0 -66
  48. data/lib/rex/modules/stacktrace.rb +0 -31
  49. data/lib/rex/modules/swap.rb +0 -13
  50. data/lib/rex/modules/to_proc.rb +0 -13
  51. data/lib/rex/modules/tuple.rb +0 -14
  52. data/lib/rex/modules/wrap.rb +0 -15
  53. data/lib/rex/numeric.rb +0 -19
  54. data/lib/rex/string.rb +0 -21
  55. data/lib/rex/symbol.rb +0 -13
  56. data/test/boolean.rb +0 -64
  57. data/test/caller_name.rb +0 -26
  58. data/test/ordinal.rb +0 -609
  59. data/test/propercase.rb +0 -40
  60. data/test/roman.rb +0 -335
  61. data/test/scrub.rb +0 -27
  62. data/test/stacktrace.rb +0 -24
  63. data/test/swap.rb +0 -44
  64. data/test/to_proc.rb +0 -24
  65. data/test/tuple.rb +0 -26
  66. data/test/wrap.rb +0 -44
@@ -0,0 +1,7 @@
1
+ === 1.0.2
2
+
3
+ * Bug fixes
4
+
5
+ * Fixed nested macros so that backslashes will work
6
+
7
+
@@ -0,0 +1,215 @@
1
+
2
+ = REX: Ruby Lex for Racc
3
+
4
+
5
+ == About
6
+
7
+ Lexical Scanner Generator for Ruby, with Racc.
8
+
9
+
10
+ == Usage
11
+
12
+ rex [options] grammarfile
13
+
14
+ -o --output-file filename designated output filename.
15
+ -s --stub append stub main for debug.
16
+ -i --ignorecase ignore char case
17
+ -C --check-only syntax check only.
18
+ --independent independent mode.
19
+ -d --debug print debug information
20
+ -h --help print usage.
21
+ --version print version.
22
+ --copyright print copyright.
23
+
24
+
25
+ == Default Output Filename
26
+
27
+ It destinate from foo.rex to foo.rex.rb.
28
+ This name is for a follow description.
29
+
30
+ require 'foo.rex'
31
+
32
+
33
+ == Grammar File Format
34
+
35
+ A definition is given in order of a header part, a rule part,
36
+ and the footer part. One or more sections are included in a rule part.
37
+ As for each section, the head of the sentence starts by the keyword.
38
+
39
+ Summary:
40
+
41
+ [Header Part]
42
+ "class" Foo
43
+ ["option"
44
+ [options] ]
45
+ ["inner"
46
+ [methods] ]
47
+ ["macro"
48
+ [macro-name regular-expression] ]
49
+ "rule"
50
+ [start-state] pattern [actions]
51
+ "end"
52
+ [Footer Part]
53
+
54
+
55
+ === Grammar File Example
56
+
57
+ class Foo
58
+ macro
59
+ BLANK \s+
60
+ DIGIT \d+
61
+ rule
62
+ {BLANK}
63
+ {DIGIT} { [:NUMBER, text.to_i] }
64
+ . { [text, text] }
65
+ end
66
+
67
+
68
+ == Header Part ( Optional )
69
+
70
+ All the contents described before the definition of a rule part are
71
+ posted to head of the output file.
72
+
73
+
74
+ == Footer Part ( Optional )
75
+
76
+ All the contents described after the definition of a rule part are
77
+ posted to tail of the output file.
78
+
79
+
80
+ == Rule Part
81
+
82
+ Rule part is from the line which begins from the "class" keyword
83
+ to the line which begins from the "end" keyword.
84
+ The class name outputted after a keyword "class" is specified.
85
+ If embellished with a module name, it will become a class in a module.
86
+ The class which inherited Racc::Parser is generated.
87
+
88
+
89
+ === Rule Header Example
90
+
91
+ class Foo
92
+ class Bar::Foo
93
+
94
+
95
+ == Option Section ( Optional )
96
+
97
+ "option" is start keyword.
98
+
99
+ "ignorecase" when pattern match, ignore char case.
100
+ "stub" append stub main for debug.
101
+ "independent" independent mode, for it is not inherited Racc.
102
+
103
+
104
+ == Inner Section ( Optional )
105
+
106
+ "inner" is start keyword.
107
+ The contents defined here are defined by the inside of the class
108
+ of the generated scanner.
109
+
110
+
111
+ == Macro Section ( Optional )
112
+
113
+ "macro" is start keyword.
114
+ One regular expression is named.
115
+ A blank character (0x20) can be included by escaping by \ .
116
+
117
+
118
+ === Macro Section Example
119
+
120
+ DIGIT \d+
121
+ IDENT [a-zA-Z_][a-zA-Z0-9_]*
122
+ BLANK [\ \t]+
123
+ REMIN \/\*
124
+ REMOUT \*\/
125
+
126
+
127
+ == Rule Section
128
+
129
+ "rule" is start keyword.
130
+
131
+ [state] pattern [actions]
132
+
133
+
134
+ === state: Start State ( Optional )
135
+
136
+ A start state is expressed with the identifier which prefaces ":".
137
+ When the continuing alphabetic character is a capital letter,
138
+ it will be in an exclusive start state.
139
+ When it is a small letter, it will be in an inclusive start state.
140
+ Initial value and default value of a start state is nil.
141
+
142
+
143
+ === pattern: String Pattern
144
+
145
+ The regular expression for specifying a character string.
146
+ The macro definition bundled with { } can be used for description
147
+ of a regular expression. A macro definition is used in order to use
148
+ a regular expression including a blank.
149
+
150
+
151
+ === actions: Processing Actions ( Optional )
152
+
153
+ Action is performed when a pattern is suited.
154
+ The processing which creates a suitable token is defined.
155
+ The arrangement whose token has the second clause of classification
156
+ and a value, or nil.
157
+ The following elements can be used in order to create a token.
158
+
159
+ lineno Line number ( Read Only )
160
+ text Matched string ( Read Only )
161
+ state Start state ( Read/Write )
162
+
163
+ Action is bundled with { }. It is the block of Ruby.
164
+ Don't use the function to change the flow of control exceeding a block.
165
+ ( return, exit, next, break, ... )
166
+ If action is omitted, the character string which matched will be canceled
167
+ and will progress to the next scan.
168
+
169
+
170
+ === Rule Part Example
171
+
172
+ {REMIN} { state = :REM ; [:REM_IN, text] }
173
+ :REM {REMOUT} { state = nil ; [:REM_OUT, text] }
174
+ :REM (.+)(?={REMOUT}) { [:COMMENT, text] }
175
+ {BLANK}
176
+ -?{DIGIT} { [:NUMBER, text.to_i] }
177
+ {WORD} { [:word, text] }
178
+ . { [text, text] }
179
+
180
+ == Comment ( Optional )
181
+
182
+ From "#" to the end of the line becomes a comment in each line.
183
+
184
+
185
+ == Usage for Generated Class
186
+
187
+ === scan_setup()
188
+
189
+ The event for initializing at the time of the execution start of a scanner.
190
+ It is redefined and used.
191
+
192
+
193
+ === scan_str( str )
194
+
195
+ Parse the string described by the defined grammar.
196
+ Token is stored in an inside.
197
+
198
+
199
+ === scan_file( filename )
200
+
201
+ Parse the file described by the defined grammar.
202
+ Token is stored in an inside.
203
+
204
+
205
+ === next_token
206
+
207
+ One token stored in the inside is taken out.
208
+ The last returns nil.
209
+
210
+
211
+ == Notice
212
+
213
+ This specification is provisional and may be changed without a preliminary
214
+ announcement.
215
+
@@ -0,0 +1,205 @@
1
+
2
+ = REX: Ruby Lex for Racc
3
+
4
+
5
+ == ����
6
+
7
+ Racc ��ʻ�Ѥ��� Ruby �Ѥλ��祹����������ġ��롣
8
+
9
+
10
+ == �Ȥ���
11
+
12
+ rex [options] grammarfile
13
+
14
+ -o --output-file filename ���ϥե�����̾����
15
+ -s --stub �ǥХå��Ѥμ�������ղ�
16
+ -i --ignorecase ��ʸ����ʸ������̤��ʤ�
17
+ -C --check-only ʸˡ�����Τ�
18
+ --independent ���¸�⡼��
19
+ -d --debug �ǥХå�����ɽ��
20
+ -h --help �Ȥ���������
21
+ --version �С������ɽ��
22
+ --copyright �������ɽ��
23
+
24
+
25
+ == �ǥե���Ȥν��ϥե�����̾
26
+
27
+ foo.rex �ˤĤ��� foo.rex.rb ����Ϥ��롣
28
+ �ʲ��Τ褦�����Ѥ���뤳�Ȥ����ꤷ�Ƥ��롣
29
+
30
+ require 'foo.rex'
31
+
32
+
33
+ == ���ϥե����빽¤
34
+
35
+ Ƭ������§���������ν��������롣
36
+ ��§���ˤϡ�ʣ�����᤬�ޤޤ�롣
37
+ ����ϡ���Ƭ��������ɤǻϤޤ롣
38
+
39
+ ���ס�
40
+
41
+ [Ƭ��]
42
+ "class" Foo
43
+ ["option"
44
+ [���ץ����] ]
45
+ ["inner"
46
+ [�᥽�å����] ]
47
+ ["macro"
48
+ [�ޥ���̾ ����ɽ��] ]
49
+ "rule"
50
+ [�������Ⱦ���] �ѥ����� [���������]
51
+ "end"
52
+ [����]
53
+
54
+
55
+ === ���ϥե����뵭����
56
+
57
+ class Foo
58
+ macro
59
+ BLANK \s+
60
+ DIGIT \d+
61
+ rule
62
+ {BLANK}
63
+ {DIGIT} { [:NUMBER, text.to_i] }
64
+ . { [text, text] }
65
+ end
66
+
67
+
68
+ == Ƭ���ʾ�ά��ǽ��
69
+
70
+ ��§������������˵��Ҥ��줿���Ƥϡ����٤ƽ��ϥե�������Ƭ��ž������롣
71
+
72
+
73
+ == �����ʾ�ά��ǽ��
74
+
75
+ ��§��������ʹߤ˵��Ҥ��줿���Ƥϡ����٤ƽ��ϥե�����������ž������롣
76
+
77
+
78
+ == ��§��
79
+
80
+ ��§���� "class" ������ɤ���Ϥޤ�Ԥ��� "end" ������ɤ���Ϥޤ�
81
+ �ԤޤǤǤ��롣
82
+ "class" ������ɤ�³���ƽ��Ϥ��륯�饹̾����ꤹ�롣
83
+ �⥸�塼��̾�ǽ�������ȡ��⥸�塼���⥯�饹�Ȥʤ롣
84
+ Racc::Parser ��Ѿ��������饹���������롣
85
+
86
+
87
+ === ��§�������
88
+
89
+ class Foo
90
+ class Bar::Foo
91
+
92
+
93
+ == ���ץ����ʾ�ά��ǽ��
94
+
95
+ ������� "option" ������ɤǻϤޤ롣
96
+
97
+ "ignorecase" ��ʸ����ʸ������̤��ʤ���
98
+ "stub" �ǥХå��Ѥμ�������ղ�
99
+ "independent" ���¸�⡼�ɡ�Racc ��Ѿ����ʤ���
100
+
101
+ == �����桼�������ɡʾ�ά��ǽ��
102
+
103
+ ������� "inner" ������ɤǻϤޤ롣
104
+ ����������������Ƥϡ���������������ʤΥ��饹���������������롣
105
+
106
+
107
+ == �ޥ�������ʾ�ά��ǽ��
108
+
109
+ ������� "macro" ������ɤǻϤޤ롣
110
+ ���֤������ɽ����̾����Ĥ��롣
111
+ \ �ǥ��������פ��뤳�ȤǶ����ޤ�뤳�Ȥ��Ǥ��롣
112
+
113
+ === �ޥ��������
114
+
115
+ DIGIT \d+
116
+ IDENT [a-zA-Z_][a-zA-Z0-9_]*
117
+ BLANK [\ \t]+
118
+ REMIN \/\*
119
+ REMOUT \*\/
120
+
121
+
122
+ == ������§
123
+
124
+ ������� "rule" ������ɤǻϤޤ롣
125
+
126
+ [state] pattern [actions]
127
+
128
+
129
+ === state: �������Ⱦ��֡ʾ�ά��ǽ��
130
+
131
+ �������Ⱦ��֤� ":" �����֤��뼱�̻Ҥ�ɽ����롣
132
+ ³���ѻ�����ʸ���ΤȤ�����¾Ū�������Ⱦ��֤Ȥʤ롣
133
+ ��ʸ���ΤȤ������Ū�������Ⱦ��֤Ȥʤ롣
134
+ �������Ⱦ��֤ν���ͤ���Ӿ�ά���ͤ� nil �Ǥ��롣
135
+
136
+
137
+ === pattern: ʸ����ѥ�����
138
+
139
+ ʸ��������ꤹ�뤿�������ɽ����
140
+ ����ɽ���ε��Ҥˤϡ���̤dz�ä��ޥ���������Ѥ��뤳�Ȥ��Ǥ��롣
141
+ �����ޤ�����ɽ�����Ѥ���ˤϡ��ޥ�������Ѥ��롣
142
+
143
+
144
+ === actions: ���������ʾ�ά��ǽ��
145
+
146
+ �ѥ������Ŭ�礹��Ȥ����������ϼ¹Ԥ���롣
147
+ Ŭ�ڤʥȡ������������������������롣
148
+ �ȡ�����ϡ����̤��ͤ�����������󡢤ޤ��� nil �Ǥ��롣
149
+ �ȡ������������뤿��˰ʲ������Ǥ����ѤǤ��롣
150
+
151
+ lineno ���Ϲ��ֹ� ( Read Only )
152
+ text ���Ф���ʸ���� ( Read Only )
153
+ state �������Ⱦ��� ( Read/Write )
154
+
155
+ ���������� { } �dz�ä� Ruby �Υ֥��å��Ǥ��롣
156
+ �֥��å���ۤ��������ή����Ѥ��뵡ǽ����Ѥ��ƤϤ����ʤ���
157
+ ( return, exit, next, break, ... )
158
+ ��������󤬾�ά�����ȡ�Ŭ�礷��ʸ������˴�����Ƽ��������˿ʤࡣ
159
+
160
+
161
+ === ������§�����
162
+
163
+ {REMIN} { state = :REM ; [:REM_IN, text] }
164
+ :REM {REMOUT} { state = nil ; [:REM_OUT, text] }
165
+ :REM (.+)(?={REMOUT}) { [:COMMENT, text] }
166
+ {BLANK}
167
+ -?{DIGIT} { [:NUMBER, text.to_i] }
168
+ {WORD} { [:word, text] }
169
+ . { [text, text] }
170
+
171
+
172
+ == �����ȡʾ�ά��ǽ��
173
+
174
+ �ƹԤˤ����� "#" ���� �����ޤǤ������Ȥˤʤ롣
175
+
176
+
177
+ == �����������饹�λȤ���
178
+
179
+ === scan_setup()
180
+
181
+ ������ʤμ¹Գ��ϻ��˽�������뤿��Υ��٥�ȡ�
182
+ ��������ƻ��Ѥ��롣
183
+
184
+ === scan_str( str )
185
+
186
+ ������줿ʸˡ�ˤ�äƵ��Ҥ��줿ʸ������᤹�롣
187
+ token ���������ݻ����롣
188
+
189
+
190
+ === scan_file( filename )
191
+
192
+ ������줿ʸˡ�ˤ�äƵ��Ҥ��줿�ե�������ɤ߹��ࡣ
193
+ token ���������ݻ����롣
194
+
195
+
196
+ === next_token
197
+
198
+ �������ݻ����� token ��ҤȤĤ��ļ��Ф���
199
+ �Ǹ�� nil ���֤���
200
+
201
+
202
+ == ����
203
+
204
+ �ܻ��ͤϻ���Ū�Ǥ��ꡢͽ��ʤ��ѹ�������礬���롣
205
+
@@ -0,0 +1,37 @@
1
+ CHANGELOG.rdoc
2
+ DOCUMENTATION.en.rdoc
3
+ DOCUMENTATION.ja.rdoc
4
+ Manifest.txt
5
+ README.ja
6
+ README.rdoc
7
+ Rakefile
8
+ bin/rex
9
+ lib/rex.rb
10
+ lib/rex/generator.rb
11
+ lib/rex/info.rb
12
+ lib/rex/rexcmd.rb
13
+ sample/a.cmd
14
+ sample/b.cmd
15
+ sample/c.cmd
16
+ sample/calc3.racc
17
+ sample/calc3.rex
18
+ sample/calc3.rex.rb
19
+ sample/calc3.tab.rb
20
+ sample/error1.rex
21
+ sample/error2.rex
22
+ sample/sample.html
23
+ sample/sample.rex
24
+ sample/sample.rex.rb
25
+ sample/sample.xhtml
26
+ sample/sample1.c
27
+ sample/sample1.rex
28
+ sample/sample2.bas
29
+ sample/sample2.rex
30
+ sample/simple.html
31
+ sample/simple.xhtml
32
+ sample/xhtmlparser.racc
33
+ sample/xhtmlparser.rex
34
+ test/assets/test.rex
35
+ test/rex-20060125.rb
36
+ test/rex-20060511.rb
37
+ test/test_generator.rb