pegex 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. data/.gemspec +21 -0
  2. data/CHANGELOG.yaml +3 -0
  3. data/Gemfile +2 -0
  4. data/Gemfile.lock +16 -0
  5. data/LICENSE +21 -0
  6. data/README.rdoc +78 -0
  7. data/Rakefile +64 -0
  8. data/lib/pegex/compiler.rb +91 -0
  9. data/lib/pegex/grammar/atoms.rb +96 -0
  10. data/lib/pegex/grammar.rb +21 -0
  11. data/lib/pegex/input.rb +41 -0
  12. data/lib/pegex/parser.rb +287 -0
  13. data/lib/pegex/pegex/ast.rb +148 -0
  14. data/lib/pegex/pegex/grammar.rb +414 -0
  15. data/lib/pegex/receiver.rb +7 -0
  16. data/lib/pegex/tree/wrap.rb +13 -0
  17. data/lib/pegex/tree.rb +17 -0
  18. data/lib/pegex.rb +18 -0
  19. data/test/compiler-checks.rb +271 -0
  20. data/test/compiler-checks.tml +271 -0
  21. data/test/compiler-equivalence.rb +79 -0
  22. data/test/compiler.rb +42 -0
  23. data/test/compiler.tml +111 -0
  24. data/test/error.rb +161 -0
  25. data/test/export_ok.rb +36 -0
  26. data/test/grammar-api.rb +21 -0
  27. data/test/lib/recursive_sort.rb +17 -0
  28. data/test/lib/test_pegex.rb +33 -0
  29. data/test/lib/testast.rb +15 -0
  30. data/test/lib/xxx.rb +13 -0
  31. data/test/tree-pegex.tml +35 -0
  32. data/test/tree.rb +47 -0
  33. data/test/tree.tml +449 -0
  34. metadata +99 -0
data/test/compiler-checks.rb ADDED
@@ -0,0 +1,271 @@
1
+ require './test/lib/test_pegex'
2
+
3
+ TestML.require_or_skip 'psych'
4
+
5
+ TestML.run do |t|
6
+ t.eval '*grammar.compile.yaml.clean == *yaml'
7
+ end
8
+
9
+ TestML.data <<'...'
10
+ === Empty Grammar
11
+ --- grammar
12
+ --- yaml
13
+ {}
14
+
15
+ === Simple Grammar
16
+ --- grammar
17
+ a: ( <b> <c>* )+
18
+ b: /x/
19
+ c: <x>
20
+
21
+ --- yaml
22
+ a:
23
+ +min: 1
24
+ .all:
25
+ - .ref: b
26
+ - +min: 0
27
+ .ref: c
28
+ b:
29
+ .rgx: x
30
+ c:
31
+ .ref: x
32
+
33
+ === Single Rule Reference
34
+ --- grammar
35
+ a: <x>
36
+ --- yaml
37
+ a:
38
+ .ref: x
39
+
40
+ === Single Rule with no brackets
41
+ --- grammar
42
+ a: x
43
+ --- yaml
44
+ a:
45
+ .ref: x
46
+
47
+ === Single Rule With Trailing Quantifier
48
+ --- grammar
49
+ a: <x>*
50
+ --- yaml
51
+ a:
52
+ +min: 0
53
+ .ref: x
54
+
55
+ === Single Rule With Trailing Quantifier (no angles)
56
+ --- grammar
57
+ a: x*
58
+ --- yaml
59
+ a:
60
+ +min: 0
61
+ .ref: x
62
+
63
+ === Single Rule With Leading Assertion
64
+ --- grammar
65
+ a: =<x>
66
+ --- yaml
67
+ a:
68
+ +asr: 1
69
+ .ref: x
70
+
71
+ === Negative and Positive Assertion
72
+ --- grammar
73
+ a: !<b> =<c>
74
+ --- yaml
75
+ a:
76
+ .all:
77
+ - +asr: -1
78
+ .ref: b
79
+ - +asr: 1
80
+ .ref: c
81
+
82
+ === Single Regex
83
+ --- grammar
84
+ a: /x/
85
+ --- yaml
86
+ a:
87
+ .rgx: x
88
+
89
+ === Single Error
90
+ --- grammar
91
+ a: `x`
92
+ --- yaml
93
+ a:
94
+ .err: x
95
+
96
+ === Skip and Wrap Marker
97
+ --- grammar
98
+ a: .<b> +<c>+ -<d>?
99
+ --- yaml
100
+ a:
101
+ .all:
102
+ - -skip: 1
103
+ .ref: b
104
+ - +min: 1
105
+ -wrap: 1
106
+ .ref: c
107
+ - +max: 1
108
+ -pass: 1
109
+ .ref: d
110
+
111
+ === Unbracketed All Group
112
+ --- grammar
113
+ a: /x/ <y>
114
+ --- yaml
115
+ a:
116
+ .all:
117
+ - .rgx: x
118
+ - .ref: y
119
+
120
+ === Unbracketed Any Group
121
+ --- grammar
122
+ a: /x/ | <y> | `z`
123
+ --- yaml
124
+ a:
125
+ .any:
126
+ - .rgx: x
127
+ - .ref: y
128
+ - .err: z
129
+
130
+ === Bracketed All Group
131
+ --- grammar
132
+ a: ( <x> <y> )
133
+ --- yaml
134
+ a:
135
+ .all:
136
+ - .ref: x
137
+ - .ref: y
138
+
139
+ === Bracketed Group With Trailing Modifier
140
+ --- grammar
141
+ a: ( <x> <y> )?
142
+ --- yaml
143
+ a:
144
+ +max: 1
145
+ .all:
146
+ - .ref: x
147
+ - .ref: y
148
+
149
+ === Bracketed Group With Leading Modifier
150
+ --- grammar
151
+ a: .( =<x> <y> )
152
+ --- yaml
153
+ a:
154
+ -skip: 1
155
+ .all:
156
+ - +asr: 1
157
+ .ref: x
158
+ - .ref: y
159
+
160
+ === Multiple Groups
161
+ --- grammar
162
+ a: ( <x> <y> ) ( <z> | /zzz/ )
163
+ --- yaml
164
+ a:
165
+ .all:
166
+ - .all:
167
+ - .ref: x
168
+ - .ref: y
169
+ - .any:
170
+ - .ref: z
171
+ - .rgx: zzz
172
+
173
+ === List Separator
174
+ --- grammar
175
+ a: <b> | <c>? %% /d/
176
+ --- yaml
177
+ a:
178
+ .any:
179
+ - .ref: b
180
+ - +max: 1
181
+ .ref: c
182
+ .sep:
183
+ +eok: 1
184
+ .rgx: d
185
+
186
+ === Separators with Quantifiers
187
+ --- grammar
188
+ a: <b>2+ % <c>* <d>* %% <e>2-3
189
+ --- yaml
190
+ a:
191
+ .all:
192
+ - +min: 2
193
+ .ref: b
194
+ .sep:
195
+ +min: 0
196
+ .ref: c
197
+ - +min: 0
198
+ .ref: d
199
+ .sep:
200
+ +eok: 1
201
+ +max: 3
202
+ +min: 2
203
+ .ref: e
204
+
205
+ === All Quantifier Forms
206
+ --- grammar
207
+ a: <b> <c>? <d>* <e>+ <f>55 <g>5+ <h>5-55
208
+ --- yaml
209
+ a:
210
+ .all:
211
+ - .ref: b
212
+ - +max: 1
213
+ .ref: c
214
+ - +min: 0
215
+ .ref: d
216
+ - +min: 1
217
+ .ref: e
218
+ - +max: 55
219
+ +min: 55
220
+ .ref: f
221
+ - +min: 5
222
+ .ref: g
223
+ - +max: 55
224
+ +min: 5
225
+ .ref: h
226
+
227
+ === Whitespace in Regex
228
+ --- grammar
229
+ a: /<DOT>* (<DASH>{3})
230
+ <BANG> <BANG>
231
+ /
232
+ --- yaml
233
+ a:
234
+ .rgx: \.*(\-{3})!!
235
+
236
+ === Directives
237
+ --- grammar
238
+ \%grammar foo
239
+ \%version 1.2.3
240
+
241
+ --- yaml
242
+ +grammar: foo
243
+ +version: 1.2.3
244
+
245
+ === Multiple Duplicate Directives
246
+ --- grammar
247
+ \%grammar foo
248
+ \%include bar
249
+ \%include baz
250
+
251
+ --- yaml
252
+ +grammar: foo
253
+ +include:
254
+ - bar
255
+ - baz
256
+
257
+ === Meta Lines
258
+ --- grammar
259
+ %grammar foo
260
+ %version 1.1.1
261
+ %extends bar bar
262
+ %include bazzy
263
+ a: /b/
264
+ --- yaml
265
+ +extends: bar bar
266
+ +grammar: foo
267
+ +include: bazzy
268
+ +version: 1.1.1
269
+ a:
270
+ .rgx: b
271
+ ...
data/test/compiler-checks.tml ADDED
@@ -0,0 +1,271 @@
1
+ #!/usr/bin/env testml
2
+
3
+ %TestML 1.0
4
+
5
+ Plan = 50;
6
+
7
+ *grammar.compile.yaml.clean == *yaml;
8
+
9
+ testml_data <<'...'
10
+ === Empty Grammar
11
+ --- grammar
12
+ --- yaml
13
+ {}
14
+
15
+ === Simple Grammar
16
+ --- grammar
17
+ a: ( <b> <c>* )+
18
+ b: /x/
19
+ c: <x>
20
+
21
+ --- yaml
22
+ a:
23
+ +min: 1
24
+ .all:
25
+ - .ref: b
26
+ - +min: 0
27
+ .ref: c
28
+ b:
29
+ .rgx: x
30
+ c:
31
+ .ref: x
32
+
33
+ === Single Rule Reference
34
+ --- grammar
35
+ a: <x>
36
+ --- yaml
37
+ a:
38
+ .ref: x
39
+
40
+ === Single Rule with no brackets
41
+ --- grammar
42
+ a: x
43
+ --- yaml
44
+ a:
45
+ .ref: x
46
+
47
+ === Single Rule With Trailing Quantifier
48
+ --- grammar
49
+ a: <x>*
50
+ --- yaml
51
+ a:
52
+ +min: 0
53
+ .ref: x
54
+
55
+ === Single Rule With Trailing Quantifier (no angles)
56
+ --- grammar
57
+ a: x*
58
+ --- yaml
59
+ a:
60
+ +min: 0
61
+ .ref: x
62
+
63
+ === Single Rule With Leading Assertion
64
+ --- grammar
65
+ a: =<x>
66
+ --- yaml
67
+ a:
68
+ +asr: 1
69
+ .ref: x
70
+
71
+ === Negative and Positive Assertion
72
+ --- grammar
73
+ a: !<b> =<c>
74
+ --- yaml
75
+ a:
76
+ .all:
77
+ - +asr: -1
78
+ .ref: b
79
+ - +asr: 1
80
+ .ref: c
81
+
82
+ === Single Regex
83
+ --- grammar
84
+ a: /x/
85
+ --- yaml
86
+ a:
87
+ .rgx: x
88
+
89
+ === Single Error
90
+ --- grammar
91
+ a: `x`
92
+ --- yaml
93
+ a:
94
+ .err: x
95
+
96
+ === Skip and Wrap Marker
97
+ --- grammar
98
+ a: .<b> +<c>+ -<d>?
99
+ --- yaml
100
+ a:
101
+ .all:
102
+ - -skip: 1
103
+ .ref: b
104
+ - +min: 1
105
+ -wrap: 1
106
+ .ref: c
107
+ - +max: 1
108
+ -pass: 1
109
+ .ref: d
110
+
111
+ === Unbracketed All Group
112
+ --- grammar
113
+ a: /x/ <y>
114
+ --- yaml
115
+ a:
116
+ .all:
117
+ - .rgx: x
118
+ - .ref: y
119
+
120
+ === Unbracketed Any Group
121
+ --- grammar
122
+ a: /x/ | <y> | `z`
123
+ --- yaml
124
+ a:
125
+ .any:
126
+ - .rgx: x
127
+ - .ref: y
128
+ - .err: z
129
+
130
+ === Bracketed All Group
131
+ --- grammar
132
+ a: ( <x> <y> )
133
+ --- yaml
134
+ a:
135
+ .all:
136
+ - .ref: x
137
+ - .ref: y
138
+
139
+ === Bracketed Group With Trailing Modifier
140
+ --- grammar
141
+ a: ( <x> <y> )?
142
+ --- yaml
143
+ a:
144
+ +max: 1
145
+ .all:
146
+ - .ref: x
147
+ - .ref: y
148
+
149
+ === Bracketed Group With Leading Modifier
150
+ --- grammar
151
+ a: .( =<x> <y> )
152
+ --- yaml
153
+ a:
154
+ -skip: 1
155
+ .all:
156
+ - +asr: 1
157
+ .ref: x
158
+ - .ref: y
159
+
160
+ === Multiple Groups
161
+ --- grammar
162
+ a: ( <x> <y> ) ( <z> | /zzz/ )
163
+ --- yaml
164
+ a:
165
+ .all:
166
+ - .all:
167
+ - .ref: x
168
+ - .ref: y
169
+ - .any:
170
+ - .ref: z
171
+ - .rgx: zzz
172
+
173
+ === List Separator
174
+ --- grammar
175
+ a: <b> | <c>? %% /d/
176
+ --- yaml
177
+ a:
178
+ .any:
179
+ - .ref: b
180
+ - +max: 1
181
+ .ref: c
182
+ .sep:
183
+ +eok: 1
184
+ .rgx: d
185
+
186
+ === Separators with Quantifiers
187
+ --- grammar
188
+ a: <b>2+ % <c>* <d>* %% <e>2-3
189
+ --- yaml
190
+ a:
191
+ .all:
192
+ - +min: 2
193
+ .ref: b
194
+ .sep:
195
+ +min: 0
196
+ .ref: c
197
+ - +min: 0
198
+ .ref: d
199
+ .sep:
200
+ +eok: 1
201
+ +max: 3
202
+ +min: 2
203
+ .ref: e
204
+
205
+ === All Quantifier Forms
206
+ --- grammar
207
+ a: <b> <c>? <d>* <e>+ <f>55 <g>5+ <h>5-55
208
+ --- yaml
209
+ a:
210
+ .all:
211
+ - .ref: b
212
+ - +max: 1
213
+ .ref: c
214
+ - +min: 0
215
+ .ref: d
216
+ - +min: 1
217
+ .ref: e
218
+ - +max: 55
219
+ +min: 55
220
+ .ref: f
221
+ - +min: 5
222
+ .ref: g
223
+ - +max: 55
224
+ +min: 5
225
+ .ref: h
226
+
227
+ === Whitespace in Regex
228
+ --- grammar
229
+ a: /<DOT>* (<DASH>{3})
230
+ <BANG> <BANG>
231
+ /
232
+ --- yaml
233
+ a:
234
+ .rgx: \.*(\-{3})!!
235
+
236
+ === Directives
237
+ --- grammar
238
+ \%grammar foo
239
+ \%version 1.2.3
240
+
241
+ --- yaml
242
+ +grammar: foo
243
+ +version: 1.2.3
244
+
245
+ === Multiple Duplicate Directives
246
+ --- grammar
247
+ \%grammar foo
248
+ \%include bar
249
+ \%include baz
250
+
251
+ --- yaml
252
+ +grammar: foo
253
+ +include:
254
+ - bar
255
+ - baz
256
+
257
+ === Meta Lines
258
+ --- grammar
259
+ %grammar foo
260
+ %version 1.1.1
261
+ %extends bar bar
262
+ %include bazzy
263
+ a: /b/
264
+ --- yaml
265
+ +extends: bar bar
266
+ +grammar: foo
267
+ +include: bazzy
268
+ +version: 1.1.1
269
+ a:
270
+ .rgx: b
271
+ ...
data/test/compiler-equivalence.rb ADDED
@@ -0,0 +1,79 @@
1
+ require './test/lib/test_pegex'
2
+
3
+ TestML.require_or_skip 'psych'
4
+
5
+ TestML.run do |t|
6
+ t.eval '*grammar1.compile.yaml == *grammar2.compile.yaml'
7
+ end
8
+
9
+ TestML.data <<'...'
10
+ === Simple Test Case
11
+ --- grammar1
12
+ a: /x/
13
+ --- grammar2
14
+ a:
15
+ /x/
16
+
17
+ === And over Or Precedence
18
+ --- grammar1
19
+ a: b c | d
20
+ --- grammar2
21
+ a: ( b c ) | d
22
+
23
+ === And/Or Precedence with joining
24
+ --- grammar1
25
+ a: b % c | d %% e
26
+ --- grammar2
27
+ a: ( b % c ) | ( d %% e )
28
+
29
+ === And/Or Precedence with grouping
30
+ --- grammar1
31
+ a:
32
+ b c
33
+ | (
34
+ d
35
+ | e
36
+ | f g h i
37
+ )
38
+ --- grammar2
39
+ a: ( b c ) | ( d | e | ( f g h i ) )
40
+
41
+ === In-Line Comments
42
+ --- grammar1
43
+ a: # test
44
+ b c # not d
45
+ /q/ # skipping to q
46
+ % e # using e here...
47
+ ; # comment -> semicolon test
48
+ --- grammar2
49
+ a: b c /q/ % e
50
+
51
+ === Token Per Line
52
+ --- SKIP: TODO
53
+ --- grammar1
54
+ a: /b/
55
+ --- grammar2
56
+ a
57
+ :
58
+ /b/
59
+
60
+ === Regex Combination
61
+ --- SKIP: TODO
62
+ --- grammar1: a: /b/ /c/
63
+ --- grammar2: a: /bc/
64
+
65
+ === Regex Combination by Reference
66
+ --- SKIP: TODO
67
+ --- grammar1
68
+ a: b /c/
69
+ b: /b/
70
+ --- grammar2: a: /bc/
71
+
72
+ === Multiple Rules Names per Definition
73
+ --- SKIP: TODO
74
+ --- grammar1
75
+ a b: /O HAI/
76
+ --- grammar2
77
+ a: /O HAI/
78
+ b: /O HAI/
79
+ ...
data/test/compiler.rb ADDED
@@ -0,0 +1,42 @@
1
+ require './test/lib/test_pegex'
2
+
3
+ TestML.require_or_skip 'psych'
4
+
5
+ TestML.data 'test/compiler.tml'
6
+
7
+ TestML.run do |t|
8
+ t.eval '*grammar', t.method('run_compiler_tests')
9
+ end
10
+
11
+ class TestPegex
12
+ def run_compiler_tests block, expr=nil
13
+ label '$BlockLabel - Compiler output matches bootstrap?'
14
+ run_test(
15
+ block,
16
+ '*grammar.compile.yaml == *grammar.compile.yaml',
17
+ )
18
+
19
+ label '$BlockLabel - Compressed grammar compiles the same?'
20
+ run_test(
21
+ block,
22
+ '*grammar.compress.compile.yaml == *grammar.compress.compile.yaml',
23
+ )
24
+
25
+ label '$BlockLabel - Compressed grammar matches uncompressed?'
26
+ run_test(
27
+ block,
28
+ '*grammar.compress.compile.yaml == *grammar.compile.yaml',
29
+ )
30
+ end
31
+
32
+ def compress grammar_text
33
+ grammar_text.gsub! /([^;])\n(\w+\s*:)/ do |m|
34
+ "#{$1};#{$2}"
35
+ end
36
+ grammar_text.gsub! /\s/, ''
37
+
38
+ # XXX mod/quant ERROR rules are too prtective here:
39
+ grammar_text.gsub! />%</, '> % <'
40
+ return "#{grammar_text}\n"
41
+ end
42
+ end