rly 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +1 -0
- data/README.md +6 -0
- data/assets/ply_dump.erb +1 -1
- data/lib/rly/lex.rb +5 -0
- data/lib/rly/parse/grammar.rb +8 -2
- data/lib/rly/parse/ply_dump.rb +3 -1
- data/lib/rly/parse/rule_parser.rb +12 -1
- data/lib/rly/parse/yacc_symbol.rb +7 -0
- data/lib/rly/version.rb +1 -1
- data/lib/rly/yacc.rb +45 -22
- data/spec/parse/calc_spec.rb +9 -6
- data/spec/parse/rule_parser_spec.rb +4 -4
- metadata +1 -1
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -16,3 +16,9 @@ Install via rubygems
|
|
16
16
|
You need to create lexer and parser classes for each grammar you want to process.
|
17
17
|
It is commonly done by subclassing {Rly::Lex} and {Rly::Parse} classes (check the
|
18
18
|
appropriate docs).
|
19
|
+
|
20
|
+
You can also read the tutorials on the wiki:
|
21
|
+
|
22
|
+
* [Calculator Tutorial Part 1: Basic lexer](https://github.com/farcaller/rly/wiki/Calculator-Tutorial-Part-1:-Basic-lexer)
|
23
|
+
* [Calculator Tutorial Part 2: Basic parser](https://github.com/farcaller/rly/wiki/Calculator-Tutorial-Part-2:-Basic-parser)
|
24
|
+
* [Calculator Tutorial Part 3: Advanced parser](https://github.com/farcaller/rly/wiki/Calculator-Tutorial-Part-3:-Advanced-parser)
|
data/assets/ply_dump.erb
CHANGED
data/lib/rly/lex.rb
CHANGED
@@ -89,6 +89,7 @@ module Rly
|
|
89
89
|
# t = lex.next # => nil
|
90
90
|
def input(input)
|
91
91
|
@input << input
|
92
|
+
nil
|
92
93
|
end
|
93
94
|
|
94
95
|
# Processes the next token in input
|
@@ -249,6 +250,7 @@ module Rly
|
|
249
250
|
else
|
250
251
|
raise ArgumentError
|
251
252
|
end
|
253
|
+
nil
|
252
254
|
end
|
253
255
|
|
254
256
|
# Specifies a list of one-char literals
|
@@ -272,6 +274,7 @@ module Rly
|
|
272
274
|
# end
|
273
275
|
def literals(lit)
|
274
276
|
@literals = lit
|
277
|
+
nil
|
275
278
|
end
|
276
279
|
|
277
280
|
# Specifies a list of one-char symbols to be ignored in input
|
@@ -297,6 +300,7 @@ module Rly
|
|
297
300
|
# end
|
298
301
|
def ignore(ign)
|
299
302
|
@ignores = ign
|
303
|
+
nil
|
300
304
|
end
|
301
305
|
|
302
306
|
# Specifies a block that should be called on error
|
@@ -325,6 +329,7 @@ module Rly
|
|
325
329
|
# end
|
326
330
|
def on_error(&block)
|
327
331
|
@error_block = block
|
332
|
+
nil
|
328
333
|
end
|
329
334
|
end
|
330
335
|
end
|
data/lib/rly/parse/grammar.rb
CHANGED
@@ -25,7 +25,7 @@ module Rly
|
|
25
25
|
@start = nil
|
26
26
|
end
|
27
27
|
|
28
|
-
def add_production(name, symbols, &block)
|
28
|
+
def add_production(name, symbols, enforced_prec=nil, &block)
|
29
29
|
raise ArgumentError unless name.downcase == name
|
30
30
|
raise ArgumentError if name == :error
|
31
31
|
|
@@ -36,7 +36,13 @@ module Rly
|
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
39
|
-
|
39
|
+
if enforced_prec
|
40
|
+
precedence = @precedence[enforced_prec]
|
41
|
+
raise RuntimeError.new("Nothing known about the precedence of '#{enforced_prec}'") unless precedence
|
42
|
+
@used_precedence[precedence] = true
|
43
|
+
else
|
44
|
+
precedence = prec_for_rightmost_terminal(symbols)
|
45
|
+
end
|
40
46
|
|
41
47
|
mapname = "#{name.to_s} -> #{symbols.to_s}"
|
42
48
|
raise ArgumentError if @prodmap[mapname]
|
data/lib/rly/parse/ply_dump.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
require "rly/parse/lr_table"
|
2
|
+
require "rly/version"
|
3
|
+
require "erb"
|
2
4
|
|
3
5
|
module Rly
|
4
6
|
|
@@ -17,7 +19,7 @@ module Rly
|
|
17
19
|
def to_s
|
18
20
|
fn = File.join(File.dirname(__FILE__), '..', '..', '..', 'assets', 'ply_dump.erb')
|
19
21
|
e = ERB.new(open(fn).read)
|
20
|
-
e.result(TinyContext.new(g: @grammar, backlog: @backlog).get_binding)
|
22
|
+
e.result(TinyContext.new(g: @grammar, backlog: @backlog, ver: Rly::VERSION).get_binding)
|
21
23
|
end
|
22
24
|
|
23
25
|
def self.stub
|
@@ -8,6 +8,7 @@ module Rly
|
|
8
8
|
return @lexer_class if @lexer_class
|
9
9
|
|
10
10
|
@lexer_class = Class.new(Lex) do
|
11
|
+
token :PREC, /\%prec/
|
11
12
|
token :ID, /[a-zA-Z_][a-zA-Z_0-9]*/
|
12
13
|
token :LITERAL, /"."|'.'/ do |t|
|
13
14
|
t.value = t.value[1]
|
@@ -25,10 +26,20 @@ module Rly
|
|
25
26
|
|
26
27
|
@grammar = Grammar.new(self.class.lexer_class.terminals)
|
27
28
|
|
29
|
+
@grammar.add_production(:grammar_def, [:grammar]) do |d, g|
|
30
|
+
d.value = g.value
|
31
|
+
end
|
32
|
+
@grammar.add_production(:grammar, [:ID, ':', :rules, :PREC, :ID]) do |g, pname, _, r, _, prec|
|
33
|
+
productions = []
|
34
|
+
r.value.each do |p|
|
35
|
+
productions << [pname.value.to_sym, p, prec.value.to_sym]
|
36
|
+
end
|
37
|
+
g.value = productions
|
38
|
+
end
|
28
39
|
@grammar.add_production(:grammar, [:ID, ':', :rules]) do |g, pname, _, r|
|
29
40
|
productions = []
|
30
41
|
r.value.each do |p|
|
31
|
-
productions << [pname.value.to_sym, p]
|
42
|
+
productions << [pname.value.to_sym, p, nil]
|
32
43
|
end
|
33
44
|
g.value = productions
|
34
45
|
end
|
data/lib/rly/version.rb
CHANGED
data/lib/rly/yacc.rb
CHANGED
@@ -2,6 +2,7 @@ require "rly/lex"
|
|
2
2
|
require "rly/parse/grammar"
|
3
3
|
require "rly/parse/yacc_production"
|
4
4
|
require "rly/parse/yacc_symbol"
|
5
|
+
require "rly/parse/ply_dump"
|
5
6
|
|
6
7
|
module Rly
|
7
8
|
class YaccError < RuntimeError; end
|
@@ -16,7 +17,11 @@ module Rly
|
|
16
17
|
@grammar = grammar
|
17
18
|
end
|
18
19
|
|
19
|
-
def
|
20
|
+
def inspect
|
21
|
+
"#<#{self.class} ...>"
|
22
|
+
end
|
23
|
+
|
24
|
+
def parse(input=nil, trace=false)
|
20
25
|
lookahead = nil
|
21
26
|
lookaheadstack = []
|
22
27
|
actions = @lr_table.lr_action
|
@@ -51,7 +56,7 @@ module Rly
|
|
51
56
|
# is already set, we just use that. Otherwise, we'll pull
|
52
57
|
# the next token off of the lookaheadstack or from the lexer
|
53
58
|
|
54
|
-
|
59
|
+
puts "State : #{state}" if trace
|
55
60
|
|
56
61
|
unless lookahead
|
57
62
|
if lookaheadstack.empty?
|
@@ -65,6 +70,8 @@ module Rly
|
|
65
70
|
end
|
66
71
|
end
|
67
72
|
|
73
|
+
puts "Stack : #{(@symstack[1..-1].map{|s|s.type}.join(' ') + ' ' + lookahead.inspect).lstrip}" if trace
|
74
|
+
|
68
75
|
# Check the action table
|
69
76
|
ltype = lookahead.type
|
70
77
|
t = actions[state][ltype]
|
@@ -75,7 +82,7 @@ module Rly
|
|
75
82
|
@statestack.push(t)
|
76
83
|
state = t
|
77
84
|
|
78
|
-
|
85
|
+
puts "Action : Shift and goto state #{t}" if trace
|
79
86
|
|
80
87
|
@symstack.push(lookahead)
|
81
88
|
lookahead = nil
|
@@ -96,11 +103,13 @@ module Rly
|
|
96
103
|
sym.type = pname
|
97
104
|
sym.value = nil
|
98
105
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
106
|
+
if trace
|
107
|
+
if plen
|
108
|
+
puts "Action : Reduce rule [#{p}] with [#{@symstack[-plen..@symstack.length].map{|s|s.value}.join(', ')}] and goto state #{-t}"
|
109
|
+
else
|
110
|
+
puts "Action : Reduce rule [#{p}] with [] and goto state #{-t}"
|
111
|
+
end
|
112
|
+
end
|
104
113
|
|
105
114
|
if plen
|
106
115
|
targ = @symstack.pop(plen)
|
@@ -118,7 +127,7 @@ module Rly
|
|
118
127
|
@statestack.pop(plen)
|
119
128
|
instance_exec(*targ, &p.block)
|
120
129
|
|
121
|
-
|
130
|
+
puts "Result : #{targ[0].value}" if trace
|
122
131
|
|
123
132
|
@symstack.push(sym)
|
124
133
|
state = goto[@statestack[-1]][pname]
|
@@ -151,7 +160,7 @@ module Rly
|
|
151
160
|
@statestack.pop(plen)
|
152
161
|
pslice[0] = instance_exec(*pslice, &p.block)
|
153
162
|
|
154
|
-
|
163
|
+
puts "Result : #{targ[0].value}" if trace
|
155
164
|
|
156
165
|
@symstack.push(sym)
|
157
166
|
state = goto[@statestack[-1]][pname]
|
@@ -176,7 +185,7 @@ module Rly
|
|
176
185
|
n = @symstack[-1]
|
177
186
|
result = n.value
|
178
187
|
|
179
|
-
|
188
|
+
puts "Done : Returning #{result}" if trace
|
180
189
|
|
181
190
|
return result
|
182
191
|
end
|
@@ -194,7 +203,7 @@ module Rly
|
|
194
203
|
# first syntax error. This function is only called if
|
195
204
|
# errorcount == 0.
|
196
205
|
if errorcount == 0 || @errorok == true
|
197
|
-
errorcount = error_count
|
206
|
+
errorcount = self.class.error_count
|
198
207
|
@errorok = false
|
199
208
|
errtoken = lookahead
|
200
209
|
errtoken = nil if errtoken.type == :"$end"
|
@@ -289,20 +298,26 @@ module Rly
|
|
289
298
|
|
290
299
|
@grammar = Grammar.new(@lex.class.terminals)
|
291
300
|
|
292
|
-
self.class.prec_rules.each do |assoc, terms|
|
293
|
-
terms.
|
301
|
+
self.class.prec_rules.each do |assoc, terms, i|
|
302
|
+
terms.each do |term|
|
294
303
|
@grammar.set_precedence(term, assoc, i)
|
295
304
|
end
|
296
305
|
end
|
297
306
|
|
298
|
-
self.class.parsed_rules.each do |
|
299
|
-
@grammar.add_production(
|
307
|
+
self.class.parsed_rules.each do |pname, p, prec, block|
|
308
|
+
@grammar.add_production(pname, p, prec, &block)
|
300
309
|
end
|
301
310
|
|
302
311
|
@grammar.set_start
|
303
312
|
|
304
313
|
@grammar.build_lritems
|
305
314
|
|
315
|
+
if self.class.store_grammar_def
|
316
|
+
d = PlyDump.new(@grammar)
|
317
|
+
gdef = d.to_s
|
318
|
+
open(self.class.store_grammar_def, 'w') { |f| f.write(gdef) }
|
319
|
+
end
|
320
|
+
|
306
321
|
@lr_table = LRTable.new(@grammar)
|
307
322
|
|
308
323
|
@lr_table.parse_table
|
@@ -311,14 +326,20 @@ module Rly
|
|
311
326
|
end
|
312
327
|
|
313
328
|
class << self
|
314
|
-
attr_accessor :rules, :grammar, :lexer_class, :prec_rules
|
329
|
+
attr_accessor :rules, :grammar, :lexer_class, :prec_rules, :error_handler, :store_grammar_def
|
330
|
+
|
331
|
+
def store_grammar(fn)
|
332
|
+
@store_grammar_def = fn
|
333
|
+
end
|
315
334
|
|
316
335
|
def rule(desc, &block)
|
317
336
|
self.rules << [desc, block]
|
337
|
+
nil
|
318
338
|
end
|
319
339
|
|
320
340
|
def lexer(&block)
|
321
341
|
@lexer_class = Class.new(Lex, &block)
|
342
|
+
nil
|
322
343
|
end
|
323
344
|
|
324
345
|
def rules
|
@@ -327,7 +348,9 @@ module Rly
|
|
327
348
|
|
328
349
|
def precedence(*prec)
|
329
350
|
assoc = prec.shift
|
330
|
-
self.prec_rules
|
351
|
+
count = self.prec_rules.length + 1
|
352
|
+
self.prec_rules << [assoc, prec, count]
|
353
|
+
nil
|
331
354
|
end
|
332
355
|
|
333
356
|
def prec_rules
|
@@ -339,13 +362,13 @@ module Rly
|
|
339
362
|
end
|
340
363
|
|
341
364
|
def parsed_rules
|
342
|
-
@parsed_rules if @parsed_rules
|
365
|
+
return @parsed_rules if @parsed_rules
|
343
366
|
|
344
367
|
@parsed_rules = []
|
345
368
|
rp = RuleParser.new
|
346
|
-
self.rules.each do |
|
347
|
-
rp.parse(
|
348
|
-
@parsed_rules << [
|
369
|
+
self.rules.each do |desc, block|
|
370
|
+
rp.parse(desc).each do |(pname, p, prec)|
|
371
|
+
@parsed_rules << [pname, p, prec, block]
|
349
372
|
end
|
350
373
|
end
|
351
374
|
@parsed_rules
|
data/spec/parse/calc_spec.rb
CHANGED
@@ -3,7 +3,7 @@ require "rly"
|
|
3
3
|
module CalcSpecExample
|
4
4
|
class CalcLex < Rly::Lex
|
5
5
|
literals '=+-*/()'
|
6
|
-
ignore " \t"
|
6
|
+
ignore " \t\n"
|
7
7
|
|
8
8
|
token :NAME, /[a-zA-Z_][a-zA-Z0-9_]*/
|
9
9
|
|
@@ -12,11 +12,10 @@ module CalcSpecExample
|
|
12
12
|
t
|
13
13
|
end
|
14
14
|
|
15
|
-
token(/\n+/) { |t| t.lexer.lineno += t.value.count("\n") }
|
16
|
-
|
17
15
|
on_error do |t|
|
18
16
|
puts "Illegal character #{t.value}"
|
19
17
|
t.lexer.pos += 1
|
18
|
+
nil
|
20
19
|
end
|
21
20
|
end
|
22
21
|
|
@@ -44,9 +43,9 @@ module CalcSpecExample
|
|
44
43
|
ex.value = e1.value.send(op.value, e2.value)
|
45
44
|
end
|
46
45
|
|
47
|
-
|
48
|
-
|
49
|
-
|
46
|
+
rule 'expression : "-" expression %prec UMINUS' do |ex, _, e|
|
47
|
+
ex.value = - e.value
|
48
|
+
end
|
50
49
|
|
51
50
|
rule 'expression : "(" expression ")"' do |ex, _, e, _|
|
52
51
|
ex.value = e.value
|
@@ -92,4 +91,8 @@ describe 'Calculator' do
|
|
92
91
|
@calc.parse('magic = 42')
|
93
92
|
@calc.parse('2 * magic').should == 84
|
94
93
|
end
|
94
|
+
|
95
|
+
it "follows special case precedence rules" do
|
96
|
+
@calc.parse('2 + - 2 + 1').should == 1
|
97
|
+
end
|
95
98
|
end
|
@@ -12,9 +12,9 @@ describe Rly::RuleParser do
|
|
12
12
|
productions = p.parse(s)
|
13
13
|
|
14
14
|
productions.length.should == 4
|
15
|
-
productions[0].should == [:expression, [:expression, '+', :expression]]
|
16
|
-
productions[1].should == [:expression, [:expression, '-', :expression]]
|
17
|
-
productions[2].should == [:expression, [:expression, '*', :expression]]
|
18
|
-
productions[3].should == [:expression, [:expression, '/', :expression]]
|
15
|
+
productions[0].should == [:expression, [:expression, '+', :expression], nil]
|
16
|
+
productions[1].should == [:expression, [:expression, '-', :expression], nil]
|
17
|
+
productions[2].should == [:expression, [:expression, '*', :expression], nil]
|
18
|
+
productions[3].should == [:expression, [:expression, '/', :expression], nil]
|
19
19
|
end
|
20
20
|
end
|