grammar 0.5 → 0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,606 @@
1
+ require 'thread'
2
+
3
+ class Grammar
4
+ class Ruby
5
+
6
+ # compile a grammar to a parser. The parser can be called with
7
+ # an input (#[] to get next element) and an output (#<< appends
8
+ # to it).
9
+ def self.compile(gram)
10
+ code = new.dump(gram)
11
+ #p(code)
12
+ #code.lined
13
+ eval(code.to_s).new
14
+ end
15
+ # dump the grammar to a Code object. #to_s get get Ruby code
16
+ # from it.
17
+ def self.dump(gram)
18
+ new.dump(gram)
19
+ end
20
+ # convert a normal Ruby object to something that can be used
21
+ # in an action block. Set +clone+ if a clone should be made
22
+ # everytime this is referenced.
23
+ def [](object, clone=false)
24
+ if clone
25
+ case object
26
+ when String,Bignum,Float
27
+ Code::Primary.new(object.inspect)
28
+ when Array,Hash
29
+ object.empty? ? Code::Primary.new(object.inspect) :
30
+ @constants[object].clone
31
+ else
32
+ code = @constants[object]
33
+ code._immediate ? code : code.clone
34
+ end
35
+ else
36
+ @constants[object]
37
+ end
38
+ end
39
+ # method call. Typically this shouldn't be needed.
40
+ # +obj+.+method+(*+args+, &+block+) should be sufficient.
41
+ def send(obj, method, *args, &block)
42
+ obj.method_missing(method, *args, &block)
43
+ end
44
+ # method call with the last argument splatted.
45
+ def send_splat(obj, method, *args, &block)
46
+ args << args.pop._splat
47
+ obj.method_missing(method, *args, &block)
48
+ end
49
+ # replacement for ruby's builtin "or"/"||" operators
50
+ def or(first, &second) # :yield:
51
+ first._or(&second)
52
+ end
53
+ # replacement for ruby's builtin "and"/"&&" operators
54
+ def and(first, &second) # :yield:
55
+ first._and(&second)
56
+ end
57
+ # replacement for ruby's builtin ";" operator
58
+ def steps(*steps)
59
+ steps.inject { |compound,step| compound._step(step) }
60
+ end
61
+ # replacement for ruby's builtin "not"/"!" operators
62
+ def not(operand)
63
+ operand._not
64
+ end
65
+ # replacement for ruby's builtin "if"/"else"/"?"/":" constructs
66
+ def if(condition, yes, &no)
67
+ condition._question(yes[], no[])
68
+ end
69
+ # appends to the engine's output
70
+ def <<(item)
71
+ @output << item
72
+ end
73
+
74
+ def initialize
75
+ @followed = 0
76
+ @consumed = 0
77
+ @output0 = Code::Primary.new("o")
78
+ @output = @output0
79
+ @input0 = Code::Primary.new("i")
80
+ @input = @input0
81
+ @lookahead0 = Code::Primary.new("la")
82
+ @lookahead = @lookahead0
83
+ @fail0 = Code::Primary.new("f")
84
+ @fail = @fail0
85
+ @failure_hard = lambda { |i, la, *expected|
86
+ Code::Always.new(@fail[i, la, *expected])
87
+ }
88
+ @failure_break = proc { Code::Always.new(Code::Implicit.break) }
89
+ @failure = @failure_hard
90
+ @expected = []
91
+ @assignments = []
92
+ @constants = Hash.new { |h, object|
93
+ code = Code[object]
94
+ if code._immediate
95
+ h[object] = code
96
+ else
97
+ h[object] = assign(code)
98
+ end
99
+ }
100
+ @method_definitions = []
101
+ @methods = {}
102
+ @consume = generate_consume
103
+ @void = nil
104
+ @scope = nil
105
+ end
106
+
107
+ def assign(code)
108
+ var = Code::Primary.new("@_#{@assignments.size}")
109
+ ensure
110
+ @assignments << var._assign(code)
111
+ end
112
+
113
+ def generate_consume
114
+ @consume = Code::SharedSteps.new(
115
+ (@output ? [@output << @lookahead] : []) <<
116
+ @lookahead._assign(@input[]) <<
117
+ Code::True
118
+ )
119
+ end
120
+
121
+ class CodeProxy
122
+ include Enumerable
123
+ def initialize(code)
124
+ @code = code
125
+ end
126
+ def inspect
127
+ @code._inspect
128
+ end
129
+ def lines
130
+ @text ||= begin
131
+ @code._lines(text=[0,""])
132
+ text
133
+ end
134
+ end
135
+ def size
136
+ lines.size>>1
137
+ end
138
+ def each(&block)
139
+ text = lines
140
+ i = 0
141
+ while i<text.size
142
+ yield((" "*text[i]).concat(text[i+=1]))
143
+ i += 1
144
+ end
145
+ end
146
+ def to_s(s="")
147
+ each { |line| s.concat(line) << ?\n }
148
+ s.chomp!
149
+ end
150
+ def to_str(s="")
151
+ to_s(s)
152
+ end
153
+ def display(port=$>)
154
+ each { |line| port.puts(line) }
155
+ end
156
+ def lined(port=$>)
157
+ f = "%#{size.to_s.size}i: "
158
+ i = 0
159
+ each { |line|
160
+ port.puts((f % (i+=1)).concat(line))
161
+ }
162
+ end
163
+ end
164
+
165
+ def dump(gram)
166
+ scope = @scope
167
+ @scope = "call"
168
+ top = gram[self]
169
+ @scope = scope
170
+ CodeProxy.new(Code[Class].new { ||
171
+ steps = []
172
+ @assignments.each { |assignment|
173
+ steps << assignment
174
+ }
175
+ steps = [Code::Steps.new(steps)._def(:initialize)] unless
176
+ steps.empty?
177
+ @method_definitions.each { |definition|
178
+ steps << definition
179
+ }
180
+ steps << top._ensure(
181
+ Code::Self.block_given?._and{
182
+ Code::Implicit.yield(@lookahead)
183
+ }
184
+ )._def(:call,
185
+ @output,
186
+ @input,
187
+ @lookahead._assign(@input[]),
188
+ @fail._assign(Code::Self.lambda { |i, la, *exp|
189
+ exp = exp[0]
190
+ Code::Implicit.raise(
191
+ Code["found: "].concat(la.inspect).
192
+ concat(Code[", expected: "]).
193
+ concat(exp.map{|e|e.inspect}.join(Code[", "]))
194
+ )
195
+ })
196
+ )
197
+ #steps << Code::Self.alias_method(Code[:call], Code[name])
198
+ steps << Code::Self.alias_method(Code[:"[]"], Code[:call])
199
+ Code::Steps.new(steps)
200
+ })
201
+ end
202
+
203
+ def failure(expected=nil)
204
+ @expected << self[expected] if expected
205
+ if @failure
206
+ begin
207
+ @failure[@input, @lookahead, *@expected]
208
+ ensure
209
+ @expected = []
210
+ end
211
+ else
212
+ Code::False
213
+ end
214
+ end
215
+ def consume(matching, expected=nil)
216
+ matching._and{@consume||Code::True}._or{failure(expected)}
217
+ ensure
218
+ if @consume
219
+ @failure = @failure_hard
220
+ @consumed += 1
221
+ end
222
+ end
223
+ def eof
224
+ @lookahead._not._or{
225
+ @expected << Code["EOF"]
226
+ failure
227
+ }
228
+ end
229
+ def any
230
+ @expected << Code["ANY"]
231
+ consume(@lookahead)
232
+ end
233
+ def match(pat)
234
+ if Range===pat
235
+ op = pat.exclude_end? ? :> : :>=
236
+ @expected << Code[pat]
237
+ consume(@lookahead._and{(self[pat.begin]<=@lookahead)._and{
238
+ self[pat.end].__send__(op, @lookahead)
239
+ }})
240
+ else
241
+ code = @constants[pat]
242
+ @expected << code
243
+ consume(if code._immediate
244
+ code.equal?(@lookahead)
245
+ else
246
+ code===@lookahead
247
+ end)
248
+ end
249
+ end
250
+ def always # :yield: engine
251
+ Code::Always.new(yield(self))
252
+ end
253
+ def output(&block) # :yield: buf[, engine]
254
+ Code::Always.new(
255
+ block_given? ? @output._assign(
256
+ block.arity==1 ? yield(@output) : yield(@output, self)
257
+ ) : @output
258
+ )
259
+ end
260
+
261
+ def alternation(gram1) # :yield: engine
262
+ failure = @failure
263
+ @failure = nil
264
+ begin
265
+ gram1[self]
266
+ ensure
267
+ failure1 = @failure
268
+ end._or{
269
+ @failure = failure
270
+ yield(self)
271
+ }
272
+ ensure
273
+ @failure = failure1 if failure1
274
+ end
275
+
276
+ def sequence(gram1) # :yield: engine
277
+ nexpected = @expected.size
278
+ @followed += 1
279
+ begin
280
+ gram1[self]
281
+ ensure
282
+ @followed -= 1
283
+ end._and{
284
+ expected = @expected
285
+ begin
286
+ if expected.size>nexpected
287
+ @expected = []
288
+ @consume ? yield(self) : Code::True
289
+ else
290
+ yield(self)
291
+ end
292
+ ensure
293
+ @expected = expected
294
+ end
295
+ }
296
+ end
297
+
298
+ def positive # :yield: engine
299
+ consume = @consume
300
+ @consume = nil
301
+ begin
302
+ yield(self)
303
+ ensure
304
+ @consume = consume
305
+ end
306
+ end
307
+
308
+ def negative # :yield: engine
309
+ consume = @consume
310
+ @consume = nil
311
+ failure = @failure
312
+ @failure = nil
313
+ begin
314
+ yield(self)._not
315
+ ensure
316
+ @failure = failure
317
+ @consume = consume
318
+ end._or{
319
+ @expected << Code["something else"]
320
+ failure()
321
+ }
322
+ end
323
+
324
+ class Var
325
+ attr_reader(:var)
326
+ def initialize(var, visitor)
327
+ @var = var
328
+ @visitor = visitor
329
+ end
330
+ def []
331
+ @visitor[]
332
+ @var
333
+ end
334
+ def <<(val)
335
+ @visitor[]
336
+ @var._assign(val)
337
+ end
338
+ end
339
+
340
+ def variables(n=nil, &block) # :yield: *vars
341
+ scopes = {}
342
+ visitor = lambda { scopes[@scope] = true }
343
+ vars2 = nil
344
+ Code::Local.new(n||block.arity) { |*vars|
345
+ vars2 = vars
346
+ yield(*vars.map { |var| Var.new(var, visitor) })[self]
347
+ }
348
+ ensure
349
+ if scopes.size>1
350
+ vars2.each { |var|
351
+ var._data.replace(assign(Code[nil])._data)
352
+ }
353
+ end
354
+ end
355
+
356
+ def call(inner=nil, &outer) # :yield: engine
357
+ @consume or return(inner ? recurse(inner, true, &outer) : outer[self])
358
+ @methods[outer] or begin
359
+ i = @method_definitions.size
360
+ @method_definitions << nil
361
+ name = "m#{i}"
362
+ output = @output
363
+ input = @input
364
+ lookahead = @lookahead
365
+ fail = @fail
366
+ failure = @failure
367
+ expected = @expected
368
+ scope = @scope
369
+ consume = @consume
370
+ begin
371
+ @output = @output0
372
+ @input = @input0
373
+ @lookahead = @lookahead0
374
+ @scope = name
375
+ @fail = @fail0
376
+ @failure = nil
377
+ @expected = []
378
+ @consume = nil
379
+ code = inner ? recurse(inner, true, &outer) : outer[self]
380
+ @expected.clear if code._always
381
+ @methods[outer] = [name,@expected]
382
+ @fail = @fail0
383
+ @failure = nil
384
+ @expected = []
385
+ generate_consume
386
+ code = inner ? recurse(inner, true, &outer) : outer[self]
387
+ @method_definitions[i] =
388
+ code._ensure(Code::Implicit.yield(@lookahead))._def(
389
+ name,
390
+ @output,
391
+ @input,
392
+ @lookahead,
393
+ @fail
394
+ )
395
+ ensure
396
+ @output = output
397
+ @input = input
398
+ @lookahead = lookahead
399
+ @consume = consume
400
+ @fail = fail
401
+ @failure = failure
402
+ @expected = expected
403
+ @scope = scope
404
+ end
405
+ end
406
+ name,expected = @methods[outer]
407
+ output = @output || (@void ||= assign(
408
+ Code[Class].new {
409
+ Code::Def.new(:<<,
410
+ [Code::Primary.new("other")],
411
+ Code::Self)
412
+ }.new
413
+ ))
414
+ call = Code::Self.method_missing(name, output, @input,
415
+ @lookahead, @fail) {
416
+ |la|
417
+ @lookahead._assign(la)
418
+ }
419
+ if expected.empty?
420
+ Code::Always.new(call)
421
+ else
422
+ call._or{ @expected.concat(expected); failure() }
423
+ end
424
+
425
+ end
426
+
427
+ def recurse(inner, expand=false, &outer) # :yield: engine
428
+ inner0 = inner.to_proc
429
+ left = false
430
+ right = false
431
+ right0 = false
432
+ middle = false
433
+ failure = @failure
434
+ followed = @followed
435
+ consumed = @consumed
436
+ nexpected = @expected.size
437
+ inner << proc {
438
+ if @consumed==consumed
439
+ @followed!=followed or raise("something must follow left recursion")
440
+ left = true
441
+ Code::False
442
+ elsif !left && @followed==followed && @failure
443
+ if !failure
444
+ right0 = true
445
+ recurse(inner, true, &outer)
446
+ else
447
+ right = true
448
+ Code::FalseTerminator
449
+ end
450
+ else
451
+ middle = true
452
+ call(inner, &outer)
453
+ end
454
+ }
455
+ body = yield(self)
456
+ if !@consume
457
+ body
458
+ elsif left
459
+ !right or raise("middle recursion should have happened")
460
+ consumed = @consumed
461
+ inner << proc {
462
+ if @consumed==consumed
463
+ @failure = @failure_break
464
+ Code::True
465
+ else
466
+ call(inner, &outer)
467
+ end
468
+ }
469
+ body._and {
470
+ begin
471
+ expected = @expected
472
+ @expected = []
473
+ Code::While.new(Code::True, yield(self))._step(Code::True)
474
+ ensure
475
+ @expected = expected
476
+ end
477
+ }
478
+ elsif right
479
+ Code::Until.new(body)._step(Code::True)
480
+ elsif !expand && (@methods.include?(outer) || !right0)
481
+ n = @expected.size-nexpected
482
+ @expected.slice!(-n, n)
483
+ call(inner, &outer)
484
+ else
485
+ body
486
+ end
487
+ ensure
488
+ inner << inner0
489
+ end
490
+
491
+ def redirect(gram, buf0, &block) # :yield: buf[, engine]
492
+ @consume or return(gram[self])
493
+ output = @output
494
+ consume = @consume
495
+ @followed += 1 if block_given?
496
+ Code::Local.new { |buf|
497
+ begin
498
+ @output = buf
499
+ @consume = generate_consume
500
+ @output._assign(self[buf0,true])._step(
501
+ gram[self]
502
+ )
503
+ ensure
504
+ @followed -= 1 if block_given?
505
+ @output = output
506
+ @consume = consume
507
+ end._and {
508
+ !block_given? ? Code::True :
509
+ Code::Always.new(block.arity==1 ? yield(buf) : yield(buf, self))
510
+ }
511
+ }
512
+ end
513
+
514
+ def discard(gram) # :yield: engine
515
+ @consume or return(gram[self])
516
+ output = @output
517
+ consume = @consume
518
+ @followed += 1 if block_given?
519
+ begin
520
+ @output = nil
521
+ @consume = generate_consume
522
+ gram[self]
523
+ ensure
524
+ @followed -= 1 if block_given?
525
+ @output = output
526
+ @consume = consume
527
+ end._and {
528
+ !block_given? ? Code::True :
529
+ Code::Always.new(@output << yield(self))
530
+ }
531
+ end
532
+
533
+ def supply(tokenizer, parser, buf0, &block) # :yield: buf[, engine]
534
+ @consume or return(Code::True)
535
+ input0 = @input
536
+ lookahead0 = @lookahead
537
+ output0 = @output
538
+ consume0 = @consume
539
+ @followed += 1 if block_given?
540
+ Code::Local.new { |buf,index,input1,lookahead1|
541
+ begin
542
+ Code::Steps.new([
543
+ buf._assign(self[buf0,true]),
544
+ index._assign(Code[-1]),
545
+ input1._assign(Code::Implicit::lambda { ||
546
+ buf[index._assign(index+Code[1])]._or { steps(
547
+ buf.slice!(Code[0], buf.size),
548
+ begin
549
+ @output = buf
550
+ @consume = generate_consume
551
+ @expected = []
552
+ tokenizer[self]
553
+ ensure
554
+ @output = output0
555
+ @consume = consume0
556
+ end,
557
+ buf[index._assign(Code[0])]
558
+ )}
559
+ }),
560
+ lookahead1._assign(input1[]),
561
+ begin
562
+ @input = input1
563
+ @lookahead = lookahead1
564
+ @consume = generate_consume
565
+ @expected = []
566
+ parser[self]
567
+ ensure
568
+ @input = input0
569
+ @lookahead = lookahead0
570
+ @consume = consume0
571
+ end
572
+ ])
573
+ ensure
574
+ @followed -= 1 if block_given?
575
+ end._and {
576
+ !block_given? ? Code::True :
577
+ steps(
578
+ buf.slice!(Code[0], index+Code[1]),
579
+ buf << lookahead1,
580
+ Code::Always.new(
581
+ block.arity==1 ? yield(buf) : yield(buf, self)
582
+ )
583
+ )
584
+ }
585
+ }
586
+ ensure
587
+ @expected = []
588
+ end
589
+
590
+ def backref(gram, &block) # :yield: n[, engine]
591
+ raise("TODO")
592
+ end
593
+
594
+ def backtrack(gram, len=nil, &block) # :yield: n[, engine]
595
+ raise("TODO")
596
+ end
597
+
598
+ def pipe(lexer, parser, buf0, max_size=nil, &block) # :yield: buf[, engine]
599
+ raise("TODO")
600
+ end
601
+
602
+ end
603
+ end
604
+
605
+ require 'grammar/ruby/code'
606
+