grammar 0.5 → 0.8

data/lib/grammar/ruby2cext.rb

@@ -0,0 +1,19 @@
+
+require 'grammar/ruby'
+require 'rubygems'
+require 'ruby2cext/eval2c'
+
+class Grammar
+class Ruby2CExt < Ruby
+    
+    def self.compile(gram)
+        code = new.dump(gram)
+        #p(code)
+        #code.lined
+        Ruby2CExtension::Eval2C.new.compile_to_proc(code.to_s).call.new
+    end
+
+end
+end
+
+

data/lib/grammar/rubycall.rb

@@ -0,0 +1,21 @@
+
+require 'grammar/ruby'
+require 'rubygems'
+
+class Grammar
+class RubyCall < Ruby
+    
+    def self.compile(gram)
+        code = new.dump(gram)
+        #p(code)
+        #code.lined
+        eval(code.to_s).new
+    end
+    def recurse(inner, expand=false, &outer) # :yield: engine
+        expand ? super : call(inner, &outer)
+    end
+
+end
+end
+
+

data/test/advanced.rb

@@ -0,0 +1,105 @@
+#!/bin/env ruby
+
+require 'stringio'
+
+require 'grammar'
+
+class Grammar
+module Test
+module Advanced
+    include Grammar::Molecules
+
+    def test_positive
+        comp = engine.compile(+E(?a) + E(?a..?z))
+        assert(comp[out="", StringIO.new("ab").method(:getc)])
+        assert_equal("a", out)
+        assert_raise(error) { comp[out="", lambda { ?b }] }
+        assert_equal("", out)
+    end
+    
+    def test_negative
+        comp = engine.compile(-E(?a) + E(?a..?z))
+        assert(comp[out="", StringIO.new("ba").method(:getc)])
+        assert_equal("b", out)
+        assert_raise(error) { comp[out="", lambda { ?a }] }
+        assert_equal("", out)
+    end
+    
+    def test_redirect
+        comp = engine.compile((E(?a)+E(?b)).redirect("") { |buf, eng|
+            eng << buf.reverse
+        })
+        assert(comp[out=[], StringIO.new("ab").method(:getc)])
+        assert_equal(["ba"], out)
+    end
+    
+    def test_discard
+        comp = engine.compile(E(?a).discard+E(?b))
+        assert(comp[out="", StringIO.new("abb").method(:getc)])
+        assert_equal("b", out)
+        assert_raise(error) { comp[out="", lambda {?b}] }
+        assert_equal("", out)
+        comp = engine.compile((E(?a)+E(?b)).discard { |e| e[:ab] } + E(?c))
+        assert(comp[out=[], StringIO.new("abca").method(:getc)])
+        assert_equal([:ab,?c], out)
+        assert_raise(error) {
+            comp[out=[], StringIO.new("a").method(:getc)]
+        }
+        assert_equal([], out)
+    end
+    
+    #def test_backref
+    #    comp = engine.compile(E(?a)+(E(?b)+E(?c)).backref { |n, engine|
+    #        engine.output[-n,n] = engine.output[-n,n].reverse
+    #    } + E(?d))
+    #    assert(comp[out="", StringIO.new("abcdab").method(:getc)])
+    #    assert_equal("acbd", out)
+    #    assert_raise(error) {
+    #        comp[out="", StringIO.new("ab").method(:getc)]
+    #    }
+    #    assert_equal("a", out)
+    #end
+    
+    def test_backtrack
+        comp = engine.compile((E(?a)+E(?b)).backtrack|(E(?a)+E(?c)))
+        assert(comp[out="", StringIO.new("abac").method(:getc)])
+        assert_equal("ab", out)
+        assert(comp[out="", StringIO.new("ac").method(:getc)])
+        assert_equal("ac", out)
+    end
+
+    def test_supply
+        token = (E(?a)|E(?b)).redirect("") { |buf, eng|
+            eng << buf.capitalize.to_sym
+        }
+        parser = E(:A) + E(:B)
+        leftover = nil
+        # TODO: leftover
+        comp = engine.compile(token.supply(parser, [])) # { |l, eng| leftover = l}
+        assert(comp[out=[], StringIO.new("abba").method(:getc)])
+        assert_equal([:A, :B], out)
+        #assert_equal([:B], leftover)
+        assert_raise(error) { comp[out=[], lambda {?a}] }
+        assert_equal([:A], out)
+    end
+    
+    def test_pipe
+        lexer = Recurse { |g| g + (E(?a)|E(?b)).redirect("") { |buf, eng|
+            eng << buf.capitalize.to_sym
+        } | NULL }
+        parser = E(:A) + E(:B)
+        leftover = nil
+        # TODO: leftover
+        comp = engine.compile(lexer.pipe(parser, [], 1)) # { |l, eng| leftover = l})
+        assert(comp[out=[], StringIO.new("abbba").method(:getc)])
+        assert_equal([:A, :B], out)
+        #assert_equal([:B, :B], leftover)
+        assert_raise(error) { comp[out=[], lambda {?a}] }
+        assert_equal([:A], out)
+    end
+    
+end
+end
+end
+
+

data/test/atoms.rb

@@ -0,0 +1,77 @@
+#!/bin/env ruby
+
+require 'stringio'
+
+require 'grammar'
+
+class Grammar
+module Test
+module Atoms
+    include Grammar::Molecules
+
+    def test_Grammar
+        gram = Grammar { |eng| eng[true] }
+        comp = engine.compile(gram)
+        assert(comp[out="", lambda {}])
+        assert_equal("", out)
+        gram << E(?a)
+        comp = engine.compile(gram)
+        assert(comp[out="", lambda {?a}])
+        assert_equal("a", out)
+        assert_raise(error) {
+            comp[out="", lambda {}]
+        }
+        assert_equal("", out)
+    end
+    
+    def test_Element
+        comp = engine.compile(Element(?a))
+        assert(comp[out="", StringIO.new("abc").method(:getc)])
+        assert_equal("a", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("bc").method(:getc)]
+        }
+        assert_equal("", out)
+        comp = engine.compile(E(?a..?z))
+        assert(comp[out="", StringIO.new("bc").method(:getc)])
+        assert_equal("b", out)
+    end
+
+    def test_Fail
+        out = ""
+        comp = engine.compile(Fail("TESTING"))
+        assert_raise(error) { comp[out, lambda {}] }
+        assert_equal("", out)
+        assert_raise(error) { comp[out="", lambda { ?a }] }
+        assert_equal("", out)
+    end
+
+    def test_NULL
+        comp = engine.compile(NULL)
+        comp[out="", lambda {}]
+        assert_equal("", out)
+        comp[out="", lambda { ?a }]
+        assert_equal("", out)
+    end
+    
+    def test_ANY
+        comp = engine.compile(ANY)
+        comp[out="", lambda { ?a }]
+        assert_equal("a", out)
+        assert_raise(error) { comp[out="", lambda { }] }
+        assert_equal("", out)
+    end
+    
+    def test_EOF
+        comp = engine.compile(EOF)
+        comp[out="", lambda {}]
+        assert_equal("", out)
+        assert_raise(error) { comp[out="", lambda { ?a }] }
+        assert_equal("", out)
+    end
+    
+end
+end
+end
+
+

data/test/basic.rb

@@ -0,0 +1,32 @@
+#!/bin/env ruby
+
+require 'stringio'
+
+require 'grammar'
+
+class Grammar
+module Test
+module Basic
+    include Grammar::Molecules
+
+    def test_plus
+        comp = engine.compile(E(?a)+E(?a..?z))
+        assert(comp[out="", StringIO.new("ab").method(:getc)])
+        assert_equal("ab", out)
+    end
+    
+    def test_or
+        comp = engine.compile(E(?a)|E(?b))
+        assert(comp[out="", StringIO.new("abc").method(:getc)])
+        assert_equal("a", out)
+        assert(comp[out="", StringIO.new("bc").method(:getc)])
+        assert_equal("b", out)
+        assert_raise(error) { comp[out="", StringIO.new("c").method(:getc)] }
+        assert_equal("", out)
+    end
+    
+end
+end
+end
+
+

data/test/composite.rb

@@ -0,0 +1,147 @@
+#!/bin/env ruby
+
+require 'stringio'
+
+require 'grammar'
+
+class Grammar
+module Test
+module Composite
+    include Grammar::Molecules
+
+    def test_Check
+        comp = engine.compile(Check {|eng| eng[1]+eng[2]==eng[3]})
+        assert(comp[out="", lambda {}])
+        comp = engine.compile(Check {|eng| eng[1]+eng[1]==eng[1]})
+        assert_raise(error) { comp[out="", lambda {}] }
+    end
+    
+    def test_not
+        comp = engine.compile(~E(?a..?z))
+        assert(comp[out="", lambda { ?A }])
+        assert_equal("A", out)
+        assert_raise(error) { comp[out="", lambda { ?a }] }
+        assert_equal("", out)
+    end
+    
+    def test_optional
+        comp = engine.compile(E(?a).optional)
+        assert(comp[out="", StringIO.new("ab").method(:getc)])
+        assert_equal("a", out)
+        assert(comp[out="", StringIO.new("b").method(:getc)])
+        assert_equal("", out)
+    end
+    
+    def test_repeat0
+        comp = engine.compile(E(?a).repeat0)
+        assert(comp[out="", lambda { }])
+        assert_equal("", out)
+        assert(comp[out="", StringIO.new("a").method(:getc)])
+        assert_equal("a", out)
+        assert(comp[out="", StringIO.new("aaa").method(:getc)])
+        assert_equal("aaa", out)
+        comp = engine.compile(E(?a).repeat0(E(?b)))
+        assert(comp[out="", StringIO.new("babaaab").method(:getc)])
+        assert_equal("b", out)
+        assert(comp[out="", StringIO.new("abaaab").method(:getc)])
+        assert_equal("ab", out)
+        assert(comp[out="", StringIO.new("aaab").method(:getc)])
+        assert_equal("aaab", out)
+    end
+    
+    def test_repeat1
+        comp = engine.compile(E(?a).repeat1)
+        out = ""
+        assert_raise(error) { comp[out, lambda { }] }
+        assert_equal("", out)
+        assert(comp[out="", StringIO.new("a").method(:getc)])
+        assert_equal("a", out)
+        assert(comp[out="", StringIO.new("aaa").method(:getc)])
+        assert_equal("aaa", out)
+        comp = engine.compile(E(?a).repeat1(E(?b)))
+        assert(comp[out="", StringIO.new("abaaabb").method(:getc)])
+        assert_equal("ab", out)
+        assert(comp[out="", StringIO.new("aaabb").method(:getc)])
+        assert_equal("aaab", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("b").method(:getc)]
+        }
+        assert_equal("", out)
+    end
+    
+    def test_times_Fixnum
+        comp = engine.compile(E(?a)*2)
+        assert(comp[out="", StringIO.new("aab").method(:getc)])
+        assert_equal("aa", out)
+        assert(comp[out="", StringIO.new("aaa").method(:getc)])
+        assert_equal("aa", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("b").method(:getc)]
+        }
+        assert_equal("", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("ab").method(:getc)]
+        }
+        assert_equal("a", out)
+    end
+    
+    def test_times_Range
+        comp = engine.compile(E(?a)*(1..2))
+        assert(comp[out="", StringIO.new("aaab").method(:getc)])
+        assert_equal("aa", out)
+        assert(comp[out="", StringIO.new("ab").method(:getc)])
+        assert_equal("a", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("b").method(:getc)]
+        }
+        assert_equal("", out)
+    end
+    
+    def test_times_Range1Inf
+        comp = engine.compile(E(?a)*(1..1.0/0)+E(?b))
+        assert(comp[out="", StringIO.new("abaaaba").method(:getc)])
+        assert_equal("ab", out)
+        assert(comp[out="", StringIO.new("aaaba").method(:getc)])
+        assert_equal("aaab", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("a").method(:getc)]
+        }
+        assert_equal("a", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("babaaaba").method(:getc)]
+        }
+        assert_equal("", out)
+    end
+    
+    def test_times_Range0Inf
+        comp = engine.compile(E(?a)*(0..1.0/0)+E(?b))
+        assert(comp[out="", StringIO.new("babaaaba").method(:getc)])
+        assert_equal("b", out)
+        assert(comp[out="", StringIO.new("abaaaba").method(:getc)])
+        assert_equal("ab", out)
+        assert(comp[out="", StringIO.new("aaaba").method(:getc)])
+        assert_equal("aaab", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("a").method(:getc)]
+        }
+        assert_equal("a", out)
+    end
+    
+    def test_group
+        comp = engine.compile((E(?a)+E(?b)).group("") + E(?c))
+        assert(comp[out=[], StringIO.new("abca").method(:getc)])
+        assert_equal(["ab",?c], out)
+        assert_raise(error) {
+            comp[out=[], StringIO.new("a").method(:getc)]
+        }
+        assert_equal([], out)
+        comp = engine.compile(E(?0..?9).repeat1.group("") { |buf, eng| buf.to_i })
+        assert(comp[out=[], StringIO.new("123").method(:getc)])
+        assert_equal([123], out)
+    end
+    
+end
+end
+end
+
+

data/test/molecules.rb

@@ -0,0 +1,125 @@
+#!/bin/env ruby
+
+require 'stringio'
+
+require 'grammar'
+
+class Grammar
+module Test
+module Molecules
+    include Grammar::Molecules
+
+    def test_Variables
+        comp = engine.compile(Variables(true) { |var| Check { var[] } })
+        assert(comp[out="", lambda {}])
+        assert_equal("", out)
+        comp = engine.compile(Variables(false) { |found|
+            (E(?a) + Grammar { |engine| found << engine[true] } | NULL) +
+            (Grammar { found[] } + E(?b) | E(?z))
+        })
+        assert(comp[out="", StringIO.new("abz").method(:getc)])
+        assert_equal("ab", out)
+        assert(comp[out="", lambda { ?z }])
+        assert_equal("z", out)
+        assert_raise(error) { comp[out="", lambda { ?b }] }
+    end
+    
+    def test_Recurse_middle
+        comp = engine.compile(Recurse { |gram| E(?() + gram + E(?)) | NULL })
+        assert(comp[out="", StringIO.new("()(())(").method(:getc)])
+        assert_equal("()", out)
+        assert(comp[out="", StringIO.new("(())(").method(:getc)])
+        assert_equal("(())", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("(").method(:getc)]
+        }
+        assert_equal("(", out)
+        assert(comp[out="", lambda {}])
+        assert_equal("", out)
+        comp = engine.compile(Recurse { |gram| E(?() + gram + E(?)) | E(?a) })
+        assert(comp[out="", StringIO.new("a(a)((a))((ab").method(:getc)])
+        assert_equal("a", out)
+        assert(comp[out="", StringIO.new("(a)((a))((ab").method(:getc)])
+        assert_equal("(a)", out)
+        assert(comp[out="", StringIO.new("((a))((ab").method(:getc)])
+        assert_equal("((a))", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("((ab").method(:getc)]
+        }
+        assert_equal("((a", out)
+    end
+    
+    def test_Recurse_right
+        comp = engine.compile(Recurse { |gram| E(?a)+gram | NULL })
+        assert(comp[out="", lambda {}])
+        assert_equal("", out)
+        assert(comp[out="", StringIO.new("a").method(:getc)])
+        assert_equal("a", out)
+        assert(comp[out="", StringIO.new("aaa").method(:getc)])
+        assert_equal("aaa", out)
+        comp = engine.compile(Recurse { |gram| E(?a) + (E(?b)|gram) })
+        assert(comp[out="", StringIO.new("abaaaba").method(:getc)])
+        assert_equal("ab", out)
+        assert(comp[out="", StringIO.new("aaaba").method(:getc)])
+        assert_equal("aaab", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("a").method(:getc)]
+        }
+        assert_equal("a", out)
+        assert_raise(error) { comp[out="", lambda { ?b }] }
+        assert_equal("", out)
+        comp = engine.compile(Recurse { |gram| E(?a) + (E(?b)|gram) } | E(?b))
+        assert(comp[out="", StringIO.new("baaaba").method(:getc)])
+        assert_equal("b", out)
+        assert(comp[out="", StringIO.new("aaaba").method(:getc)])
+        assert_equal("aaab", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("a").method(:getc)]
+        }
+        assert_equal("a", out)
+    end
+    
+    def test_Recurse_left
+        comp = engine.compile(Recurse { |gram| gram+E(?a) | NULL })
+        assert(comp[out="", lambda {}])
+        assert_equal("", out)
+        assert(comp[out="", StringIO.new("a").method(:getc)])
+        assert_equal("a", out)
+        assert(comp[out="", StringIO.new("aaa").method(:getc)])
+        assert_equal("aaa", out)
+        comp = engine.compile(Recurse { |gram| gram+E(?a) | E(?b) })
+        assert(comp[out="", lambda { ?b }])
+        assert_equal("b", out)
+        assert(comp[out="", StringIO.new("ba").method(:getc)])
+        assert_equal("ba", out)
+        assert(comp[out="", StringIO.new("baaab").method(:getc)])
+        assert_equal("baaa", out)
+        #comp = engine.compile(Recurse { |gram| gram+NULL })
+        #assert_raise(error) {
+        #    comp[out="", lambda { E(?a) }]
+        #}
+        #assert_equal("", out)
+        #comp = engine.compile(Recurse { |gram| gram })
+        #assert_raise(RuntimeError) {
+        #    comp[out="", lambda { E(?a) }]
+        #}
+        #assert_equal("", out)
+        #comp = engine.compile(Recurse { |gram| gram+NULL | NULL })
+        #assert_raise(RuntimeError) {
+        #    comp[out="", lambda { E(?a) }]
+        #}
+        #assert_equal("", out)
+        comp = engine.compile(Recurse { |gram| gram+E(?a)+E(?c) | E(?b) })
+        assert(comp[out="", StringIO.new("bacacbacab").method(:getc)])
+        assert_equal("bacac", out)
+        assert_raise(error) {
+            comp[out="", StringIO.new("bacab").method(:getc)]
+        }
+        assert_equal("baca", out)
+    end
+    
+end
+end
+end
+
+