grammar 0.5 → 0.8

data/lib/grammar/ruby.rb

@@ -0,0 +1,606 @@
+require 'thread'
+
+class Grammar
+class Ruby
+    
+    # compile a grammar to a parser.  The parser can be called with
+    # an input (#[] to get next element) and an output (#<< appends
+    # to it).
+    def self.compile(gram)
+        code = new.dump(gram)
+        #p(code)
+        #code.lined
+        eval(code.to_s).new
+    end
+    # dump the grammar to a Code object.  #to_s get get Ruby code
+    # from it.
+    def self.dump(gram)
+        new.dump(gram)
+    end
+    # convert a normal Ruby object to something that can be used
+    # in an action block.  Set +clone+ if a clone should be made
+    # everytime this is referenced.
+    def [](object, clone=false)
+        if clone
+            case object
+            when String,Bignum,Float
+                Code::Primary.new(object.inspect)
+            when Array,Hash
+                object.empty? ? Code::Primary.new(object.inspect) :
+                    @constants[object].clone
+            else
+                code = @constants[object]
+                code._immediate ? code : code.clone
+            end
+        else
+            @constants[object]
+        end
+    end
+    # method call.  Typically this shouldn't be needed.
+    # +obj+.+method+(*+args+, &+block+) should be sufficient.
+    def send(obj, method, *args, &block)
+        obj.method_missing(method, *args, &block)
+    end
+    # method call with the last argument splatted.
+    def send_splat(obj, method, *args, &block)
+        args << args.pop._splat
+        obj.method_missing(method, *args, &block)
+    end
+    # replacement for ruby's builtin "or"/"||" operators
+    def or(first, &second) # :yield:
+        first._or(&second)
+    end
+    # replacement for ruby's builtin "and"/"&&" operators
+    def and(first, &second) # :yield:
+        first._and(&second)
+    end
+    # replacement for ruby's builtin ";" operator
+    def steps(*steps)
+        steps.inject { |compound,step| compound._step(step) }
+    end
+    # replacement for ruby's builtin "not"/"!" operators
+    def not(operand)
+        operand._not
+    end
+    # replacement for ruby's builtin "if"/"else"/"?"/":" constructs
+    def if(condition, yes, &no)
+        condition._question(yes[], no[])
+    end
+    # appends to the engine's output
+    def <<(item)
+        @output << item
+    end    
+    
+    def initialize
+        @followed = 0
+        @consumed = 0
+        @output0 = Code::Primary.new("o")
+        @output = @output0
+        @input0 = Code::Primary.new("i")
+        @input = @input0
+        @lookahead0 = Code::Primary.new("la")
+        @lookahead = @lookahead0
+        @fail0 = Code::Primary.new("f")
+        @fail = @fail0
+        @failure_hard = lambda { |i, la, *expected|
+            Code::Always.new(@fail[i, la, *expected])
+        }
+        @failure_break = proc { Code::Always.new(Code::Implicit.break) }
+        @failure = @failure_hard
+        @expected = []
+        @assignments = []
+        @constants = Hash.new { |h, object|
+            code = Code[object]
+            if code._immediate
+                h[object] = code
+            else
+                h[object] = assign(code)
+            end
+        }
+        @method_definitions = []
+        @methods = {}
+        @consume = generate_consume
+        @void = nil
+        @scope = nil
+    end
+
+    def assign(code)
+        var = Code::Primary.new("@_#{@assignments.size}")
+    ensure
+        @assignments << var._assign(code)
+    end
+    
+    def generate_consume
+        @consume = Code::SharedSteps.new(
+            (@output ? [@output << @lookahead] : []) <<
+            @lookahead._assign(@input[]) <<
+            Code::True
+        )
+    end
+
+    class CodeProxy
+        include Enumerable
+        def initialize(code)
+            @code = code
+        end
+        def inspect
+            @code._inspect
+        end
+        def lines
+            @text ||= begin
+                @code._lines(text=[0,""])
+                text
+            end
+        end
+        def size
+            lines.size>>1
+        end
+        def each(&block)
+            text = lines
+            i = 0
+            while i<text.size
+                yield((" "*text[i]).concat(text[i+=1]))
+                i += 1
+            end
+        end
+        def to_s(s="")
+            each { |line| s.concat(line) << ?\n }
+            s.chomp!
+        end
+        def to_str(s="")
+            to_s(s)
+        end
+        def display(port=$>)
+            each { |line| port.puts(line) }
+        end
+        def lined(port=$>)
+            f = "%#{size.to_s.size}i: "
+            i = 0
+            each { |line|
+                port.puts((f % (i+=1)).concat(line))
+            }
+        end
+    end
+    
+    def dump(gram)
+        scope = @scope
+        @scope = "call"
+        top = gram[self]
+        @scope = scope
+        CodeProxy.new(Code[Class].new { ||
+            steps = []
+            @assignments.each { |assignment|
+                steps << assignment
+            }
+            steps = [Code::Steps.new(steps)._def(:initialize)] unless
+                steps.empty?
+            @method_definitions.each { |definition|
+                steps << definition
+            }
+            steps << top._ensure(
+                Code::Self.block_given?._and{
+                    Code::Implicit.yield(@lookahead)
+                }
+            )._def(:call,
+                @output,
+                @input,
+                @lookahead._assign(@input[]),
+                @fail._assign(Code::Self.lambda { |i, la, *exp|
+                    exp = exp[0]
+                    Code::Implicit.raise(
+                        Code["found: "].concat(la.inspect).
+                            concat(Code[", expected: "]).
+                            concat(exp.map{|e|e.inspect}.join(Code[", "]))
+                    )
+                })
+            )
+            #steps << Code::Self.alias_method(Code[:call], Code[name])
+            steps << Code::Self.alias_method(Code[:"[]"], Code[:call])
+            Code::Steps.new(steps)
+        })
+    end
+    
+    def failure(expected=nil)
+        @expected << self[expected] if expected
+        if @failure
+            begin
+                @failure[@input, @lookahead, *@expected]
+            ensure
+                @expected = []
+            end
+        else
+            Code::False
+        end
+    end
+    def consume(matching, expected=nil)
+        matching._and{@consume||Code::True}._or{failure(expected)}
+    ensure
+        if @consume
+            @failure = @failure_hard
+            @consumed += 1
+        end
+    end
+    def eof
+        @lookahead._not._or{
+            @expected << Code["EOF"]
+            failure
+        }
+    end
+    def any
+        @expected << Code["ANY"]
+        consume(@lookahead)
+    end
+    def match(pat)
+        if Range===pat
+            op = pat.exclude_end? ? :> : :>=
+            @expected << Code[pat]
+            consume(@lookahead._and{(self[pat.begin]<=@lookahead)._and{
+                self[pat.end].__send__(op, @lookahead)
+            }})
+        else
+            code = @constants[pat]
+            @expected << code
+            consume(if code._immediate
+                code.equal?(@lookahead)
+            else
+                code===@lookahead
+            end)
+        end
+    end
+    def always # :yield: engine
+        Code::Always.new(yield(self))
+    end
+    def output(&block) # :yield: buf[, engine]
+        Code::Always.new(
+            block_given? ? @output._assign(
+                block.arity==1 ? yield(@output) : yield(@output, self)
+            ) : @output
+        )
+    end
+
+    def alternation(gram1) # :yield: engine
+        failure = @failure
+        @failure = nil
+        begin
+            gram1[self]
+        ensure
+            failure1 = @failure
+        end._or{
+            @failure = failure
+            yield(self)
+        }
+    ensure
+        @failure = failure1 if failure1
+    end
+    
+    def sequence(gram1) # :yield: engine
+        nexpected = @expected.size
+        @followed += 1
+        begin
+            gram1[self]
+        ensure
+            @followed -= 1
+        end._and{
+            expected = @expected
+            begin
+                if expected.size>nexpected
+                    @expected = []
+                    @consume ? yield(self) : Code::True
+                else
+                    yield(self)
+                end
+            ensure
+                @expected = expected
+            end
+        }
+    end
+    
+    def positive # :yield: engine
+        consume = @consume
+        @consume = nil
+        begin
+            yield(self)
+        ensure
+            @consume = consume
+        end
+    end
+
+    def negative # :yield: engine
+        consume = @consume
+        @consume = nil
+        failure = @failure
+        @failure = nil
+        begin
+            yield(self)._not
+        ensure
+            @failure = failure
+            @consume = consume
+        end._or{
+            @expected << Code["something else"]
+            failure()
+        }
+    end
+    
+    class Var
+        attr_reader(:var)
+        def initialize(var, visitor)
+            @var = var
+            @visitor = visitor
+        end
+        def []
+            @visitor[]
+            @var
+        end
+        def <<(val)
+            @visitor[]
+            @var._assign(val)
+        end
+    end
+
+    def variables(n=nil, &block) # :yield: *vars
+        scopes = {}
+        visitor = lambda { scopes[@scope] = true }
+        vars2 = nil
+        Code::Local.new(n||block.arity) { |*vars|
+            vars2 = vars
+            yield(*vars.map { |var| Var.new(var, visitor) })[self]
+        }
+    ensure
+        if scopes.size>1
+            vars2.each { |var|
+                var._data.replace(assign(Code[nil])._data)
+            }
+        end
+    end
+    
+    def call(inner=nil, &outer) # :yield: engine
+        @consume or return(inner ? recurse(inner, true, &outer) : outer[self])
+        @methods[outer] or begin
+            i = @method_definitions.size
+            @method_definitions << nil
+            name = "m#{i}"
+            output = @output
+            input = @input
+            lookahead = @lookahead
+            fail = @fail
+            failure = @failure
+            expected = @expected
+            scope = @scope
+            consume = @consume
+            begin
+                @output = @output0
+                @input = @input0
+                @lookahead = @lookahead0
+                @scope = name
+                @fail = @fail0
+                @failure = nil
+                @expected = []
+                @consume = nil
+                code = inner ? recurse(inner, true, &outer) : outer[self]
+                @expected.clear if code._always
+                @methods[outer] = [name,@expected]
+                @fail = @fail0
+                @failure = nil
+                @expected = []
+                generate_consume
+                code = inner ? recurse(inner, true, &outer) : outer[self]
+                @method_definitions[i] =
+                    code._ensure(Code::Implicit.yield(@lookahead))._def(
+                        name,
+                        @output,
+                        @input,
+                        @lookahead,
+                        @fail
+                    )
+            ensure
+                @output = output
+                @input = input
+                @lookahead = lookahead
+                @consume = consume
+                @fail = fail
+                @failure = failure
+                @expected = expected
+                @scope = scope
+            end
+        end
+        name,expected = @methods[outer]
+        output = @output || (@void ||= assign(
+            Code[Class].new {
+                Code::Def.new(:<<,
+                    [Code::Primary.new("other")],
+                    Code::Self)
+            }.new
+        ))
+        call = Code::Self.method_missing(name, output, @input,
+                                         @lookahead, @fail) {
+            |la|
+            @lookahead._assign(la)
+        }
+        if expected.empty?
+            Code::Always.new(call)
+        else
+            call._or{ @expected.concat(expected); failure() }
+        end
+        
+    end
+    
+    def recurse(inner, expand=false, &outer) # :yield: engine
+        inner0 = inner.to_proc
+        left = false
+        right = false
+        right0 = false
+        middle = false
+        failure = @failure
+        followed = @followed
+        consumed = @consumed
+        nexpected = @expected.size
+        inner << proc {
+            if @consumed==consumed
+                @followed!=followed or raise("something must follow left recursion")
+                left = true
+                Code::False
+            elsif !left && @followed==followed && @failure
+                if !failure
+                    right0 = true
+                    recurse(inner, true, &outer)
+                else
+                    right = true
+                    Code::FalseTerminator
+                end
+            else
+                middle = true
+                call(inner, &outer)
+            end
+        }
+        body = yield(self)
+        if !@consume
+            body
+        elsif left
+            !right or raise("middle recursion should have happened")
+            consumed = @consumed
+            inner << proc {
+                if @consumed==consumed
+                    @failure = @failure_break
+                    Code::True
+                else
+                    call(inner, &outer)
+                end
+            }
+            body._and {
+                begin
+                    expected = @expected
+                    @expected = []
+                    Code::While.new(Code::True, yield(self))._step(Code::True)
+                ensure
+                    @expected = expected
+                end
+            }
+        elsif right
+            Code::Until.new(body)._step(Code::True)
+        elsif !expand && (@methods.include?(outer) || !right0)
+            n = @expected.size-nexpected
+            @expected.slice!(-n, n)
+            call(inner, &outer)
+        else
+            body
+        end
+    ensure
+        inner << inner0
+    end
+    
+    def redirect(gram, buf0, &block) # :yield: buf[, engine]
+        @consume or return(gram[self])
+        output = @output
+        consume = @consume
+        @followed += 1 if block_given?
+        Code::Local.new { |buf|
+            begin
+                @output = buf
+                @consume = generate_consume
+                @output._assign(self[buf0,true])._step(
+                    gram[self]
+                )
+            ensure
+                @followed -= 1 if block_given?
+                @output = output
+                @consume = consume
+            end._and {
+                !block_given? ? Code::True :
+                Code::Always.new(block.arity==1 ? yield(buf) : yield(buf, self))
+            }
+        }
+    end
+    
+    def discard(gram) # :yield: engine
+        @consume or return(gram[self])
+        output = @output
+        consume = @consume
+        @followed += 1 if block_given?
+        begin
+            @output = nil
+            @consume = generate_consume
+            gram[self]
+        ensure
+            @followed -= 1 if block_given?
+            @output = output
+            @consume = consume
+        end._and {
+            !block_given? ? Code::True :
+            Code::Always.new(@output << yield(self))
+        }
+    end
+    
+    def supply(tokenizer, parser, buf0, &block) # :yield: buf[, engine]
+        @consume or return(Code::True)
+        input0 = @input
+        lookahead0 = @lookahead
+        output0 = @output
+        consume0 = @consume
+        @followed += 1 if block_given?
+        Code::Local.new { |buf,index,input1,lookahead1|
+            begin
+                Code::Steps.new([
+                    buf._assign(self[buf0,true]),
+                    index._assign(Code[-1]),
+                    input1._assign(Code::Implicit::lambda { ||
+                        buf[index._assign(index+Code[1])]._or { steps(
+                            buf.slice!(Code[0], buf.size),
+                            begin
+                                @output = buf
+                                @consume = generate_consume
+                                @expected = []
+                                tokenizer[self]
+                            ensure
+                                @output = output0
+                                @consume = consume0
+                            end,
+                            buf[index._assign(Code[0])]
+                        )}
+                    }),
+                    lookahead1._assign(input1[]),
+                    begin
+                        @input = input1
+                        @lookahead = lookahead1
+                        @consume = generate_consume
+                        @expected = []
+                        parser[self]
+                    ensure
+                        @input = input0
+                        @lookahead = lookahead0
+                        @consume = consume0
+                    end
+                ])
+            ensure
+                @followed -= 1 if block_given?
+            end._and {
+                !block_given? ? Code::True :
+                steps(
+                    buf.slice!(Code[0], index+Code[1]),
+                    buf << lookahead1,
+                    Code::Always.new(
+                        block.arity==1 ? yield(buf) : yield(buf, self)
+                    )
+                )
+            }
+        }
+    ensure
+        @expected = []
+    end
+    
+    def backref(gram, &block) # :yield: n[, engine]
+        raise("TODO")
+    end
+    
+    def backtrack(gram, len=nil, &block) # :yield: n[, engine]
+        raise("TODO")
+    end
+    
+    def pipe(lexer, parser, buf0, max_size=nil, &block) # :yield: buf[, engine]
+        raise("TODO")
+    end
+    
+end
+end
+
+require 'grammar/ruby/code'
+