grammar 0.5 → 0.8

data/lib/grammar/ruby0.rb

@@ -0,0 +1,521 @@
+require 'thread'
+
+class Grammar
+class Ruby0
+
+    def self.compile(gram)
+        lambda { |output, input| gram[new(output, &input)] }
+    end
+    
+    def [](object, clone=false)
+        clone ? object.clone : object
+    rescue
+        object
+    end
+    def steps(*steps)
+        steps[-1]
+    end
+    def or(first) # :yield:
+        first || yield
+    end
+    def and(first) # :yield:
+        first && yield
+    end
+    def not(operand)
+        !operand
+    end
+    def if(condition, yes, &no)
+        condition ? yes[] : no[]
+    end
+    def send(obj, method, *args, &block)
+        obj.__send__(method, *args, &block)
+    end
+    def send_splat(obj, method, *args, &block)
+        args.concat(args.pop)
+        obj.__send__(method, *args, &block)
+    end
+    def <<(item)
+        @output << item
+    end
+    def output # :yield: buf
+        block_given? ? (@output = yield(@output)) : @output
+    end
+    
+    # these are mainly for internal usage
+    attr_reader(:input, :lookahead, :error)
+    
+    class Void
+        def concat(other);self;end
+        def <<(other);self;end
+    end
+    
+    class Error < Exception
+        attr_reader(:consumed)
+        attr_reader(:expected)
+        attr_reader(:list)
+        attr_accessor(:raise)
+        def initialize(raise=true)
+            @list = []
+            @expected = []
+            @consumed = 0
+            @raise = raise
+        end
+        def [](expected, found)
+            @expected << expected if expected
+            if @raise
+                @list << [@expected, found]
+                @expected = []
+                Kernel.raise(self)
+            end
+        end
+        def noraise
+            @raise
+        ensure
+            @raise = false
+        end
+        def concat(other)
+            @expected.concat(other.expected)
+            @list.concat(other.list)
+        end
+        def clear
+            @consumed += 1
+            @raise = true
+            @expected.clear
+            @list.clear
+        end
+        def each # :yield: expected, found
+            @list.each { |err|
+                yield(*err)
+            }
+        end
+        def to_str
+            s = ""
+            first = true
+            @list.each { |err|
+                s.concat(first ? "expected: " : "\nor: ")
+                first = true
+                err[0].each { |e|
+                    s << ?| unless first
+                    s.concat(e.to_s)
+                    first = false
+                }
+                s.concat(", found: ").concat(err[1].to_s)
+                first = false
+            }
+            s
+        end
+        def to_s
+            to_str
+        end
+    end
+    
+    def initialize(output=nil, lookahead=nil, input_buffer=[],
+                   error=self.class::Error.new, followed = 0, hold = false,
+                   &input)
+        @output = output
+        @input = input
+        @lookahead = lookahead || (input && input[])
+        @input_buffer = input_buffer
+        @hold = hold
+        @followed = followed
+        @error = error
+    end
+
+    def fork(buf)
+        self.class.new(buf, @lookahead, @input_buffer,
+                       @error.clone, @followed, @hold, &@input)
+    end
+    
+    def join(engine)
+        @input = engine.input
+        @lookahead = engine.lookahead
+        @error = engine.error
+    end
+    
+    def alternation(gram1) # :yield: engine
+        raise = @error.noraise
+        @followed += 1
+        begin
+            gram1[self]
+        ensure
+            @followed -= 1
+            raise1 = @error.raise
+            @error.raise = raise
+        end or yield(self)
+    ensure
+        @error.raise ||= raise1
+    end
+    
+    def sequence(gram1) # :yield: engine
+        @followed += 1
+        begin
+            gram1[self]
+        ensure
+            @followed -= 1
+        end and begin
+            # TODO: keep going with hold if nothing expected yet
+            @hold ? true : yield(self)
+        end
+    end
+    
+    def positive # :yield: engine
+        hold = @hold
+        begin
+            @hold = true
+            yield(self)
+        ensure
+            @hold = hold
+        end
+    end
+
+    def negative # :yield: engine
+        hold = @hold
+        raise = @error.noraise
+        begin
+            @hold = true
+            not yield(self)
+        ensure
+            @error.raise = raise
+            @hold = hold
+        end or failure("something else")
+    end
+    
+    # atomic parsing
+    
+    def failure(expected=nil)
+        @error[expected, @lookahead]
+    end
+    def consume
+        @hold or (
+            @error.clear
+            @output << @lookahead
+            @lookahead = @input[]
+            true
+        )
+    end
+    def eof
+        !@lookahead or failure("EOF")
+    end
+    def any
+        @lookahead ? consume : failure("ANY")
+    end
+    def match(pattern)
+        (pattern===@lookahead) ? consume : failure(pattern)
+    end
+    def always # :yield: engine
+        yield(self)
+    end
+    
+    # variables and objects
+
+    class Var
+        def initialize(val=nil)
+            @var = val
+        end
+        def []
+            @var
+        end
+        def <<(val)
+            @var = val
+        end
+    end
+    def variables(n=nil, &block) # :yield: *vars
+        vars = (0...(n||block.arity)).map { Var.new }
+        yield(*vars)[self]
+    end
+    
+    def recurse(inner, &outer) # :yield: engine
+        inner0 = inner.to_proc
+        left = false
+        right = false
+        lloop = false
+        followed = @followed
+        consumed = @error.consumed
+        inner << Proc.new {
+            if @error.consumed==consumed
+                @followed!=followed or raise("something must follow left recursion") 
+                left = true
+                lloop ? (@error.raise = true) : failure(nil)
+            elsif @followed==followed
+                right = true
+                true
+            else
+                recurse(inner, &outer)
+            end
+        }
+        yield(self) or return
+        @hold and return(true)
+        while true
+            while right
+                consumed = @error.consumed
+                right = false
+                @error.raise = true
+                yield(self) or raise("should have raised")
+            end
+            left or begin
+                !lloop or raise("left recursion not first")
+                return(true)
+            end
+            lloop = true
+            consumed = @error.consumed
+            begin
+                yield(self) or raise("should have raised")
+            rescue self.class::Error => err
+                err.equal?(@error) && err.consumed==consumed or raise(err)
+                return(true)
+            end
+            @error.consumed!=consumed or raise("infinite loop detected")
+        end
+    ensure
+        inner << inner0
+    end
+
+    def redirect(gram, buf0, &block) # :yield: buf[, engine]
+        buf = buf0.clone
+        output = @output
+        @followed += 1 if block_given?
+        begin
+            @output = buf
+            gram[self]
+        ensure
+            buf = @output
+            @followed -= 1 if block_given?
+            @output = output
+        end or return
+        if !@hold && block_given?
+            block.arity==1 ? yield(buf) : yield(buf, self)
+        end
+        true
+    end
+    
+    def discard(gram) # :yield: engine
+        buf = Void.new
+        output = @output
+        @followed += 1 if block_given?
+        begin
+            @output = buf
+            gram[self]
+        ensure
+            @followed -= 1 if block_given?
+            @output = output
+        end or return
+        if !@hold && block_given?
+            @output << yield(self)
+        end
+        true
+    end
+    
+    def backref(gram, &block) # :yield: n[, engine] 
+        tainted = @output.tainted?
+        size0 = @output.taint.size
+        @followed += 1 if block_given?
+        begin
+            ret = gram[self]
+        ensure
+            @followed -= 1 if block_given?
+            n = @output.size-size0
+            if @hold || !ret
+                @output.slice!(size0, n)
+            end
+        end or return
+        if !@hold && block_given?
+            block.arity==1 ? yield(n) : yield(n, self)
+        end
+        true
+    ensure
+        @output.untaint unless tainted
+    end
+    
+    def backtrack(gram, len=nil) # :yield: n[, engine] 
+        tainted = @output.tainted?
+        return gram[self] if @error.raise
+        len ||= -1
+        hold = @hold
+        lookahead = @lookahead
+        error = @error.clone
+        buffer = @input_buffer.clone
+        size0 = @output.taint.size
+        @followed += 1 if block_given?
+        begin
+            len = -1 if hold
+            @hold = false
+            input = @input
+            len += 1
+            @input = lambda {
+                begin
+                    c = input[]
+                ensure
+                    if (len-=1).zero?
+                        @input = input
+                        buffer = nil
+                    else
+                        buffer << c
+                    end
+                end
+            }
+            ret = gram[self]
+        rescue self.class::Error => err
+            @error = err
+            raise(err) if !buffer
+        ensure
+            @followed -= 1 if block_given?
+            @hold = hold
+            n = @output.size-size0
+            if hold || !ret
+                if buffer
+                    error.concat(@error) if !ret
+                    @lookahead = lookahead
+                    @error = error
+                    index = -1
+                    @input = lambda { buffer[index+=1] || (@input=input)[] }
+                end
+                @output.slice!(size0, n)
+                return ret
+            end
+        end or return
+        if !@hold && block_given?
+            block.arity==1 ? yield(n) : yield(n, self)
+        end
+        true
+    ensure
+        @output.untaint unless tainted
+    end
+    
+    def supply(tokenizer, parser, buf0, &block) # :yield: buf[, engine] 
+        !@hold or raise("can't hold supply")
+        buf = buf0.clone
+        engine = fork(buf)
+        @followed += 1 if block_given?
+        begin
+            index = -1
+            @input = lambda {
+                buf[index+=1] or begin
+                    buf.slice!(0, buf.size)
+                    tokenizer[engine]
+                    buf[index = 0]
+                end
+            }
+            @lookahead = input[]
+            parser[self]
+        ensure
+            @followed -= 1 if block_given?
+            lookahead = @lookahead
+            join(engine)
+        end or return
+        if block_given?
+            buf.slice!(0, index+1)
+            buf << lookahead
+            block.arity==1 ? yield(buf) : yield(buf, self)
+        end
+        true
+    end
+            
+    class PipeBuffer
+        attr_accessor(:max_size)
+        def initialize(buf, mutex, cvar, max_size)
+            @buf = buf
+            @tmp_buf = nil
+            @mutex = mutex
+            @cvar = cvar
+            @max_size = max_size || -1
+        end
+        def _produce(arg)
+            @mutex.lock
+            begin
+                @buf << arg
+                @cvar.signal
+            ensure
+                @cvar.wait(@mutex) if @buf.size==@max_size
+                @mutex.unlock
+            end
+            self
+        end
+        alias_method(:<<, :_produce)
+        def taint
+            @tmp_buf ||= @buf.class.new
+            class << self
+                remove_method(:<<)
+            end
+            super
+        end
+        def untaint
+            class << self
+                alias_method(:<<, :_produce)
+            end
+            @tmp_buf.size.times { |i| self << @tmp_buf[i] }
+            @tmp_buf.slice!(0, @tmp_buf.size)
+            super
+        end
+        def method_missing(meth, *args, &block)
+            @tmp_buf.send(meth, *args, &block)
+        end
+    end
+    
+    def pipe(lexer, parser, buf0, max_size=nil) # :yield: buf[, engine] 
+        !@hold or raise("can't hold pipe")
+        buf = buf0.clone
+        mutex = Mutex.new
+        cvar = ConditionVariable.new
+        pbuf = PipeBuffer.new(buf, mutex, cvar, max_size)
+        engine = fork(pbuf)
+        producer = Thread.new {
+            begin
+                lexer[engine]
+            ensure
+                mutex.lock
+                cvar.signal
+                cvar = nil
+                mutex.unlock
+            end
+        }
+        @followed += 1 if block_given?
+        begin
+            index = 0
+            @input = lambda {
+                mutex.lock
+                begin
+                    buf[index]
+                ensure
+                    begin
+                        if 2*(index+=1)>buf.size
+                            buf.slice!(0, index)
+                            index = 0
+                        end
+                        if cvar
+                            cvar.signal
+                            cvar.wait(mutex) if buf.empty?
+                        end
+                    ensure
+                        mutex.unlock
+                    end
+                end
+            }
+            mutex.lock
+            #mutex.sleep if buf.empty?
+            cvar.wait(mutex) if cvar && buf.empty?
+            mutex.unlock
+            @lookahead = input[]
+            parser[self]
+        ensure
+            @followed -= 1 if block_given?
+            mutex.lock
+            if cvar
+                pbuf.max_size = 1
+                cvar.wait(mutex) unless producer.stop?
+                producer.kill
+            end
+            mutex.unlock
+            lookahead = @lookahead
+            join(engine)
+        end or return
+        if block_given?
+            buf[0, index] = (buf.class.new << lookahead)
+            block.arity==1 ? yield(buf) : yield(buf, self)
+        end
+        true
+    end
+    
+end
+end
+
+