ebnf 0.1.0 → 0.2.0

data/lib/ebnf/ll1/parser.rb

@@ -20,70 +20,113 @@ module EBNF::LL1
 
     # DSL for creating terminals and productions
     module ClassMethods
+      def start_handlers; @@start_handlers || {}; end
       def production_handlers; @@production_handlers || {}; end
       def terminal_handlers; @@terminal_handlers || {}; end
       def patterns; @@patterns || []; end
-      def unescape_terms; @@unescape_terms || []; end
-
-      ##
-      # Defines a production called during different phases of parsing
-      # with data from previous production along with data defined for the
-      # current production
-      #
-      # @param [Symbol] term
-      #   Term which is a key in the branch table
-      # @yield [parse, phase, input, current]
-      # @yieldparam [Object] parse
-      #   Parser instance
-      # @yieldparam [Symbol] phase
-      #   Phase of parsing, one of :start, or :finish
-      # @yieldparam [Hash] input
-      #   A Hash containing input from the parent production
-      # @yieldparam [Hash] current
-      #   A Hash defined for the current production, during :start
-      #   may be initialized with data to pass to further productions,
-      #   during :finish, it contains data placed by earlier productions
-      # @yieldparam [Prod] block
-      #   Block passed to initialization for yielding to calling parser.
-      #   Should conform to the yield specs for #initialize
-      # Yield to generate a triple
-      def production(term, &block)
-        @@production_handlers ||= {}
-        @@production_handlers[term] = block
-      end
 
       ##
       # Defines the pattern for a terminal node and a block to be invoked
       # when ther terminal is encountered. If the block is missing, the
       # value of the terminal will be placed on the input hash to be returned
-      # to a previous production.
+      # to a previous production. Block is called in an evaluation block from
+      # the enclosing parser.
       #
       # @param [Symbol, String] term
       #   Defines a terminal production, which appears as within a sequence in the branch table
       # @param [Regexp] regexp
       #   Pattern used to scan for this terminal
       # @param [Hash] options
+      # @option options [Hash{String => String}] :map ({})
+      #   A mapping from terminals, in lower-case form, to
+      #   their canonical value
       # @option options [Boolean] :unescape
       #   Cause strings and codepoints to be unescaped.
-      # @yield [parser, term, token, input]
-      # @yieldparam [Object] parser
-      #   Parser instance
+      # @yield [term, token, input, block]
       # @yieldparam [Symbol] term
       #   A symbol indicating the production which referenced this terminal
       # @yieldparam [String] token
       #   The scanned token
       # @yieldparam [Hash] input
       #   A Hash containing input from the parent production
-      # @yieldparam [Prod] block
+      # @yieldparam [Proc] block
       #   Block passed to initialization for yielding to calling parser.
       #   Should conform to the yield specs for #initialize
       def terminal(term, regexp, options = {}, &block)
         @@patterns ||= []
-        @@patterns << [term, regexp]  # Passed in order to define evaulation sequence
+        # Passed in order to define evaulation sequence
+        @@patterns << EBNF::LL1::Lexer::Terminal.new(term, regexp, options)
         @@terminal_handlers ||= {}
         @@terminal_handlers[term] = block if block_given?
-        @@unescape_terms ||= []
-        @@unescape_terms << term if options[:unescape]
+      end
+
+      ##
+      # Defines a production called at the beggining of a particular production
+      # with data from previous production along with data defined for the
+      # current production. Block is called in an evaluation block from
+      # the enclosing parser.
+      #
+      # @param [Symbol] term
+      #   Term which is a key in the branch table
+      # @yield [input, current, block]
+      # @yieldparam [Hash] input
+      #   A Hash containing input from the parent production
+      # @yieldparam [Hash] current
+      #   A Hash defined for the current production, during :start
+      #   may be initialized with data to pass to further productions,
+      #   during :finish, it contains data placed by earlier productions
+      # @yieldparam [Proc] block
+      #   Block passed to initialization for yielding to calling parser.
+      #   Should conform to the yield specs for #initialize
+      # Yield to generate a triple
+      def start_production(term, &block)
+        @@start_handlers ||= {}
+        @@start_handlers[term] = block
+      end
+
+      ##
+      # Defines a production called when production of associated
+      # terminals and non-terminals has completed
+      # with data from previous production along with data defined for the
+      # current production. Block is called in an evaluation block from
+      # the enclosing parser.
+      #
+      # @param [Symbol] term
+      #   Term which is a key in the branch table
+      # @yield [input, current, block]
+      # @yieldparam [Hash] input
+      #   A Hash containing input from the parent production
+      # @yieldparam [Hash] current
+      #   A Hash defined for the current production, during :start
+      #   may be initialized with data to pass to further productions,
+      #   during :finish, it contains data placed by earlier productions
+      # @yieldparam [Proc] block
+      #   Block passed to initialization for yielding to calling parser.
+      #   Should conform to the yield specs for #initialize
+      # Yield to generate a triple
+      def production(term, &block)
+        @@production_handlers ||= {}
+        @@production_handlers[term] = block
+      end
+
+      # Evaluate a handler, delegating to the specified object.
+      # This is necessary so that handlers can operate within the
+      # binding context of the parser in which they're invoked.
+      # @param [Object] object
+      # @return [Object]
+      def eval_with_binding(object)
+        @@delegate = object
+        object.instance_eval {yield}
+      end
+
+      private
+
+      def method_missing(method, *args, &block)
+        if @@delegate ||= nil
+          @@delegate.send method, *args, &block
+        else
+          super
+        end
       end
     end
 
@@ -163,12 +206,12 @@ module EBNF::LL1
       @branch  = options[:branch]
       @first  = options[:first] ||= {}
       @follow  = options[:follow] ||= {}
-      @lexer   = input.is_a?(Lexer) ? input : Lexer.new(input, self.class.patterns, @options.merge(:unescape_terms => self.class.unescape_terms))
+      @lexer   = input.is_a?(Lexer) ? input : Lexer.new(input, self.class.patterns, @options)
       @productions = []
       @parse_callback = block
       @recovering = false
       @error_log = []
-      terminals = self.class.patterns.map(&:first)  # Get defined terminals to help with branching
+      terminals = self.class.patterns.map(&:type)  # Get defined terminals to help with branching
 
       # Unrecoverable errors
       raise Error, "Branch table not defined" unless @branch && @branch.length > 0
@@ -257,13 +300,18 @@ module EBNF::LL1
         # If in recovery mode, continue popping until we find a term with a follow list
         while !pushed &&
               !todo_stack.empty? &&
-              ( todo_stack.last[:terms].to_a.empty? ||
-                (@recovering && @follow[todo_stack.last[:term]].nil?))
-          debug("parse(pop)", :level => 2) {"todo #{todo_stack.last.inspect}, depth #{depth}, recovering? #{@recovering.inspect}"}
-          prod = todo_stack.last[:prod]
-          @recovering = false if @follow[prod] # Stop recovering when we might have a match
-          todo_stack.pop
-          onFinish
+              ( (terms = todo_stack.last.fetch(:terms, [])).empty? ||
+                (@recovering && @follow.fetch(terms.last, []).none? {|t| token == t}))
+          debug("parse(pop)", :level => 2) {"todo #{todo_stack.last.inspect}, depth #{depth}"}
+          if terms.empty?
+            prod = todo_stack.last[:prod]
+            todo_stack.pop
+            onFinish
+          else
+            # Stop recovering when we a production which starts with the term
+            debug("parse(pop)", :level => 2) {"recovery complete"}
+            @recovering = false
+          end
         end
       end
 
@@ -272,7 +320,11 @@ module EBNF::LL1
       # Continue popping contexts off of the stack
       while !todo_stack.empty?
         debug("parse(eof)", :level => 2) {"stack #{todo_stack.last.inspect}, depth #{depth}"}
-        if todo_stack.last[:terms].length > 0
+        # There can't be anything left to do, or if there is, it must be optional
+        last_terms = todo_stack.last[:terms]
+        if last_terms.length > 0 && last_terms.none? {|t|
+          @first.has_key?(t) && @first[t].include?(:_eps)
+        }
           error("parse(eof)",
             "End of input before end of production: stack #{todo_stack.last.inspect}, depth #{depth}"
           )
@@ -289,35 +341,72 @@ module EBNF::LL1
 
     def depth; (@productions || []).length; end
 
+    # Current ProdData element
+    def prod_data; @prod_data.last; end
+
+    # Add a single value to prod_data, allows for values to be an array
+    def add_prod_datum(sym, values)
+      case values
+      when Array
+        prod_data[sym] ||= []
+        debug("add_prod_datum(#{sym})") {"#{prod_data[sym].inspect} += #{values.inspect}"}
+        prod_data[sym] += values
+      when nil
+        return
+      else
+        prod_data[sym] ||= []
+        debug("add_prod_datum(#{sym})") {"#{prod_data[sym].inspect} << #{values.inspect}"}
+        prod_data[sym] << values
+      end
+    end
+    
+    # Add values to production data, values aranged as an array
+    def add_prod_data(sym, *values)
+      return if values.compact.empty?
+      
+      prod_data[sym] ||= []
+      prod_data[sym] += values
+      debug("add_prod_data(#{sym})") {"#{prod_data[sym].inspect} += #{values.inspect}"}
+    end
+    
   private
     # Start for production
     def onStart(prod)
-      handler = self.class.production_handlers[prod]
+      handler = self.class.start_handlers[prod]
       @productions << prod
       if handler
         # Create a new production data element, potentially allowing handler
         # to customize before pushing on the @prod_data stack
         progress("#{prod}(:start):#{@prod_data.length}") {@prod_data.last}
         data = {}
-        handler.call(self, :start, @prod_data.last, data, @parse_callback)
+        self.class.eval_with_binding(self) {
+          handler.call(@prod_data.last, data, @parse_callback)
+        }
         @prod_data << data
       else
+        # Make sure we push as many was we pop, even if there is no
+        # explicit start handler
+        @prod_data << {} if self.class.production_handlers[prod]
         progress("#{prod}(:start)") { get_token.inspect}
       end
-      #puts @prod_data.inspect
+      #puts "prod_data(s): " + @prod_data.inspect
     end
 
     # Finish of production
     def onFinish
+      #puts "prod_data(f): " + @prod_data.inspect
       prod = @productions.last
       handler = self.class.production_handlers[prod]
-      if handler
+      if handler && !@recovering
         # Pop production data element from stack, potentially allowing handler to use it
         data = @prod_data.pop
-        handler.call(self, :finish, @prod_data.last, data, @parse_callback)
+        self.class.eval_with_binding(self) {
+          handler.call(@prod_data.last, data, @parse_callback)
+        }
+        #require 'debugger'; breakpoint
         progress("#{prod}(:finish):#{@prod_data.length}") {@prod_data.last}
       else
-        progress("#{prod}(:finish)", '')
+        progress("#{prod}(:finish)", "recovering: #{@recovering.inspect}")
       end
       @productions.pop
     end
@@ -330,7 +419,9 @@ module EBNF::LL1
         # Allows catch-all for simple string terminals
         handler ||= self.class.terminal_handlers[nil] if prod.is_a?(String)
         if handler
-          handler.call(self, parentProd, token, @prod_data.last)
+          self.class.eval_with_binding(self) {
+            handler.call(parentProd, token, @prod_data.last)
+          }
           progress("#{prod}(:token)", "", :depth => (depth + 1)) {"#{token}: #{@prod_data.last}"}
         else
           progress("#{prod}(:token)", "", :depth => (depth + 1)) {token.to_s}
@@ -380,7 +471,7 @@ module EBNF::LL1
         skipped = @lexer.shift
         progress("recovery") {"skip #{skipped.inspect}"}
       end
-      debug("recovery") {"found #{token.inspect}"}
+      debug("recovery") {"found #{token.inspect} in #{first.include?(token) ? 'first' : 'follows'}"}
       
       # If the token is a first, just return it. Otherwise, it is a follow
       # and we need to skip to the end of the production
@@ -400,7 +491,7 @@ module EBNF::LL1
     def error(node, message, options = {})
       message += ", found #{options[:token].representation.inspect}" if options[:token]
       message += " at line #{@lineno}" if @lineno
-      message += ", production = #{options[:production].inspect}" if options[:production] && @options[:debug]
+      message += ", production = #{options[:production].inspect}" if options[:production]
       @error_log << message unless @recovering
       @recovering = true
       debug(node, message, options.merge(:level => 0))
@@ -446,7 +537,6 @@ module EBNF::LL1
       if @options[:debug]
         return debug(node, message, {:level => 0}.merge(options))
       else
-        str = "[#{@lineno}]#{' ' * depth}#{node}: #{message}"
         $stderr.puts("[#{@lineno}]#{' ' * depth}#{node}: #{message}")
       end
     end

data/lib/ebnf/rule.rb

@@ -186,7 +186,7 @@ module EBNF
     # @param [Symbol, class] sym
     #   Symbol matching any start element, or if it is String, any start element which is a String
     # @return [Array<Symbol, String>] list of symbol (singular), or strings which are start symbol, or nil if there are none
-    def starts_with(sym)
+    def starts_with?(sym)
       if seq? && sym === (v = expr.fetch(1, nil))
         [v]
       elsif alt? && expr.any? {|e| sym === e}
@@ -196,6 +196,12 @@ module EBNF
       end
     end
 
+    # Do the firsts of this rule include the empty string?
+    # @return [Boolean]
+    def first_includes_eps?
+      @first && @first.include?(:_eps)
+    end
+
     # Add terminal as proceding this rule
     # @param [Array<Rule>] terminals
     # @return [Integer] if number of terminals added

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ebnf
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-26 00:00:00.000000000 Z
+date: 2013-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sxp
@@ -27,6 +27,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdf
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -101,6 +117,7 @@ files:
 - etc/doap.ttl
 - etc/ebnf.ebnf
 - etc/ebnf.ll1
+- etc/ebnf.rb
 - etc/turtle.ebnf
 - etc/turtle.ll1
 - etc/turtle.rb
@@ -117,13 +134,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1950185345296250945
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25