rley 0.4.01 → 0.4.02

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17e935e339154ef6729bc4e4679c9ee9c2e81bba
-  data.tar.gz: 479a0cbdc90d6137780983e4eb5b64026e5e844b
+  metadata.gz: cd7c7f9e941ac7139080960dca0cb2a16665bcbd
+  data.tar.gz: dd4e2a8dae9fd1958427beab6c63b0e9206834e7
 SHA512:
-  metadata.gz: 453d7539813b80a7664d71dbfedc2cda89efd09be1db52515a3f5dd7fd314f61e89b7659b88c48ce08f947f8cd614fbf0cab8c9193c653a92da8cf9b76d5faf0
-  data.tar.gz: c14955b02a089aa48dba40b5d97e42aeed880e16dd846d5451af85dad387e581060c0c742b58f3365f7c0305144a1f48a50a3ad7af8feb9b9f4336faff839d85
+  metadata.gz: 7611109ca8b7d99a090e88de7eea31cc08c22250d790c2befd3b3aaa9e222450ac900d10ab733d0ee72e831da0bf7a7ad07da6f6e33e20724879659a2e45d620
+  data.tar.gz: f81cc80909844e4cbd50ec85ad803b08483e714f78734b234de13b544f891f77759cdb8df4349dea1c68a8de250d44ef860e0a8bf27dd6313da08f0cc3c49b34

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 0.4.02 / 2017-04-09
+* [NEW] Module re-organization for clearer dependencies: Classes `Token` and `TokenRange` are moved to a separate module `Tokens`.
+* [CHANGE] Code, specs, examples and `README.md` adapted to reflect the module re-organization.
+
 ### 0.4.01 / 2016-12-21
 * [NEW] File `appveyor.yml`. Add AppVeyor CI to Github commits. AppVeyor complements Travis by running builds under Windows OS.
   This permits to test the portability across operating systems.

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2016 Dimitri Geshef
+Copyright (c) 2014-2017 Dimitri Geshef
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -16,4 +16,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.

data/README.md

@@ -75,7 +75,7 @@ Installing the latest stable version is simple:
 ## A whirlwind tour of Rley
 The purpose of this section is show how to create a parser for a minimalistic
 English language subset.
-The tour is organized into the following steps:  
+The tour is organized as follows:  
 1. [Defining the language grammar](#defining-the-language-grammar)  
 2. [Creating a lexicon](#creating-a-lexicon)  
 3. [Creating a tokenizer](#creating-a-tokenizer)  
@@ -152,7 +152,7 @@ The subset of English grammar is based on an example from the NLTK book.
           raise StandardError, "Word '#{word}' not found in lexicon"
         end
         terminal = aGrammar.name2symbol[term_name]
-        Rley::Parser::Token.new(word, terminal)
+        Rley::Tokens::Token.new(word, terminal)
       end
 
       return tokens
@@ -283,5 +283,5 @@ Here are a few other ones:
 
 Copyright
 ---------
-Copyright (c) 2014-2016, Dimitri Geshef.  
+Copyright (c) 2014-2017, Dimitri Geshef.  
 __Rley__ is released under the MIT License see [LICENSE.txt](https://github.com/famished-tiger/Rley/blob/master/LICENSE.txt) for details.

data/examples/NLP/mini_en_demo.rb

@@ -64,7 +64,7 @@ def tokenizer(aTextToParse, aGrammar)
       raise StandardError, "Word '#{word}' not found in lexicon"
     end
     terminal = aGrammar.name2symbol[term_name]
-    Rley::Parser::Token.new(word, terminal)
+    Rley::Tokens::Token.new(word, terminal)
   end
   
   return tokens

data/examples/data_formats/JSON/JSON_demo.rb

@@ -29,4 +29,5 @@ end
 
 # Generate a parse forest from the parse result
 pforest = result.parse_forest
+# puts pforest.ambiguous?
 # End of file

data/examples/data_formats/JSON/JSON_lexer.rb

@@ -51,7 +51,7 @@ private
         when '{', '}', '[', ']', ',', ':'
           type_name = @@lexeme2name[curr_ch]
           token_type = name2symbol[type_name]
-          token = Rley::Parser::Token.new(curr_ch, token_type)
+          token = Rley::Tokens::Token.new(curr_ch, token_type)
 
         # LITERALS
         when '"'  # Start string delimiter found
@@ -59,7 +59,7 @@ private
           end_delimiter = scanner.getch()
           raise ScanError.new('No closing quotes (") found') if end_delimiter.nil?
           token_type = name2symbol['JSON_STRING']
-          token = Rley::Parser::Token.new(value, token_type)
+          token = Rley::Tokens::Token.new(value, token_type)
 
         when /[ftn]/  # First letter of keywords
           @scanner.pos = scanner.pos - 1 # Simulate putback
@@ -69,7 +69,7 @@ private
             raise ScanError.new("Invalid keyword: #{invalid_keyw}")
           else
             token_type = name2symbol['KEYWORD']
-            token = Rley::Parser::Token.new(keyw, token_type)
+            token = Rley::Tokens::Token.new(keyw, token_type)
           end
 
 
@@ -77,7 +77,7 @@ private
           @scanner.pos = scanner.pos - 1 # Simulate putback
           value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
           token_type = name2symbol['JSON_NUMBER']
-          token = Rley::Parser::Token.new(value, token_type)
+          token = Rley::Tokens::Token.new(value, token_type)
 
 
         else # Unknown token

data/examples/general/calc/calc_lexer.rb

@@ -51,14 +51,14 @@ private
         when '(', ')', '+', '-', '*', '/'
           type_name = @@lexeme2name[curr_ch]
           token_type = name2symbol[type_name]
-          token = Rley::Parser::Token.new(curr_ch, token_type)
+          token = Rley::Tokens::Token.new(curr_ch, token_type)
 
         # LITERALS
         when /[-0-9]/ # Start character of number literal found
           @scanner.pos = scanner.pos - 1 # Simulate putback
           value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
           token_type = name2symbol['NUMBER']
-          token = Rley::Parser::Token.new(value, token_type)
+          token = Rley::Tokens::Token.new(value, token_type)
 
 
         else # Unknown token

data/lib/rley.rb

@@ -4,7 +4,7 @@
 
 require_relative './rley/constants'
 require_relative './rley/syntax/grammar_builder'
-require_relative './rley/parser/token'
+require_relative './rley/tokens/token'
 require_relative './rley/parser/earley_parser'
 require_relative './rley/parser/gfg_earley_parser'
 require_relative './rley/parse_tree_visitor'

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.4.01'.freeze
+  Version = '0.4.02'.freeze
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm".freeze

data/lib/rley/formatter/debug.rb

@@ -41,7 +41,7 @@ module Rley # This module is used as a namespace
       # the children of a non-terminal node
       # @param _parent [NonTerminalNode]
       # @param _children [Array] array of children nodes
-      def before_children(_parent, _children)
+      def before_subnodes(_parent, _children)
         output_event(__method__, indentation)
         indent
       end
@@ -76,7 +76,7 @@ module Rley # This module is used as a namespace
       # the children of a non-terminal node.
       # @param _parent [NonTerminalNode]
       # @param _children [Array] array of children nodes
-      def after_children(_parent, _children)
+      def after_subnodes(_parent, _children)
         dedent
         output_event(__method__, indentation)
       end

data/lib/rley/formatter/json.rb

@@ -48,8 +48,8 @@ module Rley # This module is used as a namespace
       # Notification of a visit event: the visitor is about to visit
       # the children of a non-terminal node
       # @param _parent [NonTerminalNode]
-      # @param _children [Array] array of children nodes
-      def before_children(_parent, _children)
+      # @param _subnodes [Array] array of children nodes
+      def before_subnodes(_parent, _subnodes)
         print_text('[', nil)
         indent
         sibling_flags.push(false)
@@ -73,8 +73,8 @@ module Rley # This module is used as a namespace
       # Notification of a visit event: the visitor completed the visit of
       # the children of a non-terminal node.
       # @param _parent [NonTerminalNode]
-      # @param _children [Array] array of children nodes
-      def after_children(_parent, _children)
+      # @param _subnodes [Array] array of children nodes
+      def after_subnodes(_parent, _subnodes)
         sibling_flags.pop
         print_text("\n", ']')
         dedent

data/lib/rley/parse_tree_visitor.rb

@@ -50,9 +50,9 @@ module Rley # This module is used as a namespace
     def visit_nonterminal(aNonTerminalNode)
       if @traversal == :post_order
         broadcast(:before_non_terminal, aNonTerminalNode)
-        traverse_children(aNonTerminalNode)
+        traverse_subnodes(aNonTerminalNode)
       else
-        traverse_children(aNonTerminalNode)  
+        traverse_subnodes(aNonTerminalNode)  
         broadcast(:before_non_terminal, aNonTerminalNode)
       end
       broadcast(:after_non_terminal, aNonTerminalNode)
@@ -82,17 +82,17 @@ module Rley # This module is used as a namespace
 
     private
     
-    # Visit event. The visitor is about to visit the children of a non 
+    # Visit event. The visitor is about to visit the subnodes of a non 
     # terminal node.
     # @param aParentNode [NonTeminalNode] the (non-terminal) parent node.
-    def traverse_children(aParentNode)
-      children = aParentNode.children
-      broadcast(:before_children, aParentNode, children)
+    def traverse_subnodes(aParentNode)
+      subnodes = aParentNode.subnodes
+      broadcast(:before_subnodes, aParentNode, subnodes)
       
-      # Let's proceed with the visit of children
-      children.each { |a_node| a_node.accept(self) }
+      # Let's proceed with the visit of subnodes
+      subnodes.each { |a_node| a_node.accept(self) }
       
-      broadcast(:after_children, aParentNode, children)
+      broadcast(:after_subnodes, aParentNode, subnodes)
     end
 
     # Send a notification to all subscribers.

data/lib/rley/parser/base_parser.rb

@@ -1,7 +1,7 @@
 require_relative '../syntax/grammar'
 require_relative 'grm_items_builder' # Use mix-in module
 require_relative 'parse_tracer'
-require_relative 'parsing'
+
 
 module Rley # This module is used as a namespace
   module Parser # This module is used as a namespace

data/lib/rley/parser/gfg_parsing.rb

@@ -2,6 +2,7 @@ require_relative 'gfg_chart'
 require_relative 'error_reason'
 require_relative 'parse_entry_tracker'
 require_relative 'parse_forest_factory'
+require_relative 'parse_tree_factory'
 
 
 module Rley # This module is used as a namespace
@@ -148,6 +149,14 @@ module Rley # This module is used as a namespace
 
         return factory.build_parse_forest
       end
+      
+      # Factory method. Builds a ParseTree from the parse result.
+      # @return [ParseTree]
+      def parse_tree()
+        factory = ParseTreeFactory.new(self)
+
+        return factory.build_parse_tree
+      end      
 
       # Retrieve the very first parse entry added to the chart.
       # This entry corresponds to the start vertex of the GF graph

data/lib/rley/parser/parse_tree_builder.rb

@@ -1,174 +1,224 @@
-require_relative '../ptree/terminal_node'
+require_relative '../syntax/terminal'
+require_relative '../syntax/non_terminal'
+require_relative '../gfg/end_vertex'
+require_relative '../gfg/item_vertex'
+require_relative '../gfg/start_vertex'
 require_relative '../ptree/non_terminal_node'
+require_relative '../ptree/terminal_node'
 require_relative '../ptree/parse_tree'
 
-
 module Rley # This module is used as a namespace
   module Parser # This module is used as a namespace
     # Builder GoF pattern. Builder pattern builds a complex object
     # (say, a parse tree) from simpler objects (terminal and non-terminal
     # nodes) and using a step by step approach.
     class ParseTreeBuilder
-      attr_reader(:root)
-      attr_reader(:current_path)
-
-      def initialize(aStartProduction, aRange)
-        @current_path = []
-        start_symbol = aStartProduction.lhs
-        add_node(start_symbol, aRange)
-        use_production(aStartProduction, aRange)
-        move_down
-      end
+      # The sequence of input tokens
+      attr_reader(:tokens)
 
-      # Return the active node.
-      def current_node()
-        return current_path.last
-      end
+      # Link to tree object
+      attr_reader(:tree)
 
-      # Factory method.
-      def parse_tree()
-        return PTree::ParseTree.new(root)
-      end
+      # Link to current path
+      attr_reader(:curr_path)
 
+      # The last parse entry visited
+      attr_reader(:last_visitee)
 
-      # Given that the current node is also lhs of the production
-      # associated with the complete parse state,
-      # Then add the rhs constituents as child nodes of the current node.
-      # Assumption: current node is lhs of the production association
-      # with the parse state.
-      # @param aCompleteState [ParseState] A complete parse state
-      # (dot is at end of rhs)
-      def use_complete_state(aCompleteState)
-        prod = aCompleteState.dotted_rule.production
-        use_production(prod, low: aCompleteState.origin)
-      end
+      # A hash with pairs of the form: visited parse entry => tree node
+      attr_reader(:entry2node)
 
-      # Given that the current node is a non-terminal
-      # Make its last child node the current node.
-      def move_down()
-        curr_node = current_node
-        unless curr_node.is_a?(PTree::NonTerminalNode)
-          msg = "Current node isn't a non-terminal node #{curr_node.class}"
-          raise StandardError, msg
-        end
-        children = curr_node.children
-        path_increment = [children.size - 1, children.last]
-        @current_path.concat(path_increment)
-      end
 
+      def initialize(theTokens)
+        @tokens = theTokens
+        @curr_path = []
+        @entry2node = {}
+      end
 
-      # Make the predecessor of current node the
-      # new current node.
-      def move_back()
-        loop do
-          break if current_path.length == 1
-          (parent, pos) = current_path[-3, 2]
-          current_path.pop(2)
-          if pos > 0
-            new_pos = pos - 1
-            new_curr_node = parent.children[new_pos]
-            current_path << new_pos
-            current_path << new_curr_node
-          end
-          break if pos > 0 || new_curr_node.is_a?(PTree::TerminalNode)
+      def receive_event(anEvent, anEntry, anIndex)
+        # puts "Event: #{anEvent} #{anEntry} #{anIndex}"
+        if anEntry.dotted_entry?
+          process_item_entry(anEvent, anEntry, anIndex)
+        elsif anEntry.start_entry?
+          process_start_entry(anEvent, anEntry, anIndex)
+        elsif anEntry.end_entry?
+          process_end_entry(anEvent, anEntry, anIndex)
+        else
+          raise NotImplementedError
         end
+
+        @last_visitee = anEntry
       end
 
+      # Return the current_parent node
+      def curr_parent()
+        return curr_path.last
+      end
 
-      # Add a child node to the current node.
-      def add_node(aSymbol, aRange)
-        # Create the node
-        a_node = new_node(aSymbol, aRange)
+      private
 
-        # Add it to the current node
-        add_child(a_node)
+      def process_start_entry(_anEvent, _anEntry, _anIndex)
+        curr_path.pop
       end
 
-      # Set unbound endpoints of current node range
-      # to the given range.
-      def range=(aRange)
-        curr_node = current_node
-        return if curr_node.nil?
-        lower = low_bound(aRange)
-        unless lower.nil?
-          current_node.range = lower
-          if curr_node.is_a?(PTree::TerminalNode) && lower[:low]
-            current_node.range = high_bound(lower[:low] + 1)
-          end
+      def process_end_entry(anEvent, anEntry, anIndex)
+        case anEvent
+          when :visit
+            # create a node with the non-terminal
+            #   with same right extent as curr_entry_set_index
+            # add the new node as first child of current_parent
+            # append the new node to the curr_path
+            range = { low: anEntry.origin, high: anIndex }
+            non_terminal = anEntry.vertex.non_terminal
+            create_non_terminal_node(anEntry, range, non_terminal)
+            @tree = create_tree(curr_parent) unless @last_visitee
+          else
+            raise NotImplementedError
         end
-        upper = high_bound(aRange)
-        current_node.range = upper unless upper.nil?
       end
 
-      private
 
-      def new_node(aSymbol, aRange)
-        case aSymbol
-          when Syntax::Terminal
-            new_node = PTree::TerminalNode.new(aSymbol, aRange)
-          when Syntax::NonTerminal
-            new_node = PTree::NonTerminalNode.new(aSymbol, aRange)
+      def process_item_entry(anEvent, anEntry, anIndex)
+        case anEvent
+          when :visit
+            if anEntry.exit_entry?
+              # Previous entry was an end entry (X. pattern)
+              # Does the previous entry have multiple antecedent?
+              if last_visitee.end_entry? && last_visitee.antecedents.size > 1
+                # Store current path for later backtracking
+                # puts "Store backtrack context #{last_visitee}"
+                # puts "path [#{curr_path.map{|e|e.to_string(0)}.join(', ')}]"
+                entry2path_to_alt[last_visitee] = curr_path.dup
+                curr_parent.refinement = :or
+
+                create_alternative_node(anEntry)
+              end
+            end
+
+            # Does this entry have multiple antecedent?
+            if anEntry.antecedents.size > 1
+              # Store current path for later backtracking
+              # puts "Store backtrack context #{anEntry}"
+              # puts "path [#{curr_path.map{|e|e.to_string(0)}.join(', ')}]"
+              entry2path_to_alt[anEntry] = curr_path.dup
+              # curr_parent.refinement = :or
+
+              create_alternative_node(anEntry)
+            end
+
+            # Retrieve the grammar symbol before the dot (if any)
+            prev_symbol = anEntry.prev_symbol
+            case prev_symbol
+              when Syntax::Terminal
+                # Add node without changing current path
+                create_token_node(anEntry, anIndex)
+
+              when NilClass # Dot at the beginning of production
+                if anEntry.vertex.dotted_item.production.empty?
+                  # Empty rhs => create an epsilon node ...
+                  # ... without changing current path
+                  create_epsilon_node(anEntry, anIndex)
+                end
+                curr_path.pop if curr_parent.kind_of?(SPPF::AlternativeNode)
+            end
+
+          when :backtrack
+            # # Restore path
+            # @curr_path = entry2path_to_alt[anEntry].dup
+            # # puts "Special restore path [#{curr_path.map{|e|e.to_string(0)}.join(', ')}]"
+            # antecedent_index = curr_parent.subnodes.size
+            # # puts "Current parent #{curr_parent.to_string(0)}"
+            # # puts "Antecedent index #{antecedent_index}"
+
+            # create_alternative_node(anEntry)
+
+        when :revisit
+            # # Retrieve the grammar symbol before the dot (if any)
+            # prev_symbol = anEntry.prev_symbol
+            # case prev_symbol
+              # when Syntax::Terminal
+                # # Add node without changing current path
+                # create_token_node(anEntry, anIndex)
+
+              # when NilClass # Dot at the beginning of production
+                # if anEntry.vertex.dotted_item.production.empty?
+                  # # Empty rhs => create an epsilon node ...
+                  # # ... without changing current path
+                  # create_epsilon_node(anEntry, anIndex)
+                # end
+                # curr_path.pop if curr_parent.kind_of?(PTree::AlternativeNode)
+            # end
         end
+      end
 
-        return new_node
+      # Create an empty parse tree
+      def create_tree(aRootNode)
+        return Rley::PTree::ParseTree.new(aRootNode)
       end
 
-      # Add children nodes to current one.
-      # The children correspond to the members of the rhs of the production.
-      def use_production(aProduction, aRange)
-        prod = aProduction
-        curr_node = current_node
 
-        if curr_node.symbol != prod.lhs
-          snapshot = root.to_string(0)
-          msg = "Current node is a #{curr_node.symbol} instead of #{prod.lhs}."
-          raise StandardError, msg + "\n" + snapshot
-        end
-        self.range = aRange
-        prod.rhs.each { |symb| add_node(symb, {}) }
+      # Factory method. Build and return an PTree non-terminal node.
+      def create_non_terminal_node(anEntry, aRange, nonTSymb = nil)
+        non_terminal = nonTSymb.nil? ? anEntry.vertex.non_terminal : nonTSymb
+        new_node = Rley::PTree::NonTerminalNode.new(non_terminal, aRange)
+        entry2node[anEntry] = new_node
+        add_subnode(new_node)
+        # puts "FOREST ADD #{curr_parent.key if curr_parent}/#{new_node.key}"
 
-        return if curr_node.children.empty?
-        curr_node.children.first.range.assign(low: curr_node.range.low)
-        curr_node.children.last.range.assign(high: curr_node.range.high)
+        return new_node
       end
 
-      # Add the given node as child node of current node
-      def add_child(aNode)
-        curr_node = current_node
 
-        if curr_node.nil?
-          self.root = aNode
-        else
-          curr_node.children << aNode
-        end
+      # Add an alternative node to the tree
+      def create_alternative_node(anEntry)
+        vertex = anEntry.vertex
+        range = curr_parent.range
+        alternative = Rley::PTree::AlternativeNode.new(vertex, range)
+        add_subnode(alternative)
+        tree.is_ambiguous = true
+        # puts "FOREST ADD #{alternative.key}"
+
+        return alternative
       end
 
-      # Set the root node of the tree.
-      def root=(aNode)
-        @root = aNode
-        @current_path = [ @root ]
-        root.range = low_bound(0)
+      # create a token node,
+      #   with same origin as token,
+      #   with same right extent = origin + 1
+      # add the new node as first child of current_parent
+      def create_token_node(anEntry, anIndex)
+        token_position = anIndex - 1
+        curr_token = tokens[token_position]
+        new_node = PTree::TerminalNode.new(curr_token, token_position)
+        candidate = add_node_to_tree(new_node)
+        entry2node[anEntry] = candidate
+
+        return candidate
       end
 
 
-      def low_bound(aRange)
-        result = case aRange
-                   when Integer then aRange
-                   when Hash then aRange[:low]
-                   when PTree::TokenRange then aRange.low
-                 end
+      def create_epsilon_node(anEntry, anIndex)
+        new_node = PTree::EpsilonNode.new(anIndex)
+        candidate = add_node_to_tree(new_node)
+        entry2node[anEntry] = candidate
+
+        return candidate
+      end
+
+      # Add the given node if not yet present in parse tree
+      def add_node_to_tree(aNode)
+        new_node = aNode
+        # puts "FOREST ADD #{key_node}"
+        add_subnode(new_node, false)
 
-        return { low: result }
+        return new_node
       end
 
-      def high_bound(aRange)
-        result = case aRange
-                   when Integer then aRange
-                   when Hash then aRange[:high]
-                   when PTree::TokenRange then aRange.high
-                 end
 
-        return { high: result }
+      # Add the given node as sub-node of current parent node
+      # Optionally add the node to the current path
+      def add_subnode(aNode, addToPath = true)
+        curr_parent.add_subnode(aNode) unless curr_path.empty?
+        curr_path << aNode if addToPath
       end
     end # class
   end # module