rley 0.2.01 → 0.2.02

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NzlkNmE1NzA2OWEzMzVlNTNjOTBkNDI3ZGQ1MDljYTI4M2I3ZmNhNQ==
+    MzY1NzQxNzdkNGEzMDFiYzhhNmQ0M2Q5MTU3NzQxOGMyMDkxYTM4Ng==
   data.tar.gz: !binary |-
-    NGNlMGZhM2UyNjNiYmMwZTBjMGFiNmEwM2U3NWI2MjUxMzFlYmY5YQ==
+    OGUzMzg4YmE5ZGRiZmRiOWEzMTFiYjhiOTJlYmQ2M2FkNzBkOWVjOQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    MjA1YjJmYjRiNjFjZTViNmFhMDk4ZjA3MDVkOWJjNDBhMjExYjhkMDg5MTUy
-    ZTJkNzhjN2I1NjY5M2I2MWY5NzEyZmJkMjJiZTIwYjEyNTk2MzFmN2ViZWU2
-    YTYzYzkzNTUxYWE1MjU3NGI5NTk3YzAzNmI0MGNkNGExYmEyMDE=
+    MzAzY2FkM2VkNzY2NDVhZDEwNmIyYWVkZDg3MDEwYzQ4ZGM3NzMzMmMxZTdk
+    YzYyZDZhMjNmNDQyZDc5ZmRlYzBjYTQ5ZTMyMmQzZTk2YjBiMjI3N2IxOTYw
+    YjZiZGYzZWQxZjUxM2ExMTQxMzExZmIzNTJmZTZjNDhjYzcyOTk=
   data.tar.gz: !binary |-
-    N2U4Y2I5MjA0YWQyMWZmMmU4MDM0N2ZlNGU1NzA5MzViMDAxZGI1YTc5Zjcy
-    M2MzMDFmOWU5ZGI1NjllODliZTkwNjE2MmMyOTQzZjRkODBjNzUwNDg2MDJl
-    NmExN2FiMjA4ZjBhNTdkNDE4YWUxMzgxZTQ1Y2M4ZDE0OTE4NGY=
+    MWQ5MGM3YjVjNjNhY2JjMzE0MjlkMWIxZTE0NDI2NWQyMWU1MDBjMGE0ZTc2
+    MjI1NzU4OGM3NDczNWQ4ZWNjMzIzZGFjNzM1ODE0MzljZTM2MzJjYjY4Mjgz
+    YmM2ZThiODQ0Njg5N2ZiYzYzODNlZjRkNGJlZTgyY2U3YzYyNWE=

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+### 0.2.02 / 2015-02-02
+* [NEW] Examplar file `parsing_L1.rb`: demo using a (highly simplified) English grammar.
+* [NEW] Examplar file `parsing_amb.rb`: demo using an ambiguous grammar.
+* [FIX] Method `Parsing#parse_tree` now produces correct parse trees for all the examples.
+
 ### 0.2.01 / 2015-01-03
 * [CHANGE] File `.rubocop.yml`: AbcMetric setting relaxed.
 * [CHANGE] Fixed most style offenses reported by Rubocop.

data/README.md

@@ -8,9 +8,9 @@ Rley
 [![Gem Version](https://badge.fury.io/rb/rley.svg)](http://badge.fury.io/rb/rley)
 [![Dependency Status](https://gemnasium.com/famished-tiger/Rley.svg)](https://gemnasium.com/famished-tiger/Rley)
 
-### What is Rley? ###
-__Rley__ is a Ruby implementation of a Earley parser.  
-The objective is to build a parser convenient for lightweight NLP (Natural Language Processing) purposes.  
+__Rley__ is a Ruby implementation of an [Earley parser](http://en.wikipedia.org/wiki/Earley_parser).  
+The project aims to build a parser convenient for lightweight NLP (Natural Language Processing) purposes.  
+
 
 Yet another parser?  
 Yes and no. Rley doesn't aim to replace other very good programming language parsers for Ruby.
@@ -19,19 +19,22 @@ in the grammar/language they support.
 The Earley's algorithm being more general is able to parse input without imposing restriction on the context-free grammar.  
 Consult Wikipedia to learn more about Earley's parsing algorithm.  
 
-This project is in "early" stage.  
+This project is in "earley" stage.  
 ####Roadmap:
+- Add examples (including small NLP grammar)
+- Document the parser API
 - Add more validation tests and sample grammars
 - Add AST generation (and semantic actions?)
 - Add DSL for grammar specification
 - Add grammar validations
 - Add error reporting
-- Add examples (including small NLP grammar)
+
 - Add a command-line interface
 - Provide documentation and examples
 
 
+
 Copyright
 ---------
-Copyright (c) 2014, Dimitri Geshef. 
+Copyright (c) 2014-2015, Dimitri Geshef. 
 __Rley__ is released under the MIT License see [LICENSE.txt](https://github.com/famished-tiger/Rley/blob/master/LICENSE.txt) for details.

data/examples/parsers/parsing_L0.rb

@@ -14,7 +14,7 @@ require 'rley'  # Load the gem
 
 ########################################
 # Step 1. Define a grammar for a micro English-like language
-# based on Jurafky & Martin L0 language.
+# based on Jurafky & Martin L0 language (chapter 12 of the book).
 # It defines the syntax of a sentence in a language with a 
 # very limited syntax and lexicon in the context of airline reservation.
 builder = Rley::Syntax::GrammarBuilder.new

data/examples/parsers/parsing_L1.rb

@@ -0,0 +1,136 @@
+# Purpose: to demonstrate how to build and render a parse tree for the L1
+# language
+require 'pp'
+require 'rley'  # Load the gem
+
+# Steps to render a parse tree (of a valid parsed input):
+# 1. Define a grammar
+# 2. Create a tokenizer for the language
+# 3. Create a parser for that grammar
+# 4. Tokenize the input
+# 5. Let the parser process the input
+# 6. Generate a parse tree from the parse result
+# 7. Render the parse tree (in JSON)
+
+########################################
+# Step 1. Define a grammar for a miniature English-like language
+# based on Jurafky & Martin L1 language (chapter 13).
+# It defines the syntax of a sentence in a language with a 
+# very limited syntax and lexicon in the context of airline reservation.
+builder = Rley::Syntax::GrammarBuilder.new
+builder.add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
+builder.add_terminals('Aux', 'Determiner', 'Preposition')
+builder.add_production('Sentence' => 'S')
+builder.add_production('S' => %w[Aux NP VP])
+builder.add_production('S' => %w[NP VP])
+builder.add_production('S' => %w[VP])
+builder.add_production('NP' => 'Pronoun')
+builder.add_production('NP' => 'Proper-Noun')
+builder.add_production('NP' => %w[Determiner Nominal])
+builder.add_production('Nominal' => %w[Nominal Noun])
+builder.add_production('Nominal' => %w[Nominal PP])
+builder.add_production('Nominal' => 'Noun')
+builder.add_production('VP' => 'Verb')
+builder.add_production('VP' => %w[Verb NP])
+builder.add_production('VP' => %w[Verb NP PP])
+builder.add_production('VP' => %w[Verb PP])
+builder.add_production('VP' => %w[VP PP])
+builder.add_production('PP' => %w[Preposition NP])
+
+# And now build the grammar...
+grammar_l1 = builder.grammar
+
+
+########################################
+# 2. Create a tokenizer for the language
+# The tokenizer transforms the input into an array of tokens
+# This is a very simplistic implementation for demo purposes.
+
+# The lexicon is just a Hash with pairs of the form:
+# word =>terminal symbol name
+L1_lexicon = {
+  'does' => 'Aux',
+  'flight' => 'Noun',
+  'trip' => 'Noun',
+  'meal' => 'Noun',
+  'money' => 'Noun',
+  'morning' => 'Noun',
+  'is' => 'Verb',
+  'book' => 'Verb',
+  'prefer' => 'Verb',
+  'like' => 'Verb',
+  'need' => 'Verb',
+  'want' => 'Verb',
+  'fly' => 'Verb',
+  'show' => 'Verb',
+  'me' => 'Pronoun',
+  'I' => 'Pronoun',
+  'she' => 'Pronoun',
+  'you' => 'Pronoun',
+  'it' => 'Pronoun',
+  'Alaska' => 'Proper-Noun',
+  'Baltimore' => 'Proper-Noun',
+  'Chicago' => 'Proper-Noun',
+  'Houston' => 'Proper-Noun',
+  'NWA' => 'Proper-Noun',
+  'United' => 'Proper-Noun',
+  'American' => 'Proper-Noun',
+  'the' => 'Determiner',
+  'a' => 'Determiner',
+  'an' => 'Determiner',
+  'this' => 'Determiner',
+  'these' => 'Determiner',
+  'that' => 'Determiner',
+  'from' => 'Preposition',
+  'to' => 'Preposition',
+  'on' => 'Preposition',
+  'near' => 'Preposition',
+  'through' => 'Preposition'
+}
+
+# Highly simplified tokenizer implementation.
+def tokenizer(aText, aGrammar)
+  tokens = aText.scan(/\S+/).map do |word|
+    term_name = L1_lexicon[word]
+    if term_name.nil?
+      fail StandardError, "Word '#{word}' not found in lexicon"
+    end
+    terminal = aGrammar.name2symbol[term_name]
+    Rley::Parser::Token.new(word, terminal)
+  end
+  
+  return tokens
+end
+
+########################################
+# Step 3. Create a parser for that grammar
+parser = Rley::Parser::EarleyParser.new(grammar_l1)
+
+########################################
+# Step 3. Tokenize the input
+valid_input = 'I want the  flight from Alaska through Chicago to Houston'
+# Another sentence: it is a flight from Chicago
+tokens = tokenizer(valid_input, grammar_l1)
+
+########################################
+# Step 5. Let the parser process the input
+result = parser.parse(tokens)
+puts "Parsing success? #{result.success?}"
+
+
+########################################
+# Step 6. Generate a parse tree from the parse result
+ptree = result.parse_tree
+
+########################################
+# Step 7. Render the parse tree (in JSON)
+# Let's create a parse tree visitor
+visitor = Rley::ParseTreeVisitor.new(ptree)
+
+#Here we create a renderer object...
+renderer = Rley::Formatter::Json.new(STDOUT)
+
+# Now emit the parse tree as JSON on the console output
+puts "JSON rendering of the parse tree for '#{valid_input}' input:"
+renderer.render(visitor)
+# End of file

data/examples/parsers/parsing_abc.rb

@@ -1,5 +1,4 @@
 # Purpose: to demonstrate how to build and render a parse tree
-
 require 'rley'  # Load the gem
 
 # Steps to render a parse tree (of a valid parsed input):
@@ -57,6 +56,7 @@ result = parser.parse(tokens)
 # Step 6. Generate a parse tree from the parse result
 ptree = result.parse_tree
 
+
 ########################################
 # Step 7. Render the parse tree (in JSON)
 # Let's create a parse tree visitor
@@ -68,5 +68,4 @@ renderer = Rley::Formatter::Json.new(STDOUT)
 # Now emit the parse tree as JSON on the console output
 puts "JSON rendering of the parse tree for '#{valid_input}' input:"
 renderer.render(visitor)
-
 # End of file

data/examples/parsers/parsing_ambig.rb

@@ -0,0 +1,85 @@
+# Purpose: to demonstrate how to build and render a parse tree
+
+require 'pp' # TODO remove this dependency
+require 'rley'  # Load the gem
+
+# Steps to render a parse tree (of a valid parsed input):
+# 1. Define a grammar
+# 2. Create a tokenizer for the language
+# 3. Create a parser for that grammar
+# 4. Tokenize the input
+# 5. Let the parser process the input
+# 6. Generate a parse tree from the parse result
+# 7. Render the parse tree (in JSON)
+
+########################################
+# Step 1. Define a grammar for a very simple language
+# Grammar 3: A ambiguous arithmetic expression language
+# (based on example in article on Earley's algorithm in Wikipedia)
+# Let's create the grammar step-by-step with the grammar builder:
+builder = Rley::Syntax::GrammarBuilder.new
+builder.add_terminals('integer', '+', '*')
+builder.add_production('P' => 'S')
+builder.add_production('S' => %w(S + S))
+builder.add_production('S' => %w(S * S))
+builder.add_production('S' => 'L')
+builder.add_production('L' => 'integer')
+
+# And now build the grammar...
+grammar_amb = builder.grammar
+
+
+########################################
+# 2. Create a tokenizer for the language
+# The tokenizer transforms the input into an array of tokens
+def tokenizer(aText, aGrammar)
+  tokens = aText.scan(/\S+/).map do |lexeme|
+    case lexeme
+      when '+', '*'
+        terminal = aGrammar.name2symbol[lexeme]
+      when /^[-+]?\d+$/
+        terminal = aGrammar.name2symbol['integer']
+      else
+        msg = "Unknown input text '#{lexeme}'"
+        fail StandardError, msg
+    end
+    Rley::Parser::Token.new(lexeme, terminal)
+  end
+
+  return tokens
+end
+
+########################################
+# Step 3. Create a parser for that grammar
+parser = Rley::Parser::EarleyParser.new(grammar_amb)
+
+########################################
+# Step 3. Tokenize the input
+valid_input = '2 + 3 * 4'
+tokens = tokenizer(valid_input, grammar_amb)
+
+########################################
+# Step 5. Let the parser process the input
+result = parser.parse(tokens)
+puts "Parsing success? #{result.success?}"
+pp result
+
+
+########################################
+# Step 6. Generate a parse tree from the parse result
+ptree = result.parse_tree
+pp ptree
+=begin
+########################################
+# Step 7. Render the parse tree (in JSON)
+# Let's create a parse tree visitor
+visitor = Rley::ParseTreeVisitor.new(ptree)
+
+#Here we create a renderer object...
+renderer = Rley::Formatter::Json.new(STDOUT)
+
+# Now emit the parse tree as JSON on the console output
+puts "JSON rendering of the parse tree for '#{valid_input}' input:"
+renderer.render(visitor)
+=end
+# End of file

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.2.01'
+  Version = '0.2.02'
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm"

data/lib/rley/formatter/json.rb

@@ -64,10 +64,7 @@ module Rley # This module is used as a namespace
       def before_terminal(term_node)
         separator = sibling_flags[-1] ? ",\n" : "\n"
         name = term_node.symbol.name
-        if term_node.token.nil?
-          msg = "No token associated with #{name}"
-          fail StandardError, msg
-        end
+
         lexeme = term_node.token.lexeme
         print_text(separator, "{\"#{name}\": \"#{lexeme}\"}")
         sibling_flags[-1] = true

data/lib/rley/parser/parse_state_tracker.rb

@@ -45,7 +45,7 @@ module Rley # This module is used as a namespace
       end
       
       # Notification that one begins with the previous state set
-      def to_prev_state_set()
+      def to_prev_state_set()      
         self.state_set_index = state_set_index - 1
       end
     end # class

data/lib/rley/parser/parse_tree_builder.rb

@@ -24,7 +24,7 @@ module Rley # This module is used as a namespace
       def current_node()
         return current_path.last
       end
-      
+
       # Factory method.
       def parse_tree()
         return PTree::ParseTree.new(root)
@@ -59,12 +59,9 @@ module Rley # This module is used as a namespace
 
       # Make the predecessor of current node the
       # new current node.
-      def move_back()   
-        begin
-          if current_path.length == 1
-            msg = 'Cannot move further back'
-            fail StandardError, msg
-          end
+      def move_back()
+        loop do
+          break if current_path.length == 1
           (parent, pos) = current_path[-3, 2]
           current_path.pop(2)
           if pos > 0
@@ -73,7 +70,8 @@ module Rley # This module is used as a namespace
             current_path << new_pos
             current_path << new_curr_node
           end
-        end while pos == 0 && new_curr_node.is_a?(PTree::NonTerminalNode)
+          break if pos > 0 || new_curr_node.is_a?(PTree::TerminalNode)
+        end
       end
 
 
@@ -94,10 +92,10 @@ module Rley # This module is used as a namespace
         lower = low_bound(aRange)
         unless lower.nil?
           current_node.range = lower
-          if curr_node.is_a?(PTree::TerminalNode)
+          if curr_node.is_a?(PTree::TerminalNode) && lower[:low]
             current_node.range = high_bound(lower[:low] + 1)
           end
-        end  
+        end
         upper = high_bound(aRange)
         current_node.range = upper unless upper.nil?
       end
@@ -122,8 +120,9 @@ module Rley # This module is used as a namespace
         curr_node = current_node
 
         if curr_node.symbol != prod.lhs
-          msg = "Current node is a #{curr_node.symbol} instead of #{prod.lhs}"
-          fail StandardError, msg
+          snapshot = root.to_string(0)
+          msg = "Current node is a #{curr_node.symbol} instead of #{prod.lhs}."
+          fail StandardError, msg + "\n" + snapshot
         end
         self.range = aRange
         prod.rhs.each { |symb| add_node(symb, {}) }
@@ -131,7 +130,7 @@ module Rley # This module is used as a namespace
         return if curr_node.children.empty?
         curr_node.children.first.range.assign(low: curr_node.range.low)
         curr_node.children.last.range.assign(high: curr_node.range.high)
-      end      
+      end
 
       # Add the given node as child node of current node
       def add_child(aNode)

data/lib/rley/parser/parsing.rb

@@ -10,7 +10,7 @@ module Rley # This module is used as a namespace
 
       # The sequence of input token to parse
       attr_reader(:tokens)
-      
+
       def initialize(startDottedRule, theTokens)
         @tokens = theTokens.dup
         @chart = Chart.new(startDottedRule, tokens.size)
@@ -25,7 +25,7 @@ module Rley # This module is used as a namespace
         found = end_parse_state
         return !found.nil?
       end
-      
+
       # Factory method. Builds a ParseTree from the parse result.
       # @return [ParseTree]
       # Algorithm:
@@ -34,27 +34,14 @@ module Rley # This module is used as a namespace
       def parse_tree()
         state_tracker = new_state_tracker
         builder = tree_builder(state_tracker.state_set_index)
-        
+
         loop do
-          # Look at the symbol on left of the dot
+          # Retrieve the symbol on left of the dot
           curr_symbol = state_tracker.symbol_on_left
-          
-          case curr_symbol
-            when Syntax::Terminal
-              state_tracker.to_prev_state_set
-              predecessor_state_terminal(curr_symbol, state_tracker, builder)
-
-            when Syntax::NonTerminal
-              completed_state_for(curr_symbol, state_tracker, builder)
-
-            when NilClass # No symbol on the left of dot
-              # Retrieve all parse states that expect the lhs
-              new_states = states_expecting_lhs(state_tracker)
-              break if new_states.empty?
-              
-              select_expecting_state(new_states, state_tracker, builder)
-              break if builder.root == builder.current_node
-          end
+
+          # Place the symbol in the parse tree
+          done = insert_matched_symbol(state_tracker, builder)
+          break if done
         end
 
         return builder.parse_tree
@@ -145,8 +132,40 @@ module Rley # This module is used as a namespace
         return candidate_states.find(&:complete?)
       end
 
+
+      # Insert in a parse tree the symbol on the left of the
+      # current dotted rule.
+      def insert_matched_symbol(aStateTracker, aBuilder)
+        # Retrieve symbol before the dot in active parse state
+        match_symbol = aStateTracker.symbol_on_left
+
+        # Retrieve tree node being processed
+        tree_node = aBuilder.current_node
+
+        done = false
+        case [match_symbol.class, tree_node.class]
+          when [Syntax::Terminal, PTree::TerminalNode]
+            aStateTracker.to_prev_state_set
+            predecessor_state_terminal(match_symbol, aStateTracker, aBuilder)
+
+          when [NilClass, Rley::PTree::TerminalNode],
+            [NilClass, PTree::NonTerminalNode]
+            # Retrieve all parse states that expect the lhs
+            new_states = states_expecting_lhs(aStateTracker, aBuilder)
+            done = true if new_states.empty?
+            # Select an unused parse state
+            aStateTracker.select_state(new_states)
+
+          when [Syntax::NonTerminal, PTree::NonTerminalNode]
+            completed_state_for(match_symbol, aStateTracker, aBuilder)
+        end
+
+        done ||= aBuilder.root == aBuilder.current_node
+        return done
+      end
+
       private
-      
+
       # Factory method. Creates and initializes a ParseStateTracker instance.
       def new_state_tracker()
         instance = ParseStateTracker.new(chart.last_index)
@@ -154,16 +173,15 @@ module Rley # This module is used as a namespace
 
         return instance
       end
-      
-      
+
+
       # A terminal symbol is on the left of dot.
       # Go to the predecessor state for the given terminal
       def predecessor_state_terminal(_a_symb, aStateTracker, aTreeBuilder)
-        aTreeBuilder.current_node.range = { low: aStateTracker.state_set_index }
+        index = aStateTracker.state_set_index
+        aTreeBuilder.current_node.range = { low: index, high: index + 1 }
         link_node_to_token(aTreeBuilder, aStateTracker.state_set_index)
         unless aTreeBuilder.current_node.is_a?(PTree::TerminalNode)
-          pp aTreeBuilder.root
-          pp aTreeBuilder.current_node
           fail StandardError, 'Expected terminal node'
         end
         aTreeBuilder.move_back
@@ -171,40 +189,39 @@ module Rley # This module is used as a namespace
         previous_state = state_set.predecessor_state(aStateTracker.parse_state)
         aStateTracker.parse_state = previous_state
       end
-      
-      
-      # Retrieve a complete state with given symbol as lhs.
+
+
+      # Retrieve a complete state with given terminal symbol as lhs.
       def completed_state_for(a_symb, aTracker, aTreeBuilder)
         new_states = chart[aTracker.state_set_index].states_rewriting(a_symb)
-        aTracker.select_state(new_states) 
+        aTracker.select_state(new_states)
         aTreeBuilder.range = { high: aTracker.state_set_index }
         aTreeBuilder.use_complete_state(aTracker.parse_state)
         link_node_to_token(aTreeBuilder, aTracker.state_set_index - 1)
         aTreeBuilder.move_down
       end
-      
-      
-      def states_expecting_lhs(aStateTracker)
+
+
+      def states_expecting_lhs(aStateTracker, aTreeBuilder)
         lhs = aStateTracker.curr_dotted_item.production.lhs
         new_states = states_expecting(lhs, aStateTracker.state_set_index, true)
+        new_states.reject! { |st| st == aStateTracker.parse_state }
+        # Filter out parse states with incompatible range
+        if new_states.size > 1
+          previous_node = aTreeBuilder.current_path[-3]
+          new_states.select! do |parse_state|
+            parse_state.dotted_rule.production.lhs == previous_node.symbol
+          end
+        end
 
         return new_states
       end
-      
-      def select_expecting_state(theStates, aStateTracker, aTreeBuilder)
-        # Select an unused parse state
-        aStateTracker.select_state(theStates) 
-        
-        aTreeBuilder.range = { low: aStateTracker.state_set_index }
-        aTreeBuilder.move_back
-        aTreeBuilder.range = { low: aStateTracker.parse_state.origin }
-      end
-
 
       # If the current node is a terminal node
       # then link the token to that node
       def link_node_to_token(aTreeBuilder, aStateSetIndex)
         return unless aTreeBuilder.current_node.is_a?(PTree::TerminalNode)
+        return unless aTreeBuilder.current_node.token.nil?
 
         a_node = aTreeBuilder.current_node
         a_node.token = tokens[aStateSetIndex] unless a_node.token