rley 0.0.13 → 0.0.14

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NTdmZDFiY2U1OTE3NDVkOTQ5NDU5OTY4YjAyZWY5MTZhNmE4ZmU5OQ==
+    ODY2NWVkNGUzYzNlZDM0MmNhYWIyNGFkYjA2ZGU1NGYyZmNmZTkxMA==
   data.tar.gz: !binary |-
-    ZDdjZTA5ZjEwMGNhM2ZlZjIyYjg1ZDliM2E5NjNmZWIzYmU0YjRmZA==
+    N2RlYjRlNDc5OWIxZWU1YmIwMzA4MWY0MDBhYmFjYzJlZWIyZTVlNg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    YmNlNWU1Y2NkOGE1NmI4Y2YyYzkzMzRiYjY1NGNhNDNiMDQyMzMzNTI3ZWY2
-    ZjdjZGRlZjY4ZTI2NDdkNTViYTA5ZDUxOGE3YTY3NzZkMDQ5YjFjNzlmMTU1
-    YmRlMGVjNmU3ODMzN2MxMzRlYTU0YTVhMzhiYWY4NzcwZmQyYTQ=
+    NTBkMTExZWZiZGYxMWZjYTQ0MmUwMDA1ZDE4ZTBlNzYxMDhmZmRkOTdiMWNk
+    MzkwNWE0MmMzYjQzZGM5MjBiYjA3NTc3MmVkZDNmNmMzNTkxZGJiYjgxNDVl
+    ZGY1N2QwYTY3NzgxNDJjYzc4ZjU1YWRkMmYwNmVmN2JkZGE0OGY=
   data.tar.gz: !binary |-
-    MzA2ODE1MWZjN2I3ZDAxNTAzZmQ4YzNhYzJlZGU4NmU0OGEzYjI0MzEzZjBh
-    ZmI4NzMyYzEyMDlkZDcwNDRkMzk1YzM2NTc4YjhmNzUzZDEwYmRmNDQxMmIw
-    OTI4N2NmZmVjNzM1OGVhNDY4MzFmMWJhNWE3NWZjZDM3Y2QxZjQ=
+    Zjc0YzVlNGU1N2YzNmQ3ZDRlM2VmYmYyNTJhZTQxNDdkZDBkZWYyYWVlZjcy
+    NGQwODhmZTcwYzQ1YmY2Y2EzOWEyMmRlMDFlMGUyZjM3NjA4MmU0OGU1YWJj
+    MjUwYmNjOWQ3NTAxYmY4ZjNiNWVlNWQ4ZmNjOThkMmQ0NGFkZDA=

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+### 0.0.14 / 2014-11-20
+* [NEW]  `EarleyParser` now supports grammar with empty productions (i.e. nullable nonterminals).
+* [CHANGE]  (private) method `EarleyParser#prediction` updated with Ayock-Horspool improvement.
+* [CHANGE]  Moved class `DottedItem` under the `Parser` module.
+
 ### 0.0.13 / 2014-11-19
 * [NEW]  (private) method `Grammar#compute_nullable` added.
 * [CHANGE] `Grammar#initialize` constructor calls the method `Grammar#compute_nullable`

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.0.13'
+  Version = '0.0.14'
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm"

data/lib/rley/parser/dotted_item.rb

@@ -13,71 +13,73 @@ module Rley # This module is used as a namespace
   # An item with the dot at the end (i.e. after all rhs symbols)
   #  is called a reduce item.
   # An item with a dot in front of a terminal is called a shift item.
-  class DottedItem
-    # Production rule
-    attr_reader(:production)
+  module Parser   # This module is used as a namespace
+    class DottedItem
+      # Production rule
+      attr_reader(:production)
 
-    # Index of the next symbol (from the rhs) after the 'dot'.
-    # If the dot is at the end of the rhs (i.e.) there is no next
-    # symbol, then the position takes the value -1.
-    # It the rhs is empty, then the postion is -2
-    attr_reader(:position)
+      # Index of the next symbol (from the rhs) after the 'dot'.
+      # If the dot is at the end of the rhs (i.e.) there is no next
+      # symbol, then the position takes the value -1.
+      # It the rhs is empty, then the postion is -2
+      attr_reader(:position)
 
-    # @param aProduction
-    def initialize(aProduction, aPosition)
-      @production = aProduction
-      @position = valid_position(aPosition)
-    end
-    
-    # Return true if the dot position is at the start of the rhs.
-    def at_start?()
-      return position == 0 || position == -2
-    end
+      # @param aProduction
+      def initialize(aProduction, aPosition)
+        @production = aProduction
+        @position = valid_position(aPosition)
+      end
+      
+      # Return true if the dot position is at the start of the rhs.
+      def at_start?()
+        return position == 0 || position == -2
+      end
 
-    # An item with the dot at the beginning is called 
-    # predicted item  
-    alias_method :predicted_item?, :at_start?
+      # An item with the dot at the beginning is called 
+      # predicted item  
+      alias_method :predicted_item?, :at_start?
 
-    # A dotted item is called a reduce item if the dot is at the end.
-    def reduce_item?()
-      return position < 0 # Either -1 or -2
-    end
-    
-    # The non-terminal symbol that is on the left-side of the production
-    def lhs()
-      return production.lhs
-    end
-    
-    # Return the symbol after the dot.
-    # nil is returned if the dot is at the end
-    def next_symbol()
-      return (position < 0) ? nil : production.rhs[position]
-    end
+      # A dotted item is called a reduce item if the dot is at the end.
+      def reduce_item?()
+        return position < 0 # Either -1 or -2
+      end
+      
+      # The non-terminal symbol that is on the left-side of the production
+      def lhs()
+        return production.lhs
+      end
+      
+      # Return the symbol after the dot.
+      # nil is returned if the dot is at the end
+      def next_symbol()
+        return (position < 0) ? nil : production.rhs[position]
+      end
 
-    # An item with the dot in front of a terminal is called a shift item
-    def shift_item?()
-    end
+      # An item with the dot in front of a terminal is called a shift item
+      def shift_item?()
+      end
 
-    private
+      private
 
-    # Return the given after its validation.
-    def valid_position(aPosition)
-      rhs_size = production.rhs.size
-      if aPosition < 0 || aPosition > rhs_size
-        fail StandardError, 'Out of bound index'
-      end
+      # Return the given after its validation.
+      def valid_position(aPosition)
+        rhs_size = production.rhs.size
+        if aPosition < 0 || aPosition > rhs_size
+          fail StandardError, 'Out of bound index'
+        end
 
-      if rhs_size == 0
-        index = -2 # Minus 2 at start/end of empty production
-      elsif aPosition == rhs_size
-        index = -1  # Minus 1 at end of non-empty production
-      else
-        index = aPosition
-      end
+        if rhs_size == 0
+          index = -2 # Minus 2 at start/end of empty production
+        elsif aPosition == rhs_size
+          index = -1  # Minus 1 at end of non-empty production
+        else
+          index = aPosition
+        end
 
-      return index
-    end
-  end # class
+        return index
+      end
+    end # class
+  end # module
 end # module
 
 # End of file

data/lib/rley/parser/earley_parser.rb

@@ -63,7 +63,7 @@ module Rley # This module is used as a namespace
         aGrammar.rules.each do |prod|
           rhs_size = prod.rhs.size
           if rhs_size == 0
-            items << DottemItem.new(prod, 0)
+            items << DottedItem.new(prod, 0)
           else
             items += (0..rhs_size).map { |i| DottedItem.new(prod, i) }
           end

data/lib/rley/syntax/grammar_builder.rb

@@ -48,6 +48,8 @@ module Rley # This module is used as a namespace
               rhs_constituents = rhs_repr.map { |name| get_nonterminal(name) }
             when String
               rhs_constituents = [ get_nonterminal(rhs_repr) ]
+            when Terminal
+              rhs_constituents = [ rhs_repr ]
           end
           new_prod = Production.new(lhs, rhs_constituents)
           productions << new_prod

data/spec/rley/parser/earley_parser_spec.rb

@@ -5,6 +5,7 @@ require_relative '../../../lib/rley/syntax/non_terminal'
 require_relative '../../../lib/rley/syntax/production'
 require_relative '../../../lib/rley/syntax/grammar_builder'
 require_relative '../../../lib/rley/parser/token'
+require_relative '../../../lib/rley/parser/dotted_item'
 # Load the class under test
 require_relative '../../../lib/rley/parser/earley_parser'
 
@@ -373,7 +374,10 @@ module Rley # Open this namespace to avoid module qualifier prefixes
             Token.new('*', t_star),
             Token.new('4', t_int)
           ]
-          parse_result = subject.parse(tokens)
+          instance = EarleyParser.new(builder.grammar)
+          expect { instance.parse(tokens) }.not_to raise_error
+          parse_result = instance.parse(tokens)
+          expect(parse_result.success?).to eq(true)
         end
 
 
@@ -432,6 +436,149 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           state_set_3 = parse_result.chart[3]
           expect(state_set_3.states).to be_empty  # This is an error symptom
         end
+        
+        it 'should parse a grammar with nullable nonterminals' do
+          # Grammar 4: A grammar with nullable nonterminal
+          # based on example in "Parsing Techniques" book (D. Grune, C. Jabobs)
+          # Z ::= E.
+          # E ::= E Q F.
+          # E ::= F.
+          # F ::= a.
+          # Q ::= *.
+          # Q ::= /.
+          # Q ::=. 
+          t_a = Syntax::VerbatimSymbol.new('a')
+          t_star = Syntax::VerbatimSymbol.new('*')
+          t_slash = Syntax::VerbatimSymbol.new('/')
+          
+          builder = Syntax::GrammarBuilder.new
+          builder.add_terminals(t_a, t_star, t_slash)
+          builder.add_production('Z' => 'E')
+          builder.add_production('E' => %w(E Q F))
+          builder.add_production('E' => 'F')
+          builder.add_production('F' => t_a)
+          builder.add_production('Q' => t_star)
+          builder.add_production('Q' => t_slash) 
+          builder.add_production('Q' => []) # Empty production    
+          tokens = [
+            Token.new('a', t_a),
+            Token.new('a', t_a),
+            Token.new('/', t_slash),
+            Token.new('a', t_a)
+          ]
+          prod_Z = builder.productions[0]
+          prod_E1 = builder.productions[1]
+          prod_E2 = builder.productions[2]
+          prod_F = builder.productions[3]
+          prod_Q1 = builder.productions[4]
+          prod_Q2 = builder.productions[5]
+          prod_Q3 = builder.productions[6]
+          
+          instance = EarleyParser.new(builder.grammar)
+          expect { instance.parse(tokens) }.not_to raise_error
+          parse_result = instance.parse(tokens)
+          expect(parse_result.success?).to eq(true)
+          
+          ###################### S(0) == . a a / a
+          # Expectation chart[0]:
+          # (1) S -> . E, 0 # start rule
+          # (2) E -> . E Q F, 0 # predict from (1)
+          # (3) E -> . F, 0 # predict from (1)
+          # (4) F -> . a    # predict from (3)
+          expectations = [
+            { origin: 0, production: prod_Z, dot: 0 },
+            { origin: 0, production: prod_E1, dot: 0 },
+            { origin: 0, production: prod_E2, dot: 0 },
+            { origin: 0, production: prod_F, dot: 0 }
+          ]
+          compare_state_set(parse_result.chart[0], expectations)
+
+          ###################### S(1) == a . a / a
+          # Expectation chart[1]:
+          # (1) F -> a ., 0 # scan from S(0) 4
+          # (2) E -> F ., 0 # complete from (1) and S(0) 3 
+          # (3) S -> E ., 0 # complete from (2) and S(0) 1
+          # (4) E -> E . Q F, 0  # complete from (2) and S(0) 2
+          # (5) Q -> . *, 1  # Predict from (4)
+          # (6) Q -> . /, 1  # Predict from (4)
+          # (7) Q -> ., 1  # Predict from (4)
+          # (8) E -> E Q . F, 0  # Modified predict from (4)
+          # (9) F -> . a, 1 # Predict from (8)
+          expectations = [
+            { origin: 0, production: prod_F, dot: -1 },
+            { origin: 0, production: prod_E2, dot: -1 },
+            { origin: 0, production: prod_Z, dot: -1 },
+            { origin: 0, production: prod_E1, dot: 1 },
+            { origin: 1, production: prod_Q1, dot: 0 },
+            { origin: 1, production: prod_Q2, dot: 0 },
+            { origin: 1, production: prod_Q3, dot: -2 },
+            { origin: 0, production: prod_E1, dot: 2 },
+            { origin: 1, production: prod_F, dot: 0 }
+          ]
+          compare_state_set(parse_result.chart[1], expectations)
+
+          ###################### S(2) == a a . / a
+          # Expectation chart[2]:
+          # (1) F -> a ., 1 # scan from S(1) 9
+          # (2) E -> E Q F ., 0  # complete from (1) and S(1) 8
+          # (3) S -> E ., 0 # complete from (1) and S(0) 1
+          # (4) E -> E . Q F, 0  # complete from (1) and S(0) 2
+          # (5) Q -> . *, 2  # Predict from (4)
+          # (6) Q -> . /, 2  # Predict from (4)
+          # (7) Q -> ., 2  # Predict from (4)
+          # (8) E -> E Q . F, 0  # Complete from (5) and S(1) 4
+          # (9) F -> . a, 1 # Predict from (8)
+          expectations = [
+            { origin: 1, production: prod_F, dot: -1 },
+            { origin: 0, production: prod_E1, dot: -1 },
+            { origin: 0, production: prod_Z, dot: -1 },
+            { origin: 0, production: prod_E1, dot: 1 },
+            { origin: 2, production: prod_Q1, dot: 0 },
+            { origin: 2, production: prod_Q2, dot: 0 },
+            { origin: 2, production: prod_Q3, dot: -2 },
+            { origin: 0, production: prod_E1, dot: 2 },
+            { origin: 2, production: prod_F, dot: 0 },
+          ]
+          compare_state_set(parse_result.chart[2], expectations)
+
+          ###################### S(3) == a a / . a
+          # Expectation chart[3]:
+          # (1) Q -> / ., 2 # scan from S(2) 6
+          # (2) E -> E Q . F, 0  # complete from (1) and S(1) 4
+          # (3) F -> . a, 3 # Predict from (2)
+          expectations = [
+            { origin: 2, production: prod_Q2, dot: -1 },
+            { origin: 0, production: prod_E1, dot: 2 },
+            { origin: 3, production: prod_F, dot: 0 }
+          ]
+          compare_state_set(parse_result.chart[3], expectations)
+
+          
+          ###################### S(4) == a a / a .
+          # Expectation chart[4]:
+          # (1) F -> a ., 3 # scan from S(3) 3
+          # (2) E -> E Q F ., 0  # complete from (1) and S(3) 2
+          # (3) S -> E ., 0 # complete from (2) and S(0) 1
+          # (4) E -> E . Q F, 0  # complete from (2) and S(0) 2
+          # (5) Q -> . *, 3  # Predict from (4)
+          # (6) Q -> . /, 3  # Predict from (4)
+          # (7) Q -> ., 3  # Predict from (4)
+          # (8) E -> E Q . F, 0  # Modified predict from (4)
+          # (9) F -> . a, 4 # Predict from (8)
+          expectations = [
+            { origin: 3, production: prod_F, dot: -1 },
+            { origin: 0, production: prod_E1, dot: -1 },
+            { origin: 0, production: prod_Z, dot: -1 },
+            { origin: 0, production: prod_E1, dot: 1 },
+            { origin: 4, production: prod_Q1, dot: 0 },
+            { origin: 4, production: prod_Q2, dot: 0 },
+            { origin: 4, production: prod_Q3, dot: -2 },
+            { origin: 0, production: prod_E1, dot: 2 },
+            { origin: 4, production: prod_F, dot: 0 },
+          ]
+          compare_state_set(parse_result.chart[4], expectations)
+          
+        end
       end # context
 
     end # describe

data/spec/rley/syntax/grammar_builder_spec.rb

@@ -11,7 +11,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
         it 'should be created without argument' do
           expect { GrammarBuilder.new }.not_to raise_error
         end
-        
+
         it 'should have no grammar symbols at start' do
             expect(subject.symbols).to be_empty
         end
@@ -58,14 +58,14 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           expect(subject.symbols['c']).to eq(c)
         end
       end # context
-      
+
       context 'Adding productions:' do
         subject do
           instance = GrammarBuilder.new
           instance.add_terminals('a', 'b', 'c')
           instance
         end
-        
+
         it 'should add a valid production' do
           # case of a rhs representation that consists of one name
           expect { subject.add_production('S' => 'A') }.not_to raise_error
@@ -73,22 +73,22 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           new_prod = subject.productions[0]
           expect(new_prod.lhs).to eq(subject['S'])
           expect(new_prod.rhs[0]).to eq(subject['A'])
-          
+
           subject.add_production('A' => %w(a A c))
           expect(subject.productions.size).to eq(2)
           new_prod = subject.productions.last
           expect(new_prod.lhs).to eq(subject['A'])
           expect_rhs = [ subject['a'], subject['A'], subject['c'] ]
           expect(new_prod.rhs.members).to eq(expect_rhs)
-          
+
           subject.add_production('A' => ['b'])
           expect(subject.productions.size).to eq(3)
           new_prod = subject.productions.last
           expect(new_prod.lhs).to eq(subject['A'])
           expect(new_prod.rhs[0]).to eq(subject['b'])
-        end 
+        end
       end # context
-      
+
       context 'Building grammar:' do
         subject do
           instance = GrammarBuilder.new
@@ -98,20 +98,20 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           instance.add_production('A' => ['b'])
           instance
         end
-        
+
         it 'should build a grammar' do
           expect(subject.grammar).to be_kind_of(Grammar)
           grm = subject.grammar
           expect(grm.rules).to eq(subject.productions)
         end
-        
+
         it 'should complain in absence of symbols' do
           instance = GrammarBuilder.new
           err = StandardError
           msg = 'No symbol found for grammar'
           expect { instance.grammar }.to raise_error(err, msg)
         end
-        
+
         it 'should complain in absence of productions' do
           instance = GrammarBuilder.new
           instance.add_terminals('a', 'b', 'c')
@@ -119,7 +119,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           msg = 'No production found for grammar'
           expect { instance.grammar }.to raise_error(err, msg)
         end
-        
+
         it 'should complain when non-terminal has no production' do
           instance = GrammarBuilder.new
           instance.add_terminals('a', 'b', 'c')
@@ -128,6 +128,34 @@ module Rley # Open this namespace to avoid module qualifier prefixes
           msg = 'Nonterminal A not rewritten'
           expect { instance.grammar }.to raise_error(err, msg)
         end
+
+        it 'should build a grammar with nullable nonterminals' do
+          # Grammar 4: A grammar with nullable nonterminal
+          # based on example in "Parsing Techniques" book (D. Grune, C. Jabobs)
+          # S ::= E.
+          # E ::= E Q F.
+          # E ::= F.
+          # F ::= a.
+          # Q ::= *.
+          # Q ::= /.
+          # Q ::=.
+          t_a = VerbatimSymbol.new('a')
+          t_star = VerbatimSymbol.new('*')
+          t_slash = VerbatimSymbol.new('/')
+
+          builder = GrammarBuilder.new
+          builder.add_terminals(t_a, t_star, t_slash)
+          builder.add_production('S' => 'E')
+          builder.add_production('E' => %w(E Q F))
+          builder.add_production('E' => 'F')
+          builder.add_production('F' => t_a)
+          builder.add_production('Q' => t_star)
+          builder.add_production('Q' => t_slash)
+          builder.add_production('Q' => []) # Empty production
+
+          expect { builder.grammar }.not_to raise_error
+          expect(builder.productions.last).to be_empty
+        end
       end
 
     end # describe

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-19 00:00:00.000000000 Z
+date: 2014-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake