rley 0.5.10 → 0.5.11

data/examples/general/SRL/spec/tokenizer_spec.rb

@@ -62,6 +62,18 @@ module SRL
       end       
     end # context
     
+    context 'String literal tokenization:' do
+      it "should recognize 'literally ...'" do
+        input = 'literally "hello"'
+        subject.scanner.string = input
+        expectations = [
+          ['LITERALLY', 'literally'],
+          ['STRING_LIT', 'hello']
+        ]
+        match_expectations(subject, expectations)
+      end    
+    end # context    
+    
     context 'Character range tokenization:' do
       it "should recognize 'letter from ... to ...'" do
         input = 'letter a to f'

data/examples/general/left.rb

@@ -0,0 +1,36 @@
+# Purpose: define a grammar with left-recursive rule
+require 'rley' # Load Rley library
+
+# Instantiate a builder object that will build the grammar for us
+builder = Rley::Syntax::GrammarBuilder.new do
+  add_terminals('DOT')
+  
+  # Grammar with left recursive rule.
+	rule 'l_dots' => []
+	rule 'l_dots' => %w[l_dots DOT]  
+end
+
+# And now, let's build the grammar...
+grammar = builder.grammar
+
+# Highly simplified tokenizer implementation.
+def tokenizer(aText, aGrammar)
+	tokens = aText.scan(/\./).map do |dot|
+	  terminal = aGrammar.name2symbol['DOT']
+	  Rley::Lexical::Token.new(dot, terminal)
+	end
+
+	return tokens
+end
+
+input_to_parse = '.' * 500 # Input = 500 consecutive dots
+
+parser = Rley::Parser::GFGEarleyParser.new(grammar)
+tokens = tokenizer(input_to_parse, grammar)
+result = parser.parse(tokens)
+
+puts "Parsing successful? #{result.success?}"
+unless result.success?
+  puts result.failure_reason.message
+  exit(1)
+end

data/examples/general/right.rb

@@ -0,0 +1,36 @@
+# Purpose: define a grammar with right-recursive rule
+require 'rley' # Load Rley library
+
+# Instantiate a builder object that will build the grammar for us
+builder = Rley::Syntax::GrammarBuilder.new do
+  add_terminals('DOT')
+  
+  # Grammar with left recursive rule.
+	rule 'r_dots' => []
+	rule 'r_dots' => %w[DOT r_dots]  
+end
+
+# And now, let's build the grammar...
+grammar = builder.grammar
+
+# Highly simplified tokenizer implementation.
+def tokenizer(aText, aGrammar)
+	tokens = aText.scan(/\./).map do |dot|
+	  terminal = aGrammar.name2symbol['DOT']
+	  Rley::Lexical::Token.new(dot, terminal)
+	end
+
+	return tokens
+end
+
+input_to_parse = '.' * 500 # Input = 500 consecutive dots
+
+parser = Rley::Parser::GFGEarleyParser.new(grammar)
+tokens = tokenizer(input_to_parse, grammar)
+result = parser.parse(tokens) # Takes about 20 seconds on my computer!!!!
+
+puts "Parsing successful? #{result.success?}"
+unless result.success?
+  puts result.failure_reason.message
+  exit(1)
+end

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.5.10'.freeze
+  Version = '0.5.11'.freeze
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm".freeze

data/lib/rley/gfg/edge.rb

@@ -4,17 +4,28 @@ module Rley # This module is used as a namespace
     # Responsibilities:
     # - To know the successor vertex
     class Edge
-      # The destination vertex of the edge .
+      # @return [Vertex] The destination vertex of the edge .
       attr_reader :successor
 
+      # Construct a directed edge between two given vertices
+      # @param [Vertex]
+      # @param [Vertex]
       def initialize(thePredecessor, theSuccessor)
         @successor = theSuccessor
         thePredecessor.add_edge(self)
       end
 
+      # @return [String]
       def to_s()
         " --> #{successor.label}"
       end
+      
+      # Returns a string containing a human-readable representation of the 
+      # production.
+      # @return [String]
+      def inspect()
+        to_s
+      end
     end # class
   end # module
 end # module

data/lib/rley/gfg/grm_flow_graph.rb

@@ -35,10 +35,11 @@ module Rley # This module is used as a namespace
     # A GFG has three types of directed edges linking the vertices.
     # call edge, return edge and scan edge.
     class GrmFlowGraph
-      # The set of all vertices in the graph
+      # @return [Array<Vertex>] The set of all vertices in the graph
       attr_reader :vertices
 
       # The vertex marked as start node of the graph
+      # @return [StartVertex>]
       attr_reader :start_vertex
 
       # A Hash with pairs of the form: non-terminal symbol => start node
@@ -57,6 +58,25 @@ module Rley # This module is used as a namespace
 
         build_graph(theDottedItems)
       end
+      
+      # Returns a string containing a human-readable representation of the 
+      # production.
+      # @return [String]
+      def inspect()
+        result = "#<#{self.class.name}:#{self.object_id}"
+        result << ' @vertices=['
+        list = vertices.map { |v| "#<#{v.selfie}>" }
+        result << list.join(', ')
+        result << '] '
+        edges = []
+        vertices.each do |v| 
+          edges << v.edges do |e|
+            result << "#{v.object_id} #{e.inspect}"
+          end
+        end
+        result << "edges=[#{edges.join(",\n ")}]>"
+        return result
+      end
 
       # Retrieve the vertex with given vertex label.
       # @param aVertexLabel [String] the label of a vertex from the graph

data/lib/rley/gfg/item_vertex.rb

@@ -74,7 +74,7 @@ module Rley # This module is used as a namespace
       # @return [Syntax::GrmSymbol] The non-terminal symbol at left side of production.
       def lhs()
         return dotted_item.lhs
-      end
+      end 
     end # class
   end # module
 end # module

data/lib/rley/gfg/non_terminal_vertex.rb

@@ -26,7 +26,7 @@ module Rley # This module is used as a namespace
       # A start vertex may accept an indegree and outdegree greater than one
       def check_add_edge(anEdge)
         return anEdge
-      end      
+      end
     end # class
   end # module
 end # module

data/lib/rley/gfg/start_vertex.rb

@@ -12,6 +12,7 @@ module Rley # This module is used as a namespace
         super(aNonTerminal)
       end
 
+      # @return [String]
       def label()
         return ".#{non_terminal}"
       end       

data/lib/rley/gfg/vertex.rb

@@ -28,6 +28,28 @@ module Rley # This module is used as a namespace
         return false # Default implementation
       end
       
+      # Returns a string containing a human-readable representation of the 
+      # vertex.
+      # @return [String]
+      def inspect()
+        result = '#<'
+        result << selfie
+        edges.each { |e| result << e.inspect }
+        result << specific_inspect()
+        result << '>'
+        
+        return result
+      end
+      
+      # Returns a string containing a human-readable representation of the 
+      # vertex without the edges.
+      # @return [String]      
+      def selfie()
+        result = "#{self.class.name}:#{self.object_id}"
+        result << %Q[ label="#{self.label}"]
+        return result
+      end
+      
       # Retrieve the grammar symbol before the dot.
       # @return [GrmSymbol, NilClass] The symbol or otherwise nil.
       def prev_symbol()
@@ -49,6 +71,11 @@ module Rley # This module is used as a namespace
         raise StandardError, 'At most one edge accepted' unless edges.empty?
         return anEdge
       end
+      
+      def specific_inspect()
+        return ''
+      end      
+      
     end # class
   end # module
 end # module

data/lib/rley/lexical/token.rb

@@ -22,6 +22,7 @@ module Rley # This module is used as a namespace
       # @param theLexeme [String] the lexeme (= piece of text from input)
       # @param aTerminal [Syntax::Terminal] The terminal symbol corresponding to the lexeme.
       def initialize(theLexeme, aTerminal)
+        raise 'Internal error: nil terminal symbol detected' if aTerminal.nil?
         @lexeme = theLexeme
         @terminal = aTerminal
       end

data/lib/rley/parser/error_reason.rb

@@ -16,7 +16,7 @@ module Rley # Module used as a namespace
 
       # Returns the result of invoking reason.to_s.
       def message()
-        return to_s
+        return self.to_s
       end
 
       # Return this reason's class name and message
@@ -51,6 +51,7 @@ module Rley # Module used as a namespace
 
       def initialize(aPosition, lastToken, expectedTerminals)
         super(aPosition)
+        raise StandardError, 'Internal error: nil token' if lastToken.nil?
         @last_token = lastToken.dup
         @expected_terminals = expectedTerminals.dup
       end

data/lib/rley/parser/gfg_chart.rb

@@ -44,7 +44,14 @@ module Rley # This module is used as a namespace
       end
 
       # Push a parse entry for the chart entry with given index
+      # @param anIndex [Integer] The rank of the token in the input stream.
+      # @return [ParseEntry] the passed parse entry if it is pushed
       def push_entry(aVertex, anOrigin, anIndex, _reason)
+        # puts "push_entry:"
+        # puts "  aVertex #{aVertex.inspect}"
+        # puts "  anOrigin: #{anOrigin}"
+        # puts "  anIndex: #{anIndex}"
+        # puts "  _reason: #{_reason}"
         new_entry = ParseEntry.new(aVertex, anOrigin)
         pushed = self[anIndex].push_entry(new_entry)
 
@@ -66,6 +73,13 @@ module Rley # This module is used as a namespace
 
         # Retrieve all the end entries (i.e. of the form
         last_entries = sets[last_index].entries.select(&:end_entry?)
+        # last_entries.each_with_index do |entry, index|
+          # if entry.nil?
+            # puts "Nil entry at index #{index}"
+          # else
+            # puts entry
+          # end
+        # end
 
         # ... now find the end vertex for start symbol and with origin at zero.
         success_entries = last_entries.select do |entry|

data/lib/rley/parser/gfg_earley_parser.rb

@@ -46,7 +46,6 @@ module Rley # This module is used as a namespace
             break unless scan_success
           end
         end
-        
         result.done # End of parsing process
         return result
       end

data/lib/rley/parser/gfg_parsing.rb

@@ -28,7 +28,7 @@ module Rley # This module is used as a namespace
       # @return [Hash{ParseEntry => Array<ParseEntry>}]
       attr_reader(:antecedence)
 
-      # The reason of a parse failure
+      # @return [ErrorReason] The reason of a parse failure
       attr_reader(:failure_reason)
 
       # Constructor
@@ -163,8 +163,9 @@ module Rley # This module is used as a namespace
       # Return true if the parse was successful (= input tokens
       # followed the syntax specified by the grammar)
       def success?()
+        return false if @failure_reason
         return chart.accepting_entry ? true : false
-      end
+      end      
 
       # Return true if there are more than one complete state
       # for the same lhs and same origin in any state set.
@@ -210,7 +211,7 @@ module Rley # This module is used as a namespace
       end
 
       # A notification that the parsing reached an end
-      def done
+      def done()
         # Parse not successful and no reason identified
         # Assuming that parse failed because of a premature end      
         premature_end unless success? || failure_reason

data/lib/rley/parser/parse_entry.rb

@@ -8,22 +8,39 @@ module Rley # This module is used as a namespace
     # - To know whether the vertex is a start, end or item vertex
     # - To know the next symbol to expect
     class ParseEntry
-      # Link to a vertex of the GFG
+      # @return [GFG::Vertex] Link to a vertex of the GFG
       attr_reader(:vertex)
 
-      # Links to preceding parse entries
+      # @return [Array<ParseEntry>] Links to preceding parse entries
       attr_reader(:antecedents)
 
       # the position in the input that matches the beginning of the rhs
       # of the production.
+      # @return [Integer]
       attr_reader(:origin)
 
+      # @param aVertex [GFG::Vertex]
+      # @param theOrigin [Integer]
       def initialize(aVertex, theOrigin)
         @vertex = valid_vertex(aVertex)
         @origin = theOrigin
         @antecedents = []
       end
 
+      # Returns a string containing a human-readable representation of the
+      # production.
+      # @return [String]
+      def inspect()
+        result = selfie()
+        result << " @antecedents=["
+        antecedents.each do |antec|
+          result << antec.selfie
+        end
+        result << ']>'
+
+        return result
+      end
+
       # Add a link to an antecedent parse entry
       def add_antecedent(anAntecedent)
         antecedents << anAntecedent
@@ -75,7 +92,7 @@ module Rley # This module is used as a namespace
       def next_symbol()
         return vertex.next_symbol
       end
-      
+
       # Return true if the entry has no antecedent entry
       def orphan?()
         return antecedents.empty?
@@ -125,9 +142,22 @@ module Rley # This module is used as a namespace
         return vertex.label + " | #{origin}"
       end
 
+      protected
+
+      # Returns a human-readable and partial representation of itself.
+      # @return [String]
+      def selfie()
+        result = "#<#{self.class.name}:#{self.object_id}"
+        result << " @vertex=<#{vertex.class.name}:#{vertex.object_id}"
+        result << " label=#{vertex.label}>"
+        result << " @origin=#{origin}"
+
+        return result
+      end
 
       private
 
+
       # Return the validated GFG vertex
       def valid_vertex(aVertex)
         raise StandardError, 'GFG vertex cannot be nil' if aVertex.nil?

data/lib/rley/parser/parse_entry_set.rb

@@ -1,4 +1,4 @@
-require 'forwardable' # Delegation
+require 'forwardable' # For the Delegation pattern
 
 require_relative '../syntax/terminal'
 require_relative '../syntax/non_terminal'
@@ -11,14 +11,26 @@ module Rley # This module is used as a namespace
       extend Forwardable
       def_delegators :entries, :empty?, :size, :first, :last, :pop, :each
 
-      # The set of parse entries
+      # @return [Array<ParseEntry>] The set of parse entries
       attr_reader :entries
 
+      # Constructor.
       def initialize()
         @entries = []
         @entries4term = Hash.new { |hash, key| hash[key] = [] }
         @entries4n_term = Hash.new { |hash, key| hash[key] = [] }
       end
+      
+      # Returns a string containing a human-readable representation of the 
+      # set of parse entries.
+      # @return [String]
+      def inspect()
+        result = "#<#{self.class.name}:#{self.object_id}"
+        result << ' @entries=['
+        entries.each { |e| result << e.inspect }
+        result << ']>'
+        return result
+      end
 
       # Access the entry at given position
       def [](index)

data/lib/rley/parser/parse_tree_builder.rb

@@ -135,7 +135,7 @@ module Rley # This module is used as a namespace
               process_middle_entry(anEntry, anIndex)
             end
           else
-            $stderr.puts "waiko '#{anEvent}'"
+            $stderr.puts "Internal Errore '#{anEvent}'"
             raise NotImplementedError
         end
       end