dendroid 0.0.9 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 171ca5c202ea7a9d5c156086cecee352539c4f0551556175cf748328d3fa0983
-  data.tar.gz: a81434f0fe610433cce7d6f1213dc3a25d06f7264fda882ae4b5ec4867514b31
+  metadata.gz: 2564f1269225e08732a9f995b10ebbbbf4710b0a1b0aea73e7fe4b486c34a1aa
+  data.tar.gz: db15f965e9365276ffc576435d514cd6c9170a8727c7fafe1425a9de7ed3e0cd
 SHA512:
-  metadata.gz: 7894b4b5abcabde582c9e16dc93df8e5aa77b9fed8691ea52901c802e957558c4353e009544b53d352e45d145807e2957e43468a1783cca92326b52eddddb140
-  data.tar.gz: b4a8415a997fb45005668b03087dc6347db11dd692c87e6da5c9fa6cd0d609ee31b630e51e896627cf95b1a57a8f60f919dde9cf18c8fba4a4f08191dc28004b
+  metadata.gz: 2517fd57cca364571e19ddd183d53fcd4fd642f0cf83ecc58ef0f62e5c7512c343cc3db4f31ee621dad5009386db0161e7c2f67944820dd36cb2a253a4d7af80
+  data.tar.gz: 24b77e7c0c5e97df315102c3434dddd251eacab96efaa3d194006c3874f6d260aeafa076ec5b6bd1bed296fbc675bef00e2fcf49a1e4516c20cc6b3e3b0aefdb

data/.rubocop.yml

@@ -20,7 +20,7 @@ Metrics/CyclomaticComplexity:
 
 Metrics/MethodLength:
   Enabled: true
-  Max: 30
+  Max: 60
   
 Metrics/PerceivedComplexity:
   Enabled: true
@@ -32,5 +32,8 @@ Naming/MethodParameterName:
 Naming/VariableName:  
   Enabled: false
 
+Style/AccessorGrouping:
+  Enabled: false
+
 Style/CommentedKeyword:
   Enabled: false

data/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## [Unreleased]
 
+## [0.0.11] - 2023-11-02
+Added Earley recognizer and its ancillary classes.
+
+### Added
+- Class `Chart` and its spec file
+- Class `EItem` and its spec file
+- Class `ItemSet` and its spec file
+- Class `Recognizer` and its spec file
+
+### Changed
+- RSpec tests: moved module `SampleGrammars` to separate file in folder `support`
+
+## [0.0.10] - 2023-11-01
+Added missing class and method documentation, fixed some `Rubocop` offenses.
+
+
 ## [0.0.9] - 2023-11-01
 Added classes for tokenization and grammar analysis.
 

data/dendroid.gemspec

@@ -7,8 +7,8 @@ Gem::Specification.new do |s|
     path = ::File.dirname(libpath) + ::File::SEPARATOR
     ::File.read("#{path}version.txt").strip
   end
-  s.summary     = 'WIP. A Ruby implementation of a Earley parser'
-  s.description = 'WIP. A Ruby implementation of a Earley parser'
+  s.summary     = 'WIP. A Ruby implementation of an Earley parser'
+  s.description = 'WIP. A Ruby implementation of an Earley parser'
   s.authors     = ['Dimitri Geshef']
   s.email       = 'famished.tiger@yahoo.com'
   s.files       = Dir['bin/dendroid',

data/lib/dendroid/grm_analysis/alternative_item.rb

@@ -61,11 +61,10 @@ module Dendroid
       private
 
       def valid_position(aPosition)
-        raise Exception if aPosition < 0 || aPosition > rule.alternatives[alt_index].size
+        raise StandardError if aPosition.negative? || aPosition > rule.alternatives[alt_index].size
 
         aPosition
       end
     end # class
   end # module
 end # module
-

data/lib/dendroid/grm_analysis/choice_items.rb

@@ -52,8 +52,9 @@ module Dendroid
       def next_item(anItem)
         items_arr = items[anItem.alt_index]
         return nil if anItem == items_arr.last
+
         items_arr[anItem.position + 1]
       end
     end # module
   end # module
-end # module
+end # module

data/lib/dendroid/grm_analysis/dotted_item.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Dendroid
+  # This module contains classes that from the analysis of grammar rules help to build objects
+  # needed by a recognizer or a parser for the language.
   module GrmAnalysis
     # For a given production rule, a dotted item represents a recognition state.
     # The dot partitions the rhs of the rule in two parts:

data/lib/dendroid/grm_analysis/grm_analyzer.rb

@@ -5,19 +5,34 @@ require_relative '../grm_analysis/choice_items'
 
 module Dendroid
   module GrmAnalysis
-    # An analyzer performs an analysis of the rules of a given grammar
-    #
+    # An analyzer performs an analysis of the grammar rules and
+    # build objects (dotted items, first and follow sets) to be used
+    # by a recognizer or a parser.
     class GrmAnalyzer
+      # @return [Dendroid::Syntax::Grammar] The grammar subjected to analysis
       attr_reader :grammar
       attr_reader :items
       attr_reader :production2items
       attr_reader :symbol2productions
+
+      # @return [Dendroid::Syntax::Terminal] The pseudo-terminal `__epsilon` (for empty string)
       attr_reader :epsilon
+
+      # @return [Dendroid::Syntax::Terminal] The pseudo-terminal `$$` for end of input stream
       attr_reader :endmarker
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to FIRST SETS mapping
       attr_reader :first_sets
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to PREDICT SETS mapping
       attr_reader :predict_sets
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to FOLLOW SETS mapping
       attr_reader :follow_sets
 
+      # Constructor.
+      # Build dotted items, first, follow sets for the given grammar
+      # @param aGrammar [Dendroid::Syntax::Grammar]
       def initialize(aGrammar)
         @grammar = aGrammar
         @items = []
@@ -34,6 +49,8 @@ module Dendroid
         build_follow_sets
       end
 
+      # The next item of a given dotted item
+      # @param aDottedItem [DottedItem]
       def next_item(aDottedItem)
         prod = aDottedItem.rule
         prod.next_item(aDottedItem)
@@ -72,7 +89,7 @@ module Dendroid
             else
               first_head.merge(sequence_first(prod.body.members))
             end
-            changed = true if (first_head.size > pre_first_size)
+            changed = true if first_head.size > pre_first_size
           end
         end until !changed
       end
@@ -84,7 +101,7 @@ module Dendroid
           elsif symb.nullable?
             first_sets[symb] = Set.new([epsilon])
           else
-            first_sets[symb]  = Set.new
+            first_sets[symb] = Set.new
           end
         end
       end
@@ -115,7 +132,7 @@ module Dendroid
 
                 head = prod.head
                 head_follow = follow_sets[head]
-                trailer = Set.new
+                # trailer = Set.new
                 last = true
                 last_index = body.size - 1
                 last_index.downto(0) do |i|
@@ -130,7 +147,7 @@ module Dendroid
                     follow_sets[symbol].merge(head_follow) if symbol.nullable?
                     last = false
                   else
-                    symbol_seq = body.slice(i+1, last_index - i)
+                    symbol_seq = body.slice(i + 1, last_index - i)
                     trailer_first = sequence_first(symbol_seq)
                     contains_epsilon = trailer_first.include? epsilon
                     trailer_first.delete(epsilon) if contains_epsilon
@@ -146,7 +163,7 @@ module Dendroid
 
               head = prod.head
               head_follow = follow_sets[head]
-              trailer = Set.new
+              # trailer = Set.new
               last = true
               last_index = body.size - 1
               last_index.downto(0) do |i|
@@ -161,7 +178,7 @@ module Dendroid
                   follow_sets[symbol].merge(head_follow) if symbol.nullable?
                   last = false
                 else
-                  symbol_seq = body.slice(i+1, last_index - i)
+                  symbol_seq = body.slice(i + 1, last_index - i)
                   trailer_first = sequence_first(symbol_seq)
                   contains_epsilon = trailer_first.include? epsilon
                   trailer_first.delete(epsilon) if contains_epsilon
@@ -179,7 +196,7 @@ module Dendroid
         grammar.symbols.each do |symb|
           next if symb.terminal?
 
-          follow_sets[symb]  = Set.new
+          follow_sets[symb] = Set.new
         end
 
         # Initialize FOLLOW(start symbol) with end marker

data/lib/dendroid/grm_analysis/production_items.rb

@@ -14,7 +14,7 @@ module Dendroid
         @items = if empty?
                    [DottedItem.new(self, 0)]
                  else
-                   (0..body.size).reduce([]) do | result,pos|
+                   (0..body.size).reduce([]) do |result, pos|
                      result << GrmAnalysis::DottedItem.new(self, pos)
                    end
                  end
@@ -47,6 +47,7 @@ module Dendroid
       # @return [GrmAnalysis::DottedItem|NilClass]
       def next_item(anItem)
         return nil if anItem == @items.last
+
         @items[anItem.position + 1]
       end
     end # module

data/lib/dendroid/lexical/token.rb

@@ -24,7 +24,7 @@ module Dendroid
       attr_reader :position
 
       # @return [String] The name of terminal symbol matching the text.
-      attr :terminal
+      attr_reader :terminal
 
       # Constructor.
       # @param original [String] the piece of text from input

data/lib/dendroid/lexical/token_position.rb

@@ -2,19 +2,27 @@
 
 module Dendroid
   module Lexical
+    # Keeps track of the position of a token in the input stream.
     class TokenPosition
+      # @return [Integer] The line number where the token begins
       attr_reader :lineno
+
+      # @return [Integer] The column number where the token begins
       attr_reader :column
 
+      # Constructor
+      # @param line [Integer] The line number where the token begins
+      # @param col [Integer] The column number where the token begins
       def initialize(line, col)
         @lineno = line
         @column = col
       end
 
+      # Return the position of the start of the token in line:col format
+      # @return [String]
       def to_s
         "#{lineno}:#{column}"
       end
     end # class
   end # module
 end # module
-

data/lib/dendroid/recognizer/chart.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative 'item_set'
+
+module Dendroid
+  module Recognizer
+    # Also called a parse table.
+    # Assuming that n == number of input tokens,
+    # then the chart is an array with n + 1 entry sets.
+    class Chart
+      extend Forwardable
+
+      # @return [Array<Recognizer::ItemSet>] The array of item sets
+      attr_reader :item_sets
+
+      # @return [Boolean] Indicates whether the recognizer successfully processed the whole input
+      attr_writer :success
+
+      # @return [StandardError] The exception class in case of an error found by the recognizer
+      attr_accessor :failure_class
+
+      # @return [String] The error message
+      attr_accessor :failure_reason
+
+      def_delegators :@item_sets, :[], :last, :size
+
+      # Constructor
+      # Initialize the chart with one empty item set.
+      def initialize
+        @item_sets = []
+        @success = false
+        append_new_set
+      end
+
+      # Add a new empty item set at the end of the array of item sets
+      def append_new_set()
+        item_sets << ItemSet.new
+      end
+
+      # Add an EItem to the last item set
+      # @param e_item [EItem]
+      def seed_last_set(e_item)
+        item_sets.last.add_item(e_item)
+      end
+
+      # Return true if the input text is valid according to the grammar.
+      # @return [Boolean]
+      def successful?
+        @success
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/recognizer/e_item.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Dendroid
+  module Recognizer
+    # An Earley item is essentially a pair consisting of a dotted item and the rank of a token.
+    # It helps to keep track the progress of an Earley recognizer.
+    class EItem
+      extend Forwardable
+
+      # @return [Dendroid::GrmAnalysis::DottedItem]
+      attr_reader :dotted_item
+
+      # @return [Integer] the rank of the token that correspond to the start of the rule.
+      attr_reader :origin
+
+      def_delegators :@dotted_item, :completed?, :expecting?, :next_symbol, :pre_scan?
+
+      # @param aDottedItem [Dendroid::GrmAnalysis::DottedItem]
+      # @param origin [Integer]
+      def initialize(aDottedItem, origin)
+        @dotted_item = aDottedItem
+        @origin = origin
+      end
+
+      # @return [Dendroid::Syntax::NonTerminal] the head of the production rule
+      def lhs
+        dotted_item.rule.lhs
+      end
+
+      # Equality test.
+      # @return [Boolean] true iff dotted items and origins are equal
+      def ==(other)
+        return true if eql?(other)
+
+        di = dotted_item
+        (origin == other.origin) && (di == other.dotted_item)
+      end
+
+      # @return [String] the text representation of the Earley item
+      def to_s
+        "#{dotted_item} @ #{origin}"
+      end
+    end # class
+  end # module
+end # module
+

data/lib/dendroid/recognizer/item_set.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Dendroid
+  module Recognizer
+    # Holds the EItem identified by the recognizer when processing at token at given rank.
+    class ItemSet
+      extend Forwardable
+
+      # @return [Recognizer::EItem]
+      attr_reader :items
+      def_delegators :@items, :clear, :each, :empty?, :select, :size
+
+      def initialize
+        @items = []
+      end
+
+      # Add an Early item to the set
+      # @param anItem [Recognizer::EItem]
+      def add_item(anItem)
+        @items << anItem unless items.include? anItem
+      end
+
+      # Find the items that expect a given grammar symbol
+      # @param aSymbol [Denroid::Syntax::GrmSymbol]
+      # @return [void]
+      def items_expecting(aSymbol)
+        items.select { |itm| itm.expecting?(aSymbol) }
+      end
+
+      # Return a text representation of the item set
+      # @return [String]
+      def to_s
+        items.join("\n")
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/recognizer/recognizer.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+require_relative '../grm_analysis/grm_analyzer'
+require_relative 'e_item'
+require_relative 'chart'
+
+module Dendroid
+  # This module host classes needed to implement an Earley recognizer
+  module Recognizer
+    # A recognizer determines whether the input text complies to the grammar (syntax) rules.
+    # This class implements the Earley recognition algorithm.
+    class Recognizer
+      # @return [GrmAnalysis::GrmAnalyzer]
+      attr_reader :grm_analysis
+
+      # @return [Object]
+      attr_reader :tokenizer
+
+      def initialize(grammar, tokenizer)
+        @grm_analysis = GrmAnalysis::GrmAnalyzer.new(grammar)
+        @tokenizer = tokenizer
+      end
+
+      def run(source)
+        tokenizer.input = source
+        tok = tokenizer.next_token
+        if tok.nil? && !grm_analysis.grammar.start_symbol.nullable?
+          chart = new_chart
+          chart.failure_class = StandardError
+          chart.failure_reason = 'Error: Input may not be empty nor blank.'
+          chart
+        else
+          earley_parse(tok)
+        end
+      end
+
+      def earley_parse(initial_token)
+        chart = new_chart
+        tokens = [initial_token]
+        predicted_symbols = [Set.new]
+        eos_reached = initial_token.nil?
+        rank = 0
+
+        loop do
+          eos_reached = advance_next_token(tokens, predicted_symbols) unless eos_reached
+
+          advance = false
+          curr_rank = rank
+          curr_set = chart[curr_rank]
+          curr_set.each do |entry|
+            # For each entry, do either completer, scanner or predictor action
+            tick = do_entry_action(chart, entry, curr_rank, tokens, :genuine, predicted_symbols)
+            advance ||= tick
+          end
+
+          rank += 1 if advance
+          break if eos_reached && !advance
+          break if ! advance
+        end
+
+        determine_outcome(chart, tokens)
+        chart
+      end
+
+      private
+
+      def new_chart
+        top_symbol = grm_analysis.grammar.start_symbol
+
+        # Reminder: there might be multiple rules for the start symbol
+        prods = grm_analysis.grammar.nonterm2productions[top_symbol]
+        chart = Chart.new
+        prods.each do |prd|
+          seed_items = prd.predicted_items
+          seed_items.each { |item| chart.seed_last_set(EItem.new(item, 0)) }
+        end
+
+        chart
+      end
+
+      def advance_next_token(tokens, predicted_symbols)
+        eos_reached = false
+        tok = tokenizer.next_token
+        if tok
+          tokens << tok
+        else
+          eos_reached = true
+        end
+
+        predicted_symbols << Set.new unless eos_reached
+        eos_reached
+      end
+
+      def do_entry_action(chart, entry, rank, tokens, mode, predicted_symbols)
+        advance = false
+
+        if entry.completed?
+          completer(chart, entry, rank, tokens, mode)
+        else
+          if entry.next_symbol.terminal?
+            advance = scanner(chart, entry, rank, tokens)
+          else
+            predictor(chart, entry, rank, tokens, mode, predicted_symbols)
+          end
+        end
+
+        advance
+      end
+=begin
+    procedure PREDICTOR((A → α•Bβ, j), k)
+        for each (B → γ) in GRAMMAR_RULES_FOR(B) do
+            ADD_TO_SET((B → •γ, k), S[k])
+        end
+      Assuming next symbol is a non-terminal
+
+      Error case: next actual token matches none of the expected tokens.
+=end
+      def predictor(chart, item, rank, tokens, mode, predicted_symbols)
+        next_symbol = item.next_symbol
+        if mode == :genuine
+          predicted_symbols << Set.new if rank == predicted_symbols.size
+          predicted = predicted_symbols[rank]
+          return if predicted.include?(next_symbol)
+
+          predicted.add(next_symbol)
+        end
+
+        prods = grm_analysis.symbol2productions[next_symbol]
+        curr_set = chart[rank]
+        next_token = tokens[rank]
+        prods.each do |prd|
+          entry_items = prd.predicted_items
+          entry_items.each do |entry|
+            member = entry.next_symbol
+            if member&.terminal?
+              next unless next_token
+              next if (member.name != next_token.terminal) && mode == :genuine
+            end
+
+            new_item = EItem.new(entry, rank)
+            curr_set.add_item(new_item)
+          end
+        end
+
+        # Use trick from paper John Aycock and R. Nigel Horspool: "Practical Earley Parsing"
+        if next_symbol.nullable?
+          next_item = grm_analysis.next_item(item.dotted_item)
+          if next_item
+            new_item = EItem.new(next_item, item.origin)
+            curr_set.add_item(new_item)
+          end
+        end
+      end
+
+=begin
+    procedure SCANNER((A → α•aβ, j), k, words)
+        if j < LENGTH(words) and a ⊂ PARTS_OF_SPEECH(words[k]) then
+            ADD_TO_SET((A → αa•β, j), S[k+1])
+        end
+    Assuming next symbol is a terminal
+=end
+      def scanner(chart, scan_item, rank, tokens)
+        advance = false
+        dit = scan_item.dotted_item
+        if rank < tokens.size && dit.next_symbol.name == tokens[rank].terminal
+          new_rank = rank + 1
+          chart.append_new_set if chart[new_rank].nil?
+          next_dotted_item = grm_analysis.next_item(dit)
+          new_item = EItem.new(next_dotted_item, scan_item.origin)
+          chart[new_rank].add_item(new_item)
+          advance = true
+        end
+
+        advance
+      end
+
+=begin
+    procedure COMPLETER((B → γ•, x), k)
+        for each (A → α•Bβ, j) in S[x] do
+            ADD_TO_SET((A → αB•β, j), S[k])
+        end
+=end
+      def completer(chart, item, rank, tokens, mode)
+        origin = item.origin
+
+        curr_set = chart[rank]
+        set_at_origin = chart[origin]
+        next_token = tokens[rank]
+        callers = set_at_origin.items_expecting(item.lhs)
+        callers.each do |call_item|
+          return_item = grm_analysis.next_item(call_item.dotted_item)
+          next unless return_item
+          member = return_item.next_symbol
+          if member&.terminal? && (mode == :genuine)
+            next unless next_token
+            next if member.name != next_token.terminal
+          end
+
+          new_item = EItem.new(return_item, call_item.origin)
+          curr_set.add_item(new_item)
+        end
+      end
+
+      def seed_set(chart, rank)
+        curr_set = chart[rank]
+        previous_set = chart[rank - 1]
+        curr_set.clear
+        scan_entries = previous_set.select { |ent| ent.dotted_item.next_symbol&.terminal? }
+        scan_entries.map do |ent|
+          new_item = grm_analysis.next_item(ent.dotted_item)
+          curr_set.add_item(EItem.new(new_item, ent.origin))
+        end
+      end
+
+      def determine_outcome(chart, tokens)
+        success = false
+        if chart.size == tokens.size + 1
+          top_symbol = grm_analysis.grammar.start_symbol
+          top_rules = grm_analysis.grammar.nonterm2productions[top_symbol]
+          final_items = top_rules.reduce([]) do |items, rule|
+            items.concat(rule.reduce_items)
+          end
+          last_set = chart.item_sets.last
+          last_set.each do |entry|
+            next if ((!entry.origin.zero?) || ! final_items.include?(entry.dotted_item))
+            success = true
+          end
+        end
+
+        if !success
+          # Error detected...
+          replay_last_set(chart, tokens)
+          if chart.size < tokens.size + 1
+            # Recognizer stopped prematurely...
+            offending_token = tokens[chart.size - 1]
+            pos = offending_token.position
+            (line, col) = [pos.lineno, pos.column]
+            last_set = chart.last
+            terminals = last_set.items.reduce([]) do |result, ent|
+              result << ent.next_symbol if ent.pre_scan?
+              result
+            end
+            terminals.uniq!
+            prefix = "Syntax error at or near token line #{line}, column #{col} >>>#{offending_token.source}<<<"
+            expectation = terminals.size == 1 ? "#{terminals[0].name}" : "one of: [#{terminals.map(&:name).join(', ')}]"
+            err_msg = "#{prefix} Expected #{expectation}, found a #{offending_token.terminal} instead."
+            chart.failure_class = StandardError
+            chart.failure_reason = err_msg
+          elsif chart.size == tokens.size + 1
+            # EOS unexpected...
+            last_token = tokens.last
+            pos = last_token.position
+            (line, col) = [pos.lineno, pos.column]
+            last_set = chart.last
+            terminals = last_set.items.reduce([]) do |result, ent|
+              result << ent.next_symbol if ent.pre_scan?
+              result
+            end
+            terminals.uniq!
+
+            prefix = "Line #{line}, column #{col}: Premature end of input after '#{last_token.source}'"
+            expectation = terminals.size == 1 ? "#{terminals[0].name}" : "one of: [#{terminals.map(&:name).join(', ')}]"
+            err_msg = "#{prefix}, expected: #{expectation}."
+            chart.failure_class = StandardError
+            chart.failure_reason = err_msg
+          end
+        end
+        chart.success = success
+      end
+
+      def replay_last_set(chart, tokens)
+        rank = chart.size - 1
+        seed_set(chart, rank) # Re-initialize last set with scan entries
+
+        # Replay in full the actions for last set
+        chart[rank].each do |entry|
+          do_entry_action(chart, entry, rank, tokens, :error, [Set.new])
+        end
+      end
+    end # class
+  end # module
+end # module