dendroid 0.0.11 → 0.1.00

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2564f1269225e08732a9f995b10ebbbbf4710b0a1b0aea73e7fe4b486c34a1aa
-  data.tar.gz: db15f965e9365276ffc576435d514cd6c9170a8727c7fafe1425a9de7ed3e0cd
+  metadata.gz: 722b27a6f20e87c43de339b3f0c45e2bcc77c464d5dd9ecd56bbb686c4857b61
+  data.tar.gz: ce6ffd0c100ea7b7c336044e2877617eebb85c7bb306d5de9d1d4395200320aa
 SHA512:
-  metadata.gz: 2517fd57cca364571e19ddd183d53fcd4fd642f0cf83ecc58ef0f62e5c7512c343cc3db4f31ee621dad5009386db0161e7c2f67944820dd36cb2a253a4d7af80
-  data.tar.gz: 24b77e7c0c5e97df315102c3434dddd251eacab96efaa3d194006c3874f6d260aeafa076ec5b6bd1bed296fbc675bef00e2fcf49a1e4516c20cc6b3e3b0aefdb
+  metadata.gz: 69870ade1f77e7fe0b9faf20b7943a500abdf2b41d383a4e048438e431e6f65bf4b418806ec4ba325a6839ee4eb1337085772fe7fa5c594b59663cd653cdeac6
+  data.tar.gz: 137bbf46a71dcb603f3866f51b5be25a8557fb574937e2e1b3f8b40980cd6e7fd6a851c84b229b7b7ecb692d2899d721e3bcc041fe09e213318f46efdb041ea7

data/.rubocop.yml

@@ -12,7 +12,7 @@ Metrics/BlockLength:
 
 Metrics/ClassLength:
   Enabled: true
-  Max: 200
+  Max: 300
   
 Metrics/CyclomaticComplexity:  
   Enabled: true

data/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## [Unreleased]
 
+## [0.1.00] - 2023-11-03
+Version bump: the Earley recognizer is functional.
+
+## [0.0.12] - 2023-11-02
+Added more tests.
+
+### Added
+- Added more tests to spec file of `Grammar` class.
+- Added more tests to spec file of `Recognizer` class.
+
 ## [0.0.11] - 2023-11-02
 Added Earley recognizer and its ancillary classes.
 

data/lib/dendroid/grm_analysis/grm_analyzer.rb

@@ -13,7 +13,6 @@ module Dendroid
       attr_reader :grammar
       attr_reader :items
       attr_reader :production2items
-      attr_reader :symbol2productions
 
       # @return [Dendroid::Syntax::Terminal] The pseudo-terminal `__epsilon` (for empty string)
       attr_reader :epsilon
@@ -37,7 +36,6 @@ module Dendroid
         @grammar = aGrammar
         @items = []
         @production2items = {}
-        @symbol2productions = {}
         @epsilon = Syntax::Terminal.new(:__epsilon)
         @endmarker = Syntax::Terminal.new(:"$$")
         @first_sets = {}
@@ -56,14 +54,14 @@ module Dendroid
         prod.next_item(aDottedItem)
       end
 
+      def symbol2production(sym)
+        grammar.nonterm2production[sym]
+      end
+
       private
 
       def build_dotted_items
         grammar.rules.each do |prod|
-          lhs = prod.head
-          symbol2productions[lhs] = [] unless symbol2productions.include? lhs
-          symbol2productions[lhs] << prod
-          # production2items[prod] = []
           mixin = prod.choice? ? ChoiceItems : ProductionItems
           prod.extend(mixin)
           prod.build_items
@@ -76,33 +74,31 @@ module Dendroid
       def build_first_sets
         initialize_first_sets
 
-        begin
+        loop do
           changed = false
           grammar.rules.each do |prod|
             head = prod.head
             first_head = first_sets[head]
             pre_first_size = first_head.size
-            if prod.choice?
-              prod.alternatives.each do |alt|
-                first_head.merge(sequence_first(alt.members))
-              end
-            else
-              first_head.merge(sequence_first(prod.body.members))
+            prod.rhs.each do |seq|
+              first_head.merge(sequence_first(seq.members))
             end
             changed = true if first_head.size > pre_first_size
           end
-        end until !changed
+          break unless changed
+        end
       end
 
       def initialize_first_sets
         grammar.symbols.each do |symb|
-          if symb.terminal?
-            first_sets[symb] = Set.new([symb])
-          elsif symb.nullable?
-            first_sets[symb] = Set.new([epsilon])
-          else
-            first_sets[symb] = Set.new
-          end
+          set_arg = if symb.terminal?
+                      [symb]
+                    elsif symb.nullable?
+                      [epsilon]
+                    else
+                      nil
+                    end
+          first_sets[symb] = Set.new(set_arg)
         end
       end
 
@@ -122,43 +118,11 @@ module Dendroid
       def build_follow_sets
         initialize_follow_sets
 
-        begin
+        loop do
           changed = false
           grammar.rules.each do |prod|
-            if prod.choice?
-              prod.alternatives.each do |alt|
-                body = alt.members
-                next if body.empty?
-
-                head = prod.head
-                head_follow = follow_sets[head]
-                # trailer = Set.new
-                last = true
-                last_index = body.size - 1
-                last_index.downto(0) do |i|
-                  symbol = body[i]
-                  next if symbol.terminal?
-
-                  follow_symbol = follow_sets[symbol]
-                  size_before = follow_symbol.size
-                  if last
-                    # Rule: if last non-terminal member (symbol) is nullable
-                    # then add FOLLOW(head) to FOLLOW(symbol)
-                    follow_sets[symbol].merge(head_follow) if symbol.nullable?
-                    last = false
-                  else
-                    symbol_seq = body.slice(i + 1, last_index - i)
-                    trailer_first = sequence_first(symbol_seq)
-                    contains_epsilon = trailer_first.include? epsilon
-                    trailer_first.delete(epsilon) if contains_epsilon
-                    follow_sets[symbol].merge(trailer_first)
-                    follow_sets[symbol].merge(head_follow) if contains_epsilon
-                  end
-                  changed = true if follow_sets[symbol].size > size_before
-                end
-              end
-            else
-              body = prod.body.members
+            prod.rhs.each do |alt|
+              body = alt.members
               next if body.empty?
 
               head = prod.head
@@ -189,7 +153,8 @@ module Dendroid
               end
             end
           end
-        end until !changed
+          break unless changed
+        end
       end
 
       def initialize_follow_sets

data/lib/dendroid/recognizer/chart.rb

@@ -4,9 +4,11 @@ require_relative 'item_set'
 
 module Dendroid
   module Recognizer
-    # Also called a parse table.
-    # Assuming that n == number of input tokens,
-    # then the chart is an array with n + 1 entry sets.
+    # Also called a parse table. It records the progress of the
+    # Earley recognizer whens its verifies the compliance of the input text
+    # to the language grammar rules.
+    # It essentially consists in an array of item sets.
+    # If n is the number of input tokens then the chart has n + 1 entry sets.
     class Chart
       extend Forwardable
 
@@ -17,10 +19,10 @@ module Dendroid
       attr_writer :success
 
       # @return [StandardError] The exception class in case of an error found by the recognizer
-      attr_accessor :failure_class
+      attr_reader :failure_class
 
       # @return [String] The error message
-      attr_accessor :failure_reason
+      attr_reader :failure_reason
 
       def_delegators :@item_sets, :[], :last, :size
 
@@ -33,7 +35,7 @@ module Dendroid
       end
 
       # Add a new empty item set at the end of the array of item sets
-      def append_new_set()
+      def append_new_set
         item_sets << ItemSet.new
       end
 
@@ -48,6 +50,14 @@ module Dendroid
       def successful?
         @success
       end
+
+      # Set the error cause.
+      # @param exception_class [StandardError] Exception class
+      # @param message [String] Error message
+      def failure(exception_class, message)
+        @failure_class = exception_class
+        @failure_reason = message
+      end
     end # class
   end # module
 end # module

data/lib/dendroid/recognizer/e_item.rb

@@ -45,4 +45,3 @@ module Dendroid
     end # class
   end # module
 end # module
-

data/lib/dendroid/recognizer/item_set.rb

@@ -8,6 +8,7 @@ module Dendroid
 
       # @return [Recognizer::EItem]
       attr_reader :items
+
       def_delegators :@items, :clear, :each, :empty?, :select, :size
 
       def initialize

data/lib/dendroid/recognizer/recognizer.rb

@@ -16,24 +16,30 @@ module Dendroid
       # @return [Object]
       attr_reader :tokenizer
 
+      # @param grammar [Dendroid::Syntax::Grammar]
+      # @param tokenizer [Object]
       def initialize(grammar, tokenizer)
         @grm_analysis = GrmAnalysis::GrmAnalyzer.new(grammar)
         @tokenizer = tokenizer
       end
 
+      # Try to read the `source` text and verify that it is syntactically correct.
+      # @param source [String] Input text to recognize
+      # @return [Dendroid::Recognizer::Chart]
       def run(source)
         tokenizer.input = source
         tok = tokenizer.next_token
         if tok.nil? && !grm_analysis.grammar.start_symbol.nullable?
           chart = new_chart
-          chart.failure_class = StandardError
-          chart.failure_reason = 'Error: Input may not be empty nor blank.'
+          chart.failure(StandardError, 'Error: Input may not be empty nor blank.')
           chart
         else
           earley_parse(tok)
         end
       end
 
+      # Run the Earley algorithm
+      # @param initial_token [Dednroid::Lexical::Token]
       def earley_parse(initial_token)
         chart = new_chart
         tokens = [initial_token]
@@ -42,7 +48,7 @@ module Dendroid
         rank = 0
 
         loop do
-          eos_reached = advance_next_token(tokens, predicted_symbols) unless eos_reached
+          eos_reached ||= advance_next_token(tokens, predicted_symbols)
 
           advance = false
           curr_rank = rank
@@ -55,7 +61,7 @@ module Dendroid
 
           rank += 1 if advance
           break if eos_reached && !advance
-          break if ! advance
+          break unless advance
         end
 
         determine_outcome(chart, tokens)
@@ -67,13 +73,10 @@ module Dendroid
       def new_chart
         top_symbol = grm_analysis.grammar.start_symbol
 
-        # Reminder: there might be multiple rules for the start symbol
-        prods = grm_analysis.grammar.nonterm2productions[top_symbol]
+        prd = grm_analysis.grammar.nonterm2production[top_symbol]
         chart = Chart.new
-        prods.each do |prd|
-          seed_items = prd.predicted_items
-          seed_items.each { |item| chart.seed_last_set(EItem.new(item, 0)) }
-        end
+        seed_items = prd.predicted_items
+        seed_items.each { |item| chart.seed_last_set(EItem.new(item, 0)) }
 
         chart
       end
@@ -96,25 +99,22 @@ module Dendroid
 
         if entry.completed?
           completer(chart, entry, rank, tokens, mode)
+        elsif entry.next_symbol.terminal?
+          advance = scanner(chart, entry, rank, tokens)
         else
-          if entry.next_symbol.terminal?
-            advance = scanner(chart, entry, rank, tokens)
-          else
-            predictor(chart, entry, rank, tokens, mode, predicted_symbols)
-          end
+          predictor(chart, entry, rank, tokens, mode, predicted_symbols)
         end
 
         advance
       end
-=begin
-    procedure PREDICTOR((A → α•Bβ, j), k)
-        for each (B → γ) in GRAMMAR_RULES_FOR(B) do
-            ADD_TO_SET((B → •γ, k), S[k])
-        end
-      Assuming next symbol is a non-terminal
 
-      Error case: next actual token matches none of the expected tokens.
-=end
+      # procedure PREDICTOR((A → α•Bβ, j), k)
+      #     for each (B → γ) in GRAMMAR_RULES_FOR(B) do
+      #         ADD_TO_SET((B → •γ, k), S[k])
+      #     end
+      #   Assuming next symbol is a non-terminal
+      #
+      #   Error case: next actual token matches none of the expected tokens.
       def predictor(chart, item, rank, tokens, mode, predicted_symbols)
         next_symbol = item.next_symbol
         if mode == :genuine
@@ -125,40 +125,35 @@ module Dendroid
           predicted.add(next_symbol)
         end
 
-        prods = grm_analysis.symbol2productions[next_symbol]
         curr_set = chart[rank]
         next_token = tokens[rank]
-        prods.each do |prd|
-          entry_items = prd.predicted_items
-          entry_items.each do |entry|
-            member = entry.next_symbol
-            if member&.terminal?
-              next unless next_token
-              next if (member.name != next_token.terminal) && mode == :genuine
-            end
-
-            new_item = EItem.new(entry, rank)
-            curr_set.add_item(new_item)
+        prd = grm_analysis.symbol2production(next_symbol)
+        entry_items = prd.predicted_items
+        entry_items.each do |entry|
+          member = entry.next_symbol
+          if member&.terminal?
+            next unless next_token
+            next if (member.name != next_token.terminal) && mode == :genuine
           end
-        end
 
-        # Use trick from paper John Aycock and R. Nigel Horspool: "Practical Earley Parsing"
-        if next_symbol.nullable?
-          next_item = grm_analysis.next_item(item.dotted_item)
-          if next_item
-            new_item = EItem.new(next_item, item.origin)
-            curr_set.add_item(new_item)
-          end
+          new_item = EItem.new(entry, rank)
+          curr_set.add_item(new_item)
         end
+        # Use trick from paper John Aycock and R. Nigel Horspool: "Practical Earley Parsing"
+        return unless next_symbol.nullable?
+
+        next_item = grm_analysis.next_item(item.dotted_item)
+        return unless next_item
+
+        new_item = EItem.new(next_item, item.origin)
+        curr_set.add_item(new_item)
       end
 
-=begin
-    procedure SCANNER((A → α•aβ, j), k, words)
-        if j < LENGTH(words) and a ⊂ PARTS_OF_SPEECH(words[k]) then
-            ADD_TO_SET((A → αa•β, j), S[k+1])
-        end
-    Assuming next symbol is a terminal
-=end
+      # procedure SCANNER((A → α•aβ, j), k, words)
+      #     if j < LENGTH(words) and a ⊂ PARTS_OF_SPEECH(words[k]) then
+      #         ADD_TO_SET((A → αa•β, j), S[k+1])
+      #     end
+      # Assuming next symbol is a terminal
       def scanner(chart, scan_item, rank, tokens)
         advance = false
         dit = scan_item.dotted_item
@@ -174,12 +169,10 @@ module Dendroid
         advance
       end
 
-=begin
-    procedure COMPLETER((B → γ•, x), k)
-        for each (A → α•Bβ, j) in S[x] do
-            ADD_TO_SET((A → αB•β, j), S[k])
-        end
-=end
+      # procedure COMPLETER((B → γ•, x), k)
+      #     for each (A → α•Bβ, j) in S[x] do
+      #         ADD_TO_SET((A → αB•β, j), S[k])
+      #     end
       def completer(chart, item, rank, tokens, mode)
         origin = item.origin
 
@@ -190,6 +183,7 @@ module Dendroid
         callers.each do |call_item|
           return_item = grm_analysis.next_item(call_item.dotted_item)
           next unless return_item
+
           member = return_item.next_symbol
           if member&.terminal? && (mode == :genuine)
             next unless next_token
@@ -216,18 +210,17 @@ module Dendroid
         success = false
         if chart.size == tokens.size + 1
           top_symbol = grm_analysis.grammar.start_symbol
-          top_rules = grm_analysis.grammar.nonterm2productions[top_symbol]
-          final_items = top_rules.reduce([]) do |items, rule|
-            items.concat(rule.reduce_items)
-          end
+          top_rule = grm_analysis.grammar.nonterm2production[top_symbol]
+          final_items = top_rule.reduce_items
           last_set = chart.item_sets.last
           last_set.each do |entry|
-            next if ((!entry.origin.zero?) || ! final_items.include?(entry.dotted_item))
+            next if !entry.origin.zero? || !final_items.include?(entry.dotted_item)
+
             success = true
           end
         end
 
-        if !success
+        unless success
           # Error detected...
           replay_last_set(chart, tokens)
           if chart.size < tokens.size + 1
@@ -235,39 +228,37 @@ module Dendroid
             offending_token = tokens[chart.size - 1]
             pos = offending_token.position
             (line, col) = [pos.lineno, pos.column]
-            last_set = chart.last
-            terminals = last_set.items.reduce([]) do |result, ent|
-              result << ent.next_symbol if ent.pre_scan?
-              result
-            end
-            terminals.uniq!
+            terminals = expected_terminals(chart)
             prefix = "Syntax error at or near token line #{line}, column #{col} >>>#{offending_token.source}<<<"
-            expectation = terminals.size == 1 ? "#{terminals[0].name}" : "one of: [#{terminals.map(&:name).join(', ')}]"
+            expectation = terminals.size == 1 ? terminals[0].name.to_s : "one of: [#{terminals.map(&:name).join(', ')}]"
             err_msg = "#{prefix} Expected #{expectation}, found a #{offending_token.terminal} instead."
-            chart.failure_class = StandardError
-            chart.failure_reason = err_msg
+            chart.failure(StandardError, err_msg)
           elsif chart.size == tokens.size + 1
             # EOS unexpected...
             last_token = tokens.last
             pos = last_token.position
             (line, col) = [pos.lineno, pos.column]
-            last_set = chart.last
-            terminals = last_set.items.reduce([]) do |result, ent|
-              result << ent.next_symbol if ent.pre_scan?
-              result
-            end
-            terminals.uniq!
-
+            terminals = expected_terminals(chart)
             prefix = "Line #{line}, column #{col}: Premature end of input after '#{last_token.source}'"
-            expectation = terminals.size == 1 ? "#{terminals[0].name}" : "one of: [#{terminals.map(&:name).join(', ')}]"
+            expectation = terminals.size == 1 ? terminals[0].name.to_s : "one of: [#{terminals.map(&:name).join(', ')}]"
             err_msg = "#{prefix}, expected: #{expectation}."
-            chart.failure_class = StandardError
-            chart.failure_reason = err_msg
+            chart.failure(StandardError, err_msg)
           end
         end
         chart.success = success
       end
 
+      def expected_terminals(chart)
+        last_set = chart.last
+        terminals = last_set.items.reduce([]) do |result, ent|
+          result << ent.next_symbol if ent.pre_scan?
+          result
+        end
+        terminals.uniq!
+
+        terminals
+      end
+
       def replay_last_set(chart, tokens)
         rank = chart.size - 1
         seed_set(chart, rank) # Re-initialize last set with scan entries
@@ -279,4 +270,4 @@ module Dendroid
       end
     end # class
   end # module
-end # module
+end # module