dendroid 0.1.00 → 0.2.01

data/lib/dendroid/parsing/parser.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require 'weakref'
+require_relative '../grm_dsl/base_grm_builder'
+require_relative '../utils/base_tokenizer'
+require_relative '../recognizer/recognizer'
+require_relative 'chart_walker'
+require_relative 'parse_tree_visitor'
+require_relative '../formatters/bracket_notation'
+require_relative '../formatters/ascii_tree'
+
+module Dendroid
+  module Parsing
+    class Parser
+    end # class
+  end # module
+end # module
+
+def grammar_l1
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Grammar inspired from Wikipedia entry on Earley parsing
+    declare_terminals('PLUS', 'STAR', 'INTEGER')
+    rule('p' => 's')
+    rule('s' => ['s PLUS m', 'm'])
+    rule('m' => ['m STAR t', 't'])
+    rule('t' => 'INTEGER')
+  end
+
+  builder.grammar
+end
+
+def grammar_l31
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Ambiguous arithmetical expression language
+    # This language is compatible with tokenizer L1
+    declare_terminals('PLUS', 'STAR', 'INTEGER')
+    rule('p' => 's')
+    rule('s' => ['s PLUS s', 's STAR s', 'INTEGER'])
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l1
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
+
+    scan_verbatim(['+', '*'])
+    scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
+  end
+end
+
+def retrieve_success_item(chart, grammar)
+  last_item_set = chart.item_sets.last
+  result = nil
+  last_item_set.items.reverse_each do |itm|
+    if itm.origin.zero? && itm.dotted_item.completed? && itm.dotted_item.rule.lhs == grammar.start_symbol
+      result = itm
+      break
+    end
+  end
+
+  result
+end
+
+def grammar_l10
+  # Grammar with left recursive rule
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    declare_terminals('a')
+
+    rule 'A' => ['A a', '']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l10
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'a' => :a })
+
+    scan_verbatim(['a'])
+  end
+end
+
+def grammar_l11
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Grammar with right-recursive rule
+    declare_terminals('a')
+
+    rule 'A' => ['a A', '']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l11
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'a' => :a })
+
+    scan_verbatim(['a'])
+  end
+end
+
+def grammar_l8
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # (based on grammar G2 from paper Masaru Tomita "An Efficient Context-Free Parsing Algorithm
+    #   for Natural Languages")
+    declare_terminals('x')
+
+    rule 'S' => ['S S', 'x']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l8
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'x' => :x })
+
+    scan_verbatim(['x'])
+  end
+end
+
+########################################
+# Entry point
+########################################
+kode = 3
+
+case kode
+when 0
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l1, tokenizer_l1)
+  chart = recognizer.run('2 + 3 * 4')
+  succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  # formatter = BracketNotation.new($stdout)
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 1
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l10, tokenizer_l10)
+  chart = recognizer.run('a a a a a')
+  succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 2
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l11, tokenizer_l11)
+  chart = recognizer.run('a a a a a')
+  succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 3
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l31, tokenizer_l1)
+  chart = recognizer.run('2 + 3 * 4')
+  succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 4
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l8, tokenizer_l8)
+  chart = recognizer.run('x x x x')
+  succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+end

data/lib/dendroid/parsing/terminal_node.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'parse_node'
+
+module Dendroid
+  module Parsing
+    class TerminalNode < ParseNode
+      # @return [Dendroid::Syntax::Terminal] Terminal symbol of matching token.
+      attr_reader :symbol
+
+      # @return [Dendroid::Lexical::Token] Matching input token object.
+      attr_reader :token
+
+      def initialize(sym, tok, rank)
+        super(rank, rank + 1)
+        @symbol = sym
+        @token = tok
+      end
+
+      # Part of the 'visitee' role in Visitor design pattern.
+      # @param aVisitor[ParseTreeVisitor] the visitor
+      def accept(aVisitor)
+        aVisitor.visit_terminal(self)
+      end
+
+      def to_s
+        display_val = token.is_a?(Dendroid::Lexical::Literal) ? ": #{token.value}" : ''
+        "#{symbol.name}#{display_val} #{super}"
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/parsing/walk_progress.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require_relative 'and_node'
+require_relative 'or_node'
+require_relative 'terminal_node'
+require_relative 'empty_rule_node'
+
+module Dendroid
+  module Parsing
+    class WalkProgress
+      attr_accessor :state
+      attr_accessor :curr_rank
+      attr_reader :curr_item
+      attr_accessor :predecessor
+      attr_reader :parents
+
+      # rubocop: disable Metrics/CyclomaticComplexity
+      # rubocop: disable Metrics/PerceivedComplexity
+
+      def initialize(start_rank, start_item, parents)
+        @state = :New
+        @curr_rank = start_rank
+        @curr_item = start_item
+        @predecessor = nil
+        @parents = parents
+      end
+
+      # Factory method.
+      def initialize_copy(orig)
+        @state = orig.state
+        @curr_rank = orig.curr_rank
+        @curr_item = orig.curr_item
+        @predecessor = nil
+        @parents = orig.parents.dup
+      end
+
+      def fork(thePredecessor)
+        @state = :Forking
+        @predecessor = thePredecessor
+      end
+
+      def curr_item=(anEntry)
+        raise StandardError if anEntry.nil?
+
+        @curr_item = anEntry
+      end
+
+      def add_node_empty(anEntry)
+        node_empty = EmptyRuleNode.new(anEntry, curr_rank)
+        add_child_node(node_empty)
+      end
+
+      # Add a terminal node for terminal at current rank as a child of last parent
+      def add_terminal_node(token)
+        @curr_rank -= 1
+        term_node = TerminalNode.new(curr_item.prev_symbol, token, curr_rank)
+        add_child_node(term_node)
+      end
+
+      # Add an AND node for given entry as a child of last parent
+      def push_and_node(anEntry)
+        node = ANDNode.new(anEntry, curr_rank)
+        raise StandardError unless anEntry.rule == node.rule # Fails
+
+        add_child_node(node)
+        parents.push(node)
+
+        node
+      end
+
+      def push_or_node(origin, arity)
+        node = OrNode.new(curr_item.prev_symbol, origin, curr_rank, arity)
+        add_child_node(node)
+        parents.push(node)
+
+        node
+      end
+
+      def add_child_node(aNode)
+        parents.last.add_child(aNode, curr_item.position - 1)
+        aNode
+      end
+
+      # Do the given EItems match one of the parent?
+      # Matching = corresponds to the same rule and range
+      # @return [Array<EItem>]
+      def match_parent?(entries, stop_at_first)
+        matching = []
+        min_origin = entries[0].origin
+        first_iteration = true
+        offset = 0
+
+        parents.reverse_each do |node|
+          if node.is_a?(OrNode)
+            offset += 1
+            next
+          end
+          entries.each do |ent|
+            min_origin = ent.origin if first_iteration && ent.origin < min_origin
+            next unless node.match(ent)
+
+            matching << [ent, offset]
+            break if stop_at_first
+          end
+          first_iteration = false
+          break if stop_at_first && !matching.empty?
+
+          # Stop loop when parent.origin < min(entries.origin)
+          break if node.range[0] < min_origin
+
+          offset += 1
+        end
+
+        matching
+      end
+    end # class
+
+    # rubocop: enable Metrics/CyclomaticComplexity
+    # rubocop: enable Metrics/PerceivedComplexity
+  end # module
+end # module

data/lib/dendroid/recognizer/chart.rb

@@ -15,6 +15,9 @@ module Dendroid
       # @return [Array<Recognizer::ItemSet>] The array of item sets
       attr_reader :item_sets
 
+      # @return [Array<Dendroid::Lexical::Token>] The input tokens
+      attr_accessor :tokens
+
       # @return [Boolean] Indicates whether the recognizer successfully processed the whole input
       attr_writer :success
 

data/lib/dendroid/recognizer/e_item.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require 'weakref'
 
 module Dendroid
   module Recognizer
@@ -9,19 +10,27 @@ module Dendroid
     class EItem
       extend Forwardable
 
+      # (Weak) reference to the dotted item
       # @return [Dendroid::GrmAnalysis::DottedItem]
       attr_reader :dotted_item
 
       # @return [Integer] the rank of the token that correspond to the start of the rule.
       attr_reader :origin
 
-      def_delegators :@dotted_item, :completed?, :expecting?, :next_symbol, :pre_scan?
+      # TODO: :predictor, :completer, :scanner
+      attr_accessor :algo
+
+      # @return [Array<WeakRef>] predecessors sorted by decreasing origin value
+      attr_accessor :predecessors
+
+      def_delegators :@dotted_item, :completed?, :expecting?, :next_symbol, :pre_scan?, :position, :prev_symbol, :rule
 
       # @param aDottedItem [Dendroid::GrmAnalysis::DottedItem]
       # @param origin [Integer]
       def initialize(aDottedItem, origin)
-        @dotted_item = aDottedItem
+        @dotted_item = WeakRef.new(aDottedItem)
         @origin = origin
+        @predecessors = []
       end
 
       # @return [Dendroid::Syntax::NonTerminal] the head of the production rule
@@ -42,6 +51,16 @@ module Dendroid
       def to_s
         "#{dotted_item} @ #{origin}"
       end
+
+      alias inspect to_s
+
+      def add_predecessor(pred)
+        if predecessors.size > 1 && pred.origin < predecessors[0].origin
+          predecessors.insert(2, WeakRef.new(pred))
+        else
+          predecessors.unshift(WeakRef.new(pred))
+        end
+      end
     end # class
   end # module
 end # module

data/lib/dendroid/recognizer/item_set.rb

@@ -15,10 +15,15 @@ module Dendroid
         @items = []
       end
 
-      # Add an Early item to the set
+      # Add an Earley item to the set if not yet present.
       # @param anItem [Recognizer::EItem]
+      # @return [Recognizer::EItem] the item in the set
       def add_item(anItem)
-        @items << anItem unless items.include? anItem
+        idx = items.find_index anItem
+        return items[idx] if idx
+
+        @items << anItem
+        anItem
       end
 
       # Find the items that expect a given grammar symbol

data/lib/dendroid/recognizer/recognizer.rb

@@ -5,7 +5,7 @@ require_relative 'e_item'
 require_relative 'chart'
 
 module Dendroid
-  # This module host classes needed to implement an Earley recognizer
+  # This module host classes needed to implement an Earley recognizer.
   module Recognizer
     # A recognizer determines whether the input text complies to the grammar (syntax) rules.
     # This class implements the Earley recognition algorithm.
@@ -16,6 +16,10 @@ module Dendroid
       # @return [Object]
       attr_reader :tokenizer
 
+      # rubocop: disable Metrics/AbcSize
+      # rubocop: disable Metrics/CyclomaticComplexity
+      # rubocop: disable Metrics/PerceivedComplexity
+
       # @param grammar [Dendroid::Syntax::Grammar]
       # @param tokenizer [Object]
       def initialize(grammar, tokenizer)
@@ -39,7 +43,7 @@ module Dendroid
       end
 
       # Run the Earley algorithm
-      # @param initial_token [Dednroid::Lexical::Token]
+      # @param initial_token [Dendroid::Lexical::Token]
       def earley_parse(initial_token)
         chart = new_chart
         tokens = [initial_token]
@@ -64,7 +68,8 @@ module Dendroid
           break unless advance
         end
 
-        determine_outcome(chart, tokens)
+        chart.tokens = tokens
+        determine_outcome(chart)
         chart
       end
 
@@ -76,7 +81,11 @@ module Dendroid
         prd = grm_analysis.grammar.nonterm2production[top_symbol]
         chart = Chart.new
         seed_items = prd.predicted_items
-        seed_items.each { |item| chart.seed_last_set(EItem.new(item, 0)) }
+        seed_items.each do |item|
+          entry = EItem.new(item, 0)
+          entry.algo = :predictor
+          chart.seed_last_set(entry)
+        end
 
         chart
       end
@@ -115,29 +124,28 @@ module Dendroid
       #   Assuming next symbol is a non-terminal
       #
       #   Error case: next actual token matches none of the expected tokens.
-      def predictor(chart, item, rank, tokens, mode, predicted_symbols)
+      def predictor(chart, item, rank, tokens, mode, _predicted_symbols)
         next_symbol = item.next_symbol
-        if mode == :genuine
-          predicted_symbols << Set.new if rank == predicted_symbols.size
-          predicted = predicted_symbols[rank]
-          return if predicted.include?(next_symbol)
-
-          predicted.add(next_symbol)
-        end
+        # if mode == :genuine
+        #   predicted_symbols << Set.new if rank == predicted_symbols.size
+        #   predicted = predicted_symbols[rank]
+        #   return if predicted.include?(next_symbol)
+        #
+        #   predicted.add(next_symbol)
+        # end
 
         curr_set = chart[rank]
         next_token = tokens[rank]
         prd = grm_analysis.symbol2production(next_symbol)
         entry_items = prd.predicted_items
+        added = []
         entry_items.each do |entry|
           member = entry.next_symbol
           if member&.terminal?
             next unless next_token
             next if (member.name != next_token.terminal) && mode == :genuine
           end
-
-          new_item = EItem.new(entry, rank)
-          curr_set.add_item(new_item)
+          added << add_item(curr_set, entry, rank, item, :predictor)
         end
         # Use trick from paper John Aycock and R. Nigel Horspool: "Practical Earley Parsing"
         return unless next_symbol.nullable?
@@ -145,8 +153,7 @@ module Dendroid
         next_item = grm_analysis.next_item(item.dotted_item)
         return unless next_item
 
-        new_item = EItem.new(next_item, item.origin)
-        curr_set.add_item(new_item)
+        add_item(curr_set, next_item, item.origin, nil, :predictor)
       end
 
       # procedure SCANNER((A → α•aβ, j), k, words)
@@ -161,8 +168,7 @@ module Dendroid
           new_rank = rank + 1
           chart.append_new_set if chart[new_rank].nil?
           next_dotted_item = grm_analysis.next_item(dit)
-          new_item = EItem.new(next_dotted_item, scan_item.origin)
-          chart[new_rank].add_item(new_item)
+          add_item(chart[new_rank], next_dotted_item, scan_item.origin, scan_item, :scanner)
           advance = true
         end
 
@@ -190,8 +196,7 @@ module Dendroid
             next if member.name != next_token.terminal
           end
 
-          new_item = EItem.new(return_item, call_item.origin)
-          curr_set.add_item(new_item)
+          add_item(curr_set, return_item, call_item.origin, item, :completer)
         end
       end
 
@@ -206,8 +211,18 @@ module Dendroid
         end
       end
 
-      def determine_outcome(chart, tokens)
+      def add_item(item_set, dotted_item, origin, predecessor, procedure)
+        new_item = EItem.new(dotted_item, origin)
+        added = item_set.add_item(new_item)
+        added.add_predecessor(predecessor) if predecessor
+        new_item.algo = procedure
+
+        added
+      end
+
+      def determine_outcome(chart)
         success = false
+        tokens = chart.tokens
         if chart.size == tokens.size + 1
           top_symbol = grm_analysis.grammar.start_symbol
           top_rule = grm_analysis.grammar.nonterm2production[top_symbol]
@@ -250,7 +265,7 @@ module Dendroid
 
       def expected_terminals(chart)
         last_set = chart.last
-        terminals = last_set.items.reduce([]) do |result, ent|
+        terminals = last_set.items.each_with_object([]) do |ent, result|
           result << ent.next_symbol if ent.pre_scan?
           result
         end
@@ -269,5 +284,9 @@ module Dendroid
         end
       end
     end # class
+
+    # rubocop: enable Metrics/AbcSize
+    # rubocop: enable Metrics/CyclomaticComplexity
+    # rubocop: enable Metrics/PerceivedComplexity
   end # module
 end # module

data/lib/dendroid/syntax/grammar.rb

@@ -39,6 +39,8 @@ module Dendroid
         add_terminals(terminals)
       end
 
+      # rubocop: disable Style/IfUnlessModifier
+
       # Add a rule to the grammar.
       # @param rule [Dendroid::Syntax::Rule]
       def add_rule(rule)
@@ -56,6 +58,8 @@ module Dendroid
         nonterm2production[rule.head] = rule
       end
 
+      # rubocop: enable Style/IfUnlessModifier
+
       # Return the start symbol for the language, that is,
       # the non-terminal symbol used to denote the top-level
       # construct of the language being defined.
@@ -211,7 +215,7 @@ module Dendroid
           backlog.subtract(to_remove)
         end
 
-        backlog.each { |i| rules[i].non_productive }
+        # backlog.each { |i| rules[i].non_productive }
         non_productive = symbols.reject(&:productive?)
         non_productive.each { |symb| symb.productive = false }
         non_productive