dendroid 0.0.12 → 0.2.00

data/lib/dendroid/parsing/parser.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require 'weakref'
+require_relative '../grm_dsl/base_grm_builder'
+require_relative '../utils/base_tokenizer'
+require_relative '../recognizer/recognizer'
+require_relative 'chart_walker'
+require_relative 'parse_tree_visitor'
+require_relative '../formatters/bracket_notation'
+require_relative '../formatters/ascii_tree'
+
+module Dendroid
+  module Parsing
+    class Parser
+    end # class
+  end # module
+end # module
+
+def grammar_l1
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Grammar inspired from Wikipedia entry on Earley parsing
+    declare_terminals('PLUS', 'STAR', 'INTEGER')
+    rule('p' => 's')
+    rule('s' => ['s PLUS m', 'm'])
+    rule('m' => ['m STAR t', 't'])
+    rule('t' => 'INTEGER')
+  end
+
+  builder.grammar
+end
+
+def grammar_l31
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Ambiguous arithmetical expression language
+    # This language is compatible with tokenizer L1
+    declare_terminals('PLUS', 'STAR', 'INTEGER')
+    rule('p' => 's')
+    rule('s' => ['s PLUS s', 's STAR s', 'INTEGER'])
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l1
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
+
+    scan_verbatim(['+', '*'])
+    scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
+  end
+end
+
+def retrieve_success_item(chart, grammar)
+  last_item_set = chart.item_sets.last
+  result = nil
+  last_item_set.items.reverse_each do |itm|
+    if itm.origin.zero? && itm.dotted_item.completed? && itm.dotted_item.rule.lhs == grammar.start_symbol
+      result = itm
+      break
+    end
+  end
+
+  result
+end
+
+def grammar_l10
+  # Grammar with left recursive rule
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    declare_terminals('a')
+
+    rule 'A' => ['A a', '']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l10
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'a' => :a })
+
+    scan_verbatim(['a'])
+  end
+end
+
+def grammar_l11
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Grammar with right-recursive rule
+    declare_terminals('a')
+
+    rule 'A' => ['a A', '']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l11
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'a' => :a })
+
+    scan_verbatim(['a'])
+  end
+end
+
+def grammar_l8
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # (based on grammar G2 from paper Masaru Tomita "An Efficient Context-Free Parsing Algorithm
+    #   for Natural Languages")
+    declare_terminals('x')
+
+    rule 'S' => ['S S', 'x']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l8
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'x' => :x })
+
+    scan_verbatim(['x'])
+  end
+end
+
+########################################
+# Entry point
+########################################
+kode = 3
+
+case kode
+when 0
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l1, tokenizer_l1)
+  chart = recognizer.run('2 + 3 * 4')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  # formatter = BracketNotation.new($stdout)
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 1
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l10, tokenizer_l10)
+  chart = recognizer.run('a a a a a')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 2
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l11, tokenizer_l11)
+  chart = recognizer.run('a a a a a')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 3
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l31, tokenizer_l1)
+  chart = recognizer.run('2 + 3 * 4')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 4
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l8, tokenizer_l8)
+  chart = recognizer.run('x x x x')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+end

data/lib/dendroid/parsing/terminal_node.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'parse_node'
+
+module Dendroid
+  module Parsing
+    class TerminalNode < ParseNode
+      # @return [Dendroid::Syntax::Terminal] Terminal symbol of matching token.
+      attr_reader :symbol
+
+      # @return [Dendroid::Lexical::Token] Matching input token object.
+      attr_reader :token
+
+      def initialize(sym, tok, rank)
+        super(rank, rank + 1)
+        @symbol = sym
+        @token = tok
+      end
+
+      # Part of the 'visitee' role in Visitor design pattern.
+      # @param aVisitor[ParseTreeVisitor] the visitor
+      def accept(aVisitor)
+        aVisitor.visit_terminal(self)
+      end
+
+      def to_s()
+        display_val = token.is_a?(Dendroid::Lexical::Literal) ? ": #{token.value}" : ''
+        "#{symbol.name}#{display_val} #{super}"
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/parsing/walk_progress.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative 'and_node'
+require_relative 'or_node'
+require_relative 'terminal_node'
+require_relative 'empty_rule_node'
+
+module Dendroid
+  module Parsing
+    class WalkProgress
+      attr_accessor :state
+      attr_accessor :curr_rank
+      attr_reader :curr_item
+      attr_accessor :predecessor
+      attr_reader :parents
+
+      def initialize(start_rank, start_item, parents)
+        @state = :New
+        @curr_rank = start_rank
+        @curr_item = start_item
+        @predecessor = nil
+        @parents = parents
+      end
+
+      # Factory method.
+      def initialize_copy(orig)
+        @state = orig.state
+        @curr_rank = orig.curr_rank
+        @curr_item = orig.curr_item
+        @predecessor = nil
+        @parents = orig.parents.dup
+      end
+
+      def fork(thePredecessor)
+        @state = :Forking
+        @predecessor = thePredecessor
+      end
+
+      def curr_item=(anEntry)
+        if anEntry.nil?
+          raise StandardError
+        else
+          @curr_item = anEntry
+        end
+      end
+
+      def add_node_empty(anEntry)
+        node_empty = EmptyRuleNode.new(anEntry, curr_rank)
+        add_child_node(node_empty)
+      end
+
+      # Add a terminal node for terminal at current rank as a child of last parent
+      def add_terminal_node(token)
+        @curr_rank -= 1
+        term_node = TerminalNode.new(curr_item.prev_symbol, token, curr_rank)
+        add_child_node(term_node)
+      end
+
+      # Add an AND node for given entry as a child of last parent
+      def push_and_node(anEntry)
+        node = ANDNode.new(anEntry, curr_rank)
+        raise StandardError unless anEntry.rule == node.rule # Fails
+        add_child_node(node)
+        parents.push(node)
+
+        node
+      end
+
+      def push_or_node(origin, arity)
+        node = OrNode.new(curr_item.prev_symbol, origin, curr_rank, arity)
+        add_child_node(node)
+        parents.push(node)
+
+        node
+      end
+
+      def add_child_node(aNode)
+        parents.last.add_child(aNode, curr_item.position - 1)
+        aNode
+      end
+
+      # Do the given EItems match one of the parent?
+      # Matching = corresponds to the same rule and range
+      # @return [Array<EItem>]
+      def match_parent?(entries, stop_at_first)
+        matching = []
+        min_origin = entries[0].origin
+        first_iteration = true
+        offset = 0
+
+        parents.reverse_each do |node|
+          if node.is_a?(OrNode)
+            offset += 1
+            next
+          end
+          entries.each do |ent|
+            if first_iteration
+              min_origin = ent.origin if ent.origin < min_origin
+            end
+            next unless node.match(ent)
+
+            matching << [ent, offset]
+            break if stop_at_first
+          end
+          first_iteration = false
+          break if stop_at_first && !matching.empty?
+
+          # Stop loop when parent.origin < min(entries.origin)
+          break if node.range[0] < min_origin
+          offset += 1
+        end
+
+        matching
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/recognizer/chart.rb

@@ -15,14 +15,17 @@ module Dendroid
       # @return [Array<Recognizer::ItemSet>] The array of item sets
       attr_reader :item_sets
 
+      # @return [Array<Dendroid::Lexical::Token>] The input tokens
+      attr_reader :tokens
+
       # @return [Boolean] Indicates whether the recognizer successfully processed the whole input
       attr_writer :success
 
       # @return [StandardError] The exception class in case of an error found by the recognizer
-      attr_accessor :failure_class
+      attr_reader :failure_class
 
       # @return [String] The error message
-      attr_accessor :failure_reason
+      attr_reader :failure_reason
 
       def_delegators :@item_sets, :[], :last, :size
 
@@ -45,11 +48,24 @@ module Dendroid
         item_sets.last.add_item(e_item)
       end
 
+      # @param input_tokens [Array<Dendroid::Lexical::Token>] The input tokens
+      def tokens=(input_tokens)
+        @tokens = input_tokens
+      end
+
       # Return true if the input text is valid according to the grammar.
       # @return [Boolean]
       def successful?
         @success
       end
+
+      # Set the error cause.
+      # @param exception_class [StandardError] Exception class
+      # @param message [String] Error message
+      def failure(exception_class, message)
+        @failure_class = exception_class
+        @failure_reason = message
+      end
     end # class
   end # module
 end # module

data/lib/dendroid/recognizer/e_item.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require 'weakref'
 
 module Dendroid
   module Recognizer
@@ -9,19 +10,27 @@ module Dendroid
     class EItem
       extend Forwardable
 
+      # (Weak) reference to the dotted item
       # @return [Dendroid::GrmAnalysis::DottedItem]
       attr_reader :dotted_item
 
       # @return [Integer] the rank of the token that correspond to the start of the rule.
       attr_reader :origin
 
-      def_delegators :@dotted_item, :completed?, :expecting?, :next_symbol, :pre_scan?
+      # TODO: :predictor, :completer, :scanner
+      attr_accessor :algo
+
+      # @return [Array<WeakRef>] predecessors sorted by decreasing origin value
+      attr_accessor :predecessors
+
+      def_delegators :@dotted_item, :completed?, :expecting?, :next_symbol, :pre_scan?, :position, :prev_symbol, :rule
 
       # @param aDottedItem [Dendroid::GrmAnalysis::DottedItem]
       # @param origin [Integer]
       def initialize(aDottedItem, origin)
-        @dotted_item = aDottedItem
+        @dotted_item = WeakRef.new(aDottedItem)
         @origin = origin
+        @predecessors = []
       end
 
       # @return [Dendroid::Syntax::NonTerminal] the head of the production rule
@@ -42,6 +51,16 @@ module Dendroid
       def to_s
         "#{dotted_item} @ #{origin}"
       end
+
+      alias inspect to_s
+
+      def add_predecessor(pred)
+        if predecessors.size > 1 && pred.origin < predecessors[0].origin
+          predecessors.insert(2, WeakRef.new(pred))
+        else
+          predecessors.unshift(WeakRef.new(pred))
+        end
+      end
     end # class
   end # module
 end # module

data/lib/dendroid/recognizer/item_set.rb

@@ -15,10 +15,15 @@ module Dendroid
         @items = []
       end
 
-      # Add an Early item to the set
+      # Add an Earley item to the set if not yet present.
       # @param anItem [Recognizer::EItem]
+      # @return [Recognizer::EItem] the item in the set
       def add_item(anItem)
-        @items << anItem unless items.include? anItem
+        idx = items.find_index anItem
+        return items[idx] if idx
+
+        @items << anItem
+        anItem
       end
 
       # Find the items that expect a given grammar symbol

data/lib/dendroid/recognizer/recognizer.rb

@@ -31,8 +31,7 @@ module Dendroid
         tok = tokenizer.next_token
         if tok.nil? && !grm_analysis.grammar.start_symbol.nullable?
           chart = new_chart
-          chart.failure_class = StandardError
-          chart.failure_reason = 'Error: Input may not be empty nor blank.'
+          chart.failure(StandardError, 'Error: Input may not be empty nor blank.')
           chart
         else
           earley_parse(tok)
@@ -40,7 +39,7 @@ module Dendroid
       end
 
       # Run the Earley algorithm
-      # @param initial_token [Dednroid::Lexical::Token]
+      # @param initial_token [Dendroid::Lexical::Token]
       def earley_parse(initial_token)
         chart = new_chart
         tokens = [initial_token]
@@ -62,10 +61,11 @@ module Dendroid
 
           rank += 1 if advance
           break if eos_reached && !advance
-          break if !advance
+          break unless advance
         end
 
-        determine_outcome(chart, tokens)
+        chart.tokens = tokens
+        determine_outcome(chart)
         chart
       end
 
@@ -74,12 +74,13 @@ module Dendroid
       def new_chart
         top_symbol = grm_analysis.grammar.start_symbol
 
-        # Reminder: there might be multiple rules for the start symbol
-        prods = grm_analysis.grammar.nonterm2productions[top_symbol]
+        prd = grm_analysis.grammar.nonterm2production[top_symbol]
         chart = Chart.new
-        prods.each do |prd|
-          seed_items = prd.predicted_items
-          seed_items.each { |item| chart.seed_last_set(EItem.new(item, 0)) }
+        seed_items = prd.predicted_items
+        seed_items.each do |item|
+          entry = EItem.new(item, 0)
+          entry.algo = :predictor
+          chart.seed_last_set(entry)
         end
 
         chart
@@ -103,12 +104,10 @@ module Dendroid
 
         if entry.completed?
           completer(chart, entry, rank, tokens, mode)
+        elsif entry.next_symbol.terminal?
+          advance = scanner(chart, entry, rank, tokens)
         else
-          if entry.next_symbol.terminal?
-            advance = scanner(chart, entry, rank, tokens)
-          else
-            predictor(chart, entry, rank, tokens, mode, predicted_symbols)
-          end
+          predictor(chart, entry, rank, tokens, mode, predicted_symbols)
         end
 
         advance
@@ -123,39 +122,36 @@ module Dendroid
       #   Error case: next actual token matches none of the expected tokens.
       def predictor(chart, item, rank, tokens, mode, predicted_symbols)
         next_symbol = item.next_symbol
-        if mode == :genuine
-          predicted_symbols << Set.new if rank == predicted_symbols.size
-          predicted = predicted_symbols[rank]
-          return if predicted.include?(next_symbol)
-
-          predicted.add(next_symbol)
-        end
+        # if mode == :genuine
+        #   predicted_symbols << Set.new if rank == predicted_symbols.size
+        #   predicted = predicted_symbols[rank]
+        #   return if predicted.include?(next_symbol)
+        #
+        #   predicted.add(next_symbol)
+        # end
 
-        prods = grm_analysis.symbol2productions[next_symbol]
         curr_set = chart[rank]
         next_token = tokens[rank]
-        prods.each do |prd|
-          entry_items = prd.predicted_items
-          entry_items.each do |entry|
-            member = entry.next_symbol
-            if member&.terminal?
-              next unless next_token
-              next if (member.name != next_token.terminal) && mode == :genuine
-            end
-
-            new_item = EItem.new(entry, rank)
-            curr_set.add_item(new_item)
+        prd = grm_analysis.symbol2production(next_symbol)
+        entry_items = prd.predicted_items
+        added = []
+        entry_items.each do |entry|
+          member = entry.next_symbol
+          if member&.terminal?
+            next unless next_token
+            next if (member.name != next_token.terminal) && mode == :genuine
           end
+          added << add_item(curr_set, entry, rank, item, :predictor)
         end
-
         # Use trick from paper John Aycock and R. Nigel Horspool: "Practical Earley Parsing"
-        if next_symbol.nullable?
-          next_item = grm_analysis.next_item(item.dotted_item)
-          if next_item
-            new_item = EItem.new(next_item, item.origin)
-            curr_set.add_item(new_item)
-          end
-        end
+        return unless next_symbol.nullable?
+
+        next_item = grm_analysis.next_item(item.dotted_item)
+        return unless next_item
+
+        special = add_item(curr_set, next_item, item.origin, nil, :predictor)
+        # special = add_item(curr_set, next_item, item.origin, added.shift, :predictor)
+        # added.each { |e|  special.add_predecessor(e) }
       end
 
       # procedure SCANNER((A → α•aβ, j), k, words)
@@ -170,8 +166,7 @@ module Dendroid
           new_rank = rank + 1
           chart.append_new_set if chart[new_rank].nil?
           next_dotted_item = grm_analysis.next_item(dit)
-          new_item = EItem.new(next_dotted_item, scan_item.origin)
-          chart[new_rank].add_item(new_item)
+          add_item(chart[new_rank], next_dotted_item, scan_item.origin, scan_item, :scanner)
           advance = true
         end
 
@@ -199,8 +194,7 @@ module Dendroid
             next if member.name != next_token.terminal
           end
 
-          new_item = EItem.new(return_item, call_item.origin)
-          curr_set.add_item(new_item)
+          add_item(curr_set, return_item, call_item.origin, item, :completer)
         end
       end
 
@@ -215,17 +209,25 @@ module Dendroid
         end
       end
 
-      def determine_outcome(chart, tokens)
+      def add_item(item_set, dotted_item, origin, predecessor, procedure)
+        new_item = EItem.new(dotted_item, origin)
+        added = item_set.add_item(new_item)
+        added.add_predecessor(predecessor) if predecessor
+        new_item.algo = procedure
+
+        added
+      end
+
+      def determine_outcome(chart)
         success = false
+        tokens = chart.tokens
         if chart.size == tokens.size + 1
           top_symbol = grm_analysis.grammar.start_symbol
-          top_rules = grm_analysis.grammar.nonterm2productions[top_symbol]
-          final_items = top_rules.reduce([]) do |items, rule|
-            items.concat(rule.reduce_items)
-          end
+          top_rule = grm_analysis.grammar.nonterm2production[top_symbol]
+          final_items = top_rule.reduce_items
           last_set = chart.item_sets.last
           last_set.each do |entry|
-            next if ((!entry.origin.zero?) || !final_items.include?(entry.dotted_item))
+            next if !entry.origin.zero? || !final_items.include?(entry.dotted_item)
 
             success = true
           end
@@ -239,39 +241,37 @@ module Dendroid
             offending_token = tokens[chart.size - 1]
             pos = offending_token.position
             (line, col) = [pos.lineno, pos.column]
-            last_set = chart.last
-            terminals = last_set.items.reduce([]) do |result, ent|
-              result << ent.next_symbol if ent.pre_scan?
-              result
-            end
-            terminals.uniq!
+            terminals = expected_terminals(chart)
             prefix = "Syntax error at or near token line #{line}, column #{col} >>>#{offending_token.source}<<<"
             expectation = terminals.size == 1 ? terminals[0].name.to_s : "one of: [#{terminals.map(&:name).join(', ')}]"
             err_msg = "#{prefix} Expected #{expectation}, found a #{offending_token.terminal} instead."
-            chart.failure_class = StandardError
-            chart.failure_reason = err_msg
+            chart.failure(StandardError, err_msg)
           elsif chart.size == tokens.size + 1
             # EOS unexpected...
             last_token = tokens.last
             pos = last_token.position
             (line, col) = [pos.lineno, pos.column]
-            last_set = chart.last
-            terminals = last_set.items.reduce([]) do |result, ent|
-              result << ent.next_symbol if ent.pre_scan?
-              result
-            end
-            terminals.uniq!
-
+            terminals = expected_terminals(chart)
             prefix = "Line #{line}, column #{col}: Premature end of input after '#{last_token.source}'"
             expectation = terminals.size == 1 ? terminals[0].name.to_s : "one of: [#{terminals.map(&:name).join(', ')}]"
             err_msg = "#{prefix}, expected: #{expectation}."
-            chart.failure_class = StandardError
-            chart.failure_reason = err_msg
+            chart.failure(StandardError, err_msg)
           end
         end
         chart.success = success
       end
 
+      def expected_terminals(chart)
+        last_set = chart.last
+        terminals = last_set.items.reduce([]) do |result, ent|
+          result << ent.next_symbol if ent.pre_scan?
+          result
+        end
+        terminals.uniq!
+
+        terminals
+      end
+
       def replay_last_set(chart, tokens)
         rank = chart.size - 1
         seed_set(chart, rank) # Re-initialize last set with scan entries