dendroid 0.1.00 → 0.2.00

data/lib/dendroid/parsing/parser.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require 'weakref'
+require_relative '../grm_dsl/base_grm_builder'
+require_relative '../utils/base_tokenizer'
+require_relative '../recognizer/recognizer'
+require_relative 'chart_walker'
+require_relative 'parse_tree_visitor'
+require_relative '../formatters/bracket_notation'
+require_relative '../formatters/ascii_tree'
+
+module Dendroid
+  module Parsing
+    class Parser
+    end # class
+  end # module
+end # module
+
+def grammar_l1
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Grammar inspired from Wikipedia entry on Earley parsing
+    declare_terminals('PLUS', 'STAR', 'INTEGER')
+    rule('p' => 's')
+    rule('s' => ['s PLUS m', 'm'])
+    rule('m' => ['m STAR t', 't'])
+    rule('t' => 'INTEGER')
+  end
+
+  builder.grammar
+end
+
+def grammar_l31
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Ambiguous arithmetical expression language
+    # This language is compatible with tokenizer L1
+    declare_terminals('PLUS', 'STAR', 'INTEGER')
+    rule('p' => 's')
+    rule('s' => ['s PLUS s', 's STAR s', 'INTEGER'])
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l1
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
+
+    scan_verbatim(['+', '*'])
+    scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
+  end
+end
+
+def retrieve_success_item(chart, grammar)
+  last_item_set = chart.item_sets.last
+  result = nil
+  last_item_set.items.reverse_each do |itm|
+    if itm.origin.zero? && itm.dotted_item.completed? && itm.dotted_item.rule.lhs == grammar.start_symbol
+      result = itm
+      break
+    end
+  end
+
+  result
+end
+
+def grammar_l10
+  # Grammar with left recursive rule
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    declare_terminals('a')
+
+    rule 'A' => ['A a', '']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l10
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'a' => :a })
+
+    scan_verbatim(['a'])
+  end
+end
+
+def grammar_l11
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # Grammar with right-recursive rule
+    declare_terminals('a')
+
+    rule 'A' => ['a A', '']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l11
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'a' => :a })
+
+    scan_verbatim(['a'])
+  end
+end
+
+def grammar_l8
+  builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+    # (based on grammar G2 from paper Masaru Tomita "An Efficient Context-Free Parsing Algorithm
+    #   for Natural Languages")
+    declare_terminals('x')
+
+    rule 'S' => ['S S', 'x']
+  end
+
+  builder.grammar
+end
+
+def tokenizer_l8
+  Dendroid::Utils::BaseTokenizer.new do
+    map_verbatim2terminal({ 'x' => :x })
+
+    scan_verbatim(['x'])
+  end
+end
+
+########################################
+# Entry point
+########################################
+kode = 3
+
+case kode
+when 0
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l1, tokenizer_l1)
+  chart = recognizer.run('2 + 3 * 4')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  # formatter = BracketNotation.new($stdout)
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 1
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l10, tokenizer_l10)
+  chart = recognizer.run('a a a a a')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 2
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l11, tokenizer_l11)
+  chart = recognizer.run('a a a a a')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 3
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l31, tokenizer_l1)
+  chart = recognizer.run('2 + 3 * 4')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+
+when 4
+  recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l8, tokenizer_l8)
+  chart = recognizer.run('x x x x')
+  succ_item =  retrieve_success_item(chart, recognizer.grm_analysis.grammar)
+  walker = Dendroid::Parsing::ChartWalker.new(chart)
+  root = walker.walk(succ_item)
+
+  formatter = Asciitree.new($stdout)
+  visitor = ParseTreeVisitor.new(root)
+  formatter.render(visitor)
+end

data/lib/dendroid/parsing/terminal_node.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'parse_node'
+
+module Dendroid
+  module Parsing
+    class TerminalNode < ParseNode
+      # @return [Dendroid::Syntax::Terminal] Terminal symbol of matching token.
+      attr_reader :symbol
+
+      # @return [Dendroid::Lexical::Token] Matching input token object.
+      attr_reader :token
+
+      def initialize(sym, tok, rank)
+        super(rank, rank + 1)
+        @symbol = sym
+        @token = tok
+      end
+
+      # Part of the 'visitee' role in Visitor design pattern.
+      # @param aVisitor[ParseTreeVisitor] the visitor
+      def accept(aVisitor)
+        aVisitor.visit_terminal(self)
+      end
+
+      def to_s()
+        display_val = token.is_a?(Dendroid::Lexical::Literal) ? ": #{token.value}" : ''
+        "#{symbol.name}#{display_val} #{super}"
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/parsing/walk_progress.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative 'and_node'
+require_relative 'or_node'
+require_relative 'terminal_node'
+require_relative 'empty_rule_node'
+
+module Dendroid
+  module Parsing
+    class WalkProgress
+      attr_accessor :state
+      attr_accessor :curr_rank
+      attr_reader :curr_item
+      attr_accessor :predecessor
+      attr_reader :parents
+
+      def initialize(start_rank, start_item, parents)
+        @state = :New
+        @curr_rank = start_rank
+        @curr_item = start_item
+        @predecessor = nil
+        @parents = parents
+      end
+
+      # Factory method.
+      def initialize_copy(orig)
+        @state = orig.state
+        @curr_rank = orig.curr_rank
+        @curr_item = orig.curr_item
+        @predecessor = nil
+        @parents = orig.parents.dup
+      end
+
+      def fork(thePredecessor)
+        @state = :Forking
+        @predecessor = thePredecessor
+      end
+
+      def curr_item=(anEntry)
+        if anEntry.nil?
+          raise StandardError
+        else
+          @curr_item = anEntry
+        end
+      end
+
+      def add_node_empty(anEntry)
+        node_empty = EmptyRuleNode.new(anEntry, curr_rank)
+        add_child_node(node_empty)
+      end
+
+      # Add a terminal node for terminal at current rank as a child of last parent
+      def add_terminal_node(token)
+        @curr_rank -= 1
+        term_node = TerminalNode.new(curr_item.prev_symbol, token, curr_rank)
+        add_child_node(term_node)
+      end
+
+      # Add an AND node for given entry as a child of last parent
+      def push_and_node(anEntry)
+        node = ANDNode.new(anEntry, curr_rank)
+        raise StandardError unless anEntry.rule == node.rule # Fails
+        add_child_node(node)
+        parents.push(node)
+
+        node
+      end
+
+      def push_or_node(origin, arity)
+        node = OrNode.new(curr_item.prev_symbol, origin, curr_rank, arity)
+        add_child_node(node)
+        parents.push(node)
+
+        node
+      end
+
+      def add_child_node(aNode)
+        parents.last.add_child(aNode, curr_item.position - 1)
+        aNode
+      end
+
+      # Do the given EItems match one of the parent?
+      # Matching = corresponds to the same rule and range
+      # @return [Array<EItem>]
+      def match_parent?(entries, stop_at_first)
+        matching = []
+        min_origin = entries[0].origin
+        first_iteration = true
+        offset = 0
+
+        parents.reverse_each do |node|
+          if node.is_a?(OrNode)
+            offset += 1
+            next
+          end
+          entries.each do |ent|
+            if first_iteration
+              min_origin = ent.origin if ent.origin < min_origin
+            end
+            next unless node.match(ent)
+
+            matching << [ent, offset]
+            break if stop_at_first
+          end
+          first_iteration = false
+          break if stop_at_first && !matching.empty?
+
+          # Stop loop when parent.origin < min(entries.origin)
+          break if node.range[0] < min_origin
+          offset += 1
+        end
+
+        matching
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/recognizer/chart.rb

@@ -15,6 +15,9 @@ module Dendroid
       # @return [Array<Recognizer::ItemSet>] The array of item sets
       attr_reader :item_sets
 
+      # @return [Array<Dendroid::Lexical::Token>] The input tokens
+      attr_reader :tokens
+
       # @return [Boolean] Indicates whether the recognizer successfully processed the whole input
       attr_writer :success
 
@@ -45,6 +48,11 @@ module Dendroid
         item_sets.last.add_item(e_item)
       end
 
+      # @param input_tokens [Array<Dendroid::Lexical::Token>] The input tokens
+      def tokens=(input_tokens)
+        @tokens = input_tokens
+      end
+
       # Return true if the input text is valid according to the grammar.
       # @return [Boolean]
       def successful?

data/lib/dendroid/recognizer/e_item.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'forwardable'
+require 'weakref'
 
 module Dendroid
   module Recognizer
@@ -9,19 +10,27 @@ module Dendroid
     class EItem
       extend Forwardable
 
+      # (Weak) reference to the dotted item
       # @return [Dendroid::GrmAnalysis::DottedItem]
       attr_reader :dotted_item
 
       # @return [Integer] the rank of the token that correspond to the start of the rule.
       attr_reader :origin
 
-      def_delegators :@dotted_item, :completed?, :expecting?, :next_symbol, :pre_scan?
+      # TODO: :predictor, :completer, :scanner
+      attr_accessor :algo
+
+      # @return [Array<WeakRef>] predecessors sorted by decreasing origin value
+      attr_accessor :predecessors
+
+      def_delegators :@dotted_item, :completed?, :expecting?, :next_symbol, :pre_scan?, :position, :prev_symbol, :rule
 
       # @param aDottedItem [Dendroid::GrmAnalysis::DottedItem]
       # @param origin [Integer]
       def initialize(aDottedItem, origin)
-        @dotted_item = aDottedItem
+        @dotted_item = WeakRef.new(aDottedItem)
         @origin = origin
+        @predecessors = []
       end
 
       # @return [Dendroid::Syntax::NonTerminal] the head of the production rule
@@ -42,6 +51,16 @@ module Dendroid
       def to_s
         "#{dotted_item} @ #{origin}"
       end
+
+      alias inspect to_s
+
+      def add_predecessor(pred)
+        if predecessors.size > 1 && pred.origin < predecessors[0].origin
+          predecessors.insert(2, WeakRef.new(pred))
+        else
+          predecessors.unshift(WeakRef.new(pred))
+        end
+      end
     end # class
   end # module
 end # module

data/lib/dendroid/recognizer/item_set.rb

@@ -15,10 +15,15 @@ module Dendroid
         @items = []
       end
 
-      # Add an Early item to the set
+      # Add an Earley item to the set if not yet present.
       # @param anItem [Recognizer::EItem]
+      # @return [Recognizer::EItem] the item in the set
       def add_item(anItem)
-        @items << anItem unless items.include? anItem
+        idx = items.find_index anItem
+        return items[idx] if idx
+
+        @items << anItem
+        anItem
       end
 
       # Find the items that expect a given grammar symbol

data/lib/dendroid/recognizer/recognizer.rb

@@ -39,7 +39,7 @@ module Dendroid
       end
 
       # Run the Earley algorithm
-      # @param initial_token [Dednroid::Lexical::Token]
+      # @param initial_token [Dendroid::Lexical::Token]
       def earley_parse(initial_token)
         chart = new_chart
         tokens = [initial_token]
@@ -64,7 +64,8 @@ module Dendroid
           break unless advance
         end
 
-        determine_outcome(chart, tokens)
+        chart.tokens = tokens
+        determine_outcome(chart)
         chart
       end
 
@@ -76,7 +77,11 @@ module Dendroid
         prd = grm_analysis.grammar.nonterm2production[top_symbol]
         chart = Chart.new
         seed_items = prd.predicted_items
-        seed_items.each { |item| chart.seed_last_set(EItem.new(item, 0)) }
+        seed_items.each do |item|
+          entry = EItem.new(item, 0)
+          entry.algo = :predictor
+          chart.seed_last_set(entry)
+        end
 
         chart
       end
@@ -117,27 +122,26 @@ module Dendroid
       #   Error case: next actual token matches none of the expected tokens.
       def predictor(chart, item, rank, tokens, mode, predicted_symbols)
         next_symbol = item.next_symbol
-        if mode == :genuine
-          predicted_symbols << Set.new if rank == predicted_symbols.size
-          predicted = predicted_symbols[rank]
-          return if predicted.include?(next_symbol)
-
-          predicted.add(next_symbol)
-        end
+        # if mode == :genuine
+        #   predicted_symbols << Set.new if rank == predicted_symbols.size
+        #   predicted = predicted_symbols[rank]
+        #   return if predicted.include?(next_symbol)
+        #
+        #   predicted.add(next_symbol)
+        # end
 
         curr_set = chart[rank]
         next_token = tokens[rank]
         prd = grm_analysis.symbol2production(next_symbol)
         entry_items = prd.predicted_items
+        added = []
         entry_items.each do |entry|
           member = entry.next_symbol
           if member&.terminal?
             next unless next_token
             next if (member.name != next_token.terminal) && mode == :genuine
           end
-
-          new_item = EItem.new(entry, rank)
-          curr_set.add_item(new_item)
+          added << add_item(curr_set, entry, rank, item, :predictor)
         end
         # Use trick from paper John Aycock and R. Nigel Horspool: "Practical Earley Parsing"
         return unless next_symbol.nullable?
@@ -145,8 +149,9 @@ module Dendroid
         next_item = grm_analysis.next_item(item.dotted_item)
         return unless next_item
 
-        new_item = EItem.new(next_item, item.origin)
-        curr_set.add_item(new_item)
+        special = add_item(curr_set, next_item, item.origin, nil, :predictor)
+        # special = add_item(curr_set, next_item, item.origin, added.shift, :predictor)
+        # added.each { |e|  special.add_predecessor(e) }
       end
 
       # procedure SCANNER((A → α•aβ, j), k, words)
@@ -161,8 +166,7 @@ module Dendroid
           new_rank = rank + 1
           chart.append_new_set if chart[new_rank].nil?
           next_dotted_item = grm_analysis.next_item(dit)
-          new_item = EItem.new(next_dotted_item, scan_item.origin)
-          chart[new_rank].add_item(new_item)
+          add_item(chart[new_rank], next_dotted_item, scan_item.origin, scan_item, :scanner)
           advance = true
         end
 
@@ -190,8 +194,7 @@ module Dendroid
             next if member.name != next_token.terminal
           end
 
-          new_item = EItem.new(return_item, call_item.origin)
-          curr_set.add_item(new_item)
+          add_item(curr_set, return_item, call_item.origin, item, :completer)
         end
       end
 
@@ -206,8 +209,18 @@ module Dendroid
         end
       end
 
-      def determine_outcome(chart, tokens)
+      def add_item(item_set, dotted_item, origin, predecessor, procedure)
+        new_item = EItem.new(dotted_item, origin)
+        added = item_set.add_item(new_item)
+        added.add_predecessor(predecessor) if predecessor
+        new_item.algo = procedure
+
+        added
+      end
+
+      def determine_outcome(chart)
         success = false
+        tokens = chart.tokens
         if chart.size == tokens.size + 1
           top_symbol = grm_analysis.grammar.start_symbol
           top_rule = grm_analysis.grammar.nonterm2production[top_symbol]

data/lib/dendroid/syntax/grammar.rb

@@ -211,7 +211,7 @@ module Dendroid
           backlog.subtract(to_remove)
         end
 
-        backlog.each { |i| rules[i].non_productive }
+        # backlog.each { |i| rules[i].non_productive }
         non_productive = symbols.reject(&:productive?)
         non_productive.each { |symb| symb.productive = false }
         non_productive

data/lib/dendroid/syntax/rule.rb

@@ -2,28 +2,60 @@
 
 module Dendroid
   module Syntax
-    # In a context-free grammar, a rule has its left-hand side (LHS)
-    # that consists solely of one non-terminal symbol.
-    # and the right-hand side (RHS) consists of one or more sequence of symbols.
-    # The symbols in RHS can be either terminal or non-terminal symbols.
-    # The rule stipulates that the LHS is equivalent to the RHS,
-    # in other words every occurrence of the LHS can be substituted to
-    # corresponding RHS.
+    # A specialization of the Rule class.
+    # A choice is a rule with multiple rhs
     class Rule
       # @return [Dendroid::Syntax::NonTerminal] The left-hand side of the rule.
       attr_reader :head
       alias lhs head
 
-      # Create a Rule instance.
-      # @param lhs [Dendroid::Syntax::NonTerminal] The left-hand side of the rule.
-      def initialize(lhs)
-        @head = valid_head(lhs)
+      # @return [Array<Dendroid::Syntax::SymbolSeq>]
+      attr_reader :alternatives
+
+      # Create a Choice instance.
+      # @param theLhs [Dendroid::Syntax::NonTerminal] The left-hand side of the rule.
+      # @param alt [Array<Dendroid::Syntax::SymbolSeq>] the alternatives (each as a sequence of symbols).
+      def initialize(theLhs, alt)
+        @head = valid_head(theLhs)
+        @alternatives = valid_alternatives(alt)
       end
 
-      # Return the text representation of the rule
+      # Return the text representation of the choice
       # @return [String]
       def to_s
-        head.to_s
+        "#{head} => #{alternatives.join(' | ')}"
+      end
+
+      # Predicate method to check whether the choice rule body is productive.
+      # It is productive when at least one of its alternative is productive.
+      # @return [Boolean]
+      def productive?
+        productive_alts = alternatives.select(&:productive?)
+        return false if productive_alts.empty?
+
+        @productive = Set.new(productive_alts)
+        head.productive = true
+      end
+
+      # Predicate method to check whether the rule has at least one empty alternative.
+      # @return [Boolean]
+      def empty?
+        alternatives.any?(&:empty?)
+      end
+
+      # Returns an array with the symbol sequence of its alternatives
+      # @return [Array<Dendroid::Syntax::SymbolSeq>]
+      def rhs
+        alternatives
+      end
+
+      # Equality operator
+      # Two production rules are equal when their head and alternatives are equal.
+      # @return [Boolean]
+      def ==(other)
+        return true if equal?(other)
+
+        (head == other.head) && (alternatives == other.alternatives)
       end
 
       # The set of all grammar symbols that occur in the rhs.
@@ -70,6 +102,32 @@ module Dendroid
 
         lhs
       end
+
+      def valid_alternatives(alt)
+        raise StandardError, "Expecting an Array, found a #{rhs.class} instead." unless alt.is_a?(Array)
+
+        if alt.size.zero?
+          # A choice must have at least two alternatives
+          raise StandardError, "The choice for `#{head}` must have at least one alternative."
+        end
+
+        # Verify that each array element is a valid symbol sequence
+        alt.each { |elem| valid_sequence(elem) }
+
+        # Fail when duplicate rhs found
+        alt_texts = alt.map(&:to_s)
+        no_duplicate = alt_texts.uniq
+        if alt_texts.size > no_duplicate.size
+          alt_texts.each_with_index do |str, i|
+            next if str == no_duplicate[i]
+
+            err_msg = "Duplicate alternatives: #{head} => #{alt_texts[i]}"
+            raise StandardError, err_msg
+          end
+        end
+
+        alt
+      end
     end # class
   end # module
 end # module