dendroid 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e94a9721000d8e9a45184107ec3488a3dad0c92337507c6753d4cbfa973400fd
-  data.tar.gz: 03bbe70d73b5e12de42b5dca4f008f1850fb9efcc4272115e5123c7397984f4a
+  metadata.gz: a762fb52e0c8ff6116f41f481b5cb7346b99050d90f77c4e133da37a31dda3b3
+  data.tar.gz: ace142c5221c038eedaab8ea9882a3fbffa34cb424be08083ee722e9e5d8c6ca
 SHA512:
-  metadata.gz: 8dde1d9509f2d713db092e5730a1c37f1aaab2a1a58b45cb46c3e28011e672d6c27d9e630262d96a5e5597de35b9278aab8b6e2b87934bf4840e024eee0d6bea
-  data.tar.gz: d927b08bf68f6c2a128c92c4724a533991799acf207cd13c381d4abea4cbbf8b340de17771c0a41c351bed3a0d6ac2cd523990d1c0bb699824ce14b9ccd97fe6
+  metadata.gz: 2746b3e1cd03e07e0045d7e41b54ed4c8a2f003df89ea45166fa0b66013f37aa9c1cf0044adc71bbe75d5958d99334b7f612d15c116954fe451f8e1e6756bfaa
+  data.tar.gz: 4917fd27358d80719b722c14a6df8636917ad827dfc29fb8201920b068393dba09dd2b9750d8b5946031a1d821bd6922c84dd27572b4a635e709faa4bfe31ceb

data/.rubocop.yml

@@ -2,9 +2,21 @@ Layout/EndOfLine:
   Enabled: true
   EnforcedStyle: lf
 
+Metrics/AbcSize:
+  Enabled: true
+  Max: 25
+
 Metrics/BlockLength:
   Enabled: true
-  Max: 50
+  Max: 75
+
+Metrics/ClassLength:
+  Enabled: true
+  Max: 200
+
+Metrics/MethodLength:
+  Enabled: true
+  Max: 20
 
 Naming/MethodParameterName:
   Enabled: false

data/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## [Unreleased]
 
+
+## [0.0.6] - 2023-10-30
+### Added
+- Class `Grammar` and its spec file
+
 ## [0.0.5] - 2023-10-28
 ### Added
 - Class `Choice` and its spec file

data/lib/dendroid/syntax/grammar.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Dendroid
+  module Syntax
+    # A grammar specifies the syntax of a language.
+    #   Formally, a grammar has:
+    #   * One start symbol,
+    #   * One or more other production rules,
+    #   * Each production has a rhs that is a sequence of grammar symbols.
+    #   * Grammar symbols are categorized into:
+    #     -terminal symbols
+    #     -non-terminal symbols
+    class Grammar
+      # The list of grammar symbols in the language.
+      # @return [Array<Dendroid::Syntax::GrmSymbol>] The terminal and non-terminal symbols.
+      attr_reader :symbols
+
+      # The list of production rules for the language.
+      # @return [Array<Dendroid::Syntax::Rule>] Array of rules for the grammar.
+      attr_reader :rules
+
+      # A Hash that maps symbol names to their grammar symbols
+      # @return [Hash{String => Dendroid::Syntax::GrmSymbol}]
+      attr_reader :name2symbol
+
+      # TODO: make nonterminal - rules one-to-one
+      # A Hash that maps symbol names to their grammar symbols
+      # @return [Hash{Dendroid::Syntax::GrmSymbol => Dendroid::Syntax::Rule}]
+      attr_reader :nonterm2productions
+
+      # Constructor.
+      # @param terminals [Array<Dendroid::Syntax::Terminal>]
+      def initialize(terminals)
+        @symbols = []
+        @name2symbol = {}
+        add_terminals(terminals)
+      end
+
+      # Add a rule to the grammar
+      # @param rule [Dendroid::Syntax::Rule]
+      def add_rule(rule)
+        if @rules.nil?
+          @rules = []
+          @nonterm2productions = {}
+        end
+        # TODO: add test for duplicate productions
+        if nonterm2productions[rule.head]&.include? rule
+          raise StandardError, "Production rule '#{production}' appears more than once in the grammar."
+        end
+        add_symbol(rule.head)
+        rule.nonterminals.each { |nonterm| add_symbol(nonterm) }
+        rules << rule
+        nonterm2productions[rule.head] = [] unless nonterm2productions.include? rule.head
+        nonterm2productions[rule.head] << rule
+      end
+
+      # Return the start symbol for the language
+      # @return [Dendroid::Syntax::NonTerminal]
+      def start_symbol
+        rules.first.lhs
+      end
+
+      # A event method to notify the grammar that all grammar rules
+      # have been entered. The grammar, in turn, reacts by validating the
+      # production rules.
+      def complete!
+        validate
+        analyze
+      end
+
+      private
+
+      # rubocop: disable Metrics/AbcSize
+      # rubocop: disable Metrics/BlockNesting
+      # rubocop: disable Metrics/MethodLength
+      # rubocop: disable Metrics/PerceivedComplexity
+      def add_terminals(terminals)
+        terminals.each { |term| add_symbol(term) }
+      end
+
+      def add_symbol(symb)
+        return if name2symbol.include? symb.name
+
+        symbols.push(symb)
+        name2symbol[symb.name] = symb
+        name2symbol[symb.name.to_s] = symb
+      end
+
+      def validate
+        at_least_one_terminal
+        are_terminals_referenced?
+        are_nonterminals_rewritten?
+        are_symbols_productive?
+        are_symbols_reachable?
+      end
+
+      def analyze
+        mark_nullable_symbols
+      end
+
+      # Does the grammar contain at least one terminal symbol?
+      def at_least_one_terminal
+        found = symbols.any?(&:terminal?)
+
+        unless found
+          err_msg = "Grammar doesn't contain any terminal symbol."
+          raise StandardError, err_msg
+        end
+      end
+
+      # Does every terminal symbol appear at least once
+      # in a rhs of a production rule?
+      def are_terminals_referenced?
+        all_terminals = Set.new(symbols.select(&:terminal?))
+        terms_in_rhs = rules.reduce(Set.new) do |collected, prd|
+          found = prd.terminals
+          collected.merge(found)
+        end
+        check_ok = all_terminals == terms_in_rhs
+        unless check_ok
+          unused_terms = all_terminals.difference(terms_in_rhs)
+          text = unused_terms.map(&:name).join("', '")
+          err_msg = "Terminal symbols '#{text}' never appear in production rules."
+          raise StandardError, err_msg
+        end
+
+        check_ok
+      end
+
+      def are_nonterminals_rewritten?
+        all_nonterminals = Set.new(symbols.reject(&:terminal?))
+
+        symbs_in_lhs = rules.reduce(Set.new) do |collected, prd|
+          collected.add(prd.head)
+        end
+        check_ok = all_nonterminals == symbs_in_lhs
+        unless check_ok
+          undefined_nterms = all_nonterminals.difference(symbs_in_lhs)
+          text = undefined_nterms.map(&:name).join("', '")
+          err_msg = "Non-terminal symbols '#{text}' never appear in head of any production rule."
+          raise StandardError, err_msg
+        end
+
+        check_ok
+      end
+
+      def are_symbols_reachable?
+        unreachable = unreachable_symbols
+        return true if unreachable.empty?
+
+        text = unreachable.to_a.map(&:name).join("', '")
+        err_msg = "Symbols '#{text}' are unreachable from start symbol."
+        raise StandardError, err_msg
+      end
+
+      def are_symbols_productive?
+        non_productive = mark_non_productive_symbols
+        return true if non_productive.empty?
+
+        text = non_productive.to_a.map(&:name).join("', '")
+        err_msg = "Symbols '#{text}' are non-productive."
+        raise StandardError, err_msg
+      end
+
+      # Are all symbols reachable from start symbol?
+      def unreachable_symbols
+        backlog = [start_symbol]
+        set_reachable = Set.new(backlog.dup)
+
+        begin
+          reachable_sym = backlog.pop
+          prods = nonterm2productions[reachable_sym]
+          prods.each do |prd|
+            # prd.body.members.each do |member|
+            prd.rhs_symbols.each do |member|
+              unless member.terminal? || set_reachable.include?(member)
+                backlog.push(member)
+              end
+              set_reachable.add(member)
+            end
+          end
+        end until backlog.empty?
+
+        all_symbols = Set.new(symbols)
+        unreachable = all_symbols - set_reachable
+      end
+
+      def mark_non_productive_symbols
+        prod_count = rules.size
+        backlog = Set.new(0...prod_count)
+        rules.each_with_index do |prd, i|
+          backlog.delete(i) if prd.productive?
+        end
+        until backlog.empty?
+          size_before = backlog.size
+          to_remove = []
+          backlog.each do |i|
+            prd = rules[i]
+            to_remove << i if prd.productive?
+          end
+          break if to_remove.empty?
+
+          backlog.subtract(to_remove)
+        end
+
+        backlog.each { |i| rules[i].non_productive }
+        non_productive = symbols.reject(&:productive?)
+        non_productive.each { |symb| symb.productive = false }
+        non_productive
+      end
+
+      def mark_nullable_symbols
+        nullable_found = false
+        sym2seqs = {}
+
+        nonterm2productions.each_pair do |sym, prods|
+          if prods.any?(&:empty?)
+            sym.nullable = nullable_found = true
+          else
+            sym2seqs[sym] = prods.map(&:rhs).flatten
+          end
+        end
+
+        if nullable_found
+          backlog = {} # { SymbolSequence => [Integer, Symbol] }
+          sym2seqs.each do |sym, seqs|
+            seqs.each { |sq| backlog[sq] = [0, sym] }
+          end
+
+          begin
+            seqs_done = []
+            backlog.each_pair do |sq, (elem_index, lhs)|
+              member = sq.members[elem_index]
+              if member.terminal?
+                seqs_done << sq # stop with this sequence: it is non-nullable
+                backlog[sq] = [-1, lhs]
+              elsif member.nullable?
+                if elem_index == sq.size - 1
+                  seqs_done << sq # end of sequence reached...
+                  backlog[sq] = [-1, lhs]
+                  lhs.nullable = true
+                else
+                  backlog[sq] = [elem_index + 1, lhs]
+                end
+              end
+            end
+            seqs_done.each do |sq|
+              if backlog.include? sq
+                (_, lhs) = backlog[sq]
+                if lhs.nullable?
+                  to_drop = sym2seqs[lhs]
+                  to_drop.each { |seq| backlog.delete(seq) }
+                else
+                  backlog.delete(sq)
+                end
+              end
+            end
+          end until backlog.empty? || seqs_done.empty?
+        end
+
+        symbols.each do |sym|
+          next if sym.terminal?
+
+          sym.nullable = false if sym.nullable.nil?
+        end
+      end
+      # rubocop: enable Metrics/AbcSize
+      # rubocop: enable Metrics/BlockNesting
+      # rubocop: enable Metrics/MethodLength
+      # rubocop: enable Metrics/PerceivedComplexity
+    end # class
+  end # module
+end # module

data/spec/dendroid/syntax/grammar_spec.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require_relative '..\..\spec_helper'
+require_relative '..\..\..\lib\dendroid\syntax\terminal'
+require_relative '..\..\..\lib\dendroid\syntax\non_terminal'
+require_relative '..\..\..\lib\dendroid\syntax\symbol_seq'
+require_relative '..\..\..\lib\dendroid\syntax\production'
+require_relative '..\..\..\lib\dendroid\syntax\choice'
+require_relative '..\..\..\lib\dendroid\syntax\grammar'
+
+describe Dendroid::Syntax::Grammar do
+  let(:int_symb) { build_terminal('INTEGER') }
+  let(:plus_symb) { build_terminal('PLUS') }
+  let(:star_symb) { build_terminal('STAR') }
+  let(:p_symb) { build_nonterminal('p') }
+  let(:s_symb) { build_nonterminal('s') }
+  let(:m_symb) { build_nonterminal('m') }
+  let(:t_symb) { build_nonterminal('t') }
+  let(:all_terminals) { [int_symb, plus_symb, star_symb] }
+
+  subject { described_class.new(all_terminals) }
+
+  def build_terminal(name)
+    Dendroid::Syntax::Terminal.new(name)
+  end
+
+  def build_nonterminal(name)
+    Dendroid::Syntax::NonTerminal.new(name)
+  end
+
+  def build_symbol_seq(symbols)
+    Dendroid::Syntax::SymbolSeq.new(symbols)
+  end
+
+  def build_production(lhs, symbols)
+    Dendroid::Syntax::Production.new(lhs, build_symbol_seq(symbols))
+  end
+
+  def build_choice(lhs, sequences)
+    Dendroid::Syntax::Choice.new(lhs, sequences.map { |arr| build_symbol_seq(arr) })
+  end
+
+  # rubocop: disable Metrics/AbcSize
+  def build_all_rules
+    rule1 = build_production(p_symb, [s_symb]) # p => s
+    rule2 = build_choice(s_symb, [[s_symb, plus_symb, m_symb], [m_symb]]) # s => s + m | m
+    rule3 = build_choice(m_symb, [[m_symb, star_symb, t_symb], [t_symb]]) # m => m * t
+    rule4 = build_production(t_symb, [int_symb]) # t => INTEGER
+    [rule1, rule2, rule3, rule4]
+  end
+  # rubocop: enable Metrics/AbcSize
+
+  context 'Initialization:' do
+    it 'is initialized with an array of terminal symbols' do
+      expect { described_class.new(all_terminals) }.not_to raise_error
+    end
+
+    it 'knows its terminal symbols' do
+      expect(subject.symbols).to eq(all_terminals)
+    end
+
+    it 'ignores about productions after initialization' do
+      expect(subject.rules).to be_nil
+    end
+
+    it 'maps a terminal name to one GrmSymbol object' do
+      expect(subject.name2symbol.values.uniq.size).to eq(all_terminals.size)
+      expect(subject.name2symbol.values.size).to eq(2 * all_terminals.size)
+      expect(subject.name2symbol[:PLUS]).to eq(plus_symb)
+      expect(subject.name2symbol['PLUS']).to eq(plus_symb)
+    end
+  end # context
+
+  context 'Adding productions:' do
+    it 'allows the addition of one production rule' do
+      rule = build_production(p_symb, [s_symb])
+      expect { subject.add_rule(rule) }.not_to raise_error
+      expect(subject.rules.size).to eq(1)
+      expect(subject.rules.first).to eq(rule)
+    end
+
+    it 'allows the addition of multiple production rules' do
+      rules = build_all_rules
+      rules.each { |rl| subject.add_rule(rl) }
+      expect(subject.rules.size).to eq(4)
+      expect(subject.rules.first).to eq(rules.first)
+      expect(subject.rules.last).to eq(rules.last)
+    end
+
+    it 'updates the set of symbols when adding production rules' do
+      rules = build_all_rules
+      rules.each { |rl| subject.add_rule(rl) }
+      [p_symb, s_symb, m_symb, t_symb].each do |symb|
+        expect(subject.symbols.include?(symb)).to be_truthy
+      end
+    end
+
+    it 'maps name of every non-terminal to its related GrmSymbol' do
+      rules = build_all_rules
+      rules.each { |rl| subject.add_rule(rl) }
+      [[:p, p_symb],
+       ['p', p_symb],
+       [:s, s_symb],
+       ['s', s_symb],
+       [:m, m_symb],
+       ['m', m_symb],
+       [:t, t_symb],
+       [:t, t_symb]].each do |(name, symb)|
+        expect(subject.name2symbol[name]).to eq(symb)
+      end
+    end
+
+    it 'maps every non-terminal to its defining productions' do
+      rules = build_all_rules
+      rules.each { |rl| subject.add_rule(rl) }
+      %i[p s m t].each do |symb_name|
+        symb = subject.name2symbol[symb_name]
+        expected_prods = subject.rules.select { |prd| prd.head == symb }
+        related_prods = subject.nonterm2productions[symb]
+        expect(related_prods).to eq(expected_prods)
+      end
+    end
+  end # context
+
+  context 'Grammar completion:' do
+    it 'detects and marks nullable symbols (I)' do
+      # Case: grammar without nullable symbols
+      rules = build_all_rules
+      rules.each { |rl| subject.add_rule(rl) }
+      subject.complete!
+      expect(subject.symbols.none?(&:nullable?)).to be_truthy
+    end
+
+    it 'detects and marks nullable symbols (II)' do
+      # Case: grammar with only nullable symbols
+      # Grammar inspired for paper "Practical Earley Parser"
+      terminal_a = build_terminal('a')
+      nterm_s_prime = build_nonterminal("S'")
+      nterm_s = build_nonterminal('S')
+      nterm_a = build_nonterminal('A')
+      nterm_e = build_nonterminal('E')
+
+      instance = described_class.new([terminal_a])
+      instance.add_rule(build_production(nterm_s_prime, [nterm_s]))
+      instance.add_rule(build_production(nterm_s, [nterm_a, nterm_a, nterm_a, nterm_a]))
+      instance.add_rule(build_choice(nterm_a, [[terminal_a], [nterm_e]]))
+      instance.add_rule(build_production(nterm_e, []))
+
+      instance.complete!
+      all_nonterminals = subject.symbols.reject(&:terminal?)
+      expect(all_nonterminals.all?(&:nullable?)).to be_truthy
+    end
+
+    it 'detects unreachable symbols' do
+      # Case: grammar without unreachable symbols
+      rules = build_all_rules
+      rules.each { |rl| subject.add_rule(rl) }
+      expect(subject.send(:unreachable_symbols)).to be_empty
+
+      # Let add's unreachable symbols
+      zed_symb = build_nonterminal('Z')
+      question_symb = build_nonterminal('?')
+      bad_rule = build_production(zed_symb, [zed_symb, question_symb, int_symb]) # Z => Z ? INTEGER
+      subject.add_rule(bad_rule)
+      unreachable = subject.send(:unreachable_symbols)
+      expect(unreachable).not_to be_empty
+      expect(unreachable).to eq(Set.new([zed_symb, question_symb]))
+    end
+
+    it 'detects non-productive symbols' do
+      # Case: grammar without non-productive symbols
+      rules = build_all_rules
+      rules.each { |rl| subject.add_rule(rl) }
+      expect(subject.send(:mark_non_productive_symbols)).to be_empty
+      expect(t_symb).to be_productive
+      expect(p_symb).to be_productive
+
+      # Grammar with non-productive symbols
+      term_a = build_terminal('a')
+      term_b = build_terminal('b')
+      term_c = build_terminal('c')
+      term_d = build_terminal('d')
+      term_e = build_terminal('e')
+      term_f = build_terminal('f')
+      nterm_A = build_nonterminal('A')
+      nterm_B = build_nonterminal('B')
+      nterm_C = build_nonterminal('C')
+      nterm_D = build_nonterminal('D')
+      nterm_E = build_nonterminal('E')
+      nterm_F = build_nonterminal('F')
+      nterm_S = build_nonterminal('S')
+      instance = described_class.new([term_a, term_b, term_c, term_d, term_e, term_f])
+      instance.add_rule(build_choice(nterm_S, [[nterm_A, nterm_B], [nterm_D, nterm_E]]))
+      instance.add_rule(build_production(nterm_A, [term_a]))
+      instance.add_rule(build_production(nterm_B, [term_b, nterm_C]))
+      instance.add_rule(build_production(nterm_C, [term_c]))
+      instance.add_rule(build_production(nterm_D, [term_d, nterm_F]))
+      instance.add_rule(build_production(nterm_E, [term_e]))
+      instance.add_rule(build_production(nterm_F, [term_f, nterm_D]))
+      nonproductive = instance.send(:mark_non_productive_symbols)
+      expect(nonproductive).not_to be_empty
+      expect(nonproductive).to eq([nterm_D, nterm_F])
+    end
+  end # context
+end # describe

data/version.txt

@@ -1 +1 @@
-0.0.5
+0.0.6

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dendroid
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-28 00:00:00.000000000 Z
+date: 2023-10-30 00:00:00.000000000 Z
 dependencies: []
 description: WIP. A Ruby implementation of a Earley parser
 email: famished.tiger@yahoo.com
@@ -23,8 +23,9 @@ files:
 - Rakefile
 - bin/dendroid
 - dendroid.gemspec
-- lib/dendroid/dendroid.rb
+- lib/dendroid.rb
 - lib/dendroid/syntax/choice.rb
+- lib/dendroid/syntax/grammar.rb
 - lib/dendroid/syntax/grm_symbol.rb
 - lib/dendroid/syntax/non_terminal.rb
 - lib/dendroid/syntax/production.rb
@@ -32,6 +33,7 @@ files:
 - lib/dendroid/syntax/symbol_seq.rb
 - lib/dendroid/syntax/terminal.rb
 - spec/dendroid/syntax/choice_spec.rb
+- spec/dendroid/syntax/grammar_spec.rb
 - spec/dendroid/syntax/grm_symbol_spec.rb
 - spec/dendroid/syntax/non_terminal_spec.rb
 - spec/dendroid/syntax/production_spec.rb