dendroid 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 171ca5c202ea7a9d5c156086cecee352539c4f0551556175cf748328d3fa0983
-  data.tar.gz: a81434f0fe610433cce7d6f1213dc3a25d06f7264fda882ae4b5ec4867514b31
+  metadata.gz: 270fc74811d70652e19c4ed42cd11138a1fe9fc413e9b1856b982edfa28c5d51
+  data.tar.gz: 280351b252bd5c4a63f3082375053ea7d3bf9a9d0d32acc055dc33cce91ed628
 SHA512:
-  metadata.gz: 7894b4b5abcabde582c9e16dc93df8e5aa77b9fed8691ea52901c802e957558c4353e009544b53d352e45d145807e2957e43468a1783cca92326b52eddddb140
-  data.tar.gz: b4a8415a997fb45005668b03087dc6347db11dd692c87e6da5c9fa6cd0d609ee31b630e51e896627cf95b1a57a8f60f919dde9cf18c8fba4a4f08191dc28004b
+  metadata.gz: 7a34047f56f1f488377afd88c4049b935d03d8a0a902cd44f8ffba3d58578c212c5ef7f0b1229192a7f4606b1d683d70ca479273d45d716d98154a38663f233f
+  data.tar.gz: 36578ffb40a0463a2e411000b24fa8005166c1ede8f6a856293c0122e44fdbb46d3758159042db0c9c4ccacf9c1bf071e49cfb86a64792b98fac8bb89447a85a

data/.rubocop.yml

@@ -20,7 +20,7 @@ Metrics/CyclomaticComplexity:
 
 Metrics/MethodLength:
   Enabled: true
-  Max: 30
+  Max: 60
   
 Metrics/PerceivedComplexity:
   Enabled: true
@@ -32,5 +32,8 @@ Naming/MethodParameterName:
 Naming/VariableName:  
   Enabled: false
 
+Style/AccessorGrouping:
+  Enabled: false
+
 Style/CommentedKeyword:
   Enabled: false

data/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+## [0.0.10] - 2023-11-01
+Added missing class and method documentation, fixed some `Rubocop` offenses.
+
+
 ## [0.0.9] - 2023-11-01
 Added classes for tokenization and grammar analysis.
 

data/dendroid.gemspec

@@ -7,8 +7,8 @@ Gem::Specification.new do |s|
     path = ::File.dirname(libpath) + ::File::SEPARATOR
     ::File.read("#{path}version.txt").strip
   end
-  s.summary     = 'WIP. A Ruby implementation of a Earley parser'
-  s.description = 'WIP. A Ruby implementation of a Earley parser'
+  s.summary     = 'WIP. A Ruby implementation of an Earley parser'
+  s.description = 'WIP. A Ruby implementation of an Earley parser'
   s.authors     = ['Dimitri Geshef']
   s.email       = 'famished.tiger@yahoo.com'
   s.files       = Dir['bin/dendroid',

data/lib/dendroid/grm_analysis/alternative_item.rb

@@ -61,11 +61,10 @@ module Dendroid
       private
 
       def valid_position(aPosition)
-        raise Exception if aPosition < 0 || aPosition > rule.alternatives[alt_index].size
+        raise StandardError if aPosition.negative? || aPosition > rule.alternatives[alt_index].size
 
         aPosition
       end
     end # class
   end # module
 end # module
-

data/lib/dendroid/grm_analysis/choice_items.rb

@@ -52,8 +52,9 @@ module Dendroid
       def next_item(anItem)
         items_arr = items[anItem.alt_index]
         return nil if anItem == items_arr.last
+
         items_arr[anItem.position + 1]
       end
     end # module
   end # module
-end # module
+end # module

data/lib/dendroid/grm_analysis/dotted_item.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Dendroid
+  # This module contains classes that from the analysis of grammar rules help to build objects
+  # needed by a recognizer or a parser for the language.
   module GrmAnalysis
     # For a given production rule, a dotted item represents a recognition state.
     # The dot partitions the rhs of the rule in two parts:

data/lib/dendroid/grm_analysis/grm_analyzer.rb

@@ -5,19 +5,34 @@ require_relative '../grm_analysis/choice_items'
 
 module Dendroid
   module GrmAnalysis
-    # An analyzer performs an analysis of the rules of a given grammar
-    #
+    # An analyzer performs an analysis of the grammar rules and
+    # build objects (dotted items, first and follow sets) to be used
+    # by a recognizer or a parser.
     class GrmAnalyzer
+      # @return [Dendroid::Syntax::Grammar] The grammar subjected to analysis
       attr_reader :grammar
       attr_reader :items
       attr_reader :production2items
       attr_reader :symbol2productions
+
+      # @return [Dendroid::Syntax::Terminal] The pseudo-terminal `__epsilon` (for empty string)
       attr_reader :epsilon
+
+      # @return [Dendroid::Syntax::Terminal] The pseudo-terminal `$$` for end of input stream
       attr_reader :endmarker
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to FIRST SETS mapping
       attr_reader :first_sets
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to PREDICT SETS mapping
       attr_reader :predict_sets
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to FOLLOW SETS mapping
       attr_reader :follow_sets
 
+      # Constructor.
+      # Build dotted items, first, follow sets for the given grammar
+      # @param aGrammar [Dendroid::Syntax::Grammar]
       def initialize(aGrammar)
         @grammar = aGrammar
         @items = []
@@ -34,6 +49,8 @@ module Dendroid
         build_follow_sets
       end
 
+      # The next item of a given dotted item
+      # @param aDottedItem [DottedItem]
       def next_item(aDottedItem)
         prod = aDottedItem.rule
         prod.next_item(aDottedItem)
@@ -72,7 +89,7 @@ module Dendroid
             else
               first_head.merge(sequence_first(prod.body.members))
             end
-            changed = true if (first_head.size > pre_first_size)
+            changed = true if first_head.size > pre_first_size
           end
         end until !changed
       end
@@ -84,7 +101,7 @@ module Dendroid
           elsif symb.nullable?
             first_sets[symb] = Set.new([epsilon])
           else
-            first_sets[symb]  = Set.new
+            first_sets[symb] = Set.new
           end
         end
       end
@@ -115,7 +132,7 @@ module Dendroid
 
                 head = prod.head
                 head_follow = follow_sets[head]
-                trailer = Set.new
+                # trailer = Set.new
                 last = true
                 last_index = body.size - 1
                 last_index.downto(0) do |i|
@@ -130,7 +147,7 @@ module Dendroid
                     follow_sets[symbol].merge(head_follow) if symbol.nullable?
                     last = false
                   else
-                    symbol_seq = body.slice(i+1, last_index - i)
+                    symbol_seq = body.slice(i + 1, last_index - i)
                     trailer_first = sequence_first(symbol_seq)
                     contains_epsilon = trailer_first.include? epsilon
                     trailer_first.delete(epsilon) if contains_epsilon
@@ -146,7 +163,7 @@ module Dendroid
 
               head = prod.head
               head_follow = follow_sets[head]
-              trailer = Set.new
+              # trailer = Set.new
               last = true
               last_index = body.size - 1
               last_index.downto(0) do |i|
@@ -161,7 +178,7 @@ module Dendroid
                   follow_sets[symbol].merge(head_follow) if symbol.nullable?
                   last = false
                 else
-                  symbol_seq = body.slice(i+1, last_index - i)
+                  symbol_seq = body.slice(i + 1, last_index - i)
                   trailer_first = sequence_first(symbol_seq)
                   contains_epsilon = trailer_first.include? epsilon
                   trailer_first.delete(epsilon) if contains_epsilon
@@ -179,7 +196,7 @@ module Dendroid
         grammar.symbols.each do |symb|
           next if symb.terminal?
 
-          follow_sets[symb]  = Set.new
+          follow_sets[symb] = Set.new
         end
 
         # Initialize FOLLOW(start symbol) with end marker

data/lib/dendroid/grm_analysis/production_items.rb

@@ -14,7 +14,7 @@ module Dendroid
         @items = if empty?
                    [DottedItem.new(self, 0)]
                  else
-                   (0..body.size).reduce([]) do | result,pos|
+                   (0..body.size).reduce([]) do |result, pos|
                      result << GrmAnalysis::DottedItem.new(self, pos)
                    end
                  end
@@ -47,6 +47,7 @@ module Dendroid
       # @return [GrmAnalysis::DottedItem|NilClass]
       def next_item(anItem)
         return nil if anItem == @items.last
+
         @items[anItem.position + 1]
       end
     end # module

data/lib/dendroid/lexical/token.rb

@@ -24,7 +24,7 @@ module Dendroid
       attr_reader :position
 
       # @return [String] The name of terminal symbol matching the text.
-      attr :terminal
+      attr_reader :terminal
 
       # Constructor.
       # @param original [String] the piece of text from input

data/lib/dendroid/lexical/token_position.rb

@@ -2,19 +2,27 @@
 
 module Dendroid
   module Lexical
+    # Keeps track of the position of a token in the input stream.
     class TokenPosition
+      # @return [Integer] The line number where the token begins
       attr_reader :lineno
+
+      # @return [Integer] The column number where the token begins
       attr_reader :column
 
+      # Constructor
+      # @param line [Integer] The line number where the token begins
+      # @param col [Integer] The column number where the token begins
       def initialize(line, col)
         @lineno = line
         @column = col
       end
 
+      # Return the position of the start of the token in line:col format
+      # @return [String]
       def to_s
         "#{lineno}:#{column}"
       end
     end # class
   end # module
 end # module
-

data/lib/dendroid/syntax/grm_symbol.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dendroid
-  # The namespace for all classes used to build a grammar.
+  # This module contains for all classes representing elements of .
   module Syntax
     # Abstract class for grammar symbols.
     # A grammar symbol is an element that appears in grammar rules.

data/lib/dendroid/utils/base_tokenizer.rb

@@ -5,44 +5,67 @@ require_relative '../lexical/token_position'
 require_relative '../lexical/literal'
 
 module Dendroid
+  # This module contains helper classes (e.g. a tokenizer generator)
   module Utils
+    # A basic tokenizer.
+    # Responsibility: break input into a sequence of token objects.
+    # This class defines a simple DSL to build a tokenizer.
     class BaseTokenizer
+      # @return [StringScanner] Low-level input scanner
       attr_reader :scanner
+
+      # @return [Integer] The current line number
       attr_reader :lineno
+
+      # @return [Integer] Position of last start of line in the input string
       attr_reader :line_start
+
+      # @return [Hash{Symbol, Array<Regexp>}]
       attr_reader :actions
 
+      # Constructor
+      # @param aBlock [Proc]
       def initialize(&aBlock)
         @scanner = StringScanner.new('')
         @actions = { skip: [], scan_verbatim: [], scan_value: [] }
         defaults
+        return unless block_given?
 
-        if block_given?
-          instance_exec(&aBlock)
-          # grammar_complete!
-        end
+        instance_exec(&aBlock)
       end
 
+      # Reset the tokenizer and set new text to tokenize
+      # @param source [String]
       def input=(source)
-        scanner.string = source
         reset
+        scanner.string = source
       end
 
+      # Reset the tokenizer
       def reset
         @lineno = 1
         @line_start = 0
+        scanner.reset
       end
 
       # action, pattern, terminal?, conversion?
       # action = skip, skip_nl, scan
+
+      # Associate the provided pattern to the action of skipping a newline and
+      # incrementing the line counter.
+      # @param pattern [Regexp]
       def skip_nl(pattern)
         actions[:skip_nl] = pattern
       end
 
+      # Associate the provided pattern with the action to skip whitespace(s).
+      # @param pattern [Regexp]
       def skip_ws(pattern)
         actions[:skip_ws] = pattern
       end
 
+      # Associate the provided pattern with the action to skip the matching text.
+      # @param pattern [Regexp]
       def skip(pattern)
         if actions[:skip].empty?
           actions[:skip] = pattern
@@ -52,6 +75,8 @@ module Dendroid
         end
       end
 
+      # Associate the provided pattern with the action to tokenize the matching text
+      # @param pattern [Regexp]
       def scan_verbatim(pattern)
         patt = normalize_pattern(pattern)
         if actions[:scan_verbatim].empty?
@@ -62,9 +87,15 @@ module Dendroid
         end
       end
 
-      def scan_value(pattern, terminal, convertion)
+      # Associate the provided pattern with the action to tokenize the matching text
+      # as an instance of the given terminal symbol and convert the matching text into
+      # a value by using the given conversion.
+      # @param pattern [Regexp]
+      # @param terminal [Dendroid::Syntax::Terminal]
+      # @param conversion [Proc] a Proc (lambda) that takes a String as argument and return a value.
+      def scan_value(pattern, terminal, conversion)
         patt = normalize_pattern(pattern)
-        tuple = [patt, terminal, convertion]
+        tuple = [patt, terminal, conversion]
         if actions[:scan_value].empty?
           actions[:scan_value] = [tuple]
         else
@@ -72,10 +103,16 @@ module Dendroid
         end
       end
 
+      # Set the mapping between a verbatim text to its corresponding terminal symbol name
+      # @param mapping [Hash{String, String}]
       def map_verbatim2terminal(mapping)
         @verbatim2terminal = mapping
       end
 
+      # rubocop: disable Metrics/AbcSize
+
+      # Return the next token (if any) from the input stream.
+      # @return [Dendroid::Lexical::Token, NilClass]
       def next_token
         token = nil
 
@@ -93,7 +130,7 @@ module Dendroid
             break
           end
 
-          tuple = actions[:scan_value].find do |(pattern, terminal, conversion)|
+          tuple = actions[:scan_value].find do |(pattern, _terminal, _conversion)|
             scanner.check(pattern)
           end
           if tuple
@@ -106,18 +143,20 @@ module Dendroid
           # Unknown token
           col = scanner.pos - line_start + 1
           erroneous = scanner.peek(1).nil? ? '' : scanner.scan(/./)
-          raise Exception, "Error: [line #{lineno}:#{col}]: Unexpected character #{erroneous}."
+          raise StandardError, "Error: [line #{lineno}:#{col}]: Unexpected character #{erroneous}."
         end
 
         token
       end
 
+      # rubocop: enable Metrics/AbcSize
+
       protected
 
       def defaults
         # Defaults
-        skip_nl /(?:\r\n)|\r|\n/  # Skip newlines
-        skip_ws /[ \t\f]+/ # Skip blanks
+        skip_nl(/(?:\r\n)|\r|\n/) # Skip newlines
+        skip_ws(/[ \t\f]+/) # Skip blanks
       end
 
       private
@@ -146,7 +185,7 @@ module Dendroid
           col = scanner.pos - lex_length - @line_start + 1
           pos = Lexical::TokenPosition.new(@lineno, col)
           token = Lexical::Token.new(text, pos, symbol_name)
-        rescue Exception => e
+        rescue StandardError => e
           puts "Failing with '#{symbol_name}' and '#{text}'"
           raise e
         end
@@ -154,15 +193,15 @@ module Dendroid
         token
       end
 
-      def value_scanned(aText, aSymbolName, convertion)
-        value = convertion.call(aText)
+      def value_scanned(aText, aSymbolName, conversion)
+        value = conversion.call(aText)
         lex_length = aText ? aText.size : 0
         col = scanner.pos - lex_length - @line_start + 1
         build_literal(aSymbolName, value, aText, col)
       end
 
       def build_literal(aSymbolName, aValue, aText, aPosition)
-        pos = if aPosition.kind_of?(Integer)
+        pos = if aPosition.is_a?(Integer)
                 col = aPosition
                 Lexical::TokenPosition.new(@lineno, col)
               else

data/spec/dendroid/grm_analysis/grm_analyzer_spec.rb

@@ -52,7 +52,8 @@ module SampleGrammars
                               '*' => :STAR,
                               '/' => :SLASH,
                               '(' => :LPAREN,
-                              ')' => :RPAREN })
+                              ')' => :RPAREN
+                            })
 
       scan_verbatim(['+', '-', '*', '/', '(', ')'])
       scan_value(/\d+/, :NUMBER, ->(txt) { txt.to_i })
@@ -67,7 +68,7 @@ module SampleGrammars
 
       rule('Z' => ['d', 'X Y Z'])
       rule('Y' => ['', 'c'])
-      rule('X' => ['Y', 'a'])
+      rule('X' => %w[Y a])
     end
 
     builder.grammar
@@ -91,7 +92,7 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
 
     it 'knows the dotted items' do
       item_count = subject.grammar.rules.reduce(0) do |count, prod|
-        count += prod.items.flatten.size
+        count + prod.items.flatten.size
       end
       expect(subject.items.size).to eq(item_count)
       expected_items = [
@@ -116,7 +117,7 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
     end
 
     it 'knows the item that follows a given dotted item' do
-      first_item = subject.items.find { |itm| itm.to_s ==  'm => . m STAR t' }
+      first_item = subject.items.find { |itm| itm.to_s == 'm => . m STAR t' }
       second = subject.next_item(first_item)
       expect(second.to_s).to eq('m => m . STAR t')
       third = subject.next_item(second)
@@ -134,9 +135,9 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
         'a' => ['a'],
         'c' => ['c'],
         'd' => ['d'],
-        'X' => ['a', 'c'], # Add epsilon
+        'X' => %w[a c], # Add epsilon
         'Y' => ['c'], # Add epsilon
-        'Z' => ['a', 'c', 'd']
+        'Z' => %w[a c d]
       }
       expectations.each_pair do |sym_name, first_names|
         symb = subject.grammar.name2symbol[sym_name]
@@ -149,8 +150,8 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
     it 'constructs the FOLLOW sets for non-terminal symbols' do
       expectations = {
         'Z' => [], # Add $$
-        'Y' => ['a', 'c', 'd'],
-        'X' => ['a', 'c', 'd']
+        'Y' => %w[a c d],
+        'X' => %w[a c d]
       }
       subject.send(:build_follow_sets)
       expectations.each_pair do |sym_name, follow_names|

data/spec/dendroid/lexical/token_position_spec.rb

@@ -28,4 +28,4 @@ describe Dendroid::Lexical::TokenPosition do
       expect(subject.to_s).to eq("#{ex_lineno}:#{ex_column}")
     end
   end # context
-end # describe
+end # describe

data/spec/dendroid/utils/base_tokenizer_spec.rb

@@ -9,7 +9,7 @@ describe Dendroid::Utils::BaseTokenizer do
 
   context 'Initialization:' do
     it 'is initialized with an optional block' do
-      expect {described_class.new }.not_to raise_error
+      expect { described_class.new }.not_to raise_error
     end
 
     it 'has a scanner at start' do
@@ -24,11 +24,11 @@ describe Dendroid::Utils::BaseTokenizer do
 
   context 'Tokenizing:' do
     subject do
-      described_class.new {
+      described_class.new do
         scan_verbatim(['+', '*'])
         scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
         map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
-      }
+      end
     end
 
     it 'generates a sequence of tokens from a simple input' do
@@ -43,7 +43,7 @@ describe Dendroid::Utils::BaseTokenizer do
       ]
       expectations.each do |tuple|
         tok = subject.next_token
-        [:pos_to_s, :source, :terminal, :value].each_with_index do |message, index|
+        %i[pos_to_s source terminal value].each_with_index do |message, index|
           expect(tok.send(message)).to eq(tuple[index]) unless tuple[index].nil?
         end
       end

data/version.txt

@@ -1 +1 @@
-0.0.9
+0.0.10

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dendroid
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Dimitri Geshef
@@ -10,7 +10,7 @@ bindir: bin
 cert_chain: []
 date: 2023-11-01 00:00:00.000000000 Z
 dependencies: []
-description: WIP. A Ruby implementation of a Earley parser
+description: WIP. A Ruby implementation of an Earley parser
 email: famished.tiger@yahoo.com
 executables: []
 extensions: []
@@ -84,5 +84,5 @@ requirements: []
 rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
-summary: WIP. A Ruby implementation of a Earley parser
+summary: WIP. A Ruby implementation of an Earley parser
 test_files: []