dendroid 0.0.8 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 989c88ea00734b7145dfc64ded1318c296a3678039213904c15ead0a306942a2
-  data.tar.gz: 73aa3efff885fcf1f71f23c7907b7c7b953fd3b174aefb9324a1326663652a9c
+  metadata.gz: 270fc74811d70652e19c4ed42cd11138a1fe9fc413e9b1856b982edfa28c5d51
+  data.tar.gz: 280351b252bd5c4a63f3082375053ea7d3bf9a9d0d32acc055dc33cce91ed628
 SHA512:
-  metadata.gz: 906746623d20819ce0f1923c8156a27c7ce58b2d926174e0a1dff611407bd873378272c2a2125bb665c8aec20c3beef787942c9fc7f0376aff22e7db4fb6a235
-  data.tar.gz: 34f3aabf74b7598a506e3acd38c9df03a7040203e8a9bda2fa02804b96ea0054d17c3044c7ca1384380bdb55af2cd80f1b9e88baa1b5c985642797cc68d80783
+  metadata.gz: 7a34047f56f1f488377afd88c4049b935d03d8a0a902cd44f8ffba3d58578c212c5ef7f0b1229192a7f4606b1d683d70ca479273d45d716d98154a38663f233f
+  data.tar.gz: 36578ffb40a0463a2e411000b24fa8005166c1ede8f6a856293c0122e44fdbb46d3758159042db0c9c4ccacf9c1bf071e49cfb86a64792b98fac8bb89447a85a

data/.rubocop.yml

@@ -20,7 +20,7 @@ Metrics/CyclomaticComplexity:
 
 Metrics/MethodLength:
   Enabled: true
-  Max: 30
+  Max: 60
   
 Metrics/PerceivedComplexity:
   Enabled: true
@@ -32,5 +32,8 @@ Naming/MethodParameterName:
 Naming/VariableName:  
   Enabled: false
 
+Style/AccessorGrouping:
+  Enabled: false
+
 Style/CommentedKeyword:
   Enabled: false

data/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## [Unreleased]
 
+## [0.0.10] - 2023-11-01
+Added missing class and method documentation, fixed some `Rubocop` offenses.
+
+
+## [0.0.9] - 2023-11-01
+Added classes for tokenization and grammar analysis.
+
+### Added
+- Class `AlternativeItem` and its spec file
+- Class `BaseTokenizer` and its spec file
+- Module `ChoiceItems` and its spec file- 
+- Class `GrmAnalyzer` and its spec file
+- Class `Literal` and its spec file
+- Module `ProductionItems` and its spec file
+- Class `Token` and its spec file
+- Class `TokenPosition` and its spec file
+
 ## [0.0.8] - 2023-10-30
 ### Added
 - Class `DottedItem` and its spec file

data/dendroid.gemspec

@@ -7,8 +7,8 @@ Gem::Specification.new do |s|
     path = ::File.dirname(libpath) + ::File::SEPARATOR
     ::File.read("#{path}version.txt").strip
   end
-  s.summary     = 'Dendroid. TODO'
-  s.description = 'WIP. A Ruby implementation of a Earley parser'
+  s.summary     = 'WIP. A Ruby implementation of an Earley parser'
+  s.description = 'WIP. A Ruby implementation of an Earley parser'
   s.authors     = ['Dimitri Geshef']
   s.email       = 'famished.tiger@yahoo.com'
   s.files       = Dir['bin/dendroid',

data/lib/dendroid/grm_analysis/alternative_item.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative 'dotted_item'
+
+module Dendroid
+  module GrmAnalysis
+    # A specialization of DottedItem specific for Choice (rule)
+    class AlternativeItem < DottedItem
+      # @return [Integer] the alternative number
+      attr_reader :alt_index
+
+      # Constructor.
+      # @param aChoice [Dendroid::Syntax::Choice]
+      # @param aPosition [Integer] Position of the dot in rhs of production.
+      # @param index [Integer] the rank of the alternative at hand
+      def initialize(aChoice, aPosition, index)
+        @alt_index = index
+        super(aChoice, aPosition)
+      end
+
+      # Return a String representation of the alternative item.
+      # @return [String]
+      def to_s
+        rhs_names = rule.alternatives[alt_index].members.map(&:to_s)
+        dotted_rhs = rhs_names.insert(position, '.')
+        "#{rule.head} => #{dotted_rhs.join(' ')}"
+      end
+
+      # Indicate whether the rhs of the alternative is empty
+      # @return [Boolean]
+      def empty?
+        rule.alternatives[alt_index].empty?
+      end
+
+      # Indicate whether the dot is at the start of rhs
+      # @return [Boolean]
+      def final_pos?
+        empty? || position == rule.alternatives[alt_index].size
+      end
+
+      alias completed? final_pos?
+
+      # Return the symbol right after the dot (if any)
+      # @return [Dendroid::Syntax::GrmSymbol, NilClass]
+      def next_symbol
+        return nil if empty? || completed?
+
+        rule.alternatives[alt_index].members[position]
+      end
+
+      # Test for equality with another dotted item.
+      # Two dotted items are equal if they refer to the same rule and
+      # have both the same rhs and dot positions.
+      # @return [Boolean]
+      def ==(other)
+        return true if eql?(other)
+
+        (position == other.position) && rule.eql?(other.rule) && (alt_index == other.alt_index)
+      end
+
+      private
+
+      def valid_position(aPosition)
+        raise StandardError if aPosition.negative? || aPosition > rule.alternatives[alt_index].size
+
+        aPosition
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/grm_analysis/choice_items.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'alternative_item'
+
+module Dendroid
+  module GrmAnalysis
+    # Mix-in module for extending the Syntax::Choice class
+    # with dotted items manipulation methods
+    module ChoiceItems
+      # Build the alternative items for this choice and assign them
+      # to the `items` attributes
+      # @return [Array<Array<GrmAnalysis::AlternativeItem>>]
+      def build_items
+        # AlternativeItem
+        @items = Array.new(alternatives.size) { |_| [] }
+        alternatives.each_with_index do |alt_seq, index|
+          if alt_seq.empty?
+            @items[index] << AlternativeItem.new(self, 0, index)
+          else
+            (0..alt_seq.size).each do |pos|
+              @items[index] << AlternativeItem.new(self, pos, index)
+            end
+          end
+        end
+      end
+
+      # Read accessor for the `items` attribute.
+      # Return the dotted items for this production
+      # @return [Array<Array<GrmAnalysis::AlternativeItem>>]
+      def items
+        @items
+      end
+
+      # Return the predicted items (i.e. the alternative items with the dot at start)
+      # for this choice.
+      # @return [Array<GrmAnalysis::AlternativeItem>]
+      def predicted_items
+        @items.map(&:first)
+      end
+
+      # Return the reduce items (i.e. the alternative items with the dot at end)
+      # for this choice.
+      # @return [Array<GrmAnalysis::AlternativeItem>]
+      def reduce_items
+        @items.map(&:last)
+      end
+
+      # Return the next item given the provided item.
+      # In other words, advance the dot by one position.
+      # @param anItem [GrmAnalysis::AlternativeItem]
+      # @return [GrmAnalysis::AlternativeItem|NilClass]
+      def next_item(anItem)
+        items_arr = items[anItem.alt_index]
+        return nil if anItem == items_arr.last
+
+        items_arr[anItem.position + 1]
+      end
+    end # module
+  end # module
+end # module

data/lib/dendroid/grm_analysis/dotted_item.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module Dendroid
+  # This module contains classes that from the analysis of grammar rules help to build objects
+  # needed by a recognizer or a parser for the language.
   module GrmAnalysis
     # For a given production rule, a dotted item represents a recognition state.
     # The dot partitions the rhs of the rule in two parts:
@@ -87,7 +89,7 @@ module Dendroid
       end
 
       # Check whether the given symbol is the same as after the dot.
-      # @param [Dendroid::Syntax::GrmSymbol]
+      # @param aSymbol [Dendroid::Syntax::GrmSymbol]
       # @return [Boolean]
       def expecting?(aSymbol)
         actual = next_symbol

data/lib/dendroid/grm_analysis/grm_analyzer.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+require_relative '../grm_analysis/production_items'
+require_relative '../grm_analysis/choice_items'
+
+module Dendroid
+  module GrmAnalysis
+    # An analyzer performs an analysis of the grammar rules and
+    # build objects (dotted items, first and follow sets) to be used
+    # by a recognizer or a parser.
+    class GrmAnalyzer
+      # @return [Dendroid::Syntax::Grammar] The grammar subjected to analysis
+      attr_reader :grammar
+      attr_reader :items
+      attr_reader :production2items
+      attr_reader :symbol2productions
+
+      # @return [Dendroid::Syntax::Terminal] The pseudo-terminal `__epsilon` (for empty string)
+      attr_reader :epsilon
+
+      # @return [Dendroid::Syntax::Terminal] The pseudo-terminal `$$` for end of input stream
+      attr_reader :endmarker
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to FIRST SETS mapping
+      attr_reader :first_sets
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to PREDICT SETS mapping
+      attr_reader :predict_sets
+
+      # @return [Hash{Syntax::NonTerminal, Array<Syntax::Terminal>}] non-terminal to FOLLOW SETS mapping
+      attr_reader :follow_sets
+
+      # Constructor.
+      # Build dotted items, first, follow sets for the given grammar
+      # @param aGrammar [Dendroid::Syntax::Grammar]
+      def initialize(aGrammar)
+        @grammar = aGrammar
+        @items = []
+        @production2items = {}
+        @symbol2productions = {}
+        @epsilon = Syntax::Terminal.new(:__epsilon)
+        @endmarker = Syntax::Terminal.new(:"$$")
+        @first_sets = {}
+        @predict_sets = {}
+        @follow_sets = {}
+
+        build_dotted_items
+        build_first_sets
+        build_follow_sets
+      end
+
+      # The next item of a given dotted item
+      # @param aDottedItem [DottedItem]
+      def next_item(aDottedItem)
+        prod = aDottedItem.rule
+        prod.next_item(aDottedItem)
+      end
+
+      private
+
+      def build_dotted_items
+        grammar.rules.each do |prod|
+          lhs = prod.head
+          symbol2productions[lhs] = [] unless symbol2productions.include? lhs
+          symbol2productions[lhs] << prod
+          # production2items[prod] = []
+          mixin = prod.choice? ? ChoiceItems : ProductionItems
+          prod.extend(mixin)
+          prod.build_items
+          rule_items = prod.items.flatten
+          items.concat(rule_items)
+          production2items[prod] = rule_items
+        end
+      end
+
+      def build_first_sets
+        initialize_first_sets
+
+        begin
+          changed = false
+          grammar.rules.each do |prod|
+            head = prod.head
+            first_head = first_sets[head]
+            pre_first_size = first_head.size
+            if prod.choice?
+              prod.alternatives.each do |alt|
+                first_head.merge(sequence_first(alt.members))
+              end
+            else
+              first_head.merge(sequence_first(prod.body.members))
+            end
+            changed = true if first_head.size > pre_first_size
+          end
+        end until !changed
+      end
+
+      def initialize_first_sets
+        grammar.symbols.each do |symb|
+          if symb.terminal?
+            first_sets[symb] = Set.new([symb])
+          elsif symb.nullable?
+            first_sets[symb] = Set.new([epsilon])
+          else
+            first_sets[symb] = Set.new
+          end
+        end
+      end
+
+      def sequence_first(symbol_seq)
+        result = Set.new
+        symbol_seq.each do |symb|
+          result.delete(epsilon)
+          result.merge(first_sets[symb])
+          break unless symb.nullable?
+        end
+
+        result
+      end
+
+      # FOLLOW(A): is the set of terminals (+ end marker) that may come after the
+      # non-terminal A.
+      def build_follow_sets
+        initialize_follow_sets
+
+        begin
+          changed = false
+          grammar.rules.each do |prod|
+            if prod.choice?
+              prod.alternatives.each do |alt|
+                body = alt.members
+                next if body.empty?
+
+                head = prod.head
+                head_follow = follow_sets[head]
+                # trailer = Set.new
+                last = true
+                last_index = body.size - 1
+                last_index.downto(0) do |i|
+                  symbol = body[i]
+                  next if symbol.terminal?
+
+                  follow_symbol = follow_sets[symbol]
+                  size_before = follow_symbol.size
+                  if last
+                    # Rule: if last non-terminal member (symbol) is nullable
+                    # then add FOLLOW(head) to FOLLOW(symbol)
+                    follow_sets[symbol].merge(head_follow) if symbol.nullable?
+                    last = false
+                  else
+                    symbol_seq = body.slice(i + 1, last_index - i)
+                    trailer_first = sequence_first(symbol_seq)
+                    contains_epsilon = trailer_first.include? epsilon
+                    trailer_first.delete(epsilon) if contains_epsilon
+                    follow_sets[symbol].merge(trailer_first)
+                    follow_sets[symbol].merge(head_follow) if contains_epsilon
+                  end
+                  changed = true if follow_sets[symbol].size > size_before
+                end
+              end
+            else
+              body = prod.body.members
+              next if body.empty?
+
+              head = prod.head
+              head_follow = follow_sets[head]
+              # trailer = Set.new
+              last = true
+              last_index = body.size - 1
+              last_index.downto(0) do |i|
+                symbol = body[i]
+                next if symbol.terminal?
+
+                follow_symbol = follow_sets[symbol]
+                size_before = follow_symbol.size
+                if last
+                  # Rule: if last non-terminal member (symbol) is nullable
+                  # then add FOLLOW(head) to FOLLOW(symbol)
+                  follow_sets[symbol].merge(head_follow) if symbol.nullable?
+                  last = false
+                else
+                  symbol_seq = body.slice(i + 1, last_index - i)
+                  trailer_first = sequence_first(symbol_seq)
+                  contains_epsilon = trailer_first.include? epsilon
+                  trailer_first.delete(epsilon) if contains_epsilon
+                  follow_sets[symbol].merge(trailer_first)
+                  follow_sets[symbol].merge(head_follow) if contains_epsilon
+                end
+                changed = true if follow_sets[symbol].size > size_before
+              end
+            end
+          end
+        end until !changed
+      end
+
+      def initialize_follow_sets
+        grammar.symbols.each do |symb|
+          next if symb.terminal?
+
+          follow_sets[symb] = Set.new
+        end
+
+        # Initialize FOLLOW(start symbol) with end marker
+        follow_sets[grammar.start_symbol].add(endmarker)
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/grm_analysis/production_items.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative 'dotted_item'
+
+module Dendroid
+  module GrmAnalysis
+    # Mix-in module for extending the Dendroid::Syntax::Production class
+    # with dotted items manipulation methods and an attribute named `items`.
+    module ProductionItems
+      # Build the dotted items for this production and assign them
+      # to the `items` attributes
+      # @return [Array<GrmAnalysis::DottedItem>]
+      def build_items
+        @items = if empty?
+                   [DottedItem.new(self, 0)]
+                 else
+                   (0..body.size).reduce([]) do |result, pos|
+                     result << GrmAnalysis::DottedItem.new(self, pos)
+                   end
+                 end
+      end
+
+      # Read accessor for the `items` attribute.
+      # Return the dotted items for this production
+      # @return [Array<GrmAnalysis::DottedItem>]
+      def items
+        @items
+      end
+
+      # Return the predicted item (i.e. the dotted item with the dot at start)
+      # for this production.
+      # @return [Array<GrmAnalysis::DottedItem>]
+      def predicted_items
+        [@items.first]
+      end
+
+      # Return the reduce item (i.e. the dotted item with the dot at end)
+      # for this production.
+      # @return [Array<GrmAnalysis::DottedItem>]
+      def reduce_items
+        [@items.last]
+      end
+
+      # Return the next item given the provided item.
+      # In other words, advance the dot by one position.
+      # @param anItem [GrmAnalysis::DottedItem]
+      # @return [GrmAnalysis::DottedItem|NilClass]
+      def next_item(anItem)
+        return nil if anItem == @items.last
+
+        @items[anItem.position + 1]
+      end
+    end # module
+  end # module
+end # module

data/lib/dendroid/lexical/literal.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative 'token'
+
+module Dendroid
+  module Lexical
+    # A literal (value) is a token that represents a data value in the parsed
+    # language. For instance, in Ruby data values such as strings, numbers,
+    # regular expression,... can appear directly in the source code as text.
+    # These are examples of literal values. One responsibility of a tokenizer/lexer is
+    # to convert the text representation into a corresponding value in a
+    # convenient format for the interpreter/compiler.
+    class Literal < Token
+      # @return [Object] The value expressed in one of the target datatype.
+      attr_reader :value
+
+      # Constructor.
+      # @param original [String] the piece of text from input
+      # @param pos [Dendroid::Lexical::TokenPosition] line, column position of token
+      # @param symbol [Dendroid::Syntax::Terminal, String]
+      # @param aValue [Object] value of the token in internal representation
+      def initialize(original, pos, symbol, aValue)
+        super(original, pos, symbol)
+        @value = aValue
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/lexical/token.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Dendroid
+  # This module contains the core classes needed for lexical analysis.
+  # The lexical analysis (tokenization) aims to transform the input stream of characters
+  # into a sequence of tokens.
+  module Lexical
+    # A (lexical) token is an object created by a tokenizer (lexer)
+    # and passed to the parser. Such token object is created when a lexer
+    # detects that a sequence of characters(a lexeme) from the input stream
+    # is an instance of a terminal grammar symbol.
+    # Say, that in a particular language, the lexeme 'foo' is an occurrence
+    # of the terminal symbol IDENTIFIER. Then the lexer will return a Token
+    # object that states the fact that 'foo' is indeed an IDENTIFIER. Basically,
+    # a Token is a pair (lexeme, terminal): it asserts that a given piece of text
+    # is an instance of given terminal symbol.
+    class Token
+      # The sequence of character(s) from the input stream that is an occurrence
+      # of the related terminal symbol.
+      # @return [String] Input substring that is an instance of the terminal.
+      attr_reader :source
+
+      # @return [TokenPosition] The position -in "editor" coordinates- of the text in the source file.
+      attr_reader :position
+
+      # @return [String] The name of terminal symbol matching the text.
+      attr_reader :terminal
+
+      # Constructor.
+      # @param original [String] the piece of text from input
+      # @param pos [Dendroid::Lexical::TokenPosition] position of the token in source file
+      # @param symbol [Dendroid::Syntax::Terminal, String]
+      #   The terminal symbol corresponding to the matching text.
+      def initialize(original, pos, symbol)
+        @source = original.dup
+        @position = pos
+        @terminal = symbol
+      end
+
+      # @return [String] The text representation of the token position
+      def pos_to_s
+        position.to_s
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/lexical/token_position.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Dendroid
+  module Lexical
+    # Keeps track of the position of a token in the input stream.
+    class TokenPosition
+      # @return [Integer] The line number where the token begins
+      attr_reader :lineno
+
+      # @return [Integer] The column number where the token begins
+      attr_reader :column
+
+      # Constructor
+      # @param line [Integer] The line number where the token begins
+      # @param col [Integer] The column number where the token begins
+      def initialize(line, col)
+        @lineno = line
+        @column = col
+      end
+
+      # Return the position of the start of the token in line:col format
+      # @return [String]
+      def to_s
+        "#{lineno}:#{column}"
+      end
+    end # class
+  end # module
+end # module

data/lib/dendroid/syntax/grm_symbol.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Dendroid
-  # The namespace for all classes used to build a grammar.
+  # This module contains for all classes representing elements of .
   module Syntax
     # Abstract class for grammar symbols.
     # A grammar symbol is an element that appears in grammar rules.

data/lib/dendroid/utils/base_tokenizer.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'strscan'
+require_relative '../lexical/token_position'
+require_relative '../lexical/literal'
+
+module Dendroid
+  # This module contains helper classes (e.g. a tokenizer generator)
+  module Utils
+    # A basic tokenizer.
+    # Responsibility: break input into a sequence of token objects.
+    # This class defines a simple DSL to build a tokenizer.
+    class BaseTokenizer
+      # @return [StringScanner] Low-level input scanner
+      attr_reader :scanner
+
+      # @return [Integer] The current line number
+      attr_reader :lineno
+
+      # @return [Integer] Position of last start of line in the input string
+      attr_reader :line_start
+
+      # @return [Hash{Symbol, Array<Regexp>}]
+      attr_reader :actions
+
+      # Constructor
+      # @param aBlock [Proc]
+      def initialize(&aBlock)
+        @scanner = StringScanner.new('')
+        @actions = { skip: [], scan_verbatim: [], scan_value: [] }
+        defaults
+        return unless block_given?
+
+        instance_exec(&aBlock)
+      end
+
+      # Reset the tokenizer and set new text to tokenize
+      # @param source [String]
+      def input=(source)
+        reset
+        scanner.string = source
+      end
+
+      # Reset the tokenizer
+      def reset
+        @lineno = 1
+        @line_start = 0
+        scanner.reset
+      end
+
+      # action, pattern, terminal?, conversion?
+      # action = skip, skip_nl, scan
+
+      # Associate the provided pattern to the action of skipping a newline and
+      # incrementing the line counter.
+      # @param pattern [Regexp]
+      def skip_nl(pattern)
+        actions[:skip_nl] = pattern
+      end
+
+      # Associate the provided pattern with the action to skip whitespace(s).
+      # @param pattern [Regexp]
+      def skip_ws(pattern)
+        actions[:skip_ws] = pattern
+      end
+
+      # Associate the provided pattern with the action to skip the matching text.
+      # @param pattern [Regexp]
+      def skip(pattern)
+        if actions[:skip].empty?
+          actions[:skip] = pattern
+        else
+          new_pattern = actions[:skip].union(pattern)
+          actions[:skip] = new_pattern
+        end
+      end
+
+      # Associate the provided pattern with the action to tokenize the matching text
+      # @param pattern [Regexp]
+      def scan_verbatim(pattern)
+        patt = normalize_pattern(pattern)
+        if actions[:scan_verbatim].empty?
+          actions[:scan_verbatim] = patt
+        else
+          new_pattern = actions[:scan_verbatim].union(patt)
+          actions[:scan_verbatim] = new_pattern
+        end
+      end
+
+      # Associate the provided pattern with the action to tokenize the matching text
+      # as an instance of the given terminal symbol and convert the matching text into
+      # a value by using the given conversion.
+      # @param pattern [Regexp]
+      # @param terminal [Dendroid::Syntax::Terminal]
+      # @param conversion [Proc] a Proc (lambda) that takes a String as argument and return a value.
+      def scan_value(pattern, terminal, conversion)
+        patt = normalize_pattern(pattern)
+        tuple = [patt, terminal, conversion]
+        if actions[:scan_value].empty?
+          actions[:scan_value] = [tuple]
+        else
+          actions[:scan_verbatim] << tuple
+        end
+      end
+
+      # Set the mapping between a verbatim text to its corresponding terminal symbol name
+      # @param mapping [Hash{String, String}]
+      def map_verbatim2terminal(mapping)
+        @verbatim2terminal = mapping
+      end
+
+      # rubocop: disable Metrics/AbcSize
+
+      # Return the next token (if any) from the input stream.
+      # @return [Dendroid::Lexical::Token, NilClass]
+      def next_token
+        token = nil
+
+        # Loop until end of input reached or token found
+        until scanner.eos?
+          if scanner.skip(actions[:skip_nl])
+            next_line_scanned
+            next
+          end
+
+          next if scanner.skip(actions[:skip_ws]) # Skip whitespaces
+
+          if (text = scanner.scan(actions[:scan_verbatim]))
+            token = verbatim_scanned(text)
+            break
+          end
+
+          tuple = actions[:scan_value].find do |(pattern, _terminal, _conversion)|
+            scanner.check(pattern)
+          end
+          if tuple
+            (pattern, terminal, conversion) = tuple
+            text = scanner.scan(pattern)
+            token = value_scanned(text, terminal, conversion)
+            break
+          end
+
+          # Unknown token
+          col = scanner.pos - line_start + 1
+          erroneous = scanner.peek(1).nil? ? '' : scanner.scan(/./)
+          raise StandardError, "Error: [line #{lineno}:#{col}]: Unexpected character #{erroneous}."
+        end
+
+        token
+      end
+
+      # rubocop: enable Metrics/AbcSize
+
+      protected
+
+      def defaults
+        # Defaults
+        skip_nl(/(?:\r\n)|\r|\n/) # Skip newlines
+        skip_ws(/[ \t\f]+/) # Skip blanks
+      end
+
+      private
+
+      def normalize_pattern(pattern)
+        case pattern
+        when String
+          Regexp.new(Regexp.escape(pattern))
+        when Array
+          regexes = pattern.map { |patt| normalize_pattern(patt) }
+          Regexp.union(regexes)
+        else
+          pattern
+        end
+      end
+
+      def next_line_scanned
+        @lineno += 1
+        @line_start = scanner.pos
+      end
+
+      def verbatim_scanned(text)
+        symbol_name = @verbatim2terminal[text]
+        begin
+          lex_length = text ? text.size : 0
+          col = scanner.pos - lex_length - @line_start + 1
+          pos = Lexical::TokenPosition.new(@lineno, col)
+          token = Lexical::Token.new(text, pos, symbol_name)
+        rescue StandardError => e
+          puts "Failing with '#{symbol_name}' and '#{text}'"
+          raise e
+        end
+
+        token
+      end
+
+      def value_scanned(aText, aSymbolName, conversion)
+        value = conversion.call(aText)
+        lex_length = aText ? aText.size : 0
+        col = scanner.pos - lex_length - @line_start + 1
+        build_literal(aSymbolName, value, aText, col)
+      end
+
+      def build_literal(aSymbolName, aValue, aText, aPosition)
+        pos = if aPosition.is_a?(Integer)
+                col = aPosition
+                Lexical::TokenPosition.new(@lineno, col)
+              else
+                aPosition
+              end
+
+        Lexical::Literal.new(aText.dup, pos, aSymbolName, aValue)
+      end
+    end # class
+  end # module
+end # module

data/spec/dendroid/grm_analysis/alternative_item_spec.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require_relative '..\..\spec_helper'
+require_relative '..\..\..\lib\dendroid\syntax\terminal'
+require_relative '..\..\..\lib\dendroid\syntax\non_terminal'
+require_relative '..\..\..\lib\dendroid\syntax\symbol_seq'
+require_relative '..\..\..\lib\dendroid\syntax\production'
+require_relative '..\..\..\lib\dendroid\grm_analysis\alternative_item'
+
+describe Dendroid::GrmAnalysis::DottedItem do
+  # TODO
+end # describe

data/spec/dendroid/grm_analysis/choice_items_spec.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative '..\..\spec_helper'
+require_relative '..\..\..\lib\dendroid\syntax\terminal'
+require_relative '..\..\..\lib\dendroid\syntax\non_terminal'
+require_relative '..\..\..\lib\dendroid\syntax\symbol_seq'
+require_relative '..\..\..\lib\dendroid\syntax\choice'
+# require_relative '..\..\..\lib\dendroid\grm_analysis\alternative_item'
+require_relative '..\..\..\lib\dendroid\grm_analysis\choice_items'
+
+describe Dendroid::GrmAnalysis::ChoiceItems do
+  let(:num_symb) { Dendroid::Syntax::Terminal.new('NUMBER') }
+  let(:plus_symb) { Dendroid::Syntax::Terminal.new('PLUS') }
+  let(:star_symb) { Dendroid::Syntax::Terminal.new('STAR') }
+  let(:expr_symb) { Dendroid::Syntax::NonTerminal.new('expression') }
+  let(:alt1) { Dendroid::Syntax::SymbolSeq.new([num_symb, plus_symb, num_symb]) }
+  let(:alt2) { Dendroid::Syntax::SymbolSeq.new([num_symb, star_symb, num_symb]) }
+  let(:alt3) { Dendroid::Syntax::SymbolSeq.new([]) }
+  subject do
+    choice = Dendroid::Syntax::Choice.new(expr_symb, [alt1, alt2, alt3])
+    choice.extend(Dendroid::GrmAnalysis::ChoiceItems)
+    choice.build_items
+    choice
+  end
+
+  context 'Methods from mix-in' do
+    it 'builds items for given choice' do
+      expect(subject.items.size).to eq(subject.alternatives.size)
+      subject.items.each_with_index do |itemz, index|
+        expect(itemz.size).to eq(subject.alternatives[index].size + 1)
+      end
+      arr_items = subject.items[1]
+      arr_items.each_with_index do |item, pos|
+        expect(item.rule).to eq(subject)
+        expect(item.position).to eq(pos)
+        expect(item.alt_index).to eq(1)
+      end
+      sole_item = subject.items[2].first # empty alternative...
+      expect(sole_item.rule).to eq(subject)
+      expect(sole_item.position).to eq(0)
+      expect(sole_item.alt_index).to eq(2)
+    end
+
+    it 'returns the first (predicted) items of the choice' do
+      expect(subject.predicted_items.size).to eq(subject.alternatives.size)
+      expectations = [
+        subject.items[0].first,
+        subject.items[1].first,
+        subject.items[2].first
+      ]
+      expect(subject.predicted_items).to eq(expectations)
+    end
+
+    it 'returns the last (reduce) items of the choice' do
+      expect(subject.reduce_items.size).to eq(subject.alternatives.size)
+      expectations = [
+        subject.items[0].last,
+        subject.items[1].last,
+        subject.items[2].last
+      ]
+      expect(subject.reduce_items).to eq(expectations)
+    end
+
+    it 'returns the consecutive item to a given one' do
+      arr_items = subject.items[1]
+      (0..arr_items.size - 1).each do |pos|
+        curr_item = arr_items[pos]
+        next_one = subject.next_item(curr_item)
+        expect(next_one).to eq(arr_items[pos + 1])
+      end
+      expect(subject.next_item(arr_items.last)).to be_nil
+    end
+  end # context
+end # describe

data/spec/dendroid/grm_analysis/grm_analyzer_spec.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../../../lib/dendroid/grm_dsl/base_grm_builder'
+require_relative '../../../lib/dendroid/grm_analysis/grm_analyzer'
+
+module SampleGrammars
+  def grammar_l1
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # Grammar inspired from Wikipedia entry on Earley parsing
+      declare_terminals('PLUS', 'STAR', 'INTEGER')
+
+      rule('p' => 's')
+      rule('s' => ['s PLUS m', 'm'])
+      rule('m' => ['m STAR t', 't'])
+      rule('t' => 'INTEGER')
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l1
+    Utils::BaseTokenizer.new do
+      map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
+
+      scan_verbatim(['+', '*'])
+      scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
+    end
+  end
+
+  def grammar_l2
+    builder = GrmDSL::BaseGrmBuilder.new do
+      # Grammar inspired from Loup Vaillant's example
+      # https://loup-vaillant.fr/tutorials/earley-parsing/recogniser
+      declare_terminals('PLUS', 'MINUS',  'STAR', 'SLASH')
+      declare_terminals('LPAREN', 'RPAREN', 'NUMBER')
+
+      rule('p' => 'sum')
+      rule('sum' => ['sum PLUS product', 'sum MINUS product', 'product'])
+      rule('product' => ['product STAR factor', 'product SLASH factor', 'factor'])
+      rule('factor' => ['LPAREN sum RPAREN', 'NUMBER'])
+    end
+
+    builder.grammar
+  end
+
+  def tokenizer_l2
+    Utils::BaseTokenizer.new do
+      map_verbatim2terminal({
+                              '+' => :PLUS,
+                              '-' => :MINUS,
+                              '*' => :STAR,
+                              '/' => :SLASH,
+                              '(' => :LPAREN,
+                              ')' => :RPAREN
+                            })
+
+      scan_verbatim(['+', '-', '*', '/', '(', ')'])
+      scan_value(/\d+/, :NUMBER, ->(txt) { txt.to_i })
+    end
+  end
+
+  def grammar_l3
+    builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
+      # Grammar inspired from Andrew Appel's example
+      # Modern Compiler Implementation in Java
+      declare_terminals('a', 'c', 'd')
+
+      rule('Z' => ['d', 'X Y Z'])
+      rule('Y' => ['', 'c'])
+      rule('X' => %w[Y a])
+    end
+
+    builder.grammar
+  end
+end # module
+
+describe Dendroid::GrmAnalysis::GrmAnalyzer do
+  include SampleGrammars
+  let(:grammar) { grammar_l1 }
+
+  subject { described_class.new(grammar) }
+
+  context 'Initialization:' do
+    it 'is initialized with a grammar' do
+      expect { described_class.new(grammar) }.not_to raise_error
+    end
+
+    it 'knows its related grammar' do
+      expect(subject.grammar).to eq(grammar)
+    end
+
+    it 'knows the dotted items' do
+      item_count = subject.grammar.rules.reduce(0) do |count, prod|
+        count + prod.items.flatten.size
+      end
+      expect(subject.items.size).to eq(item_count)
+      expected_items = [
+        'p => . s',
+        'p => s .',
+        's => . s PLUS m',
+        's => s . PLUS m',
+        's => s PLUS . m',
+        's => s PLUS m .',
+        's => . m',
+        's => m .',
+        'm => . m STAR t',
+        'm => m . STAR t',
+        'm => m STAR . t',
+        'm => m STAR t .',
+        'm => . t',
+        'm => t .',
+        't => . INTEGER',
+        't => INTEGER .'
+      ]
+      expect(subject.items.map(&:to_s)).to eq(expected_items)
+    end
+
+    it 'knows the item that follows a given dotted item' do
+      first_item = subject.items.find { |itm| itm.to_s == 'm => . m STAR t' }
+      second = subject.next_item(first_item)
+      expect(second.to_s).to eq('m => m . STAR t')
+      third = subject.next_item(second)
+      expect(third.to_s).to eq('m => m STAR . t')
+      fourth = subject.next_item(third)
+      expect(fourth.to_s).to eq('m => m STAR t .')
+      expect(subject.next_item(fourth)).to be_nil
+    end
+  end # context
+
+  context 'Provided services:' do
+    subject { described_class.new(grammar_l3) }
+    it 'constructs the FIRST sets of grammar symbols' do
+      expectations = {
+        'a' => ['a'],
+        'c' => ['c'],
+        'd' => ['d'],
+        'X' => %w[a c], # Add epsilon
+        'Y' => ['c'], # Add epsilon
+        'Z' => %w[a c d]
+      }
+      expectations.each_pair do |sym_name, first_names|
+        symb = subject.grammar.name2symbol[sym_name]
+        expected_first = first_names.map { |name| subject.grammar.name2symbol[name] }
+        expected_first << subject.epsilon if sym_name =~ /[XY]/
+        expect(subject.first_sets[symb]).to eq(Set.new(expected_first))
+      end
+    end
+
+    it 'constructs the FOLLOW sets for non-terminal symbols' do
+      expectations = {
+        'Z' => [], # Add $$
+        'Y' => %w[a c d],
+        'X' => %w[a c d]
+      }
+      subject.send(:build_follow_sets)
+      expectations.each_pair do |sym_name, follow_names|
+        symb = subject.grammar.name2symbol[sym_name]
+        expected_follow = follow_names.map { |name| subject.grammar.name2symbol[name] }
+        expected_follow << subject.endmarker if sym_name == 'Z'
+        expect(subject.follow_sets[symb]).to eq(Set.new(expected_follow))
+      end
+    end
+  end # context
+end # describe

data/spec/dendroid/grm_analysis/production_items_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../../../lib/dendroid/syntax/terminal'
+require_relative '../../../lib/dendroid/syntax/non_terminal'
+require_relative '../../../lib/dendroid/syntax/symbol_seq'
+require_relative '../../../lib/dendroid/syntax/production'
+require_relative '../../../lib/dendroid/grm_analysis/production_items'
+
+describe Dendroid::GrmAnalysis::ProductionItems do
+  let(:num_symb) { Dendroid::Syntax::Terminal.new('NUMBER') }
+  let(:plus_symb) { Dendroid::Syntax::Terminal.new('PLUS') }
+  let(:expr_symb) { Dendroid::Syntax::NonTerminal.new('expression') }
+  let(:rhs) { Dendroid::Syntax::SymbolSeq.new([num_symb, plus_symb, num_symb]) }
+  let(:empty_body) { Dendroid::Syntax::SymbolSeq.new([]) }
+  let(:prod) { Dendroid::Syntax::Production.new(expr_symb, rhs) }
+  let(:empty_prod) do
+    e = Dendroid::Syntax::Production.new(expr_symb, empty_body)
+    e.extend(Dendroid::GrmAnalysis::ProductionItems)
+    e.build_items
+    e
+  end
+
+  subject do
+    prod.extend(Dendroid::GrmAnalysis::ProductionItems)
+    prod.build_items
+    prod
+  end
+
+  context 'Methods from mix-in' do
+    it 'builds items for given non-empty production' do
+      expect(subject.items.size).to eq(subject.body.size + 1)
+      subject.items.each_with_index do |item, index|
+        expect(item.rule).to eq(subject)
+        expect(item.position).to eq(index)
+      end
+    end
+
+    it 'builds the item for given empty production' do
+      expect(empty_prod.items.size).to eq(1)
+      expect(empty_prod.items[0].rule).to eq(empty_prod)
+      expect(empty_prod.items[0].position).to eq(0)
+    end
+
+    it 'returns the first (predicted) item of the production' do
+      expect(subject.predicted_items).to eq([subject.items.first])
+      expect(empty_prod.predicted_items).to eq([empty_prod.items.first])
+    end
+
+    it 'returns the last (reduce) item of the production' do
+      expect(subject.reduce_items).to eq([subject.items.last])
+      expect(empty_prod.reduce_items).to eq([empty_prod.items.first])
+    end
+
+    # rubocop: disable Style/EachForSimpleLoop
+    it 'returns the consecutive item to a given one' do
+      (0..2).each do |pos|
+        curr_item = subject.items[pos]
+        next_one = subject.next_item(curr_item)
+        expect(next_one).to eq(subject.items[pos + 1])
+      end
+      expect(subject.next_item(subject.items[-1])).to be_nil
+
+      expect(empty_prod.next_item(empty_prod.items[-1])).to be_nil
+    end
+    # rubocop: enable Style/EachForSimpleLoop
+  end # context
+end # describe

data/spec/dendroid/lexical/literal_spec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative '..\..\spec_helper'
+require_relative '..\..\..\lib\dendroid\lexical\token_position'
+require_relative '..\..\..\lib\dendroid\lexical\literal'
+
+describe Dendroid::Lexical::Literal do
+  let(:ex_source) { '42' }
+  let(:ex_pos) { Dendroid::Lexical::TokenPosition.new(2, 5) }
+  let(:ex_terminal) { :INTEGER }
+  let(:ex_value) { 42 }
+  subject { described_class.new(ex_source, ex_pos, ex_terminal, ex_value) }
+
+  context 'Initialization:' do
+    it 'is initialized with a text, position, symbol name and value' do
+      expect { described_class.new(ex_source, ex_pos, ex_terminal, ex_value) }.not_to raise_error
+    end
+
+    it 'knows its value' do
+      expect(subject.value).to eq(ex_value)
+    end
+  end # context
+end # describe

data/spec/dendroid/lexical/token_position_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../../../lib/dendroid/lexical/token_position'
+
+describe Dendroid::Lexical::TokenPosition do
+  let(:ex_lineno) { 5 }
+  let(:ex_column) { 7 }
+
+  subject { described_class.new(ex_lineno, ex_column) }
+
+  context 'Initialization:' do
+    it 'is initialized with a line number and a column position' do
+      expect { described_class.new(ex_lineno, ex_column) }.not_to raise_error
+    end
+
+    it 'knows its line number' do
+      expect(subject.lineno).to eq(ex_lineno)
+    end
+
+    it 'knows its column number' do
+      expect(subject.column).to eq(ex_column)
+    end
+  end # context
+
+  context 'Provided services:' do
+    it 'renders a String representation of itself' do
+      expect(subject.to_s).to eq("#{ex_lineno}:#{ex_column}")
+    end
+  end # context
+end # describe

data/spec/dendroid/lexical/token_spec.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../../../lib/dendroid/lexical/token_position'
+require_relative '../../../lib/dendroid/lexical/token'
+
+describe Dendroid::Lexical::Token do
+  let(:ex_source) { 'else' }
+  let(:ex_pos) { Dendroid::Lexical::TokenPosition.new(2, 5) }
+  let(:ex_terminal) { 'ELSE' }
+  subject { described_class.new(ex_source, ex_pos, ex_terminal) }
+
+  context 'Initialization:' do
+    it 'is initialized with a text, position and symbol name' do
+      expect { described_class.new(ex_source, ex_pos, ex_terminal) }.not_to raise_error
+    end
+
+    it 'knows its source text' do
+      expect(subject.source).to eq(ex_source)
+    end
+
+    it 'knows its position' do
+      expect(subject.position).to eq(ex_pos)
+      expect(subject.pos_to_s).to eq('2:5')
+    end
+
+    it 'knows the terminal name' do
+      expect(subject.terminal).to eq(ex_terminal)
+    end
+  end # context
+end # describe

data/spec/dendroid/utils/base_tokenizer_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative '../../spec_helper'
+require_relative '../../../lib/dendroid/utils/base_tokenizer'
+
+describe Dendroid::Utils::BaseTokenizer do
+  # Implements a dotted item: expression => NUMBER . PLUS NUMBER
+  subject { described_class.new }
+
+  context 'Initialization:' do
+    it 'is initialized with an optional block' do
+      expect { described_class.new }.not_to raise_error
+    end
+
+    it 'has a scanner at start' do
+      expect(subject.scanner).to be_kind_of(StringScanner)
+    end
+
+    it 'initializes actions to defaults' do
+      expect(subject.actions).to be_member(:skip_nl)
+      expect(subject.actions).to be_member(:skip_ws)
+    end
+  end # context
+
+  context 'Tokenizing:' do
+    subject do
+      described_class.new do
+        scan_verbatim(['+', '*'])
+        scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
+        map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
+      end
+    end
+
+    it 'generates a sequence of tokens from a simple input' do
+      subject.input = '2 + 3 * 4'
+
+      expectations = [
+        ['1:1', '2', :INTEGER, 2],
+        ['1:3', '+', :PLUS, nil],
+        ['1:5', '3', :INTEGER, 3],
+        ['1:7', '*', :STAR, nil],
+        ['1:9', '4', :INTEGER, 4]
+      ]
+      expectations.each do |tuple|
+        tok = subject.next_token
+        %i[pos_to_s source terminal value].each_with_index do |message, index|
+          expect(tok.send(message)).to eq(tuple[index]) unless tuple[index].nil?
+        end
+      end
+
+      # No more token... 'next_token' method returns nil
+      expect(subject.next_token).to be_nil
+    end
+  end # context
+end # describe

data/version.txt

@@ -1 +1 @@
-0.0.8
+0.0.10

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: dendroid
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.10
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-30 00:00:00.000000000 Z
+date: 2023-11-01 00:00:00.000000000 Z
 dependencies: []
-description: WIP. A Ruby implementation of a Earley parser
+description: WIP. A Ruby implementation of an Earley parser
 email: famished.tiger@yahoo.com
 executables: []
 extensions: []
@@ -24,8 +24,15 @@ files:
 - bin/dendroid
 - dendroid.gemspec
 - lib/dendroid.rb
+- lib/dendroid/grm_analysis/alternative_item.rb
+- lib/dendroid/grm_analysis/choice_items.rb
 - lib/dendroid/grm_analysis/dotted_item.rb
+- lib/dendroid/grm_analysis/grm_analyzer.rb
+- lib/dendroid/grm_analysis/production_items.rb
 - lib/dendroid/grm_dsl/base_grm_builder.rb
+- lib/dendroid/lexical/literal.rb
+- lib/dendroid/lexical/token.rb
+- lib/dendroid/lexical/token_position.rb
 - lib/dendroid/syntax/choice.rb
 - lib/dendroid/syntax/grammar.rb
 - lib/dendroid/syntax/grm_symbol.rb
@@ -34,8 +41,16 @@ files:
 - lib/dendroid/syntax/rule.rb
 - lib/dendroid/syntax/symbol_seq.rb
 - lib/dendroid/syntax/terminal.rb
+- lib/dendroid/utils/base_tokenizer.rb
+- spec/dendroid/grm_analysis/alternative_item_spec.rb
+- spec/dendroid/grm_analysis/choice_items_spec.rb
 - spec/dendroid/grm_analysis/dotted_item_spec.rb
+- spec/dendroid/grm_analysis/grm_analyzer_spec.rb
+- spec/dendroid/grm_analysis/production_items_spec.rb
 - spec/dendroid/grm_dsl/base_grm_builder_spec.rb
+- spec/dendroid/lexical/literal_spec.rb
+- spec/dendroid/lexical/token_position_spec.rb
+- spec/dendroid/lexical/token_spec.rb
 - spec/dendroid/syntax/choice_spec.rb
 - spec/dendroid/syntax/grammar_spec.rb
 - spec/dendroid/syntax/grm_symbol_spec.rb
@@ -44,6 +59,7 @@ files:
 - spec/dendroid/syntax/rule_spec.rb
 - spec/dendroid/syntax/symbol_seq_spec.rb
 - spec/dendroid/syntax/terminal_spec.rb
+- spec/dendroid/utils/base_tokenizer_spec.rb
 - spec/spec_helper.rb
 - version.txt
 homepage: https://github.com/famished-tiger/Dendroid
@@ -68,5 +84,5 @@ requirements: []
 rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
-summary: Dendroid. TODO
+summary: WIP. A Ruby implementation of an Earley parser
 test_files: []