sequitur 0.1.10 → 0.1.11

data/lib/sequitur/symbol_sequence.rb

@@ -1,182 +1,182 @@
-module Sequitur # Module for classes implementing the Sequitur algorithm
-  # Represents a sequence (concatenation) of grammar symbols
-  # as they appear in rhs of productions
-  class SymbolSequence
-    # The sequence of symbols itself
-    attr_reader(:symbols)
-
-    # Create an empty sequence
-    def initialize()
-      @symbols = []
-    end
-    
-    # Copy constructor invoked by dup or clone methods.
-    # @param orig [SymbolSequence]
-    def initialize_copy(orig)
-      # Deep copy to avoid the aliasing of production reference
-      @symbols = orig.symbols.map do |sym|
-        sym.is_a?(Symbol) ? sym : sym.dup
-      end
-      invalidate_refs
-    end
-
-    public
-    
-    # Clear the symbol sequence.
-    def clear()
-      refs = references
-      refs.each(&:unbind)      
-      @symbols = []
-      invalidate_refs
-    end
-
-    # Tell whether the sequence is empty.
-    # @return [true / false] true only if the sequence has no symbol in it.
-    def empty?()
-      return symbols.empty?
-    end
-
-    # Count the number of elements in the sequence.
-    #  @return [Fixnum] the number of elements
-    def size()
-      return symbols.size
-    end
-    
-    # Append a grammar symbol at the end of the sequence.
-    # @param aSymbol [Object] The symbol to append.
-    def <<(aSymbol)
-      symbols << aSymbol
-      if aSymbol.is_a?(ProductionRef)
-        @memo_references ||= []
-        @memo_references << aSymbol
-      end
-    end
-
-    # Retrieve the element from the sequence at given position.
-    # @param anIndex [Fixnum] A zero-based index of the element to access.
-    def [](anIndex)
-      return symbols[anIndex]
-    end
-
-    # Equality testing.
-    # @param other [SymbolSequence or Array] the other other sequence
-    #   to compare to.
-    # @return true when an item from self equals the corresponding
-    #   item from 'other'
-    def ==(other)
-      return true if object_id == other.object_id
-
-      case other
-        when SymbolSequence
-          same = symbols == other.symbols
-        when Array
-          same = symbols == other
-      else
-        same = false
-      end
-
-      return same
-    end
-
-
-    # Select the references to production appearing in the rhs.
-    # @return [Array of ProductionRef]
-    def references()
-      @memo_references ||= symbols.select { |symb| symb.is_a?(ProductionRef) }
-      return @memo_references
-    end
-
-
-    # Select the references of the given production appearing in the rhs.
-    # @param aProduction [Production]
-    # @return [Array of ProductionRef]
-    def references_of(aProduction)
-      return [] if references.empty?
-      result = references.select { |a_ref| a_ref == aProduction }
-      return result
-    end
-
-
-    # Emit a text representation of the symbol sequence.
-    # Text is of the form: space-separated sequence of symbols.
-    # @return [String]
-    def to_string()
-      rhs_text = symbols.map do |elem|
-        case elem
-          when String then "'#{elem}'"
-          else elem.to_s
-        end
-      end
-
-      return rhs_text.join(' ')
-    end
-  
-    # Insert at position the elements from another sequence.
-    # @param position [Fixnum] A zero-based index of the symbols to replace.
-    # @param another [SymbolSequence] A production with a two-elements rhs
-    #   (a single digram).
-    def insert_at(position, another)
-      klone = another.dup
-      symbols.insert(position, *klone.symbols)
-      invalidate_refs
-    end
-    
-    # Given that the production P passed as argument has exactly 2 symbols
-    #   in its rhs s1 s2, substitute in the rhs of self all occurrences of 
-    #   s1 s2 by a reference to P.
-    # @param index [Fixnum] the position of a two symbol sequence to be replaced
-    #   by the production
-    # @param aProduction [Production or ProductionRef] a production that
-    #   consists exactly of one digram (= 2 symbols).
-    def reduce_step(index, aProduction)
-      if symbols[index].is_a?(ProductionRef)
-        symbols[index].bind_to(aProduction)
-      else
-        new_ref = ProductionRef.new(aProduction)
-        symbols[index] = new_ref
-        @memo_references ||= []
-        @memo_references << new_ref
-      end
-      index1 = index + 1
-      if symbols[index1].is_a?(ProductionRef)
-        symbols[index1].unbind
-        invalidate_refs
-      end
-      delete_at(index1)
-    end
-
-    # Remove the element at given position
-    # @param position [Fixnum] a zero-based index.
-    def delete_at(position)
-      invalidate_refs if symbols[position].is_a?(ProductionRef)
-      symbols.delete_at(position)
-    end
-
-
-    # Part of the 'visitee' role in Visitor design pattern.
-    # @param aVisitor[GrammarVisitor]
-    def accept(aVisitor)
-      aVisitor.start_visit_rhs(self)
-
-      # Let's proceed with the visit of productions
-      symbols.each do |a_symb|
-        if a_symb.is_a?(ProductionRef)
-          a_symb.accept(aVisitor)
-        else
-          aVisitor.visit_terminal(a_symb)
-        end
-      end
-
-      aVisitor.end_visit_rhs(self)
-    end
-    
-    private
-    
-    def invalidate_refs()
-      @memo_references = nil
-      @lookup_references = nil
-    end
-
-  end # class
-
-end # module
+module Sequitur # Module for classes implementing the Sequitur algorithm
+  # Represents a sequence (concatenation) of grammar symbols
+  # as they appear in rhs of productions
+  class SymbolSequence
+    # The sequence of symbols itself
+    attr_reader(:symbols)
+
+    # Create an empty sequence
+    def initialize()
+      @symbols = []
+    end
+
+    # Copy constructor invoked by dup or clone methods.
+    # @param orig [SymbolSequence]
+    def initialize_copy(orig)
+      # Deep copy to avoid the aliasing of production reference
+      @symbols = orig.symbols.map do |sym|
+        sym.is_a?(ProductionRef) ? sym.dup : sym
+      end
+      invalidate_refs
+    end
+
+    public
+
+    # Clear the symbol sequence.
+    def clear()
+      refs = references
+      refs.each(&:unbind)
+      @symbols = []
+      invalidate_refs
+    end
+
+    # Tell whether the sequence is empty.
+    # @return [true / false] true only if the sequence has no symbol in it.
+    def empty?()
+      return symbols.empty?
+    end
+
+    # Count the number of elements in the sequence.
+    #  @return [Fixnum] the number of elements
+    def size()
+      return symbols.size
+    end
+
+    # Append a grammar symbol at the end of the sequence.
+    # @param aSymbol [Object] The symbol to append.
+    def <<(aSymbol)
+      symbols << aSymbol
+      if aSymbol.is_a?(ProductionRef)
+        @memo_references ||= []
+        @memo_references << aSymbol
+      end
+    end
+
+    # Retrieve the element from the sequence at given position.
+    # @param anIndex [Fixnum] A zero-based index of the element to access.
+    def [](anIndex)
+      return symbols[anIndex]
+    end
+
+    # Equality testing.
+    # @param other [SymbolSequence or Array] the other other sequence
+    #   to compare to.
+    # @return true when an item from self equals the corresponding
+    #   item from 'other'
+    def ==(other)
+      return true if object_id == other.object_id
+
+      case other
+        when SymbolSequence
+          same = symbols == other.symbols
+        when Array
+          same = symbols == other
+      else
+        same = false
+      end
+
+      return same
+    end
+
+
+    # Select the references to production appearing in the rhs.
+    # @return [Array of ProductionRef]
+    def references()
+      @memo_references ||= symbols.select { |symb| symb.is_a?(ProductionRef) }
+      return @memo_references
+    end
+
+
+    # Select the references of the given production appearing in the rhs.
+    # @param aProduction [Production]
+    # @return [Array of ProductionRef]
+    def references_of(aProduction)
+      return [] if references.empty?
+      result = references.select { |a_ref| a_ref == aProduction }
+      return result
+    end
+
+
+    # Emit a text representation of the symbol sequence.
+    # Text is of the form: space-separated sequence of symbols.
+    # @return [String]
+    def to_string()
+      rhs_text = symbols.map do |elem|
+        case elem
+          when String then "'#{elem}'"
+          else elem.to_s
+        end
+      end
+
+      return rhs_text.join(' ')
+    end
+
+    # Insert at position the elements from another sequence.
+    # @param position [Fixnum] A zero-based index of the symbols to replace.
+    # @param another [SymbolSequence] A production with a two-elements rhs
+    #   (a single digram).
+    def insert_at(position, another)
+      klone = another.dup
+      symbols.insert(position, *klone.symbols)
+      invalidate_refs
+    end
+
+    # Given that the production P passed as argument has exactly 2 symbols
+    #   in its rhs s1 s2, substitute in the rhs of self all occurrences of
+    #   s1 s2 by a reference to P.
+    # @param index [Fixnum] the position of a two symbol sequence to be replaced
+    #   by the production
+    # @param aProduction [Production or ProductionRef] a production that
+    #   consists exactly of one digram (= 2 symbols).
+    def reduce_step(index, aProduction)
+      if symbols[index].is_a?(ProductionRef)
+        symbols[index].bind_to(aProduction)
+      else
+        new_ref = ProductionRef.new(aProduction)
+        symbols[index] = new_ref
+        @memo_references ||= []
+        @memo_references << new_ref
+      end
+      index1 = index + 1
+      if symbols[index1].is_a?(ProductionRef)
+        symbols[index1].unbind
+        invalidate_refs
+      end
+      delete_at(index1)
+    end
+
+    # Remove the element at given position
+    # @param position [Fixnum] a zero-based index.
+    def delete_at(position)
+      invalidate_refs if symbols[position].is_a?(ProductionRef)
+      symbols.delete_at(position)
+    end
+
+
+    # Part of the 'visitee' role in Visitor design pattern.
+    # @param aVisitor[GrammarVisitor]
+    def accept(aVisitor)
+      aVisitor.start_visit_rhs(self)
+
+      # Let's proceed with the visit of productions
+      symbols.each do |a_symb|
+        if a_symb.is_a?(ProductionRef)
+          a_symb.accept(aVisitor)
+        else
+          aVisitor.visit_terminal(a_symb)
+        end
+      end
+
+      aVisitor.end_visit_rhs(self)
+    end
+
+    private
+
+    def invalidate_refs()
+      @memo_references = nil
+      @lookup_references = nil
+    end
+
+  end # class
+
+end # module

data/spec/sequitur/sequitur_grammar_spec.rb

@@ -74,7 +74,7 @@ describe SequiturGrammar do
       expect(p_a.rhs).to eq([:a, :b, :c])
       expect(instance.start.rhs).to eq([p_a, p_a])
     end
-    
+
     it 'should cope with a pattern that caused an exception' do
       input = 'aaac'  # This sequence raised an exception
 
@@ -102,6 +102,78 @@ describe SequiturGrammar do
       expect(p3.rhs).to eq(['b', p2, 'e'])
     end
 
+    it 'should work with strings instead of single char input tokens' do
+      # Raw input is sequence of chars
+      raw_input = 'bbebeebebebbebee'
+
+      # Convert them into multichar strings
+      input = raw_input.chars.map do |ch|
+        'letter_' + ch
+      end
+
+      # Creation
+      instance = SequiturGrammar.new(input.to_enum)
+
+      # Expectations:
+      # S: P3 P2 P3
+      # P1: b e
+      # P2: P1 P1
+      # P3: b P2 e
+      expect(instance.productions.size).to eq(4)
+      (p1, p2, p3) = instance.productions[1..3]
+      expect(instance.start.rhs).to eq([p3, p2, p3])
+      expect(p1.rhs).to eq(%w(letter_b letter_e))
+      expect(p2.rhs).to eq([p1, p1])
+      expect(p3.rhs).to eq(['letter_b',p2, 'letter_e'])
+    end
+    
+    it 'should work with Symbol instead of single char input tokens' do
+      # Raw input is sequence of single characters
+      raw_input = 'bbebeebebebbebee'
+
+      # Convert them into symbols
+      input = raw_input.chars.map(&:to_sym)
+
+      # Creation
+      instance = SequiturGrammar.new(input.to_enum)
+
+      # Expectations:
+      # S: P3 P2 P3
+      # P1: b e
+      # P2: P1 P1
+      # P3: b P2 e
+      expect(instance.productions.size).to eq(4)
+      (p1, p2, p3) = instance.productions[1..3]
+      expect(instance.start.rhs).to eq([p3, p2, p3])
+      expect(p1.rhs).to eq([:b, :e])
+      expect(p2.rhs).to eq([p1, p1])
+      expect(p3.rhs).to eq([:b, p2, :e])
+    end
+
+
+    it 'should work with integer values as input tokens' do
+      # Raw input is sequence of hex digits
+      raw_input = 'bbebeebebebbebee'
+
+      # Convert them into Fixnums
+      input = raw_input.chars.map { |ch| ch.to_i(16) }
+
+      # Creation
+      instance = SequiturGrammar.new(input.to_enum)
+
+      # Expectations:
+      # S: P3 P2 P3
+      # P1: b e
+      # P2: P1 P1
+      # P3: b P2 e
+      expect(instance.productions.size).to eq(4)
+      (p1, p2, p3) = instance.productions[1..3]
+      expect(instance.start.rhs).to eq([p3, p2, p3])
+      expect(p1.rhs).to eq([0xb, 0xe])
+      expect(p2.rhs).to eq([p1, p1])
+      expect(p3.rhs).to eq([0xb, p2, 0xe])
+    end
+
     it 'should cope with the example from sequitur.info website' do
       input = 'abcabdabcabd'
       instance = SequiturGrammar.new(input.chars)
@@ -153,7 +225,7 @@ SNIPPET
       # 2 → h o t                                         hot
       # 3 → 10 1                                          ,↵pease_porridge_
       # 4 → c 11                                          cold
-      # 5 → 12 _ t h 8 t 10 n 12 9 d a y s _ 11 . ↵ 
+      # 5 → 12 _ t h 8 t 10 n 12 9 d a y s _ 11 . ↵
       #   in_the_pot,↵nine_days_old.↵
       # 6 → s o m 9 l i k 9 i t _                         some_like_it_
       # 7 → 10 6                                          ,↵some_like_it_
@@ -196,7 +268,7 @@ SNIPPET
       ].flatten
       expect(p12.rhs).to eq(p12_expectation)  # Rule 5 above
     end
-    
+
     it 'should work with a sequence of Ruby Symbols' do
       input = 'abcabdabcabd'.chars.map(&:to_sym)
       instance = SequiturGrammar.new(input.to_enum)