sequitur 0.0.04

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzhjZGJlNTkxNTYxNGE4ZWU2NGUzNzI1NTI3NWMxNTNjMTdhZjRhZA==
+  data.tar.gz: !binary |-
+    YzI2ZjFiYTk2NmRlYzgyZDI1MmRmNzZmZjQ3NTY3NmY2NjA1MzU2OA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NGE3ODVkMzA0MDVlOGY3MmM4NTU1MDU0NGRhYWU1ZjI1NWRlYjU4MjZkZTEw
+    YjkxMzQ0MTU2ZGRlNTg3NGE3YjkxMjMzNzYwOWVkMWNlZjE1MzI5OTIwMWEw
+    ODE4Njk0MGQ0ZTFmOTRkOTJlY2QwZDVlZTA2YTc5NWYwMzM3NGU=
+  data.tar.gz: !binary |-
+    ZDEyNDU3NjVhZTdkNDM0ZWVhN2IxYWIyZGRlZjE3YzBjZDdjOTRlYWVkZmY0
+    YjliZDJhZWJjMTZjYjUyMzg5MDZmOThjN2ZlMzBlMGI4Y2E4N2ZlMWY2ODA0
+    ODQ2NDQ0YzkzODRiNjdlOTc4NGFiYzk1ZGZlM2I5MTkyZTUwYzA=

data/.rspec

@@ -0,0 +1 @@
+--backtrace

data/.rubocop.yml

@@ -0,0 +1,74 @@
+AllCops:
+  Exclude:
+    - 'examples/**/*'
+    - 'features/**/*'
+    - 'gems/**/*'
+
+# This is disabled because some demos use UTF-8
+AsciiComments:
+  Enabled: false
+
+CaseIndentation:
+  IndentWhenRelativeTo: end
+  IndentOneStep: true
+
+# Rubocop enforces the use of is_a? instead of kind_of?
+# Which is contrary to modelling practice.
+ClassCheck:
+  Enabled: false
+  
+ClassLength:
+  Max: 250
+  CountComments: false  
+
+ConstantName:  
+  Enabled: false
+  
+CyclomaticComplexity:
+  Enabled: false
+  
+DefWithParentheses:  
+  Enabled: false
+  
+Documentation:
+  Enabled: false
+  
+EmptyLines:
+  Enabled: false
+  
+EmptyLinesAroundBody:
+  Enabled: false  
+
+Encoding:
+  Enabled: false
+
+FileName:
+  Enabled: false
+  
+IndentationWidth :
+  Enabled: false
+  
+  
+# Avoid methods longer than 50 lines of code
+MethodLength:
+  Max: 50
+  CountComments: false 
+ 
+NonNilCheck:
+  Enabled: false
+
+NumericLiterals:
+  Enabled: false
+  
+RaiseArgs:
+  Enabled: false
+  
+RedundantReturn:
+  Enabled: false
+
+SpaceInsideBrackets:
+  Enabled: false
+
+TrailingWhitespace:
+  Enabled: false
+  

data/.ruby-gemset

@@ -0,0 +1 @@
+sequitur

data/.ruby-version

@@ -0,0 +1 @@
+1.9.3

data/.simplecov

@@ -0,0 +1,7 @@
+# .simplecov
+# Configuration
+
+SimpleCov.start do
+  # Remove all files that match /spec/ in their path
+  add_filter "/spec/"
+end

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+# TODO: add JRuby (after enabling ObjectSpace)
+rvm:
+  - 2.1.0
+  - 2.0.0
+  - 1.9.3
+  - 1.9.2
+
+gemfile:
+  - Gemfile
+  
+# whitelist
+branches:  
+  only:
+    - master

data/.yardopts

@@ -0,0 +1,6 @@
+--exclude examples --exclude features --exclude spec
+--no-private
+--markup markdown
+-
+Changelog.md
+License.txt

data/CHANGELOG.md

@@ -0,0 +1,12 @@
+### 0.0.04 / 2014-08-24
+* [FIX] `.travis.yml`: removed JRuby from Travis CI. Rationale: ObjectSpace class is disabled!
+
+### 0.0.03 / 2014-08-24
+* [FIX] `Rakefile`: removed unused Cucumber-based task
+
+### 0.0.02 / 2014-08-24
+* [CHANGE] `README.md`: added badge from Travis CI
+
+### 0.0.01 / 2014-08-24
+
+* [FEATURE] Initial public working version

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+# Prevent Bundler to load the dependencies from our .gemspec file
+
+
+group :development do
+  gem 'rake',  '>= 0.8.0' 
+  gem 'rspec', '>= 3.0.0'
+  gem 'simplecov', '>= 0.5.0'
+end

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2014 Dimitri Geshef
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,10 @@
+Sequitur
+===========
+_Ruby gem implementing the Sequitur algorithm_  
+[Homepage](https://github.com/famished-tiger/Sequitur)  
+
+[![Build Status](https://travis-ci.org/famished-tiger/Sequitur.svg?branch=master)](https://travis-ci.org/famished-tiger/Sequitur)
+
+Copyright
+---------
+Copyright (c) 2014, Dimitri Geshef. Sequitur is released under the MIT License see [LICENSE.txt](https://github.com/famished-tiger/Sequitur/blob/master/LICENSE.txt) for details.

data/Rakefile

@@ -0,0 +1,31 @@
+require 'rubygems'
+require_relative './lib/sequitur/constants'
+
+namespace :gem do
+
+desc 'Push the gem to rubygems.org'
+task :push do
+  system("gem push sequitur-#{Sequitur::Version}.gem")
+end
+
+end # namespace
+
+# Testing-specific tasks
+
+# RSpec as testing tool
+require 'rspec/core/rake_task'
+desc 'Run RSpec'
+RSpec::Core::RakeTask.new do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+
+# Run RSpec tests
+desc 'Run tests, with RSpec'
+task test: [:spec]
+
+
+# Default rake task
+task default: :test
+
+# End of file

data/lib/sequitur.rb

@@ -0,0 +1,8 @@
+# File: sequitur.rb
+# This file acts as a jumping-off point for loading dependencies expected
+# for a MSequitur client.
+
+require_relative './sequitur/constants'
+require_relative './sequitur/sequitur-grammar'
+
+# End of file

data/lib/sequitur/constants.rb

@@ -0,0 +1,26 @@
+# File: constants.rb
+# Purpose: definition of Sequitur constants.
+
+module Sequitur # Module used as a namespace
+  # The version number of the gem.
+  Version = '0.0.04'
+
+  # Brief description of the gem.
+  Description = 'Ruby implementation of the Sequitur algorithm'
+
+  # Constant Sequitur::RootDir contains the absolute path of Sequitur's
+  # root directory. Note: it also ends with a slash character.
+  unless defined?(RootDir)
+    # The initialisation of constant RootDir is guarded in order
+    # to avoid multiple initialisation (not allowed for constants)
+
+    # The root folder of Sequitur.
+    RootDir = begin
+      require 'pathname' # Load Pathname class from standard library
+      rootdir = Pathname(__FILE__).dirname.parent.parent.expand_path
+      rootdir.to_s + '/' # Append trailing slash character to it
+    end
+  end
+end # module
+
+# End of file

data/lib/sequitur/digram.rb

@@ -0,0 +1,29 @@
+module Sequitur # Module for classes implementing the Sequitur algorithm
+
+# A digram is a sequence of two grammar symbols (terminal or not).
+class Digram
+  # The sequence of two consecutive grammar symbols.
+  attr_reader(:symbols)
+  
+  # The object id of the production that contains this digram in its rhs.
+  attr_reader(:production_id)
+  
+  # An unique Hash key of the digram
+  attr_reader(:key)
+  
+  # Constructor.
+  def initialize(symbol1, symbol2, aProduction)
+    @symbols = [symbol1, symbol2]
+    @key = "#{symbol1.hash.to_s(16)}:#{symbol2.hash.to_s(16)}" 
+    @production_id = aProduction.object_id
+  end
+  
+  # Return the production object of this digram
+  def production()
+    ObjectSpace._id2ref(production_id)
+  end
+end # class
+
+end # module
+
+# End of file

data/lib/sequitur/dynamic-grammar.rb

@@ -0,0 +1,126 @@
+require_relative 'production'
+
+module Sequitur # Module for classes implementing the Sequitur algorithm
+
+class DynamicGrammar
+  # Link to the root - start - production.
+  attr_reader(:root)
+
+  # The set of production rules of the grammar
+  attr_reader(:productions)
+  
+  # nodoc Trace the execution of the algorithm.
+  attr(:trace, true)
+
+
+  # Constructor.
+  # Build a grammar with one empty rule as start/root rule
+  def initialize()
+    @root = Production.new
+    @productions = [ root ]
+    @trace = false
+  end
+
+  public
+
+  # Emit a text representation of the grammar.
+  # Each production rule is emitted per line.
+  def to_string()
+    rule_text = productions.map(&:to_string).join("\n")
+    return rule_text
+  end
+
+
+  # Add a production to the grammar.
+  def add_production(aProduction)
+    # TODO: remove output
+    puts "Adding #{aProduction.object_id}" if trace
+    puts aProduction.to_string if trace
+    check_rhs_of(aProduction) # TODO: configurable check
+    productions << aProduction
+  end
+
+
+  # Remove a production from the grammar
+  def delete_production(anIndex)
+    puts "Before production removal #{productions[anIndex].object_id}" if trace
+    puts to_string if trace
+    prod = productions.delete_at(anIndex)
+    # TODO: remove output
+    puts prod.to_string if trace
+    prod.clear_rhs
+
+    check_backrefs  # TODO: configurable check
+
+    return prod
+  end
+
+
+  # Add the given token to the grammar.
+  def add_token(aToken)
+    append_symbol_to(root, aToken)
+  end
+
+  protected
+
+  def append_symbol_to(aProduction, aSymbol)
+    aProduction.append_symbol(aSymbol)
+  end
+
+
+  # Check that any production reference in rhs is
+  # pointing to a production of the grammar
+  def check_rhs_of(aProduction)
+    aProduction.references.each do |symb|
+      next if productions.include?(symb)
+
+      msg = "Production #{aProduction.object_id} refers to "
+      msg << "production #{symb.object_id}"
+      msg << " that is not part of the grammar."
+      fail StandardError, msg
+    end
+  end
+
+  # Check the invariants:
+  # Every back reference must must point to a production of the grammar
+  # Every back reference count must be equal to the number
+  # of occurrences in the referencing production.
+  def check_backrefs()
+    return if productions.size < 2
+
+    all_but_root = productions[1...productions.size]
+    all_but_root.each do |a_prod|
+      a_prod.backrefs.each do |other_prod_id, count|
+        begin
+          other_prod = ObjectSpace._id2ref(other_prod_id)
+        rescue RangeError => exc
+          msg = "Production #{a_prod.object_id} has a backref to "
+          msg << "recycled production #{other_prod_id}."
+          msg << "\n#{to_string}"
+          $stderr.puts msg
+          raise exc
+        end
+        found = productions.find { |elem| elem == other_prod }
+        unless found
+          msg = "Production #{a_prod.object_id} is referenced by the "
+          msg << "unknown production (#{other_prod_id})."
+          msg << "\n#{to_string}"
+          fail StandardError, msg
+        end
+
+        unless count == found.rhs.count { |symb| symb == a_prod }
+          msg = "Production #{a_prod.object_id} has a count mismatch"
+          msg << "\nIt expects #{count} references in rhs of #{other_prod_id} "
+          msg << "but actual count is #{other_prod.rhs.count}."
+          msg << "\n#{to_string}"
+          fail StandardError, msg
+        end
+      end
+    end
+  end
+
+end # class
+
+end # module
+
+# End of file

data/lib/sequitur/production.rb

@@ -0,0 +1,202 @@
+require_relative 'digram'
+
+module Sequitur # Module for classes implementing the Sequitur algorithm
+
+
+# In a context-free grammar, a production is a rule in which
+# its left-hand side (LHS) consists solely of a non-terminal symbol
+# and the right-hand side (RHS) consists of a sequence of symbols.
+# The symbols in RHS can be either terminal or non-terminal symbols.
+# The rule stipulates that the LHS is equivalent to the RHS,
+# in other words every occurrence of the LHS can be substituted to
+# corresponding RHS.
+# The object id of the production is taken as its LHS.
+class Production
+  # The right-hand side (rhs) consists of a sequence of grammar symbols
+  attr_reader(:rhs)
+
+  # A Hash with pairs of the form:
+  # production id => reference count
+  # Where the reference count is the number of times this production
+  # appears in the rhs of the production with given id.
+  attr_reader(:backrefs)
+
+  # Constructor. Build a production with an empty RHS.
+  def initialize()
+    clear_rhs
+    @backrefs = {}
+  end
+
+  public
+
+  # Is the rhs empty?
+  def empty?
+    return rhs.empty?
+  end
+
+
+  # Return the set of productions appearing in the rhs
+  def references()
+    return rhs.select { |symb| symb.kind_of?(Production) }
+  end
+
+
+  # Does the rhs have exactly one digram only (= 2 symbols)?
+  def single_digram?
+    return rhs.size == 2
+  end
+
+
+  # Detect whether the last digram occurs twice
+  # Assumption: when a digram occurs twice in a production then it must occur
+  # at the end of the rhs
+  def repeated_digram?
+    return false if rhs.size < 3
+
+    my_digrams = digrams
+    all_keys = my_digrams.map(&:key)
+    last_key = all_keys.pop
+    same_key_found = all_keys.index(last_key)
+    return !same_key_found.nil?
+  end
+  
+  # Return the last digram appearing in the RHS.
+  def last_digram()
+    return nil if rhs.size < 2
+    
+    return Digram.new(rhs[-2], rhs[-1], self)
+  end
+
+
+
+  # The back reference count is the number of times this production
+  # appears in the rhs of all the productions of the grammar
+  def refcount()
+    total = backrefs.values.reduce(0) do |sub_result, count|
+      sub_result += count
+    end
+
+    return total
+  end
+
+  # Add a back reference to the given production.
+  def add_backref(aProduction)
+    prod_id = aProduction.object_id
+
+    count = backrefs.fetch(prod_id, 0)
+    backrefs[prod_id] = count + 1
+    return count
+  end
+
+  # Decrement the reference count for the given production.
+  # If result is zero, then the entry is removed from the Hash.
+  def remove_backref(aProduction)
+    prod_id = aProduction.object_id
+
+    count = backrefs.fetch(prod_id)
+    fail StandardError if count < 1
+
+    if count > 1
+      backrefs[prod_id] = count - 1
+    else
+      backrefs.delete(prod_id)
+    end
+
+    return count
+  end
+
+  # Emit a text representation of the production rule.
+  # Text is of the form:
+  # object id of production : rhs as space-separated sequence of symbols.
+  def to_string()
+    rhs_text = rhs.map do |elem|
+      case elem
+        when String then "'#{elem}'"
+        when Production then "#{elem.object_id}"
+      else "#{elem}"
+      end
+    end
+
+    return "#{object_id} : #{rhs_text.join(' ')}."
+  end
+
+  # Return the digrams for this production as if
+  # the given symbol is appended at the end of the rhs
+  def calc_append_symbol(aSymbol)
+    return [] if empty?
+
+    return digrams + [ Digram.new(rhs.last, aSymbol, self) ]
+  end
+
+  def append_symbol(aSymbol)
+    aSymbol.add_backref(self) if aSymbol.kind_of?(Production)
+    rhs << aSymbol
+  end
+
+  # Clear the right-hand side.
+  # Any referenced production has its back reference counter decremented
+  def clear_rhs()
+    if rhs
+      refs = references
+      refs.each { |a_ref| a_ref.remove_backref(self) }
+    end
+    @rhs = []
+  end
+
+  # Return the list digrams found in rhs of this production.
+  def digrams()
+    return [] if rhs.size < 2
+
+    result = []
+    rhs.each_cons(2) { |couple| result << Digram.new(*couple, self) }
+
+    return result
+  end
+
+  # Substitute in self all occurence of the digram that
+  # appears in the rhs of the other production
+  # Pre-condition:
+  # another has a rhs with exactly one digram (= a two-symbol sequence).
+  def replace_digram(another)
+    # Find the positions where the digram occur in rhs
+    (symb1, symb2) = another.rhs
+    indices = [ -2 ] # Dummy index!
+
+    (0...rhs.size).each do |i|
+      next if i == indices.last + 1
+      indices << i if (rhs[i] == symb1) && (rhs[i + 1] == symb2)
+    end
+    indices.shift
+
+    pos = indices.reverse
+
+    # Replace the two symbol sequence by the production
+    pos.each do |index|
+      rhs[index].remove_backref(self) if rhs[index].kind_of?(Production)
+      rhs[index] = another
+      index1 = index + 1
+      rhs[index1].remove_backref(self) if rhs[index1].kind_of?(Production)
+      rhs.delete_at(index1)
+      another.add_backref(self)
+    end
+  end
+
+  # Replace every occurrence of 'another' production in rhs by
+  # the rhs of 'another'.
+  def replace_production(another)
+    (0...rhs.size).to_a.reverse.each do |index|
+      next unless rhs[index] == another
+      rhs.insert(index + 1, *another.rhs)
+      another.rhs.each do |new_symb|
+        new_symb.add_backref(self) if new_symb.kind_of?(Production)
+      end
+      another.remove_backref(self)
+      rhs.delete_at(index)
+    end
+  end
+
+end # class
+
+end # module
+
+# End of file