raingrams 0.0.9 → 0.1.0

data/lib/raingrams/ngram.rb

@@ -1,20 +1,53 @@
+require 'raingrams/extensions'
+
 module Raingrams
   class Ngram < Array
 
-    def initialize(objs)
-      super(objs.map { |obj| obj.to_gram })
+    #
+    # Creates a new Ngram object with the specified _objects_.
+    #
+    def initialize(objects)
+      super(objects.map { |obj| obj.to_gram })
+    end
+
+    #
+    # Creates a new Ngram object from the specified _objects_.
+    #
+    def self.[](*objects)
+      self.new(objects)
+    end
+
+    #
+    # Creates a new Ngram object by appending the specified _grams_ to the
+    # ngram.
+    #
+    def +(grams)
+      if grams.kind_of?(Array)
+        return self.class.new(super(grams.map { |gram|
+          gram.to_gram
+        }))
+      else
+        return self.class.new(super([grams.to_gram]))
+      end
     end
 
-    def self.[](*objs)
-      self.new(objs)
+    def <<(gram)
+      super(gram.to_gram)
     end
 
+    #
+    # Returns the prefix of the ngram.
+    #
     def prefix
       self[0...length-1]
     end
 
-    def prefixed_by?(ngram)
-      prefix==ngram
+    #
+    # Returns +true+ if the ngram is prefixed by the specified
+    # _smaller_ngram_.
+    #
+    def prefixed_by?(smaller_ngram)
+      prefix == smaller_ngram
     end
 
     def postfix
@@ -22,21 +55,25 @@ module Raingrams
     end
 
     def postfixed_by?(ngram)
-      postfix==ngram
+      postfix == ngram
     end
 
     def starts_with?(obj)
-      self[0]==obj.to_gram
+      self.first == obj.to_gram
     end
 
     def ends_with?(obj)
-      self[-1]==obj.to_gram
+      self.last == obj.to_gram
     end
 
     def include?(obj)
       super(obj.to_gram)
     end
 
+    def includes?(*grams)
+      (self & grams) == grams
+    end
+
     def flatten
       self.dup
     end
@@ -49,5 +86,9 @@ module Raingrams
       join(', ')
     end
 
+    def inspect
+      'Ngram[' + self.map { |gram| gram.inspect }.join(', ') + ']'
+    end
+
   end
 end

data/lib/raingrams/ngram_set.rb

@@ -0,0 +1,43 @@
+require 'raingrams/ngram'
+
+require 'set'
+
+module Raingrams
+  class NgramSet < Set
+
+    def select(&block)
+      selected_ngrams = self.class.new
+
+      each do |ngram|
+        selected_ngrams << ngram if block.call(ngram)
+      end
+
+      return selected_ngrams
+    end
+
+    def prefixed_by(prefix)
+      select { |ngram| ngram.prefixed_by?(prefix) }
+    end
+
+    def postfixed_by(postfix)
+      select { |ngram| ngram.postfixed_by?(postfix) }
+    end
+
+    def starts_with(gram)
+      select { |ngram| ngram.starts_with?(gram) }
+    end
+
+    def ends_with(gram)
+      select { |ngram| ngram.ends_with?(gram) }
+    end
+
+    def including(gram)
+      select { |ngram| ngram.include?(gram) }
+    end
+
+    def includes(*grams)
+      select { |ngram| ngram.includes?(*grams) }
+    end
+
+  end
+end

data/lib/raingrams/open_vocabulary/model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/open_vocabulary/open_model'
+require 'raingrams/model'
+
+module Raingrams
+  module OpenVocabulary
+    class Model < Raingrams::Model
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/open_model.rb

@@ -7,14 +7,18 @@ module Raingrams
       # The fixed lexicon of this model
       attr_reader :lexicon
 
-      def initialize(opts={},&block)
-        @lexicon = opts[:lexicon] || []
+      def initialize(options={},&block)
+        @lexicon = (options[:lexicon] || [])
 
-        super(opts,&block)
+        @lexicon.map! do |word|
+          word.to_gram
+        end
+
+        super(options,&block)
       end
 
       def within_lexicon?(gram)
-        @lexicon.include?(gram)
+        @lexicon.include?(gram.to_gram)
       end
 
       def train_ngram(ngram)

data/lib/raingrams/open_vocabulary.rb

@@ -1,4 +1,3 @@
-require 'raingrams/openvocabulary/unigram_model'
 require 'raingrams/openvocabulary/multigram_model'
 require 'raingrams/openvocabulary/bigram_model'
 require 'raingrams/openvocabulary/trigram_model'

data/lib/raingrams/pentagram_model.rb

@@ -1,13 +1,9 @@
-require 'raingrams/multigram_model'
+require 'raingrams/model'
 
 module Raingrams
-  class PentagramModel < MultigramModel
+  class PentagramModel < Model
 
-    def initialize(opts={},&block)
-      opts[:ngram_size] = 5
-
-      super(opts,&block)
-    end
+    ngram_size 5
 
   end
 end

data/lib/raingrams/probability_table.rb

@@ -0,0 +1,153 @@
+module Raingrams
+  class ProbabilityTable
+
+    # Indicates wether the table needs to be rebuilt
+    attr_reader :dirty
+
+    # Frequencies of grams
+    attr_reader :frequencies
+
+    # Probabilities of grams
+    attr_reader :probabilities
+
+    #
+    # Creates a new empty ProbabilityTable object.
+    #
+    def initialize
+      @dirty = false
+      @total = 0
+      @frequencies = {}
+      @probabilities = {}
+    end
+
+    #
+    # Returns +true+ if the probability table is dirty and needs to be
+    # rebuilt, returns +false+ otherwise.
+    #
+    def dirty?
+      @dirty == true
+    end
+
+    #
+    # Returns +true+ if the probability table contains the specified _gram_,
+    # returns +false+ otherwise.
+    #
+    def has_gram?(gram)
+      @frequencies.has_key?(gram)
+    end
+
+    #
+    # Returns the grams within the probability table.
+    #
+    def grams
+      @frequencies.keys
+    end
+
+    #
+    # Iterates over each gram in the probability table, passing each to the
+    # given _block_.
+    #
+    def each_gram(&block)
+      @frequencies.each_key(&block)
+    end
+
+    #
+    # Returns the frequency of the specified _gram_. Returns +0+ by default.
+    #
+    def frequency_of(gram)
+      @frequencies[gram] || 0
+    end
+
+    #
+    # Returns the probability of the specified _gram_ occurring. Returns
+    # <tt>0.0</tt> by default.
+    #
+    def probability_of(gram)
+      @probabilities[gram] || 0.0
+    end
+
+    alias [] probability_of
+
+    #
+    # Sets the frequency of the specified _gram_ to the specified _value_.
+    #
+    def set_count(gram,value)
+      @dirty = true
+      @frequencies[gram] = value
+    end
+
+    #
+    # Increments the frequency of the specified _gram_ and marks the
+    # probability table as dirty.
+    #
+    def count(gram)
+      @dirty = true
+
+      unless @frequencies.has_key?(gram)
+        @frequencies[gram] = 0
+      end
+
+      return @frequencies[gram] += 1
+    end
+
+    #
+    # Calculates the total via the summation of the frequencies. Also
+    # marks the probability table as dirty.
+    #
+    def total
+      if @dirty
+        @total = @frequencies.values.inject do |sum,freq|
+          sum + freq
+        end
+      end
+
+      return @total
+    end
+
+    #
+    # Builds the probability table using the recorded frequencies, if the
+    # table is marked as dirty.
+    #
+    def build
+      if @dirty
+        current_total = total.to_f
+
+        @frequencies.each do |gram,count|
+          @probabilities[gram] = count.to_f / current_total
+        end
+
+        @dirty = false
+      end
+
+      return self
+    end
+
+    #
+    # Returns +true+ if the probability table is empty, returns +false+
+    # otherwise.
+    #
+    def empty?
+      @total == 0
+    end
+
+    #
+    # Clears the probability table.
+    #
+    def clear
+      @total = 0
+      @frequencies.clear
+      @probabilities.clear
+
+      return self
+    end
+
+    def inspect
+      if @dirty
+        "#<ProbabilityTable @total=#{@total} @frequencies=#{@frequencies.inspect}>"
+      else
+        @probabilities.inspect
+      end
+    end
+
+  end
+end

data/lib/raingrams/quadgram_model.rb

@@ -1,13 +1,9 @@
-require 'raingrams/multigram_model'
+require 'raingrams/model'
 
 module Raingrams
-  class QuadgramModel < MultigramModel
+  class QuadgramModel < Model
 
-    def initialize(opts={},&block)
-      opts[:ngram_size] = 4
-
-      super(opts,&block)
-    end
+    ngram_size 4
 
   end
 end

data/lib/raingrams/raingrams.rb

@@ -1,31 +1,21 @@
-require 'raingrams/unigram_model'
-require 'raingrams/multigram_model'
-require 'raingrams/open_vocabulary/unigram_model'
-require 'raingrams/open_vocabulary/multigram_model'
+require 'raingrams/model'
+require 'raingrams/open_vocabulary/model'
 
 module Raingrams
-  def Raingrams.closed_vocabulary_model(opts={},&block)
-    if opts[:ngram_size]==1
-      return UnigramModel.new(opts,&block)
-    else
-      return MultigramModel.new(opts,&block)
-    end
+  def Raingrams.closed_vocabulary_model(options={},&block)
+    Model.new(options,&block)
   end
 
-  def Raingrams.open_vocabulary_model(opts={},&block)
-    if opts[:ngram_size]==1
-      return OpenVocabulary::UnigramModel.new(opts,&block)
-    else
-      return OpenVocabulary::MultigramModel.new(opts,&block)
-    end
+  def Raingrams.open_vocabulary_model(options={},&block)
+    OpenVocabulary::Model.new(options,&block)
   end
 
-  def Raingrams.model(opts={},&block)
-    case opts[:vocabulary]
+  def Raingrams.model(options={},&block)
+    case options[:vocabulary]
     when :open, 'open'
-      return Raingrams.open_vocabulary_model(opts,&block)
+      return Raingrams.open_vocabulary_model(options,&block)
     else
-      return Raingrams.closed_vocabulary_model(opts,&block)
+      return Raingrams.closed_vocabulary_model(options,&block)
     end
   end
 end

data/lib/raingrams/tokens/start_sentence.rb

@@ -4,8 +4,8 @@ module Raingrams
   module Tokens
     class StartSentence < Token
 
-      def self.to_s
-        '<s>'
+      def initialize
+        super('<s>')
       end
 
     end

data/lib/raingrams/tokens/stop_sentence.rb

@@ -4,8 +4,8 @@ module Raingrams
   module Tokens
     class StopSentence < Token
 
-      def self.to_s
-        '</s>'
+      def initialize
+        super('</s>')
       end
 
     end

data/lib/raingrams/tokens/token.rb

@@ -2,16 +2,60 @@ module Raingrams
   module Tokens
     class Token
 
-      def self.*(length)
+      # Gram form of the token
+      attr_reader :gram
+
+      #
+      # Creates a new Token object with the specified _gram_.
+      #
+      def initialize(gram)
+        @gram = gram
+      end
+
+      def to_gram
+        self
+      end
+
+      #
+      # Creates an Array of the specified _length_ containing the token.
+      #
+      def *(length)
         [self] * length
       end
 
-      def self.to_sym
-        self.to_s.to_sym
+      #
+      # Returns +true+ if the token has the same gram as the _other_ token,
+      # returns +false+ otherwise.
+      #
+      def eql?(other)
+        if other.kind_of?(Token)
+          return (@gram == other.gram)
+        end
+
+        return false
+      end
+
+      alias == eql?
+
+      #
+      # Returns the String form of the token.
+      #
+      def to_s
+        @gram.to_s
+      end
+
+      #
+      # Returns the Symbol form of the token.
+      #
+      def to_sym
+        @gram.to_sym
       end
 
-      def self.inspect
-        self.to_s
+      #
+      # Returns the String form of the token.
+      #
+      def inspect
+        @gram.to_s
       end
 
     end

data/lib/raingrams/tokens/unknown.rb

@@ -4,8 +4,8 @@ module Raingrams
   module Tokens
     class Unknown < Token
 
-      def self.to_s
-        '<unknown>'
+      def initialize
+        super('<unknown>')
       end
 
     end

data/lib/raingrams/tokens.rb

@@ -2,3 +2,4 @@ require 'raingrams/tokens/token'
 require 'raingrams/tokens/start_sentence'
 require 'raingrams/tokens/stop_sentence'
 require 'raingrams/tokens/unknown'
+require 'raingrams/tokens/tokens'

data/lib/raingrams/trigram_model.rb

@@ -1,13 +1,9 @@
-require 'raingrams/multigram_model'
+require 'raingrams/model'
 
 module Raingrams
-  class TrigramModel < MultigramModel
+  class TrigramModel < Model
 
-    def initialize(opts={},&block)
-      opts[:ngram_size] = 3
-
-      super(opts,&block)
-    end
+    ngram_size 3
 
   end
 end

data/lib/raingrams/version.rb

@@ -1,3 +1,3 @@
 module Raingrams
-  VERSION = '0.0.9'
+  VERSION = '0.1.0'
 end

data/lib/raingrams.rb

@@ -1,7 +1,7 @@
 require 'raingrams/extensions'
 require 'raingrams/raingrams'
 require 'raingrams/ngram'
-require 'raingrams/unigram_model'
+require 'raingrams/model'
 require 'raingrams/bigram_model'
 require 'raingrams/trigram_model'
 require 'raingrams/quadgram_model'

data/spec/ngram_set_spec.rb

@@ -0,0 +1,54 @@
+require 'raingrams/ngram_set'
+
+require 'spec_helper'
+
+describe NgramSet do
+  before(:all) do
+    @ngrams = NgramSet[
+      Ngram[:the, :dog],
+      Ngram[:dog, :jumped],
+      Ngram[:jumped, :through],
+      Ngram[:through, :the],
+      Ngram[:the, :hoop]
+    ]
+  end
+
+  it "should select ngrams from the set" do
+    @ngrams.select { |ngram|
+      ngram.starts_with?(:the)
+    }.should == NgramSet[Ngram[:the, :dog], Ngram[:the, :hoop]]
+  end
+
+  it "should select ngrams with a specified prefixed" do
+    @ngrams.prefixed_by(Ngram[:dog]).should == NgramSet[
+      Ngram[:dog, :jumped]
+    ]
+  end
+
+  it "should select ngrams with a specified postfix" do
+    @ngrams.postfixed_by(Ngram[:through]).should == NgramSet[
+      Ngram[:jumped, :through]
+    ]
+  end
+
+  it "should select ngrams starting with a specified gram" do
+    @ngrams.starts_with(:jumped).should == NgramSet[Ngram[:jumped, :through]]
+  end
+
+  it "should select ngrams ending with a specified gram" do
+    @ngrams.ends_with(:dog).should == NgramSet[Ngram[:the, :dog]]
+  end
+
+  it "should select ngrams including a specified gram" do
+    @ngrams.including(:dog).should == NgramSet[
+      Ngram[:the, :dog],
+      Ngram[:dog, :jumped]
+    ]
+  end
+
+  it "should select ngrams which includes specified grams" do
+    @ngrams.includes(:the, :dog).should == NgramSet[
+      Ngram[:the, :dog],
+    ]
+  end
+end

data/spec/ngram_spec.rb

@@ -0,0 +1,29 @@
+require 'raingrams/ngram'
+
+require 'spec_helper'
+
+describe Ngram do
+  before(:all) do
+    @ngram = Ngram[:one, :two, :three]
+  end
+
+  it "should have a prefix" do
+    @ngram.prefix.should == Ngram[:one, :two]
+  end
+
+  it "should have a postfix" do
+    @ngram.postfix.should == Ngram[:two, :three]
+  end
+
+  it "should begin with a gram" do
+    @ngram.starts_with?(:one).should == true
+  end
+
+  it "should end with a gram" do
+    @ngram.ends_with?(:three).should == true
+  end
+
+  it "should include certain grams" do
+    @ngram.includes?(:one, :three).should == true
+  end
+end