raingrams 0.1.0 → 0.1.1

data/spec/bigram_model_spec.rb

@@ -0,0 +1,111 @@
+require 'raingrams/bigram_model'
+
+require 'spec_helper'
+require 'model_examples'
+
+describe BigramModel do
+  before(:all) do
+    @model = BigramModel.train_with_text(Training.text_for(:snowcrash))
+  end
+
+  it_should_behave_like "Model"
+
+  it "should return ngrams from specified words" do
+    words = %w{Why is the Deliverator so equipped}
+    ngrams = [
+      Ngram[:Why, :is],
+      Ngram[:is, :the],
+      Ngram[:the, :Deliverator],
+      Ngram[:Deliverator, :so],
+      Ngram[:so, :equipped]
+    ]
+
+    @model.ngrams_from_words(words).should == ngrams
+  end
+
+  it "should return common ngrams from words" do
+    words = %w{The Deliverator is a future Archetype}
+    ngrams = [
+      Ngram[:The, :Deliverator],
+      Ngram[:Deliverator, :is],
+      Ngram[:is, :a]
+    ]
+
+    @model.common_ngrams_from_words(words).should == ngrams
+  end
+
+  it "should return common ngrams from a specified fragment of text" do
+    fragment = %{The Deliverator is a future Archetype}
+    ngrams = [
+      Ngram[:The, :Deliverator],
+      Ngram[:Deliverator, :is],
+      Ngram[:is, :a]
+    ]
+
+    @model.common_ngrams_from_fragment(fragment).should == ngrams
+  end
+
+  it "should return common ngrams from a specified sentence" do
+    sentence = %{The Deliverator is a future Archetype.}
+    ngrams = [
+      Ngram[Tokens.start, Tokens.start],
+      Ngram[Tokens.start, :The],
+      Ngram[:The, :Deliverator],
+      Ngram[:Deliverator, :is],
+      Ngram[:is, :a],
+      Ngram[Tokens.stop, Tokens.stop]
+    ]
+
+    @model.common_ngrams_from_sentence(sentence).should == ngrams
+  end
+
+  it "should have a frequency for a specified ngram" do
+    ngram = Ngram[:teensy, :darts]
+
+    @model.frequency_of_ngram(ngram).should == 1
+  end
+
+  it "should have a probability for a specified ngram" do
+    ngram = Ngram[:teensy, :darts]
+
+    @model.probability_of_ngram(ngram).should == 1.0
+  end
+
+  it "should have a frequency for specified ngrams" do
+    ngrams = NgramSet[
+      Ngram[:but, :excess],
+      Ngram[:freshly, :napalmed],
+      Ngram[:sintered, :armorgel]
+    ]
+
+    @model.frequency_of_ngrams(ngrams).should == 3
+  end
+
+  it "should have a probability of specified ngrams" do
+    ngrams = NgramSet[
+      Ngram[:The, :Deliverator],
+      Ngram[:Deliverator, :belongs],
+      Ngram[:belongs, :to]
+    ]
+
+    @model.probability_of_ngrams(ngrams).to_s.should == '0.0112293144208038'
+  end
+
+  it "should have a probability for a specified fragment of text" do
+    fragment = %{The Deliverator belongs to}
+
+    @model.fragment_probability(fragment).to_s.should == '0.0112293144208038'
+  end
+
+  it "should have a probability for a specified sentence" do
+    sentence = %{The Deliverator used to make software.}
+
+    @model.sentence_probability(sentence).to_s.should == '4.10042780102381e-07'
+  end
+
+  it "should have a probability for specified text" do
+    text = %{The Deliverator used to make software. Still does, sometimes.}
+
+    @model.text_probability(text).to_s.should == '2.40635434332383e-10'
+  end
+end

data/spec/helpers/training.rb

@@ -0,0 +1,8 @@
+module Training
+  def Training.text_for(name)
+    name = name.to_sym
+    path = File.join(File.dirname(__FILE__),'..','training',"#{name}.txt")
+
+    return File.read(path)
+  end
+end

data/spec/helpers.rb

@@ -0,0 +1 @@
+require 'helpers/training'

data/spec/model_examples.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+shared_examples_for "Model" do
+  it "should have ngrams" do
+    @model.ngrams.each do |ngram|
+      @model.has_ngram?(ngram).should == true
+    end
+  end
+
+  it "should be able to iterate through all ngrams" do
+    @model.each_ngram do |ngram|
+      @model.has_ngram?(ngram).should == true
+    end
+  end
+
+  it "should be able to select ngrams with certain properties" do
+    ngrams = @model.ngrams_with do |ngram|
+      ngram.include?(:the)
+    end
+
+    ngrams.each do |ngram|
+      ngram.include?(:the).should == true
+    end
+  end
+
+  it "should be able to select ngrams starting with a specified gram" do
+    @model.ngrams_starting_with(:filtering).each do |ngram|
+      ngram.starts_with?(:filtering).should == true
+    end
+  end
+
+  it "should be able to select ngrams ending with a specified gram" do
+    @model.ngrams_ending_with(:sword).each do |ngram|
+      ngram.ends_with?(:sword).should == true
+    end
+  end
+
+  it "should be able to select ngrams including any of the specified grams" do
+    @model.ngrams_including_any(:The, :Deliverator).each do |ngram|
+      ngram.includes_any?(:The, :Deliverator).should == true
+    end
+  end
+
+  it "should be able to select ngrams including all of the specified grams" do
+    @model.ngrams_including_all(:activated, :charcoal).each do |ngram|
+      ngram.includes_all?(:activated, :charcoal).should == true
+    end
+  end
+
+  it "should have grams" do
+    @model.grams.each do |gram|
+      @model.has_gram?(gram).should == true
+    end
+  end
+
+  it "should provide a random ngram" do
+    @model.has_ngram?(@model.random_ngram).should == true
+  end
+
+  it "should generate a random sentence" do
+    sentence = @model.random_sentence
+
+    @model.ngrams_from_sentence(sentence).each do |ngram|
+      @model.has_ngram?(ngram).should == true
+    end
+  end
+
+  it "should generate a random paragraph" do
+    paragraph = @model.random_paragraph
+
+    @model.ngrams_from_paragraph(paragraph).each do |ngram|
+      @model.has_ngram?(ngram).should == true
+    end
+  end
+
+  it "should generate a random text" do
+    text = @model.random_text
+
+    @model.ngrams_from_text(text).each do |ngram|
+      @model.has_ngram?(ngram).should == true
+    end
+  end
+end

data/spec/model_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+require 'raingrams/model'
+
+describe Model do
+  before(:all) do
+    @model = Model.new(:ngram_size => 2)
+
+    @phone_number_model = Model.new(
+      :ngram_size => 2,
+      :ignore_phone_numbers => true
+    )
+
+    @references_model = Model.new(
+      :ngram_size => 2,
+      :ignore_references => true
+    )
+
+    @case_model = Model.new(
+      :ngram_size => 2,
+      :ignore_case => true
+    )
+
+    @punctuation_model = Model.new(
+      :ngram_size => 2,
+      :ignore_punctuation => false
+    )
+  end
+
+  it "should parse text into sentences" do
+    text = %{The Deliverator belongs to an elite order, a hallowed sub-category. He's got esprit up to here.}
+    sentences = [
+      "The Deliverator belongs to an elite order, a hallowed sub-category.",
+      "He's got esprit up to here."
+    ]
+
+    @model.parse_text(text).should == sentences
+  end
+
+  it "should parse words from a sentence" do
+    sentence = %{The Deliverator is in touch with the road, starts like a bad day, stops on a peseta.}
+    words = %w{The Deliverator is in touch with the road starts like a bad day stops on a peseta}
+
+    @model.parse_sentence(sentence).should == words
+  end
+
+  it "should ignore URLs by default while parsing a sentence" do
+    sentence = %{Click on the following link: http://www.example.com/}
+    words = %w{Click on the following link}
+
+    @model.parse_sentence(sentence).should == words
+  end
+
+  it "should ignore short URIs by default while parsing a sentence" do
+    sentence = %{Click on the following link: jabber://}
+    words = %w{Click on the following link}
+
+    @model.parse_sentence(sentence).should == words
+  end
+
+  it "should ignore complex HTTP URLs by default while parsing a sentence" do
+    sentence = %{Click on the following link: http://www.google.com/search?hl=en&client=firefox-a&rls=org.mozilla:en-US:official&hs=jU&q=ruby+datamapper&start=20&sa=N}
+    words = %w{Click on the following link}
+
+    @model.parse_sentence(sentence).should == words
+  end
+
+  it "may ignore phone numbers while parsing a sentence" do
+    sentence = %{Call me before 12, 1-888-444-2222.}
+    words = %w{Call me before 12}
+
+    @phone_number_model.parse_sentence(sentence).should == words
+  end
+
+  it "may ignore long-distance phone numbers while parsing a sentence" do
+    sentence = %{Call me before 12, 1-444-2222.}
+    words = %w{Call me before 12}
+
+    @phone_number_model.parse_sentence(sentence).should == words
+  end
+
+  it "may ignore short phone numbers while parsing a sentence" do
+    sentence = %{Call me before 12, 444-2222.}
+    words = %w{Call me before 12}
+
+    @phone_number_model.parse_sentence(sentence).should == words
+  end
+
+  it "may ignore RFC style references while parsing a sentence" do
+    sentence = %{As one can see, it has failed [1].}
+    words = %w{As one can see it has failed}
+
+    @references_model.parse_sentence(sentence).should == words
+  end
+
+  it "should ignore punctuation by default while parsing a sentence" do
+    sentence = %{Oh, they used to argue over times, many corporate driver-years lost to it: homeowners, red-faced and sweaty with their own lies, stinking of Old Spice and job-related stress, standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink, I swear, can't you guys tell time?}
+    words = %w{
+      Oh they used to argue over times many corporate driver-years lost to it homeowners red-faced and sweaty with their own lies stinking of Old Spice and job-related stress standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink I swear can't you guys tell time
+    }
+
+    @model.parse_sentence(sentence).should == words
+  end
+
+  it "may ignore case while parsing a sentence" do
+    sentence = %{The Deliverator is in touch with the road, starts like a bad day, stops on a peseta.}
+    words = %w{the deliverator is in touch with the road starts like a bad day stops on a peseta}
+
+    @case_model.parse_sentence(sentence).should == words
+  end
+
+  it "may preserve punctuation while parsing a sentence" do
+    sentence = %{Oh, they used to argue over times, many corporate driver-years lost to it: homeowners, red-faced and sweaty with their own lies, stinking of Old Spice and job-related stress, standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink, I swear, can't you guys tell time?}
+    words = %w{Oh, they used to argue over times, many corporate driver-years lost to it: homeowners, red-faced and sweaty with their own lies, stinking of Old Spice and job-related stress, standing in their glowing yellow doorways brandishing their Seikos and waving at the clock over the kitchen sink, I swear, can't you guys tell time?}
+
+    @punctuation_model.parse_sentence(sentence).should == words
+  end
+end

data/spec/ngram_set_spec.rb

@@ -46,9 +46,18 @@ describe NgramSet do
     ]
   end
 
-  it "should select ngrams which includes specified grams" do
-    @ngrams.includes(:the, :dog).should == NgramSet[
+  it "should select ngrams which include any of the specified grams" do
+    @ngrams.including_any(:the, :dog).should == NgramSet[
       Ngram[:the, :dog],
+      Ngram[:dog, :jumped],
+      Ngram[:through, :the],
+      Ngram[:the, :hoop]
+    ]
+  end
+
+  it "should select ngrams which include all of the specified grams" do
+    @ngrams.including_all(:the, :dog).should == NgramSet[
+      Ngram[:the, :dog]
     ]
   end
 end

data/spec/ngram_spec.rb

@@ -24,6 +24,6 @@ describe Ngram do
   end
 
   it "should include certain grams" do
-    @ngram.includes?(:one, :three).should == true
+    @ngram.includes_all?(:one, :three).should == true
   end
 end

data/spec/pentagram_model_spec.rb

@@ -0,0 +1,101 @@
+require 'raingrams/pentagram_model'
+
+require 'spec_helper'
+require 'model_examples'
+
+describe PentagramModel do
+  before(:all) do
+    @model = PentagramModel.build do |model|
+      model.train_with_text(Training.text_for(:snowcrash))
+    end
+  end
+
+  it_should_behave_like "Model"
+
+  it "should return ngrams from specified words" do
+    words = %w{Why is the Deliverator so equipped}
+    ngrams = [
+      Ngram[:Why, :is, :the, :Deliverator, :so],
+      Ngram[:is, :the, :Deliverator, :so, :equipped]
+    ]
+
+    @model.ngrams_from_words(words).should == ngrams
+  end
+
+  it "should return common ngrams from words" do
+    words = %w{The Deliverator is a future Archetype}
+    ngrams = []
+
+    @model.common_ngrams_from_words(words).should == ngrams
+  end
+
+  it "should return common ngrams from a specified fragment of text" do
+    fragment = %{The Deliverator is a future Archetype}
+    ngrams = []
+
+    @model.common_ngrams_from_fragment(fragment).should == ngrams
+  end
+
+  it "should return common ngrams from a specified sentence" do
+    sentence = %{The Deliverator is a future Archetype.}
+    ngrams = [
+      Ngram[Tokens.start, Tokens.start, Tokens.start, Tokens.start, Tokens.start],
+      Ngram[Tokens.start, Tokens.start, Tokens.start, Tokens.start, :The],
+      Ngram[Tokens.start, Tokens.start, Tokens.start, :The, :Deliverator],
+      Ngram[Tokens.start, Tokens.start, :The, :Deliverator, :is],
+      Ngram[Tokens.start, :The, :Deliverator, :is, :a],
+      Ngram[Tokens.stop, Tokens.stop, Tokens.stop, Tokens.stop, Tokens.stop]
+    ]
+
+    @model.common_ngrams_from_sentence(sentence).should == ngrams
+  end
+
+  it "should have a frequency for a specified ngram" do
+    ngram = Ngram[:it, :fires, :teensy, :darts, :that]
+
+    @model.frequency_of_ngram(ngram).should == 1
+  end
+
+  it "should have a probability for a specified ngram" do
+    ngram = Ngram[:it, :fires, :teensy, :darts, :that]
+
+    @model.probability_of_ngram(ngram).should == 1.0
+  end
+
+  it "should have a frequency for specified ngrams" do
+    ngrams = NgramSet[
+      Ngram[:but, :excess, :perspiration, :wafts, :through],
+      Ngram[:through, :a, :freshly, :napalmed, :forest],
+      Ngram[:the, :suit, :has, :sintered, :armorgel]
+    ]
+
+    @model.frequency_of_ngrams(ngrams).should == 3
+  end
+
+  it "should have a probability of specified ngrams" do
+    ngrams = NgramSet[
+      Ngram[:The, :Deliverator, :belongs, :to, :an],
+      Ngram[:Deliverator, :belongs, :to, :an, :elite]
+    ]
+
+    @model.probability_of_ngrams(ngrams).to_s.should == '1.0'
+  end
+
+  it "should have a probability for a specified fragment of text" do
+    fragment = %{The Deliverator belongs to an}
+
+    @model.fragment_probability(fragment).to_s.should == '1.0'
+  end
+
+  it "should have a probability for a specified sentence" do
+    sentence = %{So now he has this other job.}
+
+    @model.sentence_probability(sentence).to_s.should == '0.00117370892018779'
+  end
+
+  it "should have a probability for specified text" do
+    text = %{So now he has this other job. No brightness or creativity involved-but no cooperation either.}
+
+    @model.text_probability(text).to_s.should == '2.75518525865679e-06'
+  end
+end

data/spec/quadgram_model_spec.rb

@@ -0,0 +1,106 @@
+require 'raingrams/quadgram_model'
+
+require 'spec_helper'
+require 'model_examples'
+
+describe QuadgramModel do
+  before(:all) do
+    @model = QuadgramModel.build do |model|
+      model.train_with_text(Training.text_for(:snowcrash))
+    end
+  end
+
+  it_should_behave_like "Model"
+
+  it "should return ngrams from specified words" do
+    words = %w{Why is the Deliverator so equipped}
+    ngrams = [
+      Ngram[:Why, :is, :the, :Deliverator],
+      Ngram[:is, :the, :Deliverator, :so],
+      Ngram[:the, :Deliverator, :so, :equipped]
+    ]
+
+    @model.ngrams_from_words(words).should == ngrams
+  end
+
+  it "should return common ngrams from words" do
+    words = %w{The Deliverator is a future Archetype}
+    ngrams = [
+      Ngram[:The, :Deliverator, :is, :a]
+    ]
+
+    @model.common_ngrams_from_words(words).should == ngrams
+  end
+
+  it "should return common ngrams from a specified fragment of text" do
+    fragment = %{The Deliverator is a future Archetype}
+    ngrams = [
+      Ngram[:The, :Deliverator, :is, :a]
+    ]
+
+    @model.common_ngrams_from_fragment(fragment).should == ngrams
+  end
+
+  it "should return common ngrams from a specified sentence" do
+    sentence = %{The Deliverator is a future Archetype.}
+    ngrams = [
+      Ngram[Tokens.start, Tokens.start, Tokens.start, Tokens.start],
+      Ngram[Tokens.start, Tokens.start, Tokens.start, :The],
+      Ngram[Tokens.start, Tokens.start, :The, :Deliverator],
+      Ngram[Tokens.start, :The, :Deliverator, :is],
+      Ngram[:The, :Deliverator, :is, :a],
+      Ngram[Tokens.stop, Tokens.stop, Tokens.stop, Tokens.stop]
+    ]
+
+    @model.common_ngrams_from_sentence(sentence).should == ngrams
+  end
+
+  it "should have a frequency for a specified ngram" do
+    ngram = Ngram[:it, :fires, :teensy, :darts]
+
+    @model.frequency_of_ngram(ngram).should == 1
+  end
+
+  it "should have a probability for a specified ngram" do
+    ngram = Ngram[:it, :fires, :teensy, :darts]
+
+    @model.probability_of_ngram(ngram).should == 1.0
+  end
+
+  it "should have a frequency for specified ngrams" do
+    ngrams = NgramSet[
+      Ngram[:but, :excess, :perspiration, :wafts],
+      Ngram[:a, :freshly, :napalmed, :forest],
+      Ngram[:suit, :has, :sintered, :armorgel]
+    ]
+
+    @model.frequency_of_ngrams(ngrams).should == 3
+  end
+
+  it "should have a probability of specified ngrams" do
+    ngrams = NgramSet[
+      Ngram[:The, :Deliverator, :belongs, :to],
+      Ngram[:Deliverator, :belongs, :to, :an]
+    ]
+
+    @model.probability_of_ngrams(ngrams).to_s.should == '1.0'
+  end
+
+  it "should have a probability for a specified fragment of text" do
+    fragment = %{The Deliverator belongs to}
+
+    @model.fragment_probability(fragment).to_s.should == '1.0'
+  end
+
+  it "should have a probability for a specified sentence" do
+    sentence = %{So now he has this other job.}
+
+    @model.sentence_probability(sentence).to_s.should == '0.00117370892018779'
+  end
+
+  it "should have a probability for specified text" do
+    text = %{So now he has this other job. No brightness or creativity involved-but no cooperation either.}
+
+    @model.text_probability(text).to_s.should == '2.75518525865679e-06'
+  end
+end

data/spec/spec_helper.rb

@@ -3,3 +3,5 @@ gem 'rspec', '>=1.1.3'
 require 'spec'
 
 include Raingrams
+
+require 'helpers'