raingrams 0.0.9 → 0.1.0

data/spec/probability_table_spec.rb

@@ -0,0 +1,94 @@
+require 'raingrams/probability_table'
+
+require 'spec_helper'
+
+describe ProbabilityTable do
+  before(:all) do
+    @grams = [:a, :b, :a, :a, :b, :c, :d, 2, 3, :a]
+
+    @table = ProbabilityTable.new
+    @grams.each { |g| @table.count(g) }
+  end
+
+  describe "empty table" do
+    before(:all) do
+      @empty_table = ProbabilityTable.new
+    end
+
+    it "should not be dirty" do
+      @empty_table.should_not be_dirty
+    end
+
+    it "should be empty" do
+      @empty_table.should be_empty
+    end
+
+    it "should not have any frequencies" do
+      @empty_table.frequencies.should be_empty
+    end
+
+    it "should have no probabilities" do
+      @empty_table.probabilities.should be_empty
+    end
+
+    it "should have no grams" do
+      @empty_table.grams.should be_empty
+    end
+  end
+
+  describe "un-built table" do
+    it "should be dirty" do
+      @table.should be_dirty
+    end
+
+    it "should have the observed grams" do
+      (@table.grams - @grams.uniq).should be_empty
+    end
+
+    it "should have non-zero frequencies" do
+      @table.frequencies.each_value do |freq|
+        freq.should > 0
+      end
+    end
+
+    it "should have non-zero frequencies for grams it has observed" do
+      @grams.uniq.each do |g|
+        @table.frequency_of(g).should > 0
+      end
+    end
+
+    it "should return a zero frequency for unknown grams" do
+      @table.frequency_of(:x).should == 0
+    end
+
+    it "should not have any probabilities yet" do
+      @table.probabilities.should be_empty
+    end
+  end
+
+  describe "built table" do
+    before(:all) do
+      @table.build
+    end
+
+    it "should not be dirty" do
+      @table.should_not be_dirty
+    end
+
+    it "should return a zero probability for unknown grams" do
+      @table.probability_of(:x).should == 0.0
+    end
+
+    it "should have non-zero probabilities" do
+      @table.probabilities.each_value do |prob|
+        prob.should > 0.0
+      end
+    end
+
+    it "should have non-zero probabilities for grams it has observed" do
+      @grams.uniq.each do |g|
+        @table.probability_of(g).should > 0.0
+      end
+    end
+  end
+end

data/spec/raingrams_spec.rb

@@ -0,0 +1,9 @@
+require 'raingrams/version'
+
+require 'spec_helper'
+
+describe Raingrams do
+  it "should have a VERSION constant" do
+    Raingrams.const_defined?('VERSION').should == true
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+gem 'rspec', '>=1.1.3'
+require 'spec'
+
+include Raingrams

data/tasks/spec.rb

@@ -0,0 +1,7 @@
+require 'spec/rake/spectask'
+
+desc "Run all specifications"
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.libs += ['lib', 'spec']
+  t.spec_opts = ['--colour', '--format', 'specdoc']
+end

metadata

@@ -1,51 +1,54 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.4
-specification_version: 1
 name: raingrams
 version: !ruby/object:Gem::Version 
-  version: 0.0.9
-date: 2008-01-09 00:00:00 -08:00
-summary: Raingrams is a flexible and general-purpose ngrams library written in Ruby
-require_paths: 
-- lib
-email: postmodern.mod3@gmail.com
-homepage: "    by Postmodern Modulus III"
-rubyforge_project: raingrams
-description: "== FEATURES/PROBLEMS:  * Supports all non-zero ngram sizes. * Supports text and non-text grams. * Supports Open and Closed vocabulary models.  == REQUIREMENTS:  == INSTALL:  $ sudo gem install raingrams"
-autorequire: 
-default_executable: 
-bindir: bin
-has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
-  requirements: 
-  - - ">"
-    - !ruby/object:Gem::Version 
-      version: 0.0.0
-  version: 
+  version: 0.1.0
 platform: ruby
-signing_key: 
-cert_chain: 
-post_install_message: 
 authors: 
 - Postmodern Modulus III
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-10-06 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.7.0
+    version: 
+description: Raingrams is a flexible and general-purpose ngrams library written in Ruby. Raingrams supports any non-zero ngram size, text/non-text grams, multiple parsing styles and open/closed vocabulary models.
+email: 
+- postmodern.mod3@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
+- TODO.txt
 files: 
 - History.txt
 - LICENSE.txt
 - Manifest.txt
 - README.txt
+- TODO.txt
 - Rakefile
 - lib/raingrams.rb
 - lib/raingrams/version.rb
 - lib/raingrams/raingrams.rb
 - lib/raingrams/exceptions/prefix_frequency_missing.rb
 - lib/raingrams/exceptions.rb
-- lib/raingrams/extensions/class.rb
-- lib/raingrams/extensions/false_class.rb
-- lib/raingrams/extensions/nil_class.rb
 - lib/raingrams/extensions/object.rb
 - lib/raingrams/extensions/string.rb
-- lib/raingrams/extensions/symbol.rb
-- lib/raingrams/extensions/true_class.rb
 - lib/raingrams/extensions.rb
 - lib/raingrams/tokens/token.rb
 - lib/raingrams/tokens/start_sentence.rb
@@ -53,47 +56,54 @@ files:
 - lib/raingrams/tokens/unknown.rb
 - lib/raingrams/tokens.rb
 - lib/raingrams/ngram.rb
+- lib/raingrams/ngram_set.rb
+- lib/raingrams/probability_table.rb
 - lib/raingrams/model.rb
-- lib/raingrams/unigram_model.rb
-- lib/raingrams/multigram_model.rb
 - lib/raingrams/bigram_model.rb
 - lib/raingrams/trigram_model.rb
 - lib/raingrams/quadgram_model.rb
 - lib/raingrams/pentagram_model.rb
 - lib/raingrams/hexagram_model.rb
 - lib/raingrams/open_vocabulary/open_model.rb
-- lib/raingrams/open_vocabulary/unigram_model.rb
-- lib/raingrams/open_vocabulary/multigram_model.rb
+- lib/raingrams/open_vocabulary/model.rb
 - lib/raingrams/open_vocabulary/bigram_model.rb
 - lib/raingrams/open_vocabulary/trigram_model.rb
 - lib/raingrams/open_vocabulary/quadgram_model.rb
 - lib/raingrams/open_vocabulary/pentagram_model.rb
 - lib/raingrams/open_vocabulary/hexagram_model.rb
 - lib/raingrams/open_vocabulary.rb
-- test/test_raingrams.rb
-test_files: 
-- test/test_raingrams.rb
+- tasks/spec.rb
+- spec/spec_helper.rb
+- spec/ngram_spec.rb
+- spec/ngram_set_spec.rb
+- spec/probability_table_spec.rb
+- spec/raingrams_spec.rb
+has_rdoc: true
+homepage: http://raingrams.rubyforge.org/
+post_install_message: 
 rdoc_options: 
 - --main
 - README.txt
-extra_rdoc_files: 
-- History.txt
-- LICENSE.txt
-- Manifest.txt
-- README.txt
-executables: []
-
-extensions: []
-
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
 requirements: []
 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 1.4.0
-    version: 
+rubyforge_project: raingrams
+rubygems_version: 1.3.0
+signing_key: 
+specification_version: 2
+summary: Raingrams is a flexible and general-purpose ngrams library written in Ruby
+test_files: []
+

data/lib/raingrams/extensions/class.rb

@@ -1,7 +0,0 @@
-class Class
-
-  def to_gram
-    self
-  end
-
-end

data/lib/raingrams/extensions/false_class.rb

@@ -1,7 +0,0 @@
-class FalseClass
-
-  def to_gram
-    self
-  end
-
-end

data/lib/raingrams/extensions/nil_class.rb

@@ -1,7 +0,0 @@
-class NilClass
-
-  def to_gram
-    self
-  end
-
-end

data/lib/raingrams/extensions/symbol.rb

@@ -1,7 +0,0 @@
-class Symbol
-
-  def to_gram
-    self
-  end
-
-end

data/lib/raingrams/extensions/true_class.rb

@@ -1,7 +0,0 @@
-class TrueClass
-
-  def to_gram
-    self
-  end
-
-end

data/lib/raingrams/multigram_model.rb

@@ -1,165 +0,0 @@
-require 'raingrams/model'
-require 'raingrams/tokens/start_sentence'
-require 'raingrams/tokens/stop_sentence'
-require 'raingrams/exceptions/prefix_frequency_missing'
-
-module Raingrams
-  class MultigramModel < Model
-
-    # Frequencies of n-1 grams
-    attr_reader :prefix_frequency
-
-    def initialize(opts={},&block)
-      @prefix_frequency = Hash.new { |hash,key| 0 }
-
-      super(opts) { |model| model.build(&block) }
-    end
-
-    def ngrams_from_words(words)
-      return (0...(words.length-@ngram_size+1)).map do |index|
-        Ngram.new(words[index,@ngram_size])
-      end
-    end
-
-    def ngrams_from_fragment(fragment)
-      ngrams_from_words(parse_sentence(fragment))
-    end
-
-    def ngrams_from_sentence(sentence)
-      ngrams_from_words(wrap_sentence(parse_sentence(sentence)))
-    end
-
-    def ngrams_from_text(text)
-      parse_text(text).inject([]) do |ngrams,sentence|
-        ngrams + ngrams_from_sentence(sentence)
-      end
-    end
-
-    def common_ngrams_from_words(words)
-      ngrams_from_words(words).select { |ngram| has_ngram?(ngram) }
-    end
-
-    def common_ngrams_from_fragment(fragment)
-      ngrams_from_fragment(words).select { |ngram| has_ngram?(ngram) }
-    end
-
-    def common_ngrams_from_sentence(sentence)
-      ngrams_from_sentence(sentence).select { |ngram| has_ngram?(ngram) }
-    end
-
-    def common_ngrams_from_text(text)
-      ngrams_from_text(text).select { |ngram| has_ngram?(ngram) }
-    end
-
-    def train_with_ngram(ngram)
-      @prefix_frequency[ngram.prefix] += 1
-      return super(ngram)
-    end
-
-    def train_with_sentence(sentence)
-      train_with_ngrams(ngrams_from_sentence(sentence))
-    end
-
-    def train_with_text(text)
-      train_with_ngrams(ngrams_from_text(text))
-    end
-
-    def build(&block)
-      clear_probabilities
-
-      block.call(self) if block
-
-      @frequency.each do |ngram,count|
-        prefix = ngram.prefix
-
-        unless @prefix_frequency[prefix]
-          raise(PrefixFrequencyMissing,"the model is missing the frequency of the ngram prefix #{prefix}",caller)
-        end
-
-        @probability[ngram] = count.to_f / @prefix_frequency[prefix].to_f
-      end
-
-      return self
-    end
-
-    def ngrams_prefixed_by(prefix)
-      ngrams_with { |ngram| ngram.prefixed_by?(prefix) }
-    end
-
-    def ngrams_postfixed_by(postfix)
-      ngrams_with { |ngram| ngram.prefixed_by?(postfix) }
-    end
-
-    def ngrams_preceeding(gram)
-      ngrams_ending_with(gram).map do |ngram|
-        ngrams_postfixed_by(ngram.prefix)
-      end
-    end
-
-    def ngrams_following(gram)
-      ngrams_starting_with(gram).map do |ngram|
-        ngrams_prefixed_by(ngram.postfix)
-      end
-    end
-
-    def grams_preceeding(gram)
-      ngrams_ending_with(gram).map do |ngram|
-        ngram[-2]
-      end
-    end
-
-    def grams_following(gram)
-      ngrams_starting_with(gram).map do |ngram|
-        ngram[1]
-      end
-    end
-
-    def fragment_probability(fragment)
-      probability_of_ngrams(ngrams_from_fragment(fragment))
-    end
-
-    def sentence_probability(sentence)
-      probability_of_ngrams(ngrams_from_sentence(sentence))
-    end
-
-    def text_probability(text)
-      probability_of_ngrams(ngrams_from_text(text))
-    end
-
-    def common_fragment_probability(fragment)
-      probability_of_ngrams(common_ngrams_from_fragment(fragment))
-    end
-
-    def common_sentence_probability(sentence)
-      probability_of_ngrams(common_ngrams_from_sentence(sentence))
-    end
-
-    def common_text_probability(fragment)
-      probability_of_ngrams(common_ngrams_from_text(text))
-    end
-
-    def similar_fragment_probability(other,fragment)
-      common_fragment_probability(fragment) * other.common_fragment_probability(fragment)
-    end
-
-    def similar_sentence_probability(other,sentence)
-      common_sentence_probability(sentence) * other.common_sentence_probability(sentence)
-    end
-
-    def similar_text_probability(other,text)
-      common_text_probability(text) * other.common_text_probability(text)
-    end
-
-    def clear
-      @prefix_frequency.clear
-      return super
-    end
-
-    protected
-
-    def wrap_sentence(sentence)
-      (Tokens::StartSentence * @ngram_size) + sentence.to_a + (Tokens::StopSentence * @ngram_size)
-    end
-
-  end
-end

data/lib/raingrams/open_vocabulary/multigram_model.rb

@@ -1,12 +0,0 @@
-require 'raingrams/multigrammodel'
-require 'raingrams/openvocabulary/openmodel'
-
-module Raingrams
-  module OpenVocabulary
-    class MultigramModel < Raingrams::MultigramModel
-
-      include OpenModel
-
-    end
-  end
-end

data/lib/raingrams/open_vocabulary/unigram_model.rb

@@ -1,12 +0,0 @@
-require 'raingrams/unigram_model'
-require 'raingrams/openvocabulary/open_model'
-
-module Raingrams
-  module OpenVocabulary
-    class UnigramModel < Raingrams::UnigramModel
-
-      include OpenModel
-
-    end
-  end
-end

data/lib/raingrams/unigram_model.rb

@@ -1,70 +0,0 @@
-require 'raingrams/model'
-
-module Raingrams
-  class UnigramModel < Model
-
-    def initialize(opts={},&block)
-      opts[:ngram_size] = 1
-
-      super(opts) { |model| model.build(&block) }
-    end
-
-    def ngrams_from_words(words)
-      words.map { |word| Ngram[word] }
-    end
-
-    def ngrams_from_fragment(fragment)
-      ngrams_from_words(parse_sentence(fragment))
-    end
-
-    def ngrams_from_sentence(sentence)
-      ngrams_from_fragment(sentence)
-    end
-
-    def ngrams_from_text(text)
-      parse_text(text).inject([]) do |ngrams,sentence|
-        ngrams + ngrams_from_sentence(sentence)
-      end
-    end
-
-    def train_with_sentence(sentence)
-      train_with_ngrams(ngrams_from_sentence(sentence))
-    end
-
-    def train_with_text(text)
-      train_with_ngrams(ngrams_from_text(text))
-    end
-
-    def gram_count
-      @frequency.values.inject do |sum,count|
-        sum + count
-      end
-    end
-
-    def build(&block)
-      clear_probabilities
-
-      block.call(self) if block
-
-      total_count = gram_count.to_f
-      @frequency.each do |ngram,count|
-        @probability[ngram] = count.to_f / total_count
-      end
-
-      return self
-    end
-
-    def fragment_probability(fragment)
-      probability_of_ngrams(ngrams_from_fragment(fragment))
-    end
-
-    def sentence_probability(sentence)
-      probability_of_ngrams(ngrams_from_sentence(sentence))
-    end
-
-    def text_probability(text)
-      probability_of_ngrams(ngrams_from_text(text))
-    end
-
-  end
-end