raingrams 0.0.9

data/History.txt

@@ -0,0 +1,7 @@
+== 0.0.9 / 2008-01-09
+
+* Initial release.
+* Supports all non-zero ngram sizes.
+* Supports text and non-text grams.
+* Supports Open and Closed vocabulary models.
+

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2007-2008 Hal Brodigan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,42 @@
+History.txt
+LICENSE.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/raingrams.rb
+lib/raingrams/version.rb
+lib/raingrams/raingrams.rb
+lib/raingrams/exceptions/prefix_frequency_missing.rb
+lib/raingrams/exceptions.rb
+lib/raingrams/extensions/class.rb
+lib/raingrams/extensions/false_class.rb
+lib/raingrams/extensions/nil_class.rb
+lib/raingrams/extensions/object.rb
+lib/raingrams/extensions/string.rb
+lib/raingrams/extensions/symbol.rb
+lib/raingrams/extensions/true_class.rb
+lib/raingrams/extensions.rb
+lib/raingrams/tokens/token.rb
+lib/raingrams/tokens/start_sentence.rb
+lib/raingrams/tokens/stop_sentence.rb
+lib/raingrams/tokens/unknown.rb
+lib/raingrams/tokens.rb
+lib/raingrams/ngram.rb
+lib/raingrams/model.rb
+lib/raingrams/unigram_model.rb
+lib/raingrams/multigram_model.rb
+lib/raingrams/bigram_model.rb
+lib/raingrams/trigram_model.rb
+lib/raingrams/quadgram_model.rb
+lib/raingrams/pentagram_model.rb
+lib/raingrams/hexagram_model.rb
+lib/raingrams/open_vocabulary/open_model.rb
+lib/raingrams/open_vocabulary/unigram_model.rb
+lib/raingrams/open_vocabulary/multigram_model.rb
+lib/raingrams/open_vocabulary/bigram_model.rb
+lib/raingrams/open_vocabulary/trigram_model.rb
+lib/raingrams/open_vocabulary/quadgram_model.rb
+lib/raingrams/open_vocabulary/pentagram_model.rb
+lib/raingrams/open_vocabulary/hexagram_model.rb
+lib/raingrams/open_vocabulary.rb
+test/test_raingrams.rb

data/README.txt

@@ -0,0 +1,46 @@
+Raingrams
+    by Postmodern Modulus III
+    http://rubyforge.net/projects/raingrams/
+
+== DESCRIPTION:
+  
+Raingrams is a flexible and general-purpose ngrams library written in Ruby.
+Raingrams supports any non-zero ngram size, text/non-text grams, multiple
+parsing styles and open/closed vocabulary models.
+
+== FEATURES/PROBLEMS:
+  
+* Supports all non-zero ngram sizes.
+* Supports text and non-text grams.
+* Supports Open and Closed vocabulary models.
+
+== REQUIREMENTS:
+
+== INSTALL:
+
+  $ sudo gem install raingrams
+
+== LICENSE:
+
+The MIT License
+
+Copyright (c) 2007-2008 Hal Brodigan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,17 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require './lib/raingrams/version.rb'
+
+Hoe.new('raingrams', Raingrams::VERSION) do |p|
+  p.rubyforge_name = 'raingrams'
+  p.author = 'Postmodern Modulus III'
+  p.email = 'postmodern.mod3@gmail.com'
+  p.summary = 'Raingrams is a flexible and general-purpose ngrams library written in Ruby'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+end
+
+# vim: syntax=Ruby

data/lib/raingrams/bigram_model.rb

@@ -0,0 +1,13 @@
+require 'raingrams/multigram_model'
+
+module Raingrams
+  class BigramModel < MultigramModel
+
+    def initialize(opts={},&block)
+      opts[:ngram_size] = 2
+
+      super(opts,&block)
+    end
+
+  end
+end

data/lib/raingrams/exceptions/prefix_frequency_missing.rb

@@ -0,0 +1,4 @@
+module Raingrams
+  class PrefixFrequencyMissing < RuntimeError
+  end
+end

data/lib/raingrams/exceptions.rb

@@ -0,0 +1 @@
+require 'raingrams/exceptions/prefix_frequency_missing.rb'

data/lib/raingrams/extensions/class.rb

@@ -0,0 +1,7 @@
+class Class
+
+  def to_gram
+    self
+  end
+
+end

data/lib/raingrams/extensions/false_class.rb

@@ -0,0 +1,7 @@
+class FalseClass
+
+  def to_gram
+    self
+  end
+
+end

data/lib/raingrams/extensions/nil_class.rb

@@ -0,0 +1,7 @@
+class NilClass
+
+  def to_gram
+    self
+  end
+
+end

data/lib/raingrams/extensions/object.rb

@@ -0,0 +1,7 @@
+class Object
+
+  def to_gram
+    self.dup.freeze
+  end
+
+end

data/lib/raingrams/extensions/string.rb

@@ -0,0 +1,7 @@
+class String
+
+  def to_gram
+    intern
+  end
+
+end

data/lib/raingrams/extensions/symbol.rb

@@ -0,0 +1,7 @@
+class Symbol
+
+  def to_gram
+    self
+  end
+
+end

data/lib/raingrams/extensions/true_class.rb

@@ -0,0 +1,7 @@
+class TrueClass
+
+  def to_gram
+    self
+  end
+
+end

data/lib/raingrams/extensions.rb

@@ -0,0 +1,7 @@
+require 'raingrams/extensions/class'
+require 'raingrams/extensions/nil_class'
+require 'raingrams/extensions/true_class'
+require 'raingrams/extensions/false_class'
+require 'raingrams/extensions/symbol'
+require 'raingrams/extensions/string'
+require 'raingrams/extensions/object'

data/lib/raingrams/hexagram_model.rb

@@ -0,0 +1,13 @@
+require 'raingrams/multigram_model'
+
+module Raingrams
+  class HexagramModel < MultigramModel
+
+    def initialize(opts={},&block)
+      opts[:ngram_size] = 6
+
+      super(opts,&block)
+    end
+
+  end
+end

data/lib/raingrams/model.rb

@@ -0,0 +1,161 @@
+require 'raingrams/ngram'
+require 'raingrams/tokens/start_sentence'
+require 'raingrams/tokens/stop_sentence'
+require 'raingrams/exceptions/prefix_frequency_missing'
+
+module Raingrams
+  class Model
+
+    # Size of ngrams to use
+    attr_reader :ngram_size
+
+    # Ignore case of parsed text
+    attr_reader :ignore_case
+
+    # Ignore the punctuation of parsed text
+    attr_reader :ignore_punc
+
+    # Ignore URLs
+    attr_reader :ignore_urls
+
+    # Ignore Phone numbers
+    attr_reader :ignore_phone_numbers
+
+    # Ignore References
+    attr_reader :ignore_references
+
+    # Convert Acronyms to names within parsed text
+    attr_reader :convert_acronyms
+
+    # Convert Abbreviations to names within parsed text
+    attr_reader :convert_abbrev
+
+    # Frequencies of observed ngrams
+    attr_reader :frequency
+
+    # Normalized table of observed ngrams
+    attr_reader :probability
+
+    def initialize(opts={},&block)
+      @ngram_size = opts[:ngram_size]
+      @ignore_case = opts[:ignore_case] || false
+      @ignore_punc = opts[:ignore_punc] || true
+      @ignore_urls = opts[:ignore_urls] || false
+      @ignore_phone_numbers = opts[:ignore_phone_numbers] || false
+      @convert_acronyms = opts[:convert_acronyms] || false
+      @convert_abbrev = opts[:convert_abbrev] || false
+
+      @frequency = Hash.new { |hash,key| 0 }
+      @probability = Hash.new { |hash,key| 0.0 }
+
+      block.call(self) if block
+    end
+
+    def parse_sentence(sentence)
+      sentence = sentence.to_s.gsub(/[\.\?!]$/,'')
+
+      if @ignore_urls
+        sentence.gsub!(/\s*\w+:\/\/\w*\s*/,' ')
+      end
+
+      if @ignore_phone_numbers
+        sentence.gsub!(/\s*(\d-)?(\d{3}-)?\d{3}-\d{4}\s*/,' ')
+      end
+
+      if @ignore_references
+        sentence.gsub!(/\s*[\d+]\s*/,' ')
+      end
+
+      if @ignore_case
+        sentence.downcase!
+      end
+
+      if @ignore_punc
+        return sentence.scan(/\w+[\.'\-\_]?\w*/)
+      else
+        return sentence.scan(/(\w+|[-_,\.;'"])/)
+      end
+    end
+
+    def parse_text(text,&block)
+      text.to_s.scan(/[^\s\.\?!][^\.\?!]*/)
+    end
+
+    def train_with_ngram(ngram)
+      @frequency[ngram] += 1
+      return self
+    end
+
+    def train_with_ngrams(ngrams=[])
+      ngrams.each { |ngram| train_with_ngram(ngram) }
+      return self
+    end
+
+    def ngrams
+      @frequency.keys
+    end
+
+    def has_ngram?(ngram)
+      ngrams.include?(ngram)
+    end
+
+    def each_ngram(&block)
+      ngrams.each(&block)
+    end
+
+    def ngrams_with(&block)
+      ngrams.select(&block)
+    end
+
+    def vocabulary
+      ngrams.flatten.uniq
+    end
+
+    def within_vocabulary?(gram)
+      each_ngrams do |ngram|
+        return true if ngram.include?(gram)
+      end
+
+      return false
+    end
+
+    def ngrams_starting_with(obj)
+      ngrams_with { |ngram| ngram.starts_with?(obj.to_gram) }
+    end
+
+    def ngrams_ending_with(gram)
+      ngrams_with { |ngram| ngram.ends_with?(gram) }
+    end
+
+    def probabilities_for(ngrams)
+      ngrams.map { |ngram| @probability[ngram] }
+    end
+
+    def probability_of_ngram(ngram)
+      @probability[ngram]
+    end
+
+    def probability_of_ngrams(ngrams)
+      probabilities_for(ngrams).inject { |joint,prob| joint * prob }
+    end
+
+    def probability_of_gram(gram)
+      probability_of_ngrams(ngrams_starting_with(gram))
+    end
+
+    def clear
+      @frequency.clear
+
+      clear_probabilities
+      return self
+    end
+
+    protected
+
+    def clear_probabilities
+      @probability.clear
+      return self
+    end
+
+  end
+end

data/lib/raingrams/multigram_model.rb

@@ -0,0 +1,165 @@
+require 'raingrams/model'
+require 'raingrams/tokens/start_sentence'
+require 'raingrams/tokens/stop_sentence'
+require 'raingrams/exceptions/prefix_frequency_missing'
+
+module Raingrams
+  class MultigramModel < Model
+
+    # Frequencies of n-1 grams
+    attr_reader :prefix_frequency
+
+    def initialize(opts={},&block)
+      @prefix_frequency = Hash.new { |hash,key| 0 }
+
+      super(opts) { |model| model.build(&block) }
+    end
+
+    def ngrams_from_words(words)
+      return (0...(words.length-@ngram_size+1)).map do |index|
+        Ngram.new(words[index,@ngram_size])
+      end
+    end
+
+    def ngrams_from_fragment(fragment)
+      ngrams_from_words(parse_sentence(fragment))
+    end
+
+    def ngrams_from_sentence(sentence)
+      ngrams_from_words(wrap_sentence(parse_sentence(sentence)))
+    end
+
+    def ngrams_from_text(text)
+      parse_text(text).inject([]) do |ngrams,sentence|
+        ngrams + ngrams_from_sentence(sentence)
+      end
+    end
+
+    def common_ngrams_from_words(words)
+      ngrams_from_words(words).select { |ngram| has_ngram?(ngram) }
+    end
+
+    def common_ngrams_from_fragment(fragment)
+      ngrams_from_fragment(words).select { |ngram| has_ngram?(ngram) }
+    end
+
+    def common_ngrams_from_sentence(sentence)
+      ngrams_from_sentence(sentence).select { |ngram| has_ngram?(ngram) }
+    end
+
+    def common_ngrams_from_text(text)
+      ngrams_from_text(text).select { |ngram| has_ngram?(ngram) }
+    end
+
+    def train_with_ngram(ngram)
+      @prefix_frequency[ngram.prefix] += 1
+      return super(ngram)
+    end
+
+    def train_with_sentence(sentence)
+      train_with_ngrams(ngrams_from_sentence(sentence))
+    end
+
+    def train_with_text(text)
+      train_with_ngrams(ngrams_from_text(text))
+    end
+
+    def build(&block)
+      clear_probabilities
+
+      block.call(self) if block
+
+      @frequency.each do |ngram,count|
+        prefix = ngram.prefix
+
+        unless @prefix_frequency[prefix]
+          raise(PrefixFrequencyMissing,"the model is missing the frequency of the ngram prefix #{prefix}",caller)
+        end
+
+        @probability[ngram] = count.to_f / @prefix_frequency[prefix].to_f
+      end
+
+      return self
+    end
+
+    def ngrams_prefixed_by(prefix)
+      ngrams_with { |ngram| ngram.prefixed_by?(prefix) }
+    end
+
+    def ngrams_postfixed_by(postfix)
+      ngrams_with { |ngram| ngram.prefixed_by?(postfix) }
+    end
+
+    def ngrams_preceeding(gram)
+      ngrams_ending_with(gram).map do |ngram|
+        ngrams_postfixed_by(ngram.prefix)
+      end
+    end
+
+    def ngrams_following(gram)
+      ngrams_starting_with(gram).map do |ngram|
+        ngrams_prefixed_by(ngram.postfix)
+      end
+    end
+
+    def grams_preceeding(gram)
+      ngrams_ending_with(gram).map do |ngram|
+        ngram[-2]
+      end
+    end
+
+    def grams_following(gram)
+      ngrams_starting_with(gram).map do |ngram|
+        ngram[1]
+      end
+    end
+
+    def fragment_probability(fragment)
+      probability_of_ngrams(ngrams_from_fragment(fragment))
+    end
+
+    def sentence_probability(sentence)
+      probability_of_ngrams(ngrams_from_sentence(sentence))
+    end
+
+    def text_probability(text)
+      probability_of_ngrams(ngrams_from_text(text))
+    end
+
+    def common_fragment_probability(fragment)
+      probability_of_ngrams(common_ngrams_from_fragment(fragment))
+    end
+
+    def common_sentence_probability(sentence)
+      probability_of_ngrams(common_ngrams_from_sentence(sentence))
+    end
+
+    def common_text_probability(fragment)
+      probability_of_ngrams(common_ngrams_from_text(text))
+    end
+
+    def similar_fragment_probability(other,fragment)
+      common_fragment_probability(fragment) * other.common_fragment_probability(fragment)
+    end
+
+    def similar_sentence_probability(other,sentence)
+      common_sentence_probability(sentence) * other.common_sentence_probability(sentence)
+    end
+
+    def similar_text_probability(other,text)
+      common_text_probability(text) * other.common_text_probability(text)
+    end
+
+    def clear
+      @prefix_frequency.clear
+      return super
+    end
+
+    protected
+
+    def wrap_sentence(sentence)
+      (Tokens::StartSentence * @ngram_size) + sentence.to_a + (Tokens::StopSentence * @ngram_size)
+    end
+
+  end
+end

data/lib/raingrams/ngram.rb

@@ -0,0 +1,53 @@
+module Raingrams
+  class Ngram < Array
+
+    def initialize(objs)
+      super(objs.map { |obj| obj.to_gram })
+    end
+
+    def self.[](*objs)
+      self.new(objs)
+    end
+
+    def prefix
+      self[0...length-1]
+    end
+
+    def prefixed_by?(ngram)
+      prefix==ngram
+    end
+
+    def postfix
+      self[1..-1]
+    end
+
+    def postfixed_by?(ngram)
+      postfix==ngram
+    end
+
+    def starts_with?(obj)
+      self[0]==obj.to_gram
+    end
+
+    def ends_with?(obj)
+      self[-1]==obj.to_gram
+    end
+
+    def include?(obj)
+      super(obj.to_gram)
+    end
+
+    def flatten
+      self.dup
+    end
+
+    def flatten!
+      self
+    end
+
+    def to_s
+      join(', ')
+    end
+
+  end
+end

data/lib/raingrams/open_vocabulary/bigram_model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/bigram_model'
+require 'raingrams/openvocabulary/open_model'
+
+module Raingrams
+  module OpenVocabulary
+    class BigramModel < Raingrams::BigramModel
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/hexagram_model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/hexagram_model'
+require 'raingrams/openvocabulary/open_model'
+
+module Raingrams
+  module OpenVocabulary
+    class HexagramModel < Raingrams::HexagramModel
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/multigram_model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/multigrammodel'
+require 'raingrams/openvocabulary/openmodel'
+
+module Raingrams
+  module OpenVocabulary
+    class MultigramModel < Raingrams::MultigramModel
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/open_model.rb

@@ -0,0 +1,34 @@
+require 'raingrams/tokens/unknown'
+
+module Raingrams
+  module OpenVocabulary
+    module OpenModel
+
+      # The fixed lexicon of this model
+      attr_reader :lexicon
+
+      def initialize(opts={},&block)
+        @lexicon = opts[:lexicon] || []
+
+        super(opts,&block)
+      end
+
+      def within_lexicon?(gram)
+        @lexicon.include?(gram)
+      end
+
+      def train_ngram(ngram)
+        ngram = ngram.map do |gram|
+          if within_lexicon?(gram)
+            gram
+          else
+            Tokens::Unknown
+          end
+        end
+
+        return super(ngram)
+      end
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/pentagram_model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/pentagram_model'
+require 'raingrams/openvocabulary/open_model'
+
+module Raingrams
+  module OpenVocabulary
+    class PentagramModel < Raingrams::PentagramModel
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/quadgram_model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/quadgram_model'
+require 'raingrams/openvocabulary/open_model'
+
+module Raingrams
+  module OpenVocabulary
+    class QuadgramModel < Raingrams::QuadgramModel
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/trigram_model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/trigram_model'
+require 'raingrams/openvocabulary/open_model'
+
+module Raingrams
+  module OpenVocabulary
+    class TrigramModel < Raingrams::TrigramModel
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary/unigram_model.rb

@@ -0,0 +1,12 @@
+require 'raingrams/unigram_model'
+require 'raingrams/openvocabulary/open_model'
+
+module Raingrams
+  module OpenVocabulary
+    class UnigramModel < Raingrams::UnigramModel
+
+      include OpenModel
+
+    end
+  end
+end

data/lib/raingrams/open_vocabulary.rb

@@ -0,0 +1,7 @@
+require 'raingrams/openvocabulary/unigram_model'
+require 'raingrams/openvocabulary/multigram_model'
+require 'raingrams/openvocabulary/bigram_model'
+require 'raingrams/openvocabulary/trigram_model'
+require 'raingrams/openvocabulary/quadgram_model'
+require 'raingrams/openvocabulary/pentagram_model'
+require 'raingrams/openvocabulary/hexagram_model'

data/lib/raingrams/pentagram_model.rb

@@ -0,0 +1,13 @@
+require 'raingrams/multigram_model'
+
+module Raingrams
+  class PentagramModel < MultigramModel
+
+    def initialize(opts={},&block)
+      opts[:ngram_size] = 5
+
+      super(opts,&block)
+    end
+
+  end
+end

data/lib/raingrams/quadgram_model.rb

@@ -0,0 +1,13 @@
+require 'raingrams/multigram_model'
+
+module Raingrams
+  class QuadgramModel < MultigramModel
+
+    def initialize(opts={},&block)
+      opts[:ngram_size] = 4
+
+      super(opts,&block)
+    end
+
+  end
+end

data/lib/raingrams/raingrams.rb

@@ -0,0 +1,31 @@
+require 'raingrams/unigram_model'
+require 'raingrams/multigram_model'
+require 'raingrams/open_vocabulary/unigram_model'
+require 'raingrams/open_vocabulary/multigram_model'
+
+module Raingrams
+  def Raingrams.closed_vocabulary_model(opts={},&block)
+    if opts[:ngram_size]==1
+      return UnigramModel.new(opts,&block)
+    else
+      return MultigramModel.new(opts,&block)
+    end
+  end
+
+  def Raingrams.open_vocabulary_model(opts={},&block)
+    if opts[:ngram_size]==1
+      return OpenVocabulary::UnigramModel.new(opts,&block)
+    else
+      return OpenVocabulary::MultigramModel.new(opts,&block)
+    end
+  end
+
+  def Raingrams.model(opts={},&block)
+    case opts[:vocabulary]
+    when :open, 'open'
+      return Raingrams.open_vocabulary_model(opts,&block)
+    else
+      return Raingrams.closed_vocabulary_model(opts,&block)
+    end
+  end
+end

data/lib/raingrams/tokens/start_sentence.rb

@@ -0,0 +1,13 @@
+require 'raingrams/tokens/token'
+
+module Raingrams
+  module Tokens
+    class StartSentence < Token
+
+      def self.to_s
+        '<s>'
+      end
+
+    end
+  end
+end

data/lib/raingrams/tokens/stop_sentence.rb

@@ -0,0 +1,13 @@
+require 'raingrams/tokens/token'
+
+module Raingrams
+  module Tokens
+    class StopSentence < Token
+
+      def self.to_s
+        '</s>'
+      end
+
+    end
+  end
+end

data/lib/raingrams/tokens/token.rb

@@ -0,0 +1,19 @@
+module Raingrams
+  module Tokens
+    class Token
+
+      def self.*(length)
+        [self] * length
+      end
+
+      def self.to_sym
+        self.to_s.to_sym
+      end
+
+      def self.inspect
+        self.to_s
+      end
+
+    end
+  end
+end

data/lib/raingrams/tokens/unknown.rb

@@ -0,0 +1,13 @@
+require 'raingrams/tokens/token'
+
+module Raingrams
+  module Tokens
+    class Unknown < Token
+
+      def self.to_s
+        '<unknown>'
+      end
+
+    end
+  end
+end

data/lib/raingrams/tokens.rb

@@ -0,0 +1,4 @@
+require 'raingrams/tokens/token'
+require 'raingrams/tokens/start_sentence'
+require 'raingrams/tokens/stop_sentence'
+require 'raingrams/tokens/unknown'

data/lib/raingrams/trigram_model.rb

@@ -0,0 +1,13 @@
+require 'raingrams/multigram_model'
+
+module Raingrams
+  class TrigramModel < MultigramModel
+
+    def initialize(opts={},&block)
+      opts[:ngram_size] = 3
+
+      super(opts,&block)
+    end
+
+  end
+end

data/lib/raingrams/unigram_model.rb

@@ -0,0 +1,70 @@
+require 'raingrams/model'
+
+module Raingrams
+  class UnigramModel < Model
+
+    def initialize(opts={},&block)
+      opts[:ngram_size] = 1
+
+      super(opts) { |model| model.build(&block) }
+    end
+
+    def ngrams_from_words(words)
+      words.map { |word| Ngram[word] }
+    end
+
+    def ngrams_from_fragment(fragment)
+      ngrams_from_words(parse_sentence(fragment))
+    end
+
+    def ngrams_from_sentence(sentence)
+      ngrams_from_fragment(sentence)
+    end
+
+    def ngrams_from_text(text)
+      parse_text(text).inject([]) do |ngrams,sentence|
+        ngrams + ngrams_from_sentence(sentence)
+      end
+    end
+
+    def train_with_sentence(sentence)
+      train_with_ngrams(ngrams_from_sentence(sentence))
+    end
+
+    def train_with_text(text)
+      train_with_ngrams(ngrams_from_text(text))
+    end
+
+    def gram_count
+      @frequency.values.inject do |sum,count|
+        sum + count
+      end
+    end
+
+    def build(&block)
+      clear_probabilities
+
+      block.call(self) if block
+
+      total_count = gram_count.to_f
+      @frequency.each do |ngram,count|
+        @probability[ngram] = count.to_f / total_count
+      end
+
+      return self
+    end
+
+    def fragment_probability(fragment)
+      probability_of_ngrams(ngrams_from_fragment(fragment))
+    end
+
+    def sentence_probability(sentence)
+      probability_of_ngrams(ngrams_from_sentence(sentence))
+    end
+
+    def text_probability(text)
+      probability_of_ngrams(ngrams_from_text(text))
+    end
+
+  end
+end

data/lib/raingrams/version.rb

@@ -0,0 +1,3 @@
+module Raingrams
+  VERSION = '0.0.9'
+end

data/lib/raingrams.rb

@@ -0,0 +1,10 @@
+require 'raingrams/extensions'
+require 'raingrams/raingrams'
+require 'raingrams/ngram'
+require 'raingrams/unigram_model'
+require 'raingrams/bigram_model'
+require 'raingrams/trigram_model'
+require 'raingrams/quadgram_model'
+require 'raingrams/pentagram_model'
+require 'raingrams/hexagram_model'
+require 'raingrams/raingrams'

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.4
+specification_version: 1
+name: raingrams
+version: !ruby/object:Gem::Version 
+  version: 0.0.9
+date: 2008-01-09 00:00:00 -08:00
+summary: Raingrams is a flexible and general-purpose ngrams library written in Ruby
+require_paths: 
+- lib
+email: postmodern.mod3@gmail.com
+homepage: "    by Postmodern Modulus III"
+rubyforge_project: raingrams
+description: "== FEATURES/PROBLEMS:  * Supports all non-zero ngram sizes. * Supports text and non-text grams. * Supports Open and Closed vocabulary models.  == REQUIREMENTS:  == INSTALL:  $ sudo gem install raingrams"
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Postmodern Modulus III
+files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/raingrams.rb
+- lib/raingrams/version.rb
+- lib/raingrams/raingrams.rb
+- lib/raingrams/exceptions/prefix_frequency_missing.rb
+- lib/raingrams/exceptions.rb
+- lib/raingrams/extensions/class.rb
+- lib/raingrams/extensions/false_class.rb
+- lib/raingrams/extensions/nil_class.rb
+- lib/raingrams/extensions/object.rb
+- lib/raingrams/extensions/string.rb
+- lib/raingrams/extensions/symbol.rb
+- lib/raingrams/extensions/true_class.rb
+- lib/raingrams/extensions.rb
+- lib/raingrams/tokens/token.rb
+- lib/raingrams/tokens/start_sentence.rb
+- lib/raingrams/tokens/stop_sentence.rb
+- lib/raingrams/tokens/unknown.rb
+- lib/raingrams/tokens.rb
+- lib/raingrams/ngram.rb
+- lib/raingrams/model.rb
+- lib/raingrams/unigram_model.rb
+- lib/raingrams/multigram_model.rb
+- lib/raingrams/bigram_model.rb
+- lib/raingrams/trigram_model.rb
+- lib/raingrams/quadgram_model.rb
+- lib/raingrams/pentagram_model.rb
+- lib/raingrams/hexagram_model.rb
+- lib/raingrams/open_vocabulary/open_model.rb
+- lib/raingrams/open_vocabulary/unigram_model.rb
+- lib/raingrams/open_vocabulary/multigram_model.rb
+- lib/raingrams/open_vocabulary/bigram_model.rb
+- lib/raingrams/open_vocabulary/trigram_model.rb
+- lib/raingrams/open_vocabulary/quadgram_model.rb
+- lib/raingrams/open_vocabulary/pentagram_model.rb
+- lib/raingrams/open_vocabulary/hexagram_model.rb
+- lib/raingrams/open_vocabulary.rb
+- test/test_raingrams.rb
+test_files: 
+- test/test_raingrams.rb
+rdoc_options: 
+- --main
+- README.txt
+extra_rdoc_files: 
+- History.txt
+- LICENSE.txt
+- Manifest.txt
+- README.txt
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.4.0
+    version: