tf-idf 1.0.0

data/.document

@@ -0,0 +1,5 @@
+README.markdown
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Marc Chung
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,26 @@
+tf-idf
+------
+
+This is a simple Tf-idf library.  The algorithm is described in:
+
+  [http://en.wikipedia.org/wiki/Tf-idf][wiki].
+
+A port of Niniane Wang's [tfidf][tfidf] library.
+
+Note on Patches/Pull Requests
+-----------------------------
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a future version unintentionally.
+* Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+Copyright
+--------
+
+Copyright (c) 2009 Marc Chung. See LICENSE for details.
+
+  [tfidf]: http://code.google.com/p/tfidf
+  
+  [wiki]: http://en.wikipedia.org/wiki/Tf-idf

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "tf-idf"
+    gem.summary = %Q{A rubygem that implements the Tf-Idf algorithm}
+    gem.description = %Q{Computes IDF for a specified term based on the corpus, or generates keywords ordered by tf-idf for a specified document.}
+    gem.email = "mchung@gmail.com"
+    gem.homepage = "http://github.com/mchung/tf-idf"
+    gem.authors = ["Marc Chung"]
+    gem.add_development_dependency "rspec"
+    gem.add_development_dependency "yard"
+    gem.add_development_dependency "cucumber"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+  spec.rcov_dir = "doc/coverage"
+end
+
+task :spec => :check_dependencies
+
+begin
+  require 'cucumber/rake/task'
+  Cucumber::Rake::Task.new(:features)
+
+  task :features => :check_dependencies
+rescue LoadError
+  task :features do
+    abort "Cucumber is not available. In order to run features, you must: sudo gem install cucumber"
+  end
+end
+
+task :default => [:spec, :features]
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/features/step_definitions/tf-idf_steps.rb

@@ -0,0 +1,49 @@
+Before do
+  @default_idf_unittest = 1.0
+end
+
+Given /^the default IDF is set to "([^\"]*)"$/ do |val|
+  @default_idf_unittest = val.to_f
+end
+
+Given /^I have loaded the sample corpus data set "([^\"]*)"$/ do |corpus_filename|
+  @corpus_filename = File.expand_path(File.dirname(__FILE__) + "/../../spec/fixtures/#{corpus_filename}")
+  @my_tfidf = TfIdf.from_corpus(@corpus_filename, @default_idf_unittest)
+end
+
+Then /^I should have a total of "([^\"]*)" documents$/ do |total_doc_count|
+  @my_tfidf.num_docs.should == total_doc_count.to_i
+end
+
+Then /^I should have a total of "([^\"]*)" term\/num_doc pairs$/ do |total_term_num_doc_count|
+  @my_tfidf.term_num_docs.size.should == total_term_num_doc_count.to_i
+end
+
+Then /^I should get the default IDF for "([^\"]*)"$/ do |term|
+  @my_tfidf.idf(term).should == @default_idf_unittest
+end
+
+Then /^the IDF for "([^\"]*)" should be greater than the IDF for "([^\"]*)"$/ do |term1, term2|
+  @my_tfidf.idf(term1).should > @my_tfidf.idf(term2)
+end
+
+Then /^the IDF for "([^\"]*)" should be equal to the IDF for "([^\"]*)"$/ do |term1, term2|
+  @my_tfidf.idf(term1).should == @my_tfidf.idf(term2)
+end
+
+Given /^the keywords "([^\"]*)"$/ do |keywords|
+  @keywords = @my_tfidf.doc_keywords(keywords)
+end
+
+Then /^"([^\"]*)" should be located at "([^\"]*)", "([^\"]*)"$/ do |term, x, y|
+  @keywords[x.to_i][y.to_i].should == term
+end
+
+
+# Then /^I should get the expected IDF for "([^\"]*)"$/ do |arg1|
+#   pending
+# end
+
+def get_expected_idf(num_docs_total, num_docs_term)
+  Math.log((1 + num_docs_total).to_f / (1 + num_docs_term))
+end

data/features/support/env.rb

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'tf-idf'
+
+require 'spec/expectations'

data/features/tf-idf.feature

@@ -0,0 +1,79 @@
+Feature: Tf-Idf
+  A user should be able to construct an IDF corpus
+
+  Scenario: A corpus should contain the total number of documents
+    Given I have loaded the sample corpus data set "tfidf_testcorpus.txt"
+    Then I should have a total of "50" documents
+
+  Scenario: A corpus should contain terms and the number of documents they are found in.
+    Given I have loaded the sample corpus data set "tfidf_testcorpus.txt"
+    Then I should have a total of "6" term/num_doc pairs
+
+  Scenario: A corpus should query IDF for nonexistent terms
+    Given I have loaded the sample corpus data set "tfidf_testcorpus.txt"
+    Then I should get the default IDF for "nonexistent"
+    Then I should get the default IDF for "THE"
+    
+  Scenario: A corpus should query IDF for existent terms
+    Given I have loaded the sample corpus data set "tfidf_testcorpus.txt"
+    Then the IDF for "a" should be greater than the IDF for "the"
+    Then the IDF for "girl" should be equal to the IDF for "moon"
+
+  Scenario: A corpus should retrieve keywords from a document, ordered by tf-idf"
+    Given the default IDF is set to "0.01"
+    And I have loaded the sample corpus data set "tfidf_testcorpus.txt"
+    Given the keywords "the spoon and the fork"
+    Then "the" should be located at "0", "0"
+    Given the keywords "the girl said hello over the phone"
+    Then "girl" should be located at "0", "0"
+    Then "phone" should be located at "1", "0"
+    Then "said" should be located at "2", "0"
+    Then "the" should be located at "3", "0"
+
+  # Scenario: A corpus should add input documents to an existing corpus"
+  #   Given I have loaded the sample corpus data set "tfidf_testcorpus.txt"
+  #   Then I should get the default IDF for "water"
+  #   Then I should get the expected IDF for "moon" when it has "1" occurrence
+  #   Then I should get the expected IDF for "said" when it has "5" occurrences
+
+
+    # it "should add input documents to an existing corpus" do
+    #   my_tfidf = TfIdf.new(@test_corpus, nil, @default_idf_unittest)
+    #   my_tfidf.idf("water").should == @default_idf_unittest
+    #   my_tfidf.idf("moon").should == get_expected_idf(my_tfidf.num_docs, 1)
+    #   my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+    # 
+    #   my_tfidf.add_input_document("water moon") # doesn't support commas
+    # 
+    #   my_tfidf.idf("water").should == get_expected_idf(my_tfidf.num_docs, 1)
+    #   my_tfidf.idf("moon").should == get_expected_idf(my_tfidf.num_docs, 2)
+    #   my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+    # end
+    # 
+    # it "should add input documents to an empty corpus" do
+    #   my_tfidf = TfIdf.new(nil, nil, @default_idf_unittest)
+    #   my_tfidf.idf("moon").should == @default_idf_unittest
+    #   my_tfidf.idf("water").should == @default_idf_unittest
+    #   my_tfidf.idf("said").should == @default_idf_unittest
+    # 
+    #   my_tfidf.add_input_document("moon")
+    #   my_tfidf.add_input_document("moon said hello")
+    # 
+    #   my_tfidf.idf("water").should == @default_idf_unittest
+    #   my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 1)
+    #   my_tfidf.idf("moon").should == get_expected_idf(my_tfidf.num_docs, 2)
+    # end
+    # 
+    # it "should observe stopwords list" do
+    #   my_tfidf = TfIdf.new(@test_corpus, @test_stopwords, @default_idf_unittest)    
+    #   my_tfidf.idf("water").should == @default_idf_unittest
+    #   my_tfidf.idf("moon").should == 0 # ignored
+    #   my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+    # 
+    #   my_tfidf.add_input_document("moon")
+    #   my_tfidf.add_input_document("moon and water")
+    # 
+    #   my_tfidf.idf("water").should == get_expected_idf(my_tfidf.num_docs, 1)
+    #   my_tfidf.idf("moon").should == 0
+    #   my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+    # end

data/lib/tf-idf.rb

@@ -0,0 +1,199 @@
+# Tf-idf class implementing http://en.wikipedia.org/wiki/Tf-idf.
+#
+# The library constructs an IDF corpus and stopword list either from
+# documents specified by the client, or by reading from input files.  It
+# computes IDF for a specified term based on the corpus, or generates
+# keywords ordered by tf-idf for a specified document.
+#
+# @author Marc Chung <mchung@gmail.com>
+# @see http://en.wikipedia.org/wiki/Tf-idf Term frequency-inverse document frequency
+class TfIdf
+
+  # @return [Integer] The total number of documents in the tf-idf corpus.
+  attr_accessor :num_docs
+  
+  # @return [Hash<String, Integer>] A histogram of terms and their term frequency.
+  attr_accessor :term_num_docs
+  
+  # @return [Array<String>] An array of stopwords.
+  attr_accessor :stopwords
+  
+  # @return [Float] The default value returned when a term is not found in the tf-idf corpus.
+  attr_accessor :idf_default
+  
+  DEFAULT_IDF = 1.5
+  
+  ##
+  # Initialize the tf-idf dictionary.  
+  #   
+  # If a corpus file is supplied, reads the idf dictionary from it, in the
+  # format of:
+  #  # of total documents
+  #  term: # of documents containing the term
+  # 
+  # If a stopword file is specified, reads the stopword list from it, in
+  # the format of one stopword per line.
+  # 
+  # The DEFAULT_IDF value is returned when a query term is not found in the
+  # IDF corpus.
+  #
+  # @param [String] corpus_filename The disk location of the IDF corpus.
+  # @param [String] stopword_filename The disk location of the stopword list.
+  # @param [Float] default_idf The value returned when a term is not found in the IDF corpus.
+  # @raise ["Corpus Not Found"] Thrown when the corpus isn't found.
+  # @raise ["Stopwords Not Found"] Thrown when the stopwords list isn't found.
+  # @return [TfIdf] A TfIdf instance loaded with the corpus.
+  def initialize(corpus_filename = nil, stopword_filename = nil, default_idf = DEFAULT_IDF)
+    self.num_docs = 0
+    self.term_num_docs = {} 
+    self.stopwords = []
+    self.idf_default = default_idf
+
+    raise "Corpus not found" if corpus_filename && !File.exists?(corpus_filename)    
+    if corpus_filename
+      entries = File.read(corpus_filename).entries
+      self.num_docs = entries.shift.strip.to_i
+      entries.each do |line|
+        tokens = line.split(":")
+        term = tokens[0].strip
+        frequency = tokens[1].strip.to_i
+        self.term_num_docs[term] = frequency
+      end
+    end
+    
+    raise "Stopwords not found" if stopword_filename && !File.exists?(stopword_filename)
+    if stopword_filename
+      self.stopwords = File.read(stopword_filename).entries.collect{|x| x.strip}
+    end
+  end
+  
+  ## 
+  # Convenience method for creating a TfIdf instance.
+  # 
+  # @param [String] corpus_filename The disk location of the IDF corpus.
+  # @return [TfIdf] A TfIdf instance loaded with the corpus.
+  def self.from_corpus(corpus_filename, default_idf = DEFAULT_IDF)
+    self.new(corpus_filename, nil, default_idf)
+  end
+  
+  ##
+  # Breaks a string into tokens. This implementation matches whole words.
+  # Clients may wish to override this behaviour with their own tokenization.
+  # strategy.
+  #
+  # @param [String] input String representation of a document
+  # @return [Array<String>] A list of tokens
+  def get_tokens(input)
+    # str.split().collect{|x| x if x =~ /[A-Za-z]+/}.compact
+    input.split.select{|x| x =~ /<a.*?\/a>|<[^\>]*>|[\w'@#]+/}
+  end
+  
+  ##
+  # Add terms in the specified document to the IDF corpus.
+  #
+  # @param [String] input String representation of a document.
+  def add_input_document(input)
+    self.num_docs += 1
+    token_set = get_tokens(input).uniq
+    token_set.each do |term|
+      if self.term_num_docs[term]
+        self.term_num_docs[term] += 1
+      else
+        self.term_num_docs[term] = 1
+      end
+    end
+  end
+  
+  ##
+  # Saves the tf-idf corpus and stopword list to the specified file.
+  # 
+  # A word is a stopword if it occurs in more than stopword_threshold% of num_docs.
+  # A threshold of 0.4, means that the word must occur in more than 40% of the documents.
+  #
+  # @param [String] idf_filename Filename.
+  # @param [String] stopword_filename Filename.
+  # @param [Float] stopword_percentage_threshold Stopword threshold. Lower threshold lower criteria.
+  def save_corpus_to_file(idf_filename, stopword_filename, stopword_percentage_threshold = 0.01)
+    File.open(idf_filename, "w") do |file|
+      file.write("#{self.num_docs}\n")
+      self.term_num_docs.each do |term, num_docs|
+        file.write("#{term}: #{num_docs}\n")
+      end
+    end
+    
+    File.open(stopword_filename, "w") do |file|
+      sorted_term_num_docs = sort_by_tfidf(self.term_num_docs)
+      sorted_term_num_docs.each do |term, num_docs|
+        # pp [term, num_docs, stopword_percentage_threshold, self.num_docs, stopword_percentage_threshold * self.num_docs, ]
+        if num_docs > stopword_percentage_threshold * self.num_docs
+          file.write("#{term}\n")
+        end
+      end
+    end
+  end
+
+  ##
+  # Retrieves the IDF for the specified term. 
+  # 
+  # This is computed with:
+  #   logarithm of ((number of documents in corpus) divided by 
+  #                 (number of documents containing this term)).
+  #
+  # @param [String] term A term in the IDF corpus.
+  # @return [Float] The IDF for the specified term.
+  def idf(term)
+    if self.stopwords.include?(term)
+      return 0
+    end
+            
+    if self.term_num_docs[term].nil?
+      return self.idf_default
+    end
+
+    return Math.log((1 + self.num_docs).to_f / 
+                    (1 + self.term_num_docs[term]))
+  end
+
+  ## 
+  # Retrieve terms and corresponding tf-idf for the specified document.
+  #
+  # The returned terms are ordered by decreasing tf-idf.
+  #
+  # @param [String] curr_doc String representation of an existing document.
+  # @return [Array] Terms ordered by decreasing tf-idf rank.
+  def doc_keywords(curr_doc)
+    tfidf = {}
+
+    tokens = self.get_tokens(curr_doc)
+    token_set = tokens.uniq
+    token_set_sz = token_set.count
+    
+    token_set.each do |term|
+      mytf = tokens.count(term).to_f / token_set_sz
+      myidf = self.idf(term)
+      tfidf[term] = mytf * myidf
+    end
+
+    sort_by_tfidf(tfidf)
+  end
+
+  ##
+  # Returns a string representation of the tf-idf corpus. 
+  # 
+  # @return [String] Contains # docs, # term and frequency.
+  def to_s
+    {:num_docs => self.num_docs, :term_num_docs => self.term_num_docs.size}.inspect
+  end
+  
+  ## 
+  # Sorts terms by decreasing tf-idf.
+  #
+  # @example Sort by tf-idf
+  #  "{'and'=>0.0025, 'fork'=>0.0025, 'the'=>0.37688590118819, 'spoon'=>1.0025}" #=>
+  #  "[['spoon', 1.0025], ['the', 0.37688590118819], ['fork', 0.0025], ['and', 0.0025]]"
+  # @return [Array<Array<String, Float>>] An array of term/IDF array pairs.
+  def sort_by_tfidf(tfidf)
+    tfidf.sort{|a, b| b[1] <=> a[1]}
+  end
+
+end

data/spec/fixtures/tfidf_testcorpus.txt

@@ -0,0 +1,7 @@
+50
+the: 23
+a: 17
+girl: 1
+moon: 1
+said: 5
+phone: 2

data/spec/fixtures/tfidf_teststopwords.txt

@@ -0,0 +1 @@
+moon

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'tf-idf'
+require 'spec'
+require 'spec/autorun'
+require 'pp'
+
+Spec::Runner.configure do |config|
+end
+
+def get_expected_idf(num_docs_total, num_docs_term)
+  Math.log((1 + num_docs_total).to_f / (1 + num_docs_term))
+end

data/spec/tf-idf_spec.rb

@@ -0,0 +1,104 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "TF-IDF library" do
+
+  before(:all) do
+    @test_corpus = File.expand_path(File.dirname(__FILE__) + '/fixtures/tfidf_testcorpus.txt')
+    @test_stopwords = File.expand_path(File.dirname(__FILE__) + '/fixtures/tfidf_teststopwords.txt')
+    @default_idf_unittest = 1.0
+  end
+
+  it "should instantiate without args" do
+    TfIdf.new
+  end
+  
+  it "should report the correct number of documents" do
+    my_tfidf = TfIdf.from_corpus(@test_corpus, @default_idf_unittest)
+    my_tfidf.num_docs.should == 50
+  end
+
+  it "should report the correct number of terms" do
+    my_tfidf = TfIdf.from_corpus(@test_corpus, @default_idf_unittest)
+    my_tfidf.term_num_docs.size.should == 6
+  end
+  
+  it "should query IDF for nonexistent terms" do
+    my_tfidf = TfIdf.from_corpus(@test_corpus, @default_idf_unittest)
+    my_tfidf.idf("nonexistent").should == @default_idf_unittest
+    my_tfidf.idf("THE").should == @default_idf_unittest
+  end
+  
+  it "should query IDF for existent terms" do
+    my_tfidf = TfIdf.from_corpus(@test_corpus, @default_idf_unittest)
+    my_tfidf.idf("a").should > my_tfidf.idf("the")
+    my_tfidf.idf("girl").should == my_tfidf.idf("moon")
+  end
+  
+  it "should retrieve keywords from a document, ordered by tf-idf" do
+    my_tfidf = TfIdf.from_corpus(@test_corpus, 0.01)
+
+    # Test retrieving keywords when there is only one keyword.
+    keywords = my_tfidf.doc_keywords("the spoon and the fork")
+    keywords[0][0].should == "the"
+    
+    # Test retrieving multiple keywords.
+    keywords = my_tfidf.doc_keywords("the girl said hello over the phone")
+    keywords[0][0].should == "girl"
+    keywords[1][0].should == "phone"
+    keywords[2][0].should == "said"
+    keywords[3][0].should == "the"
+  end
+
+  it "should add input documents to an existing corpus" do
+    my_tfidf = TfIdf.new(@test_corpus, nil, @default_idf_unittest)
+    my_tfidf.idf("water").should == @default_idf_unittest
+    my_tfidf.idf("moon").should == get_expected_idf(my_tfidf.num_docs, 1)
+    my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+
+    my_tfidf.add_input_document("water moon") # doesn't support commas
+    
+    my_tfidf.idf("water").should == get_expected_idf(my_tfidf.num_docs, 1)
+    my_tfidf.idf("moon").should == get_expected_idf(my_tfidf.num_docs, 2)
+    my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+  end
+  
+  it "should add input documents to an empty corpus" do
+    my_tfidf = TfIdf.new(nil, nil, @default_idf_unittest)
+    my_tfidf.idf("moon").should == @default_idf_unittest
+    my_tfidf.idf("water").should == @default_idf_unittest
+    my_tfidf.idf("said").should == @default_idf_unittest
+
+    my_tfidf.add_input_document("moon")
+    my_tfidf.add_input_document("moon said hello")
+
+    my_tfidf.idf("water").should == @default_idf_unittest
+    my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 1)
+    my_tfidf.idf("moon").should == get_expected_idf(my_tfidf.num_docs, 2)
+  end
+  
+  it "should observe stopwords list" do
+    my_tfidf = TfIdf.new(@test_corpus, @test_stopwords, @default_idf_unittest)    
+    my_tfidf.idf("water").should == @default_idf_unittest
+    my_tfidf.idf("moon").should == 0 # ignored
+    my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+    
+    my_tfidf.add_input_document("moon")
+    my_tfidf.add_input_document("moon and water")
+
+    my_tfidf.idf("water").should == get_expected_idf(my_tfidf.num_docs, 1)
+    my_tfidf.idf("moon").should == 0
+    my_tfidf.idf("said").should == get_expected_idf(my_tfidf.num_docs, 5)
+  end
+  
+  # Abstract out File IO
+  # it "should write the contents of the TF/IDF corpus to disk" do
+  #   my_tfidf = TfIdf.new(@test_corpus, @test_stopwords, @default_idf_unittest)    
+  #   my_tfidf.save_corpus_to_file("foo.txt", "bar.txt", 0.3)
+  #   stopwords = File.read("bar.txt").split
+  # 
+  #   stopwords.size.should == 2
+  #   stopwords.should include("a")
+  #   stopwords.should include("the")
+  # end
+
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification 
+name: tf-idf
+version: !ruby/object:Gem::Version 
+  version: 1.0.0
+platform: ruby
+authors: 
+- Marc Chung
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-29 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: cucumber
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Computes IDF for a specified term based on the corpus, or generates keywords ordered by tf-idf for a specified document.
+email: mchung@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.markdown
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.markdown
+- Rakefile
+- VERSION
+- features/step_definitions/tf-idf_steps.rb
+- features/support/env.rb
+- features/tf-idf.feature
+- lib/tf-idf.rb
+- spec/fixtures/tfidf_testcorpus.txt
+- spec/fixtures/tfidf_teststopwords.txt
+- spec/spec_helper.rb
+- spec/tf-idf_spec.rb
+has_rdoc: true
+homepage: http://github.com/mchung/tf-idf
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A rubygem that implements the Tf-Idf algorithm
+test_files: 
+- spec/spec_helper.rb
+- spec/tf-idf_spec.rb